signature:
def bls_snr ( blsdict , times , mags , errs , assumeserialbls = False , magsarefluxes = False , sigclip = 10.0 , npeaks = None , perioddeltapercent = 10 , ingressdurationfraction = 0.1 , verbose = True ) :
    '''Calculates the signal to noise ratio for each best peak in the BLS
 periodogram , along with transit depth , duration , and refit period and epoch .
 The following equation is used for SNR : :
 SNR = ( transit model depth / RMS of LC with transit model subtracted )
 * sqrt ( number of points in transit )
 Parameters
 blsdict : dict
 This is an lspinfo dict produced by either ` bls _ parallel _ pfind ` or
 ` bls _ serial _ pfind ` in this module , or by your own BLS function . If you
 provide results in a dict from an external BLS function , make sure this
 matches the form below : :
 { ' bestperiod ' : the best period value in the periodogram ,
 ' bestlspval ' : the periodogram peak associated with the best period ,
 ' nbestpeaks ' : the input value of nbestpeaks ,
 ' nbestlspvals ' : nbestpeaks - size list of best period peak values ,
 ' nbestperiods ' : nbestpeaks - size list of best periods ,
 ' lspvals ' : the full array of periodogram powers ,
 ' frequencies ' : the full array of frequencies considered ,
 ' periods ' : the full array of periods considered ,
 ' blsresult ' : list of result dicts from eebls . f wrapper functions ,
 ' stepsize ' : the actual stepsize used ,
 ' nfreq ' : the actual nfreq used ,
 ' nphasebins ' : the actual nphasebins used ,
 ' mintransitduration ' : the input mintransitduration ,
 ' maxtransitduration ' : the input maxtransitdurations ,
 ' method ' : ' bls ' - > the name of the period - finder method ,
 ' kwargs ' : { dict of all of the input kwargs for record - keeping } }
 times , mags , errs : np . array
 These contain the magnitude / flux time - series and any associated errors .
 assumeserialbls : bool
 If this is True , this function will not rerun BLS around each best peak
 in the input lspinfo dict to refit the periods and epochs . This is
 usally required for ` bls _ parallel _ pfind ` so set this to False if you use
 results from that function . The parallel method breaks up the frequency
 space into chunks for speed , and the results may not exactly match those
 from a regular BLS run .
 magsarefluxes : bool
 Set to True if the input measurements in ` mags ` are actually fluxes and
 not magnitudes .
 npeaks : int or None
 This controls how many of the periods in ` blsdict [ ' nbestperiods ' ] ` to
 find the SNR for . If it ' s None , then this will calculate the SNR for all
 of them . If it ' s an integer between 1 and
 ` len ( blsdict [ ' nbestperiods ' ] ) ` , will calculate for only the specified
 number of peak periods , starting from the best period .
 perioddeltapercent : float
 The fraction of the period provided to use to search around this
 value . This is a percentage . The period range searched will then be : :
 [ period - ( perioddeltapercent / 100.0 ) * period ,
 period + ( perioddeltapercent / 100.0 ) * period ]
 ingressdurationfraction : float
 The fraction of the transit duration to use to generate an initial value
 of the transit ingress duration for the BLS model refit . This will be
 fit by this function .
 verbose : bool
 If True , will indicate progress and any problems encountered .
 Returns
 dict
 A dict of the following form is returned : :
 { ' npeaks : the number of periodogram peaks requested to get SNR for ,
 ' period ' : list of refit best periods for each requested peak ,
 ' epoch ' : list of refit epochs ( i . e . mid - transit times ) ,
 ' snr ' : list of SNRs of the transit for each requested peak ,
 ' transitdepth ' : list of depths of the transits ,
 ' transitduration ' : list of durations of the transits ,
 ' nphasebins ' : the input value of nphasebins ,
 ' transingressbin ' : the phase bin containing transit ingress ,
 ' transegressbin ' : the phase bin containing transit egress ,
 ' allblsmodels ' : the full BLS models used along with its parameters ,
 ' allsubtractedmags ' : BLS models - phased light curves ,
 ' allphasedmags ' : the phase light curves ,
 ' allphases ' : the phase values }'''

implementation:
# figure out how many periods to work on
if ( npeaks and ( 0 < npeaks < len ( blsdict [ 'nbestperiods' ] ) ) ) :
    nperiods = npeaks
else :
    if verbose :
        LOGWARNING ( 'npeaks not specified or invalid, ' 'getting SNR for all %s BLS peaks' % len ( blsdict [ 'nbestperiods' ] ) )
    nperiods = len ( blsdict [ 'nbestperiods' ] )
nbestperiods = blsdict [ 'nbestperiods' ] [ : nperiods ]
# get rid of nans first and sigclip
stimes , smags , serrs = sigclip_magseries ( times , mags , errs , magsarefluxes = magsarefluxes , sigclip = sigclip )
# make sure there are enough points to calculate a spectrum
if len ( stimes ) > 9 and len ( smags ) > 9 and len ( serrs ) > 9 :
    nbestsnrs = [ ]
    transitdepth , transitduration = [ ] , [ ]
    nphasebins , transingressbin , transegressbin = [ ] , [ ] , [ ]
    # keep these around for diagnostics
    allsubtractedmags = [ ]
    allphasedmags = [ ]
    allphases = [ ]
    allblsmodels = [ ]
    # these are refit periods and epochs
    refitperiods = [ ]
    refitepochs = [ ]
    for period in nbestperiods : # get the period interval
        startp = period - perioddeltapercent * period / 100.0
        if startp < 0 :
            startp = period
        endp = period + perioddeltapercent * period / 100.0
        # see if we need to rerun bls _ serial _ pfind
        if not assumeserialbls : # run bls _ serial _ pfind with the kwargs copied over from the
        # initial run . replace only the startp , endp , verbose , sigclip
        # kwarg values
            prevkwargs = blsdict [ 'kwargs' ] . copy ( )
            prevkwargs [ 'verbose' ] = verbose
            prevkwargs [ 'startp' ] = startp
            prevkwargs [ 'endp' ] = endp
            prevkwargs [ 'sigclip' ] = None
            blsres = bls_serial_pfind ( stimes , smags , serrs , ** prevkwargs )
        else :
            blsres = blsdict
        thistransdepth = blsres [ 'blsresult' ] [ 'transdepth' ]
        thistransduration = blsres [ 'blsresult' ] [ 'transduration' ]
        thisbestperiod = blsres [ 'bestperiod' ]
        thistransingressbin = blsres [ 'blsresult' ] [ 'transingressbin' ]
        thistransegressbin = blsres [ 'blsresult' ] [ 'transegressbin' ]
        thisnphasebins = blsdict [ 'kwargs' ] [ 'nphasebins' ]
        stats = _get_bls_stats ( stimes , smags , serrs , thistransdepth , thistransduration , ingressdurationfraction , nphasebins , thistransingressbin , thistransegressbin , thisbestperiod , thisnphasebins , magsarefluxes = magsarefluxes , verbose = verbose )
        # update the lists with results from this peak
        nbestsnrs . append ( stats [ 'snr' ] )
        transitdepth . append ( stats [ 'transitdepth' ] )
        transitduration . append ( stats [ 'transitduration' ] )
        transingressbin . append ( stats [ 'transingressbin' ] )
        transegressbin . append ( stats [ 'transegressbin' ] )
        nphasebins . append ( stats [ 'nphasebins' ] )
        # update the refit periods and epochs
        refitperiods . append ( stats [ 'period' ] )
        refitepochs . append ( stats [ 'epoch' ] )
        # update the diagnostics
        allsubtractedmags . append ( stats [ 'subtractedmags' ] )
        allphasedmags . append ( stats [ 'phasedmags' ] )
        allphases . append ( stats [ 'phases' ] )
        allblsmodels . append ( stats [ 'blsmodel' ] )
    # done with working on each peak
# if there aren ' t enough points in the mag series , bail out
else :
    LOGERROR ( 'no good detections for these times and mags, skipping...' )
    nbestsnrs = None
    transitdepth , transitduration = None , None
    nphasebins , transingressbin , transegressbin = None , None , None
    allsubtractedmags , allphases , allphasedmags = None , None , None
return { 'npeaks' : npeaks , 'period' : refitperiods , 'epoch' : refitepochs , 'snr' : nbestsnrs , 'transitdepth' : transitdepth , 'transitduration' : transitduration , 'nphasebins' : nphasebins , 'transingressbin' : transingressbin , 'transegressbin' : transegressbin , 'allblsmodels' : allblsmodels , 'allsubtractedmags' : allsubtractedmags , 'allphasedmags' : allphasedmags , 'allphases' : allphases }


signature:
def massradius ( age , planetdist , coremass , mass = 'massjupiter' , radius = 'radiusjupiter' ) :
    '''This function gets the Fortney mass - radius relation for planets .
 Parameters
 age : float
 This should be one of : 0.3 , 1.0 , 4.5 [ in Gyr ] .
 planetdist : float
 This should be one of : 0.02 , 0.045 , 0.1 , 1.0 , 9.5 [ in AU ]
 coremass : int
 This should be one of : 0 , 10 , 25 , 50 , 100 [ in Mearth ]
 mass : { ' massjupiter ' , ' massearth ' }
 Sets the mass units .
 radius : str
 Sets the radius units . Only ' radiusjupiter ' is used for now .
 Returns
 dict
 A dict of the following form is returned : :
 { ' mass ' : an array containing the masses to plot ) ,
 ' radius ' : an array containing the radii to plot }
 These can be passed to a plotting routine to make mass - radius plot for
 the specified age , planet - star distance , and core - mass .'''

implementation:
MR = { 0.3 : MASSESRADII_0_3GYR , 1.0 : MASSESRADII_1_0GYR , 4.5 : MASSESRADII_4_5GYR }
if age not in MR :
    print ( 'given age not in Fortney 2007, returning...' )
    return
massradius = MR [ age ]
if ( planetdist in massradius ) and ( coremass in massradius [ planetdist ] ) :
    print ( 'getting % Gyr M-R for planet dist %s AU, ' 'core mass %s Mearth...' % ( age , planetdist , coremass ) )
    massradrelation = massradius [ planetdist ] [ coremass ]
    outdict = { 'mass' : array ( massradrelation [ mass ] ) , 'radius' : array ( massradrelation [ radius ] ) }
    return outdict


signature:
def _collect_tfa_stats ( task ) :
    '''This is a parallel worker to gather LC stats .
 task [ 0 ] = lcfile
 task [ 1 ] = lcformat
 task [ 2 ] = lcformatdir
 task [ 3 ] = timecols
 task [ 4 ] = magcols
 task [ 5 ] = errcols
 task [ 6 ] = custom _ bandpasses'''

implementation:
try :
    ( lcfile , lcformat , lcformatdir , timecols , magcols , errcols , custom_bandpasses ) = task
    try :
        formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
        if formatinfo :
            ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
        else :
            LOGERROR ( "can't figure out the light curve format" )
            return None
    except Exception as e :
        LOGEXCEPTION ( "can't figure out the light curve format" )
        return None
    # override the default timecols , magcols , and errcols
    # using the ones provided to the function
    if timecols is None :
        timecols = dtimecols
    if magcols is None :
        magcols = dmagcols
    if errcols is None :
        errcols = derrcols
    # get the LC into a dict
    lcdict = readerfunc ( lcfile )
    # this should handle lists / tuples being returned by readerfunc
    # we assume that the first element is the actual lcdict
    # FIXME : figure out how to not need this assumption
    if ( ( isinstance ( lcdict , ( list , tuple ) ) ) and ( isinstance ( lcdict [ 0 ] , dict ) ) ) :
        lcdict = lcdict [ 0 ]
    # collect the necessary stats for this light curve
    # 1 . number of observations
    # 2 . median mag
    # 3 . eta _ normal
    # 4 . MAD
    # 5 . objectid
    # 6 . get mags and colors from objectinfo if there ' s one in lcdict
    if 'objectid' in lcdict :
        objectid = lcdict [ 'objectid' ]
    elif 'objectinfo' in lcdict and 'objectid' in lcdict [ 'objectinfo' ] :
        objectid = lcdict [ 'objectinfo' ] [ 'objectid' ]
    elif 'objectinfo' in lcdict and 'hatid' in lcdict [ 'objectinfo' ] :
        objectid = lcdict [ 'objectinfo' ] [ 'hatid' ]
    else :
        LOGERROR ( 'no objectid present in lcdict for LC %s, ' 'using filename prefix as objectid' % lcfile )
        objectid = os . path . splitext ( os . path . basename ( lcfile ) ) [ 0 ]
    if 'objectinfo' in lcdict :
        colorfeat = starfeatures . color_features ( lcdict [ 'objectinfo' ] , deredden = False , custom_bandpasses = custom_bandpasses )
    else :
        LOGERROR ( 'no objectinfo dict in lcdict, ' 'could not get magnitudes for LC %s, ' 'cannot use for TFA template ensemble' % lcfile )
        return None
    # this is the initial dict
    resultdict = { 'objectid' : objectid , 'ra' : lcdict [ 'objectinfo' ] [ 'ra' ] , 'decl' : lcdict [ 'objectinfo' ] [ 'decl' ] , 'colorfeat' : colorfeat , 'lcfpath' : os . path . abspath ( lcfile ) , 'lcformat' : lcformat , 'lcformatdir' : lcformatdir , 'timecols' : timecols , 'magcols' : magcols , 'errcols' : errcols }
    for tcol , mcol , ecol in zip ( timecols , magcols , errcols ) :
        try : # dereference the columns and get them from the lcdict
            if '.' in tcol :
                tcolget = tcol . split ( '.' )
            else :
                tcolget = [ tcol ]
            times = _dict_get ( lcdict , tcolget )
            if '.' in mcol :
                mcolget = mcol . split ( '.' )
            else :
                mcolget = [ mcol ]
            mags = _dict_get ( lcdict , mcolget )
            if '.' in ecol :
                ecolget = ecol . split ( '.' )
            else :
                ecolget = [ ecol ]
            errs = _dict_get ( lcdict , ecolget )
            # normalize here if not using special normalization
            if normfunc is None :
                ntimes , nmags = normalize_magseries ( times , mags , magsarefluxes = magsarefluxes )
                times , mags , errs = ntimes , nmags , errs
            # get the variability features for this object
            varfeat = varfeatures . all_nonperiodic_features ( times , mags , errs )
            resultdict [ mcol ] = varfeat
        except Exception as e :
            LOGEXCEPTION ( '%s, magcol: %s, probably ran into all-nans' % ( lcfile , mcol ) )
            resultdict [ mcol ] = { 'ndet' : 0 , 'mad' : np . nan , 'eta_normal' : np . nan }
    return resultdict
except Exception as e :
    LOGEXCEPTION ( 'could not execute get_tfa_stats for task: %s' % repr ( task ) )
    return None


signature:
def _reform_templatelc_for_tfa ( task ) :
    '''This is a parallel worker that reforms light curves for TFA .
 task [ 0 ] = lcfile
 task [ 1 ] = lcformat
 task [ 2 ] = lcformatdir
 task [ 3 ] = timecol
 task [ 4 ] = magcol
 task [ 5 ] = errcol
 task [ 6 ] = timebase
 task [ 7 ] = interpolate _ type
 task [ 8 ] = sigclip'''

implementation:
try :
    ( lcfile , lcformat , lcformatdir , tcol , mcol , ecol , timebase , interpolate_type , sigclip ) = task
    try :
        formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
        if formatinfo :
            ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
        else :
            LOGERROR ( "can't figure out the light curve format" )
            return None
    except Exception as e :
        LOGEXCEPTION ( "can't figure out the light curve format" )
        return None
    # get the LC into a dict
    lcdict = readerfunc ( lcfile )
    # this should handle lists / tuples being returned by readerfunc
    # we assume that the first element is the actual lcdict
    # FIXME : figure out how to not need this assumption
    if ( ( isinstance ( lcdict , ( list , tuple ) ) ) and ( isinstance ( lcdict [ 0 ] , dict ) ) ) :
        lcdict = lcdict [ 0 ]
    outdict = { }
    # dereference the columns and get them from the lcdict
    if '.' in tcol :
        tcolget = tcol . split ( '.' )
    else :
        tcolget = [ tcol ]
    times = _dict_get ( lcdict , tcolget )
    if '.' in mcol :
        mcolget = mcol . split ( '.' )
    else :
        mcolget = [ mcol ]
    mags = _dict_get ( lcdict , mcolget )
    if '.' in ecol :
        ecolget = ecol . split ( '.' )
    else :
        ecolget = [ ecol ]
    errs = _dict_get ( lcdict , ecolget )
    # normalize here if not using special normalization
    if normfunc is None :
        ntimes , nmags = normalize_magseries ( times , mags , magsarefluxes = magsarefluxes )
    times , mags , errs = ntimes , nmags , errs
    # now we ' ll do : 1 . sigclip , 2 . reform to timebase , 3 . renorm to zero
    # 1 . sigclip as requested
    stimes , smags , serrs = sigclip_magseries ( times , mags , errs , sigclip = sigclip )
    # 2 . now , we ' ll renorm to the timebase
    mags_interpolator = spi . interp1d ( stimes , smags , kind = interpolate_type , fill_value = 'extrapolate' )
    errs_interpolator = spi . interp1d ( stimes , serrs , kind = interpolate_type , fill_value = 'extrapolate' )
    interpolated_mags = mags_interpolator ( timebase )
    interpolated_errs = errs_interpolator ( timebase )
    # 3 . renorm to zero
    magmedian = np . median ( interpolated_mags )
    renormed_mags = interpolated_mags - magmedian
    # update the dict
    outdict = { 'mags' : renormed_mags , 'errs' : interpolated_errs , 'origmags' : interpolated_mags }
    # done with this magcol
    return outdict
except Exception as e :
    LOGEXCEPTION ( 'reform LC task failed: %s' % repr ( task ) )
    return None


signature:
def tfa_templates_lclist ( lclist , lcinfo_pkl = None , outfile = None , target_template_frac = 0.1 , max_target_frac_obs = 0.25 , min_template_number = 10 , max_template_number = 1000 , max_rms = 0.15 , max_mult_above_magmad = 1.5 , max_mult_above_mageta = 1.5 , xieta_bins = 20 , mag_bandpass = 'sdssr' , custom_bandpasses = None , mag_bright_limit = 10.0 , mag_faint_limit = 12.0 , process_template_lcs = True , template_sigclip = 5.0 , template_interpolate = 'nearest' , lcformat = 'hat-sql' , lcformatdir = None , timecols = None , magcols = None , errcols = None , nworkers = NCPUS , maxworkertasks = 1000 , ) :
    '''This selects template objects for TFA .
 Selection criteria for TFA template ensemble objects :
 - not variable : use a poly fit to the mag - MAD relation and eta - normal
 variability index to get nonvar objects
 - not more than 10 % of the total number of objects in the field or
 ` max _ tfa _ templates ` at most
 - allow shuffling of the templates if the target ends up in them
 - nothing with less than the median number of observations in the field
 - sigma - clip the input time series observations
 - TODO : uniform sampling in tangent plane coordinates ( we ' ll need ra and
 decl )
 This also determines the effective cadence that all TFA LCs will be binned
 to as the template LC with the largest number of non - nan observations will
 be used . All template LCs will be renormed to zero .
 Parameters
 lclist : list of str
 This is a list of light curves to use as input to generate the template
 set .
 lcinfo _ pkl : str or None
 If provided , is a file path to a pickle file created by this function on
 a previous run containing the LC information . This will be loaded
 directly instead of having to re - run LC info collection .
 outfile : str or None
 This is the pickle filename to which the TFA template list will be
 written to . If None , a default file name will be used for this .
 target _ template _ frac : float
 This is the fraction of total objects in lclist to use for the number of
 templates .
 max _ target _ frac _ obs : float
 This sets the number of templates to generate if the number of
 observations for the light curves is smaller than the number of objects
 in the collection . The number of templates will be set to this fraction
 of the number of observations if this is the case .
 min _ template _ number : int
 This is the minimum number of templates to generate .
 max _ template _ number : int
 This is the maximum number of templates to generate . If
 ` target _ template _ frac ` times the number of objects is greater than
 ` max _ template _ number ` , only ` max _ template _ number ` templates will be
 used .
 max _ rms : float
 This is the maximum light curve RMS for an object to consider it as a
 possible template ensemble member .
 max _ mult _ above _ magmad : float
 This is the maximum multiplier above the mag - RMS fit to consider an
 object as variable and thus not part of the template ensemble .
 max _ mult _ above _ mageta : float
 This is the maximum multiplier above the mag - eta ( variable index ) fit to
 consider an object as variable and thus not part of the template
 ensemble .
 mag _ bandpass : str
 This sets the key in the light curve dict ' s objectinfo dict to use as
 the canonical magnitude for the object and apply any magnitude limits
 to .
 custom _ bandpasses : dict or None
 This can be used to provide any custom band name keys to the star
 feature collection function .
 mag _ bright _ limit : float or list of floats
 This sets the brightest mag ( in the ` mag _ bandpass ` filter ) for a
 potential member of the TFA template ensemble . If this is a single
 float , the value will be used for all magcols . If this is a list of
 floats with len = len ( magcols ) , the specific bright limits will be used
 for each magcol individually .
 mag _ faint _ limit : float or list of floats
 This sets the faintest mag ( in the ` mag _ bandpass ` filter ) for a
 potential member of the TFA template ensemble . If this is a single
 float , the value will be used for all magcols . If this is a list of
 floats with len = len ( magcols ) , the specific faint limits will be used
 for each magcol individually .
 process _ template _ lcs : bool
 If True , will reform the template light curves to the chosen
 time - base . If False , will only select light curves for templates but not
 process them . This is useful for initial exploration of how the template
 LC are selected .
 template _ sigclip : float or sequence of floats or None
 This sets the sigma - clip to be applied to the template light curves .
 template _ interpolate : str
 This sets the kwarg to pass to ` scipy . interpolate . interp1d ` to set the
 kind of interpolation to use when reforming light curves to the TFA
 template timebase .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 timecols : list of str or None
 The timecol keys to use from the lcdict in calculating the features .
 magcols : list of str or None
 The magcol keys to use from the lcdict in calculating the features .
 errcols : list of str or None
 The errcol keys to use from the lcdict in calculating the features .
 nworkers : int
 The number of parallel workers to launch .
 maxworkertasks : int
 The maximum number of tasks to run per worker before it is replaced by a
 fresh one .
 Returns
 dict
 This function returns a dict that can be passed directly to
 ` apply _ tfa _ magseries ` below . It can optionally produce a pickle with the
 same dict , which can also be passed to that function .'''

implementation:
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
# override the default timecols , magcols , and errcols
# using the ones provided to the function
if timecols is None :
    timecols = dtimecols
if magcols is None :
    magcols = dmagcols
if errcols is None :
    errcols = derrcols
LOGINFO ( 'collecting light curve information for %s objects in list...' % len ( lclist ) )
# check if we have cached results for this run
# case where we provide a cache info pkl directly
if lcinfo_pkl and os . path . exists ( lcinfo_pkl ) :
    with open ( lcinfo_pkl , 'rb' ) as infd :
        results = pickle . load ( infd )
# case where we don ' t have an info pickle or an outfile
elif ( ( not outfile ) and os . path . exists ( 'tfa-collected-lcfinfo-%s.pkl' % lcformat ) ) :
    with open ( 'tfa-collected-lcfinfo-%s.pkl' % lcformat , 'rb' ) as infd :
        results = pickle . load ( infd )
# case where we don ' t have an info pickle but do have an outfile
elif ( outfile and os . path . exists ( 'tfa-collected-lcfinfo-%s-%s' % ( lcformat , os . path . basename ( outfile ) ) ) ) :
    with open ( 'tfa-collected-lcinfo-%s-%s' % ( lcformat , os . path . basename ( outfile ) ) , 'rb' ) as infd :
        results = pickle . load ( infd )
# case where we have to redo the LC info collection
else : # first , we ' ll collect the light curve info
    tasks = [ ( x , lcformat , lcformat , timecols , magcols , errcols , custom_bandpasses ) for x in lclist ]
    pool = mp . Pool ( nworkers , maxtasksperchild = maxworkertasks )
    results = pool . map ( _collect_tfa_stats , tasks )
    pool . close ( )
    pool . join ( )
    # save these results so we don ' t have to redo if something breaks here
    if not outfile :
        with open ( 'tfa-collected-lcinfo-%s.pkl' % lcformat , 'wb' ) as outfd :
            pickle . dump ( results , outfd , pickle . HIGHEST_PROTOCOL )
    else :
        with open ( 'tfa-collected-lcinfo-%s-%s' % ( lcformat , os . path . basename ( outfile ) ) , 'wb' ) as outfd :
            pickle . dump ( results , outfd , pickle . HIGHEST_PROTOCOL )
# now , go through the light curve information
# find the center RA and center DEC - > median of all LC RAs and DECs
all_ras = np . array ( [ res [ 'ra' ] for res in results ] )
all_decls = np . array ( [ res [ 'decl' ] for res in results ] )
center_ra = np . nanmedian ( all_ras )
center_decl = np . nanmedian ( all_decls )
outdict = { 'timecols' : [ ] , 'magcols' : [ ] , 'errcols' : [ ] , 'center_ra' : center_ra , 'center_decl' : center_decl , }
# for each magcol , we ' ll generate a separate template list
for tcol , mcol , ecol in zip ( timecols , magcols , errcols ) :
    if '.' in tcol :
        tcolget = tcol . split ( '.' )
    else :
        tcolget = [ tcol ]
    # these are the containers for possible template collection LC info
    ( lcmag , lcmad , lceta , lcndet , lcobj , lcfpaths , lcra , lcdecl ) = [ ] , [ ] , [ ] , [ ] , [ ] , [ ] , [ ] , [ ]
    outdict [ 'timecols' ] . append ( tcol )
    outdict [ 'magcols' ] . append ( mcol )
    outdict [ 'errcols' ] . append ( ecol )
    # add to the collection of all light curves
    outdict [ mcol ] = { 'collection' : { 'mag' : [ ] , 'mad' : [ ] , 'eta' : [ ] , 'ndet' : [ ] , 'obj' : [ ] , 'lcf' : [ ] , 'ra' : [ ] , 'decl' : [ ] } }
    LOGINFO ( 'magcol: %s, collecting prospective template LC info...' % mcol )
    # collect the template LCs for this magcol
    for result in results : # we ' ll only append objects that have all of these elements
        try :
            thismag = result [ 'colorfeat' ] [ mag_bandpass ]
            thismad = result [ mcol ] [ 'mad' ]
            thiseta = result [ mcol ] [ 'eta_normal' ]
            thisndet = result [ mcol ] [ 'ndet' ]
            thisobj = result [ 'objectid' ]
            thislcf = result [ 'lcfpath' ]
            thisra = result [ 'ra' ]
            thisdecl = result [ 'decl' ]
            outdict [ mcol ] [ 'collection' ] [ 'mag' ] . append ( thismag )
            outdict [ mcol ] [ 'collection' ] [ 'mad' ] . append ( thismad )
            outdict [ mcol ] [ 'collection' ] [ 'eta' ] . append ( thiseta )
            outdict [ mcol ] [ 'collection' ] [ 'ndet' ] . append ( thisndet )
            outdict [ mcol ] [ 'collection' ] [ 'obj' ] . append ( thisobj )
            outdict [ mcol ] [ 'collection' ] [ 'lcf' ] . append ( thislcf )
            outdict [ mcol ] [ 'collection' ] [ 'ra' ] . append ( thisra )
            outdict [ mcol ] [ 'collection' ] [ 'decl' ] . append ( thisdecl )
            # check if we have more than one bright or faint limit elem
            if isinstance ( mag_bright_limit , ( list , tuple ) ) :
                use_bright_maglim = mag_bright_limit [ magcols . index ( mcol ) ]
            else :
                use_bright_maglim = mag_bright_limit
            if isinstance ( mag_faint_limit , ( list , tuple ) ) :
                use_faint_maglim = mag_faint_limit [ magcols . index ( mcol ) ]
            else :
                use_faint_maglim = mag_faint_limit
            # make sure the object lies in the mag limits and RMS limits we
            # set before to try to accept it into the TFA ensemble
            if ( ( use_bright_maglim < thismag < use_faint_maglim ) and ( 1.4826 * thismad < max_rms ) ) :
                lcmag . append ( thismag )
                lcmad . append ( thismad )
                lceta . append ( thiseta )
                lcndet . append ( thisndet )
                lcobj . append ( thisobj )
                lcfpaths . append ( thislcf )
                lcra . append ( thisra )
                lcdecl . append ( thisdecl )
        except Exception as e :
            pass
    # make sure we have enough LCs to work on
    if len ( lcobj ) >= min_template_number :
        LOGINFO ( 'magcol: %s, %s objects eligible for ' 'template selection after filtering on mag ' 'limits (%s, %s) and max RMS (%s)' % ( mcol , len ( lcobj ) , mag_bright_limit , mag_faint_limit , max_rms ) )
        lcmag = np . array ( lcmag )
        lcmad = np . array ( lcmad )
        lceta = np . array ( lceta )
        lcndet = np . array ( lcndet )
        lcobj = np . array ( lcobj )
        lcfpaths = np . array ( lcfpaths )
        lcra = np . array ( lcra )
        lcdecl = np . array ( lcdecl )
        sortind = np . argsort ( lcmag )
        lcmag = lcmag [ sortind ]
        lcmad = lcmad [ sortind ]
        lceta = lceta [ sortind ]
        lcndet = lcndet [ sortind ]
        lcobj = lcobj [ sortind ]
        lcfpaths = lcfpaths [ sortind ]
        lcra = lcra [ sortind ]
        lcdecl = lcdecl [ sortind ]
        # 1 . get the mag - MAD relation
        # this is needed for spline fitting
        # should take care of the pesky ' x must be strictly increasing ' bit
        splfit_ind = np . diff ( lcmag ) > 0.0
        splfit_ind = np . concatenate ( ( np . array ( [ True ] ) , splfit_ind ) )
        fit_lcmag = lcmag [ splfit_ind ]
        fit_lcmad = lcmad [ splfit_ind ]
        fit_lceta = lceta [ splfit_ind ]
        magmadfit = np . poly1d ( np . polyfit ( fit_lcmag , fit_lcmad , 2 ) )
        magmadind = lcmad / magmadfit ( lcmag ) < max_mult_above_magmad
        # 2 . get the mag - eta relation
        magetafit = np . poly1d ( np . polyfit ( fit_lcmag , fit_lceta , 2 ) )
        magetaind = magetafit ( lcmag ) / lceta < max_mult_above_mageta
        # 3 . get the median ndet
        median_ndet = np . median ( lcndet )
        ndetind = lcndet >= median_ndet
        # form the final template ensemble
        templateind = magmadind & magetaind & ndetind
        # check again if we have enough LCs in the template
        if templateind . sum ( ) >= min_template_number :
            LOGINFO ( 'magcol: %s, %s objects selectable for TFA templates' % ( mcol , templateind . sum ( ) ) )
            templatemag = lcmag [ templateind ]
            templatemad = lcmad [ templateind ]
            templateeta = lceta [ templateind ]
            templatendet = lcndet [ templateind ]
            templateobj = lcobj [ templateind ]
            templatelcf = lcfpaths [ templateind ]
            templatera = lcra [ templateind ]
            templatedecl = lcdecl [ templateind ]
            # now , check if we have no more than the required fraction of
            # TFA templates
            target_number_templates = int ( target_template_frac * len ( results ) )
            if target_number_templates > max_template_number :
                target_number_templates = max_template_number
            LOGINFO ( 'magcol: %s, selecting %s TFA templates randomly' % ( mcol , target_number_templates ) )
            # FIXME : how do we select uniformly in xi - eta ?
            # 1 . 2D histogram the data into binsize ( nx , ny )
            # 2 . random uniform select from 0 to nx - 1 , 0 to ny - 1
            # 3 . pick object from selected bin
            # 4 . continue until we have target _ number _ templates
            # 5 . make sure the same object isn ' t picked twice
            # get the xi - eta
            template_cxi , template_ceta = coordutils . xieta_from_radecl ( templatera , templatedecl , center_ra , center_decl )
            cxi_bins = np . linspace ( template_cxi . min ( ) , template_cxi . max ( ) , num = xieta_bins )
            ceta_bins = np . linspace ( template_ceta . min ( ) , template_ceta . max ( ) , num = xieta_bins )
            digitized_cxi_inds = np . digitize ( template_cxi , cxi_bins )
            digitized_ceta_inds = np . digitize ( template_ceta , ceta_bins )
            # pick target _ number _ templates indexes out of the bins
            targetind = npr . choice ( xieta_bins , target_number_templates , replace = True )
            # put together the template lists
            selected_template_obj = [ ]
            selected_template_lcf = [ ]
            selected_template_ndet = [ ]
            selected_template_ra = [ ]
            selected_template_decl = [ ]
            selected_template_mag = [ ]
            selected_template_mad = [ ]
            selected_template_eta = [ ]
            for ind in targetind :
                pass
            # select random uniform objects from the template candidates
            targetind = npr . choice ( templateobj . size , target_number_templates , replace = False )
            templatemag = templatemag [ targetind ]
            templatemad = templatemad [ targetind ]
            templateeta = templateeta [ targetind ]
            templatendet = templatendet [ targetind ]
            templateobj = templateobj [ targetind ]
            templatelcf = templatelcf [ targetind ]
            templatera = templatera [ targetind ]
            templatedecl = templatedecl [ targetind ]
            # get the max ndet so far to use that LC as the timebase
            maxndetind = templatendet == templatendet . max ( )
            timebaselcf = templatelcf [ maxndetind ] [ 0 ]
            timebasendet = templatendet [ maxndetind ] [ 0 ]
            LOGINFO ( 'magcol: %s, selected %s as template time ' 'base LC with %s observations' % ( mcol , timebaselcf , timebasendet ) )
            if process_template_lcs :
                timebaselcdict = readerfunc ( timebaselcf )
                if ( ( isinstance ( timebaselcdict , ( list , tuple ) ) ) and ( isinstance ( timebaselcdict [ 0 ] , dict ) ) ) :
                    timebaselcdict = timebaselcdict [ 0 ]
                # this is the timebase to use for all of the templates
                timebase = _dict_get ( timebaselcdict , tcolget )
            else :
                timebase = None
            # also check if the number of templates is longer than the
            # actual timebase of the observations . this will cause issues
            # with overcorrections and will probably break TFA
            if target_number_templates > timebasendet :
                LOGWARNING ( 'The number of TFA templates (%s) is ' 'larger than the number of observations ' 'of the time base (%s). This will likely ' 'overcorrect all light curves to a ' 'constant level. ' 'Will use up to %s x timebase ndet ' 'templates instead' % ( target_number_templates , timebasendet , max_target_frac_obs ) )
                # regen the templates based on the new number
                newmaxtemplates = int ( max_target_frac_obs * timebasendet )
                # choose this number out of the already chosen templates
                # randomly
                LOGWARNING ( 'magcol: %s, re-selecting %s TFA ' 'templates randomly' % ( mcol , newmaxtemplates ) )
                # FIXME : how do we select uniformly in ra - decl ?
                # 1 . 2D histogram the data into binsize ( nx , ny )
                # 2 . random uniform select from 0 to nx - 1 , 0 to ny - 1
                # 3 . pick object from selected bin
                # 4 . continue until we have target _ number _ templates
                # 5 . make sure the same object isn ' t picked twice
                # select random uniform objects from the template candidates
                targetind = npr . choice ( templateobj . size , newmaxtemplates , replace = False )
                templatemag = templatemag [ targetind ]
                templatemad = templatemad [ targetind ]
                templateeta = templateeta [ targetind ]
                templatendet = templatendet [ targetind ]
                templateobj = templateobj [ targetind ]
                templatelcf = templatelcf [ targetind ]
                templatera = templatera [ targetind ]
                templatedecl = templatedecl [ targetind ]
                # get the max ndet so far to use that LC as the timebase
                maxndetind = templatendet == templatendet . max ( )
                timebaselcf = templatelcf [ maxndetind ] [ 0 ]
                timebasendet = templatendet [ maxndetind ] [ 0 ]
                LOGWARNING ( 'magcol: %s, re-selected %s as template time ' 'base LC with %s observations' % ( mcol , timebaselcf , timebasendet ) )
                if process_template_lcs :
                    timebaselcdict = readerfunc ( timebaselcf )
                    if ( ( isinstance ( timebaselcdict , ( list , tuple ) ) ) and ( isinstance ( timebaselcdict [ 0 ] , dict ) ) ) :
                        timebaselcdict = timebaselcdict [ 0 ]
                    # this is the timebase to use for all of the templates
                    timebase = _dict_get ( timebaselcdict , tcolget )
                else :
                    timebase = None
            # end of check for ntemplates > timebase ndet
            if process_template_lcs :
                LOGINFO ( 'magcol: %s, reforming TFA template LCs to ' ' chosen timebase...' % mcol )
                # reform all template LCs to this time base , normalize to
                # zero , and sigclip as requested . this is a parallel op
                # first , we ' ll collect the light curve info
                tasks = [ ( x , lcformat , lcformatdir , tcol , mcol , ecol , timebase , template_interpolate , template_sigclip ) for x in templatelcf ]
                pool = mp . Pool ( nworkers , maxtasksperchild = maxworkertasks )
                reform_results = pool . map ( _reform_templatelc_for_tfa , tasks )
                pool . close ( )
                pool . join ( )
                # generate a 2D array for the template magseries with
                # dimensions = ( n _ objects , n _ lcpoints )
                template_magseries = np . array ( [ x [ 'mags' ] for x in reform_results ] )
                template_errseries = np . array ( [ x [ 'errs' ] for x in reform_results ] )
            else :
                template_magseries = None
                template_errseries = None
            # put everything into a templateinfo dict for this magcol
            outdict [ mcol ] . update ( { 'timebaselcf' : timebaselcf , 'timebase' : timebase , 'trendfits' : { 'mag-mad' : magmadfit , 'mag-eta' : magetafit } , 'template_objects' : templateobj , 'template_ra' : templatera , 'template_decl' : templatedecl , 'template_mag' : templatemag , 'template_mad' : templatemad , 'template_eta' : templateeta , 'template_ndet' : templatendet , 'template_magseries' : template_magseries , 'template_errseries' : template_errseries } )
            # make a KDTree on the template coordinates
            outdict [ mcol ] [ 'template_radecl_kdtree' ] = ( coordutils . make_kdtree ( templatera , templatedecl ) )
        # if we don ' t have enough , return nothing for this magcol
        else :
            LOGERROR ( 'not enough objects meeting requested ' 'MAD, eta, ndet conditions to ' 'select templates for magcol: %s' % mcol )
            continue
    else :
        LOGERROR ( 'nobjects: %s, not enough in requested mag range to ' 'select templates for magcol: %s' % ( len ( lcobj ) , mcol ) )
        continue
    # make the plots for mag - MAD / mag - eta relation and fits used
    plt . plot ( lcmag , lcmad , marker = 'o' , linestyle = 'none' , ms = 1.0 )
    modelmags = np . linspace ( lcmag . min ( ) , lcmag . max ( ) , num = 1000 )
    plt . plot ( modelmags , outdict [ mcol ] [ 'trendfits' ] [ 'mag-mad' ] ( modelmags ) )
    plt . yscale ( 'log' )
    plt . xlabel ( 'catalog magnitude' )
    plt . ylabel ( 'light curve MAD' )
    plt . title ( 'catalog mag vs. light curve MAD and fit' )
    plt . savefig ( 'catmag-%s-lcmad-fit.png' % mcol , bbox_inches = 'tight' )
    plt . close ( 'all' )
    plt . plot ( lcmag , lceta , marker = 'o' , linestyle = 'none' , ms = 1.0 )
    modelmags = np . linspace ( lcmag . min ( ) , lcmag . max ( ) , num = 1000 )
    plt . plot ( modelmags , outdict [ mcol ] [ 'trendfits' ] [ 'mag-eta' ] ( modelmags ) )
    plt . yscale ( 'log' )
    plt . xlabel ( 'catalog magnitude' )
    plt . ylabel ( 'light curve eta variable index' )
    plt . title ( 'catalog mag vs. light curve eta and fit' )
    plt . savefig ( 'catmag-%s-lceta-fit.png' % mcol , bbox_inches = 'tight' )
    plt . close ( 'all' )
# end of operating on each magcol
# save the templateinfo dict to a pickle if requested
if outfile :
    if outfile . endswith ( '.gz' ) :
        outfd = gzip . open ( outfile , 'wb' )
    else :
        outfd = open ( outfile , 'wb' )
    with outfd :
        pickle . dump ( outdict , outfd , protocol = pickle . HIGHEST_PROTOCOL )
# return the templateinfo dict
return outdict


signature:
def apply_tfa_magseries ( lcfile , timecol , magcol , errcol , templateinfo , mintemplatedist_arcmin = 10.0 , lcformat = 'hat-sql' , lcformatdir = None , interp = 'nearest' , sigclip = 5.0 ) :
    '''This applies the TFA correction to an LC given TFA template information .
 Parameters
 lcfile : str
 This is the light curve file to apply the TFA correction to .
 timecol , magcol , errcol : str
 These are the column keys in the lcdict for the LC file to apply the TFA
 correction to .
 templateinfo : dict or str
 This is either the dict produced by ` tfa _ templates _ lclist ` or the pickle
 produced by the same function .
 mintemplatedist _ arcmin : float
 This sets the minimum distance required from the target object for
 objects in the TFA template ensemble . Objects closer than this distance
 will be removed from the ensemble .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 interp : str
 This is passed to scipy . interpolate . interp1d as the kind of
 interpolation to use when reforming this light curve to the timebase of
 the TFA templates .
 sigclip : float or sequence of two floats or None
 This is the sigma clip to apply to this light curve before running TFA
 on it .
 Returns
 str
 This returns the filename of the light curve file generated after TFA
 applications . This is a pickle ( that can be read by ` lcproc . read _ pklc ` )
 in the same directory as ` lcfile ` . The ` magcol ` will be encoded in the
 filename , so each ` magcol ` in ` lcfile ` gets its own output file .'''

implementation:
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
# get the templateinfo from a pickle if necessary
if isinstance ( templateinfo , str ) and os . path . exists ( templateinfo ) :
    with open ( templateinfo , 'rb' ) as infd :
        templateinfo = pickle . load ( infd )
lcdict = readerfunc ( lcfile )
if ( ( isinstance ( lcdict , ( tuple , list ) ) ) and isinstance ( lcdict [ 0 ] , dict ) ) :
    lcdict = lcdict [ 0 ]
objectid = lcdict [ 'objectid' ]
# this is the initial template array
tmagseries = templateinfo [ magcol ] [ 'template_magseries' ] [ : : ]
# if the object itself is in the template ensemble , remove it
if objectid in templateinfo [ magcol ] [ 'template_objects' ] :
    LOGWARNING ( 'object %s found in the TFA template ensemble, removing...' % objectid )
    templateind = templateinfo [ magcol ] [ 'template_objects' ] == objectid
    # get the objects in the tmagseries not corresponding to the current
    # object ' s index
    tmagseries = tmagseries [ ~ templateind , : ]
# check if there are close matches to the current object in the templates
object_matches = coordutils . conesearch_kdtree ( templateinfo [ magcol ] [ 'template_radecl_kdtree' ] , lcdict [ 'objectinfo' ] [ 'ra' ] , lcdict [ 'objectinfo' ] [ 'decl' ] , mintemplatedist_arcmin / 60.0 )
if len ( object_matches ) > 0 :
    LOGWARNING ( "object %s is within %.1f arcminutes of %s " "template objects. Will remove these objects " "from the template applied to this object." % ( objectid , mintemplatedist_arcmin , len ( object_matches ) ) )
    removalind = np . full ( templateinfo [ magcol ] [ 'template_objects' ] . size , False , dtype = np . bool )
    removalind [ np . array ( object_matches ) ] = True
    tmagseries = tmagseries [ ~ removalind , : ]
# finally , proceed to TFA
# this is the normal matrix
normal_matrix = np . dot ( tmagseries , tmagseries . T )
# get the inverse of the matrix
normal_matrix_inverse = spla . pinv2 ( normal_matrix )
# get the timebase from the template
timebase = templateinfo [ magcol ] [ 'timebase' ]
# use this to reform the target lc in the same manner as that for a TFA
# template LC
reformed_targetlc = _reform_templatelc_for_tfa ( ( lcfile , lcformat , lcformatdir , timecol , magcol , errcol , timebase , interp , sigclip ) )
# calculate the scalar products of the target and template magseries
scalar_products = np . dot ( tmagseries , reformed_targetlc [ 'mags' ] )
# calculate the corrections
corrections = np . dot ( normal_matrix_inverse , scalar_products )
# finally , get the corrected time series for the target object
corrected_magseries = ( reformed_targetlc [ 'origmags' ] - np . dot ( tmagseries . T , corrections ) )
outdict = { 'times' : timebase , 'mags' : corrected_magseries , 'errs' : reformed_targetlc [ 'errs' ] , 'mags_median' : np . median ( corrected_magseries ) , 'mags_mad' : np . median ( np . abs ( corrected_magseries - np . median ( corrected_magseries ) ) ) , 'work' : { 'tmagseries' : tmagseries , 'normal_matrix' : normal_matrix , 'normal_matrix_inverse' : normal_matrix_inverse , 'scalar_products' : scalar_products , 'corrections' : corrections , 'reformed_targetlc' : reformed_targetlc } , }
# we ' ll write back the tfa times and mags to the lcdict
lcdict [ 'tfa' ] = outdict
outfile = os . path . join ( os . path . dirname ( lcfile ) , '%s-tfa-%s-pklc.pkl' % ( squeeze ( objectid ) . replace ( ' ' , '-' ) , magcol ) )
with open ( outfile , 'wb' ) as outfd :
    pickle . dump ( lcdict , outfd , pickle . HIGHEST_PROTOCOL )
return outfile


signature:
def _parallel_tfa_worker ( task ) :
    '''This is a parallel worker for the function below .
 task [ 0 ] = lcfile
 task [ 1 ] = timecol
 task [ 2 ] = magcol
 task [ 3 ] = errcol
 task [ 4 ] = templateinfo
 task [ 5 ] = lcformat
 task [ 6 ] = lcformatdir
 task [ 6 ] = interp
 task [ 7 ] = sigclip'''

implementation:
( lcfile , timecol , magcol , errcol , templateinfo , lcformat , lcformatdir , interp , sigclip , mintemplatedist_arcmin ) = task
try :
    res = apply_tfa_magseries ( lcfile , timecol , magcol , errcol , templateinfo , lcformat = lcformat , lcformatdir = lcformatdir , interp = interp , sigclip = sigclip , mintemplatedist_arcmin = mintemplatedist_arcmin )
    if res :
        LOGINFO ( '%s -> %s TFA OK' % ( lcfile , res ) )
    return res
except Exception as e :
    LOGEXCEPTION ( 'TFA failed for %s' % lcfile )
    return None


signature:
def parallel_tfa_lclist ( lclist , templateinfo , timecols = None , magcols = None , errcols = None , lcformat = 'hat-sql' , lcformatdir = None , interp = 'nearest' , sigclip = 5.0 , mintemplatedist_arcmin = 10.0 , nworkers = NCPUS , maxworkertasks = 1000 ) :
    '''This applies TFA in parallel to all LCs in the given list of file names .
 Parameters
 lclist : str
 This is a list of light curve files to apply TFA correction to .
 templateinfo : dict or str
 This is either the dict produced by ` tfa _ templates _ lclist ` or the pickle
 produced by the same function .
 timecols : list of str or None
 The timecol keys to use from the lcdict in applying TFA corrections .
 magcols : list of str or None
 The magcol keys to use from the lcdict in applying TFA corrections .
 errcols : list of str or None
 The errcol keys to use from the lcdict in applying TFA corrections .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 interp : str
 This is passed to scipy . interpolate . interp1d as the kind of
 interpolation to use when reforming the light curves to the timebase of
 the TFA templates .
 sigclip : float or sequence of two floats or None
 This is the sigma clip to apply to the light curves before running TFA
 on it .
 mintemplatedist _ arcmin : float
 This sets the minimum distance required from the target object for
 objects in the TFA template ensemble . Objects closer than this distance
 will be removed from the ensemble .
 nworkers : int
 The number of parallel workers to launch
 maxworkertasks : int
 The maximum number of tasks per worker allowed before it ' s replaced by a
 fresh one .
 Returns
 dict
 Contains the input file names and output TFA light curve filenames per
 input file organized by each ` magcol ` in ` magcols ` .'''

implementation:
# open the templateinfo first
if isinstance ( templateinfo , str ) and os . path . exists ( templateinfo ) :
    with open ( templateinfo , 'rb' ) as infd :
        templateinfo = pickle . load ( infd )
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
# override the default timecols , magcols , and errcols
# using the ones provided to the function
# we ' ll get the defaults from the templateinfo object
if timecols is None :
    timecols = templateinfo [ 'timecols' ]
if magcols is None :
    magcols = templateinfo [ 'magcols' ]
if errcols is None :
    errcols = templateinfo [ 'errcols' ]
outdict = { }
# run by magcol
for t , m , e in zip ( timecols , magcols , errcols ) :
    tasks = [ ( x , t , m , e , templateinfo , lcformat , lcformatdir , interp , sigclip ) for x in lclist ]
    pool = mp . Pool ( nworkers , maxtasksperchild = maxworkertasks )
    results = pool . map ( _parallel_tfa_worker , tasks )
    pool . close ( )
    pool . join ( )
    outdict [ m ] = results
return outdict


signature:
def parallel_tfa_lcdir ( lcdir , templateinfo , lcfileglob = None , timecols = None , magcols = None , errcols = None , lcformat = 'hat-sql' , lcformatdir = None , interp = 'nearest' , sigclip = 5.0 , mintemplatedist_arcmin = 10.0 , nworkers = NCPUS , maxworkertasks = 1000 ) :
    '''This applies TFA in parallel to all LCs in a directory .
 Parameters
 lcdir : str
 This is the directory containing the light curve files to process . .
 templateinfo : dict or str
 This is either the dict produced by ` tfa _ templates _ lclist ` or the pickle
 produced by the same function .
 lcfileglob : str or None
 The UNIX file glob to use when searching for light curve files in
 ` lcdir ` . If None , the default file glob associated with registered LC
 format provided is used .
 timecols : list of str or None
 The timecol keys to use from the lcdict in applying TFA corrections .
 magcols : list of str or None
 The magcol keys to use from the lcdict in applying TFA corrections .
 errcols : list of str or None
 The errcol keys to use from the lcdict in applying TFA corrections .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 interp : str
 This is passed to scipy . interpolate . interp1d as the kind of
 interpolation to use when reforming the light curves to the timebase of
 the TFA templates .
 sigclip : float or sequence of two floats or None
 This is the sigma clip to apply to the light curves before running TFA
 on it .
 mintemplatedist _ arcmin : float
 This sets the minimum distance required from the target object for
 objects in the TFA template ensemble . Objects closer than this distance
 will be removed from the ensemble .
 nworkers : int
 The number of parallel workers to launch
 maxworkertasks : int
 The maximum number of tasks per worker allowed before it ' s replaced by a
 fresh one .
 Returns
 dict
 Contains the input file names and output TFA light curve filenames per
 input file organized by each ` magcol ` in ` magcols ` .'''

implementation:
# open the templateinfo first
if isinstance ( templateinfo , str ) and os . path . exists ( templateinfo ) :
    with open ( templateinfo , 'rb' ) as infd :
        templateinfo = pickle . load ( infd )
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
# find all the files matching the lcglob in lcdir
if lcfileglob is None :
    lcfileglob = dfileglob
lclist = sorted ( glob . glob ( os . path . join ( lcdir , lcfileglob ) ) )
return parallel_tfa_lclist ( lclist , templateinfo , timecols = timecols , magcols = magcols , errcols = errcols , lcformat = lcformat , lcformatdir = None , interp = interp , sigclip = sigclip , mintemplatedist_arcmin = mintemplatedist_arcmin , nworkers = nworkers , maxworkertasks = maxworkertasks )


signature:
def update_checkplot_objectinfo ( cpf , fast_mode = False , findercmap = 'gray_r' , finderconvolve = None , deredden_object = True , custom_bandpasses = None , gaia_submit_timeout = 10.0 , gaia_submit_tries = 3 , gaia_max_timeout = 180.0 , gaia_mirror = None , complete_query_later = True , lclistpkl = None , nbrradiusarcsec = 60.0 , maxnumneighbors = 5 , plotdpi = 100 , findercachedir = '~/.astrobase/stamp-cache' , verbose = True ) :
    '''This updates a checkplot objectinfo dict .
 Useful in cases where a previous round of GAIA / finderchart / external catalog
 acquisition failed . This will preserve the following keys in the checkplot
 if they exist : :
 comments
 varinfo
 objectinfo . objecttags
 Parameters
 cpf : str
 The path to the checkplot pickle to update .
 fast _ mode : bool or float
 This runs the external catalog operations in a " fast " mode , with short
 timeouts and not trying to hit external catalogs that take a long time
 to respond . See the docstring for
 : py : func : ` astrobase . checkplot . pkl _ utils . _ pkl _ finder _ objectinfo ` for
 details on how this works . If this is True , will run in " fast " mode with
 default timeouts ( 5 seconds in most cases ) . If this is a float , will run
 in " fast " mode with the provided timeout value in seconds .
 findercmap : str or matplotlib . cm . ColorMap object
 The Colormap object to use for the finder chart image .
 finderconvolve : astropy . convolution . Kernel object or None
 If not None , the Kernel object to use for convolving the finder image .
 deredden _ objects : bool
 If this is True , will use the 2MASS DUST service to get extinction
 coefficients in various bands , and then try to deredden the magnitudes
 and colors of the object already present in the checkplot ' s objectinfo
 dict .
 custom _ bandpasses : dict
 This is a dict used to provide custom bandpass definitions for any
 magnitude measurements in the objectinfo dict that are not automatically
 recognized by the ` varclass . starfeatures . color _ features ` function . See
 its docstring for details on the required format .
 gaia _ submit _ timeout : float
 Sets the timeout in seconds to use when submitting a request to look up
 the object ' s information to the GAIA service . Note that if ` fast _ mode `
 is set , this is ignored .
 gaia _ submit _ tries : int
 Sets the maximum number of times the GAIA services will be contacted to
 obtain this object ' s information . If ` fast _ mode ` is set , this is
 ignored , and the services will be contacted only once ( meaning that a
 failure to respond will be silently ignored and no GAIA data will be
 added to the checkplot ' s objectinfo dict ) .
 gaia _ max _ timeout : float
 Sets the timeout in seconds to use when waiting for the GAIA service to
 respond to our request for the object ' s information . Note that if
 ` fast _ mode ` is set , this is ignored .
 gaia _ mirror : str
 This sets the GAIA mirror to use . This is a key in the
 : py : data : ` astrobase . services . gaia . GAIA _ URLS ` dict which defines the URLs
 to hit for each mirror .
 complete _ query _ later : bool
 If this is True , saves the state of GAIA queries that are not yet
 complete when ` gaia _ max _ timeout ` is reached while waiting for the GAIA
 service to respond to our request . A later call for GAIA info on the
 same object will attempt to pick up the results from the existing query
 if it ' s completed . If ` fast _ mode ` is True , this is ignored .
 lclistpkl : dict or str
 If this is provided , must be a dict resulting from reading a catalog
 produced by the ` lcproc . catalogs . make _ lclist ` function or a str path
 pointing to the pickle file produced by that function . This catalog is
 used to find neighbors of the current object in the current light curve
 collection . Looking at neighbors of the object within the radius
 specified by ` nbrradiusarcsec ` is useful for light curves produced by
 instruments that have a large pixel scale , so are susceptible to
 blending of variability and potential confusion of neighbor variability
 with that of the actual object being looked at . If this is None , no
 neighbor lookups will be performed .
 nbrradiusarcsec : float
 The radius in arcseconds to use for a search conducted around the
 coordinates of this object to look for any potential confusion and
 blending of variability amplitude caused by their proximity .
 maxnumneighbors : int
 The maximum number of neighbors that will have their light curves and
 magnitudes noted in this checkplot as potential blends with the target
 object .
 plotdpi : int
 The resolution in DPI of the plots to generate in this function
 ( e . g . the finder chart , etc . )
 findercachedir : str
 The path to the astrobase cache directory for finder chart downloads
 from the NASA SkyView service .
 verbose : bool
 If True , will indicate progress and warn about potential problems .
 Returns
 str
 Path to the updated checkplot pickle file .'''

implementation:
cpd = _read_checkplot_picklefile ( cpf )
if cpd [ 'objectinfo' ] [ 'objecttags' ] is not None :
    objecttags = cpd [ 'objectinfo' ] [ 'objecttags' ] [ : : ]
else :
    objecttags = None
varinfo = deepcopy ( cpd [ 'varinfo' ] )
if 'comments' in cpd and cpd [ 'comments' ] is not None :
    comments = cpd [ 'comments' ] [ : : ]
else :
    comments = None
newcpd = _pkl_finder_objectinfo ( cpd [ 'objectinfo' ] , varinfo , findercmap , finderconvolve , cpd [ 'sigclip' ] , cpd [ 'normto' ] , cpd [ 'normmingap' ] , fast_mode = fast_mode , deredden_object = deredden_object , custom_bandpasses = custom_bandpasses , gaia_submit_timeout = gaia_submit_timeout , gaia_submit_tries = gaia_submit_tries , gaia_max_timeout = gaia_max_timeout , gaia_mirror = gaia_mirror , complete_query_later = complete_query_later , lclistpkl = lclistpkl , nbrradiusarcsec = nbrradiusarcsec , maxnumneighbors = maxnumneighbors , plotdpi = plotdpi , findercachedir = findercachedir , verbose = verbose )
# don ' t update neighbors or finder chart if the new one is bad
if ( newcpd [ 'finderchart' ] is None and cpd [ 'finderchart' ] is not None ) :
    newcpd [ 'finderchart' ] = deepcopy ( cpd [ 'finderchart' ] )
if ( newcpd [ 'neighbors' ] is None and cpd [ 'neighbors' ] is not None ) :
    newcpd [ 'neighbors' ] = deepcopy ( cpd [ 'neighbors' ] )
# if there ' s existing GAIA info , don ' t overwrite if the new objectinfo dict
# doesn ' t have any
if ( ( 'failed' in newcpd [ 'objectinfo' ] [ 'gaia_status' ] or ( 'gaiaid' in newcpd [ 'objectinfo' ] and newcpd [ 'objectinfo' ] [ 'gaiaid' ] is None ) ) and 'ok' in cpd [ 'objectinfo' ] [ 'gaia_status' ] ) :
    newcpd [ 'objectinfo' ] [ 'gaia_status' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_status' ] )
    if 'gaiaid' in cpd [ 'objectinfo' ] :
        newcpd [ 'objectinfo' ] [ 'gaiaid' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaiaid' ] )
    newcpd [ 'objectinfo' ] [ 'gaiamag' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaiamag' ] )
    newcpd [ 'objectinfo' ] [ 'gaia_absmag' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_absmag' ] )
    newcpd [ 'objectinfo' ] [ 'gaia_parallax' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_parallax' ] )
    newcpd [ 'objectinfo' ] [ 'gaia_parallax_err' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_parallax_err' ] )
    newcpd [ 'objectinfo' ] [ 'gaia_pmra' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_pmra' ] )
    newcpd [ 'objectinfo' ] [ 'gaia_pmra_err' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_pmra_err' ] )
    newcpd [ 'objectinfo' ] [ 'gaia_pmdecl' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_pmdecl' ] )
    newcpd [ 'objectinfo' ] [ 'gaia_pmdecl_err' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_pmdecl_err' ] )
if ( not np . isfinite ( newcpd [ 'objectinfo' ] [ 'gaia_neighbors' ] ) and np . isfinite ( cpd [ 'objectinfo' ] [ 'gaia_neighbors' ] ) ) :
    newcpd [ 'objectinfo' ] [ 'gaia_neighbors' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_neighbors' ] )
if ( not np . isfinite ( newcpd [ 'objectinfo' ] [ 'gaia_closest_distarcsec' ] ) and np . isfinite ( cpd [ 'objectinfo' ] [ 'gaia_closest_distarcsec' ] ) ) :
    newcpd [ 'objectinfo' ] [ 'gaia_closest_distarcsec' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_closest_gmagdiff' ] )
if ( not np . isfinite ( newcpd [ 'objectinfo' ] [ 'gaia_closest_gmagdiff' ] ) and np . isfinite ( cpd [ 'objectinfo' ] [ 'gaia_closest_gmagdiff' ] ) ) :
    newcpd [ 'objectinfo' ] [ 'gaia_closest_gmagdiff' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_closest_gmagdiff' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_ids' ] is None and cpd [ 'objectinfo' ] [ 'gaia_ids' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_ids' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_ids' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_xypos' ] is None and cpd [ 'objectinfo' ] [ 'gaia_xypos' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_xypos' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_xypos' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_mags' ] is None and cpd [ 'objectinfo' ] [ 'gaia_mags' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_mags' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_mags' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_parallaxes' ] is None and cpd [ 'objectinfo' ] [ 'gaia_parallaxes' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_parallaxes' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_parallaxes' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_parallax_errs' ] is None and cpd [ 'objectinfo' ] [ 'gaia_parallax_errs' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_parallax_errs' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_parallax_errs' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_pmras' ] is None and cpd [ 'objectinfo' ] [ 'gaia_pmras' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_pmras' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_pmras' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_pmra_errs' ] is None and cpd [ 'objectinfo' ] [ 'gaia_pmra_errs' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_pmra_errs' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_pmra_errs' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_pmdecls' ] is None and cpd [ 'objectinfo' ] [ 'gaia_pmdecls' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_pmdecls' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_pmdecls' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_pmdecl_errs' ] is None and cpd [ 'objectinfo' ] [ 'gaia_pmdecl_errs' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_pmdecl_errs' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_pmdecl_errs' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_absolute_mags' ] is None and cpd [ 'objectinfo' ] [ 'gaia_absolute_mags' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_absolute_mags' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_absolute_mags' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaiak_colors' ] is None and cpd [ 'objectinfo' ] [ 'gaiak_colors' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaiak_colors' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaiak_colors' ] )
if ( newcpd [ 'objectinfo' ] [ 'gaia_dists' ] is None and cpd [ 'objectinfo' ] [ 'gaia_dists' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'gaia_dists' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'gaia_dists' ] )
# don ' t overwrite good SIMBAD info with bad
if ( 'failed' in newcpd [ 'objectinfo' ] [ 'simbad_status' ] and 'ok' in cpd [ 'objectinfo' ] [ 'simbad_status' ] ) :
    newcpd [ 'objectinfo' ] [ 'simbad_status' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'simbad_status' ] )
if ( newcpd [ 'objectinfo' ] [ 'simbad_nmatches' ] is None and cpd [ 'objectinfo' ] [ 'simbad_nmatches' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'simbad_nmatches' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'simbad_nmatches' ] )
if ( newcpd [ 'objectinfo' ] [ 'simbad_mainid' ] is None and cpd [ 'objectinfo' ] [ 'simbad_mainid' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'simbad_mainid' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'simbad_mainid' ] )
if ( newcpd [ 'objectinfo' ] [ 'simbad_objtype' ] is None and cpd [ 'objectinfo' ] [ 'simbad_objtype' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'simbad_objtype' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'simbad_objtype' ] )
if ( newcpd [ 'objectinfo' ] [ 'simbad_allids' ] is None and cpd [ 'objectinfo' ] [ 'simbad_allids' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'simbad_allids' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'simbad_allids' ] )
if ( newcpd [ 'objectinfo' ] [ 'simbad_distarcsec' ] is None and cpd [ 'objectinfo' ] [ 'simbad_distarcsec' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'simbad_distarcsec' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'simbad_distarcsec' ] )
if ( newcpd [ 'objectinfo' ] [ 'simbad_best_mainid' ] is None and cpd [ 'objectinfo' ] [ 'simbad_best_mainid' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'simbad_best_mainid' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'simbad_best_mainid' ] )
if ( newcpd [ 'objectinfo' ] [ 'simbad_best_objtype' ] is None and cpd [ 'objectinfo' ] [ 'simbad_best_objtype' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'simbad_best_objtype' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'simbad_best_objtype' ] )
if ( newcpd [ 'objectinfo' ] [ 'simbad_best_allids' ] is None and cpd [ 'objectinfo' ] [ 'simbad_best_allids' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'simbad_best_allids' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'simbad_best_allids' ] )
if ( newcpd [ 'objectinfo' ] [ 'simbad_best_distarcsec' ] is None and cpd [ 'objectinfo' ] [ 'simbad_best_distarcsec' ] is not None ) :
    newcpd [ 'objectinfo' ] [ 'simbad_best_distarcsec' ] = deepcopy ( cpd [ 'objectinfo' ] [ 'simbad_best_distarcsec' ] )
# update the objectinfo dict
cpd . update ( newcpd )
cpd [ 'objectinfo' ] [ 'objecttags' ] = objecttags
cpd [ 'comments' ] = comments
newcpf = _write_checkplot_picklefile ( cpd , outfile = cpf )
return newcpf


signature:
def _read_pklc ( lcfile ) :
    '''This just reads a light curve pickle file .
 Parameters
 lcfile : str
 The file name of the pickle to open .
 Returns
 dict
 This returns an lcdict .'''

implementation:
if lcfile . endswith ( '.gz' ) :
    try :
        with gzip . open ( lcfile , 'rb' ) as infd :
            lcdict = pickle . load ( infd )
    except UnicodeDecodeError :
        with gzip . open ( lcfile , 'rb' ) as infd :
            lcdict = pickle . load ( infd , encoding = 'latin1' )
else :
    try :
        with open ( lcfile , 'rb' ) as infd :
            lcdict = pickle . load ( infd )
    except UnicodeDecodeError :
        with open ( lcfile , 'rb' ) as infd :
            lcdict = pickle . load ( infd , encoding = 'latin1' )
return lcdict


signature:
def _check_extmodule ( module , formatkey ) :
    '''This imports the module specified .
 Used to dynamically import Python modules that are needed to support LC
 formats not natively supported by astrobase .
 Parameters
 module : str
 This is either :
 - a Python module import path , e . g . ' astrobase . lcproc . catalogs ' or
 - a path to a Python file , e . g . ' / astrobase / hatsurveys / hatlc . py '
 that contains the Python module that contains functions used to open
 ( and optionally normalize ) a custom LC format that ' s not natively
 supported by astrobase .
 formatkey : str
 A str used as the unique ID of this LC format for all lcproc functions
 and can be used to look it up later and import the correct functions
 needed to support it for lcproc operations . For example , we use
 ' kep - fits ' as a the specifier for Kepler FITS light curves , which can be
 read by the ` astrobase . astrokep . read _ kepler _ fitslc ` function as
 specified by the ` < astrobase install path > / data / lcformats / kep - fits . json `
 LC format specification JSON .
 Returns
 Python module
 This returns a Python module if it ' s able to successfully import it .'''

implementation:
try :
    if os . path . exists ( module ) :
        sys . path . append ( os . path . dirname ( module ) )
        importedok = importlib . import_module ( os . path . basename ( module . replace ( '.py' , '' ) ) )
    else :
        importedok = importlib . import_module ( module )
except Exception as e :
    LOGEXCEPTION ( 'could not import the module: %s for LC format: %s. ' 'check the file path or fully qualified module name?' % ( module , formatkey ) )
    importedok = False
return importedok


signature:
def register_lcformat ( formatkey , fileglob , timecols , magcols , errcols , readerfunc_module , readerfunc , readerfunc_kwargs = None , normfunc_module = None , normfunc = None , normfunc_kwargs = None , magsarefluxes = False , overwrite_existing = False , lcformat_dir = '~/.astrobase/lcformat-jsons' ) :
    '''This adds a new LC format to the astrobase LC format registry .
 Allows handling of custom format light curves for astrobase lcproc
 drivers . Once the format is successfully registered , light curves should
 work transparently with all of the functions in this module , by simply
 calling them with the ` formatkey ` in the ` lcformat ` keyword argument .
 LC format specifications are generated as JSON files . astrobase comes with
 several of these in ` < astrobase install path > / data / lcformats ` . LC formats
 you add by using this function will have their specifiers written to the
 ` ~ / . astrobase / lcformat - jsons ` directory in your home directory .
 Parameters
 formatkey : str
 A str used as the unique ID of this LC format for all lcproc functions
 and can be used to look it up later and import the correct functions
 needed to support it for lcproc operations . For example , we use
 ' kep - fits ' as a the specifier for Kepler FITS light curves , which can be
 read by the ` astrobase . astrokep . read _ kepler _ fitslc ` function as
 specified by the ` < astrobase install path > / data / lcformats / kep - fits . json `
 LC format specification JSON produced by ` register _ lcformat ` .
 fileglob : str
 The default UNIX fileglob to use to search for light curve files in this
 LC format . This is a string like ' * - whatever - ? ? ? - * . * ? ? - . lc ' .
 timecols , magcols , errcols : list of str
 These are all lists of strings indicating which keys in the lcdict
 produced by your ` lcreader _ func ` that will be extracted and used by
 lcproc functions for processing . The lists must all have the same
 dimensions , e . g . if timecols = [ ' timecol1 ' , ' timecol2 ' ] , then magcols
 must be something like [ ' magcol1 ' , ' magcol2 ' ] and errcols must be
 something like [ ' errcol1 ' , ' errcol2 ' ] . This allows you to process
 multiple apertures or multiple types of measurements in one go .
 Each element in these lists can be a simple key , e . g . ' time ' ( which
 would correspond to lcdict [ ' time ' ] ) , or a composite key ,
 e . g . ' aperture1 . times . rjd ' ( which would correspond to
 lcdict [ ' aperture1 ' ] [ ' times ' ] [ ' rjd ' ] ) . See the examples in the lcformat
 specification JSON files in ` < astrobase install path > / data / lcformats ` .
 readerfunc _ module : str
 This is either :
 - a Python module import path , e . g . ' astrobase . lcproc . catalogs ' or
 - a path to a Python file , e . g . ' / astrobase / hatsurveys / hatlc . py '
 that contains the Python module that contains functions used to open
 ( and optionally normalize ) a custom LC format that ' s not natively
 supported by astrobase .
 readerfunc : str
 This is the function name in ` readerfunc _ module ` to use to read light
 curves in the custom format . This MUST always return a dictionary ( the
 ' lcdict ' ) with the following signature ( the keys listed below are
 required , but others are allowed ) : :
 { ' objectid ' : this object ' s identifier as a string ,
 ' objectinfo ' : { ' ra ' : this object ' s right ascension in decimal deg ,
 ' decl ' : this object ' s declination in decimal deg ,
 ' ndet ' : the number of observations in this LC ,
 ' objectid ' : the object ID again for legacy reasons } ,
 . . . other time columns , mag columns go in as their own keys }
 normfunc _ kwargs : dict or None
 This is a dictionary containing any kwargs to pass through to
 the light curve norm function .
 normfunc _ module : str or None
 This is either :
 - a Python module import path , e . g . ' astrobase . lcproc . catalogs ' or
 - a path to a Python file , e . g . ' / astrobase / hatsurveys / hatlc . py '
 - None , in which case we ' ll use default normalization
 that contains the Python module that contains functions used to
 normalize a custom LC format that ' s not natively supported by astrobase .
 normfunc : str or None
 This is the function name in ` normfunc _ module ` to use to normalize light
 curves in the custom format . If None , the default normalization method
 used by lcproc is to find gaps in the time - series , normalize
 measurements grouped by these gaps to zero , then normalize the entire
 magnitude time series to global time series median using the
 ` astrobase . lcmath . normalize _ magseries ` function .
 If this is provided , the normalization function should take and return
 an lcdict of the same form as that produced by ` readerfunc ` above . For
 an example of a specific normalization function , see
 ` normalize _ lcdict _ by _ inst ` in the ` astrobase . hatsurveys . hatlc ` module .
 normfunc _ kwargs : dict or None
 This is a dictionary containing any kwargs to pass through to
 the light curve normalization function .
 magsarefluxes : bool
 If this is True , then all lcproc functions will treat the measurement
 columns in the lcdict produced by your ` readerfunc ` as flux instead of
 mags , so things like default normalization and sigma - clipping will be
 done correctly . If this is False , magnitudes will be treated as
 magnitudes .
 overwrite _ existing : bool
 If this is True , this function will overwrite any existing LC format
 specification JSON with the same name as that provided in the
 ` formatkey ` arg . This can be used to update LC format specifications
 while keeping the ` formatkey ` the same .
 lcformat _ dir : str
 This specifies the directory where the the LC format specification JSON
 produced by this function will be written . By default , this goes to the
 ` . astrobase / lcformat - jsons ` directory in your home directory .
 Returns
 str
 Returns the file path to the generated LC format specification JSON
 file .'''

implementation:
LOGINFO ( 'adding %s to LC format registry...' % formatkey )
# search for the lcformat _ dir and create it if it doesn ' t exist
lcformat_dpath = os . path . abspath ( os . path . expanduser ( lcformat_dir ) )
if not os . path . exists ( lcformat_dpath ) :
    os . makedirs ( lcformat_dpath )
lcformat_jsonpath = os . path . join ( lcformat_dpath , '%s.json' % formatkey )
if os . path . exists ( lcformat_jsonpath ) and not overwrite_existing :
    LOGERROR ( 'There is an existing lcformat JSON: %s ' 'for this formatkey: %s and ' 'overwrite_existing = False, skipping...' % ( lcformat_jsonpath , formatkey ) )
    return None
# see if we can import the reader module
readermodule = _check_extmodule ( readerfunc_module , formatkey )
if not readermodule :
    LOGERROR ( "could not import the required " "module: %s to read %s light curves" % ( readerfunc_module , formatkey ) )
    return None
# then , get the function we need to read the light curve
try :
    getattr ( readermodule , readerfunc )
    readerfunc_in = readerfunc
except AttributeError :
    LOGEXCEPTION ( 'Could not get the specified reader ' 'function: %s for lcformat: %s ' 'from module: %s' % ( formatkey , readerfunc_module , readerfunc ) )
    raise
# see if we can import the normalization module
if normfunc_module :
    normmodule = _check_extmodule ( normfunc_module , formatkey )
    if not normmodule :
        LOGERROR ( "could not import the required " "module: %s to normalize %s light curves" % ( normfunc_module , formatkey ) )
        return None
else :
    normmodule = None
# finally , get the function we need to normalize the light curve
if normfunc_module and normfunc :
    try :
        getattr ( normmodule , normfunc )
        normfunc_in = normfunc
    except AttributeError :
        LOGEXCEPTION ( 'Could not get the specified norm ' 'function: %s for lcformat: %s ' 'from module: %s' % ( normfunc , formatkey , normfunc_module ) )
        raise
else :
    normfunc_in = None
# if we made it to here , then everything ' s good . generate the JSON
# structure
formatdict = { 'fileglob' : fileglob , 'timecols' : timecols , 'magcols' : magcols , 'errcols' : errcols , 'magsarefluxes' : magsarefluxes , 'lcreader_module' : readerfunc_module , 'lcreader_func' : readerfunc_in , 'lcreader_kwargs' : readerfunc_kwargs , 'lcnorm_module' : normfunc_module , 'lcnorm_func' : normfunc_in , 'lcnorm_kwargs' : normfunc_kwargs }
# write this to the lcformat directory
with open ( lcformat_jsonpath , 'w' ) as outfd :
    json . dump ( formatdict , outfd , indent = 4 )
return lcformat_jsonpath


signature:
def get_lcformat ( formatkey , use_lcformat_dir = None ) :
    '''This loads an LC format description from a previously - saved JSON file .
 Parameters
 formatkey : str
 The key used to refer to the LC format . This is part of the JSON file ' s
 name , e . g . the format key ' hat - csv ' maps to the format JSON file :
 ' < astrobase install path > / data / lcformats / hat - csv . json ' .
 use _ lcformat _ dir : str or None
 If provided , must be the path to a directory that contains the
 corresponding lcformat JSON file for ` formatkey ` . If this is None , this
 function will look for lcformat JSON files corresponding to the given
 ` formatkey ` :
 - first , in the directory specified in this kwarg ,
 - if not found there , in the home directory : ~ / . astrobase / lcformat - jsons
 - if not found there , in : < astrobase install path > / data / lcformats
 Returns
 tuple
 A tuple of the following form is returned : :
 ( fileglob : the file glob of the associated LC files ,
 readerfunc _ in : the imported Python function for reading LCs ,
 timecols : list of time col keys to get from the lcdict ,
 magcols : list of mag col keys to get from the lcdict ,
 errcols : list of err col keys to get from the lcdict ,
 magsarefluxes : True if the measurements are fluxes not mags ,
 normfunc _ in : the imported Python function for normalizing LCs )
 All ` astrobase . lcproc ` functions can then use this tuple to dynamically
 import your LC reader and normalization functions to work with your LC
 format transparently .'''

implementation:
if isinstance ( use_lcformat_dir , str ) : # look for the lcformat JSON
    lcformat_jsonpath = os . path . join ( use_lcformat_dir , '%s.json' % formatkey )
    if not os . path . exists ( lcformat_jsonpath ) :
        lcformat_jsonpath = os . path . join ( os . path . expanduser ( '~/.astrobase/lcformat-jsons' ) , '%s.json' % formatkey )
        if not os . path . exists ( lcformat_jsonpath ) :
            install_path = os . path . dirname ( __file__ )
            install_path = os . path . abspath ( os . path . join ( install_path , '..' , 'data' , 'lcformats' ) )
            lcformat_jsonpath = os . path . join ( install_path , '%s.json' % formatkey )
            if not os . path . exists ( lcformat_jsonpath ) :
                LOGERROR ( 'could not find an lcformat JSON ' 'for formatkey: %s in any of: ' 'use_lcformat_dir, home directory, ' 'astrobase installed data directory' % formatkey )
                return None
else :
    lcformat_jsonpath = os . path . join ( os . path . expanduser ( '~/.astrobase/lcformat-jsons' ) , '%s.json' % formatkey )
    if not os . path . exists ( lcformat_jsonpath ) :
        install_path = os . path . dirname ( __file__ )
        install_path = os . path . abspath ( os . path . join ( install_path , '..' , 'data' , 'lcformats' ) )
        lcformat_jsonpath = os . path . join ( install_path , '%s.json' % formatkey )
        if not os . path . exists ( lcformat_jsonpath ) :
            LOGERROR ( 'could not find an lcformat JSON ' 'for formatkey: %s in any of: ' 'use_lcformat_dir, home directory, ' 'astrobase installed data directory' % formatkey )
            return None
# load the found lcformat JSON
with open ( lcformat_jsonpath ) as infd :
    lcformatdict = json . load ( infd )
readerfunc_module = lcformatdict [ 'lcreader_module' ]
readerfunc = lcformatdict [ 'lcreader_func' ]
readerfunc_kwargs = lcformatdict [ 'lcreader_kwargs' ]
normfunc_module = lcformatdict [ 'lcnorm_module' ]
normfunc = lcformatdict [ 'lcnorm_func' ]
normfunc_kwargs = lcformatdict [ 'lcnorm_kwargs' ]
fileglob = lcformatdict [ 'fileglob' ]
timecols = lcformatdict [ 'timecols' ]
magcols = lcformatdict [ 'magcols' ]
errcols = lcformatdict [ 'errcols' ]
magsarefluxes = lcformatdict [ 'magsarefluxes' ]
# import all the required bits
# see if we can import the reader module
readermodule = _check_extmodule ( readerfunc_module , formatkey )
if not readermodule :
    LOGERROR ( "could not import the required " "module: %s to read %s light curves" % ( readerfunc_module , formatkey ) )
    return None
# then , get the function we need to read the light curve
try :
    readerfunc_in = getattr ( readermodule , readerfunc )
except AttributeError :
    LOGEXCEPTION ( 'Could not get the specified reader ' 'function: %s for lcformat: %s ' 'from module: %s' % ( formatkey , readerfunc_module , readerfunc ) )
    raise
# see if we can import the normalization module
if normfunc_module :
    normmodule = _check_extmodule ( normfunc_module , formatkey )
    if not normmodule :
        LOGERROR ( "could not import the required " "module: %s to normalize %s light curves" % ( normfunc_module , formatkey ) )
        return None
else :
    normmodule = None
# finally , get the function we need to normalize the light curve
if normfunc_module and normfunc :
    try :
        normfunc_in = getattr ( normmodule , normfunc )
    except AttributeError :
        LOGEXCEPTION ( 'Could not get the specified norm ' 'function: %s for lcformat: %s ' 'from module: %s' % ( formatkey , normfunc_module , normfunc ) )
        raise
else :
    normfunc_in = None
# add in any optional kwargs that need to be there for readerfunc
if isinstance ( readerfunc_kwargs , dict ) :
    readerfunc_in = partial ( readerfunc_in , ** readerfunc_kwargs )
# add in any optional kwargs that need to be there for normfunc
if normfunc_in is not None :
    if isinstance ( normfunc_kwargs , dict ) :
        normfunc_in = partial ( normfunc_in , ** normfunc_kwargs )
# assemble the return tuple
# this can be used directly by other lcproc functions
returntuple = ( fileglob , readerfunc_in , timecols , magcols , errcols , magsarefluxes , normfunc_in , )
return returntuple


signature:
def ec2_ssh ( ip_address , keypem_file , username = 'ec2-user' , raiseonfail = False ) :
    """This opens an SSH connection to the EC2 instance at ` ip _ address ` .
 Parameters
 ip _ address : str
 IP address of the AWS EC2 instance to connect to .
 keypem _ file : str
 The path to the keypair PEM file generated by AWS to allow SSH
 connections .
 username : str
 The username to use to login to the EC2 instance .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 paramiko . SSHClient
 This has all the usual ` paramiko ` functionality :
 - Use ` SSHClient . exec _ command ( command , environment = None ) ` to exec a
 shell command .
 - Use ` SSHClient . open _ sftp ( ) ` to get a ` SFTPClient ` for the server . Then
 call SFTPClient . get ( ) and . put ( ) to copy files from and to the server ."""

implementation:
c = paramiko . client . SSHClient ( )
c . load_system_host_keys ( )
c . set_missing_host_key_policy ( paramiko . client . AutoAddPolicy )
# load the private key from the AWS keypair pem
privatekey = paramiko . RSAKey . from_private_key_file ( keypem_file )
# connect to the server
try :
    c . connect ( ip_address , pkey = privatekey , username = 'ec2-user' )
    return c
except Exception as e :
    LOGEXCEPTION ( 'could not connect to EC2 instance at %s ' 'using keyfile: %s and user: %s' % ( ip_address , keypem_file , username ) )
    if raiseonfail :
        raise
    return None


signature:
def s3_get_file ( bucket , filename , local_file , altexts = None , client = None , raiseonfail = False ) :
    """This gets a file from an S3 bucket .
 Parameters
 bucket : str
 The AWS S3 bucket name .
 filename : str
 The full filename of the file to get from the bucket
 local _ file : str
 Path to where the downloaded file will be stored .
 altexts : None or list of str
 If not None , this is a list of alternate extensions to try for the file
 other than the one provided in ` filename ` . For example , to get anything
 that ' s an . sqlite where . sqlite . gz is expected , use altexts = [ ' ' ] to
 strip the . gz .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 str
 Path to the downloaded filename or None if the download was
 unsuccessful ."""

implementation:
if not client :
    client = boto3 . client ( 's3' )
try :
    client . download_file ( bucket , filename , local_file )
    return local_file
except Exception as e :
    if altexts is not None :
        for alt_extension in altexts :
            split_ext = os . path . splitext ( filename )
            check_file = split_ext [ 0 ] + alt_extension
            try :
                client . download_file ( bucket , check_file , local_file . replace ( split_ext [ - 1 ] , alt_extension ) )
                return local_file . replace ( split_ext [ - 1 ] , alt_extension )
            except Exception as e :
                pass
    else :
        LOGEXCEPTION ( 'could not download s3://%s/%s' % ( bucket , filename ) )
        if raiseonfail :
            raise
        return None


signature:
def s3_get_url ( url , altexts = None , client = None , raiseonfail = False ) :
    """This gets a file from an S3 bucket based on its s3 : / / URL .
 Parameters
 url : str
 S3 URL to download . This should begin with ' s3 : / / ' .
 altexts : None or list of str
 If not None , this is a list of alternate extensions to try for the file
 other than the one provided in ` filename ` . For example , to get anything
 that ' s an . sqlite where . sqlite . gz is expected , use altexts = [ ' ' ] to
 strip the . gz .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 str
 Path to the downloaded filename or None if the download was
 unsuccessful . The file will be downloaded into the current working
 directory and will have a filename = = basename of the file on S3."""

implementation:
bucket_item = url . replace ( 's3://' , '' )
bucket_item = bucket_item . split ( '/' )
bucket = bucket_item [ 0 ]
filekey = '/' . join ( bucket_item [ 1 : ] )
return s3_get_file ( bucket , filekey , bucket_item [ - 1 ] , altexts = altexts , client = client , raiseonfail = raiseonfail )


signature:
def s3_put_file ( local_file , bucket , client = None , raiseonfail = False ) :
    """This uploads a file to S3.
 Parameters
 local _ file : str
 Path to the file to upload to S3.
 bucket : str
 The AWS S3 bucket to upload the file to .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 str or None
 If the file upload is successful , returns the s3 : / / URL of the uploaded
 file . If it failed , will return None ."""

implementation:
if not client :
    client = boto3 . client ( 's3' )
try :
    client . upload_file ( local_file , bucket , os . path . basename ( local_file ) )
    return 's3://%s/%s' % ( bucket , os . path . basename ( local_file ) )
except Exception as e :
    LOGEXCEPTION ( 'could not upload %s to bucket: %s' % ( local_file , bucket ) )
    if raiseonfail :
        raise
    return None


signature:
def s3_delete_file ( bucket , filename , client = None , raiseonfail = False ) :
    """This deletes a file from S3.
 Parameters
 bucket : str
 The AWS S3 bucket to delete the file from .
 filename : str
 The full file name of the file to delete , including any prefixes .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 str or None
 If the file was successfully deleted , will return the delete - marker
 ( https : / / docs . aws . amazon . com / AmazonS3 / latest / dev / DeleteMarker . html ) . If
 it wasn ' t , returns None"""

implementation:
if not client :
    client = boto3 . client ( 's3' )
try :
    resp = client . delete_object ( Bucket = bucket , Key = filename )
    if not resp :
        LOGERROR ( 'could not delete file %s from bucket %s' % ( filename , bucket ) )
    else :
        return resp [ 'DeleteMarker' ]
except Exception as e :
    LOGEXCEPTION ( 'could not delete file %s from bucket %s' % ( filename , bucket ) )
    if raiseonfail :
        raise
    return None


signature:
def sqs_create_queue ( queue_name , options = None , client = None ) :
    """This creates an SQS queue .
 Parameters
 queue _ name : str
 The name of the queue to create .
 options : dict or None
 A dict of options indicate extra attributes the queue should have .
 See the SQS docs for details . If None , no custom attributes will be
 attached to the queue .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 Returns
 dict
 This returns a dict of the form : :
 { ' url ' : SQS URL of the queue ,
 ' name ' : name of the queue }"""

implementation:
if not client :
    client = boto3 . client ( 'sqs' )
try :
    if isinstance ( options , dict ) :
        resp = client . create_queue ( QueueName = queue_name , Attributes = options )
    else :
        resp = client . create_queue ( QueueName = queue_name )
    if resp is not None :
        return { 'url' : resp [ 'QueueUrl' ] , 'name' : queue_name }
    else :
        LOGERROR ( 'could not create the specified queue: %s with options: %s' % ( queue_name , options ) )
        return None
except Exception as e :
    LOGEXCEPTION ( 'could not create the specified queue: %s with options: %s' % ( queue_name , options ) )
    return None


signature:
def sqs_delete_queue ( queue_url , client = None ) :
    """This deletes an SQS queue given its URL
 Parameters
 queue _ url : str
 The SQS URL of the queue to delete .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 Returns
 bool
 True if the queue was deleted successfully . False otherwise ."""

implementation:
if not client :
    client = boto3 . client ( 'sqs' )
try :
    client . delete_queue ( QueueUrl = queue_url )
    return True
except Exception as e :
    LOGEXCEPTION ( 'could not delete the specified queue: %s' % ( queue_url , ) )
    return False


signature:
def sqs_put_item ( queue_url , item , delay_seconds = 0 , client = None , raiseonfail = False ) :
    """This pushes a dict serialized to JSON to the specified SQS queue .
 Parameters
 queue _ url : str
 The SQS URL of the queue to push the object to .
 item : dict
 The dict passed in here will be serialized to JSON .
 delay _ seconds : int
 The amount of time in seconds the pushed item will be held before going
 ' live ' and being visible to all queue consumers .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 boto3 . Response or None
 If the item was successfully put on the queue , will return the response
 from the service . If it wasn ' t , will return None ."""

implementation:
if not client :
    client = boto3 . client ( 'sqs' )
try :
    json_msg = json . dumps ( item )
    resp = client . send_message ( QueueUrl = queue_url , MessageBody = json_msg , DelaySeconds = delay_seconds , )
    if not resp :
        LOGERROR ( 'could not send item to queue: %s' % queue_url )
        return None
    else :
        return resp
except Exception as e :
    LOGEXCEPTION ( 'could not send item to queue: %s' % queue_url )
    if raiseonfail :
        raise
    return None