signature:
def epd_kepler_lightcurve ( lcdict , xccol = 'mom_centr1' , yccol = 'mom_centr2' , timestoignore = None , filterflags = True , writetodict = True , epdsmooth = 5 ) :
    '''This runs EPD on the Kepler light curve .
 Following Huang et al . 2015 , we fit the following EPD function to a smoothed
 light curve , and then subtract it to obtain EPD corrected magnitudes : :
 f = c0 +
 c1 * sin ( 2 * pi * x ) + c2 * cos ( 2 * pi * x ) + c3 * sin ( 2 * pi * y ) + c4 * cos ( 2 * pi * y ) +
 c5 * sin ( 4 * pi * x ) + c6 * cos ( 4 * pi * x ) + c7 * sin ( 4 * pi * y ) + c8 * cos ( 4 * pi * y ) +
 c9 * bgv + c10 * bge
 By default , this function removes points in the Kepler LC that have ANY
 quality flags set .
 Parameters
 lcdict : lcdict
 An ` lcdict ` produced by ` consolidate _ kepler _ fitslc ` or
 ` read _ kepler _ fitslc ` .
 xcol , ycol : str
 Indicates the x and y coordinate column names to use from the Kepler LC
 in the EPD fit .
 timestoignore : list of tuples
 This is of the form : :
 [ ( time1 _ start , time1 _ end ) , ( time2 _ start , time2 _ end ) , . . . ]
 and indicates the start and end times to mask out of the final
 lcdict . Use this to remove anything that wasn ' t caught by the quality
 flags .
 filterflags : bool
 If True , will remove any measurements that have non - zero quality flags
 present . This usually indicates an issue with the instrument or
 spacecraft .
 writetodict : bool
 If writetodict is True , adds the following columns to the lcdict : :
 epd _ time = time array
 epd _ sapflux = uncorrected flux before EPD
 epd _ epdsapflux = corrected flux after EPD
 epd _ epdsapcorr = EPD flux corrections
 epd _ bkg = background array
 epd _ bkg _ err = background errors array
 epd _ xcc = xcoord array
 epd _ ycc = ycoord array
 epd _ quality = quality flag array
 and updates the ' columns ' list in the lcdict as well .
 epdsmooth : int
 Sets the number of light curve points to smooth over when generating the
 EPD fit function .
 Returns
 tuple
 Returns a tuple of the form : ( times , epdfluxes , fitcoeffs , epdfit )'''

implementation:
times , fluxes , background , background_err = ( lcdict [ 'time' ] , lcdict [ 'sap' ] [ 'sap_flux' ] , lcdict [ 'sap' ] [ 'sap_bkg' ] , lcdict [ 'sap' ] [ 'sap_bkg_err' ] )
xcc = lcdict [ xccol ]
ycc = lcdict [ yccol ]
flags = lcdict [ 'sap_quality' ]
# filter all bad LC points as noted by quality flags
if filterflags :
    nbefore = times . size
    filterind = flags == 0
    times = times [ filterind ]
    fluxes = fluxes [ filterind ]
    background = background [ filterind ]
    background_err = background_err [ filterind ]
    xcc = xcc [ filterind ]
    ycc = ycc [ filterind ]
    flags = flags [ filterind ]
    nafter = times . size
    LOGINFO ( 'applied quality flag filter, ndet before = %s, ndet after = %s' % ( nbefore , nafter ) )
# remove nans
find = ( npisfinite ( xcc ) & npisfinite ( ycc ) & npisfinite ( times ) & npisfinite ( fluxes ) & npisfinite ( background ) & npisfinite ( background_err ) )
nbefore = times . size
times = times [ find ]
fluxes = fluxes [ find ]
background = background [ find ]
background_err = background_err [ find ]
xcc = xcc [ find ]
ycc = ycc [ find ]
flags = flags [ find ]
nafter = times . size
LOGINFO ( 'removed nans, ndet before = %s, ndet after = %s' % ( nbefore , nafter ) )
# exclude all times in timestoignore
if ( timestoignore and isinstance ( timestoignore , list ) and len ( timestoignore ) > 0 ) :
    exclind = npfull_like ( times , True )
    nbefore = times . size
    # apply all the masks
    for ignoretime in timestoignore :
        time0 , time1 = ignoretime [ 0 ] , ignoretime [ 1 ]
        thismask = ( times > time0 ) & ( times < time1 )
        exclind = exclind & thismask
    # quantities after masks have been applied
    times = times [ exclind ]
    fluxes = fluxes [ exclind ]
    background = background [ exclind ]
    background_err = background_err [ exclind ]
    xcc = xcc [ exclind ]
    ycc = ycc [ exclind ]
    flags = flags [ exclind ]
    nafter = times . size
    LOGINFO ( 'removed timestoignore, ndet before = %s, ndet after = %s' % ( nbefore , nafter ) )
# now that we ' re all done , we can do EPD
# first , smooth the light curve
smoothedfluxes = median_filter ( fluxes , size = epdsmooth )
# initial fit coeffs
initcoeffs = npones ( 11 )
# fit the the smoothed mags and find better coeffs
leastsqfit = leastsq ( _epd_residual , initcoeffs , args = ( smoothedfluxes , xcc , ycc , background , background_err ) )
# if the fit succeeds , then get the EPD fluxes
if leastsqfit [ - 1 ] in ( 1 , 2 , 3 , 4 ) :
    fitcoeffs = leastsqfit [ 0 ]
    epdfit = _epd_function ( fitcoeffs , fluxes , xcc , ycc , background , background_err )
    epdfluxes = npmedian ( fluxes ) + fluxes - epdfit
    # write these to the dictionary if requested
    if writetodict :
        lcdict [ 'epd' ] = { }
        lcdict [ 'epd' ] [ 'time' ] = times
        lcdict [ 'epd' ] [ 'sapflux' ] = fluxes
        lcdict [ 'epd' ] [ 'epdsapflux' ] = epdfluxes
        lcdict [ 'epd' ] [ 'epdsapcorr' ] = epdfit
        lcdict [ 'epd' ] [ 'bkg' ] = background
        lcdict [ 'epd' ] [ 'bkg_err' ] = background_err
        lcdict [ 'epd' ] [ 'xcc' ] = xcc
        lcdict [ 'epd' ] [ 'ycc' ] = ycc
        lcdict [ 'epd' ] [ 'quality' ] = flags
        for newcol in [ 'epd.time' , 'epd.sapflux' , 'epd.epdsapflux' , 'epd.epdsapcorr' , 'epd.bkg' , 'epd.bkg.err' , 'epd.xcc' , 'epd.ycc' , 'epd.quality' ] :
            if newcol not in lcdict [ 'columns' ] :
                lcdict [ 'columns' ] . append ( newcol )
    return times , epdfluxes , fitcoeffs , epdfit
else :
    LOGERROR ( 'could not fit EPD function to light curve' )
    return None , None , None , None


signature:
def rfepd_kepler_lightcurve ( lcdict , xccol = 'mom_centr1' , yccol = 'mom_centr2' , timestoignore = None , filterflags = True , writetodict = True , epdsmooth = 23 , decorr = 'xcc,ycc' , nrftrees = 200 ) :
    '''This uses a ` RandomForestRegressor ` to fit and decorrelate Kepler light
 curves .
 Fits the X and Y positions , the background , and background error .
 By default , this function removes points in the Kepler LC that have ANY
 quality flags set .
 Parameters
 lcdict : lcdict
 An ` lcdict ` produced by ` consolidate _ kepler _ fitslc ` or
 ` read _ kepler _ fitslc ` .
 xcol , ycol : str
 Indicates the x and y coordinate column names to use from the Kepler LC
 in the EPD fit .
 timestoignore : list of tuples
 This is of the form : :
 [ ( time1 _ start , time1 _ end ) , ( time2 _ start , time2 _ end ) , . . . ]
 and indicates the start and end times to mask out of the final
 lcdict . Use this to remove anything that wasn ' t caught by the quality
 flags .
 filterflags : bool
 If True , will remove any measurements that have non - zero quality flags
 present . This usually indicates an issue with the instrument or
 spacecraft .
 writetodict : bool
 If writetodict is True , adds the following columns to the lcdict : :
 rfepd _ time = time array
 rfepd _ sapflux = uncorrected flux before EPD
 rfepd _ epdsapflux = corrected flux after EPD
 rfepd _ epdsapcorr = EPD flux corrections
 rfepd _ bkg = background array
 rfepd _ bkg _ err = background errors array
 rfepd _ xcc = xcoord array
 rfepd _ ycc = ycoord array
 rfepd _ quality = quality flag array
 and updates the ' columns ' list in the lcdict as well .
 epdsmooth : int
 Sets the number of light curve points to smooth over when generating the
 EPD fit function .
 decorr : { ' xcc , ycc ' , ' bgv , bge ' , ' xcc , ycc , bgv , bge ' }
 Indicates whether to use the x , y coords alone ; background value and
 error alone ; or x , y coords and background value , error in combination as
 the features to training the ` RandomForestRegressor ` on and perform the
 fit .
 nrftrees : int
 The number of trees to use in the ` RandomForestRegressor ` .
 Returns
 tuple
 Returns a tuple of the form : ( times , corrected _ fluxes , flux _ corrections )'''

implementation:
times , fluxes , background , background_err = ( lcdict [ 'time' ] , lcdict [ 'sap' ] [ 'sap_flux' ] , lcdict [ 'sap' ] [ 'sap_bkg' ] , lcdict [ 'sap' ] [ 'sap_bkg_err' ] )
xcc = lcdict [ xccol ]
ycc = lcdict [ yccol ]
flags = lcdict [ 'sap_quality' ]
# filter all bad LC points as noted by quality flags
if filterflags :
    nbefore = times . size
    filterind = flags == 0
    times = times [ filterind ]
    fluxes = fluxes [ filterind ]
    background = background [ filterind ]
    background_err = background_err [ filterind ]
    xcc = xcc [ filterind ]
    ycc = ycc [ filterind ]
    flags = flags [ filterind ]
    nafter = times . size
    LOGINFO ( 'applied quality flag filter, ndet before = %s, ' 'ndet after = %s' % ( nbefore , nafter ) )
# remove nans
find = ( npisfinite ( xcc ) & npisfinite ( ycc ) & npisfinite ( times ) & npisfinite ( fluxes ) & npisfinite ( background ) & npisfinite ( background_err ) )
nbefore = times . size
times = times [ find ]
fluxes = fluxes [ find ]
background = background [ find ]
background_err = background_err [ find ]
xcc = xcc [ find ]
ycc = ycc [ find ]
flags = flags [ find ]
nafter = times . size
LOGINFO ( 'removed nans, ndet before = %s, ndet after = %s' % ( nbefore , nafter ) )
# exclude all times in timestoignore
if ( timestoignore and isinstance ( timestoignore , list ) and len ( timestoignore ) > 0 ) :
    exclind = npfull_like ( times , True )
    nbefore = times . size
    # apply all the masks
    for ignoretime in timestoignore :
        time0 , time1 = ignoretime [ 0 ] , ignoretime [ 1 ]
        thismask = ( times > time0 ) & ( times < time1 )
        exclind = exclind & thismask
    # quantities after masks have been applied
    times = times [ exclind ]
    fluxes = fluxes [ exclind ]
    background = background [ exclind ]
    background_err = background_err [ exclind ]
    xcc = xcc [ exclind ]
    ycc = ycc [ exclind ]
    flags = flags [ exclind ]
    nafter = times . size
    LOGINFO ( 'removed timestoignore, ndet before = %s, ndet after = %s' % ( nbefore , nafter ) )
# now that we ' re all done , we can do EPD
# set up the regressor
RFR = RandomForestRegressor ( n_estimators = nrftrees )
if decorr == 'xcc,ycc,bgv,bge' : # collect the features and target variable
    features = npcolumn_stack ( ( xcc , ycc , background , background_err ) )
elif decorr == 'xcc,ycc' : # collect the features and target variable
    features = npcolumn_stack ( ( xcc , ycc ) )
elif decorr == 'bgv,bge' : # collect the features and target variable
    features = npcolumn_stack ( ( background , background_err ) )
else :
    LOGERROR ( "couldn't understand decorr, not decorrelating..." )
    return None
# smooth the light curve
if epdsmooth :
    smoothedfluxes = median_filter ( fluxes , size = epdsmooth )
else :
    smoothedfluxes = fluxes
# fit , then generate the predicted values , then get corrected values
RFR . fit ( features , smoothedfluxes )
flux_corrections = RFR . predict ( features )
corrected_fluxes = npmedian ( fluxes ) + fluxes - flux_corrections
# remove the random forest to save RAM
del RFR
# write these to the dictionary if requested
if writetodict :
    lcdict [ 'rfepd' ] = { }
    lcdict [ 'rfepd' ] [ 'time' ] = times
    lcdict [ 'rfepd' ] [ 'sapflux' ] = fluxes
    lcdict [ 'rfepd' ] [ 'epdsapflux' ] = corrected_fluxes
    lcdict [ 'rfepd' ] [ 'epdsapcorr' ] = flux_corrections
    lcdict [ 'rfepd' ] [ 'bkg' ] = background
    lcdict [ 'rfepd' ] [ 'bkg_err' ] = background_err
    lcdict [ 'rfepd' ] [ 'xcc' ] = xcc
    lcdict [ 'rfepd' ] [ 'ycc' ] = ycc
    lcdict [ 'rfepd' ] [ 'quality' ] = flags
    for newcol in [ 'rfepd.time' , 'rfepd.sapflux' , 'rfepd.epdsapflux' , 'rfepd.epdsapcorr' , 'rfepd.bkg' , 'rfepd.bkg.err' , 'rfepd.xcc' , 'rfepd.ycc' , 'rfepd.quality' ] :
        if newcol not in lcdict [ 'columns' ] :
            lcdict [ 'columns' ] . append ( newcol )
return times , corrected_fluxes , flux_corrections


signature:
def detrend_centroid ( lcd , detrend = 'legendre' , sigclip = None , mingap = 0.5 ) :
    '''Detrends the x and y coordinate centroids for a Kepler light curve .
 Given an ` lcdict ` for a single quarter of Kepler data , returned by
 ` read _ kepler _ fitslc ` , this function returns this same dictionary ,
 appending detrended centroid _ x and centroid _ y values .
 Here " detrended " means " finite , SAP quality flag set to 0 , sigma clipped ,
 timegroups selected based on ` mingap ` day gaps , then fit vs time by a
 legendre polynomial of lowish degree " .
 Parameters
 lcd : lcdict
 An ` lcdict ` generated by the ` read _ kepler _ fitslc ` function .
 detrend : { ' legendre ' }
 Method by which to detrend the LC . ' legendre ' is the only thing
 implemented at the moment .
 sigclip : None or float or int or sequence of floats / ints
 Determines the type and amount of sigma - clipping done on the light curve
 to remove outliers . If None , no sigma - clipping is performed . If a two
 element sequence of floats / ints , the first element corresponds to the
 fainter sigma - clip limit , and the second element corresponds to the
 brighter sigma - clip limit .
 mingap : float
 Number of days by which to define " timegroups " ( for individual fitting
 each of timegroup , and to eliminate " burn - in " of Kepler spacecraft . For
 long cadence data , 0.5 days is typical .
 Returns
 tuple
 This is of the form ` ( lcd , errflag ) ` , where :
 ` lcd ` : an ` lcdict ` with the new key ` lcd [ ' centroids ' ] ` , containing the
 detrended times , ( centroid _ x , centroid _ y ) values , and their errors .
 ` errflag ` : boolean error flag , could be raised at various points .'''

implementation:
qnum = npunique ( lcd [ 'quarter' ] )
try :
    assert qnum . size == 1 , 'lcd should be for a unique quarter'
    assert detrend == 'legendre'
    qnum = int ( qnum )
except Exception as e :
    errflag = True
# Get finite , QUALITY _ FLAG ! = 0 times , centroids , and their errors .
# Fraquelli & Thompson ( 2012 ) , or perhaps also newer papers , give the list
# of exclusions for quality flags .
nbefore = lcd [ 'time' ] . size
# " ctd " for centroid .
times = lcd [ 'time' ] [ lcd [ 'sap_quality' ] == 0 ]
# Kepler Archive Manual KDMC - 10008-006 , pg 18 . MOM _ CENTR1 is the * column *
# value for the flux - weighted centroid , MOM _ CENTR2 is the row value .
ctd_x = lcd [ 'mom_centr2' ] [ lcd [ 'sap_quality' ] == 0 ]
ctd_y = lcd [ 'mom_centr1' ] [ lcd [ 'sap_quality' ] == 0 ]
ctd_x_err = lcd [ 'mom_centr2_err' ] [ lcd [ 'sap_quality' ] == 0 ]
ctd_y_err = lcd [ 'mom_centr1_err' ] [ lcd [ 'sap_quality' ] == 0 ]
find = npisfinite ( times ) & npisfinite ( ctd_x ) & npisfinite ( ctd_y )
find &= ( npisfinite ( ctd_x_err ) ) & ( npisfinite ( ctd_y_err ) )
f_times , f_ctd_x , f_ctd_y = times [ find ] , ctd_x [ find ] , ctd_y [ find ]
f_ctd_x_err , f_ctd_y_err = ctd_x_err [ find ] , ctd_y_err [ find ]
# Sigma clip whopping outliers . It ' d be better to have a general purpose
# function for this , but sigclip _ magseries works .
stimes_x , s_ctd_x , s_ctd_x_err = sigclip_magseries ( f_times , f_ctd_x , f_ctd_x_err , magsarefluxes = True , sigclip = 30.0 )
stimes_y , s_ctd_y , s_ctd_y_err = sigclip_magseries ( f_times , f_ctd_y , f_ctd_y_err , magsarefluxes = True , sigclip = 30.0 )
# Get times and centroids where everything is finite and sigma clipped .
mask_x = npin1d ( stimes_x , stimes_y )
s_times , s_ctd_x , s_ctd_x_err = ( stimes_x [ mask_x ] , s_ctd_x [ mask_x ] , s_ctd_x_err [ mask_x ] )
mask_y = npin1d ( stimes_y , stimes_x )
tmp , s_ctd_y , s_ctd_y_err = ( stimes_y [ mask_y ] , s_ctd_y [ mask_y ] , s_ctd_y_err [ mask_y ] )
try :
    np . testing . assert_array_equal ( s_times , tmp )
    assert len ( s_ctd_y ) == len ( s_times )
    assert len ( s_ctd_y_err ) == len ( s_times )
    assert len ( s_ctd_x ) == len ( s_times )
    assert len ( s_ctd_x_err ) == len ( s_times )
except AssertionError :
    return lcd , True
nqflag = s_times . size
# Drop intra - quarter and interquarter gaps in the timeseries . These are the
# same limits set by Armstrong et al ( 2014 ) : split each quarter ' s
# timegroups by whether points are within 0.5 day limits . Then drop points
# within 0.5 days of any boundary . Finally , since the interquarter burn - in
# time is more like 1 day , drop a further 0.5 days from the edges of each
# quarter . A nicer way to implement this would be with numpy masks , but
# this approach just constructs the full arrays for any given quarter .
ngroups , groups = find_lc_timegroups ( s_times , mingap = mingap )
tmp_times , tmp_ctd_x , tmp_ctd_y = [ ] , [ ] , [ ]
tmp_ctd_x_err , tmp_ctd_y_err = [ ] , [ ]
for group in groups :
    tg_times = s_times [ group ]
    tg_ctd_x = s_ctd_x [ group ]
    tg_ctd_y = s_ctd_y [ group ]
    tg_ctd_x_err = s_ctd_x_err [ group ]
    tg_ctd_y_err = s_ctd_y_err [ group ]
    try :
        sel = ( ( tg_times > npmin ( tg_times ) + mingap ) & ( tg_times < npmax ( tg_times ) - mingap ) )
    except ValueError : # If tgtimes is empty , continue to next timegroup .
        continue
    tmp_times . append ( tg_times [ sel ] )
    tmp_ctd_x . append ( tg_ctd_x [ sel ] )
    tmp_ctd_y . append ( tg_ctd_y [ sel ] )
    tmp_ctd_x_err . append ( tg_ctd_x_err [ sel ] )
    tmp_ctd_y_err . append ( tg_ctd_y_err [ sel ] )
s_times , s_ctd_x , s_ctd_y , s_ctd_x_err , s_ctd_y_err = ( nparray ( [ ] ) , nparray ( [ ] ) , nparray ( [ ] ) , nparray ( [ ] ) , nparray ( [ ] ) )
# N . b . : works fine with empty arrays .
for ix , _ in enumerate ( tmp_times ) :
    s_times = npappend ( s_times , tmp_times [ ix ] )
    s_ctd_x = npappend ( s_ctd_x , tmp_ctd_x [ ix ] )
    s_ctd_y = npappend ( s_ctd_y , tmp_ctd_y [ ix ] )
    s_ctd_x_err = npappend ( s_ctd_x_err , tmp_ctd_x_err [ ix ] )
    s_ctd_y_err = npappend ( s_ctd_y_err , tmp_ctd_y_err [ ix ] )
# Extra inter - quarter burn - in of 0.5 days .
try :
    s_ctd_x = s_ctd_x [ ( s_times > ( npmin ( s_times ) + mingap ) ) & ( s_times < ( npmax ( s_times ) - mingap ) ) ]
except Exception as e : # Case : s _ times is wonky , all across this quarter . ( Implemented because
# of a rare bug with a singleton s _ times array ) .
    LOGERROR ( 'DETREND FAILED, qnum {:d}' . format ( qnum ) )
    return npnan , True
s_ctd_y = s_ctd_y [ ( s_times > ( npmin ( s_times ) + mingap ) ) & ( s_times < ( npmax ( s_times ) - mingap ) ) ]
s_ctd_x_err = s_ctd_x_err [ ( s_times > ( npmin ( s_times ) + mingap ) ) & ( s_times < ( npmax ( s_times ) - mingap ) ) ]
s_ctd_y_err = s_ctd_y_err [ ( s_times > ( npmin ( s_times ) + mingap ) ) & ( s_times < ( npmax ( s_times ) - mingap ) ) ]
# Careful to do this last . . .
s_times = s_times [ ( s_times > ( npmin ( s_times ) + mingap ) ) & ( s_times < ( npmax ( s_times ) - mingap ) ) ]
nafter = s_times . size
LOGINFO ( 'CLIPPING (SAP), qnum: {:d}' . format ( qnum ) + '\nndet before qflag & sigclip: {:d} ({:.3g}),' . format ( nbefore , 1. ) + '\nndet after qflag & finite & sigclip: {:d} ({:.3g})' . format ( nqflag , nqflag / float ( nbefore ) ) + '\nndet after dropping pts near gaps: {:d} ({:.3g})' . format ( nafter , nafter / float ( nbefore ) ) )
# DETREND : fit a " low " order legendre series ( see
# " legendredeg _ vs _ npts _ per _ timegroup _ ctd . pdf " ) , and save it to the output
# dictionary . Save the fit ( residuals to be computed after ) .
ctd_dtr = { }
if detrend == 'legendre' :
    mingap = 0.5
    # days
    ngroups , groups = find_lc_timegroups ( s_times , mingap = mingap )
    tmpctdxlegfit , tmpctdylegfit , legdegs = [ ] , [ ] , [ ]
    for group in groups :
        tg_times = s_times [ group ]
        tg_ctd_x = s_ctd_x [ group ]
        tg_ctd_x_err = s_ctd_x_err [ group ]
        tg_ctd_y = s_ctd_y [ group ]
        tg_ctd_y_err = s_ctd_y_err [ group ]
        legdeg = _get_legendre_deg_ctd ( len ( tg_times ) )
        tg_ctd_x_fit , _ , _ = _legendre_dtr ( tg_times , tg_ctd_x , tg_ctd_x_err , legendredeg = legdeg )
        tg_ctd_y_fit , _ , _ = _legendre_dtr ( tg_times , tg_ctd_y , tg_ctd_y_err , legendredeg = legdeg )
        tmpctdxlegfit . append ( tg_ctd_x_fit )
        tmpctdylegfit . append ( tg_ctd_y_fit )
        legdegs . append ( legdeg )
    fit_ctd_x , fit_ctd_y = nparray ( [ ] ) , nparray ( [ ] )
    for ix , _ in enumerate ( tmpctdxlegfit ) :
        fit_ctd_x = npappend ( fit_ctd_x , tmpctdxlegfit [ ix ] )
        fit_ctd_y = npappend ( fit_ctd_y , tmpctdylegfit [ ix ] )
ctd_dtr = { 'times' : s_times , 'ctd_x' : s_ctd_x , 'ctd_x_err' : s_ctd_x_err , 'fit_ctd_x' : fit_ctd_x , 'ctd_y' : s_ctd_y , 'ctd_y_err' : s_ctd_y_err , 'fit_ctd_y' : fit_ctd_y }
lcd [ 'ctd_dtr' ] = ctd_dtr
return lcd , False


signature:
def get_centroid_offsets ( lcd , t_ing_egr , oot_buffer_time = 0.1 , sample_factor = 3 ) :
    '''After running ` detrend _ centroid ` , this gets positions of centroids during
 transits , and outside of transits .
 These positions can then be used in a false positive analysis .
 This routine requires knowing the ingress and egress times for every
 transit of interest within the quarter this routine is being called for .
 There is currently no astrobase routine that automates this for periodic
 transits ( it must be done in a calling routine ) .
 To get out of transit centroids , this routine takes points outside of the
 " buffer " set by ` oot _ buffer _ time ` , sampling 3x as many points on either
 side of the transit as are in the transit ( or however many are specified by
 ` sample _ factor ` ) .
 Parameters
 lcd : lcdict
 An ` lcdict ` generated by the ` read _ kepler _ fitslc ` function . We assume
 that the ` detrend _ centroid ` function has been run on this ` lcdict ` .
 t _ ing _ egr : list of tuples
 This is of the form : :
 [ ( ingress time of i ^ th transit , egress time of i ^ th transit ) ]
 for i the transit number index in this quarter ( starts at zero at the
 beginning of every quarter ) . Assumes units of BJD .
 oot _ buffer _ time : float
 Number of days away from ingress and egress times to begin sampling " out
 of transit " centroid points . The number of out of transit points to take
 per transit is 3x the number of points in transit .
 sample _ factor : float
 The size of out of transit window from which to sample .
 Returns
 dict
 This is a dictionary keyed by transit number ( i . e . , the same index as
 ` t _ ing _ egr ` ) , where each key contains the following value : :
 { ' ctd _ x _ in _ tra ' : ctd _ x _ in _ tra ,
 ' ctd _ y _ in _ tra ' : ctd _ y _ in _ tra ,
 ' ctd _ x _ oot ' : ctd _ x _ oot ,
 ' ctd _ y _ oot ' : ctd _ y _ oot ,
 ' npts _ in _ tra ' : len ( ctd _ x _ in _ tra ) ,
 ' npts _ oot ' : len ( ctd _ x _ oot ) ,
 ' in _ tra _ times ' : in _ tra _ times ,
 ' oot _ times ' : oot _ times }'''

implementation:
# NOTE :
# Bryson + ( 2013 ) gives a more complicated and more correct approach to this
# problem , computing offsets relative to positions defined on the SKY . This
# requires using a Kepler focal plane geometry model . I don ' t have that
# model , or know how to get it . So I use a simpler approach .
qnum = int ( np . unique ( lcd [ 'quarter' ] ) )
LOGINFO ( 'Getting centroid offsets (qnum: {:d})...' . format ( qnum ) )
# Kepler pixel scale , cf .
# https : / / keplerscience . arc . nasa . gov / the - kepler - space - telescope . html
arcsec_per_px = 3.98
# Get the residuals ( units : pixel offset ) .
times = lcd [ 'ctd_dtr' ] [ 'times' ]
ctd_resid_x = lcd [ 'ctd_dtr' ] [ 'ctd_x' ] - lcd [ 'ctd_dtr' ] [ 'fit_ctd_x' ]
ctd_resid_y = lcd [ 'ctd_dtr' ] [ 'ctd_y' ] - lcd [ 'ctd_dtr' ] [ 'fit_ctd_y' ]
# Return results in " centroid dictionary " ( has keys of transit number ) .
cd = { }
for ix , ( t_ing , t_egr ) in enumerate ( t_ing_egr ) : # We have in - transit times as input .
    in_tra_times = times [ ( times > t_ing ) & ( times < t_egr ) ]
    # Compute out of transit times on either side of the in - transit times .
    transit_dur = t_egr - t_ing
    oot_window_len = sample_factor * transit_dur
    oot_before = times [ ( times < ( t_ing - oot_buffer_time ) ) & ( times > ( t_ing - oot_buffer_time - oot_window_len ) ) ]
    oot_after = times [ ( times > ( t_egr + oot_buffer_time ) ) & ( times < ( t_egr + oot_buffer_time + oot_window_len ) ) ]
    oot_times = npconcatenate ( [ oot_before , oot_after ] )
    mask_tra = npin1d ( times , in_tra_times )
    mask_oot = npin1d ( times , oot_times )
    # Convert to units of arcseconds .
    ctd_x_in_tra = ctd_resid_x [ mask_tra ] * arcsec_per_px
    ctd_y_in_tra = ctd_resid_y [ mask_tra ] * arcsec_per_px
    ctd_x_oot = ctd_resid_x [ mask_oot ] * arcsec_per_px
    ctd_y_oot = ctd_resid_y [ mask_oot ] * arcsec_per_px
    cd [ ix ] = { 'ctd_x_in_tra' : ctd_x_in_tra , 'ctd_y_in_tra' : ctd_y_in_tra , 'ctd_x_oot' : ctd_x_oot , 'ctd_y_oot' : ctd_y_oot , 'npts_in_tra' : len ( ctd_x_in_tra ) , 'npts_oot' : len ( ctd_x_oot ) , 'in_tra_times' : in_tra_times , 'oot_times' : oot_times }
LOGINFO ( 'Got centroid offsets (qnum: {:d}).' . format ( qnum ) )
return cd


signature:
def _get_legendre_deg_ctd ( npts ) :
    '''This is a helper function for centroid detrending .'''

implementation:
from scipy . interpolate import interp1d
degs = nparray ( [ 4 , 5 , 6 , 10 , 15 ] )
pts = nparray ( [ 1e2 , 3e2 , 5e2 , 1e3 , 3e3 ] )
fn = interp1d ( pts , degs , kind = 'linear' , bounds_error = False , fill_value = ( min ( degs ) , max ( degs ) ) )
legendredeg = int ( npfloor ( fn ( npts ) ) )
return legendredeg


signature:
def _legendre_dtr ( x , y , y_err , legendredeg = 10 ) :
    '''This calculates the residual and chi - sq values for a Legendre
 function fit .
 Parameters
 x : np . array
 Array of the independent variable .
 y : np . array
 Array of the dependent variable .
 y _ err : np . array
 Array of errors associated with each ` y ` value . Used to calculate fit
 weights .
 legendredeg : int
 The degree of the Legendre function to use when fitting .
 Returns
 tuple
 The tuple returned is of the form : ( fit _ y , fitchisq , fitredchisq )'''

implementation:
try :
    p = Legendre . fit ( x , y , legendredeg )
    fit_y = p ( x )
except Exception as e :
    fit_y = npzeros_like ( y )
fitchisq = npsum ( ( ( fit_y - y ) * ( fit_y - y ) ) / ( y_err * y_err ) )
nparams = legendredeg + 1
fitredchisq = fitchisq / ( len ( y ) - nparams - 1 )
LOGINFO ( 'legendre detrend applied. chisq = %.5f, reduced chisq = %.5f' % ( fitchisq , fitredchisq ) )
return fit_y , fitchisq , fitredchisq


signature:
def timebinlc ( lcfile , binsizesec , outdir = None , lcformat = 'hat-sql' , lcformatdir = None , timecols = None , magcols = None , errcols = None , minbinelems = 7 ) :
    '''This bins the given light curve file in time using the specified bin size .
 Parameters
 lcfile : str
 The file name to process .
 binsizesec : float
 The time bin - size in seconds .
 outdir : str or None
 If this is a str , the output LC will be written to ` outdir ` . If this is
 None , the output LC will be written to the same directory as ` lcfile ` .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curve file .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 timecols , magcols , errcols : lists of str
 The keys in the lcdict produced by your light curve reader function that
 correspond to the times , mags / fluxes , and associated measurement errors
 that will be used as inputs to the binning process . If these are None ,
 the default values for ` timecols ` , ` magcols ` , and ` errcols ` for your
 light curve format will be used here .
 minbinelems : int
 The minimum number of time - bin elements required to accept a time - bin as
 valid for the output binned light curve .
 Returns
 str
 The name of the output pickle file with the binned LC .
 Writes the output binned light curve to a pickle that contains the
 lcdict with an added ` lcdict [ ' binned ' ] [ magcol ] ` key , which contains the
 binned times , mags / fluxes , and errs as
 ` lcdict [ ' binned ' ] [ magcol ] [ ' times ' ] ` , ` lcdict [ ' binned ' ] [ magcol ] [ ' mags ' ] ` ,
 and ` lcdict [ ' epd ' ] [ magcol ] [ ' errs ' ] ` for each ` magcol ` provided in the
 input or default ` magcols ` value for this light curve format .'''

implementation:
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
# override the default timecols , magcols , and errcols
# using the ones provided to the function
if timecols is None :
    timecols = dtimecols
if magcols is None :
    magcols = dmagcols
if errcols is None :
    errcols = derrcols
# get the LC into a dict
lcdict = readerfunc ( lcfile )
# this should handle lists / tuples being returned by readerfunc
# we assume that the first element is the actual lcdict
# FIXME : figure out how to not need this assumption
if ( ( isinstance ( lcdict , ( list , tuple ) ) ) and ( isinstance ( lcdict [ 0 ] , dict ) ) ) :
    lcdict = lcdict [ 0 ]
# skip already binned light curves
if 'binned' in lcdict :
    LOGERROR ( 'this light curve appears to be binned already, skipping...' )
    return None
lcdict [ 'binned' ] = { }
for tcol , mcol , ecol in zip ( timecols , magcols , errcols ) : # dereference the columns and get them from the lcdict
    if '.' in tcol :
        tcolget = tcol . split ( '.' )
    else :
        tcolget = [ tcol ]
    times = _dict_get ( lcdict , tcolget )
    if '.' in mcol :
        mcolget = mcol . split ( '.' )
    else :
        mcolget = [ mcol ]
    mags = _dict_get ( lcdict , mcolget )
    if '.' in ecol :
        ecolget = ecol . split ( '.' )
    else :
        ecolget = [ ecol ]
    errs = _dict_get ( lcdict , ecolget )
    # normalize here if not using special normalization
    if normfunc is None :
        ntimes , nmags = normalize_magseries ( times , mags , magsarefluxes = magsarefluxes )
        times , mags , errs = ntimes , nmags , errs
    # now bin the mag series as requested
    binned = time_bin_magseries_with_errs ( times , mags , errs , binsize = binsizesec , minbinelems = minbinelems )
    # put this into the special binned key of the lcdict
    lcdict [ 'binned' ] [ mcol ] = { 'times' : binned [ 'binnedtimes' ] , 'mags' : binned [ 'binnedmags' ] , 'errs' : binned [ 'binnederrs' ] , 'nbins' : binned [ 'nbins' ] , 'timebins' : binned [ 'jdbins' ] , 'binsizesec' : binsizesec }
# done with binning for all magcols , now generate the output file
# this will always be a pickle
if outdir is None :
    outdir = os . path . dirname ( lcfile )
outfile = os . path . join ( outdir , '%s-binned%.1fsec-%s.pkl' % ( squeeze ( lcdict [ 'objectid' ] ) . replace ( ' ' , '-' ) , binsizesec , lcformat ) )
with open ( outfile , 'wb' ) as outfd :
    pickle . dump ( lcdict , outfd , protocol = pickle . HIGHEST_PROTOCOL )
return outfile


signature:
def timebinlc_worker ( task ) :
    '''This is a parallel worker for the function below .
 Parameters
 task : tuple
 This is of the form : :
 task [ 0 ] = lcfile
 task [ 1 ] = binsizesec
 task [ 3 ] = { ' outdir ' , ' lcformat ' , ' lcformatdir ' ,
 ' timecols ' , ' magcols ' , ' errcols ' , ' minbinelems ' }
 Returns
 str
 The output pickle file with the binned LC if successful . None otherwise .'''

implementation:
lcfile , binsizesec , kwargs = task
try :
    binnedlc = timebinlc ( lcfile , binsizesec , ** kwargs )
    LOGINFO ( '%s binned using %s sec -> %s OK' % ( lcfile , binsizesec , binnedlc ) )
    return binnedlc
except Exception as e :
    LOGEXCEPTION ( 'failed to bin %s using binsizesec = %s' % ( lcfile , binsizesec ) )
    return None


signature:
def parallel_timebin ( lclist , binsizesec , maxobjects = None , outdir = None , lcformat = 'hat-sql' , lcformatdir = None , timecols = None , magcols = None , errcols = None , minbinelems = 7 , nworkers = NCPUS , maxworkertasks = 1000 ) :
    '''This time - bins all the LCs in the list using the specified bin size .
 Parameters
 lclist : list of str
 The input LCs to process .
 binsizesec : float
 The time bin size to use in seconds .
 maxobjects : int or None
 If provided , LC processing will stop at ` lclist [ maxobjects ] ` .
 outdir : str or None
 The directory where output LCs will be written . If None , will write to
 the same directory as the input LCs .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curve file .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 timecols , magcols , errcols : lists of str
 The keys in the lcdict produced by your light curve reader function that
 correspond to the times , mags / fluxes , and associated measurement errors
 that will be used as inputs to the binning process . If these are None ,
 the default values for ` timecols ` , ` magcols ` , and ` errcols ` for your
 light curve format will be used here .
 minbinelems : int
 The minimum number of time - bin elements required to accept a time - bin as
 valid for the output binned light curve .
 nworkers : int
 Number of parallel workers to launch .
 maxworkertasks : int
 The maximum number of tasks a parallel worker will complete before being
 replaced to guard against memory leaks .
 Returns
 dict
 The returned dict contains keys = input LCs , vals = output LCs .'''

implementation:
if outdir and not os . path . exists ( outdir ) :
    os . mkdir ( outdir )
if maxobjects is not None :
    lclist = lclist [ : maxobjects ]
tasks = [ ( x , binsizesec , { 'outdir' : outdir , 'lcformat' : lcformat , 'lcformatdir' : lcformatdir , 'timecols' : timecols , 'magcols' : magcols , 'errcols' : errcols , 'minbinelems' : minbinelems } ) for x in lclist ]
pool = mp . Pool ( nworkers , maxtasksperchild = maxworkertasks )
results = pool . map ( timebinlc_worker , tasks )
pool . close ( )
pool . join ( )
resdict = { os . path . basename ( x ) : y for ( x , y ) in zip ( lclist , results ) }
return resdict


signature:
def parallel_timebin_lcdir ( lcdir , binsizesec , maxobjects = None , outdir = None , lcformat = 'hat-sql' , lcformatdir = None , timecols = None , magcols = None , errcols = None , minbinelems = 7 , nworkers = NCPUS , maxworkertasks = 1000 ) :
    '''This time bins all the light curves in the specified directory .
 Parameters
 lcdir : list of str
 Directory containing the input LCs to process .
 binsizesec : float
 The time bin size to use in seconds .
 maxobjects : int or None
 If provided , LC processing will stop at ` lclist [ maxobjects ] ` .
 outdir : str or None
 The directory where output LCs will be written . If None , will write to
 the same directory as the input LCs .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curve file .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 timecols , magcols , errcols : lists of str
 The keys in the lcdict produced by your light curve reader function that
 correspond to the times , mags / fluxes , and associated measurement errors
 that will be used as inputs to the binning process . If these are None ,
 the default values for ` timecols ` , ` magcols ` , and ` errcols ` for your
 light curve format will be used here .
 minbinelems : int
 The minimum number of time - bin elements required to accept a time - bin as
 valid for the output binned light curve .
 nworkers : int
 Number of parallel workers to launch .
 maxworkertasks : int
 The maximum number of tasks a parallel worker will complete before being
 replaced to guard against memory leaks .
 Returns
 dict
 The returned dict contains keys = input LCs , vals = output LCs .'''

implementation:
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( fileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
lclist = sorted ( glob . glob ( os . path . join ( lcdir , fileglob ) ) )
return parallel_timebin ( lclist , binsizesec , maxobjects = maxobjects , outdir = outdir , lcformat = lcformat , timecols = timecols , magcols = magcols , errcols = errcols , minbinelems = minbinelems , nworkers = nworkers , maxworkertasks = maxworkertasks )


signature:
def get_varfeatures ( lcfile , outdir , timecols = None , magcols = None , errcols = None , mindet = 1000 , lcformat = 'hat-sql' , lcformatdir = None ) :
    '''This runs : py : func : ` astrobase . varclass . varfeatures . all _ nonperiodic _ features `
 on a single LC file .
 Parameters
 lcfile : str
 The input light curve to process .
 outfile : str
 The filename of the output variable features pickle that will be
 generated .
 timecols : list of str or None
 The timecol keys to use from the lcdict in calculating the features .
 magcols : list of str or None
 The magcol keys to use from the lcdict in calculating the features .
 errcols : list of str or None
 The errcol keys to use from the lcdict in calculating the features .
 mindet : int
 The minimum number of LC points required to generate variability
 features .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 Returns
 str
 The generated variability features pickle for the input LC , with results
 for each magcol in the input ` magcol ` or light curve format ' s default
 ` magcol ` list .'''

implementation:
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
# override the default timecols , magcols , and errcols
# using the ones provided to the function
if timecols is None :
    timecols = dtimecols
if magcols is None :
    magcols = dmagcols
if errcols is None :
    errcols = derrcols
try : # get the LC into a dict
    lcdict = readerfunc ( lcfile )
    # this should handle lists / tuples being returned by readerfunc
    # we assume that the first element is the actual lcdict
    # FIXME : figure out how to not need this assumption
    if ( ( isinstance ( lcdict , ( list , tuple ) ) ) and ( isinstance ( lcdict [ 0 ] , dict ) ) ) :
        lcdict = lcdict [ 0 ]
    resultdict = { 'objectid' : lcdict [ 'objectid' ] , 'info' : lcdict [ 'objectinfo' ] , 'lcfbasename' : os . path . basename ( lcfile ) }
    # normalize using the special function if specified
    if normfunc is not None :
        lcdict = normfunc ( lcdict )
    for tcol , mcol , ecol in zip ( timecols , magcols , errcols ) : # dereference the columns and get them from the lcdict
        if '.' in tcol :
            tcolget = tcol . split ( '.' )
        else :
            tcolget = [ tcol ]
        times = _dict_get ( lcdict , tcolget )
        if '.' in mcol :
            mcolget = mcol . split ( '.' )
        else :
            mcolget = [ mcol ]
        mags = _dict_get ( lcdict , mcolget )
        if '.' in ecol :
            ecolget = ecol . split ( '.' )
        else :
            ecolget = [ ecol ]
        errs = _dict_get ( lcdict , ecolget )
        # normalize here if not using special normalization
        if normfunc is None :
            ntimes , nmags = normalize_magseries ( times , mags , magsarefluxes = magsarefluxes )
            times , mags , errs = ntimes , nmags , errs
        # make sure we have finite values
        finind = np . isfinite ( times ) & np . isfinite ( mags ) & np . isfinite ( errs )
        # make sure we have enough finite values
        if mags [ finind ] . size < mindet :
            LOGINFO ( 'not enough LC points: %s in normalized %s LC: %s' % ( mags [ finind ] . size , mcol , os . path . basename ( lcfile ) ) )
            resultdict [ mcol ] = None
        else : # get the features for this magcol
            lcfeatures = varfeatures . all_nonperiodic_features ( times , mags , errs )
            resultdict [ mcol ] = lcfeatures
    # now that we ' ve collected all the magcols , we can choose which is the
    # " best " magcol . this is defined as the magcol that gives us the
    # smallest LC MAD .
    try :
        magmads = np . zeros ( len ( magcols ) )
        for mind , mcol in enumerate ( magcols ) :
            if '.' in mcol :
                mcolget = mcol . split ( '.' )
            else :
                mcolget = [ mcol ]
            magmads [ mind ] = resultdict [ mcol ] [ 'mad' ]
        # smallest MAD index
        bestmagcolind = np . where ( magmads == np . min ( magmads ) ) [ 0 ]
        resultdict [ 'bestmagcol' ] = magcols [ bestmagcolind ]
    except Exception as e :
        resultdict [ 'bestmagcol' ] = None
    outfile = os . path . join ( outdir , 'varfeatures-%s.pkl' % squeeze ( resultdict [ 'objectid' ] ) . replace ( ' ' , '-' ) )
    with open ( outfile , 'wb' ) as outfd :
        pickle . dump ( resultdict , outfd , protocol = 4 )
    return outfile
except Exception as e :
    LOGEXCEPTION ( 'failed to get LC features for %s because: %s' % ( os . path . basename ( lcfile ) , e ) )
    return None


signature:
def _varfeatures_worker ( task ) :
    '''This wraps varfeatures .'''

implementation:
try :
    ( lcfile , outdir , timecols , magcols , errcols , mindet , lcformat , lcformatdir ) = task
    return get_varfeatures ( lcfile , outdir , timecols = timecols , magcols = magcols , errcols = errcols , mindet = mindet , lcformat = lcformat , lcformatdir = lcformatdir )
except Exception as e :
    return None


signature:
def serial_varfeatures ( lclist , outdir , maxobjects = None , timecols = None , magcols = None , errcols = None , mindet = 1000 , lcformat = 'hat-sql' , lcformatdir = None ) :
    '''This runs variability feature extraction for a list of LCs .
 Parameters
 lclist : list of str
 The list of light curve file names to process .
 outdir : str
 The directory where the output varfeatures pickle files will be written .
 maxobjects : int
 The number of LCs to process from ` lclist ` .
 timecols : list of str or None
 The timecol keys to use from the lcdict in calculating the features .
 magcols : list of str or None
 The magcol keys to use from the lcdict in calculating the features .
 errcols : list of str or None
 The errcol keys to use from the lcdict in calculating the features .
 mindet : int
 The minimum number of LC points required to generate variability
 features .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 Returns
 list of str
 List of the generated variability features pickles for the input LCs ,
 with results for each magcol in the input ` magcol ` or light curve
 format ' s default ` magcol ` list .'''

implementation:
if maxobjects :
    lclist = lclist [ : maxobjects ]
tasks = [ ( x , outdir , timecols , magcols , errcols , mindet , lcformat , lcformatdir ) for x in lclist ]
for task in tqdm ( tasks ) :
    result = _varfeatures_worker ( task )
return result


signature:
def parallel_varfeatures ( lclist , outdir , maxobjects = None , timecols = None , magcols = None , errcols = None , mindet = 1000 , lcformat = 'hat-sql' , lcformatdir = None , nworkers = NCPUS ) :
    '''This runs variable feature extraction in parallel for all LCs in ` lclist ` .
 Parameters
 lclist : list of str
 The list of light curve file names to process .
 outdir : str
 The directory where the output varfeatures pickle files will be written .
 maxobjects : int
 The number of LCs to process from ` lclist ` .
 timecols : list of str or None
 The timecol keys to use from the lcdict in calculating the features .
 magcols : list of str or None
 The magcol keys to use from the lcdict in calculating the features .
 errcols : list of str or None
 The errcol keys to use from the lcdict in calculating the features .
 mindet : int
 The minimum number of LC points required to generate variability
 features .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 nworkers : int
 The number of parallel workers to launch .
 Returns
 dict
 A dict with key : val pairs of input LC file name : the generated
 variability features pickles for each of the input LCs , with results for
 each magcol in the input ` magcol ` or light curve format ' s default
 ` magcol ` list .'''

implementation:
# make sure to make the output directory if it doesn ' t exist
if not os . path . exists ( outdir ) :
    os . makedirs ( outdir )
if maxobjects :
    lclist = lclist [ : maxobjects ]
tasks = [ ( x , outdir , timecols , magcols , errcols , mindet , lcformat , lcformatdir ) for x in lclist ]
with ProcessPoolExecutor ( max_workers = nworkers ) as executor :
    resultfutures = executor . map ( varfeatures_worker , tasks )
results = [ x for x in resultfutures ]
resdict = { os . path . basename ( x ) : y for ( x , y ) in zip ( lclist , results ) }
return resdict


signature:
def parallel_varfeatures_lcdir ( lcdir , outdir , fileglob = None , maxobjects = None , timecols = None , magcols = None , errcols = None , recursive = True , mindet = 1000 , lcformat = 'hat-sql' , lcformatdir = None , nworkers = NCPUS ) :
    '''This runs parallel variable feature extraction for a directory of LCs .
 Parameters
 lcdir : str
 The directory of light curve files to process .
 outdir : str
 The directory where the output varfeatures pickle files will be written .
 fileglob : str or None
 The file glob to use when looking for light curve files in ` lcdir ` . If
 None , the default file glob associated for this LC format will be used .
 maxobjects : int
 The number of LCs to process from ` lclist ` .
 timecols : list of str or None
 The timecol keys to use from the lcdict in calculating the features .
 magcols : list of str or None
 The magcol keys to use from the lcdict in calculating the features .
 errcols : list of str or None
 The errcol keys to use from the lcdict in calculating the features .
 mindet : int
 The minimum number of LC points required to generate variability
 features .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 nworkers : int
 The number of parallel workers to launch .
 Returns
 dict
 A dict with key : val pairs of input LC file name : the generated
 variability features pickles for each of the input LCs , with results for
 each magcol in the input ` magcol ` or light curve format ' s default
 ` magcol ` list .'''

implementation:
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
if not fileglob :
    fileglob = dfileglob
# now find the files
LOGINFO ( 'searching for %s light curves in %s ...' % ( lcformat , lcdir ) )
if recursive is False :
    matching = glob . glob ( os . path . join ( lcdir , fileglob ) )
else : # use recursive glob for Python 3.5 +
    if sys . version_info [ : 2 ] > ( 3 , 4 ) :
        matching = glob . glob ( os . path . join ( lcdir , '**' , fileglob ) , recursive = True )
    # otherwise , use os . walk and glob
    else : # use os . walk to go through the directories
        walker = os . walk ( lcdir )
        matching = [ ]
        for root , dirs , _files in walker :
            for sdir in dirs :
                searchpath = os . path . join ( root , sdir , fileglob )
                foundfiles = glob . glob ( searchpath )
                if foundfiles :
                    matching . extend ( foundfiles )
# now that we have all the files , process them
if matching and len ( matching ) > 0 :
    LOGINFO ( 'found %s light curves, getting varfeatures...' % len ( matching ) )
    return parallel_varfeatures ( matching , outdir , maxobjects = maxobjects , timecols = timecols , magcols = magcols , errcols = errcols , mindet = mindet , lcformat = lcformat , lcformatdir = lcformatdir , nworkers = nworkers )
else :
    LOGERROR ( 'no light curve files in %s format found in %s' % ( lcformat , lcdir ) )
    return None


signature:
def checkplot_pickle_to_png ( checkplotin , outfile , extrarows = None ) :
    '''This reads the checkplot pickle or dict provided , and writes out a PNG .
 The output PNG contains most of the information in the input checkplot
 pickle / dict , and can be used to quickly glance through the highlights
 instead of having to review the checkplot with the ` checkplotserver `
 webapp . This is useful for exporting read - only views of finalized checkplots
 from the ` checkplotserver ` as well , to share them with other people .
 The PNG has 4 x N tiles : :
 [ finder ] [ objectinfo ] [ varinfo / comments ] [ unphased LC ]
 [ periodogram1 ] [ phased LC P1 ] [ phased LC P2 ] [ phased LC P3 ]
 [ periodogram2 ] [ phased LC P1 ] [ phased LC P2 ] [ phased LC P3 ]
 [ periodogramN ] [ phased LC P1 ] [ phased LC P2 ] [ phased LC P3 ]
 for N independent period - finding methods producing :
 - periodogram1,2,3 . . . N : the periodograms from each method
 - phased LC P1 , P2 , P3 : the phased lightcurves using the best 3 peaks in each
 periodogram
 Parameters
 checkplotin : dict or str
 This is either a checkplotdict produced by
 : py : func : ` astrobase . checkplot . pkl . checkplot _ dict ` or a checkplot pickle
 file produced by : py : func : ` astrobase . checkplot . pkl . checkplot _ pickle ` .
 outfile : str
 The filename of the output PNG file to create .
 extrarows : list of tuples
 This is a list of 4 - element tuples containing paths to PNG files that
 will be added to the end of the rows generated from the checkplotin
 pickle / dict . Each tuple represents a row in the final output PNG
 file . If there are less than 4 elements per tuple , the missing elements
 will be filled in with white - space . If there are more than 4 elements
 per tuple , only the first four will be used .
 The purpose of this kwarg is to incorporate periodograms and phased LC
 plots ( in the form of PNGs ) generated from an external period - finding
 function or program ( like VARTOOLS ) to allow for comparison with
 astrobase results .
 NOTE : the PNG files specified in ` extrarows ` here will be added to those
 already present in the input checkplotdict [ ' externalplots ' ] if that is
 None because you passed in a similar list of external plots to the
 : py : func : ` astrobase . checkplot . pkl . checkplot _ pickle ` function earlier . In
 this case , ` extrarows ` can be used to add even more external plots if
 desired .
 Each external plot PNG will be resized to 750 x 480 pixels to fit into
 an output image cell .
 By convention , each 4 - element tuple should contain :
 - a periodiogram PNG
 - phased LC PNG with 1st best peak period from periodogram
 - phased LC PNG with 2nd best peak period from periodogram
 - phased LC PNG with 3rd best peak period from periodogram
 Example of extrarows : :
 [ ( ' / path / to / external / bls - periodogram . png ' ,
 ' / path / to / external / bls - phasedlc - plot - bestpeak . png ' ,
 ' / path / to / external / bls - phasedlc - plot - peak2 . png ' ,
 ' / path / to / external / bls - phasedlc - plot - peak3 . png ' ) ,
 ( ' / path / to / external / pdm - periodogram . png ' ,
 ' / path / to / external / pdm - phasedlc - plot - bestpeak . png ' ,
 ' / path / to / external / pdm - phasedlc - plot - peak2 . png ' ,
 ' / path / to / external / pdm - phasedlc - plot - peak3 . png ' ) ,
 Returns
 str
 The absolute path to the generated checkplot PNG .'''

implementation:
# figure out if the checkplotpickle is a filename
# python 3
if sys . version_info [ : 2 ] > ( 3 , 2 ) :
    if ( isinstance ( checkplotin , str ) and os . path . exists ( checkplotin ) ) :
        cpd = _read_checkplot_picklefile ( checkplotin )
    elif isinstance ( checkplotin , dict ) :
        cpd = checkplotin
    else :
        LOGERROR ( 'checkplotin: %s of type %s is not a ' 'valid checkplot filename (or does not exist), or a dict' % ( os . path . abspath ( checkplotin ) , type ( checkplotin ) ) )
        return None
# check for unicode in python 2.7
else : # get the current checkplotdict
    if ( ( isinstance ( checkplotin , str ) or isinstance ( checkplotin , unicode ) ) and os . path . exists ( checkplotin ) ) :
        cpd = _read_checkplot_picklefile ( checkplotin )
    elif isinstance ( checkplotin , dict ) :
        cpd = checkplotin
    else :
        LOGERROR ( 'checkplotin: %s of type %s is not a ' 'valid checkplot filename (or does not exist), or a dict' % ( os . path . abspath ( checkplotin ) , type ( checkplotin ) ) )
        return None
# figure out the dimensions of the output png
# each cell is 750 x 480 pixels
# a row is made of four cells
# - the first row is for object info
# - the rest are for periodograms and phased LCs , one row per method
# if there are more than three phased LC plots per method , we ' ll only plot 3
if 'pfmethods' in cpd :
    cplspmethods = cpd [ 'pfmethods' ]
else :
    cplspmethods = [ ]
    for pfm in METHODSHORTLABELS :
        if pfm in cpd :
            cplspmethods . append ( pfm )
cprows = len ( cplspmethods )
# add in any extra rows from neighbors
if 'neighbors' in cpd and cpd [ 'neighbors' ] and len ( cpd [ 'neighbors' ] ) > 0 :
    nbrrows = len ( cpd [ 'neighbors' ] )
else :
    nbrrows = 0
# add in any extra rows from keyword arguments
if extrarows and len ( extrarows ) > 0 :
    erows = len ( extrarows )
else :
    erows = 0
# add in any extra rows from the checkplot dict
if ( 'externalplots' in cpd and cpd [ 'externalplots' ] and len ( cpd [ 'externalplots' ] ) > 0 ) :
    cpderows = len ( cpd [ 'externalplots' ] )
else :
    cpderows = 0
totalwidth = 3000
totalheight = 480 + ( cprows + erows + nbrrows + cpderows ) * 480
# this is the output PNG
outimg = Image . new ( 'RGBA' , ( totalwidth , totalheight ) , ( 255 , 255 , 255 , 255 ) )
# now fill in the rows of the output png . we ' ll use Pillow to build up the
# output image from the already stored plots and stuff in the checkplot
# dict .
# row 1 , cell 1 : finder chart #
if cpd [ 'finderchart' ] :
    finder = Image . open ( _base64_to_file ( cpd [ 'finderchart' ] , None , writetostrio = True ) )
    bigfinder = finder . resize ( ( 450 , 450 ) , Image . ANTIALIAS )
    outimg . paste ( bigfinder , ( 150 , 20 ) )
# row 1 , cell 2 : object information #
# find the font we need from the package data
fontpath = os . path . abspath ( os . path . join ( os . path . dirname ( __file__ ) , '..' , 'cpserver' , 'cps-assets' , 'DejaVuSans.ttf' ) )
# load the font
if os . path . exists ( fontpath ) :
    cpfontnormal = ImageFont . truetype ( fontpath , 20 )
    cpfontlarge = ImageFont . truetype ( fontpath , 28 )
else :
    LOGWARNING ( 'could not find bundled ' 'DejaVu Sans font in the astrobase package ' 'data, using ugly defaults...' )
    cpfontnormal = ImageFont . load_default ( )
    cpfontlarge = ImageFont . load_default ( )
# the image draw object
objinfodraw = ImageDraw . Draw ( outimg )
# write out the object information
# objectid
objinfodraw . text ( ( 625 , 25 ) , cpd [ 'objectid' ] if cpd [ 'objectid' ] else 'no objectid' , font = cpfontlarge , fill = ( 0 , 0 , 255 , 255 ) )
# twomass id
if 'twomassid' in cpd [ 'objectinfo' ] :
    objinfodraw . text ( ( 625 , 60 ) , ( '2MASS J%s' % cpd [ 'objectinfo' ] [ 'twomassid' ] if cpd [ 'objectinfo' ] [ 'twomassid' ] else '' ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# ndet
if 'ndet' in cpd [ 'objectinfo' ] :
    objinfodraw . text ( ( 625 , 85 ) , ( 'LC points: %s' % cpd [ 'objectinfo' ] [ 'ndet' ] if cpd [ 'objectinfo' ] [ 'ndet' ] is not None else '' ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
else :
    objinfodraw . text ( ( 625 , 85 ) , ( 'LC points: %s' % cpd [ 'magseries' ] [ 'times' ] . size ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# coords and PM
objinfodraw . text ( ( 625 , 125 ) , ( 'Coords and PM' ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
if 'ra' in cpd [ 'objectinfo' ] and 'decl' in cpd [ 'objectinfo' ] :
    objinfodraw . text ( ( 900 , 125 ) , ( ( 'RA, Dec: %.3f, %.3f' % ( cpd [ 'objectinfo' ] [ 'ra' ] , cpd [ 'objectinfo' ] [ 'decl' ] ) ) if ( cpd [ 'objectinfo' ] [ 'ra' ] is not None and cpd [ 'objectinfo' ] [ 'decl' ] is not None ) else '' ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
else :
    objinfodraw . text ( ( 900 , 125 ) , 'RA, Dec: nan, nan' , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
if 'propermotion' in cpd [ 'objectinfo' ] :
    objinfodraw . text ( ( 900 , 150 ) , ( ( 'Total PM: %.5f mas/yr' % cpd [ 'objectinfo' ] [ 'propermotion' ] ) if ( cpd [ 'objectinfo' ] [ 'propermotion' ] is not None ) else '' ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
else :
    objinfodraw . text ( ( 900 , 150 ) , 'Total PM: nan' , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
if 'rpmj' in cpd [ 'objectinfo' ] :
    objinfodraw . text ( ( 900 , 175 ) , ( ( 'Reduced PM [Jmag]: %.3f' % cpd [ 'objectinfo' ] [ 'rpmj' ] ) if ( cpd [ 'objectinfo' ] [ 'rpmj' ] is not None ) else '' ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
else :
    objinfodraw . text ( ( 900 , 175 ) , 'Reduced PM [Jmag]: nan' , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# here , we have to deal with two generations of objectinfo dicts
# first , deal with the new generation of objectinfo dicts
if 'available_dereddened_bands' in cpd [ 'objectinfo' ] : # first , we deal with the bands and mags
# magnitudes
    objinfodraw . text ( ( 625 , 200 ) , 'Magnitudes' , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
    # process the various bands
    # if dereddened mags aren ' t available , use the observed mags
    if len ( cpd [ 'objectinfo' ] [ 'available_bands' ] ) > 0 : # we ' ll get all the available mags
        for bandind , band , label in zip ( range ( len ( cpd [ 'objectinfo' ] [ 'available_bands' ] ) ) , cpd [ 'objectinfo' ] [ 'available_bands' ] , cpd [ 'objectinfo' ] [ 'available_band_labels' ] ) :
            thisbandmag = cpd [ 'objectinfo' ] [ band ]
            # we ' ll draw stuff in three rows depending on the number of
            # bands we have to use
            if bandind in ( 0 , 1 , 2 , 3 , 4 ) :
                thispos = ( 900 + 125 * bandind , 200 )
                objinfodraw . text ( thispos , '%s: %.3f' % ( label , thisbandmag ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
            elif bandind in ( 5 , 6 , 7 , 8 , 9 ) :
                rowbandind = bandind - 5
                thispos = ( 900 + 125 * rowbandind , 225 )
                objinfodraw . text ( thispos , '%s: %.3f' % ( label , thisbandmag ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
            else :
                rowbandind = bandind - 10
                thispos = ( 900 + 125 * rowbandind , 250 )
                objinfodraw . text ( thispos , '%s: %.3f' % ( label , thisbandmag ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
    # next , deal with the colors
    # colors
    if ( 'dereddened' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'dereddened' ] is True ) :
        deredlabel = "(dereddened)"
    else :
        deredlabel = ""
    objinfodraw . text ( ( 625 , 275 ) , 'Colors %s' % deredlabel , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
    if len ( cpd [ 'objectinfo' ] [ 'available_colors' ] ) > 0 : # we ' ll get all the available mags ( dereddened versions preferred )
        for colorind , color , colorlabel in zip ( range ( len ( cpd [ 'objectinfo' ] [ 'available_colors' ] ) ) , cpd [ 'objectinfo' ] [ 'available_colors' ] , cpd [ 'objectinfo' ] [ 'available_color_labels' ] ) :
            thiscolor = cpd [ 'objectinfo' ] [ color ]
            # we ' ll draw stuff in three rows depending on the number of
            # bands we have to use
            if colorind in ( 0 , 1 , 2 , 3 , 4 ) :
                thispos = ( 900 + 150 * colorind , 275 )
                objinfodraw . text ( thispos , '%s: %.3f' % ( colorlabel , thiscolor ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
            elif colorind in ( 5 , 6 , 7 , 8 , 9 ) :
                thisrowind = colorind - 5
                thispos = ( 900 + 150 * thisrowind , 300 )
                objinfodraw . text ( thispos , '%s: %.3f' % ( colorlabel , thiscolor ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
            elif colorind in ( 10 , 11 , 12 , 13 , 14 ) :
                thisrowind = colorind - 10
                thispos = ( 900 + 150 * thisrowind , 325 )
                objinfodraw . text ( thispos , '%s: %.3f' % ( colorlabel , thiscolor ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
            else :
                thisrowind = colorind - 15
                thispos = ( 900 + 150 * thisrowind , 350 )
                objinfodraw . text ( thispos , '%s: %.3f' % ( colorlabel , thiscolor ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# otherwise , deal with older generation of checkplots
else :
    objinfodraw . text ( ( 625 , 200 ) , ( 'Magnitudes' ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
    objinfodraw . text ( ( 900 , 200 ) , ( 'gri: %.3f, %.3f, %.3f' % ( ( cpd [ 'objectinfo' ] [ 'sdssg' ] if ( 'sdssg' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'sdssg' ] is not None ) else npnan ) , ( cpd [ 'objectinfo' ] [ 'sdssr' ] if ( 'sdssr' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'sdssr' ] is not None ) else npnan ) , ( cpd [ 'objectinfo' ] [ 'sdssi' ] if ( 'sdssi' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'sdssi' ] is not None ) else npnan ) ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
    objinfodraw . text ( ( 900 , 225 ) , ( 'JHK: %.3f, %.3f, %.3f' % ( ( cpd [ 'objectinfo' ] [ 'jmag' ] if ( 'jmag' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'jmag' ] is not None ) else npnan ) , ( cpd [ 'objectinfo' ] [ 'hmag' ] if ( 'hmag' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'hmag' ] is not None ) else npnan ) , ( cpd [ 'objectinfo' ] [ 'kmag' ] if ( 'kmag' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'kmag' ] is not None ) else npnan ) ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
    objinfodraw . text ( ( 900 , 250 ) , ( 'BV: %.3f, %.3f' % ( ( cpd [ 'objectinfo' ] [ 'bmag' ] if ( 'bmag' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'bmag' ] is not None ) else npnan ) , ( cpd [ 'objectinfo' ] [ 'vmag' ] if ( 'vmag' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'vmag' ] is not None ) else npnan ) ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
    # colors
    if ( 'dereddened' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'dereddened' ] is True ) :
        deredlabel = "(dereddened)"
    else :
        deredlabel = ""
    objinfodraw . text ( ( 625 , 275 ) , 'Colors %s' % deredlabel , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
    objinfodraw . text ( ( 900 , 275 ) , ( 'B - V: %.3f, V - K: %.3f' % ( ( cpd [ 'objectinfo' ] [ 'bvcolor' ] if ( 'bvcolor' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'bvcolor' ] is not None ) else npnan ) , ( cpd [ 'objectinfo' ] [ 'vkcolor' ] if ( 'vkcolor' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'vkcolor' ] is not None ) else npnan ) ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
    objinfodraw . text ( ( 900 , 300 ) , ( 'i - J: %.3f, g - K: %.3f' % ( ( cpd [ 'objectinfo' ] [ 'ijcolor' ] if ( 'ijcolor' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'ijcolor' ] is not None ) else npnan ) , ( cpd [ 'objectinfo' ] [ 'gkcolor' ] if ( 'gkcolor' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'gkcolor' ] is not None ) else npnan ) ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
    objinfodraw . text ( ( 900 , 325 ) , ( 'J - K: %.3f' % ( ( cpd [ 'objectinfo' ] [ 'jkcolor' ] if ( 'jkcolor' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'jkcolor' ] is not None ) else npnan ) , ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# rest of the object information
# color classification
if ( 'color_classes' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'color_classes' ] ) :
    objinfodraw . text ( ( 625 , 375 ) , ( 'star classification by color: %s' % ( ', ' . join ( cpd [ 'objectinfo' ] [ 'color_classes' ] ) ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# GAIA neighbors
if ( ( 'gaia_neighbors' in cpd [ 'objectinfo' ] ) and ( cpd [ 'objectinfo' ] [ 'gaia_neighbors' ] is not None ) and ( np . isfinite ( cpd [ 'objectinfo' ] [ 'gaia_neighbors' ] ) ) and ( 'searchradarcsec' in cpd [ 'objectinfo' ] ) and ( cpd [ 'objectinfo' ] [ 'searchradarcsec' ] ) ) :
    objinfodraw . text ( ( 625 , 400 ) , ( '%s GAIA close neighbors within %.1f arcsec' % ( cpd [ 'objectinfo' ] [ 'gaia_neighbors' ] , cpd [ 'objectinfo' ] [ 'searchradarcsec' ] ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# closest GAIA neighbor
if ( ( 'gaia_closest_distarcsec' in cpd [ 'objectinfo' ] ) and ( cpd [ 'objectinfo' ] [ 'gaia_closest_distarcsec' ] is not None ) and ( np . isfinite ( cpd [ 'objectinfo' ] [ 'gaia_closest_distarcsec' ] ) ) and ( 'gaia_closest_gmagdiff' in cpd [ 'objectinfo' ] ) and ( cpd [ 'objectinfo' ] [ 'gaia_closest_gmagdiff' ] is not None ) and ( np . isfinite ( cpd [ 'objectinfo' ] [ 'gaia_closest_gmagdiff' ] ) ) ) :
    objinfodraw . text ( ( 625 , 425 ) , ( 'closest GAIA neighbor is %.1f arcsec away, ' 'GAIA mag (obj-nbr): %.3f' % ( cpd [ 'objectinfo' ] [ 'gaia_closest_distarcsec' ] , cpd [ 'objectinfo' ] [ 'gaia_closest_gmagdiff' ] ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# object tags
if 'objecttags' in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ 'objecttags' ] :
    objtagsplit = cpd [ 'objectinfo' ] [ 'objecttags' ] . split ( ',' )
    # write three tags per line
    nobjtaglines = int ( np . ceil ( len ( objtagsplit ) / 3.0 ) )
    for objtagline in range ( nobjtaglines ) :
        objtagslice = ',' . join ( objtagsplit [ objtagline * 3 : objtagline * 3 + 3 ] )
        objinfodraw . text ( ( 625 , 450 + objtagline * 25 ) , objtagslice , font = cpfontnormal , fill = ( 135 , 54 , 0 , 255 ) )
# row 1 , cell 3 : variability info and comments #
# objectisvar
objisvar = cpd [ 'varinfo' ] [ 'objectisvar' ]
if objisvar == '0' :
    objvarflag = 'Variable star flag not set'
elif objisvar == '1' :
    objvarflag = 'Object is probably a variable star'
elif objisvar == '2' :
    objvarflag = 'Object is probably not a variable star'
elif objisvar == '3' :
    objvarflag = 'Not sure if this object is a variable star'
elif objisvar is None :
    objvarflag = 'Variable star flag not set'
elif objisvar is True :
    objvarflag = 'Object is probably a variable star'
elif objisvar is False :
    objvarflag = 'Object is probably not a variable star'
else :
    objvarflag = 'Variable star flag: %s' % objisvar
objinfodraw . text ( ( 1650 , 125 ) , objvarflag , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# period
objinfodraw . text ( ( 1650 , 150 ) , ( 'Period [days]: %.6f' % ( cpd [ 'varinfo' ] [ 'varperiod' ] if cpd [ 'varinfo' ] [ 'varperiod' ] is not None else np . nan ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# epoch
objinfodraw . text ( ( 1650 , 175 ) , ( 'Epoch [JD]: %.6f' % ( cpd [ 'varinfo' ] [ 'varepoch' ] if cpd [ 'varinfo' ] [ 'varepoch' ] is not None else np . nan ) ) , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# variability tags
if cpd [ 'varinfo' ] [ 'vartags' ] :
    vartagsplit = cpd [ 'varinfo' ] [ 'vartags' ] . split ( ',' )
    # write three tags per line
    nvartaglines = int ( np . ceil ( len ( vartagsplit ) / 3.0 ) )
    for vartagline in range ( nvartaglines ) :
        vartagslice = ',' . join ( vartagsplit [ vartagline * 3 : vartagline * 3 + 3 ] )
        objinfodraw . text ( ( 1650 , 225 + vartagline * 25 ) , vartagslice , font = cpfontnormal , fill = ( 135 , 54 , 0 , 255 ) )
# object comments
if 'comments' in cpd and cpd [ 'comments' ] :
    commentsplit = cpd [ 'comments' ] . split ( ' ' )
    # write 10 words per line
    ncommentlines = int ( np . ceil ( len ( commentsplit ) / 10.0 ) )
    for commentline in range ( ncommentlines ) :
        commentslice = ' ' . join ( commentsplit [ commentline * 10 : commentline * 10 + 10 ] )
        objinfodraw . text ( ( 1650 , 325 + commentline * 25 ) , commentslice , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# this handles JSON - ified checkplots returned by LCC server
elif 'objectcomments' in cpd and cpd [ 'objectcomments' ] :
    commentsplit = cpd [ 'objectcomments' ] . split ( ' ' )
    # write 10 words per line
    ncommentlines = int ( np . ceil ( len ( commentsplit ) / 10.0 ) )
    for commentline in range ( ncommentlines ) :
        commentslice = ' ' . join ( commentsplit [ commentline * 10 : commentline * 10 + 10 ] )
        objinfodraw . text ( ( 1650 , 325 + commentline * 25 ) , commentslice , font = cpfontnormal , fill = ( 0 , 0 , 0 , 255 ) )
# row 1 , cell 4 : unphased light curve #
if ( cpd [ 'magseries' ] and 'plot' in cpd [ 'magseries' ] and cpd [ 'magseries' ] [ 'plot' ] ) :
    magseries = Image . open ( _base64_to_file ( cpd [ 'magseries' ] [ 'plot' ] , None , writetostrio = True ) )
    outimg . paste ( magseries , ( 750 * 3 , 0 ) )
# this handles JSON - ified checkplots from LCC server
elif ( 'magseries' in cpd and isinstance ( cpd [ 'magseries' ] , str ) ) :
    magseries = Image . open ( _base64_to_file ( cpd [ 'magseries' ] , None , writetostrio = True ) )
    outimg . paste ( magseries , ( 750 * 3 , 0 ) )
# the rest of the rows in cpd #
for lspmethodind , lspmethod in enumerate ( cplspmethods ) : # the periodogram comes first #
    if ( cpd [ lspmethod ] and cpd [ lspmethod ] [ 'periodogram' ] ) :
        pgram = Image . open ( _base64_to_file ( cpd [ lspmethod ] [ 'periodogram' ] , None , writetostrio = True ) )
        outimg . paste ( pgram , ( 0 , 480 + 480 * lspmethodind ) )
    # best phased LC comes next #
    if ( cpd [ lspmethod ] and 0 in cpd [ lspmethod ] and cpd [ lspmethod ] [ 0 ] ) :
        plc1 = Image . open ( _base64_to_file ( cpd [ lspmethod ] [ 0 ] [ 'plot' ] , None , writetostrio = True ) )
        outimg . paste ( plc1 , ( 750 , 480 + 480 * lspmethodind ) )
    # this handles JSON - ified checkplots from LCC server
    elif ( cpd [ lspmethod ] and 'phasedlc0' in cpd [ lspmethod ] and isinstance ( cpd [ lspmethod ] [ 'phasedlc0' ] [ 'plot' ] , str ) ) :
        plc1 = Image . open ( _base64_to_file ( cpd [ lspmethod ] [ 'phasedlc0' ] [ 'plot' ] , None , writetostrio = True ) )
        outimg . paste ( plc1 , ( 750 , 480 + 480 * lspmethodind ) )
    # 2nd best phased LC comes next #
    if ( cpd [ lspmethod ] and 1 in cpd [ lspmethod ] and cpd [ lspmethod ] [ 1 ] ) :
        plc2 = Image . open ( _base64_to_file ( cpd [ lspmethod ] [ 1 ] [ 'plot' ] , None , writetostrio = True ) )
        outimg . paste ( plc2 , ( 750 * 2 , 480 + 480 * lspmethodind ) )
    # this handles JSON - ified checkplots from LCC server
    elif ( cpd [ lspmethod ] and 'phasedlc1' in cpd [ lspmethod ] and isinstance ( cpd [ lspmethod ] [ 'phasedlc1' ] [ 'plot' ] , str ) ) :
        plc2 = Image . open ( _base64_to_file ( cpd [ lspmethod ] [ 'phasedlc1' ] [ 'plot' ] , None , writetostrio = True ) )
        outimg . paste ( plc2 , ( 750 * 2 , 480 + 480 * lspmethodind ) )
    # 3rd best phased LC comes next #
    if ( cpd [ lspmethod ] and 2 in cpd [ lspmethod ] and cpd [ lspmethod ] [ 2 ] ) :
        plc3 = Image . open ( _base64_to_file ( cpd [ lspmethod ] [ 2 ] [ 'plot' ] , None , writetostrio = True ) )
        outimg . paste ( plc3 , ( 750 * 3 , 480 + 480 * lspmethodind ) )
    # this handles JSON - ified checkplots from LCC server
    elif ( cpd [ lspmethod ] and 'phasedlc2' in cpd [ lspmethod ] and isinstance ( cpd [ lspmethod ] [ 'phasedlc2' ] [ 'plot' ] , str ) ) :
        plc3 = Image . open ( _base64_to_file ( cpd [ lspmethod ] [ 'phasedlc2' ] [ 'plot' ] , None , writetostrio = True ) )
        outimg . paste ( plc3 , ( 750 * 3 , 480 + 480 * lspmethodind ) )
# # ALL DONE WITH BUILDING PNG # #
# add in any extra rows #
# from the keyword arguments
if erows > 0 :
    for erowind , erow in enumerate ( extrarows ) : # make sure we never go above 4 plots in a row
        for ecolind , ecol in enumerate ( erow [ : 4 ] ) :
            eplot = Image . open ( ecol )
            eplotresized = eplot . resize ( ( 750 , 480 ) , Image . ANTIALIAS )
            outimg . paste ( eplotresized , ( 750 * ecolind , ( cprows + 1 ) * 480 + 480 * erowind ) )
# from the checkplotdict
if cpderows > 0 :
    for cpderowind , cpderow in enumerate ( cpd [ 'externalplots' ] ) : # make sure we never go above 4 plots in a row
        for cpdecolind , cpdecol in enumerate ( cpderow [ : 4 ] ) :
            cpdeplot = Image . open ( cpdecol )
            cpdeplotresized = cpdeplot . resize ( ( 750 , 480 ) , Image . ANTIALIAS )
            outimg . paste ( cpdeplotresized , ( 750 * cpdecolind , ( cprows + 1 ) * 480 + ( erows * 480 ) + 480 * cpderowind ) )
# from neighbors :
if nbrrows > 0 : # we have four tiles
# tile 1 : neighbor objectid , ra , decl , distance , unphased LC
# tile 2 : phased LC for gls
# tile 3 : phased LC for pdm
# tile 4 : phased LC for any other period finding method
# the priority is like so : [ ' bls ' , ' mav ' , ' aov ' , ' win ' ]
    for nbrind , nbr in enumerate ( cpd [ 'neighbors' ] ) : # figure out which period finding methods are available for this
    # neighbor . make sure to match the ones from the actual object in
    # order of priority : ' gls ' , ' pdm ' , ' bls ' , ' aov ' , ' mav ' , ' acf ' , ' win '
        nbrlspmethods = [ ]
        for lspmethod in cpd [ 'pfmethods' ] :
            if lspmethod in nbr :
                nbrlspmethods . append ( lspmethod )
        # restrict to top three in priority
        nbrlspmethods = nbrlspmethods [ : 3 ]
        try : # first panel : neighbor objectid , ra , decl , distance , unphased
        # LC
            nbrlc = Image . open ( _base64_to_file ( nbr [ 'magseries' ] [ 'plot' ] , None , writetostrio = True ) )
            outimg . paste ( nbrlc , ( 750 * 0 , ( cprows + 1 ) * 480 + ( erows * 480 ) + ( cpderows * 480 ) + 480 * nbrind ) )
            # overlay the objectinfo
            objinfodraw . text ( ( 98 , ( cprows + 1 ) * 480 + ( erows * 480 ) + ( cpderows * 480 ) + 480 * nbrind + 15 ) , ( 'N%s: %s' % ( nbrind + 1 , nbr [ 'objectid' ] ) ) , font = cpfontlarge , fill = ( 0 , 0 , 255 , 255 ) )
            # overlay the objectinfo
            objinfodraw . text ( ( 98 , ( cprows + 1 ) * 480 + ( erows * 480 ) + ( cpderows * 480 ) + 480 * nbrind + 50 ) , ( '(RA, DEC) = (%.3f, %.3f), distance: %.1f arcsec' % ( nbr [ 'ra' ] , nbr [ 'decl' ] , nbr [ 'dist' ] ) ) , font = cpfontnormal , fill = ( 0 , 0 , 255 , 255 ) )
            # second panel : phased LC for gls
            lsp1lc = Image . open ( _base64_to_file ( nbr [ nbrlspmethods [ 0 ] ] [ 0 ] [ 'plot' ] , None , writetostrio = True ) )
            outimg . paste ( lsp1lc , ( 750 * 1 , ( cprows + 1 ) * 480 + ( erows * 480 ) + ( cpderows * 480 ) + 480 * nbrind ) )
            # second panel : phased LC for gls
            lsp2lc = Image . open ( _base64_to_file ( nbr [ nbrlspmethods [ 1 ] ] [ 0 ] [ 'plot' ] , None , writetostrio = True ) )
            outimg . paste ( lsp2lc , ( 750 * 2 , ( cprows + 1 ) * 480 + ( erows * 480 ) + ( cpderows * 480 ) + 480 * nbrind ) )
            # second panel : phased LC for gls
            lsp3lc = Image . open ( _base64_to_file ( nbr [ nbrlspmethods [ 2 ] ] [ 0 ] [ 'plot' ] , None , writetostrio = True ) )
            outimg . paste ( lsp3lc , ( 750 * 3 , ( cprows + 1 ) * 480 + ( erows * 480 ) + ( cpderows * 480 ) + 480 * nbrind ) )
        except Exception as e :
            LOGERROR ( 'neighbor %s does not have a magseries plot, ' 'measurements are probably all nan' % nbr [ 'objectid' ] )
            # overlay the objectinfo
            objinfodraw . text ( ( 98 , ( cprows + 1 ) * 480 + ( erows * 480 ) + ( cpderows * 480 ) + 480 * nbrind + 15 ) , ( 'N%s: %s' % ( nbrind + 1 , nbr [ 'objectid' ] ) ) , font = cpfontlarge , fill = ( 0 , 0 , 255 , 255 ) )
            if 'ra' in nbr and 'decl' in nbr and 'dist' in nbr : # overlay the objectinfo
                objinfodraw . text ( ( 98 , ( cprows + 1 ) * 480 + ( erows * 480 ) + ( cpderows * 480 ) + 480 * nbrind + 50 ) , ( '(RA, DEC) = (%.3f, %.3f), distance: %.1f arcsec' % ( nbr [ 'ra' ] , nbr [ 'decl' ] , nbr [ 'dist' ] ) ) , font = cpfontnormal , fill = ( 0 , 0 , 255 , 255 ) )
            elif 'objectinfo' in nbr : # overlay the objectinfo
                objinfodraw . text ( ( 98 , ( cprows + 1 ) * 480 + ( erows * 480 ) + ( cpderows * 480 ) + 480 * nbrind + 50 ) , ( '(RA, DEC) = (%.3f, %.3f), distance: %.1f arcsec' % ( nbr [ 'objectinfo' ] [ 'ra' ] , nbr [ 'objectinfo' ] [ 'decl' ] , nbr [ 'objectinfo' ] [ 'distarcsec' ] ) ) , font = cpfontnormal , fill = ( 0 , 0 , 255 , 255 ) )
# # WRITE FINAL PNG # #
# check if the output filename is actually an instance of StringIO
if sys . version_info [ : 2 ] < ( 3 , 0 ) :
    is_strio = isinstance ( outfile , cStringIO . InputType )
else :
    is_strio = isinstance ( outfile , StrIO )
if not is_strio : # check if we ' ve stupidly copied over the same filename as the input
# pickle to expected output file
    if outfile . endswith ( 'pkl' ) :
        LOGWARNING ( 'expected output PNG filename ends with .pkl, ' 'changed to .png' )
        outfile = outfile . replace ( '.pkl' , '.png' )
outimg . save ( outfile , format = 'PNG' , optimize = True )
if not is_strio :
    if os . path . exists ( outfile ) :
        LOGINFO ( 'checkplot pickle -> checkplot PNG: %s OK' % outfile )
        return outfile
    else :
        LOGERROR ( 'failed to write checkplot PNG' )
        return None
else :
    LOGINFO ( 'checkplot pickle -> StringIO instance OK' )
    return outfile


signature:
def cp2png ( checkplotin , extrarows = None ) :
    '''This is just a shortened form of the function above for convenience .
 This only handles pickle files as input .
 Parameters
 checkplotin : str
 File name of a checkplot pickle file to convert to a PNG .
 extrarows : list of tuples
 This is a list of 4 - element tuples containing paths to PNG files that
 will be added to the end of the rows generated from the checkplotin
 pickle / dict . Each tuple represents a row in the final output PNG
 file . If there are less than 4 elements per tuple , the missing elements
 will be filled in with white - space . If there are more than 4 elements
 per tuple , only the first four will be used .
 The purpose of this kwarg is to incorporate periodograms and phased LC
 plots ( in the form of PNGs ) generated from an external period - finding
 function or program ( like VARTOOLS ) to allow for comparison with
 astrobase results .
 NOTE : the PNG files specified in ` extrarows ` here will be added to those
 already present in the input ` checkplotdict [ ' externalplots ' ] ` if that is
 None because you passed in a similar list of external plots to the
 : py : func : ` astrobase . checkplot . pkl . checkplot _ pickle ` function earlier . In
 this case , ` extrarows ` can be used to add even more external plots if
 desired .
 Each external plot PNG will be resized to 750 x 480 pixels to fit into
 an output image cell .
 By convention , each 4 - element tuple should contain :
 - a periodiogram PNG
 - phased LC PNG with 1st best peak period from periodogram
 - phased LC PNG with 2nd best peak period from periodogram
 - phased LC PNG with 3rd best peak period from periodogram
 Example of extrarows : :
 [ ( ' / path / to / external / bls - periodogram . png ' ,
 ' / path / to / external / bls - phasedlc - plot - bestpeak . png ' ,
 ' / path / to / external / bls - phasedlc - plot - peak2 . png ' ,
 ' / path / to / external / bls - phasedlc - plot - peak3 . png ' ) ,
 ( ' / path / to / external / pdm - periodogram . png ' ,
 ' / path / to / external / pdm - phasedlc - plot - bestpeak . png ' ,
 ' / path / to / external / pdm - phasedlc - plot - peak2 . png ' ,
 ' / path / to / external / pdm - phasedlc - plot - peak3 . png ' ) ,
 Returns
 str
 The absolute path to the generated checkplot PNG .'''

implementation:
if checkplotin . endswith ( '.gz' ) :
    outfile = checkplotin . replace ( '.pkl.gz' , '.png' )
else :
    outfile = checkplotin . replace ( '.pkl' , '.png' )
return checkplot_pickle_to_png ( checkplotin , outfile , extrarows = extrarows )


signature:
def flare_model ( flareparams , times , mags , errs ) :
    '''This is a flare model function , similar to Kowalski + 2011.
 From the paper by Pitkin + 2014:
 http : / / adsabs . harvard . edu / abs / 2014MNRAS . 445.2268P
 Parameters
 flareparams : list of float
 This defines the flare model : :
 [ amplitude ,
 flare _ peak _ time ,
 rise _ gaussian _ stdev ,
 decay _ time _ constant ]
 where :
 ` amplitude ` : the maximum flare amplitude in mags or flux . If flux , then
 amplitude should be positive . If mags , amplitude should be negative .
 ` flare _ peak _ time ` : time at which the flare maximum happens .
 ` rise _ gaussian _ stdev ` : the stdev of the gaussian describing the rise of
 the flare .
 ` decay _ time _ constant ` : the time constant of the exponential fall of the
 flare .
 times , mags , errs : np . array
 The input time - series of measurements and associated errors for which
 the model will be generated . The times will be used to generate
 model mags .
 Returns
 ( modelmags , times , mags , errs ) : tuple
 Returns the model mags evaluated at the input time values . Also returns
 the input ` times ` , ` mags ` , and ` errs ` .'''

implementation:
( amplitude , flare_peak_time , rise_gaussian_stdev , decay_time_constant ) = flareparams
zerolevel = np . median ( mags )
modelmags = np . full_like ( times , zerolevel )
# before peak gaussian rise . . .
modelmags [ times < flare_peak_time ] = ( mags [ times < flare_peak_time ] + amplitude * np . exp ( - ( ( times [ times < flare_peak_time ] - flare_peak_time ) * ( times [ times < flare_peak_time ] - flare_peak_time ) ) / ( 2.0 * rise_gaussian_stdev * rise_gaussian_stdev ) ) )
# after peak exponential decay . . .
modelmags [ times > flare_peak_time ] = ( mags [ times > flare_peak_time ] + amplitude * np . exp ( - ( ( times [ times > flare_peak_time ] - flare_peak_time ) ) / ( decay_time_constant ) ) )
return modelmags , times , mags , errs


signature:
def flare_model_residual ( flareparams , times , mags , errs ) :
    '''This returns the residual between model mags and the actual mags .
 Parameters
 flareparams : list of float
 This defines the flare model : :
 [ amplitude ,
 flare _ peak _ time ,
 rise _ gaussian _ stdev ,
 decay _ time _ constant ]
 where :
 ` amplitude ` : the maximum flare amplitude in mags or flux . If flux , then
 amplitude should be positive . If mags , amplitude should be negative .
 ` flare _ peak _ time ` : time at which the flare maximum happens .
 ` rise _ gaussian _ stdev ` : the stdev of the gaussian describing the rise of
 the flare .
 ` decay _ time _ constant ` : the time constant of the exponential fall of the
 flare .
 times , mags , errs : np . array
 The input time - series of measurements and associated errors for which
 the model will be generated . The times will be used to generate
 model mags .
 Returns
 np . array
 The residuals between the input ` mags ` and generated ` modelmags ` ,
 weighted by the measurement errors in ` errs ` .'''

implementation:
modelmags , _ , _ , _ = flare_model ( flareparams , times , mags , errs )
return ( mags - modelmags ) / errs


signature:
def cache_clean_handler ( min_age_hours = 1 ) :
    """This periodically cleans up the ~ / . astrobase cache to save us from
 disk - space doom .
 Parameters
 min _ age _ hours : int
 Files older than this number of hours from the current time will be
 deleted .
 Returns
 Nothing ."""

implementation:
# find the files to delete
cmd = ( "find ~ec2-user/.astrobase -type f -mmin +{mmin} -exec rm -v '{{}}' \;" )
mmin = '%.1f' % ( min_age_hours * 60.0 )
cmd = cmd . format ( mmin = mmin )
try :
    proc = subprocess . run ( cmd , shell = True , stdout = subprocess . PIPE )
    ndeleted = len ( proc . stdout . decode ( ) . split ( '\n' ) )
    LOGWARNING ( 'cache clean: %s files older than %s hours deleted' % ( ndeleted , min_age_hours ) )
except Exception as e :
    LOGEXCEPTION ( 'cache clean: could not delete old files' )


signature:
def shutdown_check_handler ( ) :
    """This checks the AWS instance data URL to see if there ' s a pending
 shutdown for the instance .
 This is useful for AWS spot instances . If there is a pending shutdown posted
 to the instance data URL , we ' ll use the result of this function break out of
 the processing loop and shut everything down ASAP before the instance dies .
 Returns
 bool
 - True if the instance is going to die soon .
 - False if the instance is still safe ."""

implementation:
url = 'http://169.254.169.254/latest/meta-data/spot/instance-action'
try :
    resp = requests . get ( url , timeout = 1.0 )
    resp . raise_for_status ( )
    stopinfo = resp . json ( )
    if 'action' in stopinfo and stopinfo [ 'action' ] in ( 'stop' , 'terminate' , 'hibernate' ) :
        stoptime = stopinfo [ 'time' ]
        LOGWARNING ( 'instance is going to %s at %s' % ( stopinfo [ 'action' ] , stoptime ) )
        resp . close ( )
        return True
    else :
        resp . close ( )
        return False
except HTTPError as e :
    resp . close ( )
    return False
except Exception as e :
    resp . close ( )
    return False


signature:
def runcp_producer_loop ( lightcurve_list , input_queue , input_bucket , result_queue , result_bucket , pfresult_list = None , runcp_kwargs = None , process_list_slice = None , purge_queues_when_done = False , delete_queues_when_done = False , download_when_done = True , save_state_when_done = True , s3_client = None , sqs_client = None ) :
    """This sends checkplot making tasks to the input queue and monitors the
 result queue for task completion .
 Parameters
 lightcurve _ list : str or list of str
 This is either a string pointing to a file containing a list of light
 curves filenames to process or the list itself . The names must
 correspond to the full filenames of files stored on S3 , including all
 prefixes , but not include the ' s3 : / / < bucket name > / ' bit ( these will be
 added automatically ) .
 input _ queue : str
 This is the name of the SQS queue which will receive processing tasks
 generated by this function . The queue URL will automatically be obtained
 from AWS .
 input _ bucket : str
 The name of the S3 bucket containing the light curve files to process .
 result _ queue : str
 This is the name of the SQS queue that this function will listen to for
 messages from the workers as they complete processing on their input
 elements . This function will attempt to match input sent to the
 ` input _ queue ` with results coming into the ` result _ queue ` so it knows
 how many objects have been successfully processed . If this function
 receives task results that aren ' t in its own input queue , it will
 acknowledge them so they complete successfully , but not download them
 automatically . This handles leftover tasks completing from a previous
 run of this function .
 result _ bucket : str
 The name of the S3 bucket which will receive the results from the
 workers .
 pfresult _ list : list of str or None
 This is a list of periodfinder result pickle S3 URLs associated with
 each light curve . If provided , this will be used to add in phased light
 curve plots to each checkplot pickle . If this is None , the worker loop
 will produce checkplot pickles that only contain object information ,
 neighbor information , and unphased light curves .
 runcp _ kwargs : dict
 This is a dict used to pass any extra keyword arguments to the
 ` lcproc . checkplotgen . runcp ` function that will be run by the worker
 loop .
 process _ list _ slice : list
 This is used to index into the input light curve list so a subset of the
 full list can be processed in this specific run of this function .
 Use None for a slice index elem to emulate single slice spec behavior :
 process _ list _ slice = [ 10 , None ] - > lightcurve _ list [ 10 : ]
 process _ list _ slice = [ None , 500 ] - > lightcurve _ list [ : 500]
 purge _ queues _ when _ done : bool
 If this is True , and this function exits ( either when all done , or when
 it is interrupted with a Ctrl + C ) , all outstanding elements in the
 input / output queues that have not yet been acknowledged by workers or by
 this function will be purged . This effectively cancels all outstanding
 work .
 delete _ queues _ when _ done : bool
 If this is True , and this function exits ( either when all done , or when
 it is interrupted with a Ctrl + C ' ) , all outstanding work items will be
 purged from the input / queues and the queues themselves will be deleted .
 download _ when _ done : bool
 If this is True , the generated checkplot pickle for each input work item
 will be downloaded immediately to the current working directory when the
 worker functions report they ' re done with it .
 save _ state _ when _ done : bool
 If this is True , will save the current state of the work item queue and
 the work items acknowledged as completed to a pickle in the current
 working directory . Call the ` runcp _ producer _ loop _ savedstate ` function
 below to resume processing from this saved state later .
 s3 _ client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its S3 download operations . Alternatively , pass in an existing
 ` boto3 . Client ` instance to re - use it here .
 sqs _ client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its SQS operations . Alternatively , pass in an existing
 ` boto3 . Client ` instance to re - use it here .
 Returns
 dict or str
 Returns the current work state as a dict or str path to the generated
 work state pickle depending on if ` save _ state _ when _ done ` is True ."""

implementation:
if not sqs_client :
    sqs_client = boto3 . client ( 'sqs' )
if not s3_client :
    s3_client = boto3 . client ( 's3' )
if isinstance ( lightcurve_list , str ) and os . path . exists ( lightcurve_list ) : # get the LC list
    with open ( lightcurve_list , 'r' ) as infd :
        lclist = infd . readlines ( )
    lclist = [ x . replace ( '\n' , '' ) for x in lclist if len ( x ) > 0 ]
    if process_list_slice is not None :
        lclist = lclist [ process_list_slice [ 0 ] : process_list_slice [ 1 ] ]
    lclist = [ x [ 1 : ] for x in lclist if x . startswith ( '/' ) ]
    lclist = [ 's3://%s/%s' % ( input_bucket , x ) for x in lclist ]
# this handles direct invocation using lists of s3 : / / urls of light curves
elif isinstance ( lightcurve_list , list ) :
    lclist = lightcurve_list
# set up the input and output queues
# check if the queues by the input and output names given exist already
# if they do , go ahead and use them
# if they don ' t , make new ones .
try :
    inq = sqs_client . get_queue_url ( QueueName = input_queue )
    inq_url = inq [ 'QueueUrl' ]
    LOGINFO ( 'input queue already exists, skipping creation...' )
except ClientError as e :
    inq = awsutils . sqs_create_queue ( input_queue , client = sqs_client )
    inq_url = inq [ 'url' ]
try :
    outq = sqs_client . get_queue_url ( QueueName = result_queue )
    outq_url = outq [ 'QueueUrl' ]
    LOGINFO ( 'result queue already exists, skipping creation...' )
except ClientError as e :
    outq = awsutils . sqs_create_queue ( result_queue , client = sqs_client )
    outq_url = outq [ 'url' ]
LOGINFO ( 'input queue: %s' % inq_url )
LOGINFO ( 'output queue: %s' % outq_url )
# wait until queues are up
LOGINFO ( 'waiting for queues to become ready...' )
time . sleep ( 10.0 )
# for each item in the lightcurve _ list , send it to the input queue and wait
# until it ' s done to send another one
if pfresult_list is None :
    pfresult_list = [ None for x in lclist ]
for lc , pf in zip ( lclist , pfresult_list ) :
    this_item = { 'target' : lc , 'action' : 'runcp' , 'args' : ( pf , ) , 'kwargs' : runcp_kwargs if runcp_kwargs is not None else { } , 'outbucket' : result_bucket , 'outqueue' : outq_url }
    resp = awsutils . sqs_put_item ( inq_url , this_item , client = sqs_client )
    if resp :
        LOGINFO ( 'sent %s to queue: %s' % ( lc , inq_url ) )
# now block until all objects are done
done_objects = { }
LOGINFO ( 'all items queued, waiting for results...' )
# listen to the kill and term signals and raise KeyboardInterrupt when
# called
signal . signal ( signal . SIGINT , kill_handler )
signal . signal ( signal . SIGTERM , kill_handler )
while len ( list ( done_objects . keys ( ) ) ) < len ( lclist ) :
    try :
        result = awsutils . sqs_get_item ( outq_url , client = sqs_client )
        if result is not None and len ( result ) > 0 :
            recv = result [ 0 ]
            try :
                processed_object = recv [ 'item' ] [ 'target' ]
            except KeyError :
                LOGWARNING ( 'unknown target in received item: %s' % recv )
                processed_object = 'unknown-lc'
            cpf = recv [ 'item' ] [ 'cpf' ]
            receipt = recv [ 'receipt_handle' ]
            if processed_object in lclist :
                if processed_object not in done_objects :
                    done_objects [ processed_object ] = [ cpf ]
                else :
                    done_objects [ processed_object ] . append ( cpf )
                LOGINFO ( 'done with %s -> %s' % ( processed_object , cpf ) )
                if download_when_done :
                    getobj = awsutils . awsutils . s3_get_url ( cpf , client = s3_client )
                    LOGINFO ( 'downloaded %s -> %s' % ( cpf , getobj ) )
            else :
                LOGWARNING ( 'processed object returned is not in ' 'queued target list, probably from an ' 'earlier run. accepting but not downloading.' )
            awsutils . sqs_delete_item ( outq_url , receipt )
    except KeyboardInterrupt as e :
        LOGWARNING ( 'breaking out of producer wait-loop' )
        break
# delete the input and output queues when we ' re done
LOGINFO ( 'done with processing.' )
time . sleep ( 1.0 )
if purge_queues_when_done :
    LOGWARNING ( 'purging queues at exit, please wait 10 seconds...' )
    sqs_client . purge_queue ( QueueUrl = inq_url )
    sqs_client . purge_queue ( QueueUrl = outq_url )
    time . sleep ( 10.0 )
if delete_queues_when_done :
    LOGWARNING ( 'deleting queues at exit' )
    awsutils . sqs_delete_queue ( inq_url )
    awsutils . sqs_delete_queue ( outq_url )
work_state = { 'done' : done_objects , 'in_progress' : list ( set ( lclist ) - set ( done_objects . keys ( ) ) ) , 'args' : ( ( os . path . abspath ( lightcurve_list ) if isinstance ( lightcurve_list , str ) else lightcurve_list ) , input_queue , input_bucket , result_queue , result_bucket ) , 'kwargs' : { 'pfresult_list' : pfresult_list , 'runcp_kwargs' : runcp_kwargs , 'process_list_slice' : process_list_slice , 'download_when_done' : download_when_done , 'purge_queues_when_done' : purge_queues_when_done , 'save_state_when_done' : save_state_when_done , 'delete_queues_when_done' : delete_queues_when_done } }
if save_state_when_done :
    with open ( 'runcp-queue-producer-loop-state.pkl' , 'wb' ) as outfd :
        pickle . dump ( work_state , outfd , pickle . HIGHEST_PROTOCOL )
# at the end , return the done _ objects dict
# also return the list of unprocessed items if any
return work_state