signature:
def aovhm_theta ( times , mags , errs , frequency , nharmonics , magvariance ) :
    '''This calculates the harmonic AoV theta statistic for a frequency .
 This is a mostly faithful translation of the inner loop in ` aovper . f90 ` . See
 the following for details :
 - http : / / users . camk . edu . pl / alex /
 - Schwarzenberg - Czerny ( ` 1996
 < http : / / iopscience . iop . org / article / 10.1086/309985 / meta > ` _ )
 Schwarzenberg - Czerny ( 1996 ) equation 11 : :
 theta _ prefactor = ( K - 2N - 1 ) / ( 2N )
 theta _ top = sum ( c _ n * c _ n ) ( from n = 0 to n = 2N )
 theta _ bot = variance ( timeseries ) - sum ( c _ n * c _ n ) ( from n = 0 to n = 2N )
 theta = theta _ prefactor * ( theta _ top / theta _ bot )
 N = number of harmonics ( nharmonics )
 K = length of time series ( times . size )
 Parameters
 times , mags , errs : np . array
 The input time - series to calculate the test statistic for . These should
 all be of nans / infs and be normalized to zero .
 frequency : float
 The test frequency to calculate the statistic for .
 nharmonics : int
 The number of harmonics to calculate up to . The recommended range is 4 to
 magvariance : float
 This is the ( weighted by errors ) variance of the magnitude time
 series . We provide it as a pre - calculated value here so we don ' t have to
 re - calculate it for every worker .
 Returns
 aov _ harmonic _ theta : float
 THe value of the harmonic AoV theta for the specified test ` frequency ` .'''

implementation:
period = 1.0 / frequency
ndet = times . size
two_nharmonics = nharmonics + nharmonics
# phase with test period
phasedseries = phase_magseries_with_errs ( times , mags , errs , period , times [ 0 ] , sort = True , wrap = False )
# get the phased quantities
phase = phasedseries [ 'phase' ]
pmags = phasedseries [ 'mags' ]
perrs = phasedseries [ 'errs' ]
# this is sqrt ( 1.0 / errs ^ 2 ) - > the weights
pweights = 1.0 / perrs
# multiply by 2.0 * PI ( for omega * time )
phase = phase * 2.0 * pi_value
# this is the z complex vector
z = npcos ( phase ) + 1.0j * npsin ( phase )
# multiply phase with N
phase = nharmonics * phase
# this is the psi complex vector
psi = pmags * pweights * ( npcos ( phase ) + 1j * npsin ( phase ) )
# this is the initial value of z ^ n
zn = 1.0 + 0.0j
# this is the initial value of phi
phi = pweights + 0.0j
# initialize theta to zero
theta_aov = 0.0
# go through all the harmonics now up to 2N
for _ in range ( two_nharmonics ) : # this is < phi , phi >
    phi_dot_phi = npsum ( phi * phi . conjugate ( ) )
    # this is the alpha _ n numerator
    alpha = npsum ( pweights * z * phi )
    # this is < phi , psi > . make sure to use npvdot and NOT npdot to get
    # complex conjugate of first vector as expected for complex vectors
    phi_dot_psi = npvdot ( phi , psi )
    # make sure phi _ dot _ phi is not zero
    phi_dot_phi = npmax ( [ phi_dot_phi , 10.0e-9 ] )
    # this is the expression for alpha _ n
    alpha = alpha / phi_dot_phi
    # update theta _ aov for this harmonic
    theta_aov = ( theta_aov + npabs ( phi_dot_psi ) * npabs ( phi_dot_psi ) / phi_dot_phi )
    # use the recurrence relation to find the next phi
    phi = phi * z - alpha * zn * phi . conjugate ( )
    # update z ^ n
    zn = zn * z
# done with all harmonics , calculate the theta _ aov for this freq
# the max below makes sure that magvariance - theta _ aov > zero
theta_aov = ( ( ndet - two_nharmonics - 1.0 ) * theta_aov / ( two_nharmonics * npmax ( [ magvariance - theta_aov , 1.0e-9 ] ) ) )
return theta_aov


signature:
def _aovhm_theta_worker ( task ) :
    '''This is a parallel worker for the function below .
 Parameters
 tasks : tuple
 This is of the form below : :
 task [ 0 ] = times
 task [ 1 ] = mags
 task [ 2 ] = errs
 task [ 3 ] = frequency
 task [ 4 ] = nharmonics
 task [ 5 ] = magvariance
 Returns
 harmonic _ aov _ theta : float
 The value of the harmonic AoV statistic for the test frequency used .
 If something goes wrong with the calculation , nan is returned .'''

implementation:
times , mags , errs , frequency , nharmonics , magvariance = task
try :
    theta = aovhm_theta ( times , mags , errs , frequency , nharmonics , magvariance )
    return theta
except Exception as e :
    return npnan


signature:
def open ( self , database , user , password , host ) :
    '''This opens a new database connection .
 Parameters
 database : str
 Name of the database to connect to .
 user : str
 User name of the database server user .
 password : str
 Password for the database server user .
 host : str
 Database hostname or IP address to connect to .'''

implementation:
try :
    self . connection = pg . connect ( user = user , password = password , database = database , host = host )
    LOGINFO ( 'postgres connection successfully ' 'created, using DB %s, user %s' % ( database , user ) )
    self . database = database
    self . user = user
except Exception as e :
    LOGEXCEPTION ( 'postgres connection failed, ' 'using DB %s, user %s' % ( database , user ) )
    self . database = None
    self . user = None


signature:
def open_default ( self ) :
    '''This opens the database connection using the default database parameters
 given in the ~ / . astrobase / astrobase . conf file .'''

implementation:
if HAVECONF :
    self . open ( DBDATA , DBUSER , DBPASS , DBHOST )
else :
    LOGERROR ( "no default DB connection config found in lcdb.conf, " "this function won't work otherwise" )


signature:
def autocommit ( self ) :
    '''This sets the database connection to autocommit . Must be called before
 any cursors have been instantiated .'''

implementation:
if len ( self . cursors . keys ( ) ) == 0 :
    self . connection . autocommit = True
else :
    raise AttributeError ( 'database cursors are already active, ' 'cannot switch to autocommit now' )


signature:
def cursor ( self , handle , dictcursor = False ) :
    '''This gets or creates a DB cursor for the current DB connection .
 Parameters
 handle : str
 The name of the cursor to look up in the existing list or if it
 doesn ' t exist , the name to be used for a new cursor to be returned .
 dictcursor : bool
 If True , returns a cursor where each returned row can be addressed
 as a dictionary by column name .
 Returns
 psycopg2 . Cursor instance'''

implementation:
if handle in self . cursors :
    return self . cursors [ handle ]
else :
    if dictcursor :
        self . cursors [ handle ] = self . connection . cursor ( cursor_factory = psycopg2 . extras . DictCursor )
    else :
        self . cursors [ handle ] = self . connection . cursor ( )
    return self . cursors [ handle ]


signature:
def newcursor ( self , dictcursor = False ) :
    '''This creates a DB cursor for the current DB connection using a
 randomly generated handle . Returns a tuple with cursor and handle .
 Parameters
 dictcursor : bool
 If True , returns a cursor where each returned row can be addressed
 as a dictionary by column name .
 Returns
 tuple
 The tuple is of the form ( handle , psycopg2 . Cursor instance ) .'''

implementation:
handle = hashlib . sha256 ( os . urandom ( 12 ) ) . hexdigest ( )
if dictcursor :
    self . cursors [ handle ] = self . connection . cursor ( cursor_factory = psycopg2 . extras . DictCursor )
else :
    self . cursors [ handle ] = self . connection . cursor ( )
    return ( self . cursors [ handle ] , handle )


signature:
def commit ( self ) :
    '''This just calls the connection ' s commit method .'''

implementation:
if not self . connection . closed :
    self . connection . commit ( )
else :
    raise AttributeError ( 'postgres connection to %s is closed' % self . database )


signature:
def rollback ( self ) :
    '''This just calls the connection ' s commit method .'''

implementation:
if not self . connection . closed :
    self . connection . rollback ( )
else :
    raise AttributeError ( 'postgres connection to %s is closed' % self . database )


signature:
def close_cursor ( self , handle ) :
    '''Closes the cursor specified and removes it from the ` self . cursors `
 dictionary .'''

implementation:
if handle in self . cursors :
    self . cursors [ handle ] . close ( )
else :
    raise KeyError ( 'cursor with handle %s was not found' % handle )


signature:
def trapezoid_transit_func ( transitparams , times , mags , errs , get_ntransitpoints = False ) :
    '''This returns a trapezoid transit - shaped function .
 Suitable for first order modeling of transit signals .
 Parameters
 transitparams : list of float
 This contains the transiting planet trapezoid model : :
 transitparams = [ transitperiod ( time ) ,
 transitepoch ( time ) ,
 transitdepth ( flux or mags ) ,
 transitduration ( phase ) ,
 ingressduration ( phase ) ]
 All of these will then have fitted values after the fit is done .
 - for magnitudes - > ` transitdepth ` should be < 0
 - for fluxes - > ` transitdepth ` should be > 0
 times , mags , errs : np . array
 The input time - series of measurements and associated errors for which
 the transit model will be generated . The times will be used to generate
 model mags , and the input ` times ` , ` mags ` , and ` errs ` will be resorted
 by model phase and returned .
 Returns
 ( modelmags , phase , ptimes , pmags , perrs ) : tuple
 Returns the model mags and phase values . Also returns the input ` times ` ,
 ` mags ` , and ` errs ` sorted by the model ' s phase .'''

implementation:
( transitperiod , transitepoch , transitdepth , transitduration , ingressduration ) = transitparams
# generate the phases
iphase = ( times - transitepoch ) / transitperiod
iphase = iphase - np . floor ( iphase )
phasesortind = np . argsort ( iphase )
phase = iphase [ phasesortind ]
ptimes = times [ phasesortind ]
pmags = mags [ phasesortind ]
perrs = errs [ phasesortind ]
zerolevel = np . median ( pmags )
modelmags = np . full_like ( phase , zerolevel )
halftransitduration = transitduration / 2.0
bottomlevel = zerolevel - transitdepth
slope = transitdepth / ingressduration
# the four contact points of the eclipse
firstcontact = 1.0 - halftransitduration
secondcontact = firstcontact + ingressduration
thirdcontact = halftransitduration - ingressduration
fourthcontact = halftransitduration
# # the phase indices # #
# during ingress
ingressind = ( phase > firstcontact ) & ( phase < secondcontact )
# at transit bottom
bottomind = ( phase > secondcontact ) | ( phase < thirdcontact )
# during egress
egressind = ( phase > thirdcontact ) & ( phase < fourthcontact )
# count the number of points in transit
in_transit_points = ingressind | bottomind | egressind
n_transit_points = np . sum ( in_transit_points )
# set the mags
modelmags [ ingressind ] = zerolevel - slope * ( phase [ ingressind ] - firstcontact )
modelmags [ bottomind ] = bottomlevel
modelmags [ egressind ] = bottomlevel + slope * ( phase [ egressind ] - thirdcontact )
if get_ntransitpoints :
    return modelmags , phase , ptimes , pmags , perrs , n_transit_points
else :
    return modelmags , phase , ptimes , pmags , perrs


signature:
def trapezoid_transit_residual ( transitparams , times , mags , errs ) :
    '''This returns the residual between the modelmags and the actual mags .
 Parameters
 transitparams : list of float
 This contains the transiting planet trapezoid model : :
 transitparams = [ transitperiod ( time ) ,
 transitepoch ( time ) ,
 transitdepth ( flux or mags ) ,
 transitduration ( phase ) ,
 ingressduration ( phase ) ]
 All of these will then have fitted values after the fit is done .
 - for magnitudes - > ` transitdepth ` should be < 0
 - for fluxes - > ` transitdepth ` should be > 0
 times , mags , errs : np . array
 The input time - series of measurements and associated errors for which
 the transit model will be generated . The times will be used to generate
 model mags , and the input ` times ` , ` mags ` , and ` errs ` will be resorted
 by model phase and returned .
 Returns
 np . array
 The residuals between the input ` mags ` and generated ` modelmags ` ,
 weighted by the measurement errors in ` errs ` .'''

implementation:
modelmags , phase , ptimes , pmags , perrs = ( trapezoid_transit_func ( transitparams , times , mags , errs ) )
# this is now a weighted residual taking into account the measurement err
return ( pmags - modelmags ) / perrs


signature:
def tap_query ( querystr , simbad_mirror = 'simbad' , returnformat = 'csv' , forcefetch = False , cachedir = '~/.astrobase/simbad-cache' , verbose = True , timeout = 10.0 , refresh = 2.0 , maxtimeout = 90.0 , maxtries = 3 , complete_query_later = False , jitter = 5.0 ) :
    '''This queries the SIMBAD TAP service using the ADQL query string provided .
 Parameters
 querystr : str
 This is the ADQL query string . See :
 http : / / www . ivoa . net / documents / ADQL / 2.0 for the specification .
 simbad _ mirror : str
 This is the key used to select a SIMBAD mirror from the
 ` SIMBAD _ URLS ` dict above . If set , the specified mirror will be used . If
 None , a random mirror chosen from that dict will be used .
 returnformat : { ' csv ' , ' votable ' , ' json ' }
 The returned file format to request from the GAIA catalog service .
 forcefetch : bool
 If this is True , the query will be retried even if cached results for
 it exist .
 cachedir : str
 This points to the directory where results will be downloaded .
 verbose : bool
 If True , will indicate progress and warn of any issues .
 timeout : float
 This sets the amount of time in seconds to wait for the service to
 respond to our initial request .
 refresh : float
 This sets the amount of time in seconds to wait before checking if the
 result file is available . If the results file isn ' t available after
 ` refresh ` seconds have elapsed , the function will wait for ` refresh `
 seconds continuously , until ` maxtimeout ` is reached or the results file
 becomes available .
 maxtimeout : float
 The maximum amount of time in seconds to wait for a result to become
 available after submitting our query request .
 maxtries : int
 The maximum number of tries ( across all mirrors tried ) to make to either
 submit the request or download the results , before giving up .
 complete _ query _ later : bool
 If set to True , a submitted query that does not return a result before
 ` maxtimeout ` has passed will be cancelled but its input request
 parameters and the result URL provided by the service will be saved . If
 this function is then called later with these same input request
 parameters , it will check if the query finally finished and a result is
 available . If so , will download the results instead of submitting a new
 query . If it ' s not done yet , will start waiting for results again . To
 force launch a new query with the same request parameters , set the
 ` forcefetch ` kwarg to True .
 jitter : float
 This is used to control the scale of the random wait in seconds before
 starting the query . Useful in parallelized situations .
 Returns
 dict
 This returns a dict of the following form : :
 { ' params ' : dict of the input params used for the query ,
 ' provenance ' : ' cache ' or ' new download ' ,
 ' result ' : path to the file on disk with the downloaded data table }'''

implementation:
# get the default params
inputparams = TAP_PARAMS . copy ( )
# update them with our input params
inputparams [ 'QUERY' ] = querystr [ : : ]
if returnformat in RETURN_FORMATS :
    inputparams [ 'FORMAT' ] = returnformat
else :
    LOGWARNING ( 'unknown result format: %s requested, using CSV' % returnformat )
    inputparams [ 'FORMAT' ] = 'csv'
# see if the cachedir exists
if '~' in cachedir :
    cachedir = os . path . expanduser ( cachedir )
if not os . path . exists ( cachedir ) :
    os . makedirs ( cachedir )
# generate the cachefname and look for it
xcachekey = '-' . join ( [ repr ( inputparams [ x ] ) for x in sorted ( inputparams . keys ( ) ) ] )
cachekey = hashlib . sha256 ( xcachekey . encode ( ) ) . hexdigest ( )
cachefname = os . path . join ( cachedir , '%s.%s' % ( cachekey , RETURN_FORMATS [ returnformat ] ) )
provenance = 'cache'
incomplete_qpklf = os . path . join ( cachedir , 'incomplete-query-%s' % cachekey )
# # COMPLETE A QUERY THAT MAY BE RUNNING # #
# first , check if this query can be resurrected
if ( not forcefetch and complete_query_later and os . path . exists ( incomplete_qpklf ) ) :
    with open ( incomplete_qpklf , 'rb' ) as infd :
        incomplete_qinfo = pickle . load ( infd )
    LOGWARNING ( 'complete_query_later = True, and ' 'this query was not completed on a ' 'previous run, will check if it is done now...' )
    # get the status URL and go into a loop to see if the query completed
    waitdone = False
    timeelapsed = 0.0
    simbad_mirror = incomplete_qinfo [ 'simbad_mirror' ]
    status_url = incomplete_qinfo [ 'status_url' ]
    phasekeyword = incomplete_qinfo [ 'phase_keyword' ]
    resultkeyword = incomplete_qinfo [ 'result_keyword' ]
    while not waitdone :
        if timeelapsed > maxtimeout :
            LOGERROR ( 'SIMBAD TAP query still not done ' 'after waiting %s seconds for results.\n' 'status URL is: %s' % ( maxtimeout , repr ( inputparams ) , status_url ) )
            return None
        try :
            resreq = requests . get ( status_url , timeout = timeout )
            resreq . raise_for_status ( )
            # parse the response XML and get the job status
            resxml = parseString ( resreq . text )
            jobstatuselem = ( resxml . getElementsByTagName ( phasekeyword ) [ 0 ] )
            jobstatus = jobstatuselem . firstChild . toxml ( )
            if jobstatus == 'COMPLETED' :
                if verbose :
                    LOGINFO ( 'SIMBAD query completed, ' 'retrieving results...' )
                waitdone = True
            # if we ' re not done yet , then wait some more
            elif jobstatus != 'ERROR' :
                if verbose :
                    LOGINFO ( 'elapsed time: %.1f, ' 'current status: %s, ' 'status URL: %s, waiting...' % ( timeelapsed , jobstatus , status_url ) )
                time . sleep ( refresh )
                timeelapsed = timeelapsed + refresh
            # if the JOB failed , then bail out immediately
            else :
                LOGERROR ( 'SIMBAD TAP query failed due to a server error.\n' 'status URL: %s\n' 'status contents: %s' % ( status_url , resreq . text ) )
                # since this job failed , remove the incomplete query pickle
                # so we can try this from scratch
                os . remove ( incomplete_qpklf )
                return None
        except requests . exceptions . Timeout as e :
            LOGEXCEPTION ( 'SIMBAD query timed out while waiting for status ' 'download results.\n' 'query: %s\n' 'status URL: %s' % ( repr ( inputparams ) , status_url ) )
            return None
        except Exception as e :
            LOGEXCEPTION ( 'SIMBAD query failed while waiting for status\n' 'query: %s\n' 'status URL: %s\n' 'status contents: %s' % ( repr ( inputparams ) , status_url , resreq . text ) )
            # if the query fails completely , then either the status URL
            # doesn ' t exist any more or something else went wrong . we ' ll
            # remove the incomplete query pickle so we can try this from
            # scratch
            os . remove ( incomplete_qpklf )
            return None
    # at this point , we should be ready to get the query results
    LOGINFO ( 'query completed, retrieving results...' )
    result_url_elem = resxml . getElementsByTagName ( resultkeyword ) [ 0 ]
    result_url = result_url_elem . getAttribute ( 'xlink:href' )
    result_nrows = result_url_elem . getAttribute ( 'rows' )
    try :
        resreq = requests . get ( result_url , timeout = timeout )
        resreq . raise_for_status ( )
        if cachefname . endswith ( '.gz' ) :
            with gzip . open ( cachefname , 'wb' ) as outfd :
                for chunk in resreq . iter_content ( chunk_size = 65536 ) :
                    outfd . write ( chunk )
        else :
            with open ( cachefname , 'wb' ) as outfd :
                for chunk in resreq . iter_content ( chunk_size = 65536 ) :
                    outfd . write ( chunk )
        if verbose :
            LOGINFO ( 'done. rows in result: %s' % result_nrows )
        tablefname = cachefname
        provenance = 'cache'
        # return a dict pointing to the result file
        # we ' ll parse this later
        resdict = { 'params' : inputparams , 'provenance' : provenance , 'result' : tablefname }
        # all went well , so we ' ll remove the incomplete query pickle
        os . remove ( incomplete_qpklf )
        return resdict
    except requests . exceptions . Timeout as e :
        LOGEXCEPTION ( 'SIMBAD query timed out while trying to ' 'download results.\n' 'query: %s\n' 'result URL: %s' % ( repr ( inputparams ) , result_url ) )
        return None
    except Exception as e :
        LOGEXCEPTION ( 'SIMBAD query failed because of an error ' 'while trying to download results.\n' 'query: %s\n' 'result URL: %s\n' 'response status code: %s' % ( repr ( inputparams ) , result_url , resreq . status_code ) )
        # if the result download fails , then either the result URL doesn ' t
        # exist any more or something else went wrong . we ' ll remove the
        # incomplete query pickle so we can try this from scratch
        os . remove ( incomplete_qpklf )
        return None
# # RUN A NEW QUERY # #
# otherwise , we check the cache if it ' s done already , or run it again if not
if forcefetch or ( not os . path . exists ( cachefname ) ) :
    provenance = 'new download'
    time . sleep ( random . randint ( 1 , jitter ) )
    # generate a jobid here and update the input params
    jobid = 'ab-simbad-%i' % time . time ( )
    inputparams [ 'JOBNAME' ] = jobid
    inputparams [ 'JOBDESCRIPTION' ] = 'astrobase-simbad-tap-ADQL-query'
    try :
        waitdone = False
        timeelapsed = 0.0
        # set the simbad mirror to use
        if simbad_mirror is not None and simbad_mirror in SIMBAD_URLS :
            tapurl = SIMBAD_URLS [ simbad_mirror ] [ 'url' ]
            resultkeyword = SIMBAD_URLS [ simbad_mirror ] [ 'resultkeyword' ]
            phasekeyword = SIMBAD_URLS [ simbad_mirror ] [ 'phasekeyword' ]
            randkey = simbad_mirror
            # sub in a table name if this is left unresolved in the input
            # query
            if '{table}' in querystr :
                inputparams [ 'QUERY' ] = ( querystr . format ( table = SIMBAD_URLS [ simbad_mirror ] [ 'table' ] ) )
        else :
            randkey = random . choice ( list ( SIMBAD_URLS . keys ( ) ) )
            tapurl = SIMBAD_URLS [ randkey ] [ 'url' ]
            resultkeyword = SIMBAD_URLS [ randkey ] [ 'resultkeyword' ]
            phasekeyword = SIMBAD_URLS [ randkey ] [ 'phasekeyword' ]
            # sub in a table name if this is left unresolved in the input
            # query
            if '{table}' in querystr :
                inputparams [ 'QUERY' ] = ( querystr . format ( table = SIMBAD_URLS [ randkey ] [ 'table' ] ) )
            if verbose :
                LOGINFO ( 'using SIMBAD mirror TAP URL: %s' % tapurl )
        # send the query and get status
        if verbose :
            LOGINFO ( 'submitting SIMBAD TAP query request for input params: %s' % repr ( inputparams ) )
        # here , we ' ll make sure the SIMBAD mirror works before doing
        # anything else
        mirrorok = False
        ntries = 1
        while ( not mirrorok ) :
            if ntries > maxtries :
                LOGERROR ( 'maximum number of allowed SIMBAD query ' 'submission tries (%s) reached, bailing out...' % maxtries )
                return None
            try :
                req = requests . post ( tapurl , data = inputparams , timeout = timeout )
                resp_status = req . status_code
                req . raise_for_status ( )
                mirrorok = True
            # this handles immediate 503s
            except requests . exceptions . HTTPError as e :
                LOGWARNING ( 'SIMBAD TAP server: %s not responding, ' 'trying another mirror...' % tapurl )
                mirrorok = False
                # for now , we have only one SIMBAD mirror to hit , so we ' ll
                # wait a random time between 1 and 5 seconds to hit it again
                remainingmirrors = list ( SIMBAD_URLS . keys ( ) )
                waittime = random . choice ( range ( 1 , 6 ) )
                time . sleep ( waittime )
                randkey = remainingmirrors [ 0 ]
                tapurl = SIMBAD_URLS [ randkey ] [ 'url' ]
                resultkeyword = SIMBAD_URLS [ randkey ] [ 'resultkeyword' ]
                phasekeyword = SIMBAD_URLS [ randkey ] [ 'phasekeyword' ]
                if '{table}' in querystr :
                    inputparams [ 'QUERY' ] = ( querystr . format ( table = SIMBAD_URLS [ randkey ] [ 'table' ] ) )
            # this handles initial query submission timeouts
            except requests . exceptions . Timeout as e :
                LOGWARNING ( 'SIMBAD TAP query submission timed out, ' 'mirror is probably down. Trying another mirror...' )
                mirrorok = False
                # for now , we have only one SIMBAD mirror to hit , so we ' ll
                # wait a random time between 1 and 5 seconds to hit it again
                remainingmirrors = list ( SIMBAD_URLS . keys ( ) )
                waittime = random . choice ( range ( 1 , 6 ) )
                time . sleep ( waittime )
                randkey = remainingmirrors [ 0 ]
                tapurl = SIMBAD_URLS [ randkey ] [ 'url' ]
                resultkeyword = SIMBAD_URLS [ randkey ] [ 'resultkeyword' ]
                phasekeyword = SIMBAD_URLS [ randkey ] [ 'phasekeyword' ]
                if '{table}' in querystr :
                    inputparams [ 'QUERY' ] = ( querystr . format ( table = SIMBAD_URLS [ randkey ] [ 'table' ] ) )
            # update the number of submission tries
            ntries = ntries + 1
        # NOTE : python - requests follows the " 303 See Other " redirect
        # automatically , so we get the XML status doc immediately . We don ' t
        # need to look up the location of it in the initial response ' s
        # header as in the SIMBAD example .
        status_url = req . url
        # parse the response XML and get the job status
        resxml = parseString ( req . text )
        jobstatuselem = resxml . getElementsByTagName ( phasekeyword )
        if jobstatuselem :
            jobstatuselem = jobstatuselem [ 0 ]
        else :
            LOGERROR ( 'could not parse job phase using ' 'keyword %s in result XML' % phasekeyword )
            LOGERROR ( '%s' % req . txt )
            req . close ( )
            return None
        jobstatus = jobstatuselem . firstChild . toxml ( )
        # if the job completed already , jump down to retrieving results
        if jobstatus == 'COMPLETED' :
            if verbose :
                LOGINFO ( 'SIMBAD query completed, ' 'retrieving results...' )
                waitdone = True
        elif jobstatus == 'ERROR' :
            if verbose :
                LOGERROR ( 'SIMBAD query failed immediately ' '(probably an ADQL error): %s, ' 'status URL: %s, status contents: %s' % ( repr ( inputparams ) , status_url , req . text ) )
                return None
        # we wait for the job to complete if it ' s not done already
        else :
            if verbose :
                LOGINFO ( 'request submitted successfully, ' 'current status is: %s. ' 'waiting for results...' % jobstatus )
            while not waitdone :
                if timeelapsed > maxtimeout :
                    LOGERROR ( 'SIMBAD TAP query timed out ' 'after waiting %s seconds for results.\n' 'request was: %s\n' 'status URL is: %s\n' 'last status was: %s' % ( maxtimeout , repr ( inputparams ) , status_url , jobstatus ) )
                    # here , we ' ll check if we ' re allowed to sleep on a query
                    # for a bit and return to it later if the last status
                    # was QUEUED or EXECUTING
                    if complete_query_later and jobstatus in ( 'EXECUTING' , 'QUEUED' ) : # write a pickle with the query params that we can
                    # pick up later to finish this query
                        incomplete_qpklf = os . path . join ( cachedir , 'incomplete-query-%s' % cachekey )
                        with open ( incomplete_qpklf , 'wb' ) as outfd :
                            savedict = inputparams . copy ( )
                            savedict [ 'status_url' ] = status_url
                            savedict [ 'last_status' ] = jobstatus
                            savedict [ 'simbad_mirror' ] = simbad_mirror
                            savedict [ 'phase_keyword' ] = phasekeyword
                            savedict [ 'result_keyword' ] = resultkeyword
                            pickle . dump ( savedict , outfd , pickle . HIGHEST_PROTOCOL )
                        LOGINFO ( 'complete_query_later = True, ' 'last state of query was: %s, ' 'will resume later if this function ' 'is called again with the same query' % jobstatus )
                    return None
                time . sleep ( refresh )
                timeelapsed = timeelapsed + refresh
                try :
                    resreq = requests . get ( status_url , timeout = timeout )
                    resreq . raise_for_status ( )
                    # parse the response XML and get the job status
                    resxml = parseString ( resreq . text )
                    jobstatuselem = ( resxml . getElementsByTagName ( phasekeyword ) [ 0 ] )
                    jobstatus = jobstatuselem . firstChild . toxml ( )
                    if jobstatus == 'COMPLETED' :
                        if verbose :
                            LOGINFO ( 'SIMBAD query completed, ' 'retrieving results...' )
                        waitdone = True
                    else :
                        if verbose :
                            LOGINFO ( 'elapsed time: %.1f, ' 'current status: %s, ' 'status URL: %s, waiting...' % ( timeelapsed , jobstatus , status_url ) )
                        continue
                except requests . exceptions . Timeout as e :
                    LOGEXCEPTION ( 'SIMBAD query timed out while waiting for results ' 'download results.\n' 'query: %s\n' 'status URL: %s' % ( repr ( inputparams ) , status_url ) )
                    return None
                except Exception as e :
                    LOGEXCEPTION ( 'SIMBAD query failed while waiting for results\n' 'query: %s\n' 'status URL: %s\n' 'status contents: %s' % ( repr ( inputparams ) , status_url , resreq . text ) )
                    return None
        # at this point , we should be ready to get the query results
        result_url_elem = resxml . getElementsByTagName ( resultkeyword ) [ 0 ]
        result_url = result_url_elem . getAttribute ( 'xlink:href' )
        result_nrows = result_url_elem . getAttribute ( 'rows' )
        try :
            resreq = requests . get ( result_url , timeout = timeout )
            resreq . raise_for_status ( )
            if cachefname . endswith ( '.gz' ) :
                with gzip . open ( cachefname , 'wb' ) as outfd :
                    for chunk in resreq . iter_content ( chunk_size = 65536 ) :
                        outfd . write ( chunk )
            else :
                with open ( cachefname , 'wb' ) as outfd :
                    for chunk in resreq . iter_content ( chunk_size = 65536 ) :
                        outfd . write ( chunk )
            if verbose :
                LOGINFO ( 'done. rows in result: %s' % result_nrows )
            tablefname = cachefname
        except requests . exceptions . Timeout as e :
            LOGEXCEPTION ( 'SIMBAD query timed out while trying to ' 'download results.\n' 'query: %s\n' 'result URL: %s' % ( repr ( inputparams ) , result_url ) )
            return None
        except Exception as e :
            LOGEXCEPTION ( 'SIMBAD query failed because of an error ' 'while trying to download results.\n' 'query: %s\n' 'result URL: %s\n' 'response status code: %s' % ( repr ( inputparams ) , result_url , resreq . status_code ) )
            return None
    except requests . exceptions . HTTPError as e :
        LOGEXCEPTION ( 'SIMBAD TAP query failed.\nrequest status was: ' '%s.\nquery was: %s' % ( resp_status , repr ( inputparams ) ) )
        return None
    except requests . exceptions . Timeout as e :
        LOGERROR ( 'SIMBAD TAP query submission timed out, ' 'site is probably down. Request was: ' '%s' % repr ( inputparams ) )
        return None
    except Exception as e :
        LOGEXCEPTION ( 'SIMBAD TAP query request failed for ' '%s' % repr ( inputparams ) )
        if 'resxml' in locals ( ) :
            LOGERROR ( 'HTTP response from service:\n%s' % req . text )
        return None
# # GET RESULTS FROM CACHE # #
else :
    if verbose :
        LOGINFO ( 'getting cached SIMBAD query result for ' 'request: %s' % ( repr ( inputparams ) ) )
    tablefname = cachefname
    # try to open the cached file to make sure it ' s OK
    try :
        infd = gzip . open ( cachefname , 'rb' )
        simbad_objectnames = np . genfromtxt ( infd , names = True , delimiter = ',' , dtype = 'U20,f8,f8,U20,U20,U20,i8,U600,f8' , usecols = ( 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 ) , comments = '?' , # object names can have ' # ' in them
        )
        infd . close ( )
    except Exception as e :
        LOGEXCEPTION ( 'could not read cached SIMBAD result file: %s, ' 'fetching from server again' % cachefname )
        return tap_query ( querystr , simbad_mirror = simbad_mirror , returnformat = returnformat , forcefetch = True , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout )
# all done with retrieval , now return the result dict
# return a dict pointing to the result file
# we ' ll parse this later
resdict = { 'params' : inputparams , 'provenance' : provenance , 'result' : tablefname }
return resdict


signature:
def objectnames_conesearch ( racenter , declcenter , searchradiusarcsec , simbad_mirror = 'simbad' , returnformat = 'csv' , forcefetch = False , cachedir = '~/.astrobase/simbad-cache' , verbose = True , timeout = 10.0 , refresh = 2.0 , maxtimeout = 90.0 , maxtries = 1 , complete_query_later = True ) :
    '''This queries the SIMBAD TAP service for a list of object names near the
 coords . This is effectively a " reverse " name resolver ( i . e . this does the
 opposite of SESAME ) .
 Parameters
 racenter , declcenter : float
 The cone - search center coordinates in decimal degrees
 searchradiusarcsec : float
 The radius in arcseconds to search around the center coordinates .
 simbad _ mirror : str
 This is the key used to select a SIMBAD mirror from the
 ` SIMBAD _ URLS ` dict above . If set , the specified mirror will be used . If
 None , a random mirror chosen from that dict will be used .
 returnformat : { ' csv ' , ' votable ' , ' json ' }
 The returned file format to request from the GAIA catalog service .
 forcefetch : bool
 If this is True , the query will be retried even if cached results for
 it exist .
 cachedir : str
 This points to the directory where results will be downloaded .
 verbose : bool
 If True , will indicate progress and warn of any issues .
 timeout : float
 This sets the amount of time in seconds to wait for the service to
 respond to our initial request .
 refresh : float
 This sets the amount of time in seconds to wait before checking if the
 result file is available . If the results file isn ' t available after
 ` refresh ` seconds have elapsed , the function will wait for ` refresh `
 seconds continuously , until ` maxtimeout ` is reached or the results file
 becomes available .
 maxtimeout : float
 The maximum amount of time in seconds to wait for a result to become
 available after submitting our query request .
 maxtries : int
 The maximum number of tries ( across all mirrors tried ) to make to either
 submit the request or download the results , before giving up .
 complete _ query _ later : bool
 If set to True , a submitted query that does not return a result before
 ` maxtimeout ` has passed will be cancelled but its input request
 parameters and the result URL provided by the service will be saved . If
 this function is then called later with these same input request
 parameters , it will check if the query finally finished and a result is
 available . If so , will download the results instead of submitting a new
 query . If it ' s not done yet , will start waiting for results again . To
 force launch a new query with the same request parameters , set the
 ` forcefetch ` kwarg to True .
 Returns
 dict
 This returns a dict of the following form : :
 { ' params ' : dict of the input params used for the query ,
 ' provenance ' : ' cache ' or ' new download ' ,
 ' result ' : path to the file on disk with the downloaded data table }'''

implementation:
# this was generated using the example at :
# http : / / simbad . u - strasbg . fr / simbad / sim - tap and the table diagram at :
# http : / / simbad . u - strasbg . fr / simbad / tap / tapsearch . html
query = ( "select a.oid, a.ra, a.dec, a.main_id, a.otype_txt, " "a.coo_bibcode, a.nbref, b.ids as all_ids, " "(DISTANCE(POINT('ICRS', a.ra, a.dec), " "POINT('ICRS', {ra_center:.5f}, {decl_center:.5f})))*3600.0 " "AS dist_arcsec " "from basic a join ids b on a.oid = b.oidref where " "CONTAINS(POINT('ICRS',a.ra, a.dec)," "CIRCLE('ICRS',{ra_center:.5f},{decl_center:.5f}," "{search_radius:.6f}))=1 " "ORDER by dist_arcsec asc " )
formatted_query = query . format ( ra_center = racenter , decl_center = declcenter , search_radius = searchradiusarcsec / 3600.0 )
return tap_query ( formatted_query , simbad_mirror = simbad_mirror , returnformat = returnformat , forcefetch = forcefetch , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout , maxtries = maxtries , complete_query_later = complete_query_later )


signature:
def xmatch_cplist_external_catalogs ( cplist , xmatchpkl , xmatchradiusarcsec = 2.0 , updateexisting = True , resultstodir = None ) :
    '''This xmatches external catalogs to a collection of checkplots .
 Parameters
 cplist : list of str
 This is the list of checkplot pickle files to process .
 xmatchpkl : str
 The filename of a pickle prepared beforehand with the
 ` checkplot . pkl _ xmatch . load _ xmatch _ external _ catalogs ` function ,
 containing collected external catalogs to cross - match the objects in the
 input ` cplist ` against .
 xmatchradiusarcsec : float
 The match radius to use for the cross - match in arcseconds .
 updateexisting : bool
 If this is True , will only update the ` xmatch ` dict in each checkplot
 pickle with any new cross - matches to the external catalogs . If False ,
 will overwrite the ` xmatch ` dict with results from the current run .
 resultstodir : str or None
 If this is provided , then it must be a directory to write the resulting
 checkplots to after xmatch is done . This can be used to keep the
 original checkplots in pristine condition for some reason .
 Returns
 dict
 Returns a dict with keys = input checkplot pickle filenames and vals =
 xmatch status dict for each checkplot pickle .'''

implementation:
# load the external catalog
with open ( xmatchpkl , 'rb' ) as infd :
    xmd = pickle . load ( infd )
# match each object . this is fairly fast , so this is not parallelized at the
# moment
status_dict = { }
for cpf in cplist :
    cpd = _read_checkplot_picklefile ( cpf )
    try : # match in place
        xmatch_external_catalogs ( cpd , xmd , xmatchradiusarcsec = xmatchradiusarcsec , updatexmatch = updateexisting )
        for xmi in cpd [ 'xmatch' ] :
            if cpd [ 'xmatch' ] [ xmi ] [ 'found' ] :
                LOGINFO ( 'checkplot %s: %s matched to %s, ' 'match dist: %s arcsec' % ( os . path . basename ( cpf ) , cpd [ 'objectid' ] , cpd [ 'xmatch' ] [ xmi ] [ 'name' ] , cpd [ 'xmatch' ] [ xmi ] [ 'distarcsec' ] ) )
            if not resultstodir :
                outcpf = _write_checkplot_picklefile ( cpd , outfile = cpf )
            else :
                xcpf = os . path . join ( resultstodir , os . path . basename ( cpf ) )
                outcpf = _write_checkplot_picklefile ( cpd , outfile = xcpf )
        status_dict [ cpf ] = outcpf
    except Exception as e :
        LOGEXCEPTION ( 'failed to match objects for %s' % cpf )
        status_dict [ cpf ] = None
return status_dict


signature:
def xmatch_cpdir_external_catalogs ( cpdir , xmatchpkl , cpfileglob = 'checkplot-*.pkl*' , xmatchradiusarcsec = 2.0 , updateexisting = True , resultstodir = None ) :
    '''This xmatches external catalogs to all checkplots in a directory .
 Parameters
 cpdir : str
 This is the directory to search in for checkplots .
 xmatchpkl : str
 The filename of a pickle prepared beforehand with the
 ` checkplot . pkl _ xmatch . load _ xmatch _ external _ catalogs ` function ,
 containing collected external catalogs to cross - match the objects in the
 input ` cplist ` against .
 cpfileglob : str
 This is the UNIX fileglob to use in searching for checkplots .
 xmatchradiusarcsec : float
 The match radius to use for the cross - match in arcseconds .
 updateexisting : bool
 If this is True , will only update the ` xmatch ` dict in each checkplot
 pickle with any new cross - matches to the external catalogs . If False ,
 will overwrite the ` xmatch ` dict with results from the current run .
 resultstodir : str or None
 If this is provided , then it must be a directory to write the resulting
 checkplots to after xmatch is done . This can be used to keep the
 original checkplots in pristine condition for some reason .
 Returns
 dict
 Returns a dict with keys = input checkplot pickle filenames and vals =
 xmatch status dict for each checkplot pickle .'''

implementation:
cplist = glob . glob ( os . path . join ( cpdir , cpfileglob ) )
return xmatch_cplist_external_catalogs ( cplist , xmatchpkl , xmatchradiusarcsec = xmatchradiusarcsec , updateexisting = updateexisting , resultstodir = resultstodir )


signature:
def colormagdiagram_cplist ( cplist , outpkl , color_mag1 = [ 'gaiamag' , 'sdssg' ] , color_mag2 = [ 'kmag' , 'kmag' ] , yaxis_mag = [ 'gaia_absmag' , 'rpmj' ] ) :
    '''This makes color - mag diagrams for all checkplot pickles in the provided
 list .
 Can make an arbitrary number of CMDs given lists of x - axis colors and y - axis
 mags to use .
 Parameters
 cplist : list of str
 This is the list of checkplot pickles to process .
 outpkl : str
 The filename of the output pickle that will contain the color - mag
 information for all objects in the checkplots specified in ` cplist ` .
 color _ mag1 : list of str
 This a list of the keys in each checkplot ' s ` objectinfo ` dict that will
 be used as color _ 1 in the equation : :
 x - axis color = color _ mag1 - color _ mag2
 color _ mag2 : list of str
 This a list of the keys in each checkplot ' s ` objectinfo ` dict that will
 be used as color _ 2 in the equation : :
 x - axis color = color _ mag1 - color _ mag2
 yaxis _ mag : list of str
 This is a list of the keys in each checkplot ' s ` objectinfo ` dict that
 will be used as the ( absolute ) magnitude y - axis of the color - mag
 diagrams .
 Returns
 str
 The path to the generated CMD pickle file for the collection of objects
 in the input checkplot list .
 Notes
 This can make many CMDs in one go . For example , the default kwargs for
 ` color _ mag ` , ` color _ mag2 ` , and ` yaxis _ mag ` result in two CMDs generated and
 written to the output pickle file :
 - CMD1 - > gaiamag - kmag on the x - axis vs gaia _ absmag on the y - axis
 - CMD2 - > sdssg - kmag on the x - axis vs rpmj ( J reduced PM ) on the y - axis'''

implementation:
# first , we ' ll collect all of the info
cplist_objectids = [ ]
cplist_mags = [ ]
cplist_colors = [ ]
for cpf in cplist :
    cpd = _read_checkplot_picklefile ( cpf )
    cplist_objectids . append ( cpd [ 'objectid' ] )
    thiscp_mags = [ ]
    thiscp_colors = [ ]
    for cm1 , cm2 , ym in zip ( color_mag1 , color_mag2 , yaxis_mag ) :
        if ( ym in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ ym ] is not None ) :
            thiscp_mags . append ( cpd [ 'objectinfo' ] [ ym ] )
        else :
            thiscp_mags . append ( np . nan )
        if ( cm1 in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ cm1 ] is not None and cm2 in cpd [ 'objectinfo' ] and cpd [ 'objectinfo' ] [ cm2 ] is not None ) :
            thiscp_colors . append ( cpd [ 'objectinfo' ] [ cm1 ] - cpd [ 'objectinfo' ] [ cm2 ] )
        else :
            thiscp_colors . append ( np . nan )
    cplist_mags . append ( thiscp_mags )
    cplist_colors . append ( thiscp_colors )
# convert these to arrays
cplist_objectids = np . array ( cplist_objectids )
cplist_mags = np . array ( cplist_mags )
cplist_colors = np . array ( cplist_colors )
# prepare the outdict
cmddict = { 'objectids' : cplist_objectids , 'mags' : cplist_mags , 'colors' : cplist_colors , 'color_mag1' : color_mag1 , 'color_mag2' : color_mag2 , 'yaxis_mag' : yaxis_mag }
# save the pickled figure and dict for fast retrieval later
with open ( outpkl , 'wb' ) as outfd :
    pickle . dump ( cmddict , outfd , pickle . HIGHEST_PROTOCOL )
plt . close ( 'all' )
return cmddict


signature:
def colormagdiagram_cpdir ( cpdir , outpkl , cpfileglob = 'checkplot*.pkl*' , color_mag1 = [ 'gaiamag' , 'sdssg' ] , color_mag2 = [ 'kmag' , 'kmag' ] , yaxis_mag = [ 'gaia_absmag' , 'rpmj' ] ) :
    '''This makes CMDs for all checkplot pickles in the provided directory .
 Can make an arbitrary number of CMDs given lists of x - axis colors and y - axis
 mags to use .
 Parameters
 cpdir : list of str
 This is the directory to get the list of input checkplot pickles from .
 outpkl : str
 The filename of the output pickle that will contain the color - mag
 information for all objects in the checkplots specified in ` cplist ` .
 cpfileglob : str
 The UNIX fileglob to use to search for checkplot pickle files .
 color _ mag1 : list of str
 This a list of the keys in each checkplot ' s ` objectinfo ` dict that will
 be used as color _ 1 in the equation : :
 x - axis color = color _ mag1 - color _ mag2
 color _ mag2 : list of str
 This a list of the keys in each checkplot ' s ` objectinfo ` dict that will
 be used as color _ 2 in the equation : :
 x - axis color = color _ mag1 - color _ mag2
 yaxis _ mag : list of str
 This is a list of the keys in each checkplot ' s ` objectinfo ` dict that
 will be used as the ( absolute ) magnitude y - axis of the color - mag
 diagrams .
 Returns
 str
 The path to the generated CMD pickle file for the collection of objects
 in the input checkplot directory .
 Notes
 This can make many CMDs in one go . For example , the default kwargs for
 ` color _ mag ` , ` color _ mag2 ` , and ` yaxis _ mag ` result in two CMDs generated and
 written to the output pickle file :
 - CMD1 - > gaiamag - kmag on the x - axis vs gaia _ absmag on the y - axis
 - CMD2 - > sdssg - kmag on the x - axis vs rpmj ( J reduced PM ) on the y - axis'''

implementation:
cplist = glob . glob ( os . path . join ( cpdir , cpfileglob ) )
return colormagdiagram_cplist ( cplist , outpkl , color_mag1 = color_mag1 , color_mag2 = color_mag2 , yaxis_mag = yaxis_mag )


signature:
def add_cmd_to_checkplot ( cpx , cmdpkl , require_cmd_magcolor = True , save_cmd_pngs = False ) :
    '''This adds CMD figures to a checkplot dict or pickle .
 Looks up the CMDs in ` cmdpkl ` , adds the object from ` cpx ` as a gold ( - ish )
 star in the plot , and then saves the figure to a base64 encoded PNG , which
 can then be read and used by the ` checkplotserver ` .
 Parameters
 cpx : str or dict
 This is the input checkplot pickle or dict to add the CMD to .
 cmdpkl : str or dict
 The CMD pickle generated by the ` colormagdiagram _ cplist ` or
 ` colormagdiagram _ cpdir ` functions above , or the dict produced by reading
 this pickle in .
 require _ cmd _ magcolor : bool
 If this is True , a CMD plot will not be made if the color and mag keys
 required by the CMD are not present or are nan in this checkplot ' s
 objectinfo dict .
 save _ cmd _ png : bool
 If this is True , then will save the CMD plots that were generated and
 added back to the checkplotdict as PNGs to the same directory as
 ` cpx ` . If ` cpx ` is a dict , will save them to the current working
 directory .
 Returns
 str or dict
 If ` cpx ` was a str filename of checkplot pickle , this will return that
 filename to indicate that the CMD was added to the file . If ` cpx ` was a
 checkplotdict , this will return the checkplotdict with a new key called
 ' colormagdiagram ' containing the base64 encoded PNG binary streams of
 all CMDs generated .'''

implementation:
# get the checkplot
if isinstance ( cpx , str ) and os . path . exists ( cpx ) :
    cpdict = _read_checkplot_picklefile ( cpx )
elif isinstance ( cpx , dict ) :
    cpdict = cpx
else :
    LOGERROR ( 'unknown type of checkplot provided as the cpx arg' )
    return None
# get the CMD
if isinstance ( cmdpkl , str ) and os . path . exists ( cmdpkl ) :
    with open ( cmdpkl , 'rb' ) as infd :
        cmd = pickle . load ( infd )
elif isinstance ( cmdpkl , dict ) :
    cmd = cmdpkl
cpdict [ 'colormagdiagram' ] = { }
# get the mags and colors from the CMD dict
cplist_mags = cmd [ 'mags' ]
cplist_colors = cmd [ 'colors' ]
# now make the CMD plots for each color - mag combination in the CMD
for c1 , c2 , ym , ind in zip ( cmd [ 'color_mag1' ] , cmd [ 'color_mag2' ] , cmd [ 'yaxis_mag' ] , range ( len ( cmd [ 'color_mag1' ] ) ) ) : # get these from the checkplot for this object
    if ( c1 in cpdict [ 'objectinfo' ] and cpdict [ 'objectinfo' ] [ c1 ] is not None ) :
        c1mag = cpdict [ 'objectinfo' ] [ c1 ]
    else :
        c1mag = np . nan
    if ( c2 in cpdict [ 'objectinfo' ] and cpdict [ 'objectinfo' ] [ c2 ] is not None ) :
        c2mag = cpdict [ 'objectinfo' ] [ c2 ]
    else :
        c2mag = np . nan
    if ( ym in cpdict [ 'objectinfo' ] and cpdict [ 'objectinfo' ] [ ym ] is not None ) :
        ymmag = cpdict [ 'objectinfo' ] [ ym ]
    else :
        ymmag = np . nan
    if ( require_cmd_magcolor and not ( np . isfinite ( c1mag ) and np . isfinite ( c2mag ) and np . isfinite ( ymmag ) ) ) :
        LOGWARNING ( "required color: %s-%s or mag: %s are not " "in this checkplot's objectinfo dict " "(objectid: %s), skipping CMD..." % ( c1 , c2 , ym , cpdict [ 'objectid' ] ) )
        continue
    # make the CMD for this color - mag combination
    try :
        thiscmd_title = r'%s-%s/%s' % ( CMD_LABELS [ c1 ] , CMD_LABELS [ c2 ] , CMD_LABELS [ ym ] )
        # make the scatter plot
        fig = plt . figure ( figsize = ( 10 , 8 ) )
        plt . plot ( cplist_colors [ : , ind ] , cplist_mags [ : , ind ] , rasterized = True , marker = 'o' , linestyle = 'none' , mew = 0 , ms = 3 )
        # put this object on the plot
        plt . plot ( [ c1mag - c2mag ] , [ ymmag ] , ms = 20 , color = '#b0ff05' , marker = '*' , mew = 0 )
        plt . xlabel ( r'$%s - %s$' % ( CMD_LABELS [ c1 ] , CMD_LABELS [ c2 ] ) )
        plt . ylabel ( r'$%s$' % CMD_LABELS [ ym ] )
        plt . title ( '%s - $%s$ CMD' % ( cpdict [ 'objectid' ] , thiscmd_title ) )
        plt . gca ( ) . invert_yaxis ( )
        # now save the figure to StrIO and put it back in the checkplot
        cmdpng = StrIO ( )
        plt . savefig ( cmdpng , bbox_inches = 'tight' , pad_inches = 0.0 , format = 'png' )
        cmdpng . seek ( 0 )
        cmdb64 = base64 . b64encode ( cmdpng . read ( ) )
        cmdpng . close ( )
        plt . close ( 'all' )
        plt . gcf ( ) . clear ( )
        cpdict [ 'colormagdiagram' ] [ '%s-%s/%s' % ( c1 , c2 , ym ) ] = cmdb64
        # if we ' re supposed to export to PNG , do so
        if save_cmd_pngs :
            if isinstance ( cpx , str ) :
                outpng = os . path . join ( os . path . dirname ( cpx ) , 'cmd-%s-%s-%s.%s.png' % ( cpdict [ 'objectid' ] , c1 , c2 , ym ) )
            else :
                outpng = 'cmd-%s-%s-%s.%s.png' % ( cpdict [ 'objectid' ] , c1 , c2 , ym )
            _base64_to_file ( cmdb64 , outpng )
    except Exception as e :
        LOGEXCEPTION ( 'CMD for %s-%s/%s does not exist in %s, skipping...' % ( c1 , c2 , ym , cmdpkl ) )
        continue
# end of making CMDs
if isinstance ( cpx , str ) :
    cpf = _write_checkplot_picklefile ( cpdict , outfile = cpx , protocol = 4 )
    return cpf
elif isinstance ( cpx , dict ) :
    return cpdict


signature:
def add_cmds_cplist ( cplist , cmdpkl , require_cmd_magcolor = True , save_cmd_pngs = False ) :
    '''This adds CMDs for each object in cplist .
 Parameters
 cplist : list of str
 This is the input list of checkplot pickles to add the CMDs to .
 cmdpkl : str
 This is the filename of the CMD pickle created previously .
 require _ cmd _ magcolor : bool
 If this is True , a CMD plot will not be made if the color and mag keys
 required by the CMD are not present or are nan in each checkplot ' s
 objectinfo dict .
 save _ cmd _ pngs : bool
 If this is True , then will save the CMD plots that were generated and
 added back to the checkplotdict as PNGs to the same directory as
 ` cpx ` .
 Returns
 Nothing .'''

implementation:
# load the CMD first to save on IO
with open ( cmdpkl , 'rb' ) as infd :
    cmd = pickle . load ( infd )
for cpf in cplist :
    add_cmd_to_checkplot ( cpf , cmd , require_cmd_magcolor = require_cmd_magcolor , save_cmd_pngs = save_cmd_pngs )


signature:
def add_cmds_cpdir ( cpdir , cmdpkl , cpfileglob = 'checkplot*.pkl*' , require_cmd_magcolor = True , save_cmd_pngs = False ) :
    '''This adds CMDs for each object in cpdir .
 Parameters
 cpdir : list of str
 This is the directory to search for checkplot pickles .
 cmdpkl : str
 This is the filename of the CMD pickle created previously .
 cpfileglob : str
 The UNIX fileglob to use when searching for checkplot pickles to operate
 on .
 require _ cmd _ magcolor : bool
 If this is True , a CMD plot will not be made if the color and mag keys
 required by the CMD are not present or are nan in each checkplot ' s
 objectinfo dict .
 save _ cmd _ pngs : bool
 If this is True , then will save the CMD plots that were generated and
 added back to the checkplotdict as PNGs to the same directory as
 ` cpx ` .
 Returns
 Nothing .'''

implementation:
cplist = glob . glob ( os . path . join ( cpdir , cpfileglob ) )
return add_cmds_cplist ( cplist , cmdpkl , require_cmd_magcolor = require_cmd_magcolor , save_cmd_pngs = save_cmd_pngs )


signature:
def cp_objectinfo_worker ( task ) :
    '''This is a parallel worker for ` parallel _ update _ cp _ objectinfo ` .
 Parameters
 task : tuple
 - task [ 0 ] = checkplot pickle file
 - task [ 1 ] = kwargs
 Returns
 str
 The name of the checkplot file that was updated . None if the update
 fails for some reason .'''

implementation:
cpf , cpkwargs = task
try :
    newcpf = update_checkplot_objectinfo ( cpf , ** cpkwargs )
    return newcpf
except Exception as e :
    LOGEXCEPTION ( 'failed to update objectinfo for %s' % cpf )
    return None


signature:
def parallel_update_objectinfo_cplist ( cplist , liststartindex = None , maxobjects = None , nworkers = NCPUS , fast_mode = False , findercmap = 'gray_r' , finderconvolve = None , deredden_object = True , custom_bandpasses = None , gaia_submit_timeout = 10.0 , gaia_submit_tries = 3 , gaia_max_timeout = 180.0 , gaia_mirror = None , complete_query_later = True , lclistpkl = None , nbrradiusarcsec = 60.0 , maxnumneighbors = 5 , plotdpi = 100 , findercachedir = '~/.astrobase/stamp-cache' , verbose = True ) :
    '''This updates objectinfo for a list of checkplots .
 Useful in cases where a previous round of GAIA / finderchart / external catalog
 acquisition failed . This will preserve the following keys in the checkplots
 if they exist :
 comments
 varinfo
 objectinfo . objecttags
 Parameters
 cplist : list of str
 A list of checkplot pickle file names to update .
 liststartindex : int
 The index of the input list to start working at .
 maxobjects : int
 The maximum number of objects to process in this run . Use this with
 ` liststartindex ` to effectively distribute working on a large list of
 input checkplot pickles over several sessions or machines .
 nworkers : int
 The number of parallel workers that will work on the checkplot
 update process .
 fast _ mode : bool or float
 This runs the external catalog operations in a " fast " mode , with short
 timeouts and not trying to hit external catalogs that take a long time
 to respond . See the docstring for
 ` checkplot . pkl _ utils . _ pkl _ finder _ objectinfo ` for details on how this
 works . If this is True , will run in " fast " mode with default timeouts ( 5
 seconds in most cases ) . If this is a float , will run in " fast " mode with
 the provided timeout value in seconds .
 findercmap : str or matplotlib . cm . Colormap object
 findercmap : str or matplotlib . cm . ColorMap object
 The Colormap object to use for the finder chart image .
 finderconvolve : astropy . convolution . Kernel object or None
 If not None , the Kernel object to use for convolving the finder image .
 deredden _ objects : bool
 If this is True , will use the 2MASS DUST service to get extinction
 coefficients in various bands , and then try to deredden the magnitudes
 and colors of the object already present in the checkplot ' s objectinfo
 dict .
 custom _ bandpasses : dict
 This is a dict used to provide custom bandpass definitions for any
 magnitude measurements in the objectinfo dict that are not automatically
 recognized by the ` varclass . starfeatures . color _ features ` function . See
 its docstring for details on the required format .
 gaia _ submit _ timeout : float
 Sets the timeout in seconds to use when submitting a request to look up
 the object ' s information to the GAIA service . Note that if ` fast _ mode `
 is set , this is ignored .
 gaia _ submit _ tries : int
 Sets the maximum number of times the GAIA services will be contacted to
 obtain this object ' s information . If ` fast _ mode ` is set , this is
 ignored , and the services will be contacted only once ( meaning that a
 failure to respond will be silently ignored and no GAIA data will be
 added to the checkplot ' s objectinfo dict ) .
 gaia _ max _ timeout : float
 Sets the timeout in seconds to use when waiting for the GAIA service to
 respond to our request for the object ' s information . Note that if
 ` fast _ mode ` is set , this is ignored .
 gaia _ mirror : str
 This sets the GAIA mirror to use . This is a key in the
 ` services . gaia . GAIA _ URLS ` dict which defines the URLs to hit for each
 mirror .
 complete _ query _ later : bool
 If this is True , saves the state of GAIA queries that are not yet
 complete when ` gaia _ max _ timeout ` is reached while waiting for the GAIA
 service to respond to our request . A later call for GAIA info on the
 same object will attempt to pick up the results from the existing query
 if it ' s completed . If ` fast _ mode ` is True , this is ignored .
 lclistpkl : dict or str
 If this is provided , must be a dict resulting from reading a catalog
 produced by the ` lcproc . catalogs . make _ lclist ` function or a str path
 pointing to the pickle file produced by that function . This catalog is
 used to find neighbors of the current object in the current light curve
 collection . Looking at neighbors of the object within the radius
 specified by ` nbrradiusarcsec ` is useful for light curves produced by
 instruments that have a large pixel scale , so are susceptible to
 blending of variability and potential confusion of neighbor variability
 with that of the actual object being looked at . If this is None , no
 neighbor lookups will be performed .
 nbrradiusarcsec : float
 The radius in arcseconds to use for a search conducted around the
 coordinates of this object to look for any potential confusion and
 blending of variability amplitude caused by their proximity .
 maxnumneighbors : int
 The maximum number of neighbors that will have their light curves and
 magnitudes noted in this checkplot as potential blends with the target
 object .
 plotdpi : int
 The resolution in DPI of the plots to generate in this function
 ( e . g . the finder chart , etc . )
 findercachedir : str
 The path to the astrobase cache directory for finder chart downloads
 from the NASA SkyView service .
 verbose : bool
 If True , will indicate progress and warn about potential problems .
 Returns
 list of str
 Paths to the updated checkplot pickle file .'''

implementation:
# work around the Darwin segfault after fork if no network activity in
# main thread bug : https : / / bugs . python . org / issue30385 # msg293958
if sys . platform == 'darwin' :
    import requests
    requests . get ( 'http://captive.apple.com/hotspot-detect.html' )
# handle the start and end indices
if ( liststartindex is not None ) and ( maxobjects is None ) :
    cplist = cplist [ liststartindex : ]
elif ( liststartindex is None ) and ( maxobjects is not None ) :
    cplist = cplist [ : maxobjects ]
elif ( liststartindex is not None ) and ( maxobjects is not None ) :
    cplist = ( cplist [ liststartindex : liststartindex + maxobjects ] )
tasks = [ ( x , { 'fast_mode' : fast_mode , 'findercmap' : findercmap , 'finderconvolve' : finderconvolve , 'deredden_object' : deredden_object , 'custom_bandpasses' : custom_bandpasses , 'gaia_submit_timeout' : gaia_submit_timeout , 'gaia_submit_tries' : gaia_submit_tries , 'gaia_max_timeout' : gaia_max_timeout , 'gaia_mirror' : gaia_mirror , 'complete_query_later' : complete_query_later , 'lclistpkl' : lclistpkl , 'nbrradiusarcsec' : nbrradiusarcsec , 'maxnumneighbors' : maxnumneighbors , 'plotdpi' : plotdpi , 'findercachedir' : findercachedir , 'verbose' : verbose } ) for x in cplist ]
resultfutures = [ ]
results = [ ]
with ProcessPoolExecutor ( max_workers = nworkers ) as executor :
    resultfutures = executor . map ( cp_objectinfo_worker , tasks )
results = [ x for x in resultfutures ]
executor . shutdown ( )
return results


signature:
def parallel_update_objectinfo_cpdir ( cpdir , cpglob = 'checkplot-*.pkl*' , liststartindex = None , maxobjects = None , nworkers = NCPUS , fast_mode = False , findercmap = 'gray_r' , finderconvolve = None , deredden_object = True , custom_bandpasses = None , gaia_submit_timeout = 10.0 , gaia_submit_tries = 3 , gaia_max_timeout = 180.0 , gaia_mirror = None , complete_query_later = True , lclistpkl = None , nbrradiusarcsec = 60.0 , maxnumneighbors = 5 , plotdpi = 100 , findercachedir = '~/.astrobase/stamp-cache' , verbose = True ) :
    '''This updates the objectinfo for a directory of checkplot pickles .
 Useful in cases where a previous round of GAIA / finderchart / external catalog
 acquisition failed . This will preserve the following keys in the checkplots
 if they exist :
 comments
 varinfo
 objectinfo . objecttags
 Parameters
 cpdir : str
 The directory to look for checkplot pickles in .
 cpglob : str
 The UNIX fileglob to use when searching for checkplot pickle files .
 liststartindex : int
 The index of the input list to start working at .
 maxobjects : int
 The maximum number of objects to process in this run . Use this with
 ` liststartindex ` to effectively distribute working on a large list of
 input checkplot pickles over several sessions or machines .
 nworkers : int
 The number of parallel workers that will work on the checkplot
 update process .
 fast _ mode : bool or float
 This runs the external catalog operations in a " fast " mode , with short
 timeouts and not trying to hit external catalogs that take a long time
 to respond . See the docstring for
 ` checkplot . pkl _ utils . _ pkl _ finder _ objectinfo ` for details on how this
 works . If this is True , will run in " fast " mode with default timeouts ( 5
 seconds in most cases ) . If this is a float , will run in " fast " mode with
 the provided timeout value in seconds .
 findercmap : str or matplotlib . cm . Colormap object
 findercmap : str or matplotlib . cm . ColorMap object
 The Colormap object to use for the finder chart image .
 finderconvolve : astropy . convolution . Kernel object or None
 If not None , the Kernel object to use for convolving the finder image .
 deredden _ objects : bool
 If this is True , will use the 2MASS DUST service to get extinction
 coefficients in various bands , and then try to deredden the magnitudes
 and colors of the object already present in the checkplot ' s objectinfo
 dict .
 custom _ bandpasses : dict
 This is a dict used to provide custom bandpass definitions for any
 magnitude measurements in the objectinfo dict that are not automatically
 recognized by the ` varclass . starfeatures . color _ features ` function . See
 its docstring for details on the required format .
 gaia _ submit _ timeout : float
 Sets the timeout in seconds to use when submitting a request to look up
 the object ' s information to the GAIA service . Note that if ` fast _ mode `
 is set , this is ignored .
 gaia _ submit _ tries : int
 Sets the maximum number of times the GAIA services will be contacted to
 obtain this object ' s information . If ` fast _ mode ` is set , this is
 ignored , and the services will be contacted only once ( meaning that a
 failure to respond will be silently ignored and no GAIA data will be
 added to the checkplot ' s objectinfo dict ) .
 gaia _ max _ timeout : float
 Sets the timeout in seconds to use when waiting for the GAIA service to
 respond to our request for the object ' s information . Note that if
 ` fast _ mode ` is set , this is ignored .
 gaia _ mirror : str
 This sets the GAIA mirror to use . This is a key in the
 ` services . gaia . GAIA _ URLS ` dict which defines the URLs to hit for each
 mirror .
 complete _ query _ later : bool
 If this is True , saves the state of GAIA queries that are not yet
 complete when ` gaia _ max _ timeout ` is reached while waiting for the GAIA
 service to respond to our request . A later call for GAIA info on the
 same object will attempt to pick up the results from the existing query
 if it ' s completed . If ` fast _ mode ` is True , this is ignored .
 lclistpkl : dict or str
 If this is provided , must be a dict resulting from reading a catalog
 produced by the ` lcproc . catalogs . make _ lclist ` function or a str path
 pointing to the pickle file produced by that function . This catalog is
 used to find neighbors of the current object in the current light curve
 collection . Looking at neighbors of the object within the radius
 specified by ` nbrradiusarcsec ` is useful for light curves produced by
 instruments that have a large pixel scale , so are susceptible to
 blending of variability and potential confusion of neighbor variability
 with that of the actual object being looked at . If this is None , no
 neighbor lookups will be performed .
 nbrradiusarcsec : float
 The radius in arcseconds to use for a search conducted around the
 coordinates of this object to look for any potential confusion and
 blending of variability amplitude caused by their proximity .
 maxnumneighbors : int
 The maximum number of neighbors that will have their light curves and
 magnitudes noted in this checkplot as potential blends with the target
 object .
 plotdpi : int
 The resolution in DPI of the plots to generate in this function
 ( e . g . the finder chart , etc . )
 findercachedir : str
 The path to the astrobase cache directory for finder chart downloads
 from the NASA SkyView service .
 verbose : bool
 If True , will indicate progress and warn about potential problems .
 Returns
 list of str
 Paths to the updated checkplot pickle file .'''

implementation:
cplist = sorted ( glob . glob ( os . path . join ( cpdir , cpglob ) ) )
return parallel_update_objectinfo_cplist ( cplist , liststartindex = liststartindex , maxobjects = maxobjects , nworkers = nworkers , fast_mode = fast_mode , findercmap = findercmap , finderconvolve = finderconvolve , deredden_object = deredden_object , custom_bandpasses = custom_bandpasses , gaia_submit_timeout = gaia_submit_timeout , gaia_submit_tries = gaia_submit_tries , gaia_max_timeout = gaia_max_timeout , gaia_mirror = gaia_mirror , complete_query_later = complete_query_later , lclistpkl = lclistpkl , nbrradiusarcsec = nbrradiusarcsec , maxnumneighbors = maxnumneighbors , plotdpi = plotdpi , findercachedir = findercachedir , verbose = verbose )


signature:
def checkplot_infokey_worker ( task ) :
    '''This gets the required keys from the requested file .
 Parameters
 task : tuple
 Task is a two element tuple : :
 - task [ 0 ] is the dict to work on
 - task [ 1 ] is a list of lists of str indicating all the key address to
 extract items from the dict for
 Returns
 list
 This is a list of all of the items at the requested key addresses .'''

implementation:
cpf , keys = task
cpd = _read_checkplot_picklefile ( cpf )
resultkeys = [ ]
for k in keys :
    try :
        resultkeys . append ( _dict_get ( cpd , k ) )
    except Exception as e :
        resultkeys . append ( np . nan )
return resultkeys


signature:
def main ( ) :
    '''This is the main function of this script .
 The current script args are shown below : :
 Usage : checkplotlist [ - h ] [ - - search SEARCH ] [ - - sortby SORTBY ]
 [ - - filterby FILTERBY ] [ - - splitout SPLITOUT ]
 [ - - outprefix OUTPREFIX ] [ - - maxkeyworkers MAXKEYWORKERS ]
 { pkl , png } cpdir
 This makes a checkplot file list for use with the checkplot - viewer . html
 ( for checkplot PNGs ) or the checkplotserver . py ( for checkplot pickles )
 webapps .
 positional arguments :
 { pkl , png } type of checkplot to search for : pkl - > checkplot
 pickles , png - > checkplot PNGs
 cpdir directory containing the checkplots to process
 optional arguments :
 - h , - - help show this help message and exit
 - - search SEARCH file glob prefix to use when searching for checkplots ,
 default : ' * checkplot * ' , ( the extension is added
 automatically - . png or . pkl )
 - - sortby SORTBY the sort key and order to use when sorting
 - - filterby FILTERBY the filter key and condition to use when filtering .
 you can specify this multiple times to filter by
 several keys at once . all filters are joined with a
 logical AND operation in the order they ' re given .
 - - splitout SPLITOUT if there are more than SPLITOUT objects in the target
 directory ( default : 5000 ) , checkplotlist will split
 the output JSON into multiple files . this helps keep
 the checkplotserver webapp responsive .
 - - outprefix OUTPREFIX
 a prefix string to use for the output JSON file ( s ) .
 use this to separate out different sort orders or
 filter conditions , for example . if this isn ' t
 provided , but - - sortby or - - filterby are , will use
 those to figure out the output files ' prefixes
 - - maxkeyworkers MAXKEYWORKERS
 the number of parallel workers that will be launched
 to retrieve checkplot key values used for sorting and
 filtering ( default : 2)'''

implementation:
# # PARSE THE ARGS # #
aparser = argparse . ArgumentParser ( epilog = PROGEPILOG , description = PROGDESC , formatter_class = argparse . RawDescriptionHelpFormatter , )
aparser . add_argument ( 'cptype' , action = 'store' , choices = [ 'pkl' , 'png' ] , type = str , help = ( "type of checkplot to search for: pkl -> checkplot pickles, " "png -> checkplot PNGs" ) )
aparser . add_argument ( 'cpdir' , action = 'store' , type = str , help = ( "directory containing the checkplots to process" ) )
# TODO : here , make the - - search kwarg an array ( i . e . allow multiple search
# statements ) . the use of this will be to make checkplotserver able to load
# more than one checkplot per object ( i . e . different mag types - - epd
# vs . tfa - - or different bands - - r vs . i - - at the SAME time ) .
# TODO : we ' ll fix checkplotserver and its js so there ' s a vertical tab
# column between the left period / epoch / tags panel and main
# periodogram / phased - LCs panel on the right . the user will be able to flip
# between tabs to look at the object in all loaded alternative checkplots .
# TODO : need to also think about to sort / filter ; for now let ' s make it so
# the sorting works on a chosen checkplot search list , if we give - - search
# ' checkplot * iep1 ' and - - search ' checkplot * itf1 ' , specify - - sortpkls and
# - - filterpkls kwargs , which match the given globs for the - - search
# kwargs . e . g . we ' d specify - - sortpkls ' checkplot * iep1 ' to sort everything
# by the specified - - sortby values in those pickles .
# TODO : we ' ll have to change the output JSON so it ' s primarily by objectid
# instead of checkplot filenames . each objectid will have its own list of
# checkplots to use for the frontend .
aparser . add_argument ( '--search' , action = 'store' , default = '*checkplot*' , type = str , help = ( "file glob prefix to use when searching for checkplots, " "default: '%(default)s', " "(the extension is added automatically - .png or .pkl)" ) )
aparser . add_argument ( '--sortby' , action = 'store' , type = str , help = ( "the sort key and order to use when sorting" ) )
aparser . add_argument ( '--filterby' , action = 'append' , type = str , help = ( "the filter key and condition to use when filtering. " "you can specify this multiple times to filter by " "several keys at once. all filters are joined with a " "logical AND operation in the order they're given." ) )
aparser . add_argument ( '--splitout' , action = 'store' , type = int , default = 5000 , help = ( "if there are more than SPLITOUT objects in " "the target directory (default: %(default)s), " "checkplotlist will split the output JSON into multiple files. " "this helps keep the checkplotserver webapp responsive." ) )
aparser . add_argument ( '--outprefix' , action = 'store' , type = str , help = ( "a prefix string to use for the output JSON file(s). " "use this to separate out different sort orders " "or filter conditions, for example. " "if this isn't provided, but --sortby or --filterby are, " "will use those to figure out the output files' prefixes" ) )
aparser . add_argument ( '--maxkeyworkers' , action = 'store' , type = int , default = int ( CPU_COUNT / 4.0 ) , help = ( "the number of parallel workers that will be launched " "to retrieve checkplot key values used for " "sorting and filtering (default: %(default)s)" ) )
args = aparser . parse_args ( )
checkplotbasedir = args . cpdir
fileglob = args . search
splitout = args . splitout
outprefix = args . outprefix if args . outprefix else None
# see if there ' s a sorting order
if args . sortby :
    sortkey , sortorder = args . sortby . split ( '|' )
    if outprefix is None :
        outprefix = args . sortby
else :
    sortkey , sortorder = 'objectid' , 'asc'
# see if there ' s a filter condition
if args . filterby :
    filterkeys , filterconditions = [ ] , [ ]
    # load all the filters
    for filt in args . filterby :
        f = filt . split ( '|' )
        filterkeys . append ( f [ 0 ] )
        filterconditions . append ( f [ 1 ] )
    # generate the output file ' s prefix
    if outprefix is None :
        outprefix = '-' . join ( args . filterby )
    else :
        outprefix = '%s-%s' % ( '-' . join ( args . filterby ) , outprefix )
else :
    filterkeys , filterconditions = None , None
if args . cptype == 'pkl' :
    checkplotext = 'pkl'
elif args . cptype == 'png' :
    checkplotext = 'png'
else :
    print ( "unknown format for checkplots: %s! can't continue!" % args . cptype )
    sys . exit ( 1 )
# # NOW START WORKING # #
currdir = os . getcwd ( )
checkplotglob = os . path . join ( checkplotbasedir , '%s.%s' % ( fileglob , checkplotext ) )
print ( 'searching for checkplots: %s' % checkplotglob )
searchresults = glob . glob ( checkplotglob )
if searchresults :
    print ( 'found %s checkplot files in dir: %s' % ( len ( searchresults ) , checkplotbasedir ) )
    # see if we should sort the searchresults in some special order
    # this requires an arg on the commandline of the form :
    # ' < sortkey > - < asc | desc > '
    # where sortkey is some key in the checkplot pickle :
    # this can be a simple key : e . g . objectid
    # or it can be a composite key : e . g . varinfo . varfeatures . stetsonj
    # and sortorder is either ' asc ' or desc ' for ascending / descending sort
    # we only support a single condition conditions are of the form :
    # ' < filterkey > - < condition > @ < operand > ' where < condition > is one of : ' ge ' ,
    # ' gt ' , ' le ' , ' lt ' , ' eq ' and < operand > is a string , float , or int to use
    # when applying < condition >
    # first , take care of sort keys
    sortdone = False
    # second , take care of any filters
    filterok = False
    filterstatements = [ ]
    # make sure we only run these operations on checkplot pickles
    if ( ( args . cptype == 'pkl' ) and ( ( sortkey and sortorder ) or ( filterkeys and filterconditions ) ) ) :
        keystoget = [ ]
        # handle sorting
        if ( sortkey and sortorder ) :
            print ( 'sorting checkplot pickles by %s in order: %s' % ( sortkey , sortorder ) )
            # dereference the sort key
            sortkeys = sortkey . split ( '.' )
            # if there are any integers in the sortkeys strings , interpret
            # these to mean actual integer indexes of lists or integer keys
            # for dicts this allows us to move into arrays easily by
            # indexing them
            if sys . version_info [ : 2 ] < ( 3 , 4 ) :
                sortkeys = [ ( int ( x ) if x . isdigit ( ) else x ) for x in sortkeys ]
            else :
                sortkeys = [ ( int ( x ) if x . isdecimal ( ) else x ) for x in sortkeys ]
            keystoget . append ( sortkeys )
        # handle filtering
        if ( filterkeys and filterconditions ) :
            print ( 'filtering checkplot pickles by %s using: %s' % ( filterkeys , filterconditions ) )
            # add all the filtkeys to the list of keys to get
            for fdk in filterkeys : # dereference the filter dict key
                fdictkeys = fdk . split ( '.' )
                fdictkeys = [ ( int ( x ) if x . isdecimal ( ) else x ) for x in fdictkeys ]
                keystoget . append ( fdictkeys )
        print ( 'retrieving checkplot info using %s workers...' % args . maxkeyworkers )
        # launch the key retrieval
        pool = mp . Pool ( args . maxkeyworkers )
        tasks = [ ( x , keystoget ) for x in searchresults ]
        keytargets = pool . map ( checkplot_infokey_worker , tasks )
        pool . close ( )
        pool . join ( )
        # now that we have keys , we need to use them
        # keys will be returned in the order we put them into keystoget
        # if keystoget is more than 1 element , then it ' s either sorting
        # followed by filtering ( multiple ) . . .
        if ( len ( keystoget ) > 1 and ( sortkey and sortorder ) and ( filterkeys and filterconditions ) ) : # the first elem is sort key targets
            sorttargets = [ x [ 0 ] for x in keytargets ]
            # all of the rest are filter targets
            filtertargets = [ x [ 1 : ] for x in keytargets ]
        # otherwise , it ' s just multiple filters
        elif ( len ( keystoget ) > 1 and ( not ( sortkey and sortorder ) ) and ( filterkeys and filterconditions ) ) :
            sorttargets = None
            filtertargets = keytargets
        # if there ' s only one element in keytoget , then it ' s either just a
        # sort target . . .
        elif ( len ( keystoget ) == 1 and ( sortkey and sortorder ) and ( not ( filterkeys and filterconditions ) ) ) :
            sorttargets = keytargets
            filtertargets = None
        # or it ' s just a filter target
        elif ( len ( keystoget ) == 1 and ( filterkeys and filterconditions ) and ( not ( sortkey and sortorder ) ) ) :
            sorttargets = None
            filtertargets = keytargets
        # turn the search results into an np . array before we do
        # sorting / filtering
        searchresults = np . array ( searchresults )
        if sorttargets :
            sorttargets = np . ravel ( np . array ( sorttargets ) )
            sortind = np . argsort ( sorttargets )
            if sortorder == 'desc' :
                sortind = sortind [ : : - 1 ]
            # sort the search results in the requested order
            searchresults = searchresults [ sortind ]
            sortdone = True
        if filtertargets : # don ' t forget to also sort the filtertargets in the same order
        # as sorttargets so we can get the correct objects to filter .
        # now figure out the filter conditions : < condition > @ < operand >
        # where < condition > is one of : ' ge ' , ' gt ' , ' le ' , ' lt ' , ' eq ' and
        # < operand > is a string , float , or int to use when applying
        # < condition >
            finalfilterind = [ ]
            for ind , fcond in enumerate ( filterconditions ) :
                thisftarget = np . array ( [ x [ ind ] for x in filtertargets ] )
                if ( sortdone ) :
                    thisftarget = thisftarget [ sortind ]
                try :
                    foperator , foperand = fcond . split ( '@' )
                    foperator = FILTEROPS [ foperator ]
                    # we ' ll do a straight eval of the filter
                    # yes : this is unsafe
                    filterstr = ( 'np.isfinite(thisftarget) & (thisftarget %s %s)' % ( foperator , foperand ) )
                    filterind = eval ( filterstr )
                    # add this filter to the finalfilterind
                    finalfilterind . append ( filterind )
                    # update the filterstatements
                    filterstatements . append ( '%s %s %s' % ( filterkeys [ ind ] , foperator , foperand ) )
                except Exception as e :
                    print ( 'ERR! could not understand filter spec: %s' '\nexception was: %s' % ( args . filterby [ ind ] , e ) )
                    print ( 'WRN! not applying broken filter' )
            # DONE with evaluating each filter , get final results below
            # column stack the overall filter ind
            finalfilterind = np . column_stack ( finalfilterind )
            # do a logical AND across the rows
            finalfilterind = np . all ( finalfilterind , axis = 1 )
            # these are the final results after ANDing all the filters
            filterresults = searchresults [ finalfilterind ]
            # make sure we got some results
            if filterresults . size > 0 :
                print ( 'filters applied: %s -> objects found: %s ' % ( repr ( args . filterby ) , filterresults . size ) )
                searchresults = filterresults
                filterok = True
            # otherwise , applying all of the filters killed everything
            else :
                print ( 'WRN! filtering failed! %s -> ZERO objects found!' % ( repr ( args . filterby ) , ) )
                print ( 'WRN! not applying any filters' )
        # all done with sorting and filtering
        # turn the searchresults back into a list
        searchresults = searchresults . tolist ( )
        # if there ' s no special sort order defined , use the usual sort order
        # at the end after filtering
        if not ( sortkey and sortorder ) :
            print ( 'WRN! no special sort key and order/' 'filter key and condition specified, ' 'sorting checkplot pickles ' 'using usual alphanumeric sort...' )
            searchresults = sorted ( searchresults )
            sortkey = 'filename'
            sortorder = 'asc'
    nchunks = int ( len ( searchresults ) / splitout ) + 1
    searchchunks = [ searchresults [ x * splitout : x * splitout + splitout ] for x in range ( nchunks ) ]
    if nchunks > 1 :
        print ( 'WRN! more than %s checkplots in final list, ' 'splitting into %s chunks' % ( splitout , nchunks ) )
    # if the filter failed , zero out filterkey
    if ( filterkeys and filterconditions ) and not filterok :
        filterstatements = [ ]
    # generate the output
    for chunkind , chunk in enumerate ( searchchunks ) : # figure out if we need to split the JSON file
        outjson = os . path . abspath ( os . path . join ( currdir , '%scheckplot-filelist%s.json' % ( ( '%s-' % outprefix if outprefix is not None else '' ) , ( '-%02i' % chunkind if len ( searchchunks ) > 1 else '' ) , ) ) )
        outjson = outjson . replace ( '|' , '_' )
        outjson = outjson . replace ( '@' , '_' )
        # ask if the checkplot list JSON should be updated
        if os . path . exists ( outjson ) :
            if sys . version_info [ : 2 ] < ( 3 , 0 ) :
                answer = raw_input ( 'There is an existing ' 'checkplot list file in this ' 'directory:\n    %s\nDo you want to ' 'overwrite it completely? (default: no) [y/n] ' % outjson )
            else :
                answer = input ( 'There is an existing ' 'checkplot list file in this ' 'directory:\n    %s\nDo you want to ' 'overwrite it completely? (default: no) [y/n] ' % outjson )
            # if it ' s OK to overwrite , then do so
            if answer and answer == 'y' :
                with open ( outjson , 'w' ) as outfd :
                    print ( 'WRN! completely overwriting ' 'existing checkplot list %s' % outjson )
                    outdict = { 'checkplots' : chunk , 'nfiles' : len ( chunk ) , 'sortkey' : sortkey , 'sortorder' : sortorder , 'filterstatements' : filterstatements }
                    json . dump ( outdict , outfd )
            # if it ' s not OK to overwrite , then
            else : # read in the outjson , and add stuff to it for objects that
            # don ' t have an entry
                print ( 'only updating existing checkplot list ' 'file with any new checkplot pickles' )
                with open ( outjson , 'r' ) as infd :
                    indict = json . load ( infd )
                # update the checkplot list , sortorder , and sortkey only
                indict [ 'checkplots' ] = chunk
                indict [ 'nfiles' ] = len ( chunk )
                indict [ 'sortkey' ] = sortkey
                indict [ 'sortorder' ] = sortorder
                indict [ 'filterstatements' ] = filterstatements
                # write the updated to back to the file
                with open ( outjson , 'w' ) as outfd :
                    json . dump ( indict , outfd )
        # if this is a new output file
        else :
            with open ( outjson , 'w' ) as outfd :
                outdict = { 'checkplots' : chunk , 'nfiles' : len ( chunk ) , 'sortkey' : sortkey , 'sortorder' : sortorder , 'filterstatements' : filterstatements }
                json . dump ( outdict , outfd )
        if os . path . exists ( outjson ) :
            print ( 'checkplot file list written to %s' % outjson )
        else :
            print ( 'ERR! writing the checkplot file list failed!' )
else :
    print ( 'ERR! no checkplots found in %s' % checkplotbasedir )


signature:
def _gaussian ( x , amp , loc , std ) :
    '''This is a simple gaussian .
 Parameters
 x : np . array
 The items at which the Gaussian is evaluated .
 amp : float
 The amplitude of the Gaussian .
 loc : float
 The central value of the Gaussian .
 std : float
 The standard deviation of the Gaussian .
 Returns
 np . array
 Returns the Gaussian evaluated at the items in ` x ` , using the provided
 parameters of ` amp ` , ` loc ` , and ` std ` .'''

implementation: return amp * np . exp ( - ( ( x - loc ) * ( x - loc ) ) / ( 2.0 * std * std ) )


signature:
def _double_inverted_gaussian ( x , amp1 , loc1 , std1 , amp2 , loc2 , std2 ) :
    '''This is a double inverted gaussian .
 Parameters
 x : np . array
 The items at which the Gaussian is evaluated .
 amp1 , amp2 : float
 The amplitude of Gaussian 1 and Gaussian 2.
 loc1 , loc2 : float
 The central value of Gaussian 1 and Gaussian 2.
 std1 , std2 : float
 The standard deviation of Gaussian 1 and Gaussian 2.
 Returns
 np . array
 Returns a double inverted Gaussian function evaluated at the items in
 ` x ` , using the provided parameters of ` amp ` , ` loc ` , and ` std ` for two
 component Gaussians 1 and 2.'''

implementation:
gaussian1 = - _gaussian ( x , amp1 , loc1 , std1 )
gaussian2 = - _gaussian ( x , amp2 , loc2 , std2 )
return gaussian1 + gaussian2


signature:
def invgauss_eclipses_func ( ebparams , times , mags , errs ) :
    '''This returns a double eclipse shaped function .
 Suitable for first order modeling of eclipsing binaries .
 Parameters
 ebparams : list of float
 This contains the parameters for the eclipsing binary : :
 ebparams = [ period ( time ) ,
 epoch ( time ) ,
 pdepth : primary eclipse depth ( mags ) ,
 pduration : primary eclipse duration ( phase ) ,
 psdepthratio : primary - secondary eclipse depth ratio ,
 secondaryphase : center phase of the secondary eclipse ]
 ` period ` is the period in days .
 ` epoch ` is the time of minimum in JD .
 ` pdepth ` is the depth of the primary eclipse .
 - for magnitudes - > pdepth should be < 0
 - for fluxes - > pdepth should be > 0
 ` pduration ` is the length of the primary eclipse in phase .
 ` psdepthratio ` is the ratio in the eclipse depths :
 ` depth _ secondary / depth _ primary ` . This is generally the same as the ratio
 of the ` T _ effs ` of the two stars .
 ` secondaryphase ` is the phase at which the minimum of the secondary
 eclipse is located . This effectively parameterizes eccentricity .
 All of these will then have fitted values after the fit is done .
 times , mags , errs : np . array
 The input time - series of measurements and associated errors for which
 the eclipse model will be generated . The times will be used to generate
 model mags , and the input ` times ` , ` mags ` , and ` errs ` will be resorted
 by model phase and returned .
 Returns
 ( modelmags , phase , ptimes , pmags , perrs ) : tuple
 Returns the model mags and phase values . Also returns the input ` times ` ,
 ` mags ` , and ` errs ` sorted by the model ' s phase .'''

implementation:
( period , epoch , pdepth , pduration , depthratio , secondaryphase ) = ebparams
# generate the phases
iphase = ( times - epoch ) / period
iphase = iphase - np . floor ( iphase )
phasesortind = np . argsort ( iphase )
phase = iphase [ phasesortind ]
ptimes = times [ phasesortind ]
pmags = mags [ phasesortind ]
perrs = errs [ phasesortind ]
zerolevel = np . median ( pmags )
modelmags = np . full_like ( phase , zerolevel )
primaryecl_amp = - pdepth
secondaryecl_amp = - pdepth * depthratio
primaryecl_std = pduration / 5.0
# we use 5 - sigma as full - width - > duration
secondaryecl_std = pduration / 5.0
# secondary eclipse has the same duration
halfduration = pduration / 2.0
# phase indices
primary_eclipse_ingress = ( ( phase >= ( 1.0 - halfduration ) ) & ( phase <= 1.0 ) )
primary_eclipse_egress = ( ( phase >= 0.0 ) & ( phase <= halfduration ) )
secondary_eclipse_phase = ( ( phase >= ( secondaryphase - halfduration ) ) & ( phase <= ( secondaryphase + halfduration ) ) )
# put in the eclipses
modelmags [ primary_eclipse_ingress ] = ( zerolevel + _gaussian ( phase [ primary_eclipse_ingress ] , primaryecl_amp , 1.0 , primaryecl_std ) )
modelmags [ primary_eclipse_egress ] = ( zerolevel + _gaussian ( phase [ primary_eclipse_egress ] , primaryecl_amp , 0.0 , primaryecl_std ) )
modelmags [ secondary_eclipse_phase ] = ( zerolevel + _gaussian ( phase [ secondary_eclipse_phase ] , secondaryecl_amp , secondaryphase , secondaryecl_std ) )
return modelmags , phase , ptimes , pmags , perrs


signature:
def invgauss_eclipses_residual ( ebparams , times , mags , errs ) :
    '''This returns the residual between the modelmags and the actual mags .
 Parameters
 ebparams : list of float
 This contains the parameters for the eclipsing binary : :
 ebparams = [ period ( time ) ,
 epoch ( time ) ,
 pdepth : primary eclipse depth ( mags ) ,
 pduration : primary eclipse duration ( phase ) ,
 psdepthratio : primary - secondary eclipse depth ratio ,
 secondaryphase : center phase of the secondary eclipse ]
 ` period ` is the period in days .
 ` epoch ` is the time of minimum in JD .
 ` pdepth ` is the depth of the primary eclipse .
 - for magnitudes - > ` pdepth ` should be < 0
 - for fluxes - > ` pdepth ` should be > 0
 ` pduration ` is the length of the primary eclipse in phase .
 ` psdepthratio ` is the ratio in the eclipse depths :
 ` depth _ secondary / depth _ primary ` . This is generally the same as the ratio
 of the ` T _ effs ` of the two stars .
 ` secondaryphase ` is the phase at which the minimum of the secondary
 eclipse is located . This effectively parameterizes eccentricity .
 All of these will then have fitted values after the fit is done .
 times , mags , errs : np . array
 The input time - series of measurements and associated errors for which
 the eclipse model will be generated . The times will be used to generate
 model mags , and the input ` times ` , ` mags ` , and ` errs ` will be resorted
 by model phase and returned .
 Returns
 np . array
 The residuals between the input ` mags ` and generated ` modelmags ` ,
 weighted by the measurement errors in ` errs ` .'''

implementation:
modelmags , phase , ptimes , pmags , perrs = ( invgauss_eclipses_func ( ebparams , times , mags , errs ) )
# this is now a weighted residual taking into account the measurement err
return ( pmags - modelmags ) / perrs


signature:
def convert_constants ( jmag , hmag , kmag , cjhk , cjh , cjk , chk , cj , ch , ck ) :
    '''This converts between JHK and BVRI / SDSS mags .
 Not meant to be used directly . See the functions below for more sensible
 interface . This function does the grunt work of converting from JHK to
 either BVRI or SDSS ugriz . while taking care of missing values for any of
 jmag , hmag , or kmag .
 Parameters
 jmag , hmag , kmag : float
 2MASS J , H , Ks mags to use to convert .
 cjhk , cjh , cjk , chk , cj , ch , ck : lists
 Constants to use when converting .
 Returns
 float
 The converted magnitude in SDSS or BVRI system .'''

implementation:
if jmag is not None :
    if hmag is not None :
        if kmag is not None :
            return cjhk [ 0 ] + cjhk [ 1 ] * jmag + cjhk [ 2 ] * hmag + cjhk [ 3 ] * kmag
        else :
            return cjh [ 0 ] + cjh [ 1 ] * jmag + cjh [ 2 ] * hmag
    else :
        if kmag is not None :
            return cjk [ 0 ] + cjk [ 1 ] * jmag + cjk [ 2 ] * kmag
        else :
            return cj [ 0 ] + cj [ 1 ] * jmag
else :
    if hmag is not None :
        if kmag is not None :
            return chk [ 0 ] + chk [ 1 ] * hmag + chk [ 2 ] * kmag
        else :
            return ch [ 0 ] + ch [ 1 ] * hmag
    else :
        if kmag is not None :
            return ck [ 0 ] + ck [ 1 ] * kmag
        else :
            return np . nan


signature:
def jhk_to_bmag ( jmag , hmag , kmag ) :
    '''Converts given J , H , Ks mags to a B magnitude value .
 Parameters
 jmag , hmag , kmag : float
 2MASS J , H , Ks mags of the object .
 Returns
 float
 The converted B band magnitude .'''

implementation: return convert_constants ( jmag , hmag , kmag , BJHK , BJH , BJK , BHK , BJ , BH , BK )


signature:
def jhk_to_vmag ( jmag , hmag , kmag ) :
    '''Converts given J , H , Ks mags to a V magnitude value .
 Parameters
 jmag , hmag , kmag : float
 2MASS J , H , Ks mags of the object .
 Returns
 float
 The converted V band magnitude .'''

implementation: return convert_constants ( jmag , hmag , kmag , VJHK , VJH , VJK , VHK , VJ , VH , VK )


signature:
def jhk_to_rmag ( jmag , hmag , kmag ) :
    '''Converts given J , H , Ks mags to an R magnitude value .
 Parameters
 jmag , hmag , kmag : float
 2MASS J , H , Ks mags of the object .
 Returns
 float
 The converted R band magnitude .'''

implementation: return convert_constants ( jmag , hmag , kmag , RJHK , RJH , RJK , RHK , RJ , RH , RK )


signature:
def jhk_to_imag ( jmag , hmag , kmag ) :
    '''Converts given J , H , Ks mags to an I magnitude value .
 Parameters
 jmag , hmag , kmag : float
 2MASS J , H , Ks mags of the object .
 Returns
 float
 The converted I band magnitude .'''

implementation: return convert_constants ( jmag , hmag , kmag , IJHK , IJH , IJK , IHK , IJ , IH , IK )


signature:
def jhk_to_sdssu ( jmag , hmag , kmag ) :
    '''Converts given J , H , Ks mags to an SDSS u magnitude value .
 Parameters
 jmag , hmag , kmag : float
 2MASS J , H , Ks mags of the object .
 Returns
 float
 The converted SDSS u band magnitude .'''

implementation: return convert_constants ( jmag , hmag , kmag , SDSSU_JHK , SDSSU_JH , SDSSU_JK , SDSSU_HK , SDSSU_J , SDSSU_H , SDSSU_K )


signature:
def jhk_to_sdssg ( jmag , hmag , kmag ) :
    '''Converts given J , H , Ks mags to an SDSS g magnitude value .
 Parameters
 jmag , hmag , kmag : float
 2MASS J , H , Ks mags of the object .
 Returns
 float
 The converted SDSS g band magnitude .'''

implementation: return convert_constants ( jmag , hmag , kmag , SDSSG_JHK , SDSSG_JH , SDSSG_JK , SDSSG_HK , SDSSG_J , SDSSG_H , SDSSG_K )


signature:
def jhk_to_sdssr ( jmag , hmag , kmag ) :
    '''Converts given J , H , Ks mags to an SDSS r magnitude value .
 Parameters
 jmag , hmag , kmag : float
 2MASS J , H , Ks mags of the object .
 Returns
 float
 The converted SDSS r band magnitude .'''

implementation: return convert_constants ( jmag , hmag , kmag , SDSSR_JHK , SDSSR_JH , SDSSR_JK , SDSSR_HK , SDSSR_J , SDSSR_H , SDSSR_K )


signature:
def jhk_to_sdssi ( jmag , hmag , kmag ) :
    '''Converts given J , H , Ks mags to an SDSS i magnitude value .
 Parameters
 jmag , hmag , kmag : float
 2MASS J , H , Ks mags of the object .
 Returns
 float
 The converted SDSS i band magnitude .'''

implementation: return convert_constants ( jmag , hmag , kmag , SDSSI_JHK , SDSSI_JH , SDSSI_JK , SDSSI_HK , SDSSI_J , SDSSI_H , SDSSI_K )


signature:
def jhk_to_sdssz ( jmag , hmag , kmag ) :
    '''Converts given J , H , Ks mags to an SDSS z magnitude value .
 Parameters
 jmag , hmag , kmag : float
 2MASS J , H , Ks mags of the object .
 Returns
 float
 The converted SDSS z band magnitude .'''

implementation: return convert_constants ( jmag , hmag , kmag , SDSSZ_JHK , SDSSZ_JH , SDSSZ_JK , SDSSZ_HK , SDSSZ_J , SDSSZ_H , SDSSZ_K )