signature:
def fits_finder_chart ( fitsfile , outfile , fitsext = 0 , wcsfrom = None , scale = ZScaleInterval ( ) , stretch = LinearStretch ( ) , colormap = plt . cm . gray_r , findersize = None , finder_coordlimits = None , overlay_ra = None , overlay_decl = None , overlay_pltopts = { 'marker' : 'o' , 'markersize' : 10.0 , 'markerfacecolor' : 'none' , 'markeredgewidth' : 2.0 , 'markeredgecolor' : 'red' } , overlay_zoomcontain = False , grid = False , gridcolor = 'k' ) :
    '''This makes a finder chart for a given FITS with an optional object
 position overlay .
 Parameters
 fitsfile : str
 ` fitsfile ` is the FITS file to use to make the finder chart .
 outfile : str
 ` outfile ` is the name of the output file . This can be a png or pdf or
 whatever else matplotlib can write given a filename and extension .
 fitsext : int
 Sets the FITS extension in ` fitsfile ` to use to extract the image array
 from .
 wcsfrom : str or None
 If ` wcsfrom ` is None , the WCS to transform the RA / Dec to pixel x / y will
 be taken from the FITS header of ` fitsfile ` . If this is not None , it
 must be a FITS or similar file that contains a WCS header in its first
 extension .
 scale : astropy . visualization . Interval object
 ` scale ` sets the normalization for the FITS pixel values . This is an
 astropy . visualization Interval object .
 See http : / / docs . astropy . org / en / stable / visualization / normalization . html
 for details on ` scale ` and ` stretch ` objects .
 stretch : astropy . visualization . Stretch object
 ` stretch ` sets the stretch function for mapping FITS pixel values to
 output pixel values . This is an astropy . visualization Stretch object .
 See http : / / docs . astropy . org / en / stable / visualization / normalization . html
 for details on ` scale ` and ` stretch ` objects .
 colormap : matplotlib Colormap object
 ` colormap ` is a matplotlib color map object to use for the output image .
 findersize : None or tuple of two ints
 If ` findersize ` is None , the output image size will be set by the NAXIS1
 and NAXIS2 keywords in the input ` fitsfile ` FITS header . Otherwise ,
 ` findersize ` must be a tuple with the intended x and y size of the image
 in inches ( all output images will use a DPI = 100 ) .
 finder _ coordlimits : list of four floats or None
 If not None , ` finder _ coordlimits ` sets x and y limits for the plot ,
 effectively zooming it in if these are smaller than the dimensions of
 the FITS image . This should be a list of the form : [ minra , maxra ,
 mindecl , maxdecl ] all in decimal degrees .
 overlay _ ra , overlay _ decl : np . array or None
 ` overlay _ ra ` and ` overlay _ decl ` are ndarrays containing the RA and Dec
 values to overplot on the image as an overlay . If these are both None ,
 then no overlay will be plotted .
 overlay _ pltopts : dict
 ` overlay _ pltopts ` controls how the overlay points will be plotted . This
 a dict with standard matplotlib marker , etc . kwargs as key - val pairs ,
 e . g . ' markersize ' , ' markerfacecolor ' , etc . The default options make red
 outline circles at the location of each object in the overlay .
 overlay _ zoomcontain : bool
 ` overlay _ zoomcontain ` controls if the finder chart will be zoomed to
 just contain the overlayed points . Everything outside the footprint of
 these points will be discarded .
 grid : bool
 ` grid ` sets if a grid will be made on the output image .
 gridcolor : str
 ` gridcolor ` sets the color of the grid lines . This is a usual matplotib
 color spec string .
 Returns
 str or None
 The filename of the generated output image if successful . None
 otherwise .'''

implementation:
# read in the FITS file
if wcsfrom is None :
    hdulist = pyfits . open ( fitsfile )
    img , hdr = hdulist [ fitsext ] . data , hdulist [ fitsext ] . header
    hdulist . close ( )
    frameshape = ( hdr [ 'NAXIS1' ] , hdr [ 'NAXIS2' ] )
    w = WCS ( hdr )
elif os . path . exists ( wcsfrom ) :
    hdulist = pyfits . open ( fitsfile )
    img , hdr = hdulist [ fitsext ] . data , hdulist [ fitsext ] . header
    hdulist . close ( )
    frameshape = ( hdr [ 'NAXIS1' ] , hdr [ 'NAXIS2' ] )
    w = WCS ( wcsfrom )
else :
    LOGERROR ( 'could not determine WCS info for input FITS: %s' % fitsfile )
    return None
# use the frame shape to set the output PNG ' s dimensions
if findersize is None :
    fig = plt . figure ( figsize = ( frameshape [ 0 ] / 100.0 , frameshape [ 1 ] / 100.0 ) )
else :
    fig = plt . figure ( figsize = findersize )
# set the coord limits if zoomcontain is True
# we ' ll leave 30 arcseconds of padding on each side
if ( overlay_zoomcontain and overlay_ra is not None and overlay_decl is not None ) :
    finder_coordlimits = [ overlay_ra . min ( ) - 30.0 / 3600.0 , overlay_ra . max ( ) + 30.0 / 3600.0 , overlay_decl . min ( ) - 30.0 / 3600.0 , overlay_decl . max ( ) + 30.0 / 3600.0 ]
# set the coordinate limits if provided
if finder_coordlimits and isinstance ( finder_coordlimits , ( list , tuple ) ) :
    minra , maxra , mindecl , maxdecl = finder_coordlimits
    cntra , cntdecl = ( minra + maxra ) / 2.0 , ( mindecl + maxdecl ) / 2.0
    pixelcoords = w . all_world2pix ( [ [ minra , mindecl ] , [ maxra , maxdecl ] , [ cntra , cntdecl ] ] , 1 )
    x1 , y1 , x2 , y2 = ( int ( pixelcoords [ 0 , 0 ] ) , int ( pixelcoords [ 0 , 1 ] ) , int ( pixelcoords [ 1 , 0 ] ) , int ( pixelcoords [ 1 , 1 ] ) )
    xmin = x1 if x1 < x2 else x2
    xmax = x2 if x2 > x1 else x1
    ymin = y1 if y1 < y2 else y2
    ymax = y2 if y2 > y1 else y1
    # create a new WCS with the same transform but new center coordinates
    whdr = w . to_header ( )
    whdr [ 'CRPIX1' ] = ( xmax - xmin ) / 2
    whdr [ 'CRPIX2' ] = ( ymax - ymin ) / 2
    whdr [ 'CRVAL1' ] = cntra
    whdr [ 'CRVAL2' ] = cntdecl
    whdr [ 'NAXIS1' ] = xmax - xmin
    whdr [ 'NAXIS2' ] = ymax - ymin
    w = WCS ( whdr )
else :
    xmin , xmax , ymin , ymax = 0 , hdr [ 'NAXIS2' ] , 0 , hdr [ 'NAXIS1' ]
# add the axes with the WCS projection
# this should automatically handle subimages because we fix the WCS
# appropriately above for these
fig . add_subplot ( 111 , projection = w )
if scale is not None and stretch is not None :
    norm = ImageNormalize ( img , interval = scale , stretch = stretch )
    plt . imshow ( img [ ymin : ymax , xmin : xmax ] , origin = 'lower' , cmap = colormap , norm = norm )
else :
    plt . imshow ( img [ ymin : ymax , xmin : xmax ] , origin = 'lower' , cmap = colormap )
# handle additional options
if grid :
    plt . grid ( color = gridcolor , ls = 'solid' , lw = 1.0 )
# handle the object overlay
if overlay_ra is not None and overlay_decl is not None :
    our_pltopts = dict ( transform = plt . gca ( ) . get_transform ( 'fk5' ) , marker = 'o' , markersize = 10.0 , markerfacecolor = 'none' , markeredgewidth = 2.0 , markeredgecolor = 'red' , rasterized = True , linestyle = 'none' )
    if overlay_pltopts is not None and isinstance ( overlay_pltopts , dict ) :
        our_pltopts . update ( overlay_pltopts )
    plt . gca ( ) . set_autoscale_on ( False )
    plt . gca ( ) . plot ( overlay_ra , overlay_decl , ** our_pltopts )
plt . xlabel ( 'Right Ascension [deg]' )
plt . ylabel ( 'Declination [deg]' )
# get the x and y axes objects to fix the ticks
xax = plt . gca ( ) . coords [ 0 ]
yax = plt . gca ( ) . coords [ 1 ]
yax . set_major_formatter ( 'd.ddd' )
xax . set_major_formatter ( 'd.ddd' )
# save the figure
plt . savefig ( outfile , dpi = 100.0 )
plt . close ( 'all' )
return outfile


signature:
def plot_periodbase_lsp ( lspinfo , outfile = None , plotdpi = 100 ) :
    '''Makes a plot of periodograms obtained from ` periodbase ` functions .
 This takes the output dict produced by any ` astrobase . periodbase `
 period - finder function or a pickle filename containing such a dict and makes
 a periodogram plot .
 Parameters
 lspinfo : dict or str
 If lspinfo is a dict , it must be a dict produced by an
 ` astrobase . periodbase ` period - finder function or a dict from your own
 period - finder function or routine that is of the form below with at
 least these keys : :
 { ' periods ' : np . array of all periods searched by the period - finder ,
 ' lspvals ' : np . array of periodogram power value for each period ,
 ' bestperiod ' : a float value that is the period with the highest
 peak in the periodogram , i . e . the most - likely actual
 period ,
 ' method ' : a three - letter code naming the period - finder used ; must
 be one of the keys in the ` METHODLABELS ` dict above ,
 ' nbestperiods ' : a list of the periods corresponding to periodogram
 peaks ( ` nbestlspvals ` below ) to annotate on the
 periodogram plot so they can be called out
 visually ,
 ' nbestlspvals ' : a list of the power values associated with
 periodogram peaks to annotate on the periodogram
 plot so they can be called out visually ; should be
 the same length as ` nbestperiods ` above }
 If lspinfo is a str , then it must be a path to a pickle file that
 contains a dict of the form described above .
 outfile : str or None
 If this is a str , will write the periodogram plot to the file specified
 by this string . If this is None , will write to a file called
 ' lsp - plot . png ' in the current working directory .
 plotdpi : int
 Sets the resolution in DPI of the output periodogram plot PNG file .
 Returns
 str
 Absolute path to the periodogram plot file created .'''

implementation:
# get the lspinfo from a pickle file transparently
if isinstance ( lspinfo , str ) and os . path . exists ( lspinfo ) :
    LOGINFO ( 'loading LSP info from pickle %s' % lspinfo )
    with open ( lspinfo , 'rb' ) as infd :
        lspinfo = pickle . load ( infd )
try : # get the things to plot out of the data
    periods = lspinfo [ 'periods' ]
    lspvals = lspinfo [ 'lspvals' ]
    bestperiod = lspinfo [ 'bestperiod' ]
    lspmethod = lspinfo [ 'method' ]
    # make the LSP plot on the first subplot
    plt . plot ( periods , lspvals )
    plt . xscale ( 'log' , basex = 10 )
    plt . xlabel ( 'Period [days]' )
    plt . ylabel ( PLOTYLABELS [ lspmethod ] )
    plottitle = '%s best period: %.6f d' % ( METHODSHORTLABELS [ lspmethod ] , bestperiod )
    plt . title ( plottitle )
    # show the best five peaks on the plot
    for bestperiod , bestpeak in zip ( lspinfo [ 'nbestperiods' ] , lspinfo [ 'nbestlspvals' ] ) :
        plt . annotate ( '%.6f' % bestperiod , xy = ( bestperiod , bestpeak ) , xycoords = 'data' , xytext = ( 0.0 , 25.0 ) , textcoords = 'offset points' , arrowprops = dict ( arrowstyle = "->" ) , fontsize = 'x-small' )
    # make a grid
    plt . grid ( color = '#a9a9a9' , alpha = 0.9 , zorder = 0 , linewidth = 1.0 , linestyle = ':' )
    # make the figure
    if outfile and isinstance ( outfile , str ) :
        if outfile . endswith ( '.png' ) :
            plt . savefig ( outfile , bbox_inches = 'tight' , dpi = plotdpi )
        else :
            plt . savefig ( outfile , bbox_inches = 'tight' )
        plt . close ( )
        return os . path . abspath ( outfile )
    elif dispok :
        plt . show ( )
        plt . close ( )
        return
    else :
        LOGWARNING ( 'no output file specified and no $DISPLAY set, ' 'saving to lsp-plot.png in current directory' )
        outfile = 'lsp-plot.png'
        plt . savefig ( outfile , bbox_inches = 'tight' , dpi = plotdpi )
        plt . close ( )
        return os . path . abspath ( outfile )
except Exception as e :
    LOGEXCEPTION ( 'could not plot this LSP, appears to be empty' )
    return


signature:
def read_hatpi_textlc ( lcfile ) :
    '''This reads in a textlc that is complete up to the TFA stage .'''

implementation:
if 'TF1' in lcfile :
    thiscoldefs = COLDEFS + [ ( 'itf1' , float ) ]
elif 'TF2' in lcfile :
    thiscoldefs = COLDEFS + [ ( 'itf2' , float ) ]
elif 'TF3' in lcfile :
    thiscoldefs = COLDEFS + [ ( 'itf3' , float ) ]
LOGINFO ( 'reading %s' % lcfile )
if lcfile . endswith ( '.gz' ) :
    infd = gzip . open ( lcfile , 'r' )
else :
    infd = open ( lcfile , 'r' )
with infd :
    lclines = infd . read ( ) . decode ( ) . split ( '\n' )
    lclines = [ x . split ( ) for x in lclines if ( '#' not in x and len ( x ) > 0 ) ]
    ndet = len ( lclines )
    if ndet > 0 :
        lccols = list ( zip ( * lclines ) )
        lcdict = { x [ 0 ] : y for ( x , y ) in zip ( thiscoldefs , lccols ) }
        # convert to ndarray
        for col in thiscoldefs :
            lcdict [ col [ 0 ] ] = np . array ( [ col [ 1 ] ( x ) for x in lcdict [ col [ 0 ] ] ] )
    else :
        lcdict = { }
        LOGWARNING ( 'no detections in %s' % lcfile )
        # convert to empty ndarrays
        for col in thiscoldefs :
            lcdict [ col [ 0 ] ] = np . array ( [ ] )
    # add the object ' s name to the lcdict
    hatid = HATIDREGEX . findall ( lcfile )
    lcdict [ 'objectid' ] = hatid [ 0 ] if hatid else 'unknown object'
    # add the columns to the lcdict
    lcdict [ 'columns' ] = [ x [ 0 ] for x in thiscoldefs ]
    # add some basic info similar to usual HATLCs
    lcdict [ 'objectinfo' ] = { 'ndet' : ndet , 'hatid' : hatid [ 0 ] if hatid else 'unknown object' , 'network' : 'HP' , }
    # break out the { stationid } - { framenum } { framesub } _ { ccdnum } framekey
    # into separate columns
    framekeyelems = FRAMEREGEX . findall ( '\n' . join ( lcdict [ 'frk' ] ) )
    lcdict [ 'stf' ] = np . array ( [ ( int ( x [ 0 ] ) if x [ 0 ] . isdigit ( ) else np . nan ) for x in framekeyelems ] )
    lcdict [ 'cfn' ] = np . array ( [ ( int ( x [ 1 ] ) if x [ 0 ] . isdigit ( ) else np . nan ) for x in framekeyelems ] )
    lcdict [ 'cfs' ] = np . array ( [ x [ 2 ] for x in framekeyelems ] )
    lcdict [ 'ccd' ] = np . array ( [ ( int ( x [ 3 ] ) if x [ 0 ] . isdigit ( ) else np . nan ) for x in framekeyelems ] )
    # update the column list with these columns
    lcdict [ 'columns' ] . extend ( [ 'stf' , 'cfn' , 'cfs' , 'ccd' ] )
    # add more objectinfo : ' stations ' , etc .
    lcdict [ 'objectinfo' ] [ 'network' ] = 'HP'
    lcdict [ 'objectinfo' ] [ 'stations' ] = [ 'HP%s' % x for x in np . unique ( lcdict [ 'stf' ] ) . tolist ( ) ]
return lcdict


signature:
def lcdict_to_pickle ( lcdict , outfile = None ) :
    '''This just writes the lcdict to a pickle .
 If outfile is None , then will try to get the name from the
 lcdict [ ' objectid ' ] and write to < objectid > - hptxtlc . pkl . If that fails , will
 write to a file named hptxtlc . pkl ' .'''

implementation:
if not outfile and lcdict [ 'objectid' ] :
    outfile = '%s-hplc.pkl' % lcdict [ 'objectid' ]
elif not outfile and not lcdict [ 'objectid' ] :
    outfile = 'hplc.pkl'
with open ( outfile , 'wb' ) as outfd :
    pickle . dump ( lcdict , outfd , protocol = pickle . HIGHEST_PROTOCOL )
if os . path . exists ( outfile ) :
    LOGINFO ( 'lcdict for object: %s -> %s OK' % ( lcdict [ 'objectid' ] , outfile ) )
    return outfile
else :
    LOGERROR ( 'could not make a pickle for this lcdict!' )
    return None


signature:
def read_hatpi_pklc ( lcfile ) :
    '''This just reads a pickle LC . Returns an lcdict .'''

implementation:
try :
    if lcfile . endswith ( '.gz' ) :
        infd = gzip . open ( lcfile , 'rb' )
    else :
        infd = open ( lcfile , 'rb' )
    lcdict = pickle . load ( infd )
    infd . close ( )
    return lcdict
except UnicodeDecodeError :
    if lcfile . endswith ( '.gz' ) :
        infd = gzip . open ( lcfile , 'rb' )
    else :
        infd = open ( lcfile , 'rb' )
    LOGWARNING ( 'pickle %s was probably from Python 2 ' 'and failed to load without using "latin1" encoding. ' 'This is probably a numpy issue: ' 'http://stackoverflow.com/q/11305790' % lcfile )
    lcdict = pickle . load ( infd , encoding = 'latin1' )
    infd . close ( )
    return lcdict


signature:
def concatenate_textlcs ( lclist , sortby = 'rjd' , normalize = True ) :
    '''This concatenates a list of light curves .
 Does not care about overlaps or duplicates . The light curves must all be
 from the same aperture .
 The intended use is to concatenate light curves across CCDs or instrument
 changes for a single object . These can then be normalized later using
 standard astrobase tools to search for variablity and / or periodicity .
 sortby is a column to sort the final concatenated light curve by in
 ascending order .
 If normalize is True , then each light curve ' s magnitude columns are
 normalized to zero .
 The returned lcdict has an extra column : ' lcn ' that tracks which measurement
 belongs to which input light curve . This can be used with
 lcdict [ ' concatenated ' ] which relates input light curve index to input light
 curve filepath . Finally , there is an ' nconcatenated ' key in the lcdict that
 contains the total number of concatenated light curves .'''

implementation:
# read the first light curve
lcdict = read_hatpi_textlc ( lclist [ 0 ] )
# track which LC goes where
# initial LC
lccounter = 0
lcdict [ 'concatenated' ] = { lccounter : os . path . abspath ( lclist [ 0 ] ) }
lcdict [ 'lcn' ] = np . full_like ( lcdict [ 'rjd' ] , lccounter )
# normalize if needed
if normalize :
    for col in MAGCOLS :
        if col in lcdict :
            thismedval = np . nanmedian ( lcdict [ col ] )
            # handle fluxes
            if col in ( 'ifl1' , 'ifl2' , 'ifl3' ) :
                lcdict [ col ] = lcdict [ col ] / thismedval
            # handle mags
            else :
                lcdict [ col ] = lcdict [ col ] - thismedval
# now read the rest
for lcf in lclist [ 1 : ] :
    thislcd = read_hatpi_textlc ( lcf )
    # if the columns don ' t agree , skip this LC
    if thislcd [ 'columns' ] != lcdict [ 'columns' ] :
        LOGERROR ( 'file %s does not have the ' 'same columns as first file %s, skipping...' % ( lcf , lclist [ 0 ] ) )
        continue
    # otherwise , go ahead and start concatenatin '
    else :
        LOGINFO ( 'adding %s (ndet: %s) to %s (ndet: %s)' % ( lcf , thislcd [ 'objectinfo' ] [ 'ndet' ] , lclist [ 0 ] , lcdict [ lcdict [ 'columns' ] [ 0 ] ] . size ) )
        # update LC tracking
        lccounter = lccounter + 1
        lcdict [ 'concatenated' ] [ lccounter ] = os . path . abspath ( lcf )
        lcdict [ 'lcn' ] = np . concatenate ( ( lcdict [ 'lcn' ] , np . full_like ( thislcd [ 'rjd' ] , lccounter ) ) )
        # concatenate the columns
        for col in lcdict [ 'columns' ] : # handle normalization for magnitude columns
            if normalize and col in MAGCOLS :
                thismedval = np . nanmedian ( thislcd [ col ] )
                # handle fluxes
                if col in ( 'ifl1' , 'ifl2' , 'ifl3' ) :
                    thislcd [ col ] = thislcd [ col ] / thismedval
                # handle mags
                else :
                    thislcd [ col ] = thislcd [ col ] - thismedval
            # concatenate the values
            lcdict [ col ] = np . concatenate ( ( lcdict [ col ] , thislcd [ col ] ) )
# now we ' re all done concatenatin '
# make sure to add up the ndet
lcdict [ 'objectinfo' ] [ 'ndet' ] = lcdict [ lcdict [ 'columns' ] [ 0 ] ] . size
# update the stations
lcdict [ 'objectinfo' ] [ 'stations' ] = [ 'HP%s' % x for x in np . unique ( lcdict [ 'stf' ] ) . tolist ( ) ]
# update the total LC count
lcdict [ 'nconcatenated' ] = lccounter + 1
# if we ' re supposed to sort by a column , do so
if sortby and sortby in [ x [ 0 ] for x in COLDEFS ] :
    LOGINFO ( 'sorting concatenated light curve by %s...' % sortby )
    sortind = np . argsort ( lcdict [ sortby ] )
    # sort all the measurement columns by this index
    for col in lcdict [ 'columns' ] :
        lcdict [ col ] = lcdict [ col ] [ sortind ]
    # make sure to sort the lcn index as well
    lcdict [ 'lcn' ] = lcdict [ 'lcn' ] [ sortind ]
LOGINFO ( 'done. concatenated light curve has %s detections' % lcdict [ 'objectinfo' ] [ 'ndet' ] )
return lcdict


signature:
def concatenate_textlcs_for_objectid ( lcbasedir , objectid , aperture = 'TF1' , postfix = '.gz' , sortby = 'rjd' , normalize = True , recursive = True ) :
    '''This concatenates all text LCs for an objectid with the given aperture .
 Does not care about overlaps or duplicates . The light curves must all be
 from the same aperture .
 The intended use is to concatenate light curves across CCDs or instrument
 changes for a single object . These can then be normalized later using
 standard astrobase tools to search for variablity and / or periodicity .
 lcbasedir is the directory to start searching in .
 objectid is the object to search for .
 aperture is the aperture postfix to use : ( TF1 = aperture 1,
 TF2 = aperture 2,
 TF3 = aperture 3)
 sortby is a column to sort the final concatenated light curve by in
 ascending order .
 If normalize is True , then each light curve ' s magnitude columns are
 normalized to zero , and the whole light curve is then normalized to the
 global median magnitude for each magnitude column .
 If recursive is True , then the function will search recursively in lcbasedir
 for any light curves matching the specified criteria . This may take a while ,
 especially on network filesystems .
 The returned lcdict has an extra column : ' lcn ' that tracks which measurement
 belongs to which input light curve . This can be used with
 lcdict [ ' concatenated ' ] which relates input light curve index to input light
 curve filepath . Finally , there is an ' nconcatenated ' key in the lcdict that
 contains the total number of concatenated light curves .'''

implementation:
LOGINFO ( 'looking for light curves for %s, aperture %s in directory: %s' % ( objectid , aperture , lcbasedir ) )
if recursive is False :
    matching = glob . glob ( os . path . join ( lcbasedir , '*%s*%s*%s' % ( objectid , aperture , postfix ) ) )
else : # use recursive glob for Python 3.5 +
    if sys . version_info [ : 2 ] > ( 3 , 4 ) :
        matching = glob . glob ( os . path . join ( lcbasedir , '**' , '*%s*%s*%s' % ( objectid , aperture , postfix ) ) , recursive = True )
        LOGINFO ( 'found %s files: %s' % ( len ( matching ) , repr ( matching ) ) )
    # otherwise , use os . walk and glob
    else : # use os . walk to go through the directories
        walker = os . walk ( lcbasedir )
        matching = [ ]
        for root , dirs , _files in walker :
            for sdir in dirs :
                searchpath = os . path . join ( root , sdir , '*%s*%s*%s' % ( objectid , aperture , postfix ) )
                foundfiles = glob . glob ( searchpath )
                if foundfiles :
                    matching . extend ( foundfiles )
                    LOGINFO ( 'found %s in dir: %s' % ( repr ( foundfiles ) , os . path . join ( root , sdir ) ) )
# now that we have all the files , concatenate them
# a single file will be returned as normalized
if matching and len ( matching ) > 0 :
    clcdict = concatenate_textlcs ( matching , sortby = sortby , normalize = normalize )
    return clcdict
else :
    LOGERROR ( 'did not find any light curves for %s and aperture %s' % ( objectid , aperture ) )
    return None


signature:
def concat_write_pklc ( lcbasedir , objectid , aperture = 'TF1' , postfix = '.gz' , sortby = 'rjd' , normalize = True , outdir = None , recursive = True ) :
    '''This concatenates all text LCs for the given object and writes to a pklc .
 Basically a rollup for the concatenate _ textlcs _ for _ objectid and
 lcdict _ to _ pickle functions .'''

implementation:
concatlcd = concatenate_textlcs_for_objectid ( lcbasedir , objectid , aperture = aperture , sortby = sortby , normalize = normalize , recursive = recursive )
if not outdir :
    outdir = 'pklcs'
if not os . path . exists ( outdir ) :
    os . mkdir ( outdir )
outfpath = os . path . join ( outdir , '%s-%s-pklc.pkl' % ( concatlcd [ 'objectid' ] , aperture ) )
pklc = lcdict_to_pickle ( concatlcd , outfile = outfpath )
return pklc


signature:
def parallel_concat_worker ( task ) :
    '''This is a worker for the function below .
 task [ 0 ] = lcbasedir
 task [ 1 ] = objectid
 task [ 2 ] = { ' aperture ' , ' postfix ' , ' sortby ' , ' normalize ' , ' outdir ' , ' recursive ' }'''

implementation:
lcbasedir , objectid , kwargs = task
try :
    return concat_write_pklc ( lcbasedir , objectid , ** kwargs )
except Exception as e :
    LOGEXCEPTION ( 'failed LC concatenation for %s in %s' % ( objectid , lcbasedir ) )
    return None


signature:
def parallel_concat_lcdir ( lcbasedir , objectidlist , aperture = 'TF1' , postfix = '.gz' , sortby = 'rjd' , normalize = True , outdir = None , recursive = True , nworkers = 32 , maxworkertasks = 1000 ) :
    '''This concatenates all text LCs for the given objectidlist .'''

implementation:
if not outdir :
    outdir = 'pklcs'
if not os . path . exists ( outdir ) :
    os . mkdir ( outdir )
tasks = [ ( lcbasedir , x , { 'aperture' : aperture , 'postfix' : postfix , 'sortby' : sortby , 'normalize' : normalize , 'outdir' : outdir , 'recursive' : recursive } ) for x in objectidlist ]
pool = mp . Pool ( nworkers , maxtasksperchild = maxworkertasks )
results = pool . map ( parallel_concat_worker , tasks )
pool . close ( )
pool . join ( )
return { x : y for ( x , y ) in zip ( objectidlist , results ) }


signature:
def merge_hatpi_textlc_apertures ( lclist ) :
    '''This merges all TFA text LCs with separate apertures for a single object .
 The framekey column will be used as the join column across all light curves
 in lclist . Missing values will be filled in with nans . This function assumes
 all light curves are in the format specified in COLDEFS above and readable
 by read _ hatpi _ textlc above ( i . e . have a single column for TFA mags for a
 specific aperture at the end ) .'''

implementation:
lcaps = { }
framekeys = [ ]
for lc in lclist :
    lcd = read_hatpi_textlc ( lc )
    # figure what aperture this is and put it into the lcdict . if two LCs
    # with the same aperture ( i . e . TF1 and TF1 ) are provided , the later one
    # in the lclist will overwrite the previous one ,
    for col in lcd [ 'columns' ] :
        if col . startswith ( 'itf' ) :
            lcaps [ col ] = lcd
    thisframekeys = lcd [ 'frk' ] . tolist ( )
    framekeys . extend ( thisframekeys )
# uniqify the framekeys
framekeys = sorted ( list ( set ( framekeys ) ) )


signature:
def read_hatpi_binnedlc ( binnedpklf , textlcf , timebinsec ) :
    '''This reads a binnedlc pickle produced by the HATPI prototype pipeline .
 Converts it into a standard lcdict as produced by the read _ hatpi _ textlc
 function above by using the information in unbinnedtextlc for the same
 object .
 Adds a ' binned ' key to the standard lcdict containing the binned mags , etc .'''

implementation:
LOGINFO ( 'reading binned LC %s' % binnedpklf )
# read the textlc
lcdict = read_hatpi_textlc ( textlcf )
# read the binned LC
if binnedpklf . endswith ( '.gz' ) :
    infd = gzip . open ( binnedpklf , 'rb' )
else :
    infd = open ( binnedpklf , 'rb' )
try :
    binned = pickle . load ( infd )
except Exception as e :
    infd . seek ( 0 )
    binned = pickle . load ( infd , encoding = 'latin1' )
infd . close ( )
# now that we have both , pull out the required columns from the binnedlc
blckeys = binned . keys ( )
lcdict [ 'binned' ] = { }
for key in blckeys : # get EPD stuff
    if ( key == 'epdlc' and 'AP0' in binned [ key ] and 'AP1' in binned [ key ] and 'AP2' in binned [ key ] ) : # we ' ll have to generate errors because we don ' t have any in the
    # generated binned LC .
        ap0mad = np . nanmedian ( np . abs ( binned [ key ] [ 'AP0' ] - np . nanmedian ( binned [ key ] [ 'AP0' ] ) ) )
        ap1mad = np . nanmedian ( np . abs ( binned [ key ] [ 'AP1' ] - np . nanmedian ( binned [ key ] [ 'AP1' ] ) ) )
        ap2mad = np . nanmedian ( np . abs ( binned [ key ] [ 'AP2' ] - np . nanmedian ( binned [ key ] [ 'AP2' ] ) ) )
        lcdict [ 'binned' ] [ 'iep1' ] = { 'times' : binned [ key ] [ 'RJD' ] , 'mags' : binned [ key ] [ 'AP0' ] , 'errs' : np . full_like ( binned [ key ] [ 'AP0' ] , ap0mad ) , 'nbins' : binned [ key ] [ 'nbins' ] , 'timebins' : binned [ key ] [ 'jdbins' ] , 'timebinsec' : timebinsec }
        lcdict [ 'binned' ] [ 'iep2' ] = { 'times' : binned [ key ] [ 'RJD' ] , 'mags' : binned [ key ] [ 'AP1' ] , 'errs' : np . full_like ( binned [ key ] [ 'AP1' ] , ap1mad ) , 'nbins' : binned [ key ] [ 'nbins' ] , 'timebins' : binned [ key ] [ 'jdbins' ] , 'timebinsec' : timebinsec }
        lcdict [ 'binned' ] [ 'iep3' ] = { 'times' : binned [ key ] [ 'RJD' ] , 'mags' : binned [ key ] [ 'AP2' ] , 'errs' : np . full_like ( binned [ key ] [ 'AP2' ] , ap2mad ) , 'nbins' : binned [ key ] [ 'nbins' ] , 'timebins' : binned [ key ] [ 'jdbins' ] , 'timebinsec' : timebinsec }
    # get TFA stuff for aperture 1
    if ( ( key == 'tfalc.TF1' or key == 'tfalc.TF1.gz' ) and 'AP0' in binned [ key ] ) : # we ' ll have to generate errors because we don ' t have any in the
    # generated binned LC .
        ap0mad = np . nanmedian ( np . abs ( binned [ key ] [ 'AP0' ] - np . nanmedian ( binned [ key ] [ 'AP0' ] ) ) )
        lcdict [ 'binned' ] [ 'itf1' ] = { 'times' : binned [ key ] [ 'RJD' ] , 'mags' : binned [ key ] [ 'AP0' ] , 'errs' : np . full_like ( binned [ key ] [ 'AP0' ] , ap0mad ) , 'nbins' : binned [ key ] [ 'nbins' ] , 'timebins' : binned [ key ] [ 'jdbins' ] , 'timebinsec' : timebinsec }
    # get TFA stuff for aperture 1
    if ( ( key == 'tfalc.TF2' or key == 'tfalc.TF2.gz' ) and 'AP0' in binned [ key ] ) : # we ' ll have to generate errors because we don ' t have any in the
    # generated binned LC .
        ap0mad = np . nanmedian ( np . abs ( binned [ key ] [ 'AP0' ] - np . nanmedian ( binned [ key ] [ 'AP0' ] ) ) )
        lcdict [ 'binned' ] [ 'itf2' ] = { 'times' : binned [ key ] [ 'RJD' ] , 'mags' : binned [ key ] [ 'AP0' ] , 'errs' : np . full_like ( binned [ key ] [ 'AP0' ] , ap0mad ) , 'nbins' : binned [ key ] [ 'nbins' ] , 'timebins' : binned [ key ] [ 'jdbins' ] , 'timebinsec' : timebinsec }
    # get TFA stuff for aperture 1
    if ( ( key == 'tfalc.TF3' or key == 'tfalc.TF3.gz' ) and 'AP0' in binned [ key ] ) : # we ' ll have to generate errors because we don ' t have any in the
    # generated binned LC .
        ap0mad = np . nanmedian ( np . abs ( binned [ key ] [ 'AP0' ] - np . nanmedian ( binned [ key ] [ 'AP0' ] ) ) )
        lcdict [ 'binned' ] [ 'itf3' ] = { 'times' : binned [ key ] [ 'RJD' ] , 'mags' : binned [ key ] [ 'AP0' ] , 'errs' : np . full_like ( binned [ key ] [ 'AP0' ] , ap0mad ) , 'nbins' : binned [ key ] [ 'nbins' ] , 'timebins' : binned [ key ] [ 'jdbins' ] , 'timebinsec' : timebinsec }
# all done , check if we succeeded
if lcdict [ 'binned' ] :
    return lcdict
else :
    LOGERROR ( 'no binned measurements found in %s!' % binnedpklf )
    return None


signature:
def generate_hatpi_binnedlc_pkl ( binnedpklf , textlcf , timebinsec , outfile = None ) :
    '''This reads the binned LC and writes it out to a pickle .'''

implementation:
binlcdict = read_hatpi_binnedlc ( binnedpklf , textlcf , timebinsec )
if binlcdict :
    if outfile is None :
        outfile = os . path . join ( os . path . dirname ( binnedpklf ) , '%s-hplc.pkl' % ( os . path . basename ( binnedpklf ) . replace ( 'sec-lc.pkl.gz' , '' ) ) )
    return lcdict_to_pickle ( binlcdict , outfile = outfile )
else :
    LOGERROR ( 'could not read binned HATPI LC: %s' % binnedpklf )
    return None


signature:
def parallel_gen_binnedlc_pkls ( binnedpkldir , textlcdir , timebinsec , binnedpklglob = '*binned*sec*.pkl' , textlcglob = '*.tfalc.TF1*' ) :
    '''This generates the binnedlc pkls for a directory of such files .
 FIXME : finish this'''

implementation:
binnedpkls = sorted ( glob . glob ( os . path . join ( binnedpkldir , binnedpklglob ) ) )
# find all the textlcs associated with these
textlcs = [ ]
for bpkl in binnedpkls :
    objectid = HATIDREGEX . findall ( bpkl )
    if objectid is not None :
        objectid = objectid [ 0 ]
    searchpath = os . path . join ( textlcdir , '%s-%s' % ( objectid , textlcglob ) )
    textlcf = glob . glob ( searchpath )
    if textlcf :
        textlcs . append ( textlcf )
    else :
        textlcs . append ( None )


signature:
def pklc_fovcatalog_objectinfo ( pklcdir , fovcatalog , fovcatalog_columns = [ 0 , 1 , 2 , 6 , 7 , 8 , 9 , 10 , 11 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 ] , fovcatalog_colnames = [ 'objectid' , 'ra' , 'decl' , 'jmag' , 'jmag_err' , 'hmag' , 'hmag_err' , 'kmag' , 'kmag_err' , 'bmag' , 'vmag' , 'rmag' , 'imag' , 'sdssu' , 'sdssg' , 'sdssr' , 'sdssi' , 'sdssz' ] , fovcatalog_colformats = ( 'U20,f8,f8,' 'f8,f8,' 'f8,f8,' 'f8,f8,' 'f8,f8,f8,f8,' 'f8,f8,f8,' 'f8,f8' ) ) :
    '''Adds catalog info to objectinfo key of all pklcs in lcdir .
 If fovcatalog , fovcatalog _ columns , fovcatalog _ colnames are provided , uses
 them to find all the additional information listed in the fovcatalog _ colname
 keys , and writes this info to the objectinfo key of each lcdict . This makes
 it easier for astrobase tools to work on these light curve .
 The default set up for fovcatalog is to use a text file generated by the
 HATPI pipeline before auto - calibrating a field . The format is specified as
 above in _ columns , _ colnames , and _ colformats .'''

implementation:
if fovcatalog . endswith ( '.gz' ) :
    catfd = gzip . open ( fovcatalog )
else :
    catfd = open ( fovcatalog )
# read the catalog using the colformats , etc .
fovcat = np . genfromtxt ( catfd , usecols = fovcatalog_columns , names = fovcatalog_colnames , dtype = fovcatalog_colformats )
catfd . close ( )
pklclist = sorted ( glob . glob ( os . path . join ( pklcdir , '*HAT*-pklc.pkl' ) ) )
updatedpklcs , failedpklcs = [ ] , [ ]
for pklc in pklclist :
    lcdict = read_hatpi_pklc ( pklc )
    objectid = lcdict [ 'objectid' ]
    catind = np . where ( fovcat [ 'objectid' ] == objectid )
    # if we found catalog info for this object , put it into objectinfo
    if len ( catind ) > 0 and catind [ 0 ] :
        lcdict [ 'objectinfo' ] . update ( { x : y for x , y in zip ( fovcatalog_colnames , [ np . asscalar ( fovcat [ z ] [ catind ] ) for z in fovcatalog_colnames ] ) } )
        # write the LC back to the pickle ( tempfile for atomicity )
        with open ( pklc + '-tmp' , 'wb' ) as outfd :
            pickle . dump ( lcdict , outfd , pickle . HIGHEST_PROTOCOL )
        # overwrite previous once we know it exists
        if os . path . exists ( pklc + '-tmp' ) :
            shutil . move ( pklc + '-tmp' , pklc )
            LOGINFO ( 'updated %s with catalog info for %s at %.3f, %.3f OK' % ( pklc , objectid , lcdict [ 'objectinfo' ] [ 'ra' ] , lcdict [ 'objectinfo' ] [ 'decl' ] ) )
            updatedpklcs . append ( pklc )
    # otherwise , do nothing
    else :
        failedpklcs . append ( pklc )
# end of pklclist processing
return updatedpklcs , failedpklcs


signature:
def _base64_to_file ( b64str , outfpath , writetostrio = False ) :
    '''This converts the base64 encoded string to a file .
 Parameters
 b64str : str
 A base64 encoded strin that is the output of ` base64 . b64encode ` .
 outfpath : str
 The path to where the file will be written . This should include an
 appropriate extension for the file ( e . g . a base64 encoded string that
 represents a PNG should have its ` outfpath ` end in a ' . png ' ) so the OS
 can open these files correctly .
 writetostrio : bool
 If this is True , will return a StringIO object with the binary stream
 decoded from the base64 - encoded input string ` b64str ` . This can be
 useful to embed these into other files without having to write them to
 disk .
 Returns
 str or StringIO object
 If ` writetostrio ` is False , will return the output file ' s path as a
 str . If it is True , will return a StringIO object directly . If writing
 the file fails in either case , will return None .'''

implementation:
try :
    filebytes = base64 . b64decode ( b64str )
    # if we ' re writing back to a stringio object
    if writetostrio :
        outobj = StrIO ( filebytes )
        return outobj
    # otherwise , we ' re writing to an actual file
    else :
        with open ( outfpath , 'wb' ) as outfd :
            outfd . write ( filebytes )
        if os . path . exists ( outfpath ) :
            return outfpath
        else :
            LOGERROR ( 'could not write output file: %s' % outfpath )
            return None
except Exception as e :
    LOGEXCEPTION ( 'failed while trying to convert ' 'b64 string to file %s' % outfpath )
    return None


signature:
def _read_checkplot_picklefile ( checkplotpickle ) :
    '''This reads a checkplot gzipped pickle file back into a dict .
 NOTE : the try - except is for Python 2 pickles that have numpy arrays in
 them . Apparently , these aren ' t compatible with Python 3 . See here :
 http : / / stackoverflow . com / q / 11305790
 The workaround is noted in this answer :
 http : / / stackoverflow . com / a / 41366785
 Parameters
 checkplotpickle : str
 The path to a checkplot pickle file . This can be a gzipped file ( in
 which case the file extension should end in ' . gz ' )
 Returns
 dict
 This returns a checkplotdict .'''

implementation:
if checkplotpickle . endswith ( '.gz' ) :
    try :
        with gzip . open ( checkplotpickle , 'rb' ) as infd :
            cpdict = pickle . load ( infd )
    except UnicodeDecodeError :
        with gzip . open ( checkplotpickle , 'rb' ) as infd :
            cpdict = pickle . load ( infd , encoding = 'latin1' )
else :
    try :
        with open ( checkplotpickle , 'rb' ) as infd :
            cpdict = pickle . load ( infd )
    except UnicodeDecodeError :
        with open ( checkplotpickle , 'rb' ) as infd :
            cpdict = pickle . load ( infd , encoding = 'latin1' )
return cpdict


signature:
def _write_checkplot_picklefile ( checkplotdict , outfile = None , protocol = None , outgzip = False ) :
    '''This writes the checkplotdict to a ( gzipped ) pickle file .
 Parameters
 checkplotdict : dict
 This the checkplotdict to write to the pickle file .
 outfile : None or str
 The path to the output pickle file to write . If ` outfile ` is None ,
 writes a ( gzipped ) pickle file of the form :
 checkplot - { objectid } . pkl ( . gz )
 to the current directory .
 protocol : int
 This sets the pickle file protocol to use when writing the pickle :
 If None , will choose a protocol using the following rules :
 - 4 - > default in Python > = 3.4 - fast but incompatible with Python 2
 - 3 - > default in Python 3.0-3.3 - mildly fast
 - 2 - > default in Python 2 - very slow , but compatible with Python 2/3
 The default protocol kwarg is None , this will make an automatic choice
 for pickle protocol that ' s best suited for the version of Python in
 use . Note that this will make pickles generated by Py3 incompatible with
 Py2.
 outgzip : bool
 If this is True , will gzip the output file . Note that if the ` outfile `
 str ends in a gzip , this will be automatically turned on .
 Returns
 str
 The absolute path to the written checkplot pickle file . None if writing
 fails .'''

implementation:
# figure out which protocol to use
# for Python > = 3.4 ; use v4 by default
if ( ( sys . version_info [ 0 : 2 ] >= ( 3 , 4 ) and not protocol ) or ( protocol > 2 ) ) :
    protocol = 4
elif ( ( sys . version_info [ 0 : 2 ] >= ( 3 , 0 ) and not protocol ) or ( protocol > 2 ) ) :
    protocol = 3
# for Python = = 2.7 ; use v2
elif sys . version_info [ 0 : 2 ] == ( 2 , 7 ) and not protocol :
    protocol = 2
# otherwise , if left unspecified , use the slowest but most compatible
# protocol . this will be readable by all ( most ? ) Pythons
elif not protocol :
    protocol = 0
if outgzip :
    if not outfile :
        outfile = ( 'checkplot-{objectid}.pkl.gz' . format ( objectid = squeeze ( checkplotdict [ 'objectid' ] ) . replace ( ' ' , '-' ) ) )
    with gzip . open ( outfile , 'wb' ) as outfd :
        pickle . dump ( checkplotdict , outfd , protocol = protocol )
else :
    if not outfile :
        outfile = ( 'checkplot-{objectid}.pkl' . format ( objectid = squeeze ( checkplotdict [ 'objectid' ] ) . replace ( ' ' , '-' ) ) )
    # make sure to do the right thing if ' . gz ' is in the filename but
    # outgzip was False
    if outfile . endswith ( '.gz' ) :
        LOGWARNING ( 'output filename ends with .gz but kwarg outgzip=False. ' 'will use gzip to compress the output pickle' )
        with gzip . open ( outfile , 'wb' ) as outfd :
            pickle . dump ( checkplotdict , outfd , protocol = protocol )
    else :
        with open ( outfile , 'wb' ) as outfd :
            pickle . dump ( checkplotdict , outfd , protocol = protocol )
return os . path . abspath ( outfile )


signature:
def get_phased_quantities ( stimes , smags , serrs , period ) :
    '''Does phase - folding for the mag / flux time - series given a period .
 Given finite and sigma - clipped times , magnitudes , and errors , along with the
 period at which to phase - fold the data , perform the phase - folding and
 return the phase - folded values .
 Parameters
 stimes , smags , serrs : np . array
 The sigma - clipped and finite input mag / flux time - series arrays to
 operate on .
 period : float
 The period to phase the mag / flux time - series at . stimes . min ( ) is used as
 the epoch value to fold the times - series around .
 Returns
 ( phase , pmags , perrs , ptimes , mintime ) : tuple
 The tuple returned contains the following items :
 - ` phase ` : phase - sorted values of phase at each of stimes
 - ` pmags ` : phase - sorted magnitudes at each phase
 - ` perrs ` : phase - sorted errors
 - ` ptimes ` : phase - sorted times
 - ` mintime ` : earliest time in stimes .'''

implementation:
# phase the mag series using the given period and faintest mag time
# mintime = stimes [ npwhere ( smags = = npmax ( smags ) ) ]
# phase the mag series using the given period and epoch = min ( stimes )
mintime = np . min ( stimes )
# calculate the unsorted phase , then sort it
iphase = ( stimes - mintime ) / period - np . floor ( ( stimes - mintime ) / period )
phasesortind = np . argsort ( iphase )
# these are the final quantities to use for the Fourier fits
phase = iphase [ phasesortind ]
pmags = smags [ phasesortind ]
perrs = serrs [ phasesortind ]
# get the times sorted in phase order ( useful to get the fit mag minimum
# with respect to phase - - the light curve minimum )
ptimes = stimes [ phasesortind ]
return phase , pmags , perrs , ptimes , mintime


signature:
def make_fit_plot ( phase , pmags , perrs , fitmags , period , mintime , magseriesepoch , plotfit , magsarefluxes = False , wrap = False , model_over_lc = False ) :
    '''This makes a plot of the LC model fit .
 Parameters
 phase , pmags , perrs : np . array
 The actual mag / flux time - series .
 fitmags : np . array
 The model fit time - series .
 period : float
 The period at which the phased LC was generated .
 mintime : float
 The minimum time value .
 magseriesepoch : float
 The value of time around which the phased LC was folded .
 plotfit : str
 The name of a file to write the plot to .
 magsarefluxes : bool
 Set this to True if the values in ` pmags ` and ` fitmags ` are actually
 fluxes .
 wrap : bool
 If True , will wrap the phased LC around 0.0 to make some phased LCs
 easier to look at .
 model _ over _ lc : bool
 Usually , this function will plot the actual LC over the model LC . Set
 this to True to plot the model over the actual LC ; this is most useful
 when you have a very dense light curve and want to be able to see how it
 follows the model .
 Returns
 Nothing .'''

implementation:
# set up the figure
plt . close ( 'all' )
plt . figure ( figsize = ( 8 , 4.8 ) )
if model_over_lc :
    model_z = 100
    lc_z = 0
else :
    model_z = 0
    lc_z = 100
if not wrap :
    plt . plot ( phase , fitmags , linewidth = 3.0 , color = 'red' , zorder = model_z )
    plt . plot ( phase , pmags , marker = 'o' , markersize = 1.0 , linestyle = 'none' , rasterized = True , color = 'k' , zorder = lc_z )
    # set the x axis ticks and label
    plt . gca ( ) . set_xticks ( [ 0.0 , 0.1 , 0.2 , 0.3 , 0.4 , 0.5 , 0.6 , 0.7 , 0.8 , 0.9 , 1.0 ] )
else :
    plt . plot ( np . concatenate ( [ phase - 1.0 , phase ] ) , np . concatenate ( [ fitmags , fitmags ] ) , linewidth = 3.0 , color = 'red' , zorder = model_z )
    plt . plot ( np . concatenate ( [ phase - 1.0 , phase ] ) , np . concatenate ( [ pmags , pmags ] ) , marker = 'o' , markersize = 1.0 , linestyle = 'none' , rasterized = True , color = 'k' , zorder = lc_z )
    plt . gca ( ) . set_xlim ( ( - 0.8 , 0.8 ) )
    # set the x axis ticks and label
    plt . gca ( ) . set_xticks ( [ - 0.8 , - 0.7 , - 0.6 , - 0.5 , - 0.4 , - 0.3 , - 0.2 , - 0.1 , 0.0 , 0.1 , 0.2 , 0.3 , 0.4 , 0.5 , 0.6 , 0.7 , 0.8 ] )
# set the y axis limit and label
ymin , ymax = plt . ylim ( )
if not magsarefluxes :
    plt . gca ( ) . invert_yaxis ( )
    plt . ylabel ( 'magnitude' )
else :
    plt . ylabel ( 'flux' )
plt . xlabel ( 'phase' )
plt . title ( 'period: %.6f, folded at %.6f, fit epoch: %.6f' % ( period , mintime , magseriesepoch ) )
plt . savefig ( plotfit )
plt . close ( )


signature:
def objectlist_conesearch ( racenter , declcenter , searchradiusarcsec , gaia_mirror = None , columns = ( 'source_id' , 'ra' , 'dec' , 'phot_g_mean_mag' , 'l' , 'b' , 'parallax' , 'parallax_error' , 'pmra' , 'pmra_error' , 'pmdec' , 'pmdec_error' ) , extra_filter = None , returnformat = 'csv' , forcefetch = False , cachedir = '~/.astrobase/gaia-cache' , verbose = True , timeout = 15.0 , refresh = 2.0 , maxtimeout = 300.0 , maxtries = 3 , complete_query_later = True ) :
    '''This queries the GAIA TAP service for a list of objects near the coords .
 Runs a conesearch around ` ( racenter , declcenter ) ` with radius in arcsec of
 ` searchradiusarcsec ` .
 Parameters
 racenter , declcenter : float
 The center equatorial coordinates in decimal degrees .
 searchradiusarcsec : float
 The search radius of the cone - search in arcseconds .
 gaia _ mirror : { ' gaia ' , ' heidelberg ' , ' vizier ' } or None
 This is the key used to select a GAIA catalog mirror from the
 ` GAIA _ URLS ` dict above . If set , the specified mirror will be used . If
 None , a random mirror chosen from that dict will be used .
 columns : sequence of str
 This indicates which columns from the GAIA table to request for the
 objects found within the search radius .
 extra _ filter : str or None
 If this is provided , must be a valid ADQL filter string that is used to
 further filter the cone - search results .
 returnformat : { ' csv ' , ' votable ' , ' json ' }
 The returned file format to request from the GAIA catalog service .
 forcefetch : bool
 If this is True , the query will be retried even if cached results for
 it exist .
 cachedir : str
 This points to the directory where results will be downloaded .
 verbose : bool
 If True , will indicate progress and warn of any issues .
 timeout : float
 This sets the amount of time in seconds to wait for the service to
 respond to our initial request .
 refresh : float
 This sets the amount of time in seconds to wait before checking if the
 result file is available . If the results file isn ' t available after
 ` refresh ` seconds have elapsed , the function will wait for ` refresh `
 seconds continuously , until ` maxtimeout ` is reached or the results file
 becomes available .
 maxtimeout : float
 The maximum amount of time in seconds to wait for a result to become
 available after submitting our query request .
 maxtries : int
 The maximum number of tries ( across all mirrors tried ) to make to either
 submit the request or download the results , before giving up .
 completequerylater : bool
 If set to True , a submitted query that does not return a result before
 ` maxtimeout ` has passed will be cancelled but its input request
 parameters and the result URL provided by the service will be saved . If
 this function is then called later with these same input request
 parameters , it will check if the query finally finished and a result is
 available . If so , will download the results instead of submitting a new
 query . If it ' s not done yet , will start waiting for results again . To
 force launch a new query with the same request parameters , set the
 ` forcefetch ` kwarg to True .
 Returns
 dict
 This returns a dict of the following form : :
 { ' params ' : dict of the input params used for the query ,
 ' provenance ' : ' cache ' or ' new download ' ,
 ' result ' : path to the file on disk with the downloaded data table }'''

implementation:
# this was generated using the awesome query generator at :
# https : / / gea . esac . esa . int / archive /
# NOTE : here we don ' t resolve the table name right away . this is because
# some of the GAIA mirrors use different table names , so we leave the table
# name to be resolved by the lower level tap _ query function . this is done by
# the { { table } } construct .
query = ( "select {columns}, " "(DISTANCE(POINT('ICRS', " "{{table}}.ra, {{table}}.dec), " "POINT('ICRS', {ra_center:.5f}, {decl_center:.5f})))*3600.0 " "AS dist_arcsec " "from {{table}} where " "CONTAINS(POINT('ICRS',{{table}}.ra, {{table}}.dec)," "CIRCLE('ICRS',{ra_center:.5f},{decl_center:.5f}," "{search_radius:.6f}))=1 " "{extra_filter_str}" "ORDER by dist_arcsec asc " )
if extra_filter is not None :
    extra_filter_str = ' and %s ' % extra_filter
else :
    extra_filter_str = ''
formatted_query = query . format ( ra_center = racenter , decl_center = declcenter , search_radius = searchradiusarcsec / 3600.0 , extra_filter_str = extra_filter_str , columns = ', ' . join ( columns ) )
return tap_query ( formatted_query , gaia_mirror = gaia_mirror , returnformat = returnformat , forcefetch = forcefetch , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout , maxtries = maxtries , complete_query_later = complete_query_later )


signature:
def objectlist_radeclbox ( radeclbox , gaia_mirror = None , columns = ( 'source_id' , 'ra' , 'dec' , 'phot_g_mean_mag' , 'l' , 'b' , 'parallax, parallax_error' , 'pmra' , 'pmra_error' , 'pmdec' , 'pmdec_error' ) , extra_filter = None , returnformat = 'csv' , forcefetch = False , cachedir = '~/.astrobase/gaia-cache' , verbose = True , timeout = 15.0 , refresh = 2.0 , maxtimeout = 300.0 , maxtries = 3 , complete_query_later = True ) :
    '''This queries the GAIA TAP service for a list of objects in an equatorial
 coordinate box .
 Parameters
 radeclbox : sequence of four floats
 This defines the box to search in : :
 [ ra _ min , ra _ max , decl _ min , decl _ max ]
 gaia _ mirror : { ' gaia ' , ' heidelberg ' , ' vizier ' } or None
 This is the key used to select a GAIA catalog mirror from the
 ` GAIA _ URLS ` dict above . If set , the specified mirror will be used . If
 None , a random mirror chosen from that dict will be used .
 columns : sequence of str
 This indicates which columns from the GAIA table to request for the
 objects found within the search radius .
 extra _ filter : str or None
 If this is provided , must be a valid ADQL filter string that is used to
 further filter the cone - search results .
 returnformat : { ' csv ' , ' votable ' , ' json ' }
 The returned file format to request from the GAIA catalog service .
 forcefetch : bool
 If this is True , the query will be retried even if cached results for
 it exist .
 cachedir : str
 This points to the directory where results will be downloaded .
 verbose : bool
 If True , will indicate progress and warn of any issues .
 timeout : float
 This sets the amount of time in seconds to wait for the service to
 respond to our initial request .
 refresh : float
 This sets the amount of time in seconds to wait before checking if the
 result file is available . If the results file isn ' t available after
 ` refresh ` seconds have elapsed , the function will wait for ` refresh `
 seconds continuously , until ` maxtimeout ` is reached or the results file
 becomes available .
 maxtimeout : float
 The maximum amount of time in seconds to wait for a result to become
 available after submitting our query request .
 maxtries : int
 The maximum number of tries ( across all mirrors tried ) to make to either
 submit the request or download the results , before giving up .
 completequerylater : bool
 If set to True , a submitted query that does not return a result before
 ` maxtimeout ` has passed will be cancelled but its input request
 parameters and the result URL provided by the service will be saved . If
 this function is then called later with these same input request
 parameters , it will check if the query finally finished and a result is
 available . If so , will download the results instead of submitting a new
 query . If it ' s not done yet , will start waiting for results again . To
 force launch a new query with the same request parameters , set the
 ` forcefetch ` kwarg to True .
 Returns
 dict
 This returns a dict of the following form : :
 { ' params ' : dict of the input params used for the query ,
 ' provenance ' : ' cache ' or ' new download ' ,
 ' result ' : path to the file on disk with the downloaded data table }'''

implementation:
# this was generated using the awesome query generator at :
# https : / / gea . esac . esa . int / archive /
# NOTE : here we don ' t resolve the table name right away . this is because
# some of the GAIA mirrors use different table names , so we leave the table
# name to be resolved by the lower level tap _ query function . this is done by
# the { { table } } construct .
query = ( "select {columns} from {{table}} where " "CONTAINS(POINT('ICRS',{{table}}.ra, {{table}}.dec)," "BOX('ICRS',{ra_center:.5f},{decl_center:.5f}," "{ra_width:.5f},{decl_height:.5f}))=1" "{extra_filter_str}" )
ra_min , ra_max , decl_min , decl_max = radeclbox
ra_center = ( ra_max + ra_min ) / 2.0
decl_center = ( decl_max + decl_min ) / 2.0
ra_width = ra_max - ra_min
decl_height = decl_max - decl_min
if extra_filter is not None :
    extra_filter_str = ' and %s ' % extra_filter
else :
    extra_filter_str = ''
formatted_query = query . format ( columns = ', ' . join ( columns ) , extra_filter_str = extra_filter_str , ra_center = ra_center , decl_center = decl_center , ra_width = ra_width , decl_height = decl_height )
return tap_query ( formatted_query , gaia_mirror = gaia_mirror , returnformat = returnformat , forcefetch = forcefetch , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout , maxtries = maxtries , complete_query_later = complete_query_later )


signature:
def objectid_search ( gaiaid , gaia_mirror = None , columns = ( 'source_id' , 'ra' , 'dec' , 'phot_g_mean_mag' , 'phot_bp_mean_mag' , 'phot_rp_mean_mag' , 'l' , 'b' , 'parallax, parallax_error' , 'pmra' , 'pmra_error' , 'pmdec' , 'pmdec_error' ) , returnformat = 'csv' , forcefetch = False , cachedir = '~/.astrobase/gaia-cache' , verbose = True , timeout = 15.0 , refresh = 2.0 , maxtimeout = 300.0 , maxtries = 3 , complete_query_later = True ) :
    '''This queries the GAIA TAP service for a single GAIA source ID .
 Parameters
 gaiaid : str
 The source ID of the object whose info will be collected .
 gaia _ mirror : { ' gaia ' , ' heidelberg ' , ' vizier ' } or None
 This is the key used to select a GAIA catalog mirror from the
 ` GAIA _ URLS ` dict above . If set , the specified mirror will be used . If
 None , a random mirror chosen from that dict will be used .
 columns : sequence of str
 This indicates which columns from the GAIA table to request for the
 objects found within the search radius .
 returnformat : { ' csv ' , ' votable ' , ' json ' }
 The returned file format to request from the GAIA catalog service .
 forcefetch : bool
 If this is True , the query will be retried even if cached results for
 it exist .
 cachedir : str
 This points to the directory where results will be downloaded .
 verbose : bool
 If True , will indicate progress and warn of any issues .
 timeout : float
 This sets the amount of time in seconds to wait for the service to
 respond to our initial request .
 refresh : float
 This sets the amount of time in seconds to wait before checking if the
 result file is available . If the results file isn ' t available after
 ` refresh ` seconds have elapsed , the function will wait for ` refresh `
 seconds continuously , until ` maxtimeout ` is reached or the results file
 becomes available .
 maxtimeout : float
 The maximum amount of time in seconds to wait for a result to become
 available after submitting our query request .
 maxtries : int
 The maximum number of tries ( across all mirrors tried ) to make to either
 submit the request or download the results , before giving up .
 completequerylater : bool
 If set to True , a submitted query that does not return a result before
 ` maxtimeout ` has passed will be cancelled but its input request
 parameters and the result URL provided by the service will be saved . If
 this function is then called later with these same input request
 parameters , it will check if the query finally finished and a result is
 available . If so , will download the results instead of submitting a new
 query . If it ' s not done yet , will start waiting for results again . To
 force launch a new query with the same request parameters , set the
 ` forcefetch ` kwarg to True .
 Returns
 dict
 This returns a dict of the following form : :
 { ' params ' : dict of the input params used for the query ,
 ' provenance ' : ' cache ' or ' new download ' ,
 ' result ' : path to the file on disk with the downloaded data table }'''

implementation:
# NOTE : here we don ' t resolve the table name right away . this is because
# some of the GAIA mirrors use different table names , so we leave the table
# name to be resolved by the lower level tap _ query function . this is done by
# the { { table } } construct .
query = ( "select {columns} from {{table}} where " "source_id = {gaiaid}" )
formatted_query = query . format ( columns = ', ' . join ( columns ) , gaiaid = gaiaid )
return tap_query ( formatted_query , gaia_mirror = gaia_mirror , returnformat = returnformat , forcefetch = forcefetch , cachedir = cachedir , verbose = verbose , timeout = timeout , refresh = refresh , maxtimeout = maxtimeout , maxtries = maxtries , complete_query_later = complete_query_later )


signature:
def generalized_lsp_value ( times , mags , errs , omega ) :
    '''Generalized LSP value for a single omega .
 The relations used are : :
 P ( w ) = ( 1 / YY ) * ( YC * YC / CC + YS * YS / SS )
 where : YC , YS , CC , and SS are all calculated at T
 and where : tan 2omegaT = 2 * CS / ( CC - SS )
 and where :
 Y = sum ( w _ i * y _ i )
 C = sum ( w _ i * cos ( wT _ i ) )
 S = sum ( w _ i * sin ( wT _ i ) )
 YY = sum ( w _ i * y _ i * y _ i ) - Y * Y
 YC = sum ( w _ i * y _ i * cos ( wT _ i ) ) - Y * C
 YS = sum ( w _ i * y _ i * sin ( wT _ i ) ) - Y * S
 CpC = sum ( w _ i * cos ( w _ T _ i ) * cos ( w _ T _ i ) )
 CC = CpC - C * C
 SS = ( 1 - CpC ) - S * S
 CS = sum ( w _ i * cos ( w _ T _ i ) * sin ( w _ T _ i ) ) - C * S
 Parameters
 times , mags , errs : np . array
 The time - series to calculate the periodogram value for .
 omega : float
 The frequency to calculate the periodogram value at .
 Returns
 periodogramvalue : float
 The normalized periodogram at the specified test frequency ` omega ` .'''

implementation:
one_over_errs2 = 1.0 / ( errs * errs )
W = npsum ( one_over_errs2 )
wi = one_over_errs2 / W
sin_omegat = npsin ( omega * times )
cos_omegat = npcos ( omega * times )
sin2_omegat = sin_omegat * sin_omegat
cos2_omegat = cos_omegat * cos_omegat
sincos_omegat = sin_omegat * cos_omegat
# calculate some more sums and terms
Y = npsum ( wi * mags )
C = npsum ( wi * cos_omegat )
S = npsum ( wi * sin_omegat )
CpS = npsum ( wi * sincos_omegat )
CpC = npsum ( wi * cos2_omegat )
CS = CpS - C * S
CC = CpC - C * C
SS = 1 - CpC - S * S
# use SpS = 1 - CpC
# calculate tau
tan_omega_tau_top = 2.0 * CS
tan_omega_tau_bottom = CC - SS
tan_omega_tau = tan_omega_tau_top / tan_omega_tau_bottom
tau = nparctan ( tan_omega_tau ) / ( 2.0 * omega )
YpY = npsum ( wi * mags * mags )
YpC = npsum ( wi * mags * cos_omegat )
YpS = npsum ( wi * mags * sin_omegat )
# SpS = npsum ( wi * sin2 _ omegat )
# the final terms
YY = YpY - Y * Y
YC = YpC - Y * C
YS = YpS - Y * S
periodogramvalue = ( YC * YC / CC + YS * YS / SS ) / YY
return periodogramvalue


signature:
def generalized_lsp_value_withtau ( times , mags , errs , omega ) :
    '''Generalized LSP value for a single omega .
 This uses tau to provide an arbitrary time - reference point .
 The relations used are : :
 P ( w ) = ( 1 / YY ) * ( YC * YC / CC + YS * YS / SS )
 where : YC , YS , CC , and SS are all calculated at T
 and where : tan 2omegaT = 2 * CS / ( CC - SS )
 and where :
 Y = sum ( w _ i * y _ i )
 C = sum ( w _ i * cos ( wT _ i ) )
 S = sum ( w _ i * sin ( wT _ i ) )
 YY = sum ( w _ i * y _ i * y _ i ) - Y * Y
 YC = sum ( w _ i * y _ i * cos ( wT _ i ) ) - Y * C
 YS = sum ( w _ i * y _ i * sin ( wT _ i ) ) - Y * S
 CpC = sum ( w _ i * cos ( w _ T _ i ) * cos ( w _ T _ i ) )
 CC = CpC - C * C
 SS = ( 1 - CpC ) - S * S
 CS = sum ( w _ i * cos ( w _ T _ i ) * sin ( w _ T _ i ) ) - C * S
 Parameters
 times , mags , errs : np . array
 The time - series to calculate the periodogram value for .
 omega : float
 The frequency to calculate the periodogram value at .
 Returns
 periodogramvalue : float
 The normalized periodogram at the specified test frequency ` omega ` .'''

implementation:
one_over_errs2 = 1.0 / ( errs * errs )
W = npsum ( one_over_errs2 )
wi = one_over_errs2 / W
sin_omegat = npsin ( omega * times )
cos_omegat = npcos ( omega * times )
sin2_omegat = sin_omegat * sin_omegat
cos2_omegat = cos_omegat * cos_omegat
sincos_omegat = sin_omegat * cos_omegat
# calculate some more sums and terms
Y = npsum ( wi * mags )
C = npsum ( wi * cos_omegat )
S = npsum ( wi * sin_omegat )
CpS = npsum ( wi * sincos_omegat )
CpC = npsum ( wi * cos2_omegat )
CS = CpS - C * S
CC = CpC - C * C
SS = 1 - CpC - S * S
# use SpS = 1 - CpC
# calculate tau
tan_omega_tau_top = 2.0 * CS
tan_omega_tau_bottom = CC - SS
tan_omega_tau = tan_omega_tau_top / tan_omega_tau_bottom
tau = nparctan ( tan_omega_tau ) / ( 2.0 * omega )
# now we need to calculate all the bits at tau
sin_omega_tau = npsin ( omega * ( times - tau ) )
cos_omega_tau = npcos ( omega * ( times - tau ) )
sin2_omega_tau = sin_omega_tau * sin_omega_tau
cos2_omega_tau = cos_omega_tau * cos_omega_tau
sincos_omega_tau = sin_omega_tau * cos_omega_tau
C_tau = npsum ( wi * cos_omega_tau )
S_tau = npsum ( wi * sin_omega_tau )
CpS_tau = npsum ( wi * sincos_omega_tau )
CpC_tau = npsum ( wi * cos2_omega_tau )
CS_tau = CpS_tau - C_tau * S_tau
CC_tau = CpC_tau - C_tau * C_tau
SS_tau = 1 - CpC_tau - S_tau * S_tau
# use SpS = 1 - CpC
YpY = npsum ( wi * mags * mags )
YpC_tau = npsum ( wi * mags * cos_omega_tau )
YpS_tau = npsum ( wi * mags * sin_omega_tau )
# SpS = npsum ( wi * sin2 _ omegat )
# the final terms
YY = YpY - Y * Y
YC_tau = YpC_tau - Y * C_tau
YS_tau = YpS_tau - Y * S_tau
periodogramvalue = ( YC_tau * YC_tau / CC_tau + YS_tau * YS_tau / SS_tau ) / YY
return periodogramvalue


signature:
def generalized_lsp_value_notau ( times , mags , errs , omega ) :
    '''This is the simplified version not using tau .
 The relations used are : :
 W = sum ( 1.0 / ( errs * errs ) )
 w _ i = ( 1 / W ) * ( 1 / ( errs * errs ) )
 Y = sum ( w _ i * y _ i )
 C = sum ( w _ i * cos ( wt _ i ) )
 S = sum ( w _ i * sin ( wt _ i ) )
 YY = sum ( w _ i * y _ i * y _ i ) - Y * Y
 YC = sum ( w _ i * y _ i * cos ( wt _ i ) ) - Y * C
 YS = sum ( w _ i * y _ i * sin ( wt _ i ) ) - Y * S
 CpC = sum ( w _ i * cos ( w _ t _ i ) * cos ( w _ t _ i ) )
 CC = CpC - C * C
 SS = ( 1 - CpC ) - S * S
 CS = sum ( w _ i * cos ( w _ t _ i ) * sin ( w _ t _ i ) ) - C * S
 D ( omega ) = CC * SS - CS * CS
 P ( omega ) = ( SS * YC * YC + CC * YS * YS - 2.0 * CS * YC * YS ) / ( YY * D )
 Parameters
 times , mags , errs : np . array
 The time - series to calculate the periodogram value for .
 omega : float
 The frequency to calculate the periodogram value at .
 Returns
 periodogramvalue : float
 The normalized periodogram at the specified test frequency ` omega ` .'''

implementation:
one_over_errs2 = 1.0 / ( errs * errs )
W = npsum ( one_over_errs2 )
wi = one_over_errs2 / W
sin_omegat = npsin ( omega * times )
cos_omegat = npcos ( omega * times )
sin2_omegat = sin_omegat * sin_omegat
cos2_omegat = cos_omegat * cos_omegat
sincos_omegat = sin_omegat * cos_omegat
# calculate some more sums and terms
Y = npsum ( wi * mags )
C = npsum ( wi * cos_omegat )
S = npsum ( wi * sin_omegat )
YpY = npsum ( wi * mags * mags )
YpC = npsum ( wi * mags * cos_omegat )
YpS = npsum ( wi * mags * sin_omegat )
CpC = npsum ( wi * cos2_omegat )
# SpS = npsum ( wi * sin2 _ omegat )
CpS = npsum ( wi * sincos_omegat )
# the final terms
YY = YpY - Y * Y
YC = YpC - Y * C
YS = YpS - Y * S
CC = CpC - C * C
SS = 1 - CpC - S * S
# use SpS = 1 - CpC
CS = CpS - C * S
# P ( omega ) = ( SS * YC * YC + CC * YS * YS - 2.0 * CS * YC * YS ) / ( YY * D )
# D ( omega ) = CC * SS - CS * CS
Domega = CC * SS - CS * CS
lspval = ( SS * YC * YC + CC * YS * YS - 2.0 * CS * YC * YS ) / ( YY * Domega )
return lspval


signature:
def specwindow_lsp_value ( times , mags , errs , omega ) :
    '''This calculates the peak associated with the spectral window function
 for times and at the specified omega .
 NOTE : this is classical Lomb - Scargle , not the Generalized
 Lomb - Scargle . ` mags ` and ` errs ` are silently ignored since we ' re calculating
 the periodogram of the observing window function . These are kept to present
 a consistent external API so the ` pgen _ lsp ` function below can call this
 transparently .
 Parameters
 times , mags , errs : np . array
 The time - series to calculate the periodogram value for .
 omega : float
 The frequency to calculate the periodogram value at .
 Returns
 periodogramvalue : float
 The normalized periodogram at the specified test frequency ` omega ` .'''

implementation:
norm_times = times - times . min ( )
tau = ( ( 1.0 / ( 2.0 * omega ) ) * nparctan ( npsum ( npsin ( 2.0 * omega * norm_times ) ) / npsum ( npcos ( 2.0 * omega * norm_times ) ) ) )
lspval_top_cos = ( npsum ( 1.0 * npcos ( omega * ( norm_times - tau ) ) ) * npsum ( 1.0 * npcos ( omega * ( norm_times - tau ) ) ) )
lspval_bot_cos = npsum ( ( npcos ( omega * ( norm_times - tau ) ) ) * ( npcos ( omega * ( norm_times - tau ) ) ) )
lspval_top_sin = ( npsum ( 1.0 * npsin ( omega * ( norm_times - tau ) ) ) * npsum ( 1.0 * npsin ( omega * ( norm_times - tau ) ) ) )
lspval_bot_sin = npsum ( ( npsin ( omega * ( norm_times - tau ) ) ) * ( npsin ( omega * ( norm_times - tau ) ) ) )
lspval = 0.5 * ( ( lspval_top_cos / lspval_bot_cos ) + ( lspval_top_sin / lspval_bot_sin ) )
return lspval


signature:
def pgen_lsp ( times , mags , errs , magsarefluxes = False , startp = None , endp = None , stepsize = 1.0e-4 , autofreq = True , nbestpeaks = 5 , periodepsilon = 0.1 , sigclip = 10.0 , nworkers = None , workchunksize = None , glspfunc = _glsp_worker_withtau , verbose = True ) :
    '''This calculates the generalized Lomb - Scargle periodogram .
 Uses the algorithm from Zechmeister and Kurster ( 2009 ) .
 Parameters
 times , mags , errs : np . array
 The mag / flux time - series with associated measurement errors to run the
 period - finding on .
 magsarefluxes : bool
 If the input measurement values in ` mags ` and ` errs ` are in fluxes , set
 this to True .
 startp , endp : float or None
 The minimum and maximum periods to consider for the transit search .
 stepsize : float
 The step - size in frequency to use when constructing a frequency grid for
 the period search .
 autofreq : bool
 If this is True , the value of ` stepsize ` will be ignored and the
 : py : func : ` astrobase . periodbase . get _ frequency _ grid ` function will be used
 to generate a frequency grid based on ` startp ` , and ` endp ` . If these are
 None as well , ` startp ` will be set to 0.1 and ` endp ` will be set to
 ` times . max ( ) - times . min ( ) ` .
 nbestpeaks : int
 The number of ' best ' peaks to return from the periodogram results ,
 starting from the global maximum of the periodogram peak values .
 periodepsilon : float
 The fractional difference between successive values of ' best ' periods
 when sorting by periodogram power to consider them as separate periods
 ( as opposed to part of the same periodogram peak ) . This is used to avoid
 broad peaks in the periodogram and make sure the ' best ' periods returned
 are all actually independent .
 sigclip : float or int or sequence of two floats / ints or None
 If a single float or int , a symmetric sigma - clip will be performed using
 the number provided as the sigma - multiplier to cut out from the input
 time - series .
 If a list of two ints / floats is provided , the function will perform an
 ' asymmetric ' sigma - clip . The first element in this list is the sigma
 value to use for fainter flux / mag values ; the second element in this
 list is the sigma value to use for brighter flux / mag values . For
 example , ` sigclip = [ 10 . , 3 . ] ` , will sigclip out greater than 10 - sigma
 dimmings and greater than 3 - sigma brightenings . Here the meaning of
 " dimming " and " brightening " is set by * physics * ( not the magnitude
 system ) , which is why the ` magsarefluxes ` kwarg must be correctly set .
 If ` sigclip ` is None , no sigma - clipping will be performed , and the
 time - series ( with non - finite elems removed ) will be passed through to
 the output .
 nworkers : int
 The number of parallel workers to use when calculating the periodogram .
 workchunksize : None or int
 If this is an int , will use chunks of the given size to break up the
 work for the parallel workers . If None , the chunk size is set to 1.
 glspfunc : Python function
 The worker function to use to calculate the periodogram . This can be
 used to make this function calculate the time - series sampling window
 function instead of the time - series measurements ' GLS periodogram by
 passing in ` _ glsp _ worker _ specwindow ` instead of the default
 ` _ glsp _ worker _ withtau ` function .
 verbose : bool
 If this is True , will indicate progress and details about the frequency
 grid used for the period search .
 Returns
 dict
 This function returns a dict , referred to as an ` lspinfo ` dict in other
 astrobase functions that operate on periodogram results . This is a
 standardized format across all astrobase period - finders , and is of the
 form below : :
 { ' bestperiod ' : the best period value in the periodogram ,
 ' bestlspval ' : the periodogram peak associated with the best period ,
 ' nbestpeaks ' : the input value of nbestpeaks ,
 ' nbestlspvals ' : nbestpeaks - size list of best period peak values ,
 ' nbestperiods ' : nbestpeaks - size list of best periods ,
 ' lspvals ' : the full array of periodogram powers ,
 ' periods ' : the full array of periods considered ,
 ' method ' : ' gls ' - > the name of the period - finder method ,
 ' kwargs ' : { dict of all of the input kwargs for record - keeping } }'''

implementation:
# get rid of nans first and sigclip
stimes , smags , serrs = sigclip_magseries ( times , mags , errs , magsarefluxes = magsarefluxes , sigclip = sigclip )
# get rid of zero errs
nzind = npnonzero ( serrs )
stimes , smags , serrs = stimes [ nzind ] , smags [ nzind ] , serrs [ nzind ]
# make sure there are enough points to calculate a spectrum
if len ( stimes ) > 9 and len ( smags ) > 9 and len ( serrs ) > 9 : # get the frequencies to use
    if startp :
        endf = 1.0 / startp
    else : # default start period is 0.1 day
        endf = 1.0 / 0.1
    if endp :
        startf = 1.0 / endp
    else : # default end period is length of time series
        startf = 1.0 / ( stimes . max ( ) - stimes . min ( ) )
    # if we ' re not using autofreq , then use the provided frequencies
    if not autofreq :
        omegas = 2 * pi_value * nparange ( startf , endf , stepsize )
        if verbose :
            LOGINFO ( 'using %s frequency points, start P = %.3f, end P = %.3f' % ( omegas . size , 1.0 / endf , 1.0 / startf ) )
    else : # this gets an automatic grid of frequencies to use
        freqs = get_frequency_grid ( stimes , minfreq = startf , maxfreq = endf )
        omegas = 2 * pi_value * freqs
        if verbose :
            LOGINFO ( 'using autofreq with %s frequency points, ' 'start P = %.3f, end P = %.3f' % ( omegas . size , 1.0 / freqs . max ( ) , 1.0 / freqs . min ( ) ) )
    # map to parallel workers
    if ( not nworkers ) or ( nworkers > NCPUS ) :
        nworkers = NCPUS
        if verbose :
            LOGINFO ( 'using %s workers...' % nworkers )
    pool = Pool ( nworkers )
    tasks = [ ( stimes , smags , serrs , x ) for x in omegas ]
    if workchunksize :
        lsp = pool . map ( glspfunc , tasks , chunksize = workchunksize )
    else :
        lsp = pool . map ( glspfunc , tasks )
    pool . close ( )
    pool . join ( )
    del pool
    lsp = nparray ( lsp )
    periods = 2.0 * pi_value / omegas
    # find the nbestpeaks for the periodogram : 1 . sort the lsp array by
    # highest value first 2 . go down the values until we find five
    # values that are separated by at least periodepsilon in period
    # make sure to filter out non - finite values of lsp
    finitepeakind = npisfinite ( lsp )
    finlsp = lsp [ finitepeakind ]
    finperiods = periods [ finitepeakind ]
    # make sure that finlsp has finite values before we work on it
    try :
        bestperiodind = npargmax ( finlsp )
    except ValueError :
        LOGERROR ( 'no finite periodogram values ' 'for this mag series, skipping...' )
        return { 'bestperiod' : npnan , 'bestlspval' : npnan , 'nbestpeaks' : nbestpeaks , 'nbestlspvals' : None , 'nbestperiods' : None , 'lspvals' : None , 'omegas' : omegas , 'periods' : None , 'method' : 'gls' , 'kwargs' : { 'startp' : startp , 'endp' : endp , 'stepsize' : stepsize , 'autofreq' : autofreq , 'periodepsilon' : periodepsilon , 'nbestpeaks' : nbestpeaks , 'sigclip' : sigclip } }
    sortedlspind = npargsort ( finlsp ) [ : : - 1 ]
    sortedlspperiods = finperiods [ sortedlspind ]
    sortedlspvals = finlsp [ sortedlspind ]
    # now get the nbestpeaks
    nbestperiods , nbestlspvals , peakcount = ( [ finperiods [ bestperiodind ] ] , [ finlsp [ bestperiodind ] ] , 1 )
    prevperiod = sortedlspperiods [ 0 ]
    # find the best nbestpeaks in the lsp and their periods
    for period , lspval in zip ( sortedlspperiods , sortedlspvals ) :
        if peakcount == nbestpeaks :
            break
        perioddiff = abs ( period - prevperiod )
        bestperiodsdiff = [ abs ( period - x ) for x in nbestperiods ]
        # print ( ' prevperiod = % s , thisperiod = % s , '
        # ' perioddiff = % s , peakcount = % s ' %
        # ( prevperiod , period , perioddiff , peakcount ) )
        # this ensures that this period is different from the last
        # period and from all the other existing best periods by
        # periodepsilon to make sure we jump to an entire different peak
        # in the periodogram
        if ( perioddiff > ( periodepsilon * prevperiod ) and all ( x > ( periodepsilon * period ) for x in bestperiodsdiff ) ) :
            nbestperiods . append ( period )
            nbestlspvals . append ( lspval )
            peakcount = peakcount + 1
        prevperiod = period
    return { 'bestperiod' : finperiods [ bestperiodind ] , 'bestlspval' : finlsp [ bestperiodind ] , 'nbestpeaks' : nbestpeaks , 'nbestlspvals' : nbestlspvals , 'nbestperiods' : nbestperiods , 'lspvals' : lsp , 'omegas' : omegas , 'periods' : periods , 'method' : 'gls' , 'kwargs' : { 'startp' : startp , 'endp' : endp , 'stepsize' : stepsize , 'autofreq' : autofreq , 'periodepsilon' : periodepsilon , 'nbestpeaks' : nbestpeaks , 'sigclip' : sigclip } }
else :
    LOGERROR ( 'no good detections for these times and mags, skipping...' )
    return { 'bestperiod' : npnan , 'bestlspval' : npnan , 'nbestpeaks' : nbestpeaks , 'nbestlspvals' : None , 'nbestperiods' : None , 'lspvals' : None , 'omegas' : None , 'periods' : None , 'method' : 'gls' , 'kwargs' : { 'startp' : startp , 'endp' : endp , 'stepsize' : stepsize , 'autofreq' : autofreq , 'periodepsilon' : periodepsilon , 'nbestpeaks' : nbestpeaks , 'sigclip' : sigclip } }


signature:
def specwindow_lsp ( times , mags , errs , magsarefluxes = False , startp = None , endp = None , stepsize = 1.0e-4 , autofreq = True , nbestpeaks = 5 , periodepsilon = 0.1 , sigclip = 10.0 , nworkers = None , glspfunc = _glsp_worker_specwindow , verbose = True ) :
    '''This calculates the spectral window function .
 Wraps the ` pgen _ lsp ` function above to use the specific worker for
 calculating the window - function .
 Parameters
 times , mags , errs : np . array
 The mag / flux time - series with associated measurement errors to run the
 period - finding on .
 magsarefluxes : bool
 If the input measurement values in ` mags ` and ` errs ` are in fluxes , set
 this to True .
 startp , endp : float or None
 The minimum and maximum periods to consider for the transit search .
 stepsize : float
 The step - size in frequency to use when constructing a frequency grid for
 the period search .
 autofreq : bool
 If this is True , the value of ` stepsize ` will be ignored and the
 : py : func : ` astrobase . periodbase . get _ frequency _ grid ` function will be used
 to generate a frequency grid based on ` startp ` , and ` endp ` . If these are
 None as well , ` startp ` will be set to 0.1 and ` endp ` will be set to
 ` times . max ( ) - times . min ( ) ` .
 nbestpeaks : int
 The number of ' best ' peaks to return from the periodogram results ,
 starting from the global maximum of the periodogram peak values .
 periodepsilon : float
 The fractional difference between successive values of ' best ' periods
 when sorting by periodogram power to consider them as separate periods
 ( as opposed to part of the same periodogram peak ) . This is used to avoid
 broad peaks in the periodogram and make sure the ' best ' periods returned
 are all actually independent .
 sigclip : float or int or sequence of two floats / ints or None
 If a single float or int , a symmetric sigma - clip will be performed using
 the number provided as the sigma - multiplier to cut out from the input
 time - series .
 If a list of two ints / floats is provided , the function will perform an
 ' asymmetric ' sigma - clip . The first element in this list is the sigma
 value to use for fainter flux / mag values ; the second element in this
 list is the sigma value to use for brighter flux / mag values . For
 example , ` sigclip = [ 10 . , 3 . ] ` , will sigclip out greater than 10 - sigma
 dimmings and greater than 3 - sigma brightenings . Here the meaning of
 " dimming " and " brightening " is set by * physics * ( not the magnitude
 system ) , which is why the ` magsarefluxes ` kwarg must be correctly set .
 If ` sigclip ` is None , no sigma - clipping will be performed , and the
 time - series ( with non - finite elems removed ) will be passed through to
 the output .
 nworkers : int
 The number of parallel workers to use when calculating the periodogram .
 glspfunc : Python function
 The worker function to use to calculate the periodogram . This is used to
 used to make the ` pgen _ lsp ` function calculate the time - series sampling
 window function instead of the time - series measurements ' GLS periodogram
 by passing in ` _ glsp _ worker _ specwindow ` instead of the default
 ` _ glsp _ worker ` function .
 verbose : bool
 If this is True , will indicate progress and details about the frequency
 grid used for the period search .
 Returns
 dict
 This function returns a dict , referred to as an ` lspinfo ` dict in other
 astrobase functions that operate on periodogram results . This is a
 standardized format across all astrobase period - finders , and is of the
 form below : :
 { ' bestperiod ' : the best period value in the periodogram ,
 ' bestlspval ' : the periodogram peak associated with the best period ,
 ' nbestpeaks ' : the input value of nbestpeaks ,
 ' nbestlspvals ' : nbestpeaks - size list of best period peak values ,
 ' nbestperiods ' : nbestpeaks - size list of best periods ,
 ' lspvals ' : the full array of periodogram powers ,
 ' periods ' : the full array of periods considered ,
 ' method ' : ' win ' - > the name of the period - finder method ,
 ' kwargs ' : { dict of all of the input kwargs for record - keeping } }'''

implementation:
# run the LSP using glsp _ worker _ specwindow as the worker
lspres = pgen_lsp ( times , mags , errs , magsarefluxes = magsarefluxes , startp = startp , endp = endp , autofreq = autofreq , nbestpeaks = nbestpeaks , periodepsilon = periodepsilon , stepsize = stepsize , nworkers = nworkers , sigclip = sigclip , glspfunc = glspfunc , verbose = verbose )
# update the resultdict to indicate we ' re a spectral window function
lspres [ 'method' ] = 'win'
if lspres [ 'lspvals' ] is not None : # renormalize the periodogram to between 0 and 1 like the usual GLS .
    lspmax = npnanmax ( lspres [ 'lspvals' ] )
    if npisfinite ( lspmax ) :
        lspres [ 'lspvals' ] = lspres [ 'lspvals' ] / lspmax
        lspres [ 'nbestlspvals' ] = [ x / lspmax for x in lspres [ 'nbestlspvals' ] ]
        lspres [ 'bestlspval' ] = lspres [ 'bestlspval' ] / lspmax
return lspres


signature:
def analytic_false_alarm_probability ( lspinfo , times , conservative_nfreq_eff = True , peakvals = None , inplace = True ) :
    '''This returns the analytic false alarm probabilities for periodogram
 peak values .
 The calculation follows that on page 3 of Zechmeister & Kurster ( 2009 ) : :
 FAP = 1 − [ 1 − Prob ( z > z0 ) ] * * M
 where : :
 M is the number of independent frequencies
 Prob ( z > z0 ) is the probability of peak with value > z0
 z0 is the peak value we ' re evaluating
 Parameters
 lspinfo : dict
 The dict returned by the : py : func : ` ~ astrobase . periodbase . zgls . pgen _ lsp `
 function .
 times : np . array
 The times for which the periodogram result in ` ` lspinfo ` ` was
 calculated .
 conservative _ nfreq _ eff : bool
 If True , will follow the prescription given in Schwarzenberg - Czerny
 (2003 ) :
 http : / / adsabs . harvard . edu / abs / 2003ASPC . . 292 . . 383S
 and estimate the effective number of independent frequences M _ eff as : :
 min ( N _ obs , N _ freq , DELTA _ f / delta _ f )
 peakvals : sequence or None
 The peak values for which to evaluate the false - alarm probability . If
 None , will calculate this for each of the peak values in the
 ` ` nbestpeaks ` ` key of the ` ` lspinfo ` ` dict .
 inplace : bool
 If True , puts the results of the FAP calculation into the ` ` lspinfo ` `
 dict as a list available as ` ` lspinfo [ ' falsealarmprob ' ] ` ` .
 Returns
 list
 The calculated false alarm probabilities for each of the peak values in
 ` ` peakvals ` ` .'''

implementation:
frequencies = 1.0 / lspinfo [ 'periods' ]
M = independent_freq_count ( frequencies , times , conservative = conservative_nfreq_eff )
if peakvals is None :
    peakvals = lspinfo [ 'nbestlspvals' ]
prob_exceed_vals = [ probability_peak_exceeds_value ( times , p ) for p in peakvals ]
false_alarm_probs = [ 1.0 - ( 1.0 - prob_exc ) ** M for prob_exc in prob_exceed_vals ]
if inplace :
    lspinfo [ 'falsealarmprob' ] = false_alarm_probs
return false_alarm_probs


signature:
def check_existing_apikey ( lcc_server ) :
    '''This validates if an API key for the specified LCC - Server is available .
 API keys are stored using the following file scheme : :
 ~ / . astrobase / lccs / apikey - domain . of . lccserver . org
 e . g . for the HAT LCC - Server at https : / / data . hatsurveys . org : :
 ~ / . astrobase / lccs / apikey - https - data . hatsurveys . org
 Parameters
 lcc _ server : str
 The base URL of the LCC - Server for which the existence of API keys will
 be checked .
 Returns
 ( apikey _ ok , apikey _ str , expiry ) : tuple
 The returned tuple contains the status of the API key , the API key
 itself if present , and its expiry date if present .'''

implementation:
USERHOME = os . path . expanduser ( '~' )
APIKEYFILE = os . path . join ( USERHOME , '.astrobase' , 'lccs' , 'apikey-%s' % lcc_server . replace ( 'https://' , 'https-' ) . replace ( 'http://' , 'http-' ) )
if os . path . exists ( APIKEYFILE ) : # check if this file is readable / writeable by user only
    fileperm = oct ( os . stat ( APIKEYFILE ) [ stat . ST_MODE ] )
    if fileperm == '0100600' or fileperm == '0o100600' :
        with open ( APIKEYFILE ) as infd :
            apikey , expires = infd . read ( ) . strip ( '\n' ) . split ( )
        # get today ' s datetime
        now = datetime . now ( utc )
        if sys . version_info [ : 2 ] < ( 3 , 7 ) : # this hideous incantation is required for lesser Pythons
            expdt = datetime . strptime ( expires . replace ( 'Z' , '' ) , '%Y-%m-%dT%H:%M:%S.%f' ) . replace ( tzinfo = utc )
        else :
            expdt = datetime . fromisoformat ( expires . replace ( 'Z' , '+00:00' ) )
        if now > expdt :
            LOGERROR ( 'API key has expired. expiry was on: %s' % expires )
            return False , apikey , expires
        else :
            return True , apikey , expires
    else :
        LOGWARNING ( 'The API key file %s has bad permissions ' 'and is insecure, not reading it.\n' '(you need to chmod 600 this file)' % APIKEYFILE )
        return False , None , None
else :
    LOGWARNING ( 'No LCC-Server API key ' 'found in: {apikeyfile}' . format ( apikeyfile = APIKEYFILE ) )
    return False , None , None


signature:
def get_new_apikey ( lcc_server ) :
    '''This gets a new API key from the specified LCC - Server .
 NOTE : this only gets an anonymous API key . To get an API key tied to a user
 account ( and associated privilege level ) , see the ` import _ apikey ` function
 below .
 Parameters
 lcc _ server : str
 The base URL of the LCC - Server from where the API key will be fetched .
 Returns
 ( apikey , expiry ) : tuple
 This returns a tuple with the API key and its expiry date .'''

implementation:
USERHOME = os . path . expanduser ( '~' )
APIKEYFILE = os . path . join ( USERHOME , '.astrobase' , 'lccs' , 'apikey-%s' % lcc_server . replace ( 'https://' , 'https-' ) . replace ( 'http://' , 'http-' ) )
# url for getting an API key
url = '%s/api/key' % lcc_server
# get the API key
resp = urlopen ( url )
if resp . code == 200 :
    respdict = json . loads ( resp . read ( ) )
else :
    LOGERROR ( 'could not fetch the API key from LCC-Server at: %s' % lcc_server )
    LOGERROR ( 'the HTTP status code was: %s' % resp . status_code )
    return None
# now that we have an API key dict , get the API key out of it and write it
# to the APIKEYFILE
apikey = respdict [ 'result' ] [ 'apikey' ]
expires = respdict [ 'result' ] [ 'expires' ]
# write this to the apikey file
if not os . path . exists ( os . path . dirname ( APIKEYFILE ) ) :
    os . makedirs ( os . path . dirname ( APIKEYFILE ) )
with open ( APIKEYFILE , 'w' ) as outfd :
    outfd . write ( '%s %s\n' % ( apikey , expires ) )
# chmod it to the correct value
os . chmod ( APIKEYFILE , 0o100600 )
LOGINFO ( 'key fetched successfully from: %s. expires on: %s' % ( lcc_server , expires ) )
LOGINFO ( 'written to: %s' % APIKEYFILE )
return apikey , expires


signature:
def import_apikey ( lcc_server , apikey_text_json ) :
    '''This imports an API key from text and writes it to the cache dir .
 Use this with the JSON text copied from the API key text box on your
 LCC - Server user home page . The API key will thus be tied to the privileges
 of that user account and can then access objects , datasets , and collections
 marked as private for the user only or shared with that user .
 Parameters
 lcc _ server : str
 The base URL of the LCC - Server to get the API key for .
 apikey _ text _ json : str
 The JSON string from the API key text box on the user ' s LCC - Server home
 page at ` lcc _ server / users / home ` .
 Returns
 ( apikey , expiry ) : tuple
 This returns a tuple with the API key and its expiry date .'''

implementation:
USERHOME = os . path . expanduser ( '~' )
APIKEYFILE = os . path . join ( USERHOME , '.astrobase' , 'lccs' , 'apikey-%s' % lcc_server . replace ( 'https://' , 'https-' ) . replace ( 'http://' , 'http-' ) )
respdict = json . loads ( apikey_text_json )
# now that we have an API key dict , get the API key out of it and write it
# to the APIKEYFILE
apikey = respdict [ 'apikey' ]
expires = respdict [ 'expires' ]
# write this to the apikey file
if not os . path . exists ( os . path . dirname ( APIKEYFILE ) ) :
    os . makedirs ( os . path . dirname ( APIKEYFILE ) )
with open ( APIKEYFILE , 'w' ) as outfd :
    outfd . write ( '%s %s\n' % ( apikey , expires ) )
# chmod it to the correct value
os . chmod ( APIKEYFILE , 0o100600 )
LOGINFO ( 'key fetched successfully from: %s. expires on: %s' % ( lcc_server , expires ) )
LOGINFO ( 'written to: %s' % APIKEYFILE )
return apikey , expires


signature:
def submit_post_searchquery ( url , data , apikey ) :
    '''This submits a POST query to an LCC - Server search API endpoint .
 Handles streaming of the results , and returns the final JSON stream . Also
 handles results that time out .
 Parameters
 url : str
 The URL of the search API endpoint to hit . This is something like
 ` https : / / data . hatsurveys . org / api / conesearch `
 data : dict
 A dict of the search query parameters to pass to the search service .
 apikey : str
 The API key to use to access the search service . API keys are required
 for all POST request made to an LCC - Server ' s API endpoints .
 Returns
 ( status _ flag , data _ dict , dataset _ id ) : tuple
 This returns a tuple containing the status of the request : ( ' complete ' ,
 ' failed ' , ' background ' , etc . ) , a dict parsed from the JSON result of the
 request , and a dataset ID , which can be used to reconstruct the URL on
 the LCC - Server where the results can be browsed .'''

implementation:
# first , we need to convert any columns and collections items to broken out
# params
postdata = { }
for key in data :
    if key == 'columns' :
        postdata [ 'columns[]' ] = data [ key ]
    elif key == 'collections' :
        postdata [ 'collections[]' ] = data [ key ]
    else :
        postdata [ key ] = data [ key ]
# do the urlencode with doseq = True
# we also need to encode to bytes
encoded_postdata = urlencode ( postdata , doseq = True ) . encode ( )
# if apikey is not None , add it in as an Authorization : Bearer [ apikey ]
# header
if apikey :
    headers = { 'Authorization' : 'Bearer: %s' % apikey }
else :
    headers = { }
LOGINFO ( 'submitting search query to LCC-Server API URL: %s' % url )
try : # hit the server with a POST request
    req = Request ( url , data = encoded_postdata , headers = headers )
    resp = urlopen ( req )
    if resp . code == 200 : # we ' ll iterate over the lines in the response
    # this works super - well for ND - JSON !
        for line in resp :
            data = json . loads ( line )
            msg = data [ 'message' ]
            status = data [ 'status' ]
            if status != 'failed' :
                LOGINFO ( 'status: %s, %s' % ( status , msg ) )
            else :
                LOGERROR ( 'status: %s, %s' % ( status , msg ) )
            # here , we ' ll decide what to do about the query
            # completed query or query sent to background . . .
            if status in ( 'ok' , 'background' ) :
                setid = data [ 'result' ] [ 'setid' ]
                # save the data pickle to astrobase lccs directory
                outpickle = os . path . join ( os . path . expanduser ( '~' ) , '.astrobase' , 'lccs' , 'query-%s.pkl' % setid )
                if not os . path . exists ( os . path . dirname ( outpickle ) ) :
                    os . makedirs ( os . path . dirname ( outpickle ) )
                with open ( outpickle , 'wb' ) as outfd :
                    pickle . dump ( data , outfd , pickle . HIGHEST_PROTOCOL )
                    LOGINFO ( 'saved query info to %s, use this to ' 'download results later with ' 'retrieve_dataset_files' % outpickle )
                # we ' re done at this point , return
                return status , data , data [ 'result' ] [ 'setid' ]
            # the query probably failed . . .
            elif status == 'failed' : # we ' re done at this point , return
                return status , data , None
    # if the response was not OK , then we probably failed
    else :
        try :
            data = json . load ( resp )
            msg = data [ 'message' ]
            LOGERROR ( msg )
            return 'failed' , None , None
        except Exception as e :
            LOGEXCEPTION ( 'failed to submit query to %s' % url )
            return 'failed' , None , None
except HTTPError as e :
    LOGERROR ( 'could not submit query to LCC API at: %s' % url )
    LOGERROR ( 'HTTP status code was %s, reason: %s' % ( e . code , e . reason ) )
    return 'failed' , None , None