signature:
def runcp_producer_loop_savedstate ( use_saved_state = None , lightcurve_list = None , input_queue = None , input_bucket = None , result_queue = None , result_bucket = None , pfresult_list = None , runcp_kwargs = None , process_list_slice = None , download_when_done = True , purge_queues_when_done = True , save_state_when_done = True , delete_queues_when_done = False , s3_client = None , sqs_client = None ) :
    """This wraps the function above to allow for loading previous state from a
 file .
 Parameters
 use _ saved _ state : str or None
 This is the path to the saved state pickle file produced by a previous
 run of ` runcp _ producer _ loop ` . Will get all of the arguments to run
 another instance of the loop from that pickle file . If this is None , you
 MUST provide all of the appropriate arguments to that function .
 lightcurve _ list : str or list of str or None
 This is either a string pointing to a file containing a list of light
 curves filenames to process or the list itself . The names must
 correspond to the full filenames of files stored on S3 , including all
 prefixes , but not include the ' s3 : / / < bucket name > / ' bit ( these will be
 added automatically ) .
 input _ queue : str or None
 This is the name of the SQS queue which will receive processing tasks
 generated by this function . The queue URL will automatically be obtained
 from AWS .
 input _ bucket : str or None
 The name of the S3 bucket containing the light curve files to process .
 result _ queue : str or None
 This is the name of the SQS queue that this function will listen to for
 messages from the workers as they complete processing on their input
 elements . This function will attempt to match input sent to the
 ` input _ queue ` with results coming into the ` result _ queue ` so it knows
 how many objects have been successfully processed . If this function
 receives task results that aren ' t in its own input queue , it will
 acknowledge them so they complete successfully , but not download them
 automatically . This handles leftover tasks completing from a previous
 run of this function .
 result _ bucket : str or None
 The name of the S3 bucket which will receive the results from the
 workers .
 pfresult _ list : list of str or None
 This is a list of periodfinder result pickle S3 URLs associated with
 each light curve . If provided , this will be used to add in phased light
 curve plots to each checkplot pickle . If this is None , the worker loop
 will produce checkplot pickles that only contain object information ,
 neighbor information , and unphased light curves .
 runcp _ kwargs : dict or None
 This is a dict used to pass any extra keyword arguments to the
 ` lcproc . checkplotgen . runcp ` function that will be run by the worker
 loop .
 process _ list _ slice : list or None
 This is used to index into the input light curve list so a subset of the
 full list can be processed in this specific run of this function .
 Use None for a slice index elem to emulate single slice spec behavior :
 process _ list _ slice = [ 10 , None ] - > lightcurve _ list [ 10 : ]
 process _ list _ slice = [ None , 500 ] - > lightcurve _ list [ : 500]
 purge _ queues _ when _ done : bool or None
 If this is True , and this function exits ( either when all done , or when
 it is interrupted with a Ctrl + C ) , all outstanding elements in the
 input / output queues that have not yet been acknowledged by workers or by
 this function will be purged . This effectively cancels all outstanding
 work .
 delete _ queues _ when _ done : bool or None
 If this is True , and this function exits ( either when all done , or when
 it is interrupted with a Ctrl + C ' ) , all outstanding work items will be
 purged from the input / queues and the queues themselves will be deleted .
 download _ when _ done : bool or None
 If this is True , the generated checkplot pickle for each input work item
 will be downloaded immediately to the current working directory when the
 worker functions report they ' re done with it .
 save _ state _ when _ done : bool or None
 If this is True , will save the current state of the work item queue and
 the work items acknowledged as completed to a pickle in the current
 working directory . Call the ` runcp _ producer _ loop _ savedstate ` function
 below to resume processing from this saved state later .
 s3 _ client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its S3 download operations . Alternatively , pass in an existing
 ` boto3 . Client ` instance to re - use it here .
 sqs _ client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its SQS operations . Alternatively , pass in an existing
 ` boto3 . Client ` instance to re - use it here .
 Returns
 dict or str
 Returns the current work state as a dict or str path to the generated
 work state pickle depending on if ` save _ state _ when _ done ` is True ."""

implementation:
if use_saved_state is not None and os . path . exists ( use_saved_state ) :
    with open ( use_saved_state , 'rb' ) as infd :
        saved_state = pickle . load ( infd )
    # run the producer loop using the saved state ' s todo list
    return runcp_producer_loop ( saved_state [ 'in_progress' ] , saved_state [ 'args' ] [ 1 ] , saved_state [ 'args' ] [ 2 ] , saved_state [ 'args' ] [ 3 ] , saved_state [ 'args' ] [ 4 ] , ** saved_state [ 'kwargs' ] )
else :
    return runcp_producer_loop ( lightcurve_list , input_queue , input_bucket , result_queue , result_bucket , pfresult_list = pfresult_list , runcp_kwargs = runcp_kwargs , process_list_slice = process_list_slice , download_when_done = download_when_done , purge_queues_when_done = purge_queues_when_done , save_state_when_done = save_state_when_done , delete_queues_when_done = delete_queues_when_done , s3_client = s3_client , sqs_client = sqs_client )


signature:
def runcp_consumer_loop ( in_queue_url , workdir , lclist_pkl_s3url , lc_altexts = ( '' , ) , wait_time_seconds = 5 , cache_clean_timer_seconds = 3600.0 , shutdown_check_timer_seconds = 60.0 , sqs_client = None , s3_client = None ) :
    """This runs checkplot pickle making in a loop until interrupted .
 Consumes work task items from an input queue set up by ` runcp _ producer _ loop `
 above . For the moment , we don ' t generate neighbor light curves since this
 would require a lot more S3 calls .
 Parameters
 in _ queue _ url : str
 The SQS URL of the input queue to listen to for work assignment
 messages . The task orders will include the input and output S3 bucket
 names , as well as the URL of the output queue to where this function
 will report its work - complete or work - failed status .
 workdir : str
 The directory on the local machine where this worker loop will download
 the input light curves and associated period - finder results ( if any ) ,
 process them , and produce its output checkplot pickles . These will then
 be uploaded to the specified S3 output bucket and then deleted from the
 workdir when the upload is confirmed to make it safely to S3.
 lclist _ pkl : str
 S3 URL of a catalog pickle generated by ` lcproc . catalogs . make _ lclist `
 that contains objectids and coordinates , as well as a kdtree for all of
 the objects in the current light curve collection being processed . This
 is used to look up neighbors for each object being processed .
 lc _ altexts : sequence of str
 If not None , this is a sequence of alternate extensions to try for the
 input light curve file other than the one provided in the input task
 order . For example , to get anything that ' s an . sqlite where . sqlite . gz
 is expected , use altexts = [ ' ' ] to strip the . gz .
 wait _ time _ seconds : int
 The amount of time to wait in the input SQS queue for an input task
 order . If this timeout expires and no task has been received , this
 function goes back to the top of the work loop .
 cache _ clean _ timer _ seconds : float
 The amount of time in seconds to wait before periodically removing old
 files ( such as finder chart FITS , external service result pickles ) from
 the astrobase cache directory . These accumulate as the work items are
 processed , and take up significant space , so must be removed
 periodically .
 shutdown _ check _ timer _ seconds : float
 The amount of time to wait before checking for a pending EC2 shutdown
 message for the instance this worker loop is operating on . If a shutdown
 is noticed , the worker loop is cancelled in preparation for instance
 shutdown .
 sqs _ client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its SQS operations . Alternatively , pass in an existing
 ` boto3 . Client ` instance to re - use it here .
 s3 _ client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its S3 operations . Alternatively , pass in an existing
 ` boto3 . Client ` instance to re - use it here .
 Returns
 Nothing ."""

implementation:
if not sqs_client :
    sqs_client = boto3 . client ( 'sqs' )
if not s3_client :
    s3_client = boto3 . client ( 's3' )
lclist_pklf = lclist_pkl_s3url . split ( '/' ) [ - 1 ]
if not os . path . exists ( lclist_pklf ) : # get the lclist pickle from S3 to help with neighbor queries
    lclist_pklf = awsutils . s3_get_url ( lclist_pkl_s3url , client = s3_client )
with open ( lclist_pklf , 'rb' ) as infd :
    lclistpkl = pickle . load ( infd )
# listen to the kill and term signals and raise KeyboardInterrupt when
# called
signal . signal ( signal . SIGINT , kill_handler )
signal . signal ( signal . SIGTERM , kill_handler )
shutdown_last_time = time . monotonic ( )
diskspace_last_time = time . monotonic ( )
while True :
    curr_time = time . monotonic ( )
    if ( curr_time - shutdown_last_time ) > shutdown_check_timer_seconds :
        shutdown_check = shutdown_check_handler ( )
        if shutdown_check :
            LOGWARNING ( 'instance will die soon, breaking loop' )
            break
        shutdown_last_time = time . monotonic ( )
    if ( curr_time - diskspace_last_time ) > cache_clean_timer_seconds :
        cache_clean_handler ( )
        diskspace_last_time = time . monotonic ( )
    try : # receive a single message from the inqueue
        work = awsutils . sqs_get_item ( in_queue_url , client = sqs_client , raiseonfail = True )
        # JSON deserialize the work item
        if work is not None and len ( work ) > 0 :
            recv = work [ 0 ]
            # skip any messages that don ' t tell us to runcp
            # FIXME : use the MessageAttributes for setting topics instead
            action = recv [ 'item' ] [ 'action' ]
            if action != 'runcp' :
                continue
            target = recv [ 'item' ] [ 'target' ]
            args = recv [ 'item' ] [ 'args' ]
            kwargs = recv [ 'item' ] [ 'kwargs' ]
            outbucket = recv [ 'item' ] [ 'outbucket' ]
            if 'outqueue' in recv [ 'item' ] :
                out_queue_url = recv [ 'item' ] [ 'outqueue' ]
            else :
                out_queue_url = None
            receipt = recv [ 'receipt_handle' ]
            # download the target from S3 to a file in the work directory
            try :
                lc_filename = awsutils . s3_get_url ( target , altexts = lc_altexts , client = s3_client , )
                # get the period - finder pickle if present in args
                if len ( args ) > 0 and args [ 0 ] is not None :
                    pf_pickle = awsutils . s3_get_url ( args [ 0 ] , client = s3_client )
                else :
                    pf_pickle = None
                # now runcp
                cpfs = runcp ( pf_pickle , workdir , workdir , lcfname = lc_filename , lclistpkl = lclistpkl , makeneighborlcs = False , ** kwargs )
                if cpfs and all ( os . path . exists ( x ) for x in cpfs ) :
                    LOGINFO ( 'runcp OK for LC: %s, PF: %s -> %s' % ( lc_filename , pf_pickle , cpfs ) )
                    # check if the file exists already because it ' s been
                    # processed somewhere else
                    resp = s3_client . list_objects_v2 ( Bucket = outbucket , MaxKeys = 1 , Prefix = cpfs [ 0 ] )
                    outbucket_list = resp . get ( 'Contents' , [ ] )
                    if outbucket_list and len ( outbucket_list ) > 0 :
                        LOGWARNING ( 'not uploading runcp results for %s because ' 'they exist in the output bucket already' % target )
                        awsutils . sqs_delete_item ( in_queue_url , receipt )
                        continue
                    for cpf in cpfs :
                        put_url = awsutils . s3_put_file ( cpf , outbucket , client = s3_client )
                        if put_url is not None :
                            LOGINFO ( 'result uploaded to %s' % put_url )
                            # put the S3 URL of the output into the output
                            # queue if requested
                            if out_queue_url is not None :
                                awsutils . sqs_put_item ( out_queue_url , { 'cpf' : put_url , 'target' : target , 'lc_filename' : lc_filename , 'lclistpkl' : lclist_pklf , 'kwargs' : kwargs } , raiseonfail = True )
                            # delete the result from the local directory
                            os . remove ( cpf )
                        # if the upload fails , don ' t acknowledge the
                        # message . might be a temporary S3 failure , so
                        # another worker might succeed later .
                        else :
                            LOGERROR ( 'failed to upload %s to S3' % cpf )
                    # delete the input item from the input queue to
                    # acknowledge its receipt and indicate that
                    # processing is done and successful
                    awsutils . sqs_delete_item ( in_queue_url , receipt )
                    # delete the light curve file when we ' re done with it
                    if ( ( lc_filename is not None ) and ( os . path . exists ( lc_filename ) ) ) :
                        os . remove ( lc_filename )
                # if runcp failed outright , don ' t requeue . instead , write a
                # ( ' failed - checkplot - % s . pkl ' % lc _ filename ) file to the
                # output S3 bucket .
                else :
                    LOGWARNING ( 'runcp failed for LC: %s, PF: %s' % ( lc_filename , pf_pickle ) )
                    with open ( 'failed-checkplot-%s.pkl' % lc_filename , 'wb' ) as outfd :
                        pickle . dump ( { 'in_queue_url' : in_queue_url , 'target' : target , 'lc_filename' : lc_filename , 'lclistpkl' : lclist_pklf , 'kwargs' : kwargs , 'outbucket' : outbucket , 'out_queue_url' : out_queue_url } , outfd , pickle . HIGHEST_PROTOCOL )
                    put_url = awsutils . s3_put_file ( 'failed-checkplot-%s.pkl' % lc_filename , outbucket , client = s3_client )
                    # put the S3 URL of the output into the output
                    # queue if requested
                    if out_queue_url is not None :
                        awsutils . sqs_put_item ( out_queue_url , { 'cpf' : put_url , 'lc_filename' : lc_filename , 'lclistpkl' : lclist_pklf , 'kwargs' : kwargs } , raiseonfail = True )
                    # delete the input item from the input queue to
                    # acknowledge its receipt and indicate that
                    # processing is done
                    awsutils . sqs_delete_item ( in_queue_url , receipt , raiseonfail = True )
                    # delete the light curve file when we ' re done with it
                    if ( ( lc_filename is not None ) and ( os . path . exists ( lc_filename ) ) ) :
                        os . remove ( lc_filename )
            except ClientError as e :
                LOGWARNING ( 'queues have disappeared. stopping worker loop' )
                break
            # if there ' s any other exception , put a failed response into the
            # output bucket and queue
            except Exception as e :
                LOGEXCEPTION ( 'could not process input from queue' )
                if 'lc_filename' in locals ( ) :
                    with open ( 'failed-checkplot-%s.pkl' % lc_filename , 'wb' ) as outfd :
                        pickle . dump ( { 'in_queue_url' : in_queue_url , 'target' : target , 'lc_filename' : lc_filename , 'lclistpkl' : lclist_pklf , 'kwargs' : kwargs , 'outbucket' : outbucket , 'out_queue_url' : out_queue_url } , outfd , pickle . HIGHEST_PROTOCOL )
                    put_url = awsutils . s3_put_file ( 'failed-checkplot-%s.pkl' % lc_filename , outbucket , client = s3_client )
                    # put the S3 URL of the output into the output
                    # queue if requested
                    if out_queue_url is not None :
                        awsutils . sqs_put_item ( out_queue_url , { 'cpf' : put_url , 'lc_filename' : lc_filename , 'lclistpkl' : lclist_pklf , 'kwargs' : kwargs } , raiseonfail = True )
                    if ( ( lc_filename is not None ) and ( os . path . exists ( lc_filename ) ) ) :
                        os . remove ( lc_filename )
                # delete the input item from the input queue to
                # acknowledge its receipt and indicate that
                # processing is done
                awsutils . sqs_delete_item ( in_queue_url , receipt , raiseonfail = True )
    # a keyboard interrupt kills the loop
    except KeyboardInterrupt :
        LOGWARNING ( 'breaking out of the processing loop.' )
        break
    # if the queues disappear , then the producer loop is done and we should
    # exit
    except ClientError as e :
        LOGWARNING ( 'queues have disappeared. stopping worker loop' )
        break
    # any other exception continues the loop we ' ll write the output file to
    # the output S3 bucket ( and any optional output queue ) , but add a
    # failed - * prefix to it to indicate that processing failed . FIXME : could
    # use a dead - letter queue for this instead
    except Exception as e :
        LOGEXCEPTION ( 'could not process input from queue' )
        if 'lc_filename' in locals ( ) :
            with open ( 'failed-checkplot-%s.pkl' % lc_filename , 'wb' ) as outfd :
                pickle . dump ( { 'in_queue_url' : in_queue_url , 'target' : target , 'lclistpkl' : lclist_pklf , 'kwargs' : kwargs , 'outbucket' : outbucket , 'out_queue_url' : out_queue_url } , outfd , pickle . HIGHEST_PROTOCOL )
            put_url = awsutils . s3_put_file ( 'failed-checkplot-%s.pkl' % lc_filename , outbucket , client = s3_client )
            # put the S3 URL of the output into the output
            # queue if requested
            if out_queue_url is not None :
                awsutils . sqs_put_item ( out_queue_url , { 'cpf' : put_url , 'lclistpkl' : lclist_pklf , 'kwargs' : kwargs } , raiseonfail = True )
            if ( ( lc_filename is not None ) and ( os . path . exists ( lc_filename ) ) ) :
                os . remove ( lc_filename )
        # delete the input item from the input queue to
        # acknowledge its receipt and indicate that
        # processing is done
        awsutils . sqs_delete_item ( in_queue_url , receipt , raiseonfail = True )


signature:
def runpf_consumer_loop ( in_queue_url , workdir , lc_altexts = ( '' , ) , wait_time_seconds = 5 , shutdown_check_timer_seconds = 60.0 , sqs_client = None , s3_client = None ) :
    """This runs period - finding in a loop until interrupted .
 Consumes work task items from an input queue set up by ` runpf _ producer _ loop `
 above .
 Parameters
 in _ queue _ url : str
 The SQS URL of the input queue to listen to for work assignment
 messages . The task orders will include the input and output S3 bucket
 names , as well as the URL of the output queue to where this function
 will report its work - complete or work - failed status .
 workdir : str
 The directory on the local machine where this worker loop will download
 the input light curves , process them , and produce its output
 periodfinding result pickles . These will then be uploaded to the
 specified S3 output bucket , and then deleted from the local disk .
 lc _ altexts : sequence of str
 If not None , this is a sequence of alternate extensions to try for the
 input light curve file other than the one provided in the input task
 order . For example , to get anything that ' s an . sqlite where . sqlite . gz
 is expected , use altexts = [ ' ' ] to strip the . gz .
 wait _ time _ seconds : int
 The amount of time to wait in the input SQS queue for an input task
 order . If this timeout expires and no task has been received , this
 function goes back to the top of the work loop .
 shutdown _ check _ timer _ seconds : float
 The amount of time to wait before checking for a pending EC2 shutdown
 message for the instance this worker loop is operating on . If a shutdown
 is noticed , the worker loop is cancelled in preparation for instance
 shutdown .
 sqs _ client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its SQS operations . Alternatively , pass in an existing
 ` boto3 . Client ` instance to re - use it here .
 s3 _ client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its S3 operations . Alternatively , pass in an existing
 ` boto3 . Client ` instance to re - use it here .
 Returns
 Nothing ."""

implementation:
if not sqs_client :
    sqs_client = boto3 . client ( 'sqs' )
if not s3_client :
    s3_client = boto3 . client ( 's3' )
# listen to the kill and term signals and raise KeyboardInterrupt when
# called
signal . signal ( signal . SIGINT , kill_handler )
signal . signal ( signal . SIGTERM , kill_handler )
shutdown_last_time = time . monotonic ( )
while True :
    curr_time = time . monotonic ( )
    if ( curr_time - shutdown_last_time ) > shutdown_check_timer_seconds :
        shutdown_check = shutdown_check_handler ( )
        if shutdown_check :
            LOGWARNING ( 'instance will die soon, breaking loop' )
            break
        shutdown_last_time = time . monotonic ( )
    try : # receive a single message from the inqueue
        work = awsutils . sqs_get_item ( in_queue_url , client = sqs_client , raiseonfail = True )
        # JSON deserialize the work item
        if work is not None and len ( work ) > 0 :
            recv = work [ 0 ]
            # skip any messages that don ' t tell us to runpf
            action = recv [ 'item' ] [ 'action' ]
            if action != 'runpf' :
                continue
            target = recv [ 'item' ] [ 'target' ]
            args = recv [ 'item' ] [ 'args' ]
            kwargs = recv [ 'item' ] [ 'kwargs' ]
            outbucket = recv [ 'item' ] [ 'outbucket' ]
            if 'outqueue' in recv [ 'item' ] :
                out_queue_url = recv [ 'item' ] [ 'outqueue' ]
            else :
                out_queue_url = None
            receipt = recv [ 'receipt_handle' ]
            # download the target from S3 to a file in the work directory
            try :
                lc_filename = awsutils . s3_get_url ( target , altexts = lc_altexts , client = s3_client )
                runpf_args = ( lc_filename , args [ 0 ] )
                # now runpf
                pfresult = runpf ( * runpf_args , ** kwargs )
                if pfresult and os . path . exists ( pfresult ) :
                    LOGINFO ( 'runpf OK for LC: %s -> %s' % ( lc_filename , pfresult ) )
                    # check if the file exists already because it ' s been
                    # processed somewhere else
                    resp = s3_client . list_objects_v2 ( Bucket = outbucket , MaxKeys = 1 , Prefix = pfresult )
                    outbucket_list = resp . get ( 'Contents' , [ ] )
                    if outbucket_list and len ( outbucket_list ) > 0 :
                        LOGWARNING ( 'not uploading pfresult for %s because ' 'it exists in the output bucket already' % target )
                        awsutils . sqs_delete_item ( in_queue_url , receipt )
                        continue
                    put_url = awsutils . s3_put_file ( pfresult , outbucket , client = s3_client )
                    if put_url is not None :
                        LOGINFO ( 'result uploaded to %s' % put_url )
                        # put the S3 URL of the output into the output
                        # queue if requested
                        if out_queue_url is not None :
                            awsutils . sqs_put_item ( out_queue_url , { 'pfresult' : put_url , 'target' : target , 'lc_filename' : lc_filename , 'kwargs' : kwargs } , raiseonfail = True )
                        # delete the result from the local directory
                        os . remove ( pfresult )
                    # if the upload fails , don ' t acknowledge the
                    # message . might be a temporary S3 failure , so
                    # another worker might succeed later .
                    # FIXME : add SNS bits to warn us of failures
                    else :
                        LOGERROR ( 'failed to upload %s to S3' % pfresult )
                        os . remove ( pfresult )
                    # delete the input item from the input queue to
                    # acknowledge its receipt and indicate that
                    # processing is done and successful
                    awsutils . sqs_delete_item ( in_queue_url , receipt )
                    # delete the light curve file when we ' re done with it
                    if ( ( lc_filename is not None ) and ( os . path . exists ( lc_filename ) ) ) :
                        os . remove ( lc_filename )
                # if runcp failed outright , don ' t requeue . instead , write a
                # ( ' failed - checkplot - % s . pkl ' % lc _ filename ) file to the
                # output S3 bucket .
                else :
                    LOGWARNING ( 'runpf failed for LC: %s' % ( lc_filename , ) )
                    with open ( 'failed-periodfinding-%s.pkl' % lc_filename , 'wb' ) as outfd :
                        pickle . dump ( { 'in_queue_url' : in_queue_url , 'target' : target , 'lc_filename' : lc_filename , 'kwargs' : kwargs , 'outbucket' : outbucket , 'out_queue_url' : out_queue_url } , outfd , pickle . HIGHEST_PROTOCOL )
                    put_url = awsutils . s3_put_file ( 'failed-periodfinding-%s.pkl' % lc_filename , outbucket , client = s3_client )
                    # put the S3 URL of the output into the output
                    # queue if requested
                    if out_queue_url is not None :
                        awsutils . sqs_put_item ( out_queue_url , { 'pfresult' : put_url , 'lc_filename' : lc_filename , 'kwargs' : kwargs } , raiseonfail = True )
                    # delete the input item from the input queue to
                    # acknowledge its receipt and indicate that
                    # processing is done
                    awsutils . sqs_delete_item ( in_queue_url , receipt , raiseonfail = True )
                    # delete the light curve file when we ' re done with it
                    if ( ( lc_filename is not None ) and ( os . path . exists ( lc_filename ) ) ) :
                        os . remove ( lc_filename )
            except ClientError as e :
                LOGWARNING ( 'queues have disappeared. stopping worker loop' )
                break
            # if there ' s any other exception , put a failed response into the
            # output bucket and queue
            except Exception as e :
                LOGEXCEPTION ( 'could not process input from queue' )
                if 'lc_filename' in locals ( ) :
                    with open ( 'failed-periodfinding-%s.pkl' % lc_filename , 'wb' ) as outfd :
                        pickle . dump ( { 'in_queue_url' : in_queue_url , 'target' : target , 'lc_filename' : lc_filename , 'kwargs' : kwargs , 'outbucket' : outbucket , 'out_queue_url' : out_queue_url } , outfd , pickle . HIGHEST_PROTOCOL )
                    put_url = awsutils . s3_put_file ( 'failed-periodfinding-%s.pkl' % lc_filename , outbucket , client = s3_client )
                    # put the S3 URL of the output into the output
                    # queue if requested
                    if out_queue_url is not None :
                        awsutils . sqs_put_item ( out_queue_url , { 'pfresult' : put_url , 'lc_filename' : lc_filename , 'kwargs' : kwargs } , raiseonfail = True )
                    # delete the light curve file when we ' re done with it
                    if ( ( lc_filename is not None ) and ( os . path . exists ( lc_filename ) ) ) :
                        os . remove ( lc_filename )
                # delete the input item from the input queue to
                # acknowledge its receipt and indicate that
                # processing is done
                awsutils . sqs_delete_item ( in_queue_url , receipt , raiseonfail = True )
    # a keyboard interrupt kills the loop
    except KeyboardInterrupt :
        LOGWARNING ( 'breaking out of the processing loop.' )
        break
    # if the queues disappear , then the producer loop is done and we should
    # exit
    except ClientError as e :
        LOGWARNING ( 'queues have disappeared. stopping worker loop' )
        break
    # any other exception continues the loop we ' ll write the output file to
    # the output S3 bucket ( and any optional output queue ) , but add a
    # failed - * prefix to it to indicate that processing failed . FIXME : could
    # use a dead - letter queue for this instead
    except Exception as e :
        LOGEXCEPTION ( 'could not process input from queue' )
        if 'lc_filename' in locals ( ) :
            with open ( 'failed-periodfinding-%s.pkl' % lc_filename , 'wb' ) as outfd :
                pickle . dump ( { 'in_queue_url' : in_queue_url , 'target' : target , 'kwargs' : kwargs , 'outbucket' : outbucket , 'out_queue_url' : out_queue_url } , outfd , pickle . HIGHEST_PROTOCOL )
            put_url = awsutils . s3_put_file ( 'failed-periodfinding-%s.pkl' % lc_filename , outbucket , client = s3_client )
            # put the S3 URL of the output into the output
            # queue if requested
            if out_queue_url is not None :
                awsutils . sqs_put_item ( out_queue_url , { 'cpf' : put_url , 'kwargs' : kwargs } , raiseonfail = True )
            if ( ( lc_filename is not None ) and ( os . path . exists ( lc_filename ) ) ) :
                os . remove ( lc_filename )
        # delete the input item from the input queue to
        # acknowledge its receipt and indicate that
        # processing is done
        awsutils . sqs_delete_item ( in_queue_url , receipt , raiseonfail = True )


signature:
def gaussianeb_fit_magseries ( times , mags , errs , ebparams , sigclip = 10.0 , plotfit = False , magsarefluxes = False , verbose = True ) :
    '''This fits a double inverted gaussian EB model to a magnitude time series .
 Parameters
 times , mags , errs : np . array
 The input mag / flux time - series to fit the EB model to .
 period : float
 The period to use for EB fit .
 ebparams : list of float
 This is a list containing the eclipsing binary parameters : :
 ebparams = [ period ( time ) ,
 epoch ( time ) ,
 pdepth ( mags ) ,
 pduration ( phase ) ,
 psdepthratio ,
 secondaryphase ]
 ` period ` is the period in days .
 ` epoch ` is the time of primary minimum in JD .
 ` pdepth ` is the depth of the primary eclipse :
 - for magnitudes - > ` pdepth ` should be < 0
 - for fluxes - > ` pdepth ` should be > 0
 ` pduration ` is the length of the primary eclipse in phase .
 ` psdepthratio ` is the ratio of the secondary eclipse depth to that of
 the primary eclipse .
 ` secondaryphase ` is the phase at which the minimum of the secondary
 eclipse is located . This effectively parameterizes eccentricity .
 If ` epoch ` is None , this function will do an initial spline fit to find
 an approximate minimum of the phased light curve using the given period .
 The ` pdepth ` provided is checked against the value of
 ` magsarefluxes ` . if ` magsarefluxes = True ` , the ` ebdepth ` is forced to
 be > 0 ; if ` magsarefluxes = False ` , the ` ebdepth ` is forced to be < 0.
 sigclip : float or int or sequence of two floats / ints or None
 If a single float or int , a symmetric sigma - clip will be performed using
 the number provided as the sigma - multiplier to cut out from the input
 time - series .
 If a list of two ints / floats is provided , the function will perform an
 ' asymmetric ' sigma - clip . The first element in this list is the sigma
 value to use for fainter flux / mag values ; the second element in this
 list is the sigma value to use for brighter flux / mag values . For
 example , ` sigclip = [ 10 . , 3 . ] ` , will sigclip out greater than 10 - sigma
 dimmings and greater than 3 - sigma brightenings . Here the meaning of
 " dimming " and " brightening " is set by * physics * ( not the magnitude
 system ) , which is why the ` magsarefluxes ` kwarg must be correctly set .
 If ` sigclip ` is None , no sigma - clipping will be performed , and the
 time - series ( with non - finite elems removed ) will be passed through to
 the output .
 magsarefluxes : bool
 If True , will treat the input values of ` mags ` as fluxes for purposes of
 plotting the fit and sig - clipping .
 plotfit : str or False
 If this is a string , this function will make a plot for the fit to the
 mag / flux time - series and writes the plot to the path specified here .
 ignoreinitfail : bool
 If this is True , ignores the initial failure to find a set of optimized
 Fourier parameters using the global optimization function and proceeds
 to do a least - squares fit anyway .
 verbose : bool
 If True , will indicate progress and warn of any problems .
 Returns
 dict
 This function returns a dict containing the model fit parameters , the
 minimized chi - sq value and the reduced chi - sq value . The form of this
 dict is mostly standardized across all functions in this module : :
 ' fittype ' : ' gaussianeb ' ,
 ' fitinfo ' : {
 ' initialparams ' : the initial EB params provided ,
 ' finalparams ' : the final model fit EB params ,
 ' finalparamerrs ' : formal errors in the params ,
 ' leastsqfit ' : the full tuple returned by scipy . leastsq ,
 ' fitmags ' : the model fit mags ,
 ' fitepoch ' : the epoch of minimum light for the fit ,
 ' fitchisq ' : the minimized value of the fit ' s chi - sq ,
 ' fitredchisq ' : the reduced chi - sq value ,
 ' fitplotfile ' : the output fit plot if fitplot is not None ,
 ' magseries ' : {
 ' times ' : input times in phase order of the model ,
 ' phase ' : the phases of the model mags ,
 ' mags ' : input mags / fluxes in the phase order of the model ,
 ' errs ' : errs in the phase order of the model ,
 ' magsarefluxes ' : input value of magsarefluxes kwarg'''

implementation:
stimes , smags , serrs = sigclip_magseries ( times , mags , errs , sigclip = sigclip , magsarefluxes = magsarefluxes )
# get rid of zero errs
nzind = npnonzero ( serrs )
stimes , smags , serrs = stimes [ nzind ] , smags [ nzind ] , serrs [ nzind ]
# check the ebparams
ebperiod , ebepoch , ebdepth = ebparams [ 0 : 3 ]
# check if we have a ebepoch to use
if ebepoch is None :
    if verbose :
        LOGWARNING ( 'no ebepoch given in ebparams, ' 'trying to figure it out automatically...' )
    # do a spline fit to figure out the approximate min of the LC
    try :
        spfit = spline_fit_magseries ( times , mags , errs , ebperiod , sigclip = sigclip , magsarefluxes = magsarefluxes , verbose = verbose )
        ebepoch = spfit [ 'fitinfo' ] [ 'fitepoch' ]
    # if the spline - fit fails , try a savgol fit instead
    except Exception as e :
        sgfit = savgol_fit_magseries ( times , mags , errs , ebperiod , sigclip = sigclip , magsarefluxes = magsarefluxes , verbose = verbose )
        ebepoch = sgfit [ 'fitinfo' ] [ 'fitepoch' ]
    # if everything failed , then bail out and ask for the ebepoch
    finally :
        if ebepoch is None :
            LOGERROR ( "couldn't automatically figure out the eb epoch, " "can't continue. please provide it in ebparams." )
            # assemble the returndict
            returndict = { 'fittype' : 'gaussianeb' , 'fitinfo' : { 'initialparams' : ebparams , 'finalparams' : None , 'leastsqfit' : None , 'fitmags' : None , 'fitepoch' : None , } , 'fitchisq' : npnan , 'fitredchisq' : npnan , 'fitplotfile' : None , 'magseries' : { 'phase' : None , 'times' : None , 'mags' : None , 'errs' : None , 'magsarefluxes' : magsarefluxes , } , }
            return returndict
        else :
            if ebepoch . size > 1 :
                if verbose :
                    LOGWARNING ( 'could not auto-find a single minimum ' 'for ebepoch, using the first one returned' )
                ebparams [ 1 ] = ebepoch [ 0 ]
            else :
                if verbose :
                    LOGWARNING ( 'using automatically determined ebepoch = %.5f' % ebepoch )
                ebparams [ 1 ] = ebepoch . item ( )
# next , check the ebdepth and fix it to the form required
if magsarefluxes :
    if ebdepth < 0.0 :
        ebparams [ 2 ] = - ebdepth [ 2 ]
else :
    if ebdepth > 0.0 :
        ebparams [ 2 ] = - ebdepth [ 2 ]
# finally , do the fit
try :
    leastsqfit = spleastsq ( eclipses . invgauss_eclipses_residual , ebparams , args = ( stimes , smags , serrs ) , full_output = True )
except Exception as e :
    leastsqfit = None
# if the fit succeeded , then we can return the final parameters
if leastsqfit and leastsqfit [ - 1 ] in ( 1 , 2 , 3 , 4 ) :
    finalparams = leastsqfit [ 0 ]
    covxmatrix = leastsqfit [ 1 ]
    # calculate the chisq and reduced chisq
    fitmags , phase , ptimes , pmags , perrs = eclipses . invgauss_eclipses_func ( finalparams , stimes , smags , serrs )
    fitchisq = npsum ( ( ( fitmags - pmags ) * ( fitmags - pmags ) ) / ( perrs * perrs ) )
    fitredchisq = fitchisq / ( len ( pmags ) - len ( finalparams ) - 1 )
    # get the residual variance and calculate the formal 1 - sigma errs on the
    # final parameters
    residuals = leastsqfit [ 2 ] [ 'fvec' ]
    residualvariance = ( npsum ( residuals * residuals ) / ( pmags . size - finalparams . size ) )
    if covxmatrix is not None :
        covmatrix = residualvariance * covxmatrix
        stderrs = npsqrt ( npdiag ( covmatrix ) )
    else :
        LOGERROR ( 'covxmatrix not available, fit probably failed!' )
        stderrs = None
    if verbose :
        LOGINFO ( 'final fit done. chisq = %.5f, reduced chisq = %.5f' % ( fitchisq , fitredchisq ) )
    # get the fit epoch
    fperiod , fepoch = finalparams [ : 2 ]
    # assemble the returndict
    returndict = { 'fittype' : 'gaussianeb' , 'fitinfo' : { 'initialparams' : ebparams , 'finalparams' : finalparams , 'finalparamerrs' : stderrs , 'leastsqfit' : leastsqfit , 'fitmags' : fitmags , 'fitepoch' : fepoch , } , 'fitchisq' : fitchisq , 'fitredchisq' : fitredchisq , 'fitplotfile' : None , 'magseries' : { 'phase' : phase , 'times' : ptimes , 'mags' : pmags , 'errs' : perrs , 'magsarefluxes' : magsarefluxes , } , }
    # make the fit plot if required
    if plotfit and isinstance ( plotfit , str ) :
        make_fit_plot ( phase , pmags , perrs , fitmags , fperiod , ptimes . min ( ) , fepoch , plotfit , magsarefluxes = magsarefluxes )
        returndict [ 'fitplotfile' ] = plotfit
    return returndict
# if the leastsq fit failed , return nothing
else :
    LOGERROR ( 'eb-fit: least-squared fit to the light curve failed!' )
    # assemble the returndict
    returndict = { 'fittype' : 'gaussianeb' , 'fitinfo' : { 'initialparams' : ebparams , 'finalparams' : None , 'finalparamerrs' : None , 'leastsqfit' : leastsqfit , 'fitmags' : None , 'fitepoch' : None , } , 'fitchisq' : npnan , 'fitredchisq' : npnan , 'fitplotfile' : None , 'magseries' : { 'phase' : None , 'times' : None , 'mags' : None , 'errs' : None , 'magsarefluxes' : magsarefluxes , } , }
    return returndict


signature:
def spline_fit_magseries ( times , mags , errs , period , knotfraction = 0.01 , maxknots = 30 , sigclip = 30.0 , plotfit = False , ignoreinitfail = False , magsarefluxes = False , verbose = True ) :
    '''This fits a univariate cubic spline to the phased light curve .
 This fit may be better than the Fourier fit for sharply variable objects ,
 like EBs , so can be used to distinguish them from other types of variables .
 Parameters
 times , mags , errs : np . array
 The input mag / flux time - series to fit a spline to .
 period : float
 The period to use for the spline fit .
 knotfraction : float
 The knot fraction is the number of internal knots to use for the
 spline . A value of 0.01 ( or 1 % ) of the total number of non - nan
 observations appears to work quite well , without over - fitting . maxknots
 controls the maximum number of knots that will be allowed .
 maxknots : int
 The maximum number of knots that will be used even if ` knotfraction `
 gives a value to use larger than ` maxknots ` . This helps dealing with
 over - fitting to short time - scale variations .
 sigclip : float or int or sequence of two floats / ints or None
 If a single float or int , a symmetric sigma - clip will be performed using
 the number provided as the sigma - multiplier to cut out from the input
 time - series .
 If a list of two ints / floats is provided , the function will perform an
 ' asymmetric ' sigma - clip . The first element in this list is the sigma
 value to use for fainter flux / mag values ; the second element in this
 list is the sigma value to use for brighter flux / mag values . For
 example , ` sigclip = [ 10 . , 3 . ] ` , will sigclip out greater than 10 - sigma
 dimmings and greater than 3 - sigma brightenings . Here the meaning of
 " dimming " and " brightening " is set by * physics * ( not the magnitude
 system ) , which is why the ` magsarefluxes ` kwarg must be correctly set .
 If ` sigclip ` is None , no sigma - clipping will be performed , and the
 time - series ( with non - finite elems removed ) will be passed through to
 the output .
 magsarefluxes : bool
 If True , will treat the input values of ` mags ` as fluxes for purposes of
 plotting the fit and sig - clipping .
 plotfit : str or False
 If this is a string , this function will make a plot for the fit to the
 mag / flux time - series and writes the plot to the path specified here .
 ignoreinitfail : bool
 If this is True , ignores the initial failure to find a set of optimized
 Fourier parameters using the global optimization function and proceeds
 to do a least - squares fit anyway .
 verbose : bool
 If True , will indicate progress and warn of any problems .
 Returns
 dict
 This function returns a dict containing the model fit parameters , the
 minimized chi - sq value and the reduced chi - sq value . The form of this
 dict is mostly standardized across all functions in this module : :
 ' fittype ' : ' spline ' ,
 ' fitinfo ' : {
 ' nknots ' : the number of knots used for the fit
 ' fitmags ' : the model fit mags ,
 ' fitepoch ' : the epoch of minimum light for the fit ,
 ' fitchisq ' : the minimized value of the fit ' s chi - sq ,
 ' fitredchisq ' : the reduced chi - sq value ,
 ' fitplotfile ' : the output fit plot if fitplot is not None ,
 ' magseries ' : {
 ' times ' : input times in phase order of the model ,
 ' phase ' : the phases of the model mags ,
 ' mags ' : input mags / fluxes in the phase order of the model ,
 ' errs ' : errs in the phase order of the model ,
 ' magsarefluxes ' : input value of magsarefluxes kwarg'''

implementation:
# this is required to fit the spline correctly
if errs is None :
    errs = npfull_like ( mags , 0.005 )
# sigclip the magnitude time series
stimes , smags , serrs = sigclip_magseries ( times , mags , errs , sigclip = sigclip , magsarefluxes = magsarefluxes )
# get rid of zero errs
nzind = npnonzero ( serrs )
stimes , smags , serrs = stimes [ nzind ] , smags [ nzind ] , serrs [ nzind ]
# phase the mag series
phase , pmags , perrs , ptimes , mintime = ( get_phased_quantities ( stimes , smags , serrs , period ) )
# now figure out the number of knots up to max knots ( = 100)
nobs = len ( phase )
nknots = int ( npfloor ( knotfraction * nobs ) )
nknots = maxknots if nknots > maxknots else nknots
splineknots = nplinspace ( phase [ 0 ] + 0.01 , phase [ - 1 ] - 0.01 , num = nknots )
# NOTE : newer scipy needs x to be strictly increasing . this means we should
# filter out anything that doesn ' t have np . diff ( phase ) > 0.0
# FIXME : this needs to be tested
phase_diffs_ind = npdiff ( phase ) > 0.0
incphase_ind = npconcatenate ( ( nparray ( [ True ] ) , phase_diffs_ind ) )
phase , pmags , perrs = ( phase [ incphase_ind ] , pmags [ incphase_ind ] , perrs [ incphase_ind ] )
# generate and fit the spline
spl = LSQUnivariateSpline ( phase , pmags , t = splineknots , w = 1.0 / perrs )
# calculate the spline fit to the actual phases , the chisq and red - chisq
fitmags = spl ( phase )
fitchisq = npsum ( ( ( fitmags - pmags ) * ( fitmags - pmags ) ) / ( perrs * perrs ) )
fitredchisq = fitchisq / ( len ( pmags ) - nknots - 1 )
if verbose :
    LOGINFO ( 'spline fit done. nknots = %s,  ' 'chisq = %.5f, reduced chisq = %.5f' % ( nknots , fitchisq , fitredchisq ) )
# figure out the time of light curve minimum ( i . e . the fit epoch )
# this is when the fit mag is maximum ( i . e . the faintest )
# or if magsarefluxes = True , then this is when fit flux is minimum
if not magsarefluxes :
    fitmagminind = npwhere ( fitmags == npmax ( fitmags ) )
else :
    fitmagminind = npwhere ( fitmags == npmin ( fitmags ) )
if len ( fitmagminind [ 0 ] ) > 1 :
    fitmagminind = ( fitmagminind [ 0 ] [ 0 ] , )
magseriesepoch = ptimes [ fitmagminind ]
# assemble the returndict
returndict = { 'fittype' : 'spline' , 'fitinfo' : { 'nknots' : nknots , 'fitmags' : fitmags , 'fitepoch' : magseriesepoch } , 'fitchisq' : fitchisq , 'fitredchisq' : fitredchisq , 'fitplotfile' : None , 'magseries' : { 'times' : ptimes , 'phase' : phase , 'mags' : pmags , 'errs' : perrs , 'magsarefluxes' : magsarefluxes } , }
# make the fit plot if required
if plotfit and isinstance ( plotfit , str ) :
    make_fit_plot ( phase , pmags , perrs , fitmags , period , mintime , magseriesepoch , plotfit , magsarefluxes = magsarefluxes )
    returndict [ 'fitplotfile' ] = plotfit
return returndict


signature:
def savgol_fit_magseries ( times , mags , errs , period , windowlength = None , polydeg = 2 , sigclip = 30.0 , plotfit = False , magsarefluxes = False , verbose = True ) :
    '''Fit a Savitzky - Golay filter to the magnitude / flux time series .
 SG fits successive sub - sets ( windows ) of adjacent data points with a
 low - order polynomial via least squares . At each point ( magnitude ) , it
 returns the value of the polynomial at that magnitude ' s time . This is made
 significantly cheaper than * actually * performing least squares for each
 window through linear algebra tricks that are possible when specifying the
 window size and polynomial order beforehand . Numerical Recipes Ch 14.8
 gives an overview , Eq . 14.8.6 is what Scipy has implemented .
 The idea behind Savitzky - Golay is to preserve higher moments ( > = 2 ) of the
 input data series than would be done by a simple moving window average .
 Note that the filter assumes evenly spaced data , which magnitude time series
 are not . By * pretending * the data points are evenly spaced , we introduce an
 additional noise source in the function values . This is a relatively small
 noise source provided that the changes in the magnitude values across the
 full width of the N = windowlength point window is < sqrt ( N / 2 ) times the
 measurement noise on a single point .
 TODO :
 - Find correct dof for reduced chi squared in savgol _ fit _ magseries
 Parameters
 times , mags , errs : np . array
 The input mag / flux time - series to fit the Savitsky - Golay model to .
 period : float
 The period to use for the model fit .
 windowlength : None or int
 The length of the filter window ( the number of coefficients ) . Must be
 either positive and odd , or None . ( The window is the number of points to
 the left , and to the right , of whatever point is having a polynomial fit
 to it locally ) . Bigger windows at fixed polynomial order risk lowering
 the amplitude of sharp features . If None , this routine ( arbitrarily )
 sets the ` windowlength ` for phased LCs to be either the number of finite
 data points divided by 300 , or polydeg + 3 , whichever is bigger .
 polydeg : int
 This is the order of the polynomial used to fit the samples . Must be
 less than ` windowlength ` . " Higher - order filters do better at preserving
 feature heights and widths , but do less smoothing on broader features . "
 ( Numerical Recipes ) .
 sigclip : float or int or sequence of two floats / ints or None
 If a single float or int , a symmetric sigma - clip will be performed using
 the number provided as the sigma - multiplier to cut out from the input
 time - series .
 If a list of two ints / floats is provided , the function will perform an
 ' asymmetric ' sigma - clip . The first element in this list is the sigma
 value to use for fainter flux / mag values ; the second element in this
 list is the sigma value to use for brighter flux / mag values . For
 example , ` sigclip = [ 10 . , 3 . ] ` , will sigclip out greater than 10 - sigma
 dimmings and greater than 3 - sigma brightenings . Here the meaning of
 " dimming " and " brightening " is set by * physics * ( not the magnitude
 system ) , which is why the ` magsarefluxes ` kwarg must be correctly set .
 If ` sigclip ` is None , no sigma - clipping will be performed , and the
 time - series ( with non - finite elems removed ) will be passed through to
 the output .
 magsarefluxes : bool
 If True , will treat the input values of ` mags ` as fluxes for purposes of
 plotting the fit and sig - clipping .
 plotfit : str or False
 If this is a string , this function will make a plot for the fit to the
 mag / flux time - series and writes the plot to the path specified here .
 ignoreinitfail : bool
 If this is True , ignores the initial failure to find a set of optimized
 Fourier parameters using the global optimization function and proceeds
 to do a least - squares fit anyway .
 verbose : bool
 If True , will indicate progress and warn of any problems .
 Returns
 dict
 This function returns a dict containing the model fit parameters , the
 minimized chi - sq value and the reduced chi - sq value . The form of this
 dict is mostly standardized across all functions in this module : :
 ' fittype ' : ' savgol ' ,
 ' fitinfo ' : {
 ' windowlength ' : the window length used for the fit ,
 ' polydeg ' : the polynomial degree used for the fit ,
 ' fitmags ' : the model fit mags ,
 ' fitepoch ' : the epoch of minimum light for the fit ,
 ' fitchisq ' : the minimized value of the fit ' s chi - sq ,
 ' fitredchisq ' : the reduced chi - sq value ,
 ' fitplotfile ' : the output fit plot if fitplot is not None ,
 ' magseries ' : {
 ' times ' : input times in phase order of the model ,
 ' phase ' : the phases of the model mags ,
 ' mags ' : input mags / fluxes in the phase order of the model ,
 ' errs ' : errs in the phase order of the model ,
 ' magsarefluxes ' : input value of magsarefluxes kwarg'''

implementation:
stimes , smags , serrs = sigclip_magseries ( times , mags , errs , sigclip = sigclip , magsarefluxes = magsarefluxes )
# get rid of zero errs
nzind = npnonzero ( serrs )
stimes , smags , serrs = stimes [ nzind ] , smags [ nzind ] , serrs [ nzind ]
phase , pmags , perrs , ptimes , mintime = ( get_phased_quantities ( stimes , smags , serrs , period ) )
if not isinstance ( windowlength , int ) :
    windowlength = max ( polydeg + 3 , int ( len ( phase ) / 300 ) )
    if windowlength % 2 == 0 :
        windowlength += 1
if verbose :
    LOGINFO ( 'applying Savitzky-Golay filter with ' 'window length %s and polynomial degree %s to ' 'mag series with %s observations, ' 'using period %.6f, folded at %.6f' % ( windowlength , polydeg , len ( pmags ) , period , mintime ) )
# generate the function values obtained by applying the SG filter . The
# " wrap " option is best for phase - folded LCs .
sgf = savgol_filter ( pmags , windowlength , polydeg , mode = 'wrap' )
# here the " fit " to the phases is the function produced by the
# Savitzky - Golay filter . then compute the chisq and red - chisq .
fitmags = sgf
fitchisq = npsum ( ( ( fitmags - pmags ) * ( fitmags - pmags ) ) / ( perrs * perrs ) )
# TODO : quantify dof for SG filter .
nparams = int ( len ( pmags ) / windowlength ) * polydeg
fitredchisq = fitchisq / ( len ( pmags ) - nparams - 1 )
fitredchisq = - 99.
if verbose :
    LOGINFO ( 'SG filter applied. chisq = %.5f, reduced chisq = %.5f' % ( fitchisq , fitredchisq ) )
# figure out the time of light curve minimum ( i . e . the fit epoch )
# this is when the fit mag is maximum ( i . e . the faintest )
# or if magsarefluxes = True , then this is when fit flux is minimum
if not magsarefluxes :
    fitmagminind = npwhere ( fitmags == npmax ( fitmags ) )
else :
    fitmagminind = npwhere ( fitmags == npmin ( fitmags ) )
if len ( fitmagminind [ 0 ] ) > 1 :
    fitmagminind = ( fitmagminind [ 0 ] [ 0 ] , )
magseriesepoch = ptimes [ fitmagminind ]
# assemble the returndict
returndict = { 'fittype' : 'savgol' , 'fitinfo' : { 'windowlength' : windowlength , 'polydeg' : polydeg , 'fitmags' : fitmags , 'fitepoch' : magseriesepoch } , 'fitchisq' : fitchisq , 'fitredchisq' : fitredchisq , 'fitplotfile' : None , 'magseries' : { 'times' : ptimes , 'phase' : phase , 'mags' : pmags , 'errs' : perrs , 'magsarefluxes' : magsarefluxes } }
# make the fit plot if required
if plotfit and isinstance ( plotfit , str ) :
    make_fit_plot ( phase , pmags , perrs , fitmags , period , mintime , magseriesepoch , plotfit , magsarefluxes = magsarefluxes )
    returndict [ 'fitplotfile' ] = plotfit
return returndict


signature:
def legendre_fit_magseries ( times , mags , errs , period , legendredeg = 10 , sigclip = 30.0 , plotfit = False , magsarefluxes = False , verbose = True ) :
    '''Fit an arbitrary - order Legendre series , via least squares , to the
 magnitude / flux time series .
 This is a series of the form : :
 p ( x ) = c _ 0 * L _ 0 ( x ) + c _ 1 * L _ 1 ( x ) + c _ 2 * L _ 2 ( x ) + . . . + c _ n * L _ n ( x )
 where L _ i ' s are Legendre polynomials ( also called " Legendre functions of the
 first kind " ) and c _ i ' s are the coefficients being fit .
 This function is mainly just a wrapper to
 ` numpy . polynomial . legendre . Legendre . fit ` .
 Parameters
 times , mags , errs : np . array
 The input mag / flux time - series to fit a Legendre series polynomial to .
 period : float
 The period to use for the Legendre fit .
 legendredeg : int
 This is ` n ` in the equation above , e . g . if you give ` n = 5 ` , you will
 get 6 coefficients . This number should be much less than the number of
 data points you are fitting .
 sigclip : float or int or sequence of two floats / ints or None
 If a single float or int , a symmetric sigma - clip will be performed using
 the number provided as the sigma - multiplier to cut out from the input
 time - series .
 If a list of two ints / floats is provided , the function will perform an
 ' asymmetric ' sigma - clip . The first element in this list is the sigma
 value to use for fainter flux / mag values ; the second element in this
 list is the sigma value to use for brighter flux / mag values . For
 example , ` sigclip = [ 10 . , 3 . ] ` , will sigclip out greater than 10 - sigma
 dimmings and greater than 3 - sigma brightenings . Here the meaning of
 " dimming " and " brightening " is set by * physics * ( not the magnitude
 system ) , which is why the ` magsarefluxes ` kwarg must be correctly set .
 If ` sigclip ` is None , no sigma - clipping will be performed , and the
 time - series ( with non - finite elems removed ) will be passed through to
 the output .
 magsarefluxes : bool
 If True , will treat the input values of ` mags ` as fluxes for purposes of
 plotting the fit and sig - clipping .
 plotfit : str or False
 If this is a string , this function will make a plot for the fit to the
 mag / flux time - series and writes the plot to the path specified here .
 ignoreinitfail : bool
 If this is True , ignores the initial failure to find a set of optimized
 Fourier parameters using the global optimization function and proceeds
 to do a least - squares fit anyway .
 verbose : bool
 If True , will indicate progress and warn of any problems .
 Returns
 dict
 This function returns a dict containing the model fit parameters , the
 minimized chi - sq value and the reduced chi - sq value . The form of this
 dict is mostly standardized across all functions in this module : :
 ' fittype ' : ' legendre ' ,
 ' fitinfo ' : {
 ' legendredeg ' : the Legendre polynomial degree used ,
 ' fitmags ' : the model fit mags ,
 ' fitepoch ' : the epoch of minimum light for the fit ,
 ' fitchisq ' : the minimized value of the fit ' s chi - sq ,
 ' fitredchisq ' : the reduced chi - sq value ,
 ' fitplotfile ' : the output fit plot if fitplot is not None ,
 ' magseries ' : {
 ' times ' : input times in phase order of the model ,
 ' phase ' : the phases of the model mags ,
 ' mags ' : input mags / fluxes in the phase order of the model ,
 ' errs ' : errs in the phase order of the model ,
 ' magsarefluxes ' : input value of magsarefluxes kwarg'''

implementation:
stimes , smags , serrs = sigclip_magseries ( times , mags , errs , sigclip = sigclip , magsarefluxes = magsarefluxes )
# get rid of zero errs
nzind = npnonzero ( serrs )
stimes , smags , serrs = stimes [ nzind ] , smags [ nzind ] , serrs [ nzind ]
phase , pmags , perrs , ptimes , mintime = ( get_phased_quantities ( stimes , smags , serrs , period ) )
if verbose :
    LOGINFO ( 'fitting Legendre series with ' 'maximum Legendre polynomial order %s to ' 'mag series with %s observations, ' 'using period %.6f, folded at %.6f' % ( legendredeg , len ( pmags ) , period , mintime ) )
# Least squares fit of Legendre polynomial series to the data . The window
# and domain ( see " Using the Convenience Classes " in the numpy
# documentation ) are handled automatically , scaling the times to a minimal
# domain in [ - 1,1 ] , in which Legendre polynomials are a complete basis .
p = Legendre . fit ( phase , pmags , legendredeg )
coeffs = p . coef
fitmags = p ( phase )
# Now compute the chisq and red - chisq .
fitchisq = npsum ( ( ( fitmags - pmags ) * ( fitmags - pmags ) ) / ( perrs * perrs ) )
nparams = legendredeg + 1
fitredchisq = fitchisq / ( len ( pmags ) - nparams - 1 )
if verbose :
    LOGINFO ( 'Legendre fit done. chisq = %.5f, reduced chisq = %.5f' % ( fitchisq , fitredchisq ) )
# figure out the time of light curve minimum ( i . e . the fit epoch )
# this is when the fit mag is maximum ( i . e . the faintest )
# or if magsarefluxes = True , then this is when fit flux is minimum
if not magsarefluxes :
    fitmagminind = npwhere ( fitmags == npmax ( fitmags ) )
else :
    fitmagminind = npwhere ( fitmags == npmin ( fitmags ) )
if len ( fitmagminind [ 0 ] ) > 1 :
    fitmagminind = ( fitmagminind [ 0 ] [ 0 ] , )
magseriesepoch = ptimes [ fitmagminind ]
# assemble the returndict
returndict = { 'fittype' : 'legendre' , 'fitinfo' : { 'legendredeg' : legendredeg , 'fitmags' : fitmags , 'fitepoch' : magseriesepoch , 'finalparams' : coeffs , } , 'fitchisq' : fitchisq , 'fitredchisq' : fitredchisq , 'fitplotfile' : None , 'magseries' : { 'times' : ptimes , 'phase' : phase , 'mags' : pmags , 'errs' : perrs , 'magsarefluxes' : magsarefluxes } }
# make the fit plot if required
if plotfit and isinstance ( plotfit , str ) :
    make_fit_plot ( phase , pmags , perrs , fitmags , period , mintime , magseriesepoch , plotfit , magsarefluxes = magsarefluxes )
    returndict [ 'fitplotfile' ] = plotfit
return returndict


signature:
def update_checkplotdict_nbrlcs ( checkplotdict , timecol , magcol , errcol , lcformat = 'hat-sql' , lcformatdir = None , verbose = True , ) :
    '''For all neighbors in a checkplotdict , make LCs and phased LCs .
 Parameters
 checkplotdict : dict
 This is the checkplot to process . The light curves for the neighbors to
 the object here will be extracted from the stored file paths , and this
 function will make plots of these time - series . If the object has ' best '
 periods and epochs generated by period - finder functions in this
 checkplotdict , phased light curve plots of each neighbor will be made
 using these to check the effects of blending .
 timecol , magcol , errcol : str
 The timecol , magcol , and errcol keys used to generate this object ' s
 checkplot . This is used to extract the correct times - series from the
 neighbors ' light curves .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 Returns
 dict
 The input checkplotdict is returned with the neighor light curve plots
 added in .'''

implementation:
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return checkplotdict
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return checkplotdict
if not ( 'neighbors' in checkplotdict and checkplotdict [ 'neighbors' ] and len ( checkplotdict [ 'neighbors' ] ) > 0 ) :
    LOGERROR ( 'no neighbors for %s, not updating...' % ( checkplotdict [ 'objectid' ] ) )
    return checkplotdict
# get our object ' s magkeys to compare to the neighbor
objmagkeys = { }
# handle diff generations of checkplots
if 'available_bands' in checkplotdict [ 'objectinfo' ] :
    mclist = checkplotdict [ 'objectinfo' ] [ 'available_bands' ]
else :
    mclist = ( 'bmag' , 'vmag' , 'rmag' , 'imag' , 'jmag' , 'hmag' , 'kmag' , 'sdssu' , 'sdssg' , 'sdssr' , 'sdssi' , 'sdssz' )
for mc in mclist :
    if ( mc in checkplotdict [ 'objectinfo' ] and checkplotdict [ 'objectinfo' ] [ mc ] is not None and np . isfinite ( checkplotdict [ 'objectinfo' ] [ mc ] ) ) :
        objmagkeys [ mc ] = checkplotdict [ 'objectinfo' ] [ mc ]
# if there are actually neighbors , go through them in order
for nbr in checkplotdict [ 'neighbors' ] :
    objectid , lcfpath = ( nbr [ 'objectid' ] , nbr [ 'lcfpath' ] )
    # get the light curve
    if not os . path . exists ( lcfpath ) :
        LOGERROR ( 'objectid: %s, neighbor: %s, ' 'lightcurve: %s not found, skipping...' % ( checkplotdict [ 'objectid' ] , objectid , lcfpath ) )
        continue
    lcdict = readerfunc ( lcfpath )
    # this should handle lists / tuples being returned by readerfunc
    # we assume that the first element is the actual lcdict
    # FIXME : figure out how to not need this assumption
    if ( ( isinstance ( lcdict , ( list , tuple ) ) ) and ( isinstance ( lcdict [ 0 ] , dict ) ) ) :
        lcdict = lcdict [ 0 ]
    # 0 . get this neighbor ' s magcols and get the magdiff and colordiff
    # between it and the object
    nbrmagkeys = { }
    for mc in objmagkeys :
        if ( ( 'objectinfo' in lcdict ) and ( isinstance ( lcdict [ 'objectinfo' ] , dict ) ) and ( mc in lcdict [ 'objectinfo' ] ) and ( lcdict [ 'objectinfo' ] [ mc ] is not None ) and ( np . isfinite ( lcdict [ 'objectinfo' ] [ mc ] ) ) ) :
            nbrmagkeys [ mc ] = lcdict [ 'objectinfo' ] [ mc ]
    # now calculate the magdiffs
    magdiffs = { }
    for omc in objmagkeys :
        if omc in nbrmagkeys :
            magdiffs [ omc ] = objmagkeys [ omc ] - nbrmagkeys [ omc ]
    # calculate colors and colordiffs
    colordiffs = { }
    # generate the list of colors to get
    # NOTE : here , we don ' t really bother with new / old gen checkplots
    # maybe change this later to handle arbitrary colors
    for ctrio in ( [ 'bmag' , 'vmag' , 'bvcolor' ] , [ 'vmag' , 'kmag' , 'vkcolor' ] , [ 'jmag' , 'kmag' , 'jkcolor' ] , [ 'sdssi' , 'jmag' , 'ijcolor' ] , [ 'sdssg' , 'kmag' , 'gkcolor' ] , [ 'sdssg' , 'sdssr' , 'grcolor' ] ) :
        m1 , m2 , color = ctrio
        if ( m1 in objmagkeys and m2 in objmagkeys and m1 in nbrmagkeys and m2 in nbrmagkeys ) :
            objcolor = objmagkeys [ m1 ] - objmagkeys [ m2 ]
            nbrcolor = nbrmagkeys [ m1 ] - nbrmagkeys [ m2 ]
            colordiffs [ color ] = objcolor - nbrcolor
    # finally , add all the color and magdiff info to the nbr dict
    nbr . update ( { 'magdiffs' : magdiffs , 'colordiffs' : colordiffs } )
    # process magcols
    # normalize using the special function if specified
    if normfunc is not None :
        lcdict = normfunc ( lcdict )
    try : # get the times , mags , and errs
    # dereference the columns and get them from the lcdict
        if '.' in timecol :
            timecolget = timecol . split ( '.' )
        else :
            timecolget = [ timecol ]
        times = _dict_get ( lcdict , timecolget )
        if '.' in magcol :
            magcolget = magcol . split ( '.' )
        else :
            magcolget = [ magcol ]
        mags = _dict_get ( lcdict , magcolget )
        if '.' in errcol :
            errcolget = errcol . split ( '.' )
        else :
            errcolget = [ errcol ]
        errs = _dict_get ( lcdict , errcolget )
    except KeyError :
        LOGERROR ( 'LC for neighbor: %s (target object: %s) does not ' 'have one or more of the required columns: %s, ' 'skipping...' % ( objectid , checkplotdict [ 'objectid' ] , ', ' . join ( [ timecol , magcol , errcol ] ) ) )
        continue
    # filter the input times , mags , errs ; do sigclipping and normalization
    stimes , smags , serrs = sigclip_magseries ( times , mags , errs , magsarefluxes = magsarefluxes , sigclip = 4.0 )
    # normalize here if not using special normalization
    if normfunc is None :
        ntimes , nmags = normalize_magseries ( stimes , smags , magsarefluxes = magsarefluxes )
        xtimes , xmags , xerrs = ntimes , nmags , serrs
    else :
        xtimes , xmags , xerrs = stimes , smags , serrs
    # check if this neighbor has enough finite points in its LC
    # fail early if not enough light curve points
    if ( ( xtimes is None ) or ( xmags is None ) or ( xerrs is None ) or ( xtimes . size < 49 ) or ( xmags . size < 49 ) or ( xerrs . size < 49 ) ) :
        LOGERROR ( "one or more of times, mags, errs appear to be None " "after sig-clipping. are the measurements all nan? " "can't make neighbor light curve plots " "for target: %s, neighbor: %s, neighbor LC: %s" % ( checkplotdict [ 'objectid' ] , nbr [ 'objectid' ] , nbr [ 'lcfpath' ] ) )
        continue
    # now we can start doing stuff if everything checks out
    # make an unphased mag - series plot
    nbrdict = _pkl_magseries_plot ( xtimes , xmags , xerrs , magsarefluxes = magsarefluxes )
    # update the nbr
    nbr . update ( nbrdict )
    # for each lspmethod in the checkplot , make a corresponding plot for
    # this neighbor
    # figure out the period finder methods present
    if 'pfmethods' in checkplotdict :
        pfmethods = checkplotdict [ 'pfmethods' ]
    else :
        pfmethods = [ ]
        for cpkey in checkplotdict :
            for pfkey in PFMETHODS :
                if pfkey in cpkey :
                    pfmethods . append ( pfkey )
    for lspt in pfmethods : # initialize this lspmethod entry
        nbr [ lspt ] = { }
        # we only care about the best period and its options
        operiod , oepoch = ( checkplotdict [ lspt ] [ 0 ] [ 'period' ] , checkplotdict [ lspt ] [ 0 ] [ 'epoch' ] )
        ( ophasewrap , ophasesort , ophasebin , ominbinelems , oplotxlim ) = ( checkplotdict [ lspt ] [ 0 ] [ 'phasewrap' ] , checkplotdict [ lspt ] [ 0 ] [ 'phasesort' ] , checkplotdict [ lspt ] [ 0 ] [ 'phasebin' ] , checkplotdict [ lspt ] [ 0 ] [ 'minbinelems' ] , checkplotdict [ lspt ] [ 0 ] [ 'plotxlim' ] , )
        # make the phasedlc plot for this period
        nbr = _pkl_phased_magseries_plot ( nbr , lspt . split ( '-' ) [ 1 ] , # this splits ' < pfindex > - < pfmethod > '
        0 , xtimes , xmags , xerrs , operiod , oepoch , phasewrap = ophasewrap , phasesort = ophasesort , phasebin = ophasebin , minbinelems = ominbinelems , plotxlim = oplotxlim , magsarefluxes = magsarefluxes , verbose = verbose , override_pfmethod = lspt )
# at this point , this neighbor ' s dict should be up to date with all
# info , magseries plot , and all phased LC plots
# return the updated checkplotdict
return checkplotdict


signature:
def runcp ( pfpickle , outdir , lcbasedir , lcfname = None , cprenorm = False , lclistpkl = None , nbrradiusarcsec = 60.0 , maxnumneighbors = 5 , makeneighborlcs = True , fast_mode = False , gaia_max_timeout = 60.0 , gaia_mirror = None , xmatchinfo = None , xmatchradiusarcsec = 3.0 , minobservations = 99 , sigclip = 10.0 , lcformat = 'hat-sql' , lcformatdir = None , timecols = None , magcols = None , errcols = None , skipdone = False , done_callback = None , done_callback_args = None , done_callback_kwargs = None ) :
    '''This makes a checkplot pickle for the given period - finding result pickle
 produced by ` lcproc . periodfinding . runpf ` .
 Parameters
 pfpickle : str or None
 This is the filename of the period - finding result pickle file created by
 ` lcproc . periodfinding . runpf ` . If this is None , the checkplot will be
 made anyway , but no phased LC information will be collected into the
 output checkplot pickle . This can be useful for just collecting GAIA and
 other external information and making LC plots for an object .
 outdir : str
 This is the directory to which the output checkplot pickle will be
 written .
 lcbasedir : str
 The base directory where this function will look for the light curve
 file associated with the object in the input period - finding result
 pickle file .
 lcfname : str or None
 This is usually None because we ' ll get the path to the light curve
 associated with this period - finding pickle from the pickle itself . If
 ` pfpickle ` is None , however , this function will use ` lcfname ` to look up
 the light curve file instead . If both are provided , the value of
 ` lcfname ` takes precedence .
 Providing the light curve file name in this kwarg is useful when you ' re
 making checkplots directly from light curve files and not including
 period - finder results ( perhaps because period - finding takes a long time
 for large collections of LCs ) .
 cprenorm : bool
 Set this to True if the light curves should be renormalized by
 ` checkplot . checkplot _ pickle ` . This is set to False by default because we
 do our own normalization in this function using the light curve ' s
 registered normalization function and pass the normalized times , mags ,
 errs to the ` checkplot . checkplot _ pickle ` function .
 lclistpkl : str or dict
 This is either the filename of a pickle or the actual dict produced by
 lcproc . make _ lclist . This is used to gather neighbor information .
 nbrradiusarcsec : float
 The radius in arcseconds to use for a search conducted around the
 coordinates of this object to look for any potential confusion and
 blending of variability amplitude caused by their proximity .
 maxnumneighbors : int
 The maximum number of neighbors that will have their light curves and
 magnitudes noted in this checkplot as potential blends with the target
 object .
 makeneighborlcs : bool
 If True , will make light curve and phased light curve plots for all
 neighbors to the current object found in the catalog passed in using
 ` lclistpkl ` .
 fast _ mode : bool or float
 This runs the external catalog operations in a " fast " mode , with short
 timeouts and not trying to hit external catalogs that take a long time
 to respond .
 If this is set to True , the default settings for the external requests
 will then become : :
 skyview _ lookup = False
 skyview _ timeout = 10.0
 skyview _ retry _ failed = False
 dust _ timeout = 10.0
 gaia _ submit _ timeout = 7.0
 gaia _ max _ timeout = 10.0
 gaia _ submit _ tries = 2
 complete _ query _ later = False
 search _ simbad = False
 If this is a float , will run in " fast " mode with the provided timeout
 value in seconds and the following settings : :
 skyview _ lookup = True
 skyview _ timeout = fast _ mode
 skyview _ retry _ failed = False
 dust _ timeout = fast _ mode
 gaia _ submit _ timeout = 0.66 * fast _ mode
 gaia _ max _ timeout = fast _ mode
 gaia _ submit _ tries = 2
 complete _ query _ later = False
 search _ simbad = False
 gaia _ max _ timeout : float
 Sets the timeout in seconds to use when waiting for the GAIA service to
 respond to our request for the object ' s information . Note that if
 ` fast _ mode ` is set , this is ignored .
 gaia _ mirror : str or None
 This sets the GAIA mirror to use . This is a key in the
 ` services . gaia . GAIA _ URLS ` dict which defines the URLs to hit for each
 mirror .
 xmatchinfo : str or dict
 This is either the xmatch dict produced by the function
 ` load _ xmatch _ external _ catalogs ` above , or the path to the xmatch info
 pickle file produced by that function .
 xmatchradiusarcsec : float
 This is the cross - matching radius to use in arcseconds .
 minobservations : int
 The minimum of observations the input object ' s mag / flux time - series must
 have for this function to plot its light curve and phased light
 curve . If the object has less than this number , no light curves will be
 plotted , but the checkplotdict will still contain all of the other
 information .
 sigclip : float or int or sequence of two floats / ints or None
 If a single float or int , a symmetric sigma - clip will be performed using
 the number provided as the sigma - multiplier to cut out from the input
 time - series .
 If a list of two ints / floats is provided , the function will perform an
 ' asymmetric ' sigma - clip . The first element in this list is the sigma
 value to use for fainter flux / mag values ; the second element in this
 list is the sigma value to use for brighter flux / mag values . For
 example , ` sigclip = [ 10 . , 3 . ] ` , will sigclip out greater than 10 - sigma
 dimmings and greater than 3 - sigma brightenings . Here the meaning of
 " dimming " and " brightening " is set by * physics * ( not the magnitude
 system ) , which is why the ` magsarefluxes ` kwarg must be correctly set .
 If ` sigclip ` is None , no sigma - clipping will be performed , and the
 time - series ( with non - finite elems removed ) will be passed through to
 the output .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 timecols : list of str or None
 The timecol keys to use from the lcdict in generating this checkplot .
 magcols : list of str or None
 The magcol keys to use from the lcdict in generating this checkplot .
 errcols : list of str or None
 The errcol keys to use from the lcdict in generating this checkplot .
 skipdone : bool
 This indicates if this function will skip creating checkplots that
 already exist corresponding to the current ` objectid ` and ` magcol ` . If
 ` skipdone ` is set to True , this will be done .
 done _ callback : Python function or None
 This is used to provide a function to execute after the checkplot
 pickles are generated . This is useful if you want to stream the results
 of checkplot making to some other process , e . g . directly running an
 ingestion into an LCC - Server collection . The function will always get
 the list of the generated checkplot pickles as its first arg , and all of
 the kwargs for runcp in the kwargs dict . Additional args and kwargs can
 be provided by giving a list in the ` done _ callbacks _ args ` kwarg and a
 dict in the ` done _ callbacks _ kwargs ` kwarg .
 NOTE : the function you pass in here should be pickleable by normal
 Python if you want to use it with the parallel _ cp and parallel _ cp _ lcdir
 functions below .
 done _ callback _ args : tuple or None
 If not None , contains any args to pass into the ` done _ callback `
 function .
 done _ callback _ kwargs : dict or None
 If not None , contains any kwargs to pass into the ` done _ callback `
 function .
 Returns
 list of str
 This returns a list of checkplot pickle filenames with one element for
 each ( timecol , magcol , errcol ) combination provided in the default
 lcformat config or in the timecols , magcols , errcols kwargs .'''

implementation:
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( fileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
if pfpickle is not None :
    if pfpickle . endswith ( '.gz' ) :
        infd = gzip . open ( pfpickle , 'rb' )
    else :
        infd = open ( pfpickle , 'rb' )
    pfresults = pickle . load ( infd )
    infd . close ( )
# override the default timecols , magcols , and errcols
# using the ones provided to the function
if timecols is None :
    timecols = dtimecols
if magcols is None :
    magcols = dmagcols
if errcols is None :
    errcols = derrcols
if ( ( lcfname is not None or pfpickle is None ) and os . path . exists ( lcfname ) ) :
    lcfpath = lcfname
    objectid = None
else :
    if pfpickle is not None :
        objectid = pfresults [ 'objectid' ]
        lcfbasename = pfresults [ 'lcfbasename' ]
        lcfsearchpath = os . path . join ( lcbasedir , lcfbasename )
        if os . path . exists ( lcfsearchpath ) :
            lcfpath = lcfsearchpath
        elif lcfname is not None and os . path . exists ( lcfname ) :
            lcfpath = lcfname
        else :
            LOGERROR ( 'could not find light curve for ' 'pfresult %s, objectid %s, ' 'used search path: %s, lcfname kwarg: %s' % ( pfpickle , objectid , lcfsearchpath , lcfname ) )
            return None
    else :
        LOGERROR ( "no light curve provided and pfpickle is None, " "can't continue" )
        return None
lcdict = readerfunc ( lcfpath )
# this should handle lists / tuples being returned by readerfunc
# we assume that the first element is the actual lcdict
# FIXME : figure out how to not need this assumption
if ( ( isinstance ( lcdict , ( list , tuple ) ) ) and ( isinstance ( lcdict [ 0 ] , dict ) ) ) :
    lcdict = lcdict [ 0 ]
# get the object ID from the light curve if pfpickle is None or we used
# lcfname directly
if objectid is None :
    if 'objectid' in lcdict :
        objectid = lcdict [ 'objectid' ]
    elif ( 'objectid' in lcdict [ 'objectinfo' ] and lcdict [ 'objectinfo' ] [ 'objectid' ] ) :
        objectid = lcdict [ 'objectinfo' ] [ 'objectid' ]
    elif 'hatid' in lcdict [ 'objectinfo' ] and lcdict [ 'objectinfo' ] [ 'hatid' ] :
        objectid = lcdict [ 'objectinfo' ] [ 'hatid' ]
    else :
        objectid = uuid . uuid4 ( ) . hex [ : 5 ]
        LOGWARNING ( 'no objectid found for this object, ' 'generated a random one: %s' % objectid )
# normalize using the special function if specified
if normfunc is not None :
    lcdict = normfunc ( lcdict )
cpfs = [ ]
for tcol , mcol , ecol in zip ( timecols , magcols , errcols ) : # dereference the columns and get them from the lcdict
    if '.' in tcol :
        tcolget = tcol . split ( '.' )
    else :
        tcolget = [ tcol ]
    times = _dict_get ( lcdict , tcolget )
    if '.' in mcol :
        mcolget = mcol . split ( '.' )
    else :
        mcolget = [ mcol ]
    mags = _dict_get ( lcdict , mcolget )
    if '.' in ecol :
        ecolget = ecol . split ( '.' )
    else :
        ecolget = [ ecol ]
    errs = _dict_get ( lcdict , ecolget )
    # get all the period - finder results from this magcol
    if pfpickle is not None :
        if 'pfmethods' in pfresults [ mcol ] :
            pflist = [ pfresults [ mcol ] [ x ] for x in pfresults [ mcol ] [ 'pfmethods' ] if len ( pfresults [ mcol ] [ x ] . keys ( ) ) > 0 ]
        else :
            pflist = [ ]
            for pfm in PFMETHODS :
                if ( pfm in pfresults [ mcol ] and len ( pfresults [ mcol ] [ pfm ] . keys ( ) ) > 0 ) :
                    pflist . append ( pfresults [ mcol ] [ pfm ] )
    # special case of generating a checkplot with no phased LCs
    else :
        pflist = [ ]
    # generate the output filename
    outfile = os . path . join ( outdir , 'checkplot-%s-%s.pkl' % ( squeeze ( objectid ) . replace ( ' ' , '-' ) , mcol ) )
    if skipdone and os . path . exists ( outfile ) :
        LOGWARNING ( 'skipdone = True and ' 'checkplot for this objectid/magcol combination ' 'exists already: %s, skipping...' % outfile )
        return outfile
    # make sure the checkplot has a valid objectid
    if 'objectid' not in lcdict [ 'objectinfo' ] :
        lcdict [ 'objectinfo' ] [ 'objectid' ] = objectid
    # normalize here if not using special normalization
    if normfunc is None :
        ntimes , nmags = normalize_magseries ( times , mags , magsarefluxes = magsarefluxes )
        xtimes , xmags , xerrs = ntimes , nmags , errs
    else :
        xtimes , xmags , xerrs = times , mags , errs
    # generate the checkplotdict
    cpd = checkplot_dict ( pflist , xtimes , xmags , xerrs , objectinfo = lcdict [ 'objectinfo' ] , gaia_max_timeout = gaia_max_timeout , gaia_mirror = gaia_mirror , lclistpkl = lclistpkl , nbrradiusarcsec = nbrradiusarcsec , maxnumneighbors = maxnumneighbors , xmatchinfo = xmatchinfo , xmatchradiusarcsec = xmatchradiusarcsec , sigclip = sigclip , mindet = minobservations , verbose = False , fast_mode = fast_mode , magsarefluxes = magsarefluxes , normto = cprenorm # we ' ve done the renormalization already , so this
    # should be False by default . just messes up the
    # plots otherwise , destroying LPVs in particular
    )
    if makeneighborlcs : # include any neighbor information as well
        cpdupdated = update_checkplotdict_nbrlcs ( cpd , tcol , mcol , ecol , lcformat = lcformat , verbose = False )
    else :
        cpdupdated = cpd
    # write the update checkplot dict to disk
    cpf = _write_checkplot_picklefile ( cpdupdated , outfile = outfile , protocol = pickle . HIGHEST_PROTOCOL , outgzip = False )
    cpfs . append ( cpf )
# done with checkplot making
LOGINFO ( 'done with %s -> %s' % ( objectid , repr ( cpfs ) ) )
if done_callback is not None :
    if ( done_callback_args is not None and isinstance ( done_callback_args , list ) ) :
        done_callback_args = tuple ( [ cpfs ] + done_callback_args )
    else :
        done_callback_args = ( cpfs , )
    if ( done_callback_kwargs is not None and isinstance ( done_callback_kwargs , dict ) ) :
        done_callback_kwargs . update ( dict ( fast_mode = fast_mode , lcfname = lcfname , cprenorm = cprenorm , lclistpkl = lclistpkl , nbrradiusarcsec = nbrradiusarcsec , maxnumneighbors = maxnumneighbors , gaia_max_timeout = gaia_max_timeout , gaia_mirror = gaia_mirror , xmatchinfo = xmatchinfo , xmatchradiusarcsec = xmatchradiusarcsec , minobservations = minobservations , sigclip = sigclip , lcformat = lcformat , fileglob = fileglob , readerfunc = readerfunc , normfunc = normfunc , magsarefluxes = magsarefluxes , timecols = timecols , magcols = magcols , errcols = errcols , skipdone = skipdone , ) )
    else :
        done_callback_kwargs = dict ( fast_mode = fast_mode , lcfname = lcfname , cprenorm = cprenorm , lclistpkl = lclistpkl , nbrradiusarcsec = nbrradiusarcsec , maxnumneighbors = maxnumneighbors , gaia_max_timeout = gaia_max_timeout , gaia_mirror = gaia_mirror , xmatchinfo = xmatchinfo , xmatchradiusarcsec = xmatchradiusarcsec , minobservations = minobservations , sigclip = sigclip , lcformat = lcformat , fileglob = fileglob , readerfunc = readerfunc , normfunc = normfunc , magsarefluxes = magsarefluxes , timecols = timecols , magcols = magcols , errcols = errcols , skipdone = skipdone , )
    # fire the callback
    try :
        done_callback ( * done_callback_args , ** done_callback_kwargs )
        LOGINFO ( 'callback fired successfully for %r' % cpfs )
    except Exception as e :
        LOGEXCEPTION ( 'callback function failed for %r' % cpfs )
# at the end , return the list of checkplot files generated
return cpfs


signature:
def runcp_worker ( task ) :
    '''This is the worker for running checkplots .
 Parameters
 task : tuple
 This is of the form : ( pfpickle , outdir , lcbasedir , kwargs ) .
 Returns
 list of str
 The list of checkplot pickles returned by the ` runcp ` function .'''

implementation:
pfpickle , outdir , lcbasedir , kwargs = task
try :
    return runcp ( pfpickle , outdir , lcbasedir , ** kwargs )
except Exception as e :
    LOGEXCEPTION ( ' could not make checkplots for %s: %s' % ( pfpickle , e ) )
    return None


signature:
def parallel_cp ( pfpicklelist , outdir , lcbasedir , fast_mode = False , lcfnamelist = None , cprenorm = False , lclistpkl = None , gaia_max_timeout = 60.0 , gaia_mirror = None , nbrradiusarcsec = 60.0 , maxnumneighbors = 5 , makeneighborlcs = True , xmatchinfo = None , xmatchradiusarcsec = 3.0 , sigclip = 10.0 , minobservations = 99 , lcformat = 'hat-sql' , lcformatdir = None , timecols = None , magcols = None , errcols = None , skipdone = False , done_callback = None , done_callback_args = None , done_callback_kwargs = None , liststartindex = None , maxobjects = None , nworkers = NCPUS , ) :
    '''This drives the parallel execution of ` runcp ` for a list of periodfinding
 result pickles .
 Parameters
 pfpicklelist : list of str or list of Nones
 This is the list of the filenames of the period - finding result pickles
 to process . To make checkplots using the light curves directly , set this
 to a list of Nones with the same length as the list of light curve files
 that you provide in ` lcfnamelist ` .
 outdir : str
 The directory the checkplot pickles will be written to .
 lcbasedir : str
 The base directory that this function will look in to find the light
 curves pointed to by the period - finding result files . If you ' re using
 ` lcfnamelist ` to provide a list of light curve filenames directly , this
 arg is ignored .
 lcfnamelist : list of str or None
 If this is provided , it must be a list of the input light curve
 filenames to process . These can either be associated with each input
 period - finder result pickle , or can be provided standalone to make
 checkplots without phased LC plots in them . In the second case , you must
 set ` pfpicklelist ` to a list of Nones that matches the length of
 ` lcfnamelist ` .
 cprenorm : bool
 Set this to True if the light curves should be renormalized by
 ` checkplot . checkplot _ pickle ` . This is set to False by default because we
 do our own normalization in this function using the light curve ' s
 registered normalization function and pass the normalized times , mags ,
 errs to the ` checkplot . checkplot _ pickle ` function .
 lclistpkl : str or dict
 This is either the filename of a pickle or the actual dict produced by
 lcproc . make _ lclist . This is used to gather neighbor information .
 nbrradiusarcsec : float
 The radius in arcseconds to use for a search conducted around the
 coordinates of this object to look for any potential confusion and
 blending of variability amplitude caused by their proximity .
 maxnumneighbors : int
 The maximum number of neighbors that will have their light curves and
 magnitudes noted in this checkplot as potential blends with the target
 object .
 makeneighborlcs : bool
 If True , will make light curve and phased light curve plots for all
 neighbors found in the object collection for each input object .
 fast _ mode : bool or float
 This runs the external catalog operations in a " fast " mode , with short
 timeouts and not trying to hit external catalogs that take a long time
 to respond .
 If this is set to True , the default settings for the external requests
 will then become : :
 skyview _ lookup = False
 skyview _ timeout = 10.0
 skyview _ retry _ failed = False
 dust _ timeout = 10.0
 gaia _ submit _ timeout = 7.0
 gaia _ max _ timeout = 10.0
 gaia _ submit _ tries = 2
 complete _ query _ later = False
 search _ simbad = False
 If this is a float , will run in " fast " mode with the provided timeout
 value in seconds and the following settings : :
 skyview _ lookup = True
 skyview _ timeout = fast _ mode
 skyview _ retry _ failed = False
 dust _ timeout = fast _ mode
 gaia _ submit _ timeout = 0.66 * fast _ mode
 gaia _ max _ timeout = fast _ mode
 gaia _ submit _ tries = 2
 complete _ query _ later = False
 search _ simbad = False
 gaia _ max _ timeout : float
 Sets the timeout in seconds to use when waiting for the GAIA service to
 respond to our request for the object ' s information . Note that if
 ` fast _ mode ` is set , this is ignored .
 gaia _ mirror : str or None
 This sets the GAIA mirror to use . This is a key in the
 ` services . gaia . GAIA _ URLS ` dict which defines the URLs to hit for each
 mirror .
 xmatchinfo : str or dict
 This is either the xmatch dict produced by the function
 ` load _ xmatch _ external _ catalogs ` above , or the path to the xmatch info
 pickle file produced by that function .
 xmatchradiusarcsec : float
 This is the cross - matching radius to use in arcseconds .
 minobservations : int
 The minimum of observations the input object ' s mag / flux time - series must
 have for this function to plot its light curve and phased light
 curve . If the object has less than this number , no light curves will be
 plotted , but the checkplotdict will still contain all of the other
 information .
 sigclip : float or int or sequence of two floats / ints or None
 If a single float or int , a symmetric sigma - clip will be performed using
 the number provided as the sigma - multiplier to cut out from the input
 time - series .
 If a list of two ints / floats is provided , the function will perform an
 ' asymmetric ' sigma - clip . The first element in this list is the sigma
 value to use for fainter flux / mag values ; the second element in this
 list is the sigma value to use for brighter flux / mag values . For
 example , ` sigclip = [ 10 . , 3 . ] ` , will sigclip out greater than 10 - sigma
 dimmings and greater than 3 - sigma brightenings . Here the meaning of
 " dimming " and " brightening " is set by * physics * ( not the magnitude
 system ) , which is why the ` magsarefluxes ` kwarg must be correctly set .
 If ` sigclip ` is None , no sigma - clipping will be performed , and the
 time - series ( with non - finite elems removed ) will be passed through to
 the output .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 timecols : list of str or None
 The timecol keys to use from the lcdict in generating this checkplot .
 magcols : list of str or None
 The magcol keys to use from the lcdict in generating this checkplot .
 errcols : list of str or None
 The errcol keys to use from the lcdict in generating this checkplot .
 skipdone : bool
 This indicates if this function will skip creating checkplots that
 already exist corresponding to the current ` objectid ` and ` magcol ` . If
 ` skipdone ` is set to True , this will be done .
 done _ callback : Python function or None
 This is used to provide a function to execute after the checkplot
 pickles are generated . This is useful if you want to stream the results
 of checkplot making to some other process , e . g . directly running an
 ingestion into an LCC - Server collection . The function will always get
 the list of the generated checkplot pickles as its first arg , and all of
 the kwargs for runcp in the kwargs dict . Additional args and kwargs can
 be provided by giving a list in the ` done _ callbacks _ args ` kwarg and a
 dict in the ` done _ callbacks _ kwargs ` kwarg .
 NOTE : the function you pass in here should be pickleable by normal
 Python if you want to use it with the parallel _ cp and parallel _ cp _ lcdir
 functions below .
 done _ callback _ args : tuple or None
 If not None , contains any args to pass into the ` done _ callback `
 function .
 done _ callback _ kwargs : dict or None
 If not None , contains any kwargs to pass into the ` done _ callback `
 function .
 liststartindex : int
 The index of the ` pfpicklelist ` ( and ` lcfnamelist ` if provided ) to start
 working at .
 maxobjects : int
 The maximum number of objects to process in this run . Use this with
 ` liststartindex ` to effectively distribute working on a large list of
 input period - finding result pickles ( and light curves if ` lcfnamelist `
 is also provided ) over several sessions or machines .
 nworkers : int
 The number of parallel workers that will work on the checkplot
 generation process .
 Returns
 dict
 This returns a dict with keys = input period - finding pickles and vals =
 list of the corresponding checkplot pickles produced .'''

implementation:
# work around the Darwin segfault after fork if no network activity in
# main thread bug : https : / / bugs . python . org / issue30385 # msg293958
if sys . platform == 'darwin' :
    import requests
    requests . get ( 'http://captive.apple.com/hotspot-detect.html' )
if not os . path . exists ( outdir ) :
    os . mkdir ( outdir )
# handle the start and end indices
if ( liststartindex is not None ) and ( maxobjects is None ) :
    pfpicklelist = pfpicklelist [ liststartindex : ]
    if lcfnamelist is not None :
        lcfnamelist = lcfnamelist [ liststartindex : ]
elif ( liststartindex is None ) and ( maxobjects is not None ) :
    pfpicklelist = pfpicklelist [ : maxobjects ]
    if lcfnamelist is not None :
        lcfnamelist = lcfnamelist [ : maxobjects ]
elif ( liststartindex is not None ) and ( maxobjects is not None ) :
    pfpicklelist = ( pfpicklelist [ liststartindex : liststartindex + maxobjects ] )
    if lcfnamelist is not None :
        lcfnamelist = lcfnamelist [ liststartindex : liststartindex + maxobjects ]
# if the lcfnamelist is not provided , create a dummy
if lcfnamelist is None :
    lcfnamelist = [ None ] * len ( pfpicklelist )
tasklist = [ ( x , outdir , lcbasedir , { 'lcformat' : lcformat , 'lcformatdir' : lcformatdir , 'lcfname' : y , 'timecols' : timecols , 'magcols' : magcols , 'errcols' : errcols , 'lclistpkl' : lclistpkl , 'gaia_max_timeout' : gaia_max_timeout , 'gaia_mirror' : gaia_mirror , 'nbrradiusarcsec' : nbrradiusarcsec , 'maxnumneighbors' : maxnumneighbors , 'makeneighborlcs' : makeneighborlcs , 'xmatchinfo' : xmatchinfo , 'xmatchradiusarcsec' : xmatchradiusarcsec , 'sigclip' : sigclip , 'minobservations' : minobservations , 'skipdone' : skipdone , 'cprenorm' : cprenorm , 'fast_mode' : fast_mode , 'done_callback' : done_callback , 'done_callback_args' : done_callback_args , 'done_callback_kwargs' : done_callback_kwargs } ) for x , y in zip ( pfpicklelist , lcfnamelist ) ]
resultfutures = [ ]
results = [ ]
with ProcessPoolExecutor ( max_workers = nworkers ) as executor :
    resultfutures = executor . map ( runcp_worker , tasklist )
results = [ x for x in resultfutures ]
executor . shutdown ( )
return results