signature:
def sqs_get_item ( queue_url , max_items = 1 , wait_time_seconds = 5 , client = None , raiseonfail = False ) :
    """This gets a single item from the SQS queue .
 The ` queue _ url ` is composed of some internal SQS junk plus a
 ` queue _ name ` . For our purposes ( ` lcproc _ aws . py ` ) , the queue name will be
 something like : :
 lcproc _ queue _ < action >
 where action is one of : :
 runcp
 runpf
 The item is always a JSON object : :
 { ' target ' : S3 bucket address of the file to process ,
 ' action ' : the action to perform on the file ( ' runpf ' , ' runcp ' , etc . )
 ' args ' : the action ' s args as a tuple ( not including filename , which is
 generated randomly as a temporary local file ) ,
 ' kwargs ' : the action ' s kwargs as a dict ,
 ' outbucket : S3 bucket to write the result to ,
 ' outqueue ' : SQS queue to write the processed item ' s info to ( optional ) }
 The action MUST match the < action > in the queue name for this item to be
 processed .
 Parameters
 queue _ url : str
 The SQS URL of the queue to get messages from .
 max _ items : int
 The number of items to pull from the queue in this request .
 wait _ time _ seconds : int
 This specifies how long the function should block until a message is
 received on the queue . If the timeout expires , an empty list will be
 returned . If the timeout doesn ' t expire , the function will return a list
 of items received ( up to ` max _ items ` ) .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 list of dicts or None
 For each item pulled from the queue in this request ( up to ` max _ items ` ) ,
 a dict will be deserialized from the retrieved JSON , containing the
 message items and various metadata . The most important item of the
 metadata is the ` receipt _ handle ` , which can be used to acknowledge
 receipt of all items in this request ( see ` sqs _ delete _ item ` below ) .
 If the queue pull fails outright , returns None . If no messages are
 available for this queue pull , returns an empty list ."""

implementation:
if not client :
    client = boto3 . client ( 'sqs' )
try :
    resp = client . receive_message ( QueueUrl = queue_url , AttributeNames = [ 'All' ] , MaxNumberOfMessages = max_items , WaitTimeSeconds = wait_time_seconds )
    if not resp :
        LOGERROR ( 'could not receive messages from queue: %s' % queue_url )
    else :
        messages = [ ]
        for msg in resp . get ( 'Messages' , [ ] ) :
            try :
                messages . append ( { 'id' : msg [ 'MessageId' ] , 'receipt_handle' : msg [ 'ReceiptHandle' ] , 'md5' : msg [ 'MD5OfBody' ] , 'attributes' : msg [ 'Attributes' ] , 'item' : json . loads ( msg [ 'Body' ] ) , } )
            except Exception as e :
                LOGEXCEPTION ( 'could not deserialize message ID: %s, body: %s' % ( msg [ 'MessageId' ] , msg [ 'Body' ] ) )
                continue
        return messages
except Exception as e :
    LOGEXCEPTION ( 'could not get items from queue: %s' % queue_url )
    if raiseonfail :
        raise
    return None


signature:
def sqs_delete_item ( queue_url , receipt_handle , client = None , raiseonfail = False ) :
    """This deletes a message from the queue , effectively acknowledging its
 receipt .
 Call this only when all messages retrieved from the queue have been
 processed , since this will prevent redelivery of these messages to other
 queue workers pulling fromn the same queue channel .
 Parameters
 queue _ url : str
 The SQS URL of the queue where we got the messages from . This should be
 the same queue used to retrieve the messages in ` sqs _ get _ item ` .
 receipt _ handle : str
 The receipt handle of the queue message that we ' re responding to , and
 will acknowledge receipt of . This will be present in each message
 retrieved using ` sqs _ get _ item ` .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 Nothing ."""

implementation:
if not client :
    client = boto3 . client ( 'sqs' )
try :
    client . delete_message ( QueueUrl = queue_url , ReceiptHandle = receipt_handle )
except Exception as e :
    LOGEXCEPTION ( 'could not delete message with receipt handle: ' '%s from queue: %s' % ( receipt_handle , queue_url ) )
    if raiseonfail :
        raise


signature:
def make_ec2_nodes ( security_groupid , subnet_id , keypair_name , iam_instance_profile_arn , launch_instances = 1 , ami = 'ami-04681a1dbd79675a5' , instance = 't3.micro' , ebs_optimized = True , user_data = None , wait_until_up = True , client = None , raiseonfail = False , ) :
    """This makes new EC2 worker nodes .
 This requires a security group ID attached to a VPC config and subnet , a
 keypair generated beforehand , and an IAM role ARN for the instance . See :
 https : / / docs . aws . amazon . com / cli / latest / userguide / tutorial - ec2 - ubuntu . html
 Use ` user _ data ` to launch tasks on instance launch .
 Parameters
 security _ groupid : str
 The security group ID of the AWS VPC where the instances will be
 launched .
 subnet _ id : str
 The subnet ID of the AWS VPC where the instances will be
 launched .
 keypair _ name : str
 The name of the keypair to be used to allow SSH access to all instances
 launched here . This corresponds to an already downloaded AWS keypair PEM
 file .
 iam _ instance _ profile _ arn : str
 The ARN string corresponding to the AWS instance profile that describes
 the permissions the launched instances have to access other AWS
 resources . Set this up in AWS IAM .
 launch _ instances : int
 The number of instances to launch in this request .
 ami : str
 The Amazon Machine Image ID that describes the OS the instances will use
 after launch . The default ID is Amazon Linux 2 in the US East region .
 instance : str
 The instance type to launch . See the following URL for a list of IDs :
 https : / / aws . amazon . com / ec2 / pricing / on - demand /
 ebs _ optimized : bool
 If True , will enable EBS optimization to speed up IO . This is usually
 True for all instances made available in the last couple of years .
 user _ data : str or None
 This is either the path to a file on disk that contains a shell - script
 or a string containing a shell - script that will be executed by root
 right after the instance is launched . Use to automatically set up
 workers and queues . If None , will not execute anything at instance
 start up .
 wait _ until _ up : bool
 If True , will not return from this function until all launched instances
 are verified as running by AWS .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 dict
 Returns launched instance info as a dict , keyed by instance ID ."""

implementation:
if not client :
    client = boto3 . client ( 'ec2' )
# get the user data from a string or a file
# note : boto3 will base64 encode this itself
if isinstance ( user_data , str ) and os . path . exists ( user_data ) :
    with open ( user_data , 'r' ) as infd :
        udata = infd . read ( )
elif isinstance ( user_data , str ) :
    udata = user_data
else :
    udata = ( '#!/bin/bash\necho "No user data provided. ' 'Launched instance at: %s UTC"' % datetime . utcnow ( ) . isoformat ( ) )
# fire the request
try :
    resp = client . run_instances ( ImageId = ami , InstanceType = instance , SecurityGroupIds = [ security_groupid , ] , SubnetId = subnet_id , UserData = udata , IamInstanceProfile = { 'Arn' : iam_instance_profile_arn } , InstanceInitiatedShutdownBehavior = 'terminate' , KeyName = keypair_name , MaxCount = launch_instances , MinCount = launch_instances , EbsOptimized = ebs_optimized , )
    if not resp :
        LOGERROR ( 'could not launch requested instance' )
        return None
    else :
        instance_dict = { }
        instance_list = resp . get ( 'Instances' , [ ] )
        if len ( instance_list ) > 0 :
            for instance in instance_list :
                LOGINFO ( 'launched instance ID: %s of type: %s at: %s. ' 'current state: %s' % ( instance [ 'InstanceId' ] , instance [ 'InstanceType' ] , instance [ 'LaunchTime' ] . isoformat ( ) , instance [ 'State' ] [ 'Name' ] ) )
                instance_dict [ instance [ 'InstanceId' ] ] = { 'type' : instance [ 'InstanceType' ] , 'launched' : instance [ 'LaunchTime' ] , 'state' : instance [ 'State' ] [ 'Name' ] , 'info' : instance }
        # if we ' re waiting until we ' re up , then do so
        if wait_until_up :
            ready_instances = [ ]
            LOGINFO ( 'waiting until launched instances are up...' )
            ntries = 5
            curr_try = 0
            while ( ( curr_try < ntries ) or ( len ( ready_instances ) < len ( list ( instance_dict . keys ( ) ) ) ) ) :
                resp = client . describe_instances ( InstanceIds = list ( instance_dict . keys ( ) ) , )
                if len ( resp [ 'Reservations' ] ) > 0 :
                    for resv in resp [ 'Reservations' ] :
                        if len ( resv [ 'Instances' ] ) > 0 :
                            for instance in resv [ 'Instances' ] :
                                if instance [ 'State' ] [ 'Name' ] == 'running' :
                                    ready_instances . append ( instance [ 'InstanceId' ] )
                                    instance_dict [ instance [ 'InstanceId' ] ] [ 'state' ] = 'running'
                                    instance_dict [ instance [ 'InstanceId' ] ] [ 'ip' ] = instance [ 'PublicIpAddress' ]
                                    instance_dict [ instance [ 'InstanceId' ] ] [ 'info' ] = instance
                # sleep for a bit so we don ' t hit the API too often
                curr_try = curr_try + 1
                time . sleep ( 5.0 )
            if len ( ready_instances ) == len ( list ( instance_dict . keys ( ) ) ) :
                LOGINFO ( 'all instances now up.' )
            else :
                LOGWARNING ( 'reached maximum number of tries for instance status, ' 'not all instances may be up.' )
        return instance_dict
except ClientError as e :
    LOGEXCEPTION ( 'could not launch requested instance' )
    if raiseonfail :
        raise
    return None
except Exception as e :
    LOGEXCEPTION ( 'could not launch requested instance' )
    if raiseonfail :
        raise
    return None


signature:
def delete_ec2_nodes ( instance_id_list , client = None ) :
    """This deletes EC2 nodes and terminates the instances .
 Parameters
 instance _ id _ list : list of str
 A list of EC2 instance IDs to terminate .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 Returns
 Nothing ."""

implementation:
if not client :
    client = boto3 . client ( 'ec2' )
resp = client . terminate_instances ( InstanceIds = instance_id_list )
return resp


signature:
def make_spot_fleet_cluster ( security_groupid , subnet_id , keypair_name , iam_instance_profile_arn , spot_fleet_iam_role , target_capacity = 20 , spot_price = 0.4 , expires_days = 7 , allocation_strategy = 'lowestPrice' , instance_types = SPOT_INSTANCE_TYPES , instance_weights = None , instance_ami = 'ami-04681a1dbd79675a5' , instance_user_data = None , instance_ebs_optimized = True , wait_until_up = True , client = None , raiseonfail = False ) :
    """This makes an EC2 spot - fleet cluster .
 This requires a security group ID attached to a VPC config and subnet , a
 keypair generated beforehand , and an IAM role ARN for the instance . See :
 https : / / docs . aws . amazon . com / cli / latest / userguide / tutorial - ec2 - ubuntu . html
 Use ` user _ data ` to launch tasks on instance launch .
 Parameters
 security _ groupid : str
 The security group ID of the AWS VPC where the instances will be
 launched .
 subnet _ id : str
 The subnet ID of the AWS VPC where the instances will be
 launched .
 keypair _ name : str
 The name of the keypair to be used to allow SSH access to all instances
 launched here . This corresponds to an already downloaded AWS keypair PEM
 file .
 iam _ instance _ profile _ arn : str
 The ARN string corresponding to the AWS instance profile that describes
 the permissions the launched instances have to access other AWS
 resources . Set this up in AWS IAM .
 spot _ fleet _ iam _ role : str
 This is the name of AWS IAM role that allows the Spot Fleet Manager to
 scale up and down instances based on demand and instances failing ,
 etc . Set this up in IAM .
 target _ capacity : int
 The number of instances to target in the fleet request . The fleet
 manager service will attempt to maintain this number over the lifetime
 of the Spot Fleet Request .
 spot _ price : float
 The bid price in USD for the instances . This is per hour . Keep this at
 about half the hourly on - demand price of the desired instances to make
 sure your instances aren ' t taken away by AWS when it needs capacity .
 expires _ days : int
 The number of days this request is active for . All instances launched by
 this request will live at least this long and will be terminated
 automatically after .
 allocation _ strategy : { ' lowestPrice ' , ' diversified ' }
 The allocation strategy used by the fleet manager .
 instance _ types : list of str
 List of the instance type to launch . See the following URL for a list of
 IDs : https : / / aws . amazon . com / ec2 / pricing / on - demand /
 instance _ weights : list of float or None
 If ` instance _ types ` is a list of different instance types , this is the
 relative weight applied towards launching each instance type . This can
 be used to launch a mix of instances in a defined ratio among their
 types . Doing this can make the spot fleet more resilient to AWS taking
 back the instances if it runs out of capacity .
 instance _ ami : str
 The Amazon Machine Image ID that describes the OS the instances will use
 after launch . The default ID is Amazon Linux 2 in the US East region .
 instance _ user _ data : str or None
 This is either the path to a file on disk that contains a shell - script
 or a string containing a shell - script that will be executed by root
 right after the instance is launched . Use to automatically set up
 workers and queues . If None , will not execute anything at instance
 start up .
 instance _ ebs _ optimized : bool
 If True , will enable EBS optimization to speed up IO . This is usually
 True for all instances made available in the last couple of years .
 wait _ until _ up : bool
 If True , will not return from this function until the spot fleet request
 is acknowledged by AWS .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 str or None
 This is the spot fleet request ID if successful . Otherwise , returns
 None ."""

implementation:
fleetconfig = copy . deepcopy ( SPOT_FLEET_CONFIG )
fleetconfig [ 'IamFleetRole' ] = spot_fleet_iam_role
fleetconfig [ 'AllocationStrategy' ] = allocation_strategy
fleetconfig [ 'TargetCapacity' ] = target_capacity
fleetconfig [ 'SpotPrice' ] = str ( spot_price )
fleetconfig [ 'ValidUntil' ] = ( datetime . utcnow ( ) + timedelta ( days = expires_days ) ) . strftime ( '%Y-%m-%dT%H:%M:%SZ' )
# get the user data from a string or a file
# we need to base64 encode it here
if ( isinstance ( instance_user_data , str ) and os . path . exists ( instance_user_data ) ) :
    with open ( instance_user_data , 'rb' ) as infd :
        udata = base64 . b64encode ( infd . read ( ) ) . decode ( )
elif isinstance ( instance_user_data , str ) :
    udata = base64 . b64encode ( instance_user_data . encode ( ) ) . decode ( )
else :
    udata = ( '#!/bin/bash\necho "No user data provided. ' 'Launched instance at: %s UTC"' % datetime . utcnow ( ) . isoformat ( ) )
    udata = base64 . b64encode ( udata . encode ( ) ) . decode ( )
for ind , itype in enumerate ( instance_types ) :
    thisinstance = SPOT_PERINSTANCE_CONFIG . copy ( )
    thisinstance [ 'InstanceType' ] = itype
    thisinstance [ 'ImageId' ] = instance_ami
    thisinstance [ 'SubnetId' ] = subnet_id
    thisinstance [ 'KeyName' ] = keypair_name
    thisinstance [ 'IamInstanceProfile' ] [ 'Arn' ] = iam_instance_profile_arn
    thisinstance [ 'SecurityGroups' ] [ 0 ] = { 'GroupId' : security_groupid }
    thisinstance [ 'UserData' ] = udata
    thisinstance [ 'EbsOptimized' ] = instance_ebs_optimized
    # get the instance weights
    if isinstance ( instance_weights , list ) :
        thisinstance [ 'WeightedCapacity' ] = instance_weights [ ind ]
    fleetconfig [ 'LaunchSpecifications' ] . append ( thisinstance )
# launch the fleet
if not client :
    client = boto3 . client ( 'ec2' )
try :
    resp = client . request_spot_fleet ( SpotFleetRequestConfig = fleetconfig , )
    if not resp :
        LOGERROR ( 'spot fleet request failed.' )
        return None
    else :
        spot_fleet_reqid = resp [ 'SpotFleetRequestId' ]
        LOGINFO ( 'spot fleet requested successfully. request ID: %s' % spot_fleet_reqid )
        if not wait_until_up :
            return spot_fleet_reqid
        else :
            ntries = 10
            curr_try = 0
            while curr_try < ntries :
                resp = client . describe_spot_fleet_requests ( SpotFleetRequestIds = [ spot_fleet_reqid ] )
                curr_state = resp . get ( 'SpotFleetRequestConfigs' , [ ] )
                if len ( curr_state ) > 0 :
                    curr_state = curr_state [ 0 ] [ 'SpotFleetRequestState' ]
                    if curr_state == 'active' :
                        LOGINFO ( 'spot fleet with reqid: %s is now active' % spot_fleet_reqid )
                        break
                LOGINFO ( 'spot fleet not yet active, waiting 15 seconds. ' 'try %s/%s' % ( curr_try , ntries ) )
                curr_try = curr_try + 1
                time . sleep ( 15.0 )
            return spot_fleet_reqid
except ClientError as e :
    LOGEXCEPTION ( 'could not launch spot fleet' )
    if raiseonfail :
        raise
    return None
except Exception as e :
    LOGEXCEPTION ( 'could not launch spot fleet' )
    if raiseonfail :
        raise
    return None


signature:
def delete_spot_fleet_cluster ( spot_fleet_reqid , client = None , ) :
    """This deletes a spot - fleet cluster .
 Parameters
 spot _ fleet _ reqid : str
 The fleet request ID returned by ` make _ spot _ fleet _ cluster ` .
 client : boto3 . Client or None
 If None , this function will instantiate a new ` boto3 . Client ` object to
 use in its operations . Alternatively , pass in an existing ` boto3 . Client `
 instance to re - use it here .
 Returns
 Nothing ."""

implementation:
if not client :
    client = boto3 . client ( 'ec2' )
resp = client . cancel_spot_fleet_requests ( SpotFleetRequestIds = [ spot_fleet_reqid ] , TerminateInstances = True )
return resp


signature:
def gcs_get_file ( bucketname , filename , local_file , altexts = None , client = None , service_account_json = None , raiseonfail = False ) :
    """This gets a single file from a Google Cloud Storage bucket .
 Parameters
 bucketname : str
 The name of the GCS bucket to download the file from .
 filename : str
 The full name of the file to download , including all prefixes .
 local _ file : str
 Path to where the downloaded file will be stored .
 altexts : None or list of str
 If not None , this is a list of alternate extensions to try for the file
 other than the one provided in ` filename ` . For example , to get anything
 that ' s an . sqlite where . sqlite . gz is expected , use altexts = [ ' ' ] to
 strip the . gz .
 client : google . cloud . storage . Client instance
 The instance of the Client to use to perform the download operation . If
 this is None , a new Client will be used . If this is None and
 ` service _ account _ json ` points to a downloaded JSON file with GCS
 credentials , a new Client with the provided credentials will be used . If
 this is not None , the existing Client instance will be used .
 service _ account _ json : str
 Path to a downloaded GCS credentials JSON file .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 str
 Path to the downloaded filename or None if the download was
 unsuccessful ."""

implementation:
if not client :
    if ( service_account_json is not None and os . path . exists ( service_account_json ) ) :
        client = storage . Client . from_service_account_json ( service_account_json )
    else :
        client = storage . Client ( )
try :
    bucket = client . get_bucket ( bucketname )
    blob = bucket . get_blob ( filename )
    blob . download_to_filename ( local_file )
    return local_file
except Exception as e :
    for alt_extension in altexts :
        split_ext = os . path . splitext ( filename )
        check_file = split_ext [ 0 ] + alt_extension
        try :
            bucket = client . get_bucket ( bucket )
            blob = bucket . get_blob ( check_file )
            blob . download_to_filename ( local_file . replace ( split_ext [ - 1 ] , alt_extension ) )
            return local_file . replace ( split_ext [ - 1 ] , alt_extension )
        except Exception as e :
            pass
else :
    LOGEXCEPTION ( 'could not download gs://%s/%s' % ( bucket , filename ) )
    if raiseonfail :
        raise
    return None


signature:
def gcs_get_url ( url , altexts = None , client = None , service_account_json = None , raiseonfail = False ) :
    """This gets a single file from a Google Cloud Storage bucket .
 This uses the gs : / / URL instead of a bucket name and key .
 Parameters
 url : str
 GCS URL to download . This should begin with ' gs : / / ' .
 altexts : None or list of str
 If not None , this is a list of alternate extensions to try for the file
 other than the one provided in ` filename ` . For example , to get anything
 that ' s an . sqlite where . sqlite . gz is expected , use altexts = [ ' ' ] to
 strip the . gz .
 client : google . cloud . storage . Client instance
 The instance of the Client to use to perform the download operation . If
 this is None , a new Client will be used . If this is None and
 ` service _ account _ json ` points to a downloaded JSON file with GCS
 credentials , a new Client with the provided credentials will be used . If
 this is not None , the existing Client instance will be used .
 service _ account _ json : str
 Path to a downloaded GCS credentials JSON file .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 str
 Path to the downloaded filename or None if the download was
 unsuccessful ."""

implementation:
bucket_item = url . replace ( 'gs://' , '' )
bucket_item = bucket_item . split ( '/' )
bucket = bucket_item [ 0 ]
filekey = '/' . join ( bucket_item [ 1 : ] )
return gcs_get_file ( bucket , filekey , bucket_item [ - 1 ] , altexts = altexts , client = client , service_account_json = service_account_json , raiseonfail = raiseonfail )


signature:
def gcs_put_file ( local_file , bucketname , service_account_json = None , client = None , raiseonfail = False ) :
    """This puts a single file into a Google Cloud Storage bucket .
 Parameters
 local _ file : str
 Path to the file to upload to GCS .
 bucket : str
 The GCS bucket to upload the file to .
 service _ account _ json : str
 Path to a downloaded GCS credentials JSON file .
 client : google . cloud . storage . Client instance
 The instance of the Client to use to perform the download operation . If
 this is None , a new Client will be used . If this is None and
 ` service _ account _ json ` points to a downloaded JSON file with GCS
 credentials , a new Client with the provided credentials will be used . If
 this is not None , the existing Client instance will be used .
 raiseonfail : bool
 If True , will re - raise whatever Exception caused the operation to fail
 and break out immediately .
 Returns
 str or None
 If the file upload is successful , returns the gs : / / URL of the uploaded
 file . If it failed , will return None ."""

implementation:
if not client :
    if ( service_account_json is not None and os . path . exists ( service_account_json ) ) :
        client = storage . Client . from_service_account_json ( service_account_json )
    else :
        client = storage . Client ( )
try :
    bucket = client . get_bucket ( bucketname )
    remote_blob = bucket . blob ( local_file )
    remote_blob . upload_from_filename ( local_file )
    return 'gs://%s/%s' % ( bucketname , local_file . lstrip ( '/' ) )
except Exception as e :
    LOGEXCEPTION ( 'could not upload %s to bucket %s' % ( local_file , bucket ) )
    if raiseonfail :
        raise
    return None


signature:
def read_hatlc ( hatlc ) :
    '''This reads a consolidated HAT LC written by the functions above .
 Returns a dict .'''

implementation:
lcfname = os . path . basename ( hatlc )
# unzip the files first
if '.gz' in lcfname :
    lcf = gzip . open ( hatlc , 'rb' )
elif '.bz2' in lcfname :
    lcf = bz2 . BZ2File ( hatlc , 'rb' )
else :
    lcf = open ( hatlc , 'rb' )
if '.fits' in lcfname and HAVEPYFITS :
    hdulist = pyfits . open ( lcf )
    objectinfo = hdulist [ 0 ] . header
    objectlc = hdulist [ 1 ] . data
    lccols = objectlc . columns . names
    hdulist . close ( )
    lcf . close ( )
    lcdict = { }
    for col in lccols :
        lcdict [ col ] = np . array ( objectlc [ col ] )
    lcdict [ 'hatid' ] = objectinfo [ 'hatid' ]
    lcdict [ 'twomassid' ] = objectinfo [ '2massid' ]
    lcdict [ 'ra' ] = objectinfo [ 'ra' ]
    lcdict [ 'dec' ] = objectinfo [ 'dec' ]
    lcdict [ 'mags' ] = [ objectinfo [ x ] for x in ( 'vmag' , 'rmag' , 'imag' , 'jmag' , 'hmag' , 'kmag' ) ]
    lcdict [ 'ndet' ] = objectinfo [ 'ndet' ]
    lcdict [ 'hatstations' ] = objectinfo [ 'hats' ]
    lcdict [ 'filters' ] = objectinfo [ 'filters' ]
    lcdict [ 'columns' ] = lccols
    return lcdict
elif '.fits' in lcfname and not HAVEPYFITS :
    print ( "can't read %s since we don't have the pyfits module" % lcfname )
    return
elif '.csv' in lcfname or '.hatlc' in lcfname :
    lcflines = lcf . read ( ) . decode ( ) . split ( '\n' )
    lcf . close ( )
    # now process the read - in LC
    objectdata = [ x for x in lcflines if x . startswith ( '#' ) ]
    objectlc = [ x for x in lcflines if not x . startswith ( '#' ) ]
    objectlc = [ x for x in objectlc if len ( x ) > 1 ]
    if '.csv' in lcfname :
        objectlc = [ x . split ( ',' ) for x in objectlc ]
    else :
        objectlc = [ x . split ( ) for x in objectlc ]
    # transpose split rows to get columns
    objectlc = list ( zip ( * objectlc ) )
    # read the header to figure out the object ' s info and column names
    objectdata = [ x . strip ( '#' ) for x in objectdata ]
    objectdata = [ x . strip ( ) for x in objectdata ]
    objectdata = [ x for x in objectdata if len ( x ) > 0 ]
    hatid , twomassid = objectdata [ 0 ] . split ( ' - ' )
    ra , dec = objectdata [ 1 ] . split ( ', ' )
    ra = float ( ra . split ( ' = ' ) [ - 1 ] . strip ( ' deg' ) )
    dec = float ( dec . split ( ' = ' ) [ - 1 ] . strip ( ' deg' ) )
    vmag , rmag , imag , jmag , hmag , kmag = objectdata [ 2 ] . split ( ', ' )
    vmag = float ( vmag . split ( ' = ' ) [ - 1 ] )
    rmag = float ( rmag . split ( ' = ' ) [ - 1 ] )
    imag = float ( imag . split ( ' = ' ) [ - 1 ] )
    jmag = float ( jmag . split ( ' = ' ) [ - 1 ] )
    hmag = float ( hmag . split ( ' = ' ) [ - 1 ] )
    kmag = float ( kmag . split ( ' = ' ) [ - 1 ] )
    ndet = int ( objectdata [ 3 ] . split ( ': ' ) [ - 1 ] )
    hatstations = objectdata [ 4 ] . split ( ': ' ) [ - 1 ]
    filterhead_ind = objectdata . index ( 'Filters used:' )
    columnhead_ind = objectdata . index ( 'Columns:' )
    filters = objectdata [ filterhead_ind : columnhead_ind ]
    columndefs = objectdata [ columnhead_ind + 1 : ]
    columns = [ ]
    for line in columndefs :
        colnum , colname , coldesc = line . split ( ' - ' )
        columns . append ( colname )
    lcdict = { }
    # now write all the columns to the output dictionary
    for ind , col in enumerate ( columns ) : # this formats everything nicely using our existing column
    # definitions
        lcdict [ col ] = np . array ( [ TEXTLC_OUTPUT_COLUMNS [ col ] [ 3 ] ( x ) for x in objectlc [ ind ] ] )
    # write the object metadata to the output dictionary
    lcdict [ 'hatid' ] = hatid
    lcdict [ 'twomassid' ] = twomassid . replace ( '2MASS J' , '' )
    lcdict [ 'ra' ] = ra
    lcdict [ 'dec' ] = dec
    lcdict [ 'mags' ] = [ vmag , rmag , imag , jmag , hmag , kmag ]
    lcdict [ 'ndet' ] = ndet
    lcdict [ 'hatstations' ] = hatstations . split ( ', ' )
    lcdict [ 'filters' ] = filters [ 1 : ]
    lcdict [ 'cols' ] = columns
    return lcdict


signature:
def read_fakelc ( fakelcfile ) :
    '''This just reads a pickled fake LC .
 Parameters
 fakelcfile : str
 The fake LC file to read .
 Returns
 dict
 This returns an lcdict .'''

implementation:
try :
    with open ( fakelcfile , 'rb' ) as infd :
        lcdict = pickle . load ( infd )
except UnicodeDecodeError :
    with open ( fakelcfile , 'rb' ) as infd :
        lcdict = pickle . load ( infd , encoding = 'latin1' )
return lcdict


signature:
def get_varfeatures ( simbasedir , mindet = 1000 , nworkers = None ) :
    '''This runs ` lcproc . lcvfeatures . parallel _ varfeatures ` on fake LCs in
 ` simbasedir ` .
 Parameters
 simbasedir : str
 The directory containing the fake LCs to process .
 mindet : int
 The minimum number of detections needed to accept an LC and process it .
 nworkers : int or None
 The number of parallel workers to use when extracting variability
 features from the input light curves .
 Returns
 str
 The path to the ` varfeatures ` pickle created after running the
 ` lcproc . lcvfeatures . parallel _ varfeatures ` function .'''

implementation:
# get the info from the simbasedir
with open ( os . path . join ( simbasedir , 'fakelcs-info.pkl' ) , 'rb' ) as infd :
    siminfo = pickle . load ( infd )
lcfpaths = siminfo [ 'lcfpath' ]
varfeaturedir = os . path . join ( simbasedir , 'varfeatures' )
# get the column defs for the fakelcs
timecols = siminfo [ 'timecols' ]
magcols = siminfo [ 'magcols' ]
errcols = siminfo [ 'errcols' ]
# get the column defs for the fakelcs
timecols = siminfo [ 'timecols' ]
magcols = siminfo [ 'magcols' ]
errcols = siminfo [ 'errcols' ]
# register the fakelc pklc as a custom lcproc format
# now we should be able to use all lcproc functions correctly
fakelc_formatkey = 'fake-%s' % siminfo [ 'lcformat' ]
lcproc . register_lcformat ( fakelc_formatkey , '*-fakelc.pkl' , timecols , magcols , errcols , 'astrobase.lcproc' , '_read_pklc' , magsarefluxes = siminfo [ 'magsarefluxes' ] )
# now we can use lcproc . parallel _ varfeatures directly
varinfo = lcvfeatures . parallel_varfeatures ( lcfpaths , varfeaturedir , lcformat = fakelc_formatkey , mindet = mindet , nworkers = nworkers )
with open ( os . path . join ( simbasedir , 'fakelc-varfeatures.pkl' ) , 'wb' ) as outfd :
    pickle . dump ( varinfo , outfd , pickle . HIGHEST_PROTOCOL )
return os . path . join ( simbasedir , 'fakelc-varfeatures.pkl' )


signature:
def precision ( ntp , nfp ) :
    '''This calculates precision .
 https : / / en . wikipedia . org / wiki / Precision _ and _ recall
 Parameters
 ntp : int
 The number of true positives .
 nfp : int
 The number of false positives .
 Returns
 float
 The precision calculated using ` ntp / ( ntp + nfp ) ` .'''

implementation:
if ( ntp + nfp ) > 0 :
    return ntp / ( ntp + nfp )
else :
    return np . nan


signature:
def recall ( ntp , nfn ) :
    '''This calculates recall .
 https : / / en . wikipedia . org / wiki / Precision _ and _ recall
 Parameters
 ntp : int
 The number of true positives .
 nfn : int
 The number of false negatives .
 Returns
 float
 The precision calculated using ` ntp / ( ntp + nfn ) ` .'''

implementation:
if ( ntp + nfn ) > 0 :
    return ntp / ( ntp + nfn )
else :
    return np . nan


signature:
def matthews_correl_coeff ( ntp , ntn , nfp , nfn ) :
    '''This calculates the Matthews correlation coefficent .
 https : / / en . wikipedia . org / wiki / Matthews _ correlation _ coefficient
 Parameters
 ntp : int
 The number of true positives .
 ntn : int
 The number of true negatives
 nfp : int
 The number of false positives .
 nfn : int
 The number of false negatives .
 Returns
 float
 The Matthews correlation coefficient .'''

implementation:
mcc_top = ( ntp * ntn - nfp * nfn )
mcc_bot = msqrt ( ( ntp + nfp ) * ( ntp + nfn ) * ( ntn + nfp ) * ( ntn + nfn ) )
if mcc_bot > 0 :
    return mcc_top / mcc_bot
else :
    return np . nan


signature:
def get_recovered_variables_for_magbin ( simbasedir , magbinmedian , stetson_stdev_min = 2.0 , inveta_stdev_min = 2.0 , iqr_stdev_min = 2.0 , statsonly = True ) :
    '''This runs variability selection for the given magbinmedian .
 To generate a full recovery matrix over all magnitude bins , run this
 function for each magbin over the specified stetson _ stdev _ min and
 inveta _ stdev _ min grid .
 Parameters
 simbasedir : str
 The input directory of fake LCs .
 magbinmedian : float
 The magbin to run the variable recovery for . This is an item from the
 dict from ` simbasedir / fakelcs - info . pkl : ` fakelcinfo [ ' magrms ' ] [ magcol ] `
 list for each magcol and designates which magbin to get the recovery
 stats for .
 stetson _ stdev _ min : float
 The minimum sigma above the trend in the Stetson J variability index
 distribution for this magbin to use to consider objects as variable .
 inveta _ stdev _ min : float
 The minimum sigma above the trend in the 1 / eta variability index
 distribution for this magbin to use to consider objects as variable .
 iqr _ stdev _ min : float
 The minimum sigma above the trend in the IQR variability index
 distribution for this magbin to use to consider objects as variable .
 statsonly : bool
 If this is True , only the final stats will be returned . If False , the
 full arrays used to generate the stats will also be returned .
 Returns
 dict
 The returned dict contains statistics for this magbin and if requested ,
 the full arrays used to calculate the statistics .'''

implementation:
# get the info from the simbasedir
with open ( os . path . join ( simbasedir , 'fakelcs-info.pkl' ) , 'rb' ) as infd :
    siminfo = pickle . load ( infd )
objectids = siminfo [ 'objectid' ]
varflags = siminfo [ 'isvariable' ]
sdssr = siminfo [ 'sdssr' ]
# get the column defs for the fakelcs
timecols = siminfo [ 'timecols' ]
magcols = siminfo [ 'magcols' ]
errcols = siminfo [ 'errcols' ]
# register the fakelc pklc as a custom lcproc format
# now we should be able to use all lcproc functions correctly
fakelc_formatkey = 'fake-%s' % siminfo [ 'lcformat' ]
lcproc . register_lcformat ( fakelc_formatkey , '*-fakelc.pkl' , timecols , magcols , errcols , 'astrobase.lcproc' , '_read_pklc' , magsarefluxes = siminfo [ 'magsarefluxes' ] )
# make the output directory if it doesn ' t exit
outdir = os . path . join ( simbasedir , 'recvar-threshold-pkls' )
if not os . path . exists ( outdir ) :
    os . mkdir ( outdir )
# run the variability search
varfeaturedir = os . path . join ( simbasedir , 'varfeatures' )
varthreshinfof = os . path . join ( outdir , 'varthresh-magbinmed%.2f-stet%.2f-inveta%.2f.pkl' % ( magbinmedian , stetson_stdev_min , inveta_stdev_min ) )
varthresh = varthreshold . variability_threshold ( varfeaturedir , varthreshinfof , lcformat = fakelc_formatkey , min_stetj_stdev = stetson_stdev_min , min_inveta_stdev = inveta_stdev_min , min_iqr_stdev = iqr_stdev_min , verbose = False )
# get the magbins from the varthresh info
magbins = varthresh [ 'magbins' ]
# get the magbininds
magbininds = np . digitize ( sdssr , magbins )
# bin the objects according to these magbins
binned_objectids = [ ]
binned_actualvars = [ ]
binned_actualnotvars = [ ]
# go through all the mag bins and bin up the objectids , actual variables ,
# and actual not - variables
for mbinind , _magi in zip ( np . unique ( magbininds ) , range ( len ( magbins ) - 1 ) ) :
    thisbinind = np . where ( magbininds == mbinind )
    thisbin_objectids = objectids [ thisbinind ]
    thisbin_varflags = varflags [ thisbinind ]
    thisbin_actualvars = thisbin_objectids [ thisbin_varflags ]
    thisbin_actualnotvars = thisbin_objectids [ ~ thisbin_varflags ]
    binned_objectids . append ( thisbin_objectids )
    binned_actualvars . append ( thisbin_actualvars )
    binned_actualnotvars . append ( thisbin_actualnotvars )
# this is the output dict
recdict = { 'simbasedir' : simbasedir , 'timecols' : timecols , 'magcols' : magcols , 'errcols' : errcols , 'magsarefluxes' : siminfo [ 'magsarefluxes' ] , 'stetj_min_stdev' : stetson_stdev_min , 'inveta_min_stdev' : inveta_stdev_min , 'iqr_min_stdev' : iqr_stdev_min , 'magbinmedian' : magbinmedian , }
# now , for each magcol , find the magbin corresponding to magbinmedian , and
# get its stats
for magcol in magcols : # this is the index of the matching magnitude bin for the magbinmedian
# provided
    magbinind = np . where ( np . array ( varthresh [ magcol ] [ 'binned_sdssr_median' ] ) == magbinmedian )
    magbinind = np . asscalar ( magbinind [ 0 ] )
    # get the objectids , actual vars and actual notvars in this magbin
    thisbin_objectids = binned_objectids [ magbinind ]
    thisbin_actualvars = binned_actualvars [ magbinind ]
    thisbin_actualnotvars = binned_actualnotvars [ magbinind ]
    # stetson recovered variables in this magbin
    stet_recoveredvars = varthresh [ magcol ] [ 'binned_objectids_thresh_stetsonj' ] [ magbinind ]
    # calculate TP , FP , TN , FN
    stet_recoverednotvars = np . setdiff1d ( thisbin_objectids , stet_recoveredvars )
    stet_truepositives = np . intersect1d ( stet_recoveredvars , thisbin_actualvars )
    stet_falsepositives = np . intersect1d ( stet_recoveredvars , thisbin_actualnotvars )
    stet_truenegatives = np . intersect1d ( stet_recoverednotvars , thisbin_actualnotvars )
    stet_falsenegatives = np . intersect1d ( stet_recoverednotvars , thisbin_actualvars )
    # calculate stetson recall , precision , Matthews correl coeff
    stet_recall = recall ( stet_truepositives . size , stet_falsenegatives . size )
    stet_precision = precision ( stet_truepositives . size , stet_falsepositives . size )
    stet_mcc = matthews_correl_coeff ( stet_truepositives . size , stet_truenegatives . size , stet_falsepositives . size , stet_falsenegatives . size )
    # inveta recovered variables in this magbin
    inveta_recoveredvars = varthresh [ magcol ] [ 'binned_objectids_thresh_inveta' ] [ magbinind ]
    inveta_recoverednotvars = np . setdiff1d ( thisbin_objectids , inveta_recoveredvars )
    inveta_truepositives = np . intersect1d ( inveta_recoveredvars , thisbin_actualvars )
    inveta_falsepositives = np . intersect1d ( inveta_recoveredvars , thisbin_actualnotvars )
    inveta_truenegatives = np . intersect1d ( inveta_recoverednotvars , thisbin_actualnotvars )
    inveta_falsenegatives = np . intersect1d ( inveta_recoverednotvars , thisbin_actualvars )
    # calculate inveta recall , precision , Matthews correl coeff
    inveta_recall = recall ( inveta_truepositives . size , inveta_falsenegatives . size )
    inveta_precision = precision ( inveta_truepositives . size , inveta_falsepositives . size )
    inveta_mcc = matthews_correl_coeff ( inveta_truepositives . size , inveta_truenegatives . size , inveta_falsepositives . size , inveta_falsenegatives . size )
    # iqr recovered variables in this magbin
    iqr_recoveredvars = varthresh [ magcol ] [ 'binned_objectids_thresh_iqr' ] [ magbinind ]
    iqr_recoverednotvars = np . setdiff1d ( thisbin_objectids , iqr_recoveredvars )
    iqr_truepositives = np . intersect1d ( iqr_recoveredvars , thisbin_actualvars )
    iqr_falsepositives = np . intersect1d ( iqr_recoveredvars , thisbin_actualnotvars )
    iqr_truenegatives = np . intersect1d ( iqr_recoverednotvars , thisbin_actualnotvars )
    iqr_falsenegatives = np . intersect1d ( iqr_recoverednotvars , thisbin_actualvars )
    # calculate iqr recall , precision , Matthews correl coeff
    iqr_recall = recall ( iqr_truepositives . size , iqr_falsenegatives . size )
    iqr_precision = precision ( iqr_truepositives . size , iqr_falsepositives . size )
    iqr_mcc = matthews_correl_coeff ( iqr_truepositives . size , iqr_truenegatives . size , iqr_falsepositives . size , iqr_falsenegatives . size )
    # calculate the items missed by one method but found by the other
    # methods
    stet_missed_inveta_found = np . setdiff1d ( inveta_truepositives , stet_truepositives )
    stet_missed_iqr_found = np . setdiff1d ( iqr_truepositives , stet_truepositives )
    inveta_missed_stet_found = np . setdiff1d ( stet_truepositives , inveta_truepositives )
    inveta_missed_iqr_found = np . setdiff1d ( iqr_truepositives , inveta_truepositives )
    iqr_missed_stet_found = np . setdiff1d ( stet_truepositives , iqr_truepositives )
    iqr_missed_inveta_found = np . setdiff1d ( inveta_truepositives , iqr_truepositives )
    if not statsonly :
        recdict [ magcol ] = { # stetson J alone
        'stet_recoveredvars' : stet_recoveredvars , 'stet_truepositives' : stet_truepositives , 'stet_falsepositives' : stet_falsepositives , 'stet_truenegatives' : stet_truenegatives , 'stet_falsenegatives' : stet_falsenegatives , 'stet_precision' : stet_precision , 'stet_recall' : stet_recall , 'stet_mcc' : stet_mcc , # inveta alone
        'inveta_recoveredvars' : inveta_recoveredvars , 'inveta_truepositives' : inveta_truepositives , 'inveta_falsepositives' : inveta_falsepositives , 'inveta_truenegatives' : inveta_truenegatives , 'inveta_falsenegatives' : inveta_falsenegatives , 'inveta_precision' : inveta_precision , 'inveta_recall' : inveta_recall , 'inveta_mcc' : inveta_mcc , # iqr alone
        'iqr_recoveredvars' : iqr_recoveredvars , 'iqr_truepositives' : iqr_truepositives , 'iqr_falsepositives' : iqr_falsepositives , 'iqr_truenegatives' : iqr_truenegatives , 'iqr_falsenegatives' : iqr_falsenegatives , 'iqr_precision' : iqr_precision , 'iqr_recall' : iqr_recall , 'iqr_mcc' : iqr_mcc , # true positive variables missed by one method but picked up by
        # the others
        'stet_missed_inveta_found' : stet_missed_inveta_found , 'stet_missed_iqr_found' : stet_missed_iqr_found , 'inveta_missed_stet_found' : inveta_missed_stet_found , 'inveta_missed_iqr_found' : inveta_missed_iqr_found , 'iqr_missed_stet_found' : iqr_missed_stet_found , 'iqr_missed_inveta_found' : iqr_missed_inveta_found , # bin info
        'actual_variables' : thisbin_actualvars , 'actual_nonvariables' : thisbin_actualnotvars , 'all_objectids' : thisbin_objectids , 'magbinind' : magbinind , }
    # if statsonly is set , then we only return the numbers but not the
    # arrays themselves
    else :
        recdict [ magcol ] = { # stetson J alone
        'stet_recoveredvars' : stet_recoveredvars . size , 'stet_truepositives' : stet_truepositives . size , 'stet_falsepositives' : stet_falsepositives . size , 'stet_truenegatives' : stet_truenegatives . size , 'stet_falsenegatives' : stet_falsenegatives . size , 'stet_precision' : stet_precision , 'stet_recall' : stet_recall , 'stet_mcc' : stet_mcc , # inveta alone
        'inveta_recoveredvars' : inveta_recoveredvars . size , 'inveta_truepositives' : inveta_truepositives . size , 'inveta_falsepositives' : inveta_falsepositives . size , 'inveta_truenegatives' : inveta_truenegatives . size , 'inveta_falsenegatives' : inveta_falsenegatives . size , 'inveta_precision' : inveta_precision , 'inveta_recall' : inveta_recall , 'inveta_mcc' : inveta_mcc , # iqr alone
        'iqr_recoveredvars' : iqr_recoveredvars . size , 'iqr_truepositives' : iqr_truepositives . size , 'iqr_falsepositives' : iqr_falsepositives . size , 'iqr_truenegatives' : iqr_truenegatives . size , 'iqr_falsenegatives' : iqr_falsenegatives . size , 'iqr_precision' : iqr_precision , 'iqr_recall' : iqr_recall , 'iqr_mcc' : iqr_mcc , # true positive variables missed by one method but picked up by
        # the others
        'stet_missed_inveta_found' : stet_missed_inveta_found . size , 'stet_missed_iqr_found' : stet_missed_iqr_found . size , 'inveta_missed_stet_found' : inveta_missed_stet_found . size , 'inveta_missed_iqr_found' : inveta_missed_iqr_found . size , 'iqr_missed_stet_found' : iqr_missed_stet_found . size , 'iqr_missed_inveta_found' : iqr_missed_inveta_found . size , # bin info
        'actual_variables' : thisbin_actualvars . size , 'actual_nonvariables' : thisbin_actualnotvars . size , 'all_objectids' : thisbin_objectids . size , 'magbinind' : magbinind , }
# done with per magcol
return recdict


signature:
def magbin_varind_gridsearch_worker ( task ) :
    '''This is a parallel grid search worker for the function below .'''

implementation:
simbasedir , gridpoint , magbinmedian = task
try :
    res = get_recovered_variables_for_magbin ( simbasedir , magbinmedian , stetson_stdev_min = gridpoint [ 0 ] , inveta_stdev_min = gridpoint [ 1 ] , iqr_stdev_min = gridpoint [ 2 ] , statsonly = True )
    return res
except Exception as e :
    LOGEXCEPTION ( 'failed to get info for %s' % gridpoint )
    return None


signature:
def variable_index_gridsearch_magbin ( simbasedir , stetson_stdev_range = ( 1.0 , 20.0 ) , inveta_stdev_range = ( 1.0 , 20.0 ) , iqr_stdev_range = ( 1.0 , 20.0 ) , ngridpoints = 32 , ngridworkers = None ) :
    '''This runs a variable index grid search per magbin .
 For each magbin , this does a grid search using the stetson and inveta ranges
 provided and tries to optimize the Matthews Correlation Coefficient ( best
 value is + 1.0 ) , indicating the best possible separation of variables
 vs . nonvariables . The thresholds on these two variable indexes that produce
 the largest coeff for the collection of fake LCs will probably be the ones
 that work best for actual variable classification on the real LCs .
 https : / / en . wikipedia . org / wiki / Matthews _ correlation _ coefficient
 For each grid - point , calculates the true positives , false positives , true
 negatives , false negatives . Then gets the precision and recall , confusion
 matrix , and the ROC curve for variable vs . nonvariable .
 Once we ' ve identified the best thresholds to use , we can then calculate
 variable object numbers :
 - as a function of magnitude
 - as a function of period
 - as a function of number of detections
 - as a function of amplitude of variability
 Writes everything back to ` simbasedir / fakevar - recovery . pkl ` . Use the
 plotting function below to make plots for the results .
 Parameters
 simbasedir : str
 The directory where the fake LCs are located .
 stetson _ stdev _ range : sequence of 2 floats
 The min and max values of the Stetson J variability index to generate a
 grid over these to test for the values of this index that produce the
 ' best ' recovery rate for the injected variable stars .
 inveta _ stdev _ range : sequence of 2 floats
 The min and max values of the 1 / eta variability index to generate a
 grid over these to test for the values of this index that produce the
 ' best ' recovery rate for the injected variable stars .
 iqr _ stdev _ range : sequence of 2 floats
 The min and max values of the IQR variability index to generate a
 grid over these to test for the values of this index that produce the
 ' best ' recovery rate for the injected variable stars .
 ngridpoints : int
 The number of grid points for each variability index grid . Remember that
 this function will be searching in 3D and will require lots of time to
 run if ngridpoints is too large .
 For the default number of grid points and 25000 simulated light curves ,
 this takes about 3 days to run on a 40 ( effective ) core machine with 2 x
 Xeon E5-2650v3 CPUs .
 ngridworkers : int or None
 The number of parallel grid search workers that will be launched .
 Returns
 dict
 The returned dict contains a list of recovery stats for each magbin and
 each grid point in the variability index grids that were used . This dict
 can be passed to the plotting function below to plot the results .'''

implementation:
# make the output directory where all the pkls from the variability
# threshold runs will go
outdir = os . path . join ( simbasedir , 'recvar-threshold-pkls' )
if not os . path . exists ( outdir ) :
    os . mkdir ( outdir )
# get the info from the simbasedir
with open ( os . path . join ( simbasedir , 'fakelcs-info.pkl' ) , 'rb' ) as infd :
    siminfo = pickle . load ( infd )
# get the column defs for the fakelcs
timecols = siminfo [ 'timecols' ]
magcols = siminfo [ 'magcols' ]
errcols = siminfo [ 'errcols' ]
# get the magbinmedians to use for the recovery processing
magbinmedians = siminfo [ 'magrms' ] [ magcols [ 0 ] ] [ 'binned_sdssr_median' ]
# generate the grids for stetson and inveta
stetson_grid = np . linspace ( stetson_stdev_range [ 0 ] , stetson_stdev_range [ 1 ] , num = ngridpoints )
inveta_grid = np . linspace ( inveta_stdev_range [ 0 ] , inveta_stdev_range [ 1 ] , num = ngridpoints )
iqr_grid = np . linspace ( iqr_stdev_range [ 0 ] , iqr_stdev_range [ 1 ] , num = ngridpoints )
# generate the grid
stet_inveta_iqr_grid = [ ]
for stet in stetson_grid :
    for inveta in inveta_grid :
        for iqr in iqr_grid :
            grid_point = [ stet , inveta , iqr ]
            stet_inveta_iqr_grid . append ( grid_point )
# the output dict
grid_results = { 'stetson_grid' : stetson_grid , 'inveta_grid' : inveta_grid , 'iqr_grid' : iqr_grid , 'stet_inveta_iqr_grid' : stet_inveta_iqr_grid , 'magbinmedians' : magbinmedians , 'timecols' : timecols , 'magcols' : magcols , 'errcols' : errcols , 'simbasedir' : os . path . abspath ( simbasedir ) , 'recovery' : [ ] }
# set up the pool
pool = mp . Pool ( ngridworkers )
# run the grid search per magbinmedian
for magbinmedian in magbinmedians :
    LOGINFO ( 'running stetson J-inveta grid-search ' 'for magbinmedian = %.3f...' % magbinmedian )
    tasks = [ ( simbasedir , gp , magbinmedian ) for gp in stet_inveta_iqr_grid ]
    thisbin_results = pool . map ( magbin_varind_gridsearch_worker , tasks )
    grid_results [ 'recovery' ] . append ( thisbin_results )
pool . close ( )
pool . join ( )
LOGINFO ( 'done.' )
with open ( os . path . join ( simbasedir , 'fakevar-recovery-per-magbin.pkl' ) , 'wb' ) as outfd :
    pickle . dump ( grid_results , outfd , pickle . HIGHEST_PROTOCOL )
return grid_results


signature:
def plot_varind_gridsearch_magbin_results ( gridsearch_results ) :
    '''This plots the gridsearch results from ` variable _ index _ gridsearch _ magbin ` .
 Parameters
 gridsearch _ results : dict
 This is the dict produced by ` variable _ index _ gridsearch _ magbin ` above .
 Returns
 dict
 The returned dict contains filenames of the recovery rate plots made for
 each variability index . These include plots of the precision , recall ,
 and Matthews Correlation Coefficient over each magbin and a heatmap of
 these values over the grid points of the variability index stdev values
 arrays used .'''

implementation:
# get the result pickle / dict
if ( isinstance ( gridsearch_results , str ) and os . path . exists ( gridsearch_results ) ) :
    with open ( gridsearch_results , 'rb' ) as infd :
        gridresults = pickle . load ( infd )
elif isinstance ( gridsearch_results , dict ) :
    gridresults = gridsearch_results
else :
    LOGERROR ( 'could not understand the input ' 'variable index grid-search result dict/pickle' )
    return None
plotres = { 'simbasedir' : gridresults [ 'simbasedir' ] }
recgrid = gridresults [ 'recovery' ]
simbasedir = gridresults [ 'simbasedir' ]
for magcol in gridresults [ 'magcols' ] :
    plotres [ magcol ] = { 'best_stetsonj' : [ ] , 'best_inveta' : [ ] , 'best_iqr' : [ ] , 'magbinmedians' : gridresults [ 'magbinmedians' ] }
    # go through all the magbins
    for magbinind , magbinmedian in enumerate ( gridresults [ 'magbinmedians' ] ) :
        LOGINFO ( 'plotting results for %s: magbin: %.3f' % ( magcol , magbinmedian ) )
        stet_mcc = np . array ( [ x [ magcol ] [ 'stet_mcc' ] for x in recgrid [ magbinind ] ] ) [ : : ( gridresults [ 'inveta_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ]
        stet_precision = np . array ( [ x [ magcol ] [ 'stet_precision' ] for x in recgrid [ magbinind ] ] ) [ : : ( gridresults [ 'inveta_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ]
        stet_recall = np . array ( [ x [ magcol ] [ 'stet_recall' ] for x in recgrid [ magbinind ] ] ) [ : : ( gridresults [ 'inveta_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ]
        stet_missed_inveta_found = np . array ( [ x [ magcol ] [ 'stet_missed_inveta_found' ] for x in recgrid [ magbinind ] ] ) [ : : ( gridresults [ 'inveta_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ]
        stet_missed_iqr_found = np . array ( [ x [ magcol ] [ 'stet_missed_iqr_found' ] for x in recgrid [ magbinind ] ] ) [ : : ( gridresults [ 'inveta_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ]
        inveta_mcc = np . array ( [ x [ magcol ] [ 'inveta_mcc' ] for x in recgrid [ magbinind ] ] ) [ : ( gridresults [ 'iqr_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ] [ : : gridresults [ 'inveta_grid' ] . size ]
        inveta_precision = np . array ( [ x [ magcol ] [ 'inveta_precision' ] for x in recgrid [ magbinind ] ] ) [ : ( gridresults [ 'iqr_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ] [ : : gridresults [ 'inveta_grid' ] . size ]
        inveta_recall = np . array ( [ x [ magcol ] [ 'inveta_recall' ] for x in recgrid [ magbinind ] ] ) [ : ( gridresults [ 'iqr_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ] [ : : gridresults [ 'inveta_grid' ] . size ]
        inveta_missed_stet_found = np . array ( [ x [ magcol ] [ 'inveta_missed_stet_found' ] for x in recgrid [ magbinind ] ] ) [ : ( gridresults [ 'iqr_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ] [ : : gridresults [ 'inveta_grid' ] . size ]
        inveta_missed_iqr_found = np . array ( [ x [ magcol ] [ 'inveta_missed_iqr_found' ] for x in recgrid [ magbinind ] ] ) [ : ( gridresults [ 'iqr_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ] [ : : gridresults [ 'inveta_grid' ] . size ]
        iqr_mcc = np . array ( [ x [ magcol ] [ 'iqr_mcc' ] for x in recgrid [ magbinind ] ] ) [ : ( gridresults [ 'iqr_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ] [ : gridresults [ 'inveta_grid' ] . size ]
        iqr_precision = np . array ( [ x [ magcol ] [ 'iqr_precision' ] for x in recgrid [ magbinind ] ] ) [ : ( gridresults [ 'iqr_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ] [ : gridresults [ 'inveta_grid' ] . size ]
        iqr_recall = np . array ( [ x [ magcol ] [ 'iqr_recall' ] for x in recgrid [ magbinind ] ] ) [ : ( gridresults [ 'iqr_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ] [ : gridresults [ 'inveta_grid' ] . size ]
        iqr_missed_stet_found = np . array ( [ x [ magcol ] [ 'iqr_missed_stet_found' ] for x in recgrid [ magbinind ] ] ) [ : ( gridresults [ 'iqr_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ] [ : gridresults [ 'inveta_grid' ] . size ]
        iqr_missed_inveta_found = np . array ( [ x [ magcol ] [ 'iqr_missed_inveta_found' ] for x in recgrid [ magbinind ] ] ) [ : ( gridresults [ 'iqr_grid' ] . size * gridresults [ 'stetson_grid' ] . size ) ] [ : gridresults [ 'inveta_grid' ] . size ]
        fig = plt . figure ( figsize = ( 6.4 * 5 , 4.8 * 3 ) )
        # FIRST ROW : stetson J plot
        plt . subplot ( 3 , 5 , 1 )
        if np . any ( np . isfinite ( stet_mcc ) ) :
            plt . plot ( gridresults [ 'stetson_grid' ] , stet_mcc )
            plt . xlabel ( 'stetson J stdev multiplier threshold' )
            plt . ylabel ( 'MCC' )
            plt . title ( 'MCC for stetson J' )
        else :
            plt . text ( 0.5 , 0.5 , 'stet MCC values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 2 )
        if np . any ( np . isfinite ( stet_precision ) ) :
            plt . plot ( gridresults [ 'stetson_grid' ] , stet_precision )
            plt . xlabel ( 'stetson J stdev multiplier threshold' )
            plt . ylabel ( 'precision' )
            plt . title ( 'precision for stetson J' )
        else :
            plt . text ( 0.5 , 0.5 , 'stet precision values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 3 )
        if np . any ( np . isfinite ( stet_recall ) ) :
            plt . plot ( gridresults [ 'stetson_grid' ] , stet_recall )
            plt . xlabel ( 'stetson J stdev multiplier threshold' )
            plt . ylabel ( 'recall' )
            plt . title ( 'recall for stetson J' )
        else :
            plt . text ( 0.5 , 0.5 , 'stet recall values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 4 )
        if np . any ( np . isfinite ( stet_missed_inveta_found ) ) :
            plt . plot ( gridresults [ 'stetson_grid' ] , stet_missed_inveta_found )
            plt . xlabel ( 'stetson J stdev multiplier threshold' )
            plt . ylabel ( '# objects stetson missed but inveta found' )
            plt . title ( 'stetson J missed, inveta found' )
        else :
            plt . text ( 0.5 , 0.5 , 'stet-missed/inveta-found values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 5 )
        if np . any ( np . isfinite ( stet_missed_iqr_found ) ) :
            plt . plot ( gridresults [ 'stetson_grid' ] , stet_missed_iqr_found )
            plt . xlabel ( 'stetson J stdev multiplier threshold' )
            plt . ylabel ( '# objects stetson missed but IQR found' )
            plt . title ( 'stetson J missed, IQR found' )
        else :
            plt . text ( 0.5 , 0.5 , 'stet-missed/IQR-found values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        # SECOND ROW : inveta plots
        plt . subplot ( 3 , 5 , 6 )
        if np . any ( np . isfinite ( inveta_mcc ) ) :
            plt . plot ( gridresults [ 'inveta_grid' ] , inveta_mcc )
            plt . xlabel ( 'inveta stdev multiplier threshold' )
            plt . ylabel ( 'MCC' )
            plt . title ( 'MCC for inveta' )
        else :
            plt . text ( 0.5 , 0.5 , 'inveta MCC values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 7 )
        if np . any ( np . isfinite ( inveta_precision ) ) :
            plt . plot ( gridresults [ 'inveta_grid' ] , inveta_precision )
            plt . xlabel ( 'inveta stdev multiplier threshold' )
            plt . ylabel ( 'precision' )
            plt . title ( 'precision for inveta' )
        else :
            plt . text ( 0.5 , 0.5 , 'inveta precision values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 8 )
        if np . any ( np . isfinite ( inveta_recall ) ) :
            plt . plot ( gridresults [ 'inveta_grid' ] , inveta_recall )
            plt . xlabel ( 'inveta stdev multiplier threshold' )
            plt . ylabel ( 'recall' )
            plt . title ( 'recall for inveta' )
        else :
            plt . text ( 0.5 , 0.5 , 'inveta recall values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 9 )
        if np . any ( np . isfinite ( inveta_missed_stet_found ) ) :
            plt . plot ( gridresults [ 'inveta_grid' ] , inveta_missed_stet_found )
            plt . xlabel ( 'inveta stdev multiplier threshold' )
            plt . ylabel ( '# objects inveta missed but stetson found' )
            plt . title ( 'inveta missed, stetson J found' )
        else :
            plt . text ( 0.5 , 0.5 , 'inveta-missed-stet-found values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 10 )
        if np . any ( np . isfinite ( inveta_missed_iqr_found ) ) :
            plt . plot ( gridresults [ 'inveta_grid' ] , inveta_missed_iqr_found )
            plt . xlabel ( 'inveta stdev multiplier threshold' )
            plt . ylabel ( '# objects inveta missed but IQR found' )
            plt . title ( 'inveta missed, IQR found' )
        else :
            plt . text ( 0.5 , 0.5 , 'inveta-missed-iqr-found values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        # THIRD ROW : inveta plots
        plt . subplot ( 3 , 5 , 11 )
        if np . any ( np . isfinite ( iqr_mcc ) ) :
            plt . plot ( gridresults [ 'iqr_grid' ] , iqr_mcc )
            plt . xlabel ( 'IQR stdev multiplier threshold' )
            plt . ylabel ( 'MCC' )
            plt . title ( 'MCC for IQR' )
        else :
            plt . text ( 0.5 , 0.5 , 'IQR MCC values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 12 )
        if np . any ( np . isfinite ( iqr_precision ) ) :
            plt . plot ( gridresults [ 'iqr_grid' ] , iqr_precision )
            plt . xlabel ( 'IQR stdev multiplier threshold' )
            plt . ylabel ( 'precision' )
            plt . title ( 'precision for IQR' )
        else :
            plt . text ( 0.5 , 0.5 , 'IQR precision values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 13 )
        if np . any ( np . isfinite ( iqr_recall ) ) :
            plt . plot ( gridresults [ 'iqr_grid' ] , iqr_recall )
            plt . xlabel ( 'IQR stdev multiplier threshold' )
            plt . ylabel ( 'recall' )
            plt . title ( 'recall for IQR' )
        else :
            plt . text ( 0.5 , 0.5 , 'IQR recall values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 14 )
        if np . any ( np . isfinite ( iqr_missed_stet_found ) ) :
            plt . plot ( gridresults [ 'iqr_grid' ] , iqr_missed_stet_found )
            plt . xlabel ( 'IQR stdev multiplier threshold' )
            plt . ylabel ( '# objects IQR missed but stetson found' )
            plt . title ( 'IQR missed, stetson J found' )
        else :
            plt . text ( 0.5 , 0.5 , 'iqr-missed-stet-found values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplot ( 3 , 5 , 15 )
        if np . any ( np . isfinite ( iqr_missed_inveta_found ) ) :
            plt . plot ( gridresults [ 'iqr_grid' ] , iqr_missed_inveta_found )
            plt . xlabel ( 'IQR stdev multiplier threshold' )
            plt . ylabel ( '# objects IQR missed but inveta found' )
            plt . title ( 'IQR missed, inveta found' )
        else :
            plt . text ( 0.5 , 0.5 , 'iqr-missed-inveta-found values are all nan ' 'for this magbin' , transform = plt . gca ( ) . transAxes , horizontalalignment = 'center' , verticalalignment = 'center' )
            plt . xticks ( [ ] )
            plt . yticks ( [ ] )
        plt . subplots_adjust ( hspace = 0.25 , wspace = 0.25 )
        plt . suptitle ( 'magcol: %s, magbin: %.3f' % ( magcol , magbinmedian ) )
        plotdir = os . path . join ( gridresults [ 'simbasedir' ] , 'varindex-gridsearch-plots' )
        if not os . path . exists ( plotdir ) :
            os . mkdir ( plotdir )
        gridplotf = os . path . join ( plotdir , '%s-magbin-%.3f-var-recoverygrid-permagbin.png' % ( magcol , magbinmedian ) )
        plt . savefig ( gridplotf , dpi = 100 , bbox_inches = 'tight' )
        plt . close ( 'all' )
        # get the best values of MCC , recall , precision and their associated
        # stet , inveta
        stet_mcc_maxind = np . where ( stet_mcc == np . max ( stet_mcc ) )
        stet_precision_maxind = np . where ( stet_precision == np . max ( stet_precision ) )
        stet_recall_maxind = np . where ( stet_recall == np . max ( stet_recall ) )
        best_stet_mcc = stet_mcc [ stet_mcc_maxind ]
        best_stet_precision = stet_mcc [ stet_precision_maxind ]
        best_stet_recall = stet_mcc [ stet_recall_maxind ]
        stet_with_best_mcc = gridresults [ 'stetson_grid' ] [ stet_mcc_maxind ]
        stet_with_best_precision = gridresults [ 'stetson_grid' ] [ stet_precision_maxind ]
        stet_with_best_recall = ( gridresults [ 'stetson_grid' ] [ stet_recall_maxind ] )
        inveta_mcc_maxind = np . where ( inveta_mcc == np . max ( inveta_mcc ) )
        inveta_precision_maxind = np . where ( inveta_precision == np . max ( inveta_precision ) )
        inveta_recall_maxind = ( np . where ( inveta_recall == np . max ( inveta_recall ) ) )
        best_inveta_mcc = inveta_mcc [ inveta_mcc_maxind ]
        best_inveta_precision = inveta_mcc [ inveta_precision_maxind ]
        best_inveta_recall = inveta_mcc [ inveta_recall_maxind ]
        inveta_with_best_mcc = gridresults [ 'inveta_grid' ] [ inveta_mcc_maxind ]
        inveta_with_best_precision = gridresults [ 'inveta_grid' ] [ inveta_precision_maxind ]
        inveta_with_best_recall = gridresults [ 'inveta_grid' ] [ inveta_recall_maxind ]
        iqr_mcc_maxind = np . where ( iqr_mcc == np . max ( iqr_mcc ) )
        iqr_precision_maxind = np . where ( iqr_precision == np . max ( iqr_precision ) )
        iqr_recall_maxind = ( np . where ( iqr_recall == np . max ( iqr_recall ) ) )
        best_iqr_mcc = iqr_mcc [ iqr_mcc_maxind ]
        best_iqr_precision = iqr_mcc [ iqr_precision_maxind ]
        best_iqr_recall = iqr_mcc [ iqr_recall_maxind ]
        iqr_with_best_mcc = gridresults [ 'iqr_grid' ] [ iqr_mcc_maxind ]
        iqr_with_best_precision = gridresults [ 'iqr_grid' ] [ iqr_precision_maxind ]
        iqr_with_best_recall = gridresults [ 'iqr_grid' ] [ iqr_recall_maxind ]
        plotres [ magcol ] [ magbinmedian ] = { # stetson
        'stet_grid' : gridresults [ 'stetson_grid' ] , 'stet_mcc' : stet_mcc , 'stet_precision' : stet_precision , 'stet_recall' : stet_recall , 'stet_missed_inveta_found' : stet_missed_inveta_found , 'best_stet_mcc' : best_stet_mcc , 'stet_with_best_mcc' : stet_with_best_mcc , 'best_stet_precision' : best_stet_precision , 'stet_with_best_precision' : stet_with_best_precision , 'best_stet_recall' : best_stet_recall , 'stet_with_best_recall' : stet_with_best_recall , # inveta
        'inveta_grid' : gridresults [ 'inveta_grid' ] , 'inveta_mcc' : inveta_mcc , 'inveta_precision' : inveta_precision , 'inveta_recall' : inveta_recall , 'inveta_missed_stet_found' : inveta_missed_stet_found , 'best_inveta_mcc' : best_inveta_mcc , 'inveta_with_best_mcc' : inveta_with_best_mcc , 'best_inveta_precision' : best_inveta_precision , 'inveta_with_best_precision' : inveta_with_best_precision , 'best_inveta_recall' : best_inveta_recall , 'inveta_with_best_recall' : inveta_with_best_recall , # iqr
        'iqr_grid' : gridresults [ 'iqr_grid' ] , 'iqr_mcc' : iqr_mcc , 'iqr_precision' : iqr_precision , 'iqr_recall' : iqr_recall , 'iqr_missed_stet_found' : iqr_missed_stet_found , 'best_iqr_mcc' : best_iqr_mcc , 'iqr_with_best_mcc' : iqr_with_best_mcc , 'best_iqr_precision' : best_iqr_precision , 'iqr_with_best_precision' : iqr_with_best_precision , 'best_iqr_recall' : best_iqr_recall , 'iqr_with_best_recall' : iqr_with_best_recall , # plot info
        'recoveryplot' : gridplotf }
        # recommend inveta , stetson index , and iqr for this magbin
        # if there are multiple stets , choose the smallest one
        if stet_with_best_mcc . size > 1 :
            plotres [ magcol ] [ 'best_stetsonj' ] . append ( stet_with_best_mcc [ 0 ] )
        elif stet_with_best_mcc . size > 0 :
            plotres [ magcol ] [ 'best_stetsonj' ] . append ( stet_with_best_mcc [ 0 ] )
        else :
            plotres [ magcol ] [ 'best_stetsonj' ] . append ( np . nan )
        # if there are multiple best invetas , choose the smallest one
        if inveta_with_best_mcc . size > 1 :
            plotres [ magcol ] [ 'best_inveta' ] . append ( inveta_with_best_mcc [ 0 ] )
        elif inveta_with_best_mcc . size > 0 :
            plotres [ magcol ] [ 'best_inveta' ] . append ( inveta_with_best_mcc [ 0 ] )
        else :
            plotres [ magcol ] [ 'best_inveta' ] . append ( np . nan )
        # if there are multiple best iqrs , choose the smallest one
        if iqr_with_best_mcc . size > 1 :
            plotres [ magcol ] [ 'best_iqr' ] . append ( iqr_with_best_mcc [ 0 ] )
        elif iqr_with_best_mcc . size > 0 :
            plotres [ magcol ] [ 'best_iqr' ] . append ( iqr_with_best_mcc [ 0 ] )
        else :
            plotres [ magcol ] [ 'best_iqr' ] . append ( np . nan )
# write the plotresults to a pickle
plotrespicklef = os . path . join ( simbasedir , 'varindex-gridsearch-magbin-results.pkl' )
with open ( plotrespicklef , 'wb' ) as outfd :
    pickle . dump ( plotres , outfd , pickle . HIGHEST_PROTOCOL )
# recommend the values of stetson J and inveta to use
for magcol in gridresults [ 'magcols' ] :
    LOGINFO ( 'best stdev multipliers for each %s magbin:' % magcol )
    LOGINFO ( 'magbin    inveta    stetson J    IQR' )
    for magbin , inveta , stet , iqr in zip ( plotres [ magcol ] [ 'magbinmedians' ] , plotres [ magcol ] [ 'best_inveta' ] , plotres [ magcol ] [ 'best_stetsonj' ] , plotres [ magcol ] [ 'best_iqr' ] ) :
        LOGINFO ( '%.3f    %.3f    %.3f    %.3f' % ( magbin , inveta , stet , iqr ) )
return plotres


signature:
def run_periodfinding ( simbasedir , pfmethods = ( 'gls' , 'pdm' , 'bls' ) , pfkwargs = ( { } , { } , { 'startp' : 1.0 , 'maxtransitduration' : 0.3 } ) , getblssnr = False , sigclip = 5.0 , nperiodworkers = 10 , ncontrolworkers = 4 , liststartindex = None , listmaxobjects = None ) :
    '''This runs periodfinding using several period - finders on a collection of
 fake LCs .
 As a rough benchmark , 25000 fake LCs with 10000 - - 50000 points per LC take
 about 26 days in total to run on an invocation of this function using
 GLS + PDM + BLS and 10 periodworkers and 4 controlworkers ( so all 40 ' cores ' ) on
 a 2 x Xeon E5-2660v3 machine .
 Parameters
 pfmethods : sequence of str
 This is used to specify which periodfinders to run . These must be in the
 ` lcproc . periodsearch . PFMETHODS ` dict .
 pfkwargs : sequence of dict
 This is used to provide optional kwargs to the period - finders .
 getblssnr : bool
 If this is True , will run BLS SNR calculations for each object and
 magcol . This takes a while to run , so it ' s disabled ( False ) by default .
 sigclip : float or int or sequence of two floats / ints or None
 If a single float or int , a symmetric sigma - clip will be performed using
 the number provided as the sigma - multiplier to cut out from the input
 time - series .
 If a list of two ints / floats is provided , the function will perform an
 ' asymmetric ' sigma - clip . The first element in this list is the sigma
 value to use for fainter flux / mag values ; the second element in this
 list is the sigma value to use for brighter flux / mag values . For
 example , ` sigclip = [ 10 . , 3 . ] ` , will sigclip out greater than 10 - sigma
 dimmings and greater than 3 - sigma brightenings . Here the meaning of
 " dimming " and " brightening " is set by * physics * ( not the magnitude
 system ) , which is why the ` magsarefluxes ` kwarg must be correctly set .
 If ` sigclip ` is None , no sigma - clipping will be performed , and the
 time - series ( with non - finite elems removed ) will be passed through to
 the output .
 nperiodworkers : int
 This is the number of parallel period - finding worker processes to use .
 ncontrolworkers : int
 This is the number of parallel period - finding control workers to
 use . Each control worker will launch ` nperiodworkers ` worker processes .
 liststartindex : int
 The starting index of processing . This refers to the filename list
 generated by running ` glob . glob ` on the fake LCs in ` simbasedir ` .
 maxobjects : int
 The maximum number of objects to process in this run . Use this with
 ` liststartindex ` to effectively distribute working on a large list of
 input light curves over several sessions or machines .
 Returns
 str
 The path to the output summary pickle produced by
 ` lcproc . periodsearch . parallel _ pf `'''

implementation:
# get the info from the simbasedir
with open ( os . path . join ( simbasedir , 'fakelcs-info.pkl' ) , 'rb' ) as infd :
    siminfo = pickle . load ( infd )
lcfpaths = siminfo [ 'lcfpath' ]
pfdir = os . path . join ( simbasedir , 'periodfinding' )
# get the column defs for the fakelcs
timecols = siminfo [ 'timecols' ]
magcols = siminfo [ 'magcols' ]
errcols = siminfo [ 'errcols' ]
# register the fakelc pklc as a custom lcproc format
# now we should be able to use all lcproc functions correctly
fakelc_formatkey = 'fake-%s' % siminfo [ 'lcformat' ]
lcproc . register_lcformat ( fakelc_formatkey , '*-fakelc.pkl' , timecols , magcols , errcols , 'astrobase.lcproc' , '_read_pklc' , magsarefluxes = siminfo [ 'magsarefluxes' ] )
if liststartindex :
    lcfpaths = lcfpaths [ liststartindex : ]
if listmaxobjects :
    lcfpaths = lcfpaths [ : listmaxobjects ]
pfinfo = periodsearch . parallel_pf ( lcfpaths , pfdir , lcformat = fakelc_formatkey , pfmethods = pfmethods , pfkwargs = pfkwargs , getblssnr = getblssnr , sigclip = sigclip , nperiodworkers = nperiodworkers , ncontrolworkers = ncontrolworkers )
with open ( os . path . join ( simbasedir , 'fakelc-periodsearch.pkl' ) , 'wb' ) as outfd :
    pickle . dump ( pfinfo , outfd , pickle . HIGHEST_PROTOCOL )
return os . path . join ( simbasedir , 'fakelc-periodsearch.pkl' )


signature:
def check_periodrec_alias ( actualperiod , recoveredperiod , tolerance = 1.0e-3 ) :
    '''This determines what kind of aliasing ( if any ) exists between
 ` recoveredperiod ` and ` actualperiod ` .
 Parameters
 actualperiod : float
 The actual period of the object .
 recoveredperiod : float
 The recovered period of the object .
 tolerance : float
 The absolute difference required between the input periods to mark the
 recovered period as close to the actual period .
 Returns
 str
 The type of alias determined for the input combination of periods . This
 will be CSV string with values taken from the following list , based on
 the types of alias found : :
 [ ' actual ' ,
 ' twice ' ,
 ' half ' ,
 ' ratio _ over _ 1plus ' ,
 ' ratio _ over _ 1minus ' ,
 ' ratio _ over _ 1plus _ twice ' ,
 ' ratio _ over _ 1minus _ twice ' ,
 ' ratio _ over _ 1plus _ thrice ' ,
 ' ratio _ over _ 1minus _ thrice ' ,
 ' ratio _ over _ minus1 ' ,
 ' ratio _ over _ twice _ minus1 ' ]'''

implementation:
if not ( np . isfinite ( actualperiod ) and np . isfinite ( recoveredperiod ) ) :
    LOGERROR ( "can't compare nan values for actual/recovered periods" )
    return 'unknown'
else : # # ALIAS TYPES # #
# simple ratios
    twotimes_p = actualperiod * 2.0
    half_p = actualperiod * 0.5
    # first kind of alias
    alias_1a = actualperiod / ( 1.0 + actualperiod )
    alias_1b = actualperiod / ( 1.0 - actualperiod )
    # second kind of alias
    alias_2a = actualperiod / ( 1.0 + 2.0 * actualperiod )
    alias_2b = actualperiod / ( 1.0 - 2.0 * actualperiod )
    # third kind of alias
    alias_3a = actualperiod / ( 1.0 + 3.0 * actualperiod )
    alias_3b = actualperiod / ( 1.0 - 3.0 * actualperiod )
    # fourth kind of alias
    alias_4a = actualperiod / ( actualperiod - 1.0 )
    alias_4b = actualperiod / ( 2.0 * actualperiod - 1.0 )
    aliases = np . ravel ( np . array ( [ actualperiod , twotimes_p , half_p , alias_1a , alias_1b , alias_2a , alias_2b , alias_3a , alias_3b , alias_4a , alias_4b ] ) )
    alias_labels = np . array ( ALIAS_TYPES )
    # check type of alias
    closest_alias = np . isclose ( recoveredperiod , aliases , atol = tolerance )
    if np . any ( closest_alias ) :
        closest_alias_type = alias_labels [ closest_alias ]
        return ',' . join ( closest_alias_type . tolist ( ) )
    else :
        return 'other'


signature:
def periodicvar_recovery ( fakepfpkl , simbasedir , period_tolerance = 1.0e-3 ) :
    '''Recovers the periodic variable status / info for the simulated PF result .
 - Uses simbasedir and the lcfbasename stored in fakepfpkl to figure out
 where the LC for this object is .
 - Gets the actual _ varparams , actual _ varperiod , actual _ vartype ,
 actual _ varamplitude elements from the LC .
 - Figures out if the current objectid is a periodic variable ( using
 actual _ vartype ) .
 - If it is a periodic variable , gets the canonical period assigned to it .
 - Checks if the period was recovered in any of the five best periods
 reported by any of the period - finders , checks if the period recovered was
 a harmonic of the period .
 - Returns the objectid , actual period and vartype , recovered period , and
 recovery status .
 Parameters
 fakepfpkl : str
 This is a periodfinding - < objectid > . pkl [ . gz ] file produced in the
 ` simbasedir / periodfinding ` subdirectory after ` run _ periodfinding ` above
 is done .
 simbasedir : str
 The base directory where all of the fake LCs and period - finding results
 are .
 period _ tolerance : float
 The maximum difference that this function will consider between an
 actual period ( or its aliases ) and a recovered period to consider it as
 as a ' recovered ' period .
 Returns
 dict
 Returns a dict of period - recovery results .'''

implementation:
if fakepfpkl . endswith ( '.gz' ) :
    infd = gzip . open ( fakepfpkl , 'rb' )
else :
    infd = open ( fakepfpkl , 'rb' )
fakepf = pickle . load ( infd )
infd . close ( )
# get info from the fakepf dict
objectid , lcfbasename = fakepf [ 'objectid' ] , fakepf [ 'lcfbasename' ]
lcfpath = os . path . join ( simbasedir , 'lightcurves' , lcfbasename )
# if the LC doesn ' t exist , bail out
if not os . path . exists ( lcfpath ) :
    LOGERROR ( 'light curve for %s does not exist at: %s' % ( objectid , lcfpath ) )
    return None
# now , open the fakelc
fakelc = lcproc . _read_pklc ( lcfpath )
# get the actual _ varparams , actual _ varperiod , actual _ varamplitude
actual_varparams , actual_varperiod , actual_varamplitude , actual_vartype = ( fakelc [ 'actual_varparams' ] , fakelc [ 'actual_varperiod' ] , fakelc [ 'actual_varamplitude' ] , fakelc [ 'actual_vartype' ] )
# get the moments too so we can track LC noise , etc .
actual_moments = fakelc [ 'moments' ]
# get the magcols for this LC
magcols = fakelc [ 'magcols' ]
# get the recovered info from each of the available methods
pfres = { 'objectid' : objectid , 'simbasedir' : simbasedir , 'magcols' : magcols , 'fakelc' : os . path . abspath ( lcfpath ) , 'fakepf' : os . path . abspath ( fakepfpkl ) , 'actual_vartype' : actual_vartype , 'actual_varperiod' : actual_varperiod , 'actual_varamplitude' : actual_varamplitude , 'actual_varparams' : actual_varparams , 'actual_moments' : actual_moments , 'recovery_periods' : [ ] , 'recovery_lspvals' : [ ] , 'recovery_pfmethods' : [ ] , 'recovery_magcols' : [ ] , 'recovery_status' : [ ] , 'recovery_pdiff' : [ ] , }
# populate the pfres dict with the periods , pfmethods , and magcols
for magcol in magcols :
    for pfm in lcproc . PFMETHODS :
        if pfm in fakepf [ magcol ] : # only get the unique recovered periods by using
        # period _ tolerance
            for rpi , rp in enumerate ( fakepf [ magcol ] [ pfm ] [ 'nbestperiods' ] ) :
                if ( ( not np . any ( np . isclose ( rp , np . array ( pfres [ 'recovery_periods' ] ) , rtol = period_tolerance ) ) ) and np . isfinite ( rp ) ) : # populate the recovery periods , pfmethods , and magcols
                    pfres [ 'recovery_periods' ] . append ( rp )
                    pfres [ 'recovery_pfmethods' ] . append ( pfm )
                    pfres [ 'recovery_magcols' ] . append ( magcol )
                    # normalize the periodogram peak value to between
                    # 0 and 1 so we can put in the results of multiple
                    # periodfinders on one scale
                    if pfm == 'pdm' :
                        this_lspval = ( np . max ( fakepf [ magcol ] [ pfm ] [ 'lspvals' ] ) - fakepf [ magcol ] [ pfm ] [ 'nbestlspvals' ] [ rpi ] )
                    else :
                        this_lspval = ( fakepf [ magcol ] [ pfm ] [ 'nbestlspvals' ] [ rpi ] / np . max ( fakepf [ magcol ] [ pfm ] [ 'lspvals' ] ) )
                    # add the normalized lspval to the outdict for
                    # this object as well . later , we ' ll use this to
                    # construct a periodogram for objects that were actually
                    # not variables
                    pfres [ 'recovery_lspvals' ] . append ( this_lspval )
# convert the recovery _ * lists to arrays
pfres [ 'recovery_periods' ] = np . array ( pfres [ 'recovery_periods' ] )
pfres [ 'recovery_lspvals' ] = np . array ( pfres [ 'recovery_lspvals' ] )
pfres [ 'recovery_pfmethods' ] = np . array ( pfres [ 'recovery_pfmethods' ] )
pfres [ 'recovery_magcols' ] = np . array ( pfres [ 'recovery_magcols' ] )
# now figure out recovery status
# if this is an actual periodic variable , characterize the recovery
if ( actual_vartype and actual_vartype in PERIODIC_VARTYPES and np . isfinite ( actual_varperiod ) ) :
    if pfres [ 'recovery_periods' ] . size > 0 :
        for ri in range ( pfres [ 'recovery_periods' ] . size ) :
            pfres [ 'recovery_pdiff' ] . append ( pfres [ 'recovery_periods' ] [ ri ] - np . asscalar ( actual_varperiod ) )
            # get the alias types
            pfres [ 'recovery_status' ] . append ( check_periodrec_alias ( actual_varperiod , pfres [ 'recovery_periods' ] [ ri ] , tolerance = period_tolerance ) )
        # turn the recovery _ pdiff / status lists into arrays
        pfres [ 'recovery_status' ] = np . array ( pfres [ 'recovery_status' ] )
        pfres [ 'recovery_pdiff' ] = np . array ( pfres [ 'recovery_pdiff' ] )
        # find the best recovered period and its status
        rec_absdiff = np . abs ( pfres [ 'recovery_pdiff' ] )
        best_recp_ind = rec_absdiff == rec_absdiff . min ( )
        pfres [ 'best_recovered_period' ] = ( pfres [ 'recovery_periods' ] [ best_recp_ind ] )
        pfres [ 'best_recovered_pfmethod' ] = ( pfres [ 'recovery_pfmethods' ] [ best_recp_ind ] )
        pfres [ 'best_recovered_magcol' ] = ( pfres [ 'recovery_magcols' ] [ best_recp_ind ] )
        pfres [ 'best_recovered_status' ] = ( pfres [ 'recovery_status' ] [ best_recp_ind ] )
        pfres [ 'best_recovered_pdiff' ] = ( pfres [ 'recovery_pdiff' ] [ best_recp_ind ] )
    else :
        LOGWARNING ( 'no finite periods recovered from period-finding for %s' % fakepfpkl )
        pfres [ 'recovery_status' ] = np . array ( [ 'no_finite_periods_recovered' ] )
        pfres [ 'recovery_pdiff' ] = np . array ( [ np . nan ] )
        pfres [ 'best_recovered_period' ] = np . array ( [ np . nan ] )
        pfres [ 'best_recovered_pfmethod' ] = np . array ( [ ] , dtype = np . unicode_ )
        pfres [ 'best_recovered_magcol' ] = np . array ( [ ] , dtype = np . unicode_ )
        pfres [ 'best_recovered_status' ] = np . array ( [ ] , dtype = np . unicode_ )
        pfres [ 'best_recovered_pdiff' ] = np . array ( [ np . nan ] )
# if this is not actually a variable , get the recovered period ,
# etc . anyway . this way , we can see what we need to look out for and avoid
# when getting these values for actual objects
else :
    pfres [ 'recovery_status' ] = np . array ( [ 'not_variable' ] * pfres [ 'recovery_periods' ] . size )
    pfres [ 'recovery_pdiff' ] = np . zeros ( pfres [ 'recovery_periods' ] . size )
    pfres [ 'best_recovered_period' ] = np . array ( [ np . nan ] )
    pfres [ 'best_recovered_pfmethod' ] = np . array ( [ ] , dtype = np . unicode_ )
    pfres [ 'best_recovered_magcol' ] = np . array ( [ ] , dtype = np . unicode_ )
    pfres [ 'best_recovered_status' ] = np . array ( [ 'not_variable' ] )
    pfres [ 'best_recovered_pdiff' ] = np . array ( [ np . nan ] )
return pfres


signature:
def periodrec_worker ( task ) :
    '''This is a parallel worker for running period - recovery .
 Parameters
 task : tuple
 This is used to pass args to the ` periodicvar _ recovery ` function : :
 task [ 0 ] = period - finding result pickle to work on
 task [ 1 ] = simbasedir
 task [ 2 ] = period _ tolerance
 Returns
 dict
 This is the dict produced by the ` periodicvar _ recovery ` function for the
 input period - finding result pickle .'''

implementation:
pfpkl , simbasedir , period_tolerance = task
try :
    return periodicvar_recovery ( pfpkl , simbasedir , period_tolerance = period_tolerance )
except Exception as e :
    LOGEXCEPTION ( 'periodic var recovery failed for %s' % repr ( task ) )
    return None


signature:
def parallel_periodicvar_recovery ( simbasedir , period_tolerance = 1.0e-3 , liststartind = None , listmaxobjects = None , nworkers = None ) :
    '''This is a parallel driver for ` periodicvar _ recovery ` .
 Parameters
 simbasedir : str
 The base directory where all of the fake LCs and period - finding results
 are .
 period _ tolerance : float
 The maximum difference that this function will consider between an
 actual period ( or its aliases ) and a recovered period to consider it as
 as a ' recovered ' period .
 liststartindex : int
 The starting index of processing . This refers to the filename list
 generated by running ` glob . glob ` on the period - finding result pickles in
 ` simbasedir / periodfinding ` .
 listmaxobjects : int
 The maximum number of objects to process in this run . Use this with
 ` liststartindex ` to effectively distribute working on a large list of
 input period - finding result pickles over several sessions or machines .
 nperiodworkers : int
 This is the number of parallel period - finding worker processes to use .
 Returns
 str
 Returns the filename of the pickle produced containing all of the period
 recovery results .'''

implementation:
# figure out the periodfinding pickles directory
pfpkldir = os . path . join ( simbasedir , 'periodfinding' )
if not os . path . exists ( pfpkldir ) :
    LOGERROR ( 'no "periodfinding" subdirectory in %s, can\'t continue' % simbasedir )
    return None
# find all the periodfinding pickles
pfpkl_list = glob . glob ( os . path . join ( pfpkldir , '*periodfinding*pkl*' ) )
if len ( pfpkl_list ) > 0 :
    if liststartind :
        pfpkl_list = pfpkl_list [ liststartind : ]
    if listmaxobjects :
        pfpkl_list = pfpkl_list [ : listmaxobjects ]
    tasks = [ ( x , simbasedir , period_tolerance ) for x in pfpkl_list ]
    pool = mp . Pool ( nworkers )
    results = pool . map ( periodrec_worker , tasks )
    pool . close ( )
    pool . join ( )
    resdict = { x [ 'objectid' ] : x for x in results if x is not None }
    actual_periodicvars = np . array ( [ x [ 'objectid' ] for x in results if ( x is not None and x [ 'actual_vartype' ] in PERIODIC_VARTYPES ) ] , dtype = np . unicode_ )
    recovered_periodicvars = np . array ( [ x [ 'objectid' ] for x in results if ( x is not None and 'actual' in x [ 'best_recovered_status' ] ) ] , dtype = np . unicode_ )
    alias_twice_periodicvars = np . array ( [ x [ 'objectid' ] for x in results if ( x is not None and 'twice' in x [ 'best_recovered_status' ] ) ] , dtype = np . unicode_ )
    alias_half_periodicvars = np . array ( [ x [ 'objectid' ] for x in results if ( x is not None and 'half' in x [ 'best_recovered_status' ] ) ] , dtype = np . unicode_ )
    all_objectids = [ x [ 'objectid' ] for x in results ]
    outdict = { 'simbasedir' : os . path . abspath ( simbasedir ) , 'objectids' : all_objectids , 'period_tolerance' : period_tolerance , 'actual_periodicvars' : actual_periodicvars , 'recovered_periodicvars' : recovered_periodicvars , 'alias_twice_periodicvars' : alias_twice_periodicvars , 'alias_half_periodicvars' : alias_half_periodicvars , 'details' : resdict }
    outfile = os . path . join ( simbasedir , 'periodicvar-recovery.pkl' )
    with open ( outfile , 'wb' ) as outfd :
        pickle . dump ( outdict , outfd , pickle . HIGHEST_PROTOCOL )
    return outdict
else :
    LOGERROR ( 'no periodfinding result pickles found in %s, can\'t continue' % pfpkldir )
    return None