signature:
def add_remote ( self , path , name , remote_url , use_sudo = False , user = None , fetch = True ) :
    """Add a remote Git repository into a directory .
 : param path : Path of the working copy directory . This directory must exist
 and be a Git working copy with a default remote to fetch from .
 : type path : str
 : param use _ sudo : If ` ` True ` ` execute ` ` git ` ` with
 : func : ` fabric . operations . sudo ` , else with
 : func : ` fabric . operations . run ` .
 : type use _ sudo : bool
 : param user : If ` ` use _ sudo is True ` ` , run : func : ` fabric . operations . sudo `
 with the given user . If ` ` use _ sudo is False ` ` this parameter
 has no effect .
 : type user : str
 : param name : name for the remote repository
 : type name : str
 : param remote _ url : URL of the remote repository
 : type remote _ url : str
 : param fetch : If ` ` True ` ` execute ` ` git remote add - f ` `
 : type fetch : bool"""

implementation:
if path is None :
    raise ValueError ( "Path to the working copy is needed to add a remote" )
if fetch :
    cmd = 'git remote add -f %s %s' % ( name , remote_url )
else :
    cmd = 'git remote add %s %s' % ( name , remote_url )
with cd ( path ) :
    if use_sudo and user is None :
        run_as_root ( cmd )
    elif use_sudo :
        sudo ( cmd , user = user )
    else :
        run ( cmd )


signature:
def fetch ( self , path , use_sudo = False , user = None , remote = None ) :
    """Fetch changes from the default remote repository .
 This will fetch new changesets , but will not update the contents of
 the working tree unless yo do a merge or rebase .
 : param path : Path of the working copy directory . This directory must exist
 and be a Git working copy with a default remote to fetch from .
 : type path : str
 : param use _ sudo : If ` ` True ` ` execute ` ` git ` ` with
 : func : ` fabric . operations . sudo ` , else with
 : func : ` fabric . operations . run ` .
 : type use _ sudo : bool
 : param user : If ` ` use _ sudo is True ` ` , run : func : ` fabric . operations . sudo `
 with the given user . If ` ` use _ sudo is False ` ` this parameter
 has no effect .
 : type user : str
 : type remote : Fetch this remote or default remote if is None
 : type remote : str"""

implementation:
if path is None :
    raise ValueError ( "Path to the working copy is needed to fetch from a remote repository." )
if remote is not None :
    cmd = 'git fetch %s' % remote
else :
    cmd = 'git fetch'
with cd ( path ) :
    if use_sudo and user is None :
        run_as_root ( cmd )
    elif use_sudo :
        sudo ( cmd , user = user )
    else :
        run ( cmd )


signature:
def pull ( self , path , use_sudo = False , user = None , force = False ) :
    """Fetch changes from the default remote repository and merge them .
 : param path : Path of the working copy directory . This directory must exist
 and be a Git working copy with a default remote to pull from .
 : type path : str
 : param use _ sudo : If ` ` True ` ` execute ` ` git ` ` with
 : func : ` fabric . operations . sudo ` , else with
 : func : ` fabric . operations . run ` .
 : type use _ sudo : bool
 : param user : If ` ` use _ sudo is True ` ` , run : func : ` fabric . operations . sudo `
 with the given user . If ` ` use _ sudo is False ` ` this parameter
 has no effect .
 : type user : str
 : param force : If ` ` True ` ` , append the ` ` - - force ` ` option to the command .
 : type force : bool"""

implementation:
if path is None :
    raise ValueError ( "Path to the working copy is needed to pull from a remote repository." )
options = [ ]
if force :
    options . append ( '--force' )
options = ' ' . join ( options )
cmd = 'git pull %s' % options
with cd ( path ) :
    if use_sudo and user is None :
        run_as_root ( cmd )
    elif use_sudo :
        sudo ( cmd , user = user )
    else :
        run ( cmd )


signature:
def get_logs_between_commits ( self , a , b ) :
    """Retrieves all commit messages for all commits between the given commit numbers
 on the current branch ."""

implementation:
print ( 'REAL' )
ret = self . local ( 'git --no-pager log --pretty=oneline %s...%s' % ( a , b ) , capture = True )
if self . verbose :
    print ( ret )
return str ( ret )


signature:
def get_current_commit ( self ) :
    """Retrieves the git commit number of the current head branch ."""

implementation:
with hide ( 'running' , 'stdout' , 'stderr' , 'warnings' ) :
    s = str ( self . local ( 'git rev-parse HEAD' , capture = True ) )
    self . vprint ( 'current commit:' , s )
    return s


signature:
def record_manifest ( self ) :
    """Called after a deployment to record any data necessary to detect changes
 for a future deployment ."""

implementation:
manifest = super ( GitTrackerSatchel , self ) . record_manifest ( )
manifest [ CURRENT_COMMIT ] = self . get_current_commit ( )
return manifest


signature:
def ssh_config ( self , name = '' ) :
    """Get the SSH parameters for connecting to a vagrant VM ."""

implementation:
r = self . local_renderer
with self . settings ( hide ( 'running' ) ) :
    output = r . local ( 'vagrant ssh-config %s' % name , capture = True )
config = { }
for line in output . splitlines ( ) [ 1 : ] :
    key , value = line . strip ( ) . split ( ' ' , 2 )
    config [ key ] = value
return config


signature:
def version ( self ) :
    """Get the Vagrant version ."""

implementation:
r = self . local_renderer
with self . settings ( hide ( 'running' , 'warnings' ) , warn_only = True ) :
    res = r . local ( 'vagrant --version' , capture = True )
if res . failed :
    return None
line = res . splitlines ( ) [ - 1 ]
version = re . match ( r'Vagrant (?:v(?:ersion )?)?(.*)' , line ) . group ( 1 )
return tuple ( _to_int ( part ) for part in version . split ( '.' ) )


signature:
def vagrant ( self , name = '' ) :
    """Run the following tasks on a vagrant box .
 First , you need to import this task in your ` ` fabfile . py ` ` : :
 from fabric . api import *
 from burlap . vagrant import vagrant
 @ task
 def some _ task ( ) :
 run ( ' echo hello ' )
 Then you can easily run tasks on your current Vagrant box : :
 $ fab vagrant some _ task"""

implementation:
r = self . local_renderer
config = self . ssh_config ( name )
extra_args = self . _settings_dict ( config )
r . genv . update ( extra_args )


signature:
def vagrant_settings ( self , name = '' , * args , ** kwargs ) :
    """Context manager that sets a vagrant VM
 as the remote host .
 Use this context manager inside a task to run commands
 on your current Vagrant box : :
 from burlap . vagrant import vagrant _ settings
 with vagrant _ settings ( ) :
 run ( ' hostname ' )"""

implementation:
config = self . ssh_config ( name )
extra_args = self . _settings_dict ( config )
kwargs . update ( extra_args )
return self . settings ( * args , ** kwargs )


signature:
def base_boxes ( self ) :
    """Get the list of vagrant base boxes"""

implementation: return sorted ( list ( set ( [ name for name , provider in self . _box_list ( ) ] ) ) )


signature:
def install_from_upstream ( self ) :
    """Installs Vagrant from the most recent package available from their homepage ."""

implementation:
from burlap . system import get_arch , distrib_family
r = self . local_renderer
content = urlopen ( r . env . download_url ) . read ( )
print ( len ( content ) )
matches = DOWNLOAD_LINK_PATTERN . findall ( content )
print ( matches )
arch = get_arch ( )
# e . g . ' x86_64'
family = distrib_family ( )
if family == DEBIAN :
    ext = '.deb'
    matches = [ match for match in matches if match . endswith ( ext ) and arch in match ]
    print ( 'matches:' , matches )
    assert matches , "No matches found."
    assert len ( matches ) == 1 , "Too many matches found: %s" % ( ', ' . join ( matches ) )
    r . env . final_download_url = matches [ 0 ]
    r . env . local_filename = '/tmp/vagrant%s' % ext
    r . run ( 'wget -O {local_filename} {final_download_url}' )
    r . sudo ( 'dpkg -i {local_filename}' )
else :
    raise NotImplementedError ( 'Unsupported family: %s' % family )


signature:
def distrib_id ( ) :
    """Get the OS distribution ID .
 Example : :
 from burlap . system import distrib _ id
 if distrib _ id ( ) ! = ' Debian ' :
 abort ( u " Distribution is not supported " )"""

implementation:
with settings ( hide ( 'running' , 'stdout' ) ) :
    kernel = ( run ( 'uname -s' ) or '' ) . strip ( ) . lower ( )
    if kernel == LINUX : # lsb _ release works on Ubuntu and Debian > = 6.0
    # but is not always included in other distros
        if is_file ( '/usr/bin/lsb_release' ) :
            id_ = run ( 'lsb_release --id --short' ) . strip ( ) . lower ( )
            if id in [ 'arch' , 'archlinux' ] : # old IDs used before lsb - release 1.4-14
                id_ = ARCH
            return id_
        else :
            if is_file ( '/etc/debian_version' ) :
                return DEBIAN
            elif is_file ( '/etc/fedora-release' ) :
                return FEDORA
            elif is_file ( '/etc/arch-release' ) :
                return ARCH
            elif is_file ( '/etc/redhat-release' ) :
                release = run ( 'cat /etc/redhat-release' )
                if release . startswith ( 'Red Hat Enterprise Linux' ) :
                    return REDHAT
                elif release . startswith ( 'CentOS' ) :
                    return CENTOS
                elif release . startswith ( 'Scientific Linux' ) :
                    return SLES
            elif is_file ( '/etc/gentoo-release' ) :
                return GENTOO
    elif kernel == SUNOS :
        return SUNOS


signature:
def distrib_release ( ) :
    """Get the release number of the distribution .
 Example : :
 from burlap . system import distrib _ id , distrib _ release
 if distrib _ id ( ) = = ' CentOS ' and distrib _ release ( ) = = ' 6.1 ' :
 print ( u " CentOS 6.2 has been released . Please upgrade . " )"""

implementation:
with settings ( hide ( 'running' , 'stdout' ) ) :
    kernel = ( run ( 'uname -s' ) or '' ) . strip ( ) . lower ( )
    if kernel == LINUX :
        return run ( 'lsb_release -r --short' )
    elif kernel == SUNOS :
        return run ( 'uname -v' )


signature:
def distrib_family ( ) :
    """Get the distribution family .
 Returns one of ` ` debian ` ` , ` ` redhat ` ` , ` ` arch ` ` , ` ` gentoo ` ` ,
 ` ` sun ` ` , ` ` other ` ` ."""

implementation:
distrib = ( distrib_id ( ) or '' ) . lower ( )
if distrib in [ 'debian' , 'ubuntu' , 'linuxmint' , 'elementary os' ] :
    return DEBIAN
elif distrib in [ 'redhat' , 'rhel' , 'centos' , 'sles' , 'fedora' ] :
    return REDHAT
elif distrib in [ 'sunos' ] :
    return SUN
elif distrib in [ 'gentoo' ] :
    return GENTOO
elif distrib in [ 'arch' , 'manjarolinux' ] :
    return ARCH
return 'other'


signature:
def supported_locales ( ) :
    """Gets the list of supported locales .
 Each locale is returned as a ` ` ( locale , charset ) ` ` tuple ."""

implementation:
family = distrib_family ( )
if family == 'debian' :
    return _parse_locales ( '/usr/share/i18n/SUPPORTED' )
elif family == 'arch' :
    return _parse_locales ( '/etc/locale.gen' )
elif family == 'redhat' :
    return _supported_locales_redhat ( )
else :
    raise UnsupportedFamily ( supported = [ 'debian' , 'arch' , 'redhat' ] )


signature:
def force_stop ( self ) :
    """Forcibly terminates all Celery processes ."""

implementation:
r = self . local_renderer
with self . settings ( warn_only = True ) :
    r . sudo ( 'pkill -9 -f celery' )
r . sudo ( 'rm -f /tmp/celery*.pid' )


signature:
def set_permissions ( self ) :
    """Sets ownership and permissions for Celery - related files ."""

implementation:
r = self . local_renderer
for path in r . env . paths_owned :
    r . env . path_owned = path
    r . sudo ( 'chown {celery_daemon_user}:{celery_daemon_user} {celery_path_owned}' )


signature:
def create_supervisor_services ( self , site ) :
    """This is called for each site to render a Celery config file ."""

implementation:
self . vprint ( 'create_supervisor_services:' , site )
self . set_site_specifics ( site = site )
r = self . local_renderer
if self . verbose :
    print ( 'r.env:' )
    pprint ( r . env , indent = 4 )
self . vprint ( 'r.env.has_worker:' , r . env . has_worker )
if not r . env . has_worker :
    self . vprint ( 'skipping: no celery worker' )
    return
if self . name . lower ( ) not in self . genv . services :
    self . vprint ( 'skipping: celery not enabled' )
    return
hostname = self . current_hostname
target_sites = self . genv . available_sites_by_host . get ( hostname , None )
if target_sites and site not in target_sites :
    self . vprint ( 'skipping: site not supported on this server' )
    return
self . render_paths ( )
conf_name = 'celery_%s.conf' % site
ret = r . render_to_string ( 'celery/celery_supervisor.template.conf' )
return conf_name , ret


signature:
def check_ok ( self ) :
    """Ensures all tests have passed for this branch .
 This should be called before deployment , to prevent accidental deployment of code
 that hasn ' t passed automated testing ."""

implementation:
import requests
if not self . env . check_ok :
    return
# Find current git branch .
branch_name = self . _local ( 'git rev-parse --abbrev-ref HEAD' , capture = True ) . strip ( )
check_ok_paths = self . env . check_ok_paths or { }
if branch_name in check_ok_paths :
    check = check_ok_paths [ branch_name ]
    if 'username' in check :
        auth = ( check [ 'username' ] , check [ 'password' ] )
    else :
        auth = None
    ret = requests . get ( check [ 'url' ] , auth = auth )
    passed = check [ 'text' ] in ret . content
    assert passed , 'Check failed: %s' % check [ 'url' ]


signature:
def is_present ( self , host = None ) :
    """Returns true if the given host exists on the network .
 Returns false otherwise ."""

implementation:
r = self . local_renderer
r . env . host = host or self . genv . host_string
ret = r . _local ( "getent hosts {host} | awk '{{ print $1 }}'" , capture = True ) or ''
if self . verbose :
    print ( 'ret:' , ret )
ret = ret . strip ( )
if self . verbose :
    print ( 'Host %s %s present.' % ( r . env . host , 'IS' if bool ( ret ) else 'IS NOT' ) )
ip = ret
ret = bool ( ret )
if not ret :
    return False
r . env . ip = ip
with settings ( warn_only = True ) :
    ret = r . _local ( 'ping -c 1 {ip}' , capture = True ) or ''
packet_loss = re . findall ( r'([0-9]+)% packet loss' , ret )
# print ( ' packet _ loss : ' , packet _ loss )
ip_accessible = packet_loss and int ( packet_loss [ 0 ] ) < 100
if self . verbose :
    print ( 'IP %s accessible: %s' % ( ip , ip_accessible ) )
return bool ( ip_accessible )


signature:
def purge_keys ( self ) :
    """Deletes all SSH keys on the localhost associated with the current remote host ."""

implementation:
r = self . local_renderer
r . env . default_ip = self . hostname_to_ip ( self . env . default_hostname )
r . env . home_dir = '/home/%s' % getpass . getuser ( )
r . local ( 'ssh-keygen -f "{home_dir}/.ssh/known_hosts" -R {host_string}' )
if self . env . default_hostname :
    r . local ( 'ssh-keygen -f "{home_dir}/.ssh/known_hosts" -R {default_hostname}' )
if r . env . default_ip :
    r . local ( 'ssh-keygen -f "{home_dir}/.ssh/known_hosts" -R {default_ip}' )


signature:
def find_working_password ( self , usernames = None , host_strings = None ) :
    """Returns the first working combination of username and password for the current host ."""

implementation:
r = self . local_renderer
if host_strings is None :
    host_strings = [ ]
if not host_strings :
    host_strings . append ( self . genv . host_string )
if usernames is None :
    usernames = [ ]
if not usernames :
    usernames . append ( self . genv . user )
for host_string in host_strings :
    for username in usernames :
        passwords = [ ]
        passwords . append ( self . genv . user_default_passwords [ username ] )
        passwords . append ( self . genv . user_passwords [ username ] )
        passwords . append ( self . env . default_password )
        for password in passwords :
            with settings ( warn_only = True ) :
                r . env . host_string = host_string
                r . env . password = password
                r . env . user = username
                ret = r . _local ( "sshpass -p '{password}' ssh -o StrictHostKeyChecking=no {user}@{host_string} echo hello" , capture = True )
                # print ( ' ret . return _ code : ' , ret . return _ code )
            # print ( ' ret000 : [ % s ] ' % ret )
            # code 1 = good password , but prompts needed
            # code 5 = bad password
            # code 6 = good password , but host public key is unknown
            if ret . return_code in ( 1 , 6 ) or 'hello' in ret : # Login succeeded , so we haven ' t yet changed the password , so use the default password .
                return host_string , username , password
raise Exception ( 'No working login found.' )


signature:
def needs_initrole ( self , stop_on_error = False ) :
    """Returns true if the host does not exist at the expected location and may need
 to have its initial configuration set .
 Returns false if the host exists at the expected location ."""

implementation:
ret = False
target_host_present = self . is_present ( )
if not target_host_present :
    default_host_present = self . is_present ( self . env . default_hostname )
    if default_host_present :
        if self . verbose :
            print ( 'Target host missing and default host present so host init required.' )
        ret = True
    else :
        if self . verbose :
            print ( 'Target host missing but default host also missing, ' 'so no host init required.' )
# if stop _ on _ error :
# raise Exception (
# ' Both target and default hosts missing ! '
# ' Is the machine turned on and plugged into the network ? ' )
else :
    if self . verbose :
        print ( 'Target host is present so no host init required.' )
return ret


signature:
def initrole ( self , check = True ) :
    """Called to set default password login for systems that do not yet have passwordless
 login setup ."""

implementation:
if self . env . original_user is None :
    self . env . original_user = self . genv . user
if self . env . original_key_filename is None :
    self . env . original_key_filename = self . genv . key_filename
host_string = None
user = None
password = None
if self . env . login_check :
    host_string , user , password = self . find_working_password ( usernames = [ self . genv . user , self . env . default_user ] , host_strings = [ self . genv . host_string , self . env . default_hostname ] , )
    if self . verbose :
        print ( 'host.initrole.host_string:' , host_string )
        print ( 'host.initrole.user:' , user )
        print ( 'host.initrole.password:' , password )
# needs = True
# if check :
# needs = self . needs _ initrole ( stop _ on _ error = True )
needs = False
if host_string is not None :
    self . genv . host_string = host_string
if user is not None :
    self . genv . user = user
if password is not None :
    self . genv . password = password
if not needs :
    return
assert self . env . default_hostname , 'No default hostname set.'
assert self . env . default_user , 'No default user set.'
self . genv . host_string = self . env . default_hostname
if self . env . default_hosts :
    self . genv . hosts = self . env . default_hosts
else :
    self . genv . hosts = [ self . env . default_hostname ]
self . genv . user = self . env . default_user
self . genv . password = self . env . default_password
self . genv . key_filename = self . env . default_key_filename
# If the host has been reformatted , the SSH keys will mismatch , throwing an error , so clear them .
self . purge_keys ( )
# Do a test login with the default password to determine which password we should use .
# r . env . password = self . env . default _ password
# with settings ( warn _ only = True ) :
# ret = r . _ local ( " sshpass - p ' { password } ' ssh - o StrictHostKeyChecking = no { user } @ { host _ string } echo hello " , capture = True )
# print ( ' ret . return _ code : ' , ret . return _ code )
# # print ( ' ret000 : [ % s ] ' % ret )
# # code 1 = good password , but prompts needed
# # code 5 = bad password
# # code 6 = good password , but host public key is unknown
# if ret . return _ code in ( 1 , 6 ) or ' hello ' in ret :
# # Login succeeded , so we haven ' t yet changed the password , so use the default password .
# self . genv . password = self . env . default _ password
# elif self . genv . user in self . genv . user _ passwords :
# # Otherwise , use the password or key set in the config .
# self . genv . password = self . genv . user _ passwords [ self . genv . user ]
# else :
# # Default password fails and there ' s no current password , so clear .
# self . genv . password = None
# self . genv . password = self . find _ working _ password ( )
# print ( ' host . initrole , using password : ' , self . genv . password )
# Execute post - init callbacks .
for task_name in self . env . post_initrole_tasks :
    if self . verbose :
        print ( 'Calling post initrole task %s' % task_name )
    satchel_name , method_name = task_name . split ( '.' )
    satchel = self . get_satchel ( name = satchel_name )
    getattr ( satchel , method_name ) ( )
print ( '^' * 80 )
print ( 'host.initrole.host_string:' , self . genv . host_string )
print ( 'host.initrole.user:' , self . genv . user )
print ( 'host.initrole.password:' , self . genv . password )


signature:
def iter_hostnames ( self ) :
    """Yields a list of tuples of the form ( ip , hostname ) ."""

implementation:
from burlap . common import get_hosts_retriever
if self . env . use_retriever :
    self . vprint ( 'using retriever' )
    self . vprint ( 'hosts:' , self . genv . hosts )
    retriever = get_hosts_retriever ( )
    hosts = list ( retriever ( extended = 1 ) )
    for _hostname , _data in hosts : # Skip hosts that aren ' t selected for this run .
        if self . genv . hosts and _data . ip not in self . genv . hosts and _data . public_dns_name not in self . genv . hosts and _hostname not in self . genv . hosts :
            continue
        assert _data . ip , 'Missing IP.'
        yield _data . ip , _hostname
        # _ data . public _ dns _ name
else :
    self . vprint ( 'using default' )
    for ip , hostname in self . env . hostnames . items ( ) :
        self . vprint ( 'ip lookup:' , ip , hostname )
        if ip == UNKNOWN :
            ip = self . hostname_to_ip ( hostname )
            if not ip and hostname in self . env . default_hostnames :
                ip = self . hostname_to_ip ( self . env . default_hostnames [ hostname ] )
        elif not ip [ 0 ] . isdigit ( ) :
            ip = self . hostname_to_ip ( ip )
        assert ip , 'Invalid IP.'
        yield ip , hostname


signature:
def get_public_ip ( self ) :
    """Gets the public IP for a host ."""

implementation:
r = self . local_renderer
ret = r . run ( r . env . get_public_ip_command ) or ''
ret = ret . strip ( )
print ( 'ip:' , ret )
return ret


signature:
def configure ( self , reboot = 1 ) :
    """Assigns a name to the server accessible from user space .
 Note , we add the name to / etc / hosts since not all programs use
 / etc / hostname to reliably identify the server hostname ."""

implementation:
r = self . local_renderer
for ip , hostname in self . iter_hostnames ( ) :
    self . vprint ( 'ip/hostname:' , ip , hostname )
    r . genv . host_string = ip
    r . env . hostname = hostname
    with settings ( warn_only = True ) :
        r . sudo ( 'echo "{hostname}" > /etc/hostname' )
        r . sudo ( 'echo "127.0.0.1 {hostname}" | cat - /etc/hosts > /tmp/out && mv /tmp/out /etc/hosts' )
        r . sudo ( r . env . set_hostname_command )
        if r . env . auto_reboot and int ( reboot ) :
            r . reboot ( )


signature:
def update_tickets_from_git ( self , from_commit = None , to_commit = None ) :
    """Run during a deployment .
 Looks at all commits between now and the last deployment .
 Finds all ticket numbers and updates their status in Jira ."""

implementation:
from jira import JIRA , JIRAError
# from burlap . deploy import get _ last _ current _ diffs
from burlap . git import gittracker , CURRENT_COMMIT
r = self . local_renderer
# get _ current _ commit = gittracker . get _ current _ commit
# GITTRACKER = gittracker . name . upper ( )
# Ensure this is only run once per role .
if self . genv . host_string != self . genv . hosts [ - 1 ] :
    self . vprint ( 'Not first server. Aborting.' )
    return
print ( 'self.env.update_from_git:' , self . env . update_from_git )
print ( 'self.genv.jirahelper_update_from_git:' , self . genv . jirahelper_update_from_git )
if not self . env . update_from_git :
    self . vprint ( 'Update from git disabled. Aborting.' )
    return
if not self . env . ticket_pattern :
    self . vprint ( 'No ticket pattern defined. Aborting.' )
    return
if not self . env . basic_auth_username or not self . env . basic_auth_password :
    self . vprint ( 'Username or password not given. Aborting.' )
    return
# During a deployment , we should be given these , but for testing ,
# lookup the diffs dynamically .
last = gittracker . last_manifest
current = gittracker . current_manifest
last_commit = from_commit or last . current_commit
# [ CURRENT _ COMMIT ]
print ( 'last_commit:' , last_commit )
current_commit = to_commit or current [ CURRENT_COMMIT ]
print ( 'current_commit:' , current_commit )
if not last_commit or not current_commit :
    print ( 'Missing commit ID. Aborting.' )
    return
if self . verbose :
    print ( '-' * 80 )
    print ( 'last.keys:' , last . keys ( ) )
    print ( '-' * 80 )
    print ( 'current.keys:' , current . keys ( ) )
# try :
# last _ commit = last [ ' GITTRACKER ' ] [ ' current _ commit ' ]
# except KeyError :
# return
# current _ commit = current [ ' GITTRACKER ' ] [ ' current _ commit ' ]
# Find all tickets deployed between last deployment and now .
tickets = self . get_tickets_between_commits ( current_commit , last_commit )
if self . verbose :
    print ( 'tickets:' , tickets )
# Update all tickets in Jira .
jira = JIRA ( { 'server' : self . env . server } , basic_auth = ( self . env . basic_auth_username , self . env . basic_auth_password ) )
for ticket in tickets : # Mention this Jira updated .
    r . env . role = r . genv . ROLE . lower ( )
    comment = r . format ( self . env . ticket_update_message_template )
    print ( 'Commenting on ticket %s: %s' % ( ticket , comment ) )
    if not self . dryrun :
        jira . add_comment ( ticket , comment )
    # Update ticket status .
    recheck = False
    while 1 :
        print ( 'Looking up jira ticket %s...' % ticket )
        issue = jira . issue ( ticket )
        self . vprint ( 'Ticket %s retrieved.' % ticket )
        transition_to_id = dict ( ( t [ 'name' ] , t [ 'id' ] ) for t in jira . transitions ( issue ) )
        self . vprint ( '%i allowable transitions found:' % len ( transition_to_id ) )
        pprint ( transition_to_id )
        self . vprint ( 'issue.fields.status.id:' , issue . fields . status . id )
        self . vprint ( 'issue.fields.status.name:' , issue . fields . status . name )
        jira_status_id = issue . fields . status . name . title ( )
        self . vprint ( 'jira_status_id:' , jira_status_id )
        next_transition_name = self . env . deploy_workflow . get ( jira_status_id )
        self . vprint ( 'next_transition_name:' , next_transition_name )
        next_transition_id = transition_to_id . get ( next_transition_name )
        self . vprint ( 'next_transition_id:' , next_transition_id )
        if next_transition_name :
            if issue . fields . assignee :
                if issue . fields . assignee . raw :
                    assignee_name = issue . fields . assignee . name
                else : # Get assignee name directly
                # https : / / community . atlassian . com / t5 / Jira - questions / Jira - in - Python - issue - fields - reporter - name -
                # errors - with - TypeError / qaq - p / 937924
                    assignee_name = issue . fields . assignee . _session [ 'name' ]
            else :
                assignee_name = None
            # Get new assignee by status
            new_assignee = self . env . assignee_by_status . get ( # issue . fields . status . name . title ( ) ,
            next_transition_name , assignee_name , )
            # If assigning to reporter , get reporter name .
            if new_assignee == 'reporter' :
                if issue . fields . reporter . raw :
                    new_assignee = issue . fields . reporter . name
                else : # Get reporter name directly
                # https : / / community . atlassian . com / t5 / Jira - questions / Jira - in - Python - issue - fields - reporter - name -
                # errors - with - TypeError / qaq - p / 937924
                    new_assignee = issue . fields . reporter . _session [ 'name' ]
            print ( 'Updating ticket %s to status %s (%s) and assigning it to %s.' % ( ticket , next_transition_name , next_transition_id , new_assignee ) )
            if not self . dryrun :
                if next_transition_id :
                    try :
                        jira . transition_issue ( issue , next_transition_id )
                        recheck = True
                    except AttributeError as e :
                        print ( 'Unable to transition ticket %s to %s: %s' % ( ticket , next_transition_name , e ) , file = sys . stderr )
                        traceback . print_exc ( )
                # Note assignment should happen after transition , since the assignment may
                # effect remove transitions that we need .
                try :
                    if new_assignee :
                        print ( 'Assigning ticket %s to %s.' % ( ticket , new_assignee ) )
                        jira . assign_issue ( issue , new_assignee )
                    else :
                        print ( 'No new assignee found.' )
                except JIRAError as e :
                    print ( 'Unable to reassign ticket %s to %s: %s' % ( ticket , new_assignee , e ) , file = sys . stderr )
        else :
            recheck = False
            print ( 'No transitions found for ticket %s currently in status "%s".' % ( ticket , issue . fields . status . name ) )
        if not recheck :
            break


signature:
def partitions ( device = "" ) :
    """Get a partition list for all disk or for selected device only
 Example : :
 from burlap . disk import partitions
 spart = { ' Linux ' : 0x83 , ' Swap ' : 0x82}
 parts = partitions ( )
 # parts = { ' / dev / sda1 ' : 131 , ' / dev / sda2 ' : 130 , ' / dev / sda3 ' : 131}
 r = parts [ ' / dev / sda1 ' ] = = spart [ ' Linux ' ]
 r = r and parts [ ' / dev / sda2 ' ] = = spart [ ' Swap ' ]
 if r :
 print ( " You can format these partitions " )"""

implementation:
partitions_list = { }
with settings ( hide ( 'running' , 'stdout' ) ) :
    res = run_as_root ( 'sfdisk -d %(device)s' % locals ( ) )
    spart = re . compile ( r'(?P<pname>^/.*) : .* Id=(?P<ptypeid>[0-9a-z]+)' )
    for line in res . splitlines ( ) :
        m = spart . search ( line )
        if m :
            partitions_list [ m . group ( 'pname' ) ] = int ( m . group ( 'ptypeid' ) , 16 )
return partitions_list


signature:
def getdevice_by_uuid ( uuid ) :
    """Get a HDD device by uuid
 Example : :
 from burlap . disk import getdevice _ by _ uuid
 device = getdevice _ by _ uuid ( " 356fafdc - 21d5-408e - a3e9-2b3f32cb2a8c " )
 if device :
 mount ( device , ' / mountpoint ' )"""

implementation:
with settings ( hide ( 'running' , 'warnings' , 'stdout' ) , warn_only = True ) :
    res = run_as_root ( 'blkid -U %s' % uuid )
    if not res . succeeded :
        return None
    return res


signature:
def ismounted ( device ) :
    """Check if partition is mounted
 Example : :
 from burlap . disk import ismounted
 if ismounted ( ' / dev / sda1 ' ) :
 print ( " disk sda1 is mounted " )"""

implementation:
# Check filesystem
with settings ( hide ( 'running' , 'stdout' ) ) :
    res = run_as_root ( 'mount' )
for line in res . splitlines ( ) :
    fields = line . split ( )
    if fields [ 0 ] == device :
        return True
# Check swap
with settings ( hide ( 'running' , 'stdout' ) ) :
    res = run_as_root ( 'swapon -s' )
for line in res . splitlines ( ) :
    fields = line . split ( )
    if fields [ 0 ] == device :
        return True
return False


signature:
def query ( query , use_sudo = True , ** kwargs ) :
    """Run a MySQL query ."""

implementation:
func = use_sudo and run_as_root or run
user = kwargs . get ( 'mysql_user' ) or env . get ( 'mysql_user' )
password = kwargs . get ( 'mysql_password' ) or env . get ( 'mysql_password' )
options = [ '--batch' , '--raw' , '--skip-column-names' , ]
if user :
    options . append ( '--user=%s' % quote ( user ) )
if password :
    options . append ( '--password=%s' % quote ( password ) )
options = ' ' . join ( options )
return func ( 'mysql %(options)s --execute=%(query)s' % { 'options' : options , 'query' : quote ( query ) , } )


signature:
def user_exists ( name , host = 'localhost' , ** kwargs ) :
    """Check if a MySQL user exists ."""

implementation:
with settings ( hide ( 'running' , 'stdout' , 'stderr' , 'warnings' ) , warn_only = True ) :
    res = query ( """
            use mysql;
            SELECT COUNT(*) FROM user
                WHERE User = '%(name)s' AND Host = '%(host)s';
            """ % { 'name' : name , 'host' : host , } , ** kwargs )
return res . succeeded and ( int ( res ) == 1 )


signature:
def create_user ( name , password , host = 'localhost' , ** kwargs ) :
    """Create a MySQL user .
 Example : :
 import burlap
 # Create DB user if it does not exist
 if not burlap . mysql . user _ exists ( ' dbuser ' ) :
 burlap . mysql . create _ user ( ' dbuser ' , password = ' somerandomstring ' )"""

implementation:
with settings ( hide ( 'running' ) ) :
    query ( "CREATE USER '%(name)s'@'%(host)s' IDENTIFIED BY '%(password)s';" % { 'name' : name , 'password' : password , 'host' : host } , ** kwargs )
puts ( "Created MySQL user '%s'." % name )


signature:
def database_exists ( name , ** kwargs ) :
    """Check if a MySQL database exists ."""

implementation:
with settings ( hide ( 'running' , 'stdout' , 'stderr' , 'warnings' ) , warn_only = True ) :
    res = query ( "SHOW DATABASES LIKE '%(name)s';" % { 'name' : name } , ** kwargs )
return res . succeeded and ( res == name )


signature:
def create_database ( name , owner = None , owner_host = 'localhost' , charset = 'utf8' , collate = 'utf8_general_ci' , ** kwargs ) :
    """Create a MySQL database .
 Example : :
 import burlap
 # Create DB if it does not exist
 if not burlap . mysql . database _ exists ( ' myapp ' ) :
 burlap . mysql . create _ database ( ' myapp ' , owner = ' dbuser ' )"""

implementation:
with settings ( hide ( 'running' ) ) :
    query ( "CREATE DATABASE %(name)s CHARACTER SET %(charset)s COLLATE %(collate)s;" % { 'name' : name , 'charset' : charset , 'collate' : collate } , ** kwargs )
    if owner :
        query ( "GRANT ALL PRIVILEGES ON %(name)s.* TO '%(owner)s'@'%(owner_host)s' WITH GRANT OPTION;" % { 'name' : name , 'owner' : owner , 'owner_host' : owner_host } , ** kwargs )
puts ( "Created MySQL database '%s'." % name )


signature:
def conf_path ( self ) :
    """Retrieves the path to the MySQL configuration file ."""

implementation:
from burlap . system import distrib_id , distrib_release
hostname = self . current_hostname
if hostname not in self . _conf_cache :
    self . env . conf_specifics [ hostname ] = self . env . conf_default
    d_id = distrib_id ( )
    d_release = distrib_release ( )
    for key in ( ( d_id , d_release ) , ( d_id , ) ) :
        if key in self . env . conf_specifics :
            self . _conf_cache [ hostname ] = self . env . conf_specifics [ key ]
return self . _conf_cache [ hostname ]


signature:
def prep_root_password ( self , password = None , ** kwargs ) :
    """Enters the root password prompt entries into the debconf cache
 so we can set them without user interaction .
 We keep this process separate from set _ root _ password ( ) because we also need to do
 this before installing the base MySQL package , because that will also prompt the user
 for a root login ."""

implementation:
r = self . database_renderer ( ** kwargs )
r . env . root_password = password or r . genv . get ( 'db_root_password' )
r . sudo ( "DEBIAN_FRONTEND=noninteractive dpkg --configure -a" )
r . sudo ( "debconf-set-selections <<< 'mysql-server mysql-server/root_password password {root_password}'" )
r . sudo ( "debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password {root_password}'" )


signature:
def drop_views ( self , name = None , site = None ) :
    """Drops all views ."""

implementation:
r = self . database_renderer
result = r . sudo ( "mysql --batch -v -h {db_host} " # " - u { db _ root _ username } - p ' { db _ root _ password } ' "
"-u {db_user} -p'{db_password}' " "--execute=\"SELECT GROUP_CONCAT(CONCAT(TABLE_SCHEMA,'.',table_name) SEPARATOR ', ') AS views " "FROM INFORMATION_SCHEMA.views WHERE TABLE_SCHEMA = '{db_name}' ORDER BY table_name DESC;\"" )
result = re . findall ( r'^views[\s\t\r\n]+(.*)' , result , flags = re . IGNORECASE | re . DOTALL | re . MULTILINE )
if not result :
    return
r . env . db_view_list = result [ 0 ]
# cmd = ( " mysql - v - h { db _ host } - u { db _ root _ username } - p ' { db _ root _ password } ' " \
r . sudo ( "mysql -v -h {db_host} -u {db_user} -p'{db_password}' " "--execute=\"DROP VIEW {db_view_list} CASCADE;\"" )


signature:
def exists ( self , ** kwargs ) :
    """Returns true if a database with the given name exists . False otherwise ."""

implementation:
name = kwargs . pop ( 'name' , 'default' )
site = kwargs . pop ( 'site' , None )
r = self . database_renderer ( name = name , site = site )
ret = r . run ( 'mysql -h {db_host} -u {db_root_username} ' '-p"{db_root_password}" -N -B -e "SELECT IF(\'{db_name}\'' ' IN(SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA), ' '\'exists\', \'notexists\') AS found;"' )
if ret is not None :
    ret = 'notexists' not in ( ret or 'notexists' )
if ret is not None :
    msg = '%s database on site %s %s exist.' % ( name . title ( ) , env . SITE , 'DOES' if ret else 'DOES NOT' )
    if ret :
        print ( green ( msg ) )
    else :
        print ( red ( msg ) )
    return ret


signature:
def load ( self , dump_fn = '' , prep_only = 0 , force_upload = 0 , from_local = 0 , name = None , site = None , dest_dir = None , force_host = None ) :
    """Restores a database snapshot onto the target database server .
 If prep _ only = 1 , commands for preparing the load will be generated ,
 but not the command to finally load the snapshot ."""

implementation:
r = self . database_renderer ( name = name , site = site )
r . pc ( 'Loading database snapshot.' )
# Render the snapshot filename .
r . env . dump_fn = self . get_default_db_fn ( fn_template = dump_fn , dest_dir = dest_dir ) . strip ( )
from_local = int ( from_local )
prep_only = int ( prep_only )
missing_local_dump_error = r . format ( "Database dump file {dump_fn} does not exist." )
# Copy snapshot file to target .
if self . is_local :
    r . env . remote_dump_fn = dump_fn
else :
    r . env . remote_dump_fn = '/tmp/' + os . path . split ( r . env . dump_fn ) [ - 1 ]
if not prep_only and not self . is_local : # if int ( force _ upload ) or ( not self . is _ local and not r . file _ exists ( r . env . remote _ dump _ fn ) ) :
    if not self . dryrun :
        assert os . path . isfile ( r . env . dump_fn ) , missing_local_dump_error
    # if self . verbose :
    # print ( ' Uploading MySQL database snapshot . . . ' )
    # r . put (
    # local _ path = r . env . dump _ fn ,
    # remote _ path = r . env . remote _ dump _ fn )
    self . upload_snapshot ( name = name , site = site )
if self . is_local and not prep_only and not self . dryrun :
    assert os . path . isfile ( r . env . dump_fn ) , missing_local_dump_error
if force_host :
    r . env . db_host = force_host
# Drop the database if it ' s there .
r . run ( "mysql -v -h {db_host} -u {db_root_username} -p'{db_root_password}' --execute='DROP DATABASE IF EXISTS {db_name}'" )
# Now , create the database .
r . run ( "mysqladmin -h {db_host} -u {db_root_username} -p'{db_root_password}' create {db_name}" )
# Create user
with settings ( warn_only = True ) :
    r . run ( "mysql -v -h {db_host} -u {db_root_username} -p'{db_root_password}' --execute=\"DROP USER '{db_user}'@'%%';" "FLUSH PRIVILEGES;\"" )
with settings ( warn_only = True ) :
    r . run ( "mysql -v -h {db_host} -u {db_root_username} -p'{db_root_password}' --execute=\"CREATE USER '{db_user}'@'%%' IDENTIFIED BY '{db_password}'; " "GRANT ALL PRIVILEGES ON *.* TO '{db_user}'@'%%' WITH GRANT OPTION; " "FLUSH PRIVILEGES;\"" )
self . set_collation ( name = name , site = site )
self . set_max_packet_size ( name = name , site = site )
# Run any server - specific commands ( e . g . to setup permissions ) before
# we load the data .
for command in r . env . preload_commands :
    r . run ( command )
# Restore the database content from the dump file .
if not prep_only :
    r . run ( r . env . load_command )
self . set_collation ( name = name , site = site )


signature:
def shell ( self , name = 'default' , site = None , use_root = 0 , ** kwargs ) :
    """Opens a SQL shell to the given database , assuming the configured database
 and user supports this feature ."""

implementation:
r = self . database_renderer ( name = name , site = site )
if int ( use_root ) :
    kwargs = dict ( db_user = r . env . db_root_username , db_password = r . env . db_root_password , db_host = r . env . db_host , db_name = r . env . db_name , )
    r . env . update ( kwargs )
if not name :
    r . env . db_name = ''
r . run ( '/bin/bash -i -c "mysql -u {db_user} -p\'{db_password}\' -h {db_host} {db_name}"' )


signature:
def tic_single_object_crossmatch ( ra , dec , radius ) :
    '''This does a cross - match against the TIC catalog on MAST .
 Speed tests : about 10 crossmatches per second . ( - > 3 hours for 10 ^ 5 objects
 to crossmatch ) .
 Parameters
 ra , dec : np . array
 The coordinates to cross match against , all in decimal degrees .
 radius : float
 The cross - match radius to use , in decimal degrees .
 Returns
 dict
 Returns the match results JSON from MAST loaded into a dict .'''

implementation:
for val in ra , dec , radius :
    if not isinstance ( val , float ) :
        raise AssertionError ( 'plz input ra,dec,radius in decimal degrees' )
# This is a json object
crossmatchInput = { "fields" : [ { "name" : "ra" , "type" : "float" } , { "name" : "dec" , "type" : "float" } ] , "data" : [ { "ra" : ra , "dec" : dec } ] }
request = { "service" : "Mast.Tic.Crossmatch" , "data" : crossmatchInput , "params" : { "raColumn" : "ra" , "decColumn" : "dec" , "radius" : radius } , "format" : "json" , 'removecache' : True }
headers , out_string = _mast_query ( request )
out_data = json . loads ( out_string )
return out_data


signature:
def normalized_flux_to_mag ( lcdict , columns = ( 'sap.sap_flux' , 'sap.sap_flux_err' , 'sap.sap_bkg' , 'sap.sap_bkg_err' , 'pdc.pdcsap_flux' , 'pdc.pdcsap_flux_err' ) ) :
    '''This converts the normalized fluxes in the TESS lcdicts to TESS mags .
 Uses the object ' s TESS mag stored in lcdict [ ' objectinfo ' ] [ ' tessmag ' ] : :
 mag - object _ tess _ mag = - 2.5 log ( flux / median _ flux )
 Parameters
 lcdict : lcdict
 An ` lcdict ` produced by ` read _ tess _ fitslc ` or
 ` consolidate _ tess _ fitslc ` . This must have normalized fluxes in its
 measurement columns ( use the ` normalize ` kwarg for these functions ) .
 columns : sequence of str
 The column keys of the normalized flux and background measurements in
 the ` lcdict ` to operate on and convert to magnitudes in TESS band ( T ) .
 Returns
 lcdict
 The returned ` lcdict ` will contain extra columns corresponding to
 magnitudes for each input normalized flux / background column .'''

implementation:
tess_mag = lcdict [ 'objectinfo' ] [ 'tessmag' ]
for key in columns :
    k1 , k2 = key . split ( '.' )
    if 'err' not in k2 :
        lcdict [ k1 ] [ k2 . replace ( 'flux' , 'mag' ) ] = ( tess_mag - 2.5 * np . log10 ( lcdict [ k1 ] [ k2 ] ) )
    else :
        lcdict [ k1 ] [ k2 . replace ( 'flux' , 'mag' ) ] = ( - 2.5 * np . log10 ( 1.0 - lcdict [ k1 ] [ k2 ] ) )
return lcdict


signature:
def get_time_flux_errs_from_Ames_lightcurve ( infile , lctype , cadence_min = 2 ) :
    '''Reads TESS Ames - format FITS light curve files .
 MIT TOI alerts include Ames lightcurve files . This function gets the finite ,
 nonzero times , fluxes , and errors with QUALITY = = 0.
 NOTE : the PDCSAP lightcurve typically still need " pre - whitening " after this
 step .
 . . deprecated : : 0.3.20
 This function will be removed in astrobase v0.4.2 . Use the
 ` read _ tess _ fitslc ` and ` consolidate _ tess _ fitslc ` functions instead .
 Parameters
 infile : str
 The path to ` * . fits . gz ` TOI alert file , from Ames pipeline .
 lctype : { ' PDCSAP ' , ' SAP ' }
 The type of light curve to extract from the FITS LC file .
 cadence _ min : int
 The expected frame cadence in units of minutes . Raises ValueError if you
 use the wrong cadence .
 Returns
 tuple
 The tuple returned is of the form :
 ( times , normalized ( to median ) fluxes , flux errors )'''

implementation:
warnings . warn ( "Use the astrotess.read_tess_fitslc and " "astrotess.consolidate_tess_fitslc functions instead of this function. " "This function will be removed in astrobase v0.4.2." , FutureWarning )
if lctype not in ( 'PDCSAP' , 'SAP' ) :
    raise ValueError ( 'unknown light curve type requested: %s' % lctype )
hdulist = pyfits . open ( infile )
main_hdr = hdulist [ 0 ] . header
lc_hdr = hdulist [ 1 ] . header
lc = hdulist [ 1 ] . data
if ( ( 'Ames' not in main_hdr [ 'ORIGIN' ] ) or ( 'LIGHTCURVE' not in lc_hdr [ 'EXTNAME' ] ) ) :
    raise ValueError ( 'could not understand input LC format. ' 'Is it a TESS TOI LC file?' )
time = lc [ 'TIME' ]
flux = lc [ '{:s}_FLUX' . format ( lctype ) ]
err_flux = lc [ '{:s}_FLUX_ERR' . format ( lctype ) ]
# REMOVE POINTS FLAGGED WITH :
# attitude tweaks , safe mode , coarse / earth pointing , argabrithening events ,
# reaction wheel desaturation events , cosmic rays in optimal aperture
# pixels , manual excludes , discontinuities , stray light from Earth or Moon
# in camera FoV .
# ( Note : it ' s not clear to me what a lot of these mean . Also most of these
# columns are probably not correctly propagated right now . )
sel = ( lc [ 'QUALITY' ] == 0 )
sel &= np . isfinite ( time )
sel &= np . isfinite ( flux )
sel &= np . isfinite ( err_flux )
sel &= ~ np . isnan ( time )
sel &= ~ np . isnan ( flux )
sel &= ~ np . isnan ( err_flux )
sel &= ( time != 0 )
sel &= ( flux != 0 )
sel &= ( err_flux != 0 )
time = time [ sel ]
flux = flux [ sel ]
err_flux = err_flux [ sel ]
# ensure desired cadence
lc_cadence_diff = np . abs ( np . nanmedian ( np . diff ( time ) ) * 24 * 60 - cadence_min )
if lc_cadence_diff > 1.0e-2 :
    raise ValueError ( 'the light curve is not at the required cadence specified: %.2f' % cadence_min )
fluxmedian = np . nanmedian ( flux )
flux /= fluxmedian
err_flux /= fluxmedian
return time , flux , err_flux


signature:
def read_tess_fitslc ( lcfits , headerkeys = LCHEADERKEYS , datakeys = LCDATAKEYS , sapkeys = LCSAPKEYS , pdckeys = LCPDCKEYS , topkeys = LCTOPKEYS , apkeys = LCAPERTUREKEYS , normalize = False , appendto = None , filterqualityflags = False , nanfilter = None , timestoignore = None ) :
    '''This extracts the light curve from a single TESS . lc . fits file .
 This works on the light curves available at MAST .
 TODO : look at :
 https : / / archive . stsci . edu / missions / tess / doc / EXP - TESS - ARC - ICD - TM - 0014 . pdf
 for details on the column descriptions and to fill in any other info we
 need .
 Parameters
 lcfits : str
 The filename of a MAST Kepler / K2 light curve FITS file .
 headerkeys : list
 A list of FITS header keys that will be extracted from the FITS light
 curve file . These describe the observations . The default value for this
 is given in ` LCHEADERKEYS ` above .
 datakeys : list
 A list of FITS column names that correspond to the auxiliary
 measurements in the light curve . The default is ` LCDATAKEYS ` above .
 sapkeys : list
 A list of FITS column names that correspond to the SAP flux
 measurements in the light curve . The default is ` LCSAPKEYS ` above .
 pdckeys : list
 A list of FITS column names that correspond to the PDC flux
 measurements in the light curve . The default is ` LCPDCKEYS ` above .
 topkeys : list
 A list of FITS header keys that describe the object in the light
 curve . The default is ` LCTOPKEYS ` above .
 apkeys : list
 A list of FITS header keys that describe the flux measurement apertures
 used by the TESS pipeline . The default is ` LCAPERTUREKEYS ` above .
 normalize : bool
 If True , then the light curve ' s SAP _ FLUX and PDCSAP _ FLUX measurements
 will be normalized to 1.0 by dividing out the median flux for the
 component light curve .
 appendto : lcdict or None
 If appendto is an ` lcdict ` , will append measurements of this ` lcdict ` to
 that ` lcdict ` . This is used for consolidating light curves for the same
 object across different files ( sectors / cameras / CCDs ? ) . The appending
 does not care about the time order . To consolidate light curves in time
 order , use ` consolidate _ tess _ fitslc ` below .
 filterqualityflags : bool
 If True , will remove any measurements that have non - zero quality flags
 present . This usually indicates an issue with the instrument or
 spacecraft .
 nanfilter : { ' sap ' , ' pdc ' , ' sap , pdc ' } or None
 Indicates the flux measurement type ( s ) to apply the filtering to .
 timestoignore : list of tuples or None
 This is of the form : :
 [ ( time1 _ start , time1 _ end ) , ( time2 _ start , time2 _ end ) , . . . ]
 and indicates the start and end times to mask out of the final
 lcdict . Use this to remove anything that wasn ' t caught by the quality
 flags .
 Returns
 lcdict
 Returns an ` lcdict ` ( this is useable by most astrobase functions for LC
 processing ) .'''

implementation:
# read the fits file
hdulist = pyfits . open ( lcfits )
lchdr , lcdata = hdulist [ 1 ] . header , hdulist [ 1 ] . data
lctophdr , lcaperturehdr , lcaperturedata = ( hdulist [ 0 ] . header , hdulist [ 2 ] . header , hdulist [ 2 ] . data )
hdulist . close ( )
hdrinfo = { }
# now get the values we want from the header
for key in headerkeys :
    if key in lchdr and lchdr [ key ] is not None :
        hdrinfo [ key . lower ( ) ] = lchdr [ key ]
    else :
        hdrinfo [ key . lower ( ) ] = None
# get the number of detections
ndet = lchdr [ 'NAXIS2' ]
# get the info from the topheader
for key in topkeys :
    if key in lctophdr and lctophdr [ key ] is not None :
        hdrinfo [ key . lower ( ) ] = lctophdr [ key ]
    else :
        hdrinfo [ key . lower ( ) ] = None
# get the info from the lcaperturehdr
for key in lcaperturehdr :
    if key in lcaperturehdr and lcaperturehdr [ key ] is not None :
        hdrinfo [ key . lower ( ) ] = lcaperturehdr [ key ]
    else :
        hdrinfo [ key . lower ( ) ] = None
# if we ' re appending to another lcdict
if appendto and isinstance ( appendto , dict ) :
    lcdict = appendto
    # update lcinfo
    lcdict [ 'lcinfo' ] [ 'timesys' ] . append ( hdrinfo [ 'timesys' ] )
    lcdict [ 'lcinfo' ] [ 'bjdoffset' ] . append ( hdrinfo [ 'bjdrefi' ] + hdrinfo [ 'bjdreff' ] )
    lcdict [ 'lcinfo' ] [ 'lcaperture' ] . append ( lcaperturedata )
    lcdict [ 'lcinfo' ] [ 'aperpix_used' ] . append ( hdrinfo [ 'npixsap' ] )
    lcdict [ 'lcinfo' ] [ 'aperpix_unused' ] . append ( hdrinfo [ 'npixmiss' ] )
    lcdict [ 'lcinfo' ] [ 'pixarcsec' ] . append ( ( np . abs ( hdrinfo [ 'cdelt1' ] ) + np . abs ( hdrinfo [ 'cdelt2' ] ) ) * 3600.0 / 2.0 )
    lcdict [ 'lcinfo' ] [ 'ndet' ] . append ( ndet )
    lcdict [ 'lcinfo' ] [ 'exptime' ] . append ( hdrinfo [ 'exposure' ] )
    lcdict [ 'lcinfo' ] [ 'sector' ] . append ( hdrinfo [ 'sector' ] )
    lcdict [ 'lcinfo' ] [ 'camera' ] . append ( hdrinfo [ 'camera' ] )
    lcdict [ 'lcinfo' ] [ 'ccd' ] . append ( hdrinfo [ 'ccd' ] )
    lcdict [ 'lcinfo' ] [ 'date_obs_start' ] . append ( hdrinfo [ 'date-obs' ] )
    lcdict [ 'lcinfo' ] [ 'date_obs_end' ] . append ( hdrinfo [ 'date-end' ] )
    lcdict [ 'lcinfo' ] [ 'pixel_table_id' ] . append ( hdrinfo [ 'pxtable' ] )
    lcdict [ 'lcinfo' ] [ 'origin' ] . append ( hdrinfo [ 'origin' ] )
    lcdict [ 'lcinfo' ] [ 'datarelease' ] . append ( hdrinfo [ 'data_rel' ] )
    lcdict [ 'lcinfo' ] [ 'procversion' ] . append ( hdrinfo [ 'procver' ] )
    lcdict [ 'lcinfo' ] [ 'tic_version' ] . append ( hdrinfo [ 'ticver' ] )
    lcdict [ 'lcinfo' ] [ 'cr_mitigation' ] . append ( hdrinfo [ 'crmiten' ] )
    lcdict [ 'lcinfo' ] [ 'cr_blocksize' ] . append ( hdrinfo [ 'crblksz' ] )
    lcdict [ 'lcinfo' ] [ 'cr_spocclean' ] . append ( hdrinfo [ 'crspoc' ] )
    # update the varinfo for this light curve
    lcdict [ 'varinfo' ] [ 'cdpp0_5' ] . append ( hdrinfo [ 'cdpp0_5' ] )
    lcdict [ 'varinfo' ] [ 'cdpp1_0' ] . append ( hdrinfo [ 'cdpp1_0' ] )
    lcdict [ 'varinfo' ] [ 'cdpp2_0' ] . append ( hdrinfo [ 'cdpp2_0' ] )
    lcdict [ 'varinfo' ] [ 'pdcvar' ] . append ( hdrinfo [ 'pdcvar' ] )
    lcdict [ 'varinfo' ] [ 'pdcmethod' ] . append ( hdrinfo [ 'pdcmethd' ] )
    lcdict [ 'varinfo' ] [ 'target_flux_total_flux_ratio_in_aper' ] . append ( hdrinfo [ 'crowdsap' ] )
    lcdict [ 'varinfo' ] [ 'target_flux_fraction_in_aper' ] . append ( hdrinfo [ 'flfrcsap' ] )
    # update the light curve columns now
    for key in datakeys :
        if key . lower ( ) in lcdict :
            lcdict [ key . lower ( ) ] = ( np . concatenate ( ( lcdict [ key . lower ( ) ] , lcdata [ key ] ) ) )
    for key in sapkeys :
        if key . lower ( ) in lcdict [ 'sap' ] :
            sapflux_median = np . nanmedian ( lcdata [ 'SAP_FLUX' ] )
            # normalize the current flux measurements if needed
            if normalize and key == 'SAP_FLUX' :
                thislcdata = lcdata [ key ] / sapflux_median
            elif normalize and key == 'SAP_FLUX_ERR' :
                thislcdata = lcdata [ key ] / sapflux_median
            elif normalize and key == 'SAP_BKG' :
                thislcdata = lcdata [ key ] / sapflux_median
            elif normalize and key == 'SAP_BKG_ERR' :
                thislcdata = lcdata [ key ] / sapflux_median
            else :
                thislcdata = lcdata [ key ]
            lcdict [ 'sap' ] [ key . lower ( ) ] = ( np . concatenate ( ( lcdict [ 'sap' ] [ key . lower ( ) ] , thislcdata ) ) )
    for key in pdckeys :
        if key . lower ( ) in lcdict [ 'pdc' ] :
            pdcsap_flux_median = np . nanmedian ( lcdata [ 'PDCSAP_FLUX' ] )
            # normalize the current flux measurements if needed
            if normalize and key == 'PDCSAP_FLUX' :
                thislcdata = lcdata [ key ] / pdcsap_flux_median
            elif normalize and key == 'PDCSAP_FLUX_ERR' :
                thislcdata = lcdata [ key ] / pdcsap_flux_median
            else :
                thislcdata = lcdata [ key ]
            lcdict [ 'pdc' ] [ key . lower ( ) ] = ( np . concatenate ( ( lcdict [ 'pdc' ] [ key . lower ( ) ] , thislcdata ) ) )
    # append some of the light curve information into existing numpy arrays
    # so we can sort on them later
    lcdict [ 'exptime' ] = np . concatenate ( ( lcdict [ 'exptime' ] , np . full_like ( lcdata [ 'TIME' ] , hdrinfo [ 'exposure' ] , dtype = np . float64 ) ) )
    lcdict [ 'sector' ] = np . concatenate ( ( lcdict [ 'sector' ] , np . full_like ( lcdata [ 'TIME' ] , hdrinfo [ 'sector' ] , dtype = np . int64 ) ) )
    lcdict [ 'camera' ] = np . concatenate ( ( lcdict [ 'camera' ] , np . full_like ( lcdata [ 'TIME' ] , hdrinfo [ 'camera' ] , dtype = np . int64 ) ) )
    lcdict [ 'ccd' ] = np . concatenate ( ( lcdict [ 'ccd' ] , np . full_like ( lcdata [ 'TIME' ] , hdrinfo [ 'ccd' ] , dtype = np . int64 ) ) )
    lcdict [ 'pixel_table_id' ] = np . concatenate ( ( lcdict [ 'pixel_table_id' ] , np . full_like ( lcdata [ 'TIME' ] , hdrinfo [ 'pxtable' ] , dtype = np . int64 ) ) )
    lcdict [ 'origin' ] = np . concatenate ( ( lcdict [ 'origin' ] , np . full_like ( lcdata [ 'TIME' ] , hdrinfo [ 'origin' ] , dtype = 'U100' ) ) )
    lcdict [ 'date_obs_start' ] = np . concatenate ( ( lcdict [ 'date_obs_start' ] , np . full_like ( lcdata [ 'TIME' ] , hdrinfo [ 'date-obs' ] , dtype = 'U100' ) ) )
    lcdict [ 'date_obs_end' ] = np . concatenate ( ( lcdict [ 'date_obs_end' ] , np . full_like ( lcdata [ 'TIME' ] , hdrinfo [ 'date-end' ] , dtype = 'U100' ) ) )
    lcdict [ 'procversion' ] = np . concatenate ( ( lcdict [ 'procversion' ] , np . full_like ( lcdata [ 'TIME' ] , hdrinfo [ 'procver' ] , dtype = 'U255' ) ) )
    lcdict [ 'datarelease' ] = np . concatenate ( ( lcdict [ 'datarelease' ] , np . full_like ( lcdata [ 'TIME' ] , hdrinfo [ 'data_rel' ] , dtype = np . int64 ) ) )
# otherwise , this is a new lcdict
else : # form the lcdict
# the metadata is one - elem arrays because we might add on to them later
    lcdict = { 'objectid' : hdrinfo [ 'object' ] , 'lcinfo' : { 'timesys' : [ hdrinfo [ 'timesys' ] ] , 'bjdoffset' : [ hdrinfo [ 'bjdrefi' ] + hdrinfo [ 'bjdreff' ] ] , 'exptime' : [ hdrinfo [ 'exposure' ] ] , 'lcaperture' : [ lcaperturedata ] , 'aperpix_used' : [ hdrinfo [ 'npixsap' ] ] , 'aperpix_unused' : [ hdrinfo [ 'npixmiss' ] ] , 'pixarcsec' : [ ( np . abs ( hdrinfo [ 'cdelt1' ] ) + np . abs ( hdrinfo [ 'cdelt2' ] ) ) * 3600.0 / 2.0 ] , 'ndet' : [ ndet ] , 'origin' : [ hdrinfo [ 'origin' ] ] , 'procversion' : [ hdrinfo [ 'procver' ] ] , 'datarelease' : [ hdrinfo [ 'data_rel' ] ] , 'sector' : [ hdrinfo [ 'sector' ] ] , 'camera' : [ hdrinfo [ 'camera' ] ] , 'ccd' : [ hdrinfo [ 'ccd' ] ] , 'pixel_table_id' : [ hdrinfo [ 'pxtable' ] ] , 'date_obs_start' : [ hdrinfo [ 'date-obs' ] ] , 'date_obs_end' : [ hdrinfo [ 'date-end' ] ] , 'tic_version' : [ hdrinfo [ 'ticver' ] ] , 'cr_mitigation' : [ hdrinfo [ 'crmiten' ] ] , 'cr_blocksize' : [ hdrinfo [ 'crblksz' ] ] , 'cr_spocclean' : [ hdrinfo [ 'crspoc' ] ] , } , 'objectinfo' : { 'objectid' : hdrinfo [ 'object' ] , 'ticid' : hdrinfo [ 'ticid' ] , 'tessmag' : hdrinfo [ 'tessmag' ] , 'ra' : hdrinfo [ 'ra_obj' ] , 'decl' : hdrinfo [ 'dec_obj' ] , 'pmra' : hdrinfo [ 'pmra' ] , 'pmdecl' : hdrinfo [ 'pmdec' ] , 'pmtotal' : hdrinfo [ 'pmtotal' ] , 'star_teff' : hdrinfo [ 'teff' ] , 'star_logg' : hdrinfo [ 'logg' ] , 'star_mh' : hdrinfo [ 'mh' ] , 'star_radius' : hdrinfo [ 'radius' ] , 'observatory' : 'TESS' , 'telescope' : 'TESS photometer' , } , 'varinfo' : { 'cdpp0_5' : [ hdrinfo [ 'cdpp0_5' ] ] , 'cdpp1_0' : [ hdrinfo [ 'cdpp1_0' ] ] , 'cdpp2_0' : [ hdrinfo [ 'cdpp2_0' ] ] , 'pdcvar' : [ hdrinfo [ 'pdcvar' ] ] , 'pdcmethod' : [ hdrinfo [ 'pdcmethd' ] ] , 'target_flux_total_flux_ratio_in_aper' : [ hdrinfo [ 'crowdsap' ] ] , 'target_flux_fraction_in_aper' : [ hdrinfo [ 'flfrcsap' ] ] , } , 'sap' : { } , 'pdc' : { } , }
    # get the LC columns
    for key in datakeys :
        lcdict [ key . lower ( ) ] = lcdata [ key ]
    for key in sapkeys :
        lcdict [ 'sap' ] [ key . lower ( ) ] = lcdata [ key ]
    for key in pdckeys :
        lcdict [ 'pdc' ] [ key . lower ( ) ] = lcdata [ key ]
    # turn some of the light curve information into numpy arrays so we can
    # sort on them later
    lcdict [ 'exptime' ] = np . full_like ( lcdict [ 'time' ] , lcdict [ 'lcinfo' ] [ 'exptime' ] [ 0 ] , dtype = np . float64 )
    lcdict [ 'sector' ] = np . full_like ( lcdict [ 'time' ] , lcdict [ 'lcinfo' ] [ 'sector' ] [ 0 ] , dtype = np . int64 )
    lcdict [ 'camera' ] = np . full_like ( lcdict [ 'time' ] , lcdict [ 'lcinfo' ] [ 'camera' ] [ 0 ] , dtype = np . int64 )
    lcdict [ 'ccd' ] = np . full_like ( lcdict [ 'time' ] , lcdict [ 'lcinfo' ] [ 'ccd' ] [ 0 ] , dtype = np . int64 )
    lcdict [ 'pixel_table_id' ] = np . full_like ( lcdict [ 'time' ] , lcdict [ 'lcinfo' ] [ 'pixel_table_id' ] [ 0 ] , dtype = np . int64 , )
    lcdict [ 'origin' ] = np . full_like ( lcdict [ 'time' ] , lcdict [ 'lcinfo' ] [ 'origin' ] [ 0 ] , dtype = 'U100' , )
    lcdict [ 'date_obs_start' ] = np . full_like ( lcdict [ 'time' ] , lcdict [ 'lcinfo' ] [ 'date_obs_start' ] [ 0 ] , dtype = 'U100' , )
    lcdict [ 'date_obs_end' ] = np . full_like ( lcdict [ 'time' ] , lcdict [ 'lcinfo' ] [ 'date_obs_end' ] [ 0 ] , dtype = 'U100' , )
    lcdict [ 'procversion' ] = np . full_like ( lcdict [ 'time' ] , lcdict [ 'lcinfo' ] [ 'procversion' ] [ 0 ] , dtype = 'U255' , )
    lcdict [ 'datarelease' ] = np . full_like ( lcdict [ 'time' ] , lcdict [ 'lcinfo' ] [ 'datarelease' ] [ 0 ] , dtype = np . int64 , )
    # normalize the SAP and PDCSAP fluxes , errs , and backgrounds if needed
    if normalize :
        sapflux_median = np . nanmedian ( lcdict [ 'sap' ] [ 'sap_flux' ] )
        pdcsap_flux_median = np . nanmedian ( lcdict [ 'pdc' ] [ 'pdcsap_flux' ] )
        lcdict [ 'sap' ] [ 'sap_flux' ] = ( lcdict [ 'sap' ] [ 'sap_flux' ] / sapflux_median )
        lcdict [ 'sap' ] [ 'sap_flux_err' ] = ( lcdict [ 'sap' ] [ 'sap_flux_err' ] / sapflux_median )
        lcdict [ 'sap' ] [ 'sap_bkg' ] = ( lcdict [ 'sap' ] [ 'sap_bkg' ] / sapflux_median )
        lcdict [ 'sap' ] [ 'sap_bkg_err' ] = ( lcdict [ 'sap' ] [ 'sap_bkg_err' ] / sapflux_median )
        lcdict [ 'pdc' ] [ 'pdcsap_flux' ] = ( lcdict [ 'pdc' ] [ 'pdcsap_flux' ] / pdcsap_flux_median )
        lcdict [ 'pdc' ] [ 'pdcsap_flux_err' ] = ( lcdict [ 'pdc' ] [ 'pdcsap_flux_err' ] / pdcsap_flux_median )
# # END OF LIGHT CURVE CONSTRUCTION # #
# update the lcdict columns with the actual columns
lcdict [ 'columns' ] = ( [ x . lower ( ) for x in datakeys ] + [ 'sap.%s' % x . lower ( ) for x in sapkeys ] + [ 'pdc.%s' % x . lower ( ) for x in pdckeys ] + [ 'exptime' , 'sector' , 'camera' , 'ccd' , 'pixel_table_id' , 'origin' , 'date_obs_start' , 'date_obs_end' , 'procversion' , 'datarelease' ] )
# update the ndet key in the objectinfo with the sum of all observations
lcdict [ 'objectinfo' ] [ 'ndet' ] = sum ( lcdict [ 'lcinfo' ] [ 'ndet' ] )
# filter the LC dict if requested
if ( filterqualityflags is not False or nanfilter is not None or timestoignore is not None ) :
    lcdict = filter_tess_lcdict ( lcdict , filterqualityflags , nanfilter = nanfilter , timestoignore = timestoignore )
# return the lcdict at the end
return lcdict


signature:
def consolidate_tess_fitslc ( lclist , normalize = True , filterqualityflags = False , nanfilter = None , timestoignore = None , headerkeys = LCHEADERKEYS , datakeys = LCDATAKEYS , sapkeys = LCSAPKEYS , pdckeys = LCPDCKEYS , topkeys = LCTOPKEYS , apkeys = LCAPERTUREKEYS ) :
    '''This consolidates a list of LCs for a single TIC object .
 NOTE : if light curve time arrays contain nans , these and their associated
 measurements will be sorted to the end of the final combined arrays .
 Parameters
 lclist : list of str , or str
 ` lclist ` is either a list of actual light curve files or a string that
 is valid for glob . glob to search for and generate a light curve list
 based on the file glob . This is useful for consolidating LC FITS files
 across different TESS sectors for a single TIC ID using a glob like
 ` * < TICID > * _ lc . fits ` .
 normalize : bool
 If True , then the light curve ' s SAP _ FLUX and PDCSAP _ FLUX measurements
 will be normalized to 1.0 by dividing out the median flux for the
 component light curve .
 filterqualityflags : bool
 If True , will remove any measurements that have non - zero quality flags
 present . This usually indicates an issue with the instrument or
 spacecraft .
 nanfilter : { ' sap ' , ' pdc ' , ' sap , pdc ' } or None
 Indicates the flux measurement type ( s ) to apply the filtering to .
 timestoignore : list of tuples or None
 This is of the form : :
 [ ( time1 _ start , time1 _ end ) , ( time2 _ start , time2 _ end ) , . . . ]
 and indicates the start and end times to mask out of the final
 lcdict . Use this to remove anything that wasn ' t caught by the quality
 flags .
 headerkeys : list
 A list of FITS header keys that will be extracted from the FITS light
 curve file . These describe the observations . The default value for this
 is given in ` LCHEADERKEYS ` above .
 datakeys : list
 A list of FITS column names that correspond to the auxiliary
 measurements in the light curve . The default is ` LCDATAKEYS ` above .
 sapkeys : list
 A list of FITS column names that correspond to the SAP flux
 measurements in the light curve . The default is ` LCSAPKEYS ` above .
 pdckeys : list
 A list of FITS column names that correspond to the PDC flux
 measurements in the light curve . The default is ` LCPDCKEYS ` above .
 topkeys : list
 A list of FITS header keys that describe the object in the light
 curve . The default is ` LCTOPKEYS ` above .
 apkeys : list
 A list of FITS header keys that describe the flux measurement apertures
 used by the TESS pipeline . The default is ` LCAPERTUREKEYS ` above .
 Returns
 lcdict
 Returns an ` lcdict ` ( this is useable by most astrobase functions for LC
 processing ) .'''

implementation:
# if the lclist is a string , assume that we ' re passing in a fileglob
if isinstance ( lclist , str ) :
    if sys . version_info [ : 2 ] > ( 3 , 4 ) :
        matching = glob . glob ( lclist , recursive = True )
        LOGINFO ( 'found %s LCs: %r' % ( len ( matching ) , matching ) )
    else :
        lcfitsdir = os . path . dirname ( lclist )
        lcfitsfile = os . path . basename ( lclist )
        walker = os . walk ( lcfitsdir )
        matching = [ ]
        for root , dirs , _files in walker :
            for sdir in dirs :
                searchpath = os . path . join ( root , sdir , lcfitsfile )
                foundfiles = glob . glob ( searchpath )
                if foundfiles :
                    matching . extend ( foundfiles )
                    LOGINFO ( 'found %s in dir: %s' % ( repr ( foundfiles ) , os . path . join ( root , sdir ) ) )
    if len ( matching ) == 0 :
        LOGERROR ( 'could not find any TESS LC files matching glob: %s' % lclist )
        return None
# if the lclist is an actual list of LCs , then use it directly
else :
    matching = lclist
# get the first file
consolidated = read_tess_fitslc ( matching [ 0 ] , normalize = normalize , headerkeys = LCHEADERKEYS , datakeys = LCDATAKEYS , sapkeys = LCSAPKEYS , pdckeys = LCPDCKEYS , topkeys = LCTOPKEYS , apkeys = LCAPERTUREKEYS )
# get the rest of the files
if len ( matching ) > 1 :
    for lcf in matching [ 1 : ] :
        consolidated = read_tess_fitslc ( lcf , appendto = consolidated , normalize = normalize , headerkeys = LCHEADERKEYS , datakeys = LCDATAKEYS , sapkeys = LCSAPKEYS , pdckeys = LCPDCKEYS , topkeys = LCTOPKEYS , apkeys = LCAPERTUREKEYS )
# get the sort indices . we use time for the columns and sectors for the
# bits in lcinfo and varinfo
LOGINFO ( 'sorting by time...' )
# NOTE : nans in time will be sorted to the end of the array
finiteind = np . isfinite ( consolidated [ 'time' ] )
if np . sum ( finiteind ) < consolidated [ 'time' ] . size :
    LOGWARNING ( 'some time values are nan! ' 'measurements at these times will be ' 'sorted to the end of the column arrays.' )
# get the time sort index
column_sort_ind = np . argsort ( consolidated [ 'time' ] )
# sort the columns by time
for col in consolidated [ 'columns' ] :
    if '.' in col :
        key , subkey = col . split ( '.' )
        consolidated [ key ] [ subkey ] = ( consolidated [ key ] [ subkey ] [ column_sort_ind ] )
    else :
        consolidated [ col ] = consolidated [ col ] [ column_sort_ind ]
info_sort_ind = np . argsort ( consolidated [ 'lcinfo' ] [ 'sector' ] )
# sort the keys in lcinfo
for key in consolidated [ 'lcinfo' ] :
    consolidated [ 'lcinfo' ] [ key ] = ( np . array ( consolidated [ 'lcinfo' ] [ key ] ) [ info_sort_ind ] . tolist ( ) )
# sort the keys in varinfo
for key in consolidated [ 'varinfo' ] :
    consolidated [ 'varinfo' ] [ key ] = ( np . array ( consolidated [ 'varinfo' ] [ key ] ) [ info_sort_ind ] . tolist ( ) )
# filter the LC dict if requested
# we do this at the end
if ( filterqualityflags is not False or nanfilter is not None or timestoignore is not None ) :
    consolidated = filter_tess_lcdict ( consolidated , filterqualityflags , nanfilter = nanfilter , timestoignore = timestoignore )
return consolidated


signature:
def filter_tess_lcdict ( lcdict , filterqualityflags = True , nanfilter = 'sap,pdc,time' , timestoignore = None , quiet = False ) :
    '''This filters the provided TESS ` lcdict ` , removing nans and bad
 observations .
 By default , this function removes points in the TESS LC that have ANY
 quality flags set .
 Parameters
 lcdict : lcdict
 An ` lcdict ` produced by ` consolidate _ tess _ fitslc ` or
 ` read _ tess _ fitslc ` .
 filterflags : bool
 If True , will remove any measurements that have non - zero quality flags
 present . This usually indicates an issue with the instrument or
 spacecraft .
 nanfilter : { ' sap ' , ' pdc ' , ' sap , pdc ' }
 Indicates the flux measurement type ( s ) to apply the filtering to .
 timestoignore : list of tuples or None
 This is of the form : :
 [ ( time1 _ start , time1 _ end ) , ( time2 _ start , time2 _ end ) , . . . ]
 and indicates the start and end times to mask out of the final
 lcdict . Use this to remove anything that wasn ' t caught by the quality
 flags .
 Returns
 lcdict
 Returns an ` lcdict ` ( this is useable by most astrobase functions for LC
 processing ) . The ` lcdict ` is filtered IN PLACE !'''

implementation:
cols = lcdict [ 'columns' ]
# filter all bad LC points as noted by quality flags
if filterqualityflags :
    nbefore = lcdict [ 'time' ] . size
    filterind = lcdict [ 'quality' ] == 0
    for col in cols :
        if '.' in col :
            key , subkey = col . split ( '.' )
            lcdict [ key ] [ subkey ] = lcdict [ key ] [ subkey ] [ filterind ]
        else :
            lcdict [ col ] = lcdict [ col ] [ filterind ]
    nafter = lcdict [ 'time' ] . size
    if not quiet :
        LOGINFO ( 'applied quality flag filter, ' 'ndet before = %s, ndet after = %s' % ( nbefore , nafter ) )
if nanfilter and nanfilter == 'sap,pdc,time' :
    notnanind = ( np . isfinite ( lcdict [ 'sap' ] [ 'sap_flux' ] ) & np . isfinite ( lcdict [ 'sap' ] [ 'sap_flux_err' ] ) & np . isfinite ( lcdict [ 'pdc' ] [ 'pdcsap_flux' ] ) & np . isfinite ( lcdict [ 'pdc' ] [ 'pdcsap_flux_err' ] ) & np . isfinite ( lcdict [ 'time' ] ) )
elif nanfilter and nanfilter == 'sap,time' :
    notnanind = ( np . isfinite ( lcdict [ 'sap' ] [ 'sap_flux' ] ) & np . isfinite ( lcdict [ 'sap' ] [ 'sap_flux_err' ] ) & np . isfinite ( lcdict [ 'time' ] ) )
elif nanfilter and nanfilter == 'pdc,time' :
    notnanind = ( np . isfinite ( lcdict [ 'pdc' ] [ 'pdcsap_flux' ] ) & np . isfinite ( lcdict [ 'pdc' ] [ 'pdcsap_flux_err' ] ) & np . isfinite ( lcdict [ 'time' ] ) )
elif nanfilter is None :
    pass
else :
    raise NotImplementedError
# remove nans from all columns
if nanfilter :
    nbefore = lcdict [ 'time' ] . size
    for col in cols :
        if '.' in col :
            key , subkey = col . split ( '.' )
            lcdict [ key ] [ subkey ] = lcdict [ key ] [ subkey ] [ notnanind ]
        else :
            lcdict [ col ] = lcdict [ col ] [ notnanind ]
    nafter = lcdict [ 'time' ] . size
    if not quiet :
        LOGINFO ( 'removed nans, ndet before = %s, ndet after = %s' % ( nbefore , nafter ) )
# exclude all times in timestoignore
if ( timestoignore and isinstance ( timestoignore , list ) and len ( timestoignore ) > 0 ) :
    exclind = np . full_like ( lcdict [ 'time' ] , True ) . astype ( bool )
    nbefore = exclind . size
    # get all the masks
    for ignoretime in timestoignore :
        time0 , time1 = ignoretime [ 0 ] , ignoretime [ 1 ]
        thismask = ~ ( ( lcdict [ 'time' ] >= time0 ) & ( lcdict [ 'time' ] <= time1 ) )
        exclind = exclind & thismask
    # apply the masks
    for col in cols :
        if '.' in col :
            key , subkey = col . split ( '.' )
            lcdict [ key ] [ subkey ] = lcdict [ key ] [ subkey ] [ exclind ]
        else :
            lcdict [ col ] = lcdict [ col ] [ exclind ]
    nafter = lcdict [ 'time' ] . size
    if not quiet :
        LOGINFO ( 'removed timestoignore, ndet before = %s, ndet after = %s' % ( nbefore , nafter ) )
return lcdict