signature:
def upsert_license_requests ( cursor , uuid_ , roles ) :
    """Given a ` ` uuid ` ` and list of ` ` roles ` ` ( user identifiers )
 create a license acceptance entry . If ` ` has _ accepted ` ` is supplied ,
 it will be used to assign an acceptance value to all listed ` ` uids ` ` ."""

implementation:
if not isinstance ( roles , ( list , set , tuple , ) ) :
    raise TypeError ( "``roles`` is an invalid type: {}" . format ( type ( roles ) ) )
acceptors = set ( [ x [ 'uid' ] for x in roles ] )
# Acquire a list of existing acceptors .
cursor . execute ( """\
SELECT user_id, accepted FROM license_acceptances WHERE uuid = %s""" , ( uuid_ , ) )
existing_acceptors = cursor . fetchall ( )
# Who ' s not in the existing list ?
new_acceptors = acceptors . difference ( [ x [ 0 ] for x in existing_acceptors ] )
# Insert the new licensor acceptors .
if new_acceptors :
    args = [ ]
    values_fmt = [ ]
    for uid in new_acceptors :
        has_accepted = [ x . get ( 'has_accepted' , None ) for x in roles if uid == x [ 'uid' ] ] [ 0 ]
        args . extend ( [ uuid_ , uid , has_accepted ] )
        values_fmt . append ( "(%s, %s, %s)" )
    values_fmt = ', ' . join ( values_fmt )
    cursor . execute ( """\
INSERT INTO license_acceptances (uuid, user_id, accepted)
VALUES {}""" . format ( values_fmt ) , args )
# Update any existing license acceptors
acceptors = set ( [ ( x [ 'uid' ] , x . get ( 'has_accepted' , None ) , ) for x in roles # Prevent updating newly inserted records .
if ( x [ 'uid' ] , x . get ( 'has_accepted' , None ) , ) not in new_acceptors ] )
existing_acceptors = set ( [ x for x in existing_acceptors # Prevent updating newly inserted records .
if x [ 0 ] not in new_acceptors ] )
tobe_updated_acceptors = acceptors . difference ( existing_acceptors )
for uid , has_accepted in tobe_updated_acceptors :
    cursor . execute ( """\
UPDATE license_acceptances SET accepted = %s
WHERE uuid = %s AND user_id = %s""" , ( has_accepted , uuid_ , uid , ) )


signature:
def remove_license_requests ( cursor , uuid_ , uids ) :
    """Given a ` ` uuid ` ` and list of ` ` uids ` ` ( user identifiers )
 remove the identified users ' license acceptance entries ."""

implementation:
if not isinstance ( uids , ( list , set , tuple , ) ) :
    raise TypeError ( "``uids`` is an invalid type: {}" . format ( type ( uids ) ) )
acceptors = list ( set ( uids ) )
# Remove the the entries .
cursor . execute ( """\
DELETE FROM license_acceptances
WHERE uuid = %s AND user_id = ANY(%s::text[])""" , ( uuid_ , acceptors , ) )


signature:
def upsert_role_requests ( cursor , uuid_ , roles ) :
    """Given a ` ` uuid ` ` and list of dicts containing the ` ` uid ` ` and
 ` ` role ` ` for creating a role acceptance entry . The ` ` roles ` ` dict
 can optionally contain a ` ` has _ accepted ` ` value , which will default
 to true ."""

implementation:
if not isinstance ( roles , ( list , set , tuple , ) ) :
    raise TypeError ( "``roles`` is an invalid type: {}" . format ( type ( roles ) ) )
acceptors = set ( [ ( x [ 'uid' ] , x [ 'role' ] , ) for x in roles ] )
# Acquire a list of existing acceptors .
cursor . execute ( """\
SELECT user_id, role_type, accepted
FROM role_acceptances
WHERE uuid = %s""" , ( uuid_ , ) )
existing_roles = cursor . fetchall ( )
# Who ' s not in the existing list ?
existing_acceptors = set ( [ ( r , t , ) for r , t , _ in existing_roles ] )
new_acceptors = acceptors . difference ( existing_acceptors )
# Insert the new role acceptors .
for acceptor , type_ in new_acceptors :
    has_accepted = [ x . get ( 'has_accepted' , None ) for x in roles if acceptor == x [ 'uid' ] and type_ == x [ 'role' ] ] [ 0 ]
    cursor . execute ( """\
INSERT INTO role_acceptances ("uuid", "user_id", "role_type", "accepted")
VALUES (%s, %s, %s, %s)""" , ( uuid_ , acceptor , type_ , has_accepted , ) )
# Update any existing license acceptors
acceptors = set ( [ ( x [ 'uid' ] , x [ 'role' ] , x . get ( 'has_accepted' , None ) , ) for x in roles # Prevent updating newly inserted records .
if ( x [ 'uid' ] , x . get ( 'has_accepted' , None ) , ) not in new_acceptors ] )
existing_acceptors = set ( [ x for x in existing_roles # Prevent updating newly inserted records .
if ( x [ 0 ] , x [ 1 ] , ) not in new_acceptors ] )
tobe_updated_acceptors = acceptors . difference ( existing_acceptors )
for uid , type_ , has_accepted in tobe_updated_acceptors :
    cursor . execute ( """\
UPDATE role_acceptances SET accepted = %s
WHERE uuid = %s AND user_id = %s AND role_type = %s""" , ( has_accepted , uuid_ , uid , type_ , ) )


signature:
def remove_role_requests ( cursor , uuid_ , roles ) :
    """Given a ` ` uuid ` ` and list of dicts containing the ` ` uid ` `
 ( user identifiers ) and ` ` role ` ` for removal of the identified
 users ' role acceptance entries ."""

implementation:
if not isinstance ( roles , ( list , set , tuple , ) ) :
    raise TypeError ( "``roles`` is an invalid type: {}" . format ( type ( roles ) ) )
acceptors = set ( [ ( x [ 'uid' ] , x [ 'role' ] , ) for x in roles ] )
# Remove the the entries .
for uid , role_type in acceptors :
    cursor . execute ( """\
DELETE FROM role_acceptances
WHERE uuid = %s AND user_id = %s AND role_type = %s""" , ( uuid_ , uid , role_type , ) )


signature:
def upsert_acl ( cursor , uuid_ , permissions ) :
    """Given a ` ` uuid ` ` and a set of permissions given as a
 tuple of ` ` uid ` ` and ` ` permission ` ` , upsert them into the database ."""

implementation:
if not isinstance ( permissions , ( list , set , tuple , ) ) :
    raise TypeError ( "``permissions`` is an invalid type: {}" . format ( type ( permissions ) ) )
permissions = set ( permissions )
# Acquire the existin ACL .
cursor . execute ( """\
SELECT user_id, permission
FROM document_acl
WHERE uuid = %s""" , ( uuid_ , ) )
existing = set ( [ ( r , t , ) for r , t in cursor . fetchall ( ) ] )
# Who ' s not in the existing list ?
new_entries = permissions . difference ( existing )
# Insert the new permissions .
for uid , permission in new_entries :
    cursor . execute ( """\
INSERT INTO document_acl
  ("uuid", "user_id", "permission")
VALUES (%s, %s, %s)""" , ( uuid_ , uid , permission ) )


signature:
def remove_acl ( cursor , uuid_ , permissions ) :
    """Given a ` ` uuid ` ` and a set of permissions given as a tuple
 of ` ` uid ` ` and ` ` permission ` ` , remove these entries from the database ."""

implementation:
if not isinstance ( permissions , ( list , set , tuple , ) ) :
    raise TypeError ( "``permissions`` is an invalid type: {}" . format ( type ( permissions ) ) )
permissions = set ( permissions )
# Remove the the entries .
for uid , permission in permissions :
    cursor . execute ( """\
DELETE FROM document_acl
WHERE uuid = %s AND user_id = %s AND permission = %s""" , ( uuid_ , uid , permission , ) )


signature:
def _upsert_persons ( cursor , person_ids , lookup_func ) :
    """Upsert ' s user info into the database .
 The model contains the user info as part of the role values ."""

implementation:
person_ids = list ( set ( person_ids ) )
# cleanse data
# Check for existing records to update .
cursor . execute ( "SELECT personid from persons where personid = ANY (%s)" , ( person_ids , ) )
existing_person_ids = [ x [ 0 ] for x in cursor . fetchall ( ) ]
new_person_ids = [ p for p in person_ids if p not in existing_person_ids ]
# Update existing records .
for person_id in existing_person_ids : # TODO only update based on a delta against the ' updated ' column .
    person_info = lookup_func ( person_id )
    cursor . execute ( """\
UPDATE persons
SET (personid, firstname, surname, fullname) =
    ( %(username)s, %(first_name)s, %(last_name)s,
     %(full_name)s)
WHERE personid = %(username)s""" , person_info )
# Insert new records .
# Email is an empty string because
# accounts no longer gives out user
# email info but a string datatype
# is still needed for legacy to
# properly process the persons table
for person_id in new_person_ids :
    person_info = lookup_func ( person_id )
    cursor . execute ( """\
INSERT INTO persons
(personid, firstname, surname, fullname, email)
VALUES
(%(username)s, %(first_name)s,
%(last_name)s, %(full_name)s, '')""" , person_info )


signature:
def _upsert_users ( cursor , user_ids , lookup_func ) :
    """Upsert ' s user info into the database .
 The model contains the user info as part of the role values ."""

implementation:
user_ids = list ( set ( user_ids ) )
# cleanse data
# Check for existing records to update .
cursor . execute ( "SELECT username from users where username = ANY (%s)" , ( user_ids , ) )
existing_user_ids = [ x [ 0 ] for x in cursor . fetchall ( ) ]
new_user_ids = [ u for u in user_ids if u not in existing_user_ids ]
# Update existing records .
for user_id in existing_user_ids : # TODO only update based on a delta against the ' updated ' column .
    user_info = lookup_func ( user_id )
    cursor . execute ( """\
UPDATE users
SET (updated, username, first_name, last_name, full_name, title) =
    (CURRENT_TIMESTAMP, %(username)s, %(first_name)s, %(last_name)s,
     %(full_name)s, %(title)s)
WHERE username = %(username)s""" , user_info )
# Insert new records .
for user_id in new_user_ids :
    user_info = lookup_func ( user_id )
    cursor . execute ( """\
INSERT INTO users
(username, first_name, last_name, full_name, suffix, title)
VALUES
(%(username)s, %(first_name)s, %(last_name)s, %(full_name)s,
 %(suffix)s, %(title)s)""" , user_info )


signature:
def upsert_users ( cursor , user_ids ) :
    """Given a set of user identifiers ( ` ` user _ ids ` ` ) ,
 upsert them into the database after checking accounts for
 the latest information ."""

implementation:
accounts = get_current_registry ( ) . getUtility ( IOpenstaxAccounts )
def lookup_profile ( username ) :
    profile = accounts . get_profile_by_username ( username )
    # See structure documentation at :
    # https : / / < accounts - instance > / api / docs / v1 / users / index
    if profile is None :
        raise UserFetchError ( username )
    opt_attrs = ( 'first_name' , 'last_name' , 'full_name' , 'title' , 'suffix' , )
    for attr in opt_attrs :
        profile . setdefault ( attr , None )
    return profile
_upsert_users ( cursor , user_ids , lookup_profile )
_upsert_persons ( cursor , user_ids , lookup_profile )


signature:
def notify_users ( cursor , document_id ) :
    """Notify all users about their role and / or license acceptance
 for a piece of content associated with the given ` ` document _ id ` ` ."""

implementation:
return
registry = get_current_registry ( )
accounts = registry . getUtility ( IOpenstaxAccounts )
cursor . execute ( """\
SELECT la.user_id
FROM license_acceptances AS la
WHERE
  la.uuid = (SELECT uuid FROM pending_documents WHERE id = %s)
  AND la.notified IS NULL AND (NOT la.accepted or la.accepted IS UNKNOWN)
""" , ( document_id , ) )
licensors = [ x [ 0 ] for x in cursor . fetchall ( ) ]
cursor . execute ( """\
SELECT user_id, array_agg(role_type)::text[]
FROM role_acceptances AS ra
WHERE
  ra.uuid = (SELECT uuid FROM pending_documents WHERE id = %s)
  AND ra.notified IS NULL AND (NOT ra.accepted or ra.accepted IS UNKNOWN)
GROUP BY user_id
""" , ( document_id , ) )
roles = { u : r for u , r in cursor . fetchall ( ) }
needs_notified = set ( licensors + roles . keys ( ) )
for user_id in needs_notified :
    data = { 'user_id' : user_id , 'full_name' : None , # TODO
    'licensor' : user_id in licensors , 'roles' : roles . get ( user_id , [ ] ) , }
    message = NOTIFICATION_TEMPLATE . render ( ** data )
    accounts . send_message ( user_id , NOFIFICATION_SUBJECT , message )
cursor . execute ( """\
UPDATE license_acceptances SET notified = CURRENT_TIMESTAMP
WHERE
  uuid = (SELECT uuid FROM pending_documents WHERE id = %s)
  AND user_id = ANY (%s)""" , ( document_id , licensors , ) )
# FIXME overwrites notified for all roles types a user might have .
cursor . execute ( """\
UPDATE role_acceptances SET notified = CURRENT_TIMESTAMP
WHERE
  uuid = (SELECT uuid FROM pending_documents WHERE id = %s)
  AND user_id = ANY (%s)""" , ( document_id , roles . keys ( ) , ) )


signature:
def set_post_publications_state ( cursor , module_ident , state_name , state_message = '' ) : # pragma : no cover
    """This sets the post - publication state in the database ."""

implementation:
cursor . execute ( """\
INSERT INTO post_publications
  (module_ident, state, state_message)
  VALUES (%s, %s, %s)""" , ( module_ident , state_name , state_message ) )


signature:
def update_module_state ( cursor , module_ident , state_name , recipe ) : # pragma : no cover
    """This updates the module ' s state in the database ."""

implementation:
cursor . execute ( """\
UPDATE modules
SET stateid = (
    SELECT stateid FROM modulestates WHERE statename = %s
), recipe = %s, baked = now() WHERE module_ident = %s""" , ( state_name , recipe , module_ident ) )


signature:
def get_moderation ( request ) :
    """Return the list of publications that need moderation ."""

implementation:
with db_connect ( ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        cursor . execute ( """\
SELECT row_to_json(combined_rows) FROM (
  SELECT id, created, publisher, publication_message,
         (select array_agg(row_to_json(pd))
          from pending_documents as pd
          where pd.publication_id = p.id) AS models
  FROM publications AS p
  WHERE state = 'Waiting for moderation') AS combined_rows""" )
        moderations = [ x [ 0 ] for x in cursor . fetchall ( ) ]
return moderations


signature:
def includeme ( config ) :
    """Configures the session manager"""

implementation:
settings = config . registry . settings
session_factory = SignedCookieSessionFactory ( settings [ 'session_key' ] )
config . set_session_factory ( session_factory )


signature:
def admin_print_styles ( request ) :
    """Returns a dictionary of all unique print _ styles , and their latest tag ,
 revision , and recipe _ type ."""

implementation:
styles = [ ]
# This fetches all recipes that have been used to successfully bake a
# current book plus all default recipes that have not yet been used
# as well as " bad " books that are not " current " state , but would otherwise
# be the latest / current for that book
with db_connect ( cursor_factory = DictCursor ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        cursor . execute ( """\
                WITH latest AS (SELECT print_style, recipe,
                    count(*), count(nullif(stateid, 1)) as bad
                FROM modules m
                WHERE portal_type = 'Collection'
                      AND recipe IS NOT NULL
                      AND (
                          baked IS NOT NULL OR (
                              baked IS NULL AND stateid not in (1,8)
                              )
                          )
                      AND ARRAY [major_version, minor_version] = (
                          SELECT max(ARRAY[major_version,minor_version]) FROM
                              modules where m.uuid= uuid)

                GROUP BY print_style, recipe
                ),
                defaults AS (SELECT print_style, fileid AS recipe
                FROM default_print_style_recipes d
                WHERE not exists (SELECT 1
                                  FROM latest WHERE latest.recipe = d.fileid)
                )
                SELECT coalesce(ps.print_style, '(custom)') as print_style,
                       ps.title, coalesce(ps.recipe_type, 'web') as type,
                       ps.revised, ps.tag, ps.commit_id, la.count, la.bad
                FROM latest la LEFT JOIN print_style_recipes ps ON
                                    la.print_style = ps.print_style AND
                                    la.recipe = ps.fileid
                UNION ALL
                SELECT ps.print_style, ps.title, ps.recipe_type,
                       ps.revised, ps.tag, ps.commit_id, 0 AS count, 0 AS bad
                FROM defaults de JOIN print_style_recipes ps ON
                                    de.print_style = ps.print_style AND
                                    de.recipe = ps.fileid

            ORDER BY revised desc NULLS LAST, print_style

                """ )
        for row in cursor . fetchall ( ) :
            styles . append ( { 'print_style' : row [ 'print_style' ] , 'title' : row [ 'title' ] , 'type' : row [ 'type' ] , 'revised' : row [ 'revised' ] , 'tag' : row [ 'tag' ] , 'commit_id' : row [ 'commit_id' ] , 'number' : row [ 'count' ] , 'bad' : row [ 'bad' ] , 'link' : request . route_path ( 'admin-print-style-single' , style = row [ 'print_style' ] ) } )
return { 'styles' : styles }


signature:
def admin_print_styles_single ( request ) :
    """Returns all books with any version of the given print style .
 Returns the print _ style , recipe type , num books using the print _ style ,
 along with a dictionary of the book , author , revision date , recipe ,
 tag of the print _ style , and a link to the content ."""

implementation:
style = request . matchdict [ 'style' ]
# do db search to get file id and other info on the print _ style
with db_connect ( cursor_factory = DictCursor ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        if style != '(custom)' :
            cursor . execute ( """
                    SELECT fileid, recipe_type, title
                    FROM default_print_style_recipes
                    WHERE print_style=%s
                    """ , vars = ( style , ) )
            info = cursor . fetchall ( )
            if len ( info ) < 1 :
                current_recipe = None
                recipe_type = None
                status = None
            else :
                current_recipe = info [ 0 ] [ 'fileid' ]
                recipe_type = info [ 0 ] [ 'recipe_type' ]
                status = 'current'
            cursor . execute ( """\
                    SELECT name, authors, lm.revised, lm.recipe, psr.tag,
                        f.sha1 as hash, psr.commit_id, uuid,
                        ident_hash(uuid, major_version, minor_version)
                    FROM modules as lm
                    LEFT JOIN print_style_recipes as psr
                    ON (psr.print_style = lm.print_style and
                        psr.fileid = lm.recipe)
                    LEFT JOIN files f ON psr.fileid = f.fileid
                    WHERE lm.print_style=%s
                    AND portal_type='Collection'
                    AND ARRAY [major_version, minor_version] = (
                        SELECT max(ARRAY[major_version,minor_version])
                        FROM modules WHERE lm.uuid = uuid)

                    ORDER BY psr.tag DESC;
                    """ , vars = ( style , ) )
        else :
            current_recipe = '(custom)'
            recipe_type = '(custom)'
            cursor . execute ( """\
                    SELECT name, authors, lm.revised, lm.recipe, NULL as tag,
                        f.sha1 as hash, NULL as commit_id, uuid,
                        ident_hash(uuid, major_version, minor_version)
                    FROM modules as lm
                    JOIN files f ON lm.recipe = f.fileid
                    WHERE portal_type='Collection'
                    AND NOT EXISTS (
                        SELECT 1 from print_style_recipes psr
                        WHERE psr.fileid = lm.recipe)
                    AND ARRAY [major_version, minor_version] = (
                        SELECT max(ARRAY[major_version,minor_version])
                        FROM modules WHERE lm.uuid = uuid)
                    ORDER BY uuid, recipe, revised DESC;
                    """ , vars = ( style , ) )
            status = '(custom)'
        collections = [ ]
        for row in cursor . fetchall ( ) :
            recipe = row [ 'recipe' ]
            if ( status != '(custom)' and current_recipe is not None and recipe != current_recipe ) :
                status = 'stale'
            collections . append ( { 'title' : row [ 'name' ] . decode ( 'utf-8' ) , 'authors' : row [ 'authors' ] , 'revised' : row [ 'revised' ] , 'recipe' : row [ 'hash' ] , 'recipe_link' : request . route_path ( 'get-resource' , hash = row [ 'hash' ] ) , 'tag' : row [ 'tag' ] , 'ident_hash' : row [ 'ident_hash' ] , 'link' : request . route_path ( 'get-content' , ident_hash = row [ 'ident_hash' ] ) , 'status' : status , 'status_link' : request . route_path ( 'admin-content-status-single' , uuid = row [ 'uuid' ] ) , } )
return { 'number' : len ( collections ) , 'collections' : collections , 'print_style' : style , 'recipe_type' : recipe_type }


signature:
def get_api_keys ( request ) :
    """Return the list of API keys ."""

implementation:
with db_connect ( ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        cursor . execute ( """\
SELECT row_to_json(combined_rows) FROM (
  SELECT id, key, name, groups FROM api_keys
) AS combined_rows""" )
        api_keys = [ x [ 0 ] for x in cursor . fetchall ( ) ]
return api_keys


signature:
def get_baking_statuses_sql ( get_request ) :
    """Creates SQL to get info on baking books filtered from GET request .
 All books that have ever attempted to bake will be retured if they
 pass the filters in the GET request .
 If a single book has been requested to bake multiple times there will
 be a row for each of the baking attempts .
 By default the results are sorted in descending order of when they were
 requested to bake .
 N . B . The version reported for a print - style linked recipe will the the
 lowest cnx - recipes release installed that contains the exact recipe
 used to bake that book , regardless of when the book was baked relative
 to recipe releases . E . g . if a book uses the ' physics ' recipe , and it is
 identical for versions 1.1 , 1.2 , 1.3 , and 1.4 , then it will be reported
 as version 1.1 , even if the most recent release is tagged 1.4."""

implementation:
args = { }
sort = get_request . get ( 'sort' , 'bpsa.created DESC' )
if ( len ( sort . split ( " " ) ) != 2 or sort . split ( " " ) [ 0 ] not in SORTS_DICT . keys ( ) or sort . split ( " " ) [ 1 ] not in ARROW_MATCH . keys ( ) ) :
    raise httpexceptions . HTTPBadRequest ( 'invalid sort: {}' . format ( sort ) )
if sort == "STATE ASC" or sort == "STATE DESC" :
    sort = 'bpsa.created DESC'
uuid_filter = get_request . get ( 'uuid' , '' ) . strip ( )
author_filter = get_request . get ( 'author' , '' ) . strip ( )
latest_filter = get_request . get ( 'latest' , False )
sql_filters = "WHERE"
if latest_filter :
    sql_filters += """ ARRAY [m.major_version, m.minor_version] = (
         SELECT max(ARRAY[major_version,minor_version]) FROM
                   modules where m.uuid= uuid) AND """
if uuid_filter != '' :
    args [ 'uuid' ] = uuid_filter
    sql_filters += " m.uuid=%(uuid)s AND "
if author_filter != '' :
    author_filter = author_filter . decode ( 'utf-8' )
    sql_filters += " %(author)s=ANY(m.authors) "
    args [ "author" ] = author_filter
if sql_filters . endswith ( "AND " ) :
    sql_filters = sql_filters [ : - 4 ]
if sql_filters == "WHERE" :
    sql_filters = ""
# FIXME celery AsyncResult API is soooo sloow that this page takes
# 2 min . or more to load on production . As an workaround , this code
# accesses the celery _ taskmeta table directly . Need to remove that access
# once we track enough state info ourselves . Want to track when queued ,
# started , ended , etc . for future monitoring of baking system performance
# as well .
# The ' limit 1 ' subselect is to ensure the " oldest identical version "
# for recipes released as part of cnx - recipes ( avoids one line per
# identical recipe file in different releases , for a single baking job )
statement = """
                       SELECT m.name, m.authors, m.uuid,
                       module_version(m.major_version,m.minor_version)
                          as current_version,
                       m.print_style,
                       CASE WHEN f.sha1 IS NOT NULL
                       THEN coalesce(dps.print_style,'(custom)')
                       ELSE dps.print_style
                       END AS recipe_name,
                       (select tag from print_style_recipes
                            where print_style = m.print_style
                                and fileid = m.recipe
                                order by revised asc limit 1) as recipe_tag,
                       coalesce(dps.fileid, m.recipe) as latest_recipe_id,
                       m.recipe as recipe_id,
                       f.sha1 as recipe,
                       m.module_ident,
                       ident_hash(m.uuid, m.major_version, m.minor_version),
                       bpsa.created, ctm.traceback,
                       CASE WHEN ctm.status = 'SUCCESS'
                           AND ms.statename = 'fallback'
                       THEN 'FALLBACK'
                       ELSE ctm.status
                       END as state
                FROM document_baking_result_associations AS bpsa
                INNER JOIN modules AS m USING (module_ident)
                INNER JOIN modulestates as ms USING (stateid)
                LEFT JOIN celery_taskmeta AS ctm
                    ON bpsa.result_id = ctm.task_id::uuid
                LEFT JOIN default_print_style_recipes as dps
                    ON dps.print_style = m.print_style
                LEFT JOIN latest_modules as lm
                    ON lm.uuid=m.uuid
                LEFT JOIN files f on m.recipe = f.fileid
                {}
                ORDER BY {};
                """ . format ( sql_filters , sort )
args . update ( { 'sort' : sort } )
return statement , args


signature:
def admin_content_status ( request ) :
    """Returns a dictionary with the states and info of baking books ,
 and the filters from the GET request to pre - populate the form ."""

implementation:
statement , sql_args = get_baking_statuses_sql ( request . GET )
states = [ ]
status_filters = request . params . getall ( 'status_filter' ) or [ ]
state_icons = dict ( STATE_ICONS )
with db_connect ( cursor_factory = DictCursor ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        cursor . execute ( statement , vars = sql_args )
        for row in cursor . fetchall ( ) :
            message = ''
            state = row [ 'state' ] or 'PENDING'
            if status_filters and state not in status_filters :
                continue
            if state == 'FAILURE' : # pragma : no cover
                if row [ 'traceback' ] is not None :
                    message = row [ 'traceback' ] . split ( "\n" ) [ - 2 ]
            latest_recipe = row [ 'latest_recipe_id' ]
            current_recipe = row [ 'recipe_id' ]
            if ( current_recipe is not None and current_recipe != latest_recipe ) :
                state += ' stale_recipe'
            state_icon = state
            if state [ : 7 ] == "SUCCESS" and len ( state ) > 7 :
                state_icon = 'unknown'
            states . append ( { 'title' : row [ 'name' ] . decode ( 'utf-8' ) , 'authors' : format_authors ( row [ 'authors' ] ) , 'uuid' : row [ 'uuid' ] , 'print_style' : row [ 'print_style' ] , 'print_style_link' : request . route_path ( 'admin-print-style-single' , style = row [ 'print_style' ] ) , 'recipe' : row [ 'recipe' ] , 'recipe_name' : row [ 'recipe_name' ] , 'recipe_tag' : row [ 'recipe_tag' ] , 'recipe_link' : request . route_path ( 'get-resource' , hash = row [ 'recipe' ] ) , 'created' : row [ 'created' ] , 'state' : state , 'state_message' : message , 'state_icon' : state_icons . get ( state_icon , DEFAULT_ICON ) , 'status_link' : request . route_path ( 'admin-content-status-single' , uuid = row [ 'uuid' ] ) , 'content_link' : request . route_path ( 'get-content' , ident_hash = row [ 'ident_hash' ] ) } )
sort = request . params . get ( 'sort' , 'bpsa.created DESC' )
sort_match = SORTS_DICT [ sort . split ( ' ' ) [ 0 ] ]
sort_arrow = ARROW_MATCH [ sort . split ( ' ' ) [ 1 ] ]
if sort == "STATE ASC" :
    states . sort ( key = lambda x : x [ 'state' ] )
if sort == "STATE DESC" :
    states . sort ( key = lambda x : x [ 'state' ] , reverse = True )
num_entries = request . params . get ( 'number' , 100 ) or 100
page = request . params . get ( 'page' , 1 ) or 1
try :
    page = int ( page )
    num_entries = int ( num_entries )
    start_entry = ( page - 1 ) * num_entries
except ValueError :
    raise httpexceptions . HTTPBadRequest ( 'invalid page({}) or entries per page({})' . format ( page , num_entries ) )
total_entries = len ( states )
states = states [ start_entry : start_entry + num_entries ]
returns = sql_args
returns . update ( { 'start_entry' : start_entry , 'num_entries' : num_entries , 'page' : page , 'total_entries' : total_entries , 'states' : states , 'sort_' + sort_match : sort_arrow , 'sort' : sort , 'domain' : request . host , 'latest_only' : request . GET . get ( 'latest' , False ) , 'STATE_ICONS' : STATE_ICONS , 'status_filters' : status_filters or [ i [ 0 ] for i in STATE_ICONS ] } )
return returns


signature:
def admin_content_status_single ( request ) :
    """Returns a dictionary with all the past baking statuses of a single book ."""

implementation:
uuid = request . matchdict [ 'uuid' ]
try :
    UUID ( uuid )
except ValueError :
    raise httpexceptions . HTTPBadRequest ( '{} is not a valid uuid' . format ( uuid ) )
statement , sql_args = get_baking_statuses_sql ( { 'uuid' : uuid } )
with db_connect ( cursor_factory = DictCursor ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        cursor . execute ( statement , sql_args )
        modules = cursor . fetchall ( )
        if len ( modules ) == 0 :
            raise httpexceptions . HTTPBadRequest ( '{} is not a book' . format ( uuid ) )
        states = [ ]
        collection_info = modules [ 0 ]
        for row in modules :
            message = ''
            state = row [ 'state' ] or 'PENDING'
            if state == 'FAILURE' : # pragma : no cover
                if row [ 'traceback' ] is not None :
                    message = row [ 'traceback' ]
            latest_recipe = row [ 'latest_recipe_id' ]
            current_recipe = row [ 'recipe_id' ]
            if ( latest_recipe is not None and current_recipe != latest_recipe ) :
                state += ' stale_recipe'
            states . append ( { 'version' : row [ 'current_version' ] , 'recipe' : row [ 'recipe' ] , 'created' : str ( row [ 'created' ] ) , 'state' : state , 'state_message' : message , } )
return { 'uuid' : str ( collection_info [ 'uuid' ] ) , 'title' : collection_info [ 'name' ] . decode ( 'utf-8' ) , 'authors' : format_authors ( collection_info [ 'authors' ] ) , 'print_style' : collection_info [ 'print_style' ] , 'current_recipe' : collection_info [ 'recipe_id' ] , 'current_ident' : collection_info [ 'module_ident' ] , 'current_state' : states [ 0 ] [ 'state' ] , 'states' : states }


signature:
def admin_content_status_single_POST ( request ) :
    """Retriggers baking for a given book ."""

implementation:
args = admin_content_status_single ( request )
title = args [ 'title' ]
if args [ 'current_state' ] == 'SUCCESS' :
    args [ 'response' ] = title + ' is not stale, no need to bake'
    return args
with db_connect ( ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        cursor . execute ( "SELECT stateid FROM modules WHERE module_ident=%s" , vars = ( args [ 'current_ident' ] , ) )
        data = cursor . fetchall ( )
        if len ( data ) == 0 :
            raise httpexceptions . HTTPBadRequest ( 'invalid module_ident: {}' . format ( args [ 'current_ident' ] ) )
        if data [ 0 ] [ 0 ] == 5 or data [ 0 ] [ 0 ] == 6 :
            args [ 'response' ] = title + ' is already baking/set to bake'
            return args
        cursor . execute ( """UPDATE modules SET stateid=5
                           WHERE module_ident=%s""" , vars = ( args [ 'current_ident' ] , ) )
        args [ 'response' ] = title + " set to bake!"
return args


signature:
def _insert_optional_roles ( cursor , model , ident ) :
    """Inserts the optional roles if values for the optional roles
 exist ."""

implementation:
optional_roles = [ # ( < metadata - attr > , < db - role - id > , ) ,
( 'translators' , 4 , ) , ( 'editors' , 5 , ) , ]
for attr , role_id in optional_roles :
    roles = model . metadata . get ( attr )
    if not roles : # Bail out , no roles for this type .
        continue
    usernames = [ parse_user_uri ( x [ 'id' ] ) for x in roles ]
    cursor . execute ( """\
INSERT INTO moduleoptionalroles (module_ident, roleid, personids)
VALUES (%s, %s, %s)""" , ( ident , role_id , usernames , ) )


signature:
def _insert_metadata ( cursor , model , publisher , message ) :
    """Insert a module with the given ` ` metadata ` ` ."""

implementation:
params = model . metadata . copy ( )
params [ 'publisher' ] = publisher
params [ 'publication_message' ] = message
params [ '_portal_type' ] = _model_to_portaltype ( model )
params [ 'summary' ] = str ( cnxepub . DocumentSummaryFormatter ( model ) )
# Transform person structs to id lists for database array entry .
for person_field in ATTRIBUTED_ROLE_KEYS :
    params [ person_field ] = [ parse_user_uri ( x [ 'id' ] ) for x in params . get ( person_field , [ ] ) ]
params [ 'parent_ident_hash' ] = parse_parent_ident_hash ( model )
# Assign the id and version if one is known .
if model . ident_hash is not None :
    uuid , version = split_ident_hash ( model . ident_hash , split_version = True )
    params [ '_uuid' ] = uuid
    params [ '_major_version' ] , params [ '_minor_version' ] = version
    # Lookup legacy ` ` moduleid ` ` .
    cursor . execute ( "SELECT moduleid FROM latest_modules WHERE uuid = %s" , ( uuid , ) )
    # There is the chance that a uuid and version have been set ,
    # but a previous publication does not exist . Therefore the
    # moduleid will not be found . This happens on a pre - publication .
    try :
        moduleid = cursor . fetchone ( ) [ 0 ]
    except TypeError : # NoneType
        moduleid = None
    params [ '_moduleid' ] = moduleid
    # Verify that uuid is reserved in document _ contols . If not , add it .
    cursor . execute ( "SELECT * from document_controls where uuid = %s" , ( uuid , ) )
    try :
        cursor . fetchone ( ) [ 0 ]
    except TypeError : # NoneType
        cursor . execute ( "INSERT INTO document_controls (uuid) VALUES (%s)" , ( uuid , ) )
    created = model . metadata . get ( 'created' , None )
    # Format the statement to accept the identifiers .
    stmt = MODULE_INSERTION_TEMPLATE . format ( ** { '__uuid__' : "%(_uuid)s::uuid" , '__major_version__' : "%(_major_version)s" , '__minor_version__' : "%(_minor_version)s" , '__moduleid__' : moduleid is None and "DEFAULT" or "%(_moduleid)s" , '__created__' : created is None and "DEFAULT" or "%(created)s" , } )
else :
    created = model . metadata . get ( 'created' , None )
    # Format the statement for defaults .
    stmt = MODULE_INSERTION_TEMPLATE . format ( ** { '__uuid__' : "DEFAULT" , '__major_version__' : "DEFAULT" , '__minor_version__' : "DEFAULT" , '__moduleid__' : "DEFAULT" , '__created__' : created is None and "DEFAULT" or "%(created)s" , } )
# Insert the metadata
cursor . execute ( stmt , params )
module_ident , ident_hash = cursor . fetchone ( )
# Insert optional roles
_insert_optional_roles ( cursor , model , module_ident )
return module_ident , ident_hash


signature:
def _get_file_sha1 ( file ) :
    """Return the SHA1 hash of the given a file - like object as ` ` file ` ` .
 This will seek the file back to 0 when it ' s finished ."""

implementation:
bits = file . read ( )
file . seek ( 0 )
h = hashlib . new ( 'sha1' , bits ) . hexdigest ( )
return h


signature:
def _insert_file ( cursor , file , media_type ) :
    """Upsert the ` ` file ` ` and ` ` media _ type ` ` into the files table .
 Returns the ` ` fileid ` ` and ` ` sha1 ` ` of the upserted file ."""

implementation:
resource_hash = _get_file_sha1 ( file )
cursor . execute ( "SELECT fileid FROM files WHERE sha1 = %s" , ( resource_hash , ) )
try :
    fileid = cursor . fetchone ( ) [ 0 ]
except ( IndexError , TypeError ) :
    cursor . execute ( "INSERT INTO files (file, media_type) " "VALUES (%s, %s)" "RETURNING fileid" , ( psycopg2 . Binary ( file . read ( ) ) , media_type , ) )
    fileid = cursor . fetchone ( ) [ 0 ]
return fileid , resource_hash


signature:
def _insert_resource_file ( cursor , module_ident , resource ) :
    """Insert a resource into the modules _ files table . This will
 create a new file entry or associates an existing one ."""

implementation:
with resource . open ( ) as file :
    fileid , _ = _insert_file ( cursor , file , resource . media_type )
# Is this file legitimately used twice within the same content ?
cursor . execute ( """\
select
  (fileid = %s) as is_same_file
from module_files
where module_ident = %s and filename = %s""" , ( fileid , module_ident , resource . filename , ) )
try :
    is_same_file = cursor . fetchone ( ) [ 0 ]
except TypeError : # NoneType
    is_same_file = None
if is_same_file : # All is good , bail out .
    return
elif is_same_file is not None : # pragma : no cover
# This means the file is not the same , but a filename
# conflict exists .
# FFF At this time , it is impossible to get to this logic .
    raise Exception ( "filename conflict" )
args = ( module_ident , fileid , resource . filename , )
cursor . execute ( """\
INSERT INTO module_files (module_ident, fileid, filename)
VALUES (%s, %s, %s)""" , args )


signature:
def _insert_tree ( cursor , tree , parent_id = None , index = 0 , is_collated = False ) :
    """Inserts a binder tree into the archive ."""

implementation:
if isinstance ( tree , dict ) :
    if tree [ 'id' ] == 'subcol' :
        document_id = None
        title = tree [ 'title' ]
    else :
        cursor . execute ( """\
            SELECT module_ident, name
            FROM modules
            WHERE ident_hash(uuid,major_version,minor_version) = %s
            """ , ( tree [ 'id' ] , ) )
        try :
            document_id , document_title = cursor . fetchone ( )
        except TypeError : # NoneType
            raise ValueError ( "Missing published document for '{}'." . format ( tree [ 'id' ] ) )
        if tree . get ( 'title' , None ) :
            title = tree [ 'title' ]
        else :
            title = document_title
    # TODO We haven ' t settled on a flag ( name or value )
    # to pin the node to a specific version .
    is_latest = True
    cursor . execute ( TREE_NODE_INSERT , dict ( document_id = document_id , parent_id = parent_id , title = title , child_order = index , is_latest = is_latest , is_collated = is_collated ) )
    node_id = cursor . fetchone ( ) [ 0 ]
    if 'contents' in tree :
        _insert_tree ( cursor , tree [ 'contents' ] , parent_id = node_id , is_collated = is_collated )
elif isinstance ( tree , list ) :
    for tree_node in tree :
        _insert_tree ( cursor , tree_node , parent_id = parent_id , index = tree . index ( tree_node ) , is_collated = is_collated )


signature:
def publish_model ( cursor , model , publisher , message ) :
    """Publishes the ` ` model ` ` and return its ident _ hash ."""

implementation:
publishers = publisher
if isinstance ( publishers , list ) and len ( publishers ) > 1 :
    raise ValueError ( "Only one publisher is allowed. '{}' " "were given: {}" . format ( len ( publishers ) , publishers ) )
module_ident , ident_hash = _insert_metadata ( cursor , model , publisher , message )
for resource in getattr ( model , 'resources' , [ ] ) :
    _insert_resource_file ( cursor , module_ident , resource )
if isinstance ( model , Document ) :
    html = bytes ( cnxepub . DocumentContentFormatter ( model ) )
    sha1 = hashlib . new ( 'sha1' , html ) . hexdigest ( )
    cursor . execute ( "SELECT fileid FROM files WHERE sha1 = %s" , ( sha1 , ) )
    try :
        fileid = cursor . fetchone ( ) [ 0 ]
    except TypeError :
        file_args = { 'media_type' : 'text/html' , 'data' : psycopg2 . Binary ( html ) , }
        cursor . execute ( """\
            insert into files (file, media_type)
            VALUES (%(data)s, %(media_type)s)
            returning fileid""" , file_args )
        fileid = cursor . fetchone ( ) [ 0 ]
    args = { 'module_ident' : module_ident , 'filename' : 'index.cnxml.html' , 'fileid' : fileid , }
    cursor . execute ( """\
        INSERT INTO module_files
          (module_ident, fileid, filename)
        VALUES
          (%(module_ident)s, %(fileid)s, %(filename)s)""" , args )
elif isinstance ( model , Binder ) :
    tree = cnxepub . model_to_tree ( model )
    tree = _insert_tree ( cursor , tree )
return ident_hash


signature:
def publish_composite_model ( cursor , model , parent_model , publisher , message ) :
    """Publishes the ` ` model ` ` and return its ident _ hash ."""

implementation:
if not ( isinstance ( model , CompositeDocument ) or ( isinstance ( model , Binder ) and model . metadata . get ( 'type' ) == 'composite-chapter' ) ) :
    raise ValueError ( "This function only publishes Composite" "objects. '{}' was given." . format ( type ( model ) ) )
if issequence ( publisher ) and len ( publisher ) > 1 :
    raise ValueError ( "Only one publisher is allowed. '{}' " "were given: {}" . format ( len ( publisher ) , publisher ) )
module_ident , ident_hash = _insert_metadata ( cursor , model , publisher , message )
model . id , model . metadata [ 'version' ] = split_ident_hash ( ident_hash )
model . set_uri ( 'cnx-archive' , ident_hash )
for resource in model . resources :
    _insert_resource_file ( cursor , module_ident , resource )
if isinstance ( model , CompositeDocument ) :
    html = bytes ( cnxepub . DocumentContentFormatter ( model ) )
    fileid , _ = _insert_file ( cursor , io . BytesIO ( html ) , 'text/html' )
    file_arg = { 'module_ident' : module_ident , 'parent_ident_hash' : parent_model . ident_hash , 'fileid' : fileid , }
    cursor . execute ( """\
        INSERT INTO collated_file_associations
          (context, item, fileid)
        VALUES
          ((SELECT module_ident FROM modules
            WHERE ident_hash(uuid, major_version, minor_version)
           = %(parent_ident_hash)s),
            %(module_ident)s, %(fileid)s)""" , file_arg )
return ident_hash


signature:
def publish_collated_document ( cursor , model , parent_model ) :
    """Publish a given ` module ` ' s collated content in the context of
 the ` parent _ model ` . Note , the model ' s content is expected to already
 have the collated content . This will just persist that content to
 the archive ."""

implementation:
html = bytes ( cnxepub . DocumentContentFormatter ( model ) )
sha1 = hashlib . new ( 'sha1' , html ) . hexdigest ( )
cursor . execute ( "SELECT fileid FROM files WHERE sha1 = %s" , ( sha1 , ) )
try :
    fileid = cursor . fetchone ( ) [ 0 ]
except TypeError :
    file_args = { 'media_type' : 'text/html' , 'data' : psycopg2 . Binary ( html ) , }
    cursor . execute ( """\
        INSERT INTO files (file, media_type)
        VALUES (%(data)s, %(media_type)s)
        RETURNING fileid""" , file_args )
    fileid = cursor . fetchone ( ) [ 0 ]
args = { 'module_ident_hash' : model . ident_hash , 'parent_ident_hash' : parent_model . ident_hash , 'fileid' : fileid , }
stmt = """\
INSERT INTO collated_file_associations (context, item, fileid)
VALUES
  ((SELECT module_ident FROM modules
    WHERE ident_hash(uuid, major_version, minor_version)
   = %(parent_ident_hash)s),
   (SELECT module_ident FROM modules
    WHERE ident_hash(uuid, major_version, minor_version)
   = %(module_ident_hash)s),
   %(fileid)s)"""
cursor . execute ( stmt , args )


signature:
def publish_collated_tree ( cursor , tree ) :
    """Publish a given collated ` tree ` ( containing newly added
 ` CompositeDocument ` objects and number inforation )
 alongside the original tree ."""

implementation:
tree = _insert_tree ( cursor , tree , is_collated = True )
return tree


signature:
def republish_binders ( cursor , models ) :
    """Republish the Binders that share Documents in the publication context .
 This needs to be given all the models in the publication context ."""

implementation:
documents = set ( [ ] )
binders = set ( [ ] )
history_mapping = { }
# < previous - ident - hash > : < current - ident - hash >
if not isinstance ( models , ( list , tuple , set , ) ) :
    raise TypeError ( "``models`` Must be a sequence of model objects." "We were given: {}" . format ( models ) )
for model in models :
    if isinstance ( model , ( cnxepub . Binder , ) ) :
        binders . add ( split_ident_hash ( model . ident_hash ) [ 0 ] )
        for doc in cnxepub . flatten_to_documents ( model ) :
            documents . add ( split_ident_hash ( doc . ident_hash ) )
    else :
        documents . add ( split_ident_hash ( model . ident_hash ) )
to_be_republished = [ ]
# What binders are these documents a part of ?
for ( uuid , version ) in documents :
    ident_hash = join_ident_hash ( uuid , version )
    previous_ident_hash = get_previous_publication ( cursor , ident_hash )
    if previous_ident_hash is None : # Has no prior existence .
        continue
    else :
        history_mapping [ previous_ident_hash ] = ident_hash
    cursor . execute ( """\
WITH RECURSIVE t(nodeid, parent_id, documentid, path) AS (
  SELECT tr.nodeid, tr.parent_id, tr.documentid, ARRAY[tr.nodeid]
  FROM trees tr
  WHERE tr.documentid = (
    SELECT module_ident FROM modules
    WHERE ident_hash(uuid, major_version, minor_version) = %s)
UNION ALL
  SELECT c.nodeid, c.parent_id, c.documentid, path || ARRAY[c.nodeid]
  FROM trees c JOIN t ON (c.nodeid = t.parent_id)
  WHERE not c.nodeid = ANY(t.path)
)
SELECT ident_hash(uuid, major_version, minor_version)
FROM t JOIN latest_modules m ON (t.documentid = m.module_ident)
WHERE t.parent_id IS NULL
""" , ( previous_ident_hash , ) )
    to_be_republished . extend ( [ split_ident_hash ( x [ 0 ] ) for x in cursor . fetchall ( ) ] )
to_be_republished = set ( to_be_republished )
republished_ident_hashes = [ ]
# Republish the Collections set .
for ( uuid , version ) in to_be_republished :
    if uuid in binders : # This binder is already in the publication context ,
    # don ' t try to publish it again .
        continue
    ident_hash = join_ident_hash ( uuid , version )
    bumped_version = bump_version ( cursor , uuid , is_minor_bump = True )
    republished_ident_hash = republish_collection ( cursor , ident_hash , version = bumped_version )
    # Set the identifier history .
    history_mapping [ ident_hash ] = republished_ident_hash
    rebuild_collection_tree ( cursor , ident_hash , history_mapping )
    republished_ident_hashes . append ( republished_ident_hash )
return republished_ident_hashes


signature:
def get_previous_publication ( cursor , ident_hash ) :
    """Get the previous publication of the given
 publication as an ident - hash ."""

implementation:
cursor . execute ( """\
WITH contextual_module AS (
  SELECT uuid, module_ident
  FROM modules
  WHERE ident_hash(uuid, major_version, minor_version) = %s)
SELECT ident_hash(m.uuid, m.major_version, m.minor_version)
FROM modules AS m JOIN contextual_module AS context ON (m.uuid = context.uuid)
WHERE
  m.module_ident < context.module_ident
ORDER BY revised DESC
LIMIT 1""" , ( ident_hash , ) )
try :
    previous_ident_hash = cursor . fetchone ( ) [ 0 ]
except TypeError : # NoneType
    previous_ident_hash = None
return previous_ident_hash


signature:
def bump_version ( cursor , uuid , is_minor_bump = False ) :
    """Bump to the next version of the given content identified
 by ` ` uuid ` ` . Returns the next available version as a version tuple ,
 containing major and minor version .
 If ` ` is _ minor _ bump ` ` is ` ` True ` ` the version will minor bump . That is
 1.2 becomes 1.3 in the case of Collections . And 2 becomes 3 for
 Modules regardless of this option ."""

implementation:
cursor . execute ( """\
SELECT portal_type, major_version, minor_version
FROM latest_modules
WHERE uuid = %s::uuid""" , ( uuid , ) )
type_ , major_version , minor_version = cursor . fetchone ( )
incr = 1
if type_ == 'Collection' and is_minor_bump :
    minor_version = minor_version + incr
else :
    major_version = major_version + incr
return ( major_version , minor_version , )


signature:
def republish_collection ( cursor , ident_hash , version ) :
    """Republish the collection identified as ` ` ident _ hash ` ` with
 the given ` ` version ` ` ."""

implementation:
if not isinstance ( version , ( list , tuple , ) ) :
    split_version = version . split ( '.' )
    if len ( split_version ) == 1 :
        split_version . append ( None )
    version = tuple ( split_version )
major_version , minor_version = version
cursor . execute ( """\
WITH previous AS (
  SELECT module_ident
  FROM modules
  WHERE ident_hash(uuid, major_version, minor_version) = %s),
inserted AS (
  INSERT INTO modules
    (uuid, major_version, minor_version, revised,
     portal_type, moduleid,
     name, created, language,
     submitter, submitlog,
     abstractid, licenseid, parent, parentauthors,
     authors, maintainers, licensors,
     google_analytics, buylink,
     stateid, doctype)
  SELECT
    uuid, %s, %s, CURRENT_TIMESTAMP,
    portal_type, moduleid,
    name, created, language,
    submitter, submitlog,
    abstractid, licenseid, parent, parentauthors,
    authors, maintainers, licensors,
    google_analytics, buylink,
    stateid, doctype
  FROM modules AS m JOIN previous AS p ON (m.module_ident = p.module_ident)
  RETURNING
    ident_hash(uuid, major_version, minor_version) AS ident_hash,
    module_ident),
keywords AS (
  INSERT INTO modulekeywords (module_ident, keywordid)
  SELECT i.module_ident, keywordid
  FROM modulekeywords AS mk, inserted AS i, previous AS p
  WHERE mk.module_ident = p.module_ident),
tags AS (
  INSERT INTO moduletags (module_ident, tagid)
  SELECT i.module_ident, tagid
  FROM moduletags AS mt, inserted AS i, previous AS p
  WHERE mt.module_ident = p.module_ident)
SELECT ident_hash FROM inserted""" , ( ident_hash , major_version , minor_version , ) )
repub_ident_hash = cursor . fetchone ( ) [ 0 ]
return repub_ident_hash


signature:
def rebuild_collection_tree ( cursor , ident_hash , history_map ) :
    """Create a new tree for the collection based on the old tree but with
 new document ids"""

implementation:
collection_tree_sql = """\
WITH RECURSIVE t(nodeid, parent_id, documentid, title, childorder, latest,
                 ident_hash, path) AS (
  SELECT
    tr.nodeid, tr.parent_id, tr.documentid,
    tr.title, tr.childorder, tr.latest,
    (SELECT ident_hash(uuid, major_version, minor_version)
     FROM modules
     WHERE module_ident = tr.documentid) AS ident_hash,
    ARRAY[tr.nodeid]
  FROM trees AS tr
  WHERE tr.documentid = (
    SELECT module_ident
    FROM modules
    WHERE ident_hash(uuid, major_version, minor_version) = %s)
    AND tr.is_collated = FALSE
UNION ALL
  SELECT
    c.nodeid, c.parent_id, c.documentid, c.title, c.childorder, c.latest,
    (SELECT ident_hash(uuid, major_version, minor_version)
     FROM modules
     WHERE module_ident = c.documentid) AS ident_hash,
    path || ARRAY[c.nodeid]
  FROM trees AS c JOIN t ON (c.parent_id = t.nodeid)
  WHERE not c.nodeid = ANY(t.path) AND c.is_collated = FALSE
)
SELECT row_to_json(row) FROM (SELECT * FROM t) AS row"""
tree_insert_sql = """\
INSERT INTO trees
  (nodeid, parent_id,
   documentid,
   title, childorder, latest)
VALUES
  (DEFAULT, %(parent_id)s,
   (SELECT module_ident
    FROM modules
    WHERE ident_hash(uuid, major_version, minor_version) = \
          %(ident_hash)s),
   %(title)s, %(childorder)s, %(latest)s)
RETURNING nodeid"""
def get_tree ( ) :
    cursor . execute ( collection_tree_sql , ( ident_hash , ) )
    for row in cursor . fetchall ( ) :
        yield row [ 0 ]
def insert ( fields ) :
    cursor . execute ( tree_insert_sql , fields )
    results = cursor . fetchone ( ) [ 0 ]
    return results
tree = { }
# { < current - nodeid > : { < row - data > . . . } , . . . }
children = { }
# { < nodeid > : [ < child - nodeid > , . . . ] , < child - nodeid > : [ . . . ] }
for node in get_tree ( ) :
    tree [ node [ 'nodeid' ] ] = node
    children . setdefault ( node [ 'parent_id' ] , [ ] )
    children [ node [ 'parent_id' ] ] . append ( node [ 'nodeid' ] )
def build_tree ( nodeid , parent_id ) :
    data = tree [ nodeid ]
    data [ 'parent_id' ] = parent_id
    if history_map . get ( data [ 'ident_hash' ] ) is not None and ( data [ 'latest' ] or parent_id is None ) :
        data [ 'ident_hash' ] = history_map [ data [ 'ident_hash' ] ]
    new_nodeid = insert ( data )
    for child_nodeid in children . get ( nodeid , [ ] ) :
        build_tree ( child_nodeid , new_nodeid )
root_node = children [ None ] [ 0 ]
build_tree ( root_node , None )


signature:
def publish ( request ) :
    """Accept a publication request at form value ' epub '"""

implementation:
if 'epub' not in request . POST :
    raise httpexceptions . HTTPBadRequest ( "Missing EPUB in POST body." )
is_pre_publication = asbool ( request . POST . get ( 'pre-publication' ) )
epub_upload = request . POST [ 'epub' ] . file
try :
    epub = cnxepub . EPUB . from_file ( epub_upload )
except : # noqa : E722
    raise httpexceptions . HTTPBadRequest ( 'Format not recognized.' )
# Make a publication entry in the database for status checking
# the publication . This also creates publication entries for all
# of the content in the EPUB .
with db_connect ( ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        epub_upload . seek ( 0 )
        publication_id , publications = add_publication ( cursor , epub , epub_upload , is_pre_publication )
# Poke at the publication & lookup its state .
state , messages = poke_publication_state ( publication_id )
response_data = { 'publication' : publication_id , 'mapping' : publications , 'state' : state , 'messages' : messages , }
return response_data


signature:
def get_publication ( request ) :
    """Lookup publication state"""

implementation:
publication_id = request . matchdict [ 'id' ]
state , messages = check_publication_state ( publication_id )
response_data = { 'publication' : publication_id , 'state' : state , 'messages' : messages , }
return response_data


signature:
def get_accept_license ( request ) :
    """This produces JSON data for a user ( at ` ` uid ` ` ) to view the license ( s )
 they have accepted or will need to accept for a publication ( at ` ` id ` ` ) ."""

implementation:
publication_id = request . matchdict [ 'id' ]
user_id = request . matchdict [ 'uid' ]
# FIXME Is this an active publication ?
# TODO Verify the accepting user is the one making the request .
# For each pending document , accept the license .
with db_connect ( ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        cursor . execute ( """
SELECT row_to_json(combined_rows) FROM (
SELECT
  pd.uuid AS id,
  ident_hash(pd.uuid, pd.major_version, pd.minor_version) \
    AS ident_hash,
  accepted AS is_accepted
FROM
  pending_documents AS pd
  NATURAL JOIN license_acceptances AS la
WHERE pd.publication_id = %s AND user_id = %s
) as combined_rows;""" , ( publication_id , user_id ) )
        user_documents = [ r [ 0 ] for r in cursor . fetchall ( ) ]
return { 'publication_id' : publication_id , 'user_id' : user_id , 'documents' : user_documents , }


signature:
def post_accept_license ( request ) :
    """Allows the user ( at ` ` uid ` ` ) to accept the license ( s ) for
 a publication ( at ` ` id ` ` ) ."""

implementation:
publication_id = request . matchdict [ 'id' ]
uid = request . matchdict [ 'uid' ]
# TODO Verify the accepting user is the one making the request .
# They could be authenticated but not be the license acceptor .
post_data = request . json
accepted = [ ]
denied = [ ]
try :
    documents = post_data [ 'documents' ]
    for doc_acceptance in documents :
        if doc_acceptance [ 'is_accepted' ] is None :
            continue
        elif doc_acceptance [ 'is_accepted' ] :
            accepted . append ( doc_acceptance [ 'id' ] )
        else :
            denied . append ( doc_acceptance [ 'id' ] )
except KeyError :
    raise httpexceptions . BadRequest ( "Posted data is invalid." )
# For each pending document , accept / deny the license .
with db_connect ( ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        accept_publication_license ( cursor , publication_id , uid , accepted , True )
        accept_publication_license ( cursor , publication_id , uid , denied , False )
location = request . route_url ( 'publication-license-acceptance' , id = publication_id , uid = uid )
# Poke publication to change state .
poke_publication_state ( publication_id )
return httpexceptions . HTTPFound ( location = location )


signature:
def bake_content ( request ) :
    """Invoke the baking process - trigger post - publication"""

implementation:
ident_hash = request . matchdict [ 'ident_hash' ]
try :
    id , version = split_ident_hash ( ident_hash )
except IdentHashError :
    raise httpexceptions . HTTPNotFound ( )
if not version :
    raise httpexceptions . HTTPBadRequest ( 'must specify the version' )
with db_connect ( ) as db_conn :
    with db_conn . cursor ( ) as cursor :
        cursor . execute ( """\
SELECT bool(portal_type = 'Collection'), stateid, module_ident
FROM modules
WHERE ident_hash(uuid, major_version, minor_version) = %s
""" , ( ident_hash , ) )
        try :
            is_binder , stateid , module_ident = cursor . fetchone ( )
        except TypeError :
            raise httpexceptions . HTTPNotFound ( )
        if not is_binder :
            raise httpexceptions . HTTPBadRequest ( '{} is not a book' . format ( ident_hash ) )
        if stateid == 5 :
            cursor . execute ( """\
SELECT pg_notify('post_publication',
'{"module_ident": '||%s||',
  "ident_hash": "'||%s||'",
  "timestamp": "'||CURRENT_TIMESTAMP||'"}')
""" , ( module_ident , ident_hash ) )
        else :
            cursor . execute ( """\
UPDATE modules SET stateid = 5
WHERE ident_hash(uuid, major_version, minor_version) = %s
""" , ( ident_hash , ) )


signature:
def includeme ( config ) :
    """Configures the caching manager"""

implementation:
global cache_manager
settings = config . registry . settings
cache_manager = CacheManager ( ** parse_cache_config_options ( settings ) )


signature:
def create_pg_notify_event ( notif ) :
    """A factory for creating a Postgres Notification Event
 ( an object inheriting from ` cnxpublishing . events . PGNotifyEvent ` )
 given ` notif ` , a ` psycopg2 . extensions . Notify ` object ."""

implementation:
# TODO Lookup registered events via getAllUtilitiesRegisteredFor
# for class mapping .
if notif . channel not in _CHANNEL_MAPPER :
    cls = _CHANNEL_MAPPER [ None ]
else :
    cls = _CHANNEL_MAPPER [ notif . channel ]
return cls ( notif )


signature:
def create_map ( self , type_from , type_to , mapping = None ) :
    """Method for adding mapping definitions
 : param type _ from : source type
 : param type _ to : target type
 : param mapping : dictionary of mapping definitions in a form { ' target _ property _ name ' ,
 lambda function from rhe source }
 : return : None"""

implementation:
key_from = type_from . __name__
key_to = type_to . __name__
if mapping is None :
    mapping = { }
if key_from in self . mappings :
    inner_map = self . mappings [ key_from ]
    if key_to in inner_map :
        raise ObjectMapperException ( "Mapping for {0} -> {1} already exists" . format ( key_from , key_to ) )
    else :
        inner_map [ key_to ] = mapping
else :
    self . mappings [ key_from ] = { }
    self . mappings [ key_from ] [ key_to ] = mapping


signature:
def map ( self , from_obj , to_type , ignore_case = False , allow_none = False , excluded = None ) :
    """Method for creating target object instance
 : param from _ obj : source object to be mapped from
 : param to _ type : target type
 : param ignore _ case : if set to true , ignores attribute case when performing the mapping
 : param allow _ none : if set to true , returns None if the source object is None ; otherwise throws an exception
 : param excluded : A list of fields to exclude when performing the mapping
 : return : Instance of the target class with mapped attributes"""

implementation:
if ( from_obj is None ) and allow_none :
    return None
else : # one of the tests is explicitly checking for an attribute error on _ _ dict _ _ if it ' s not set
    from_obj . __dict__
inst = to_type ( )
key_from = from_obj . __class__ . __name__
key_to = to_type . __name__
def not_private ( s ) :
    return not s . startswith ( '_' )
def not_excluded ( s ) :
    return not ( excluded and s in excluded )
from_obj_attributes = getmembers ( from_obj , lambda a : not isroutine ( a ) )
from_obj_dict = { k : v for k , v in from_obj_attributes if not_private ( k ) and not_excluded ( k ) }
to_obj_attributes = getmembers ( inst , lambda a : not isroutine ( a ) )
to_obj_dict = { k : v for k , v in to_obj_attributes if not_private ( k ) }
if ignore_case :
    from_props = CaseDict ( from_obj_dict )
    to_props = CaseDict ( to_obj_dict )
else :
    from_props = from_obj_dict
    to_props = to_obj_dict
for prop in to_props :
    if self . mappings is not None and key_from in self . mappings and key_to in self . mappings [ key_from ] :
        if prop in self . mappings [ key_from ] [ key_to ] : # take mapping function
            try :
                fnc = self . mappings [ key_from ] [ key_to ] [ prop ]
                if fnc is not None :
                    setattr ( inst , prop , fnc ( from_obj ) )
                    # none suppress mapping
            except Exception :
                raise ObjectMapperException ( "Invalid mapping function while setting property {0}.{1}" . format ( inst . __class__ . __name__ , prop ) )
        else : # try find property with the same name in the source
            if prop in from_props :
                setattr ( inst , prop , from_props [ prop ] )
                # case when target attribute is not mapped ( can be extended )
    else :
        raise ObjectMapperException ( "No mapping defined for {0} -> {1}" . format ( key_from , key_to ) )
return inst


signature:
def get ( self , key , default = _sentinel ) :
    """Gets the value from the key .
 If the key doesn ' t exist , the default value is returned , otherwise None .
 : param key : The key
 : param default : The default value
 : return : The value"""

implementation:
tup = self . _data . get ( key . lower ( ) )
if tup is not None :
    return tup [ 1 ]
elif default is not _sentinel :
    return default
else :
    return None


signature:
def pop ( self , key , default = _sentinel ) :
    """Removes the specified key and returns the corresponding value .
 If key is not found , the default is returned if given , otherwise KeyError is raised .
 : param key : The key
 : param default : The default value
 : return : The value"""

implementation:
if default is not _sentinel :
    tup = self . _data . pop ( key . lower ( ) , default )
else :
    tup = self . _data . pop ( key . lower ( ) )
if tup is not default :
    return tup [ 1 ]
else :
    return default


signature:
def reversals ( series , left = False , right = False ) :
    """Iterate reversal points in the series .
 A reversal point is a point in the series at which the first derivative
 changes sign . Reversal is undefined at the first ( last ) point because the
 derivative before ( after ) this point is undefined . The first and the last
 points may be treated as reversals by setting the optional parameters
 ` left ` and ` right ` to True .
 Parameters
 series : iterable sequence of numbers
 left : bool , optional
 If True , yield the first point in the series ( treat it as a reversal ) .
 right : bool , optional
 If True , yield the last point in the series ( treat it as a reversal ) .
 Yields
 float
 Reversal points ."""

implementation:
series = iter ( series )
x_last , x = next ( series ) , next ( series )
d_last = ( x - x_last )
if left :
    yield x_last
for x_next in series :
    if x_next == x :
        continue
    d_next = x_next - x
    if d_last * d_next < 0 :
        yield x
    x_last , x = x , x_next
    d_last = d_next
if right :
    yield x_next


signature:
def _sort_lows_and_highs ( func ) :
    "Decorator for extract _ cycles"

implementation:
@ functools . wraps ( func )
def wrapper ( * args , ** kwargs ) :
    for low , high , mult in func ( * args , ** kwargs ) :
        if low < high :
            yield low , high , mult
        else :
            yield high , low , mult
return wrapper


signature:
def extract_cycles ( series , left = False , right = False ) :
    """Iterate cycles in the series .
 Parameters
 series : iterable sequence of numbers
 left : bool , optional
 If True , treat the first point in the series as a reversal .
 right : bool , optional
 If True , treat the last point in the series as a reversal .
 Yields
 cycle : tuple
 Each tuple contains three floats ( low , high , mult ) , where low and high
 define cycle amplitude and mult equals to 1.0 for full cycles and 0.5
 for half cycles ."""

implementation:
points = deque ( )
for x in reversals ( series , left = left , right = right ) :
    points . append ( x )
    while len ( points ) >= 3 : # Form ranges X and Y from the three most recent points
        X = abs ( points [ - 2 ] - points [ - 1 ] )
        Y = abs ( points [ - 3 ] - points [ - 2 ] )
        if X < Y : # Read the next point
            break
        elif len ( points ) == 3 : # Y contains the starting point
        # Count Y as one - half cycle and discard the first point
            yield points [ 0 ] , points [ 1 ] , 0.5
            points . popleft ( )
        else : # Count Y as one cycle and discard the peak and the valley of Y
            yield points [ - 3 ] , points [ - 2 ] , 1.0
            last = points . pop ( )
            points . pop ( )
            points . pop ( )
            points . append ( last )
else : # Count the remaining ranges as one - half cycles
    while len ( points ) > 1 :
        yield points [ 0 ] , points [ 1 ] , 0.5
        points . popleft ( )


signature:
def count_cycles ( series , ndigits = None , left = False , right = False ) :
    """Count cycles in the series .
 Parameters
 series : iterable sequence of numbers
 ndigits : int , optional
 Round cycle magnitudes to the given number of digits before counting .
 left : bool , optional
 If True , treat the first point in the series as a reversal .
 right : bool , optional
 If True , treat the last point in the series as a reversal .
 Returns
 A sorted list containing pairs of cycle magnitude and count .
 One - half cycles are counted as 0.5 , so the returned counts may not be
 whole numbers ."""

implementation:
counts = defaultdict ( float )
round_ = _get_round_function ( ndigits )
for low , high , mult in extract_cycles ( series , left = left , right = right ) :
    delta = round_ ( abs ( high - low ) )
    counts [ delta ] += mult
return sorted ( counts . items ( ) )


signature:
def render ( node , strict = False ) :
    """Recipe to render a given FST node .
 The FST is composed of branch nodes which are either lists or dicts
 and of leaf nodes which are strings . Branch nodes can have other
 list , dict or leaf nodes as childs .
 To render a string , simply output it . To render a list , render each
 of its elements in order . To render a dict , you must follow the
 node ' s entry in the nodes _ rendering _ order dictionary and its
 dependents constraints .
 This function hides all this algorithmic complexity by returning
 a structured rendering recipe , whatever the type of node . But even
 better , you should subclass the RenderWalker which simplifies
 drastically working with the rendered FST .
 The recipe is a list of steps , each step correspond to a child and is actually a 3 - uple composed of the following fields :
 - ` key _ type ` is a string determining the type of the child in the second field ( ` item ` ) of the tuple . It can be one of :
 - ' constant ' : the child is a string
 - ' node ' : the child is a dict
 - ' key ' : the child is an element of a dict
 - ' list ' : the child is a list
 - ' formatting ' : the child is a list specialized in formatting
 - ` item ` is the child itself : either a string , a dict or a list .
 - ` render _ key ` gives the key used to access this child from the parent node . It ' s a string if the node is a dict or a number if its a list .
 Please note that " bool " ` key _ types ` are never rendered , that ' s why
 they are not shown here ."""

implementation:
if isinstance ( node , list ) :
    return render_list ( node )
elif isinstance ( node , dict ) :
    return render_node ( node , strict = strict )
else :
    raise NotImplementedError ( "You tried to render a %s. Only list and dicts can be rendered." % node . __class__ . __name__ )


signature:
def path_to_node ( tree , path ) :
    """FST node located at the given path"""

implementation:
if path is None :
    return None
node = tree
for key in path :
    node = child_by_key ( node , key )
return node


signature:
def before_constant ( self , constant , key ) :
    """Determine if we ' re on the targetted node .
 If the targetted column is reached , ` stop ` and ` path _ found ` are
 set . If the targetted line is passed , only ` stop ` is set . This
 prevents unnecessary tree travelling when the targetted column
 is out of bounds ."""

implementation:
newlines_split = split_on_newlines ( constant )
for c in newlines_split :
    if is_newline ( c ) :
        self . current . advance_line ( )
        # if target line is passed
        if self . current . line > self . target . line :
            return self . STOP
    else :
        advance_by = len ( c )
        if self . is_on_targetted_node ( advance_by ) :
            self . found_path = deepcopy ( self . current_path )
            return self . STOP
        self . current . advance_columns ( advance_by )


signature:
def get_prefix ( multicodec ) :
    """Returns prefix for a given multicodec
 : param str multicodec : multicodec codec name
 : return : the prefix for the given multicodec
 : rtype : byte
 : raises ValueError : if an invalid multicodec name is provided"""

implementation:
try :
    prefix = varint . encode ( NAME_TABLE [ multicodec ] )
except KeyError :
    raise ValueError ( '{} multicodec is not supported.' . format ( multicodec ) )
return prefix


signature:
def add_prefix ( multicodec , bytes_ ) :
    """Adds multicodec prefix to the given bytes input
 : param str multicodec : multicodec to use for prefixing
 : param bytes bytes _ : data to prefix
 : return : prefixed byte data
 : rtype : bytes"""

implementation:
prefix = get_prefix ( multicodec )
return b'' . join ( [ prefix , bytes_ ] )


signature:
def remove_prefix ( bytes_ ) :
    """Removes prefix from a prefixed data
 : param bytes bytes _ : multicodec prefixed data bytes
 : return : prefix removed data bytes
 : rtype : bytes"""

implementation:
prefix_int = extract_prefix ( bytes_ )
prefix = varint . encode ( prefix_int )
return bytes_ [ len ( prefix ) : ]


signature:
def get_codec ( bytes_ ) :
    """Gets the codec used for prefix the multicodec prefixed data
 : param bytes bytes _ : multicodec prefixed data bytes
 : return : name of the multicodec used to prefix
 : rtype : str"""

implementation:
prefix = extract_prefix ( bytes_ )
try :
    return CODE_TABLE [ prefix ]
except KeyError :
    raise ValueError ( 'Prefix {} not present in the lookup table' . format ( prefix ) )


signature:
def capture ( target_url , user_agent = "archiveis (https://github.com/pastpages/archiveis)" , proxies = { } ) :
    """Archives the provided URL using archive . is
 Returns the URL where the capture is stored ."""

implementation:
# Put together the URL that will save our request
domain = "http://archive.vn"
save_url = urljoin ( domain , "/submit/" )
# Configure the request headers
headers = { 'User-Agent' : user_agent , "host" : "archive.vn" , }
# Request a unique identifier for our activity
logger . debug ( "Requesting {}" . format ( domain + "/" ) )
get_kwargs = dict ( timeout = 120 , allow_redirects = True , headers = headers , )
if proxies :
    get_kwargs [ 'proxies' ] = proxies
response = requests . get ( domain + "/" , ** get_kwargs )
response . raise_for_status ( )
# It will need to be parsed from the homepage response headers
html = str ( response . content )
try :
    unique_id = html . split ( 'name="submitid' , 1 ) [ 1 ] . split ( 'value="' , 1 ) [ 1 ] . split ( '"' , 1 ) [ 0 ]
    logger . debug ( "Unique identifier: {}" . format ( unique_id ) )
except IndexError :
    logger . warn ( "Unable to extract unique identifier from archive.is. Submitting without it." )
    unique_id = None
# Send the capture request to archive . is with the unique id included
data = { "url" : target_url , "anyway" : 1 , }
if unique_id :
    data . update ( { "submitid" : unique_id } )
post_kwargs = dict ( timeout = 120 , allow_redirects = True , headers = headers , data = data )
if proxies :
    post_kwargs [ 'proxies' ] = proxies
logger . debug ( "Requesting {}" . format ( save_url ) )
response = requests . post ( save_url , ** post_kwargs )
response . raise_for_status ( )
# There are a couple ways the header can come back
if 'Refresh' in response . headers :
    memento = str ( response . headers [ 'Refresh' ] ) . split ( ';url=' ) [ 1 ]
    logger . debug ( "Memento from Refresh header: {}" . format ( memento ) )
    return memento
if 'Location' in response . headers :
    memento = response . headers [ 'Location' ]
    logger . debug ( "Memento from Location header: {}" . format ( memento ) )
    return memento
logger . debug ( "Memento not found in response headers. Inspecting history." )
for i , r in enumerate ( response . history ) :
    logger . debug ( "Inspecting history request #{}" . format ( i ) )
    logger . debug ( r . headers )
    if 'Location' in r . headers :
        memento = r . headers [ 'Location' ]
        logger . debug ( "Memento from the Location header of {} history response: {}" . format ( i + 1 , memento ) )
        return memento
# If there ' s nothing at this point , throw an error
logger . error ( "No memento returned by archive.is" )
logger . error ( "Status code: {}" . format ( response . status_code ) )
logger . error ( response . headers )
logger . error ( response . text )
raise Exception ( "No memento returned by archive.is" )


signature:
def cli ( url , user_agent ) :
    """Archives the provided URL using archive . is ."""

implementation:
kwargs = { }
if user_agent :
    kwargs [ 'user_agent' ] = user_agent
archive_url = capture ( url , ** kwargs )
click . echo ( archive_url )


signature:
def get_channel_image ( self , channel , img_size = 300 , skip_cache = False ) :
    """Get the logo for a channel"""

implementation:
from bs4 import BeautifulSoup
from wikipedia . exceptions import PageError
import re
import wikipedia
wikipedia . set_lang ( 'fr' )
if not channel :
    _LOGGER . error ( 'Channel is not set. Could not retrieve image.' )
    return
# Check if the image is in cache
if channel in self . _cache_channel_img and not skip_cache :
    img = self . _cache_channel_img [ channel ]
    _LOGGER . debug ( 'Cache hit: %s -> %s' , channel , img )
    return img
channel_info = self . get_channel_info ( channel )
query = channel_info [ 'wiki_page' ]
if not query :
    _LOGGER . debug ( 'Wiki page is not set for channel %s' , channel )
    return
_LOGGER . debug ( 'Query: %s' , query )
# If there is a max image size defined use it .
if 'max_img_size' in channel_info :
    if img_size > channel_info [ 'max_img_size' ] :
        _LOGGER . info ( 'Requested image size is bigger than the max, ' 'setting it to %s' , channel_info [ 'max_img_size' ] )
        img_size = channel_info [ 'max_img_size' ]
try :
    page = wikipedia . page ( query )
    _LOGGER . debug ( 'Wikipedia article title: %s' , page . title )
    soup = BeautifulSoup ( page . html ( ) , 'html.parser' )
    images = soup . find_all ( 'img' )
    img_src = None
    for i in images :
        if i [ 'alt' ] . startswith ( 'Image illustrative' ) :
            img_src = re . sub ( r'\d+px' , '{}px' . format ( img_size ) , i [ 'src' ] )
    img = 'https:{}' . format ( img_src ) if img_src else None
    # Cache result
    self . _cache_channel_img [ channel ] = img
    return img
except PageError :
    _LOGGER . error ( 'Could not fetch channel image for %s' , channel )


signature:
def press_key ( self , key , mode = 0 ) :
    '''modes :
 0 - > simple press
 1 - > long press
 2 - > release after long press'''

implementation:
if isinstance ( key , str ) :
    assert key in KEYS , 'No such key: {}' . format ( key )
    key = KEYS [ key ]
_LOGGER . info ( 'Press key %s' , self . __get_key_name ( key ) )
return self . rq ( '01' , OrderedDict ( [ ( 'key' , key ) , ( 'mode' , mode ) ] ) )


signature:
def forwards ( self , orm ) :
    "Write your forwards methods here ."

implementation:
# Note : Remember to use orm [ ' appname . ModelName ' ] rather than " from appname . models . . . "
for translation in orm [ 'people.PersonTranslation' ] . objects . all ( ) :
    if translation . language in [ 'en' , 'de' ] :
        translation . roman_first_name = translation . first_name
        translation . roman_last_name = translation . last_name
    else :
        translation . non_roman_first_name = translation . first_name
        translation . non_roman_last_name = translation . last_name
    translation . save ( )


implementation:
for translation in orm [ 'people.PersonTranslation' ] . objects . all ( ) :
    translation . person . roman_first_name = translation . roman_first_name
    translation . person . roman_last_name = translation . roman_last_name
    translation . person . non_roman_first_name = translation . non_roman_first_name
    translation . person . non_roman_last_name = translation . non_roman_last_name
    translation . person . save ( )


signature:
def parse ( self , scope ) :
    """Parse block node .
 args :
 scope ( Scope ) : Current scope
 raises :
 SyntaxError
 returns :
 self"""

implementation:
if not self . parsed :
    scope . push ( )
    self . name , inner = self . tokens
    scope . current = self . name
    scope . real . append ( self . name )
    if not self . name . parsed :
        self . name . parse ( scope )
    if not inner :
        inner = [ ]
    inner = list ( utility . flatten ( [ p . parse ( scope ) for p in inner if p ] ) )
    self . parsed = [ ]
    self . inner = [ ]
    if not hasattr ( self , "inner_media_queries" ) :
        self . inner_media_queries = [ ]
    for p in inner :
        if p is not None :
            if isinstance ( p , Block ) :
                if ( len ( scope ) == 2 and p . tokens [ 1 ] is not None ) :
                    p_is_mediaquery = p . name . tokens [ 0 ] == '@media'
                    # Inner block @ media . . . { . . . } is a nested media
                    # query . But double - nested media queries have to be
                    # removed and marked as well . While parsing " . foo " ,
                    # both nested " @ media print " and double - nested
                    # " @ media all " will be handled as we have to
                    # re - arrange the scope and block layout quite a bit :
                    # . foo {
                    # @ media print {
                    # color : blue ;
                    # @ media screen { font - size : 12em ; }
                    # Expected result :
                    # @ media print {
                    # . foo { color : blue ; }
                    # @ media print and screen {
                    # . foo { font - size : 12 em ; }
                    append_list = [ ]
                    reparse_p = False
                    for child in p . tokens [ 1 ] :
                        if isinstance ( child , Block ) and child . name . raw ( ) . startswith ( "@media" ) : # Remove child from the nested media query , it will be re - added to
                        # the parent with ' merged ' media query ( see above example ) .
                            p . tokens [ 1 ] . remove ( child )
                            if p_is_mediaquery : # Media query inside a & block
                            # Double - nested media query found . We remove it from ' p ' and add
                            # it to this block with a new ' name ' .
                                reparse_p = True
                                part_a = p . name . tokens [ 2 : ] [ 0 ] [ 0 ] [ 0 ]
                                part_b = child . name . tokens [ 2 : ] [ 0 ] [ 0 ]
                                new_ident_tokens = [ '@media' , ' ' , [ part_a , ( ' ' , 'and' , ' ' ) , part_b ] ]
                                # Parse child again with new @ media $ BLA { } part
                                child . tokens [ 0 ] = Identifier ( new_ident_tokens )
                                child . parsed = None
                                child = child . parse ( scope )
                            else :
                                child . block_name = p . name
                            append_list . append ( child )
                        if reparse_p :
                            p . parsed = None
                            p = p . parse ( scope )
                    if not p_is_mediaquery and not append_list :
                        self . inner . append ( p )
                    else :
                        append_list . insert ( 0 , p )
                        # This media query should occur before it ' s children
                        for media_query in append_list :
                            self . inner_media_queries . append ( media_query )
                    # NOTE ( saschpe ) : The code is not recursive but we hope that people
                    # wont use triple - nested media queries .
                else :
                    self . inner . append ( p )
            else :
                self . parsed . append ( p )
    if self . inner_media_queries : # Nested media queries , we have to remove self from scope and
    # push all nested @ media . . . { } blocks .
        scope . remove_block ( self , index = - 2 )
        for mb in self . inner_media_queries : # New inner block with current name and media block contents
            if hasattr ( mb , 'block_name' ) :
                cb_name = mb . block_name
            else :
                cb_name = self . tokens [ 0 ]
            cb = Block ( [ cb_name , mb . tokens [ 1 ] ] ) . parse ( scope )
            # Replace inner block contents with new block
            new_mb = Block ( [ mb . tokens [ 0 ] , [ cb ] ] ) . parse ( scope )
            self . inner . append ( new_mb )
            scope . add_block ( new_mb )
    scope . real . pop ( )
    scope . pop ( )
return self


signature:
def raw ( self , clean = False ) :
    """Raw block name
 args :
 clean ( bool ) : clean name
 returns :
 str"""

implementation:
try :
    return self . tokens [ 0 ] . raw ( clean )
except ( AttributeError , TypeError ) :
    pass


signature:
def fmt ( self , fills ) :
    """Format block ( CSS )
 args :
 fills ( dict ) : Fill elements
 returns :
 str ( CSS )"""

implementation:
f = "%(identifier)s%(ws)s{%(nl)s%(proplist)s}%(eb)s"
out = [ ]
name = self . name . fmt ( fills )
if self . parsed and any ( p for p in self . parsed if str ( type ( p ) ) != "<class 'lesscpy.plib.variable.Variable'>" ) :
    fills . update ( { 'identifier' : name , 'proplist' : '' . join ( [ p . fmt ( fills ) for p in self . parsed if p ] ) , } )
    out . append ( f % fills )
if hasattr ( self , 'inner' ) :
    if self . name . subparse and len ( self . inner ) > 0 : # @ media
        inner = '' . join ( [ p . fmt ( fills ) for p in self . inner ] )
        inner = inner . replace ( fills [ 'nl' ] , fills [ 'nl' ] + fills [ 'tab' ] ) . rstrip ( fills [ 'tab' ] )
        if not fills [ 'nl' ] :
            inner = inner . strip ( )
        fills . update ( { 'identifier' : name , 'proplist' : fills [ 'tab' ] + inner } )
        out . append ( f % fills )
    else :
        out . append ( '' . join ( [ p . fmt ( fills ) for p in self . inner ] ) )
return '' . join ( out )


signature:
def copy ( self ) :
    """Return a full copy of self
 returns : Block object"""

implementation:
name , inner = self . tokens
if inner :
    inner = [ u . copy ( ) if u else u for u in inner ]
if name :
    name = name . copy ( )
return Block ( [ name , inner ] , 0 )


signature:
def copy_inner ( self , scope ) :
    """Copy block contents ( properties , inner blocks ) .
 Renames inner block from current scope .
 Used for mixins .
 args :
 scope ( Scope ) : Current scope
 returns :
 list ( block contents )"""

implementation:
if self . tokens [ 1 ] :
    tokens = [ u . copy ( ) if u else u for u in self . tokens [ 1 ] ]
    out = [ p for p in tokens if p ]
    utility . rename ( out , scope , Block )
    return out
return None


signature:
def parse ( self , scope ) :
    """Parse node
 args :
 scope ( Scope ) : current scope
 raises :
 SyntaxError
 returns :
 self"""

implementation:
self . parsed = list ( utility . flatten ( self . tokens ) )
if self . parsed [ 0 ] == '@import' :
    if len ( self . parsed ) > 4 : # Media @ import
        self . parsed . insert ( 3 , ' ' )
return self


signature:
def parse ( self , scope , error = False , depth = 0 ) :
    """Parse function . We search for mixins
 first within current scope then fallback
 to global scope . The special scope . deferred
 is used when local scope mixins are called
 within parent mixins .
 If nothing is found we fallback to block - mixin
 as lessc . js allows calls to blocks and mixins to
 be inter - changable .
 clx : This method is a HACK that stems from
 poor design elsewhere . I will fix it
 when I have more time .
 args :
 scope ( Scope ) : Current scope
 returns :
 mixed"""

implementation:
res = False
ident , args = self . tokens
ident . parse ( scope )
mixins = scope . mixins ( ident . raw ( ) )
if not mixins :
    ident . parse ( None )
    mixins = scope . mixins ( ident . raw ( ) )
if depth > 64 :
    raise SyntaxError ( 'NameError `%s`' % ident . raw ( True ) )
if not mixins :
    if scope . deferred :
        store = [ t for t in scope . deferred . parsed [ - 1 ] ]
        i = 0
        while scope . deferred . parsed [ - 1 ] :
            scope . current = scope . deferred
            ident . parse ( scope )
            mixins = scope . mixins ( ident . raw ( ) )
            scope . current = None
            if mixins or i > 64 :
                break
            scope . deferred . parsed [ - 1 ] . pop ( )
            i += 1
        scope . deferred . parsed [ - 1 ] = store
if not mixins : # Fallback to blocks
    block = scope . blocks ( ident . raw ( ) )
    if not block :
        ident . parse ( None )
        block = scope . blocks ( ident . raw ( ) )
    if block :
        scope . current = scope . real [ - 1 ] if scope . real else None
        res = block . copy_inner ( scope )
        scope . current = None
if mixins :
    for mixin in mixins :
        scope . current = scope . real [ - 1 ] if scope . real else None
        res = mixin . call ( scope , args )
        if res : # Add variables to scope to support
        # closures
            [ scope . add_variable ( v ) for v in mixin . vars ]
            scope . deferred = ident
            break
if res :
    store = [ t for t in scope . deferred . parsed [ - 1 ] ] if scope . deferred else False
    tmp_res = [ ]
    for p in res :
        if p :
            if isinstance ( p , Deferred ) :
                tmp_res . append ( p . parse ( scope , depth = depth + 1 ) )
            else :
                tmp_res . append ( p . parse ( scope ) )
    res = tmp_res
    # res = [ p . parse ( scope , depth = depth + 1 ) for p in res if p ]
    while ( any ( t for t in res if isinstance ( t , Deferred ) ) ) :
        res = [ p . parse ( scope ) for p in res if p ]
    if store :
        scope . deferred . parsed [ - 1 ] = store
if error and not res :
    raise SyntaxError ( 'NameError `%s`' % ident . raw ( True ) )
return res


signature:
def ldirectory ( inpath , outpath , args , scope ) :
    """Compile all * . less files in directory
 Args :
 inpath ( str ) : Path to compile
 outpath ( str ) : Output directory
 args ( object ) : Argparse Object
 scope ( Scope ) : Scope object or None"""

implementation:
yacctab = 'yacctab' if args . debug else None
if not outpath :
    sys . exit ( "Compile directory option needs -o ..." )
else :
    if not os . path . isdir ( outpath ) :
        if args . verbose :
            print ( "Creating '%s'" % outpath , file = sys . stderr )
        if not args . dry_run :
            os . mkdir ( outpath )
less = glob . glob ( os . path . join ( inpath , '*.less' ) )
f = formatter . Formatter ( args )
for lf in less :
    outf = os . path . splitext ( os . path . basename ( lf ) )
    minx = '.min' if args . min_ending else ''
    outf = "%s/%s%s.css" % ( outpath , outf [ 0 ] , minx )
    if not args . force and os . path . exists ( outf ) :
        recompile = os . path . getmtime ( outf ) < os . path . getmtime ( lf )
    else :
        recompile = True
    if recompile :
        print ( '%s -> %s' % ( lf , outf ) )
        p = parser . LessParser ( yacc_debug = ( args . debug ) , lex_optimize = True , yacc_optimize = ( not args . debug ) , scope = scope , tabfile = yacctab , verbose = args . verbose )
        p . parse ( filename = lf , debuglevel = 0 )
        css = f . format ( p )
        if not args . dry_run :
            with open ( outf , 'w' ) as outfile :
                outfile . write ( css )
    elif args . verbose :
        print ( 'skipping %s, not modified' % lf , file = sys . stderr )
    sys . stdout . flush ( )
if args . recurse :
    [ ldirectory ( os . path . join ( inpath , name ) , os . path . join ( outpath , name ) , args , scope ) for name in os . listdir ( inpath ) if os . path . isdir ( os . path . join ( inpath , name ) ) and not name . startswith ( '.' ) and not name == outpath ]


signature:
def run ( ) :
    """Run compiler"""

implementation:
aparse = argparse . ArgumentParser ( description = 'LessCss Compiler' , epilog = '<< jtm@robot.is @_o >>' )
aparse . add_argument ( '-v' , '--version' , action = 'version' , version = VERSION_STR )
aparse . add_argument ( '-I' , '--include' , action = "store" , type = str , help = "Included less-files (comma separated)" )
aparse . add_argument ( '-V' , '--verbose' , action = "store_true" , default = False , help = "Verbose mode" )
aparse . add_argument ( '-C' , '--dont_create_dirs' , action = "store_true" , default = False , help = "Creates directories when outputing files (lessc non-compatible)" )
fgroup = aparse . add_argument_group ( 'Formatting options' )
fgroup . add_argument ( '-x' , '--minify' , action = "store_true" , default = False , help = "Minify output" )
fgroup . add_argument ( '-X' , '--xminify' , action = "store_true" , default = False , help = "Minify output, no end of block newlines" )
fgroup . add_argument ( '-t' , '--tabs' , help = "Use tabs" , action = "store_true" )
fgroup . add_argument ( '-s' , '--spaces' , help = "Number of startline spaces (default 2)" , default = 2 )
dgroup = aparse . add_argument_group ( 'Directory options' , 'Compiles all *.less files in directory that ' 'have a newer timestamp than it\'s css file.' )
dgroup . add_argument ( '-o' , '--out' , action = "store" , help = "Output directory" )
dgroup . add_argument ( '-r' , '--recurse' , action = "store_true" , help = "Recursive into subdirectorys" )
dgroup . add_argument ( '-f' , '--force' , action = "store_true" , help = "Force recompile on all files" )
dgroup . add_argument ( '-m' , '--min-ending' , action = "store_true" , default = False , help = "Add '.min' into output filename. eg, name.min.css" )
dgroup . add_argument ( '-D' , '--dry-run' , action = "store_true" , default = False , help = "Dry run, do not write files" )
group = aparse . add_argument_group ( 'Debugging' )
group . add_argument ( '-g' , '--debug' , action = "store_true" , default = False , help = "Debugging information" )
group . add_argument ( '-S' , '--scopemap' , action = "store_true" , default = False , help = "Scopemap" )
group . add_argument ( '-L' , '--lex-only' , action = "store_true" , default = False , help = "Run lexer on target" )
group . add_argument ( '-N' , '--no-css' , action = "store_true" , default = False , help = "No css output" )
aparse . add_argument ( 'target' , help = "less file or directory" )
aparse . add_argument ( 'output' , nargs = '?' , help = "output file path" )
args = aparse . parse_args ( )
try :
    if args . lex_only :
        lex = lexer . LessLexer ( )
        ll = lex . file ( args . target )
        while True :
            tok = ll . token ( )
            if not tok :
                break
            if hasattr ( tok , "lexer" ) : # literals don ' t have the lexer attribute
                print ( tok , "State:" , tok . lexer . lexstate )
            else :
                print ( tok )
        print ( 'EOF' )
        sys . exit ( )
    yacctab = 'yacctab' if args . debug else None
    scope = None
    if args . include :
        for u in args . include . split ( ',' ) :
            if os . path . exists ( u ) :
                p = parser . LessParser ( yacc_debug = ( args . debug ) , lex_optimize = True , yacc_optimize = ( not args . debug ) , tabfile = yacctab , verbose = args . verbose )
                p . parse ( filename = u , debuglevel = args . debug )
                if not scope :
                    scope = p . scope
                else :
                    scope . update ( p . scope )
            else :
                sys . exit ( 'included file `%s` not found ...' % u )
            sys . stdout . flush ( )
    p = None
    f = formatter . Formatter ( args )
    if not os . path . exists ( args . target ) :
        sys . exit ( "Target not found '%s' ..." % args . target )
    if os . path . isdir ( args . target ) :
        ldirectory ( args . target , args . out , args , scope )
        if args . dry_run :
            print ( 'Dry run, nothing done.' , file = sys . stderr )
    else :
        p = parser . LessParser ( yacc_debug = ( args . debug ) , lex_optimize = True , yacc_optimize = ( not args . debug ) , scope = copy . deepcopy ( scope ) , verbose = args . verbose )
        p . parse ( filename = args . target , debuglevel = args . debug )
        if args . scopemap :
            args . no_css = True
            p . scopemap ( )
        if not args . no_css and p :
            out = f . format ( p )
            if args . output :
                if not args . dont_create_dirs and not os . path . exists ( os . path . dirname ( args . output ) ) :
                    try :
                        os . makedirs ( os . path . dirname ( args . output ) )
                    except OSError as exc : # Guard against race condition
                        if exc . errno != errno . EEXIST :
                            raise
                with open ( args . output , "w" ) as f :
                    f . write ( out )
            else :
                print ( out )
except ( KeyboardInterrupt , SystemExit , IOError ) :
    sys . exit ( '\nAborting...' )


signature:
def parse ( self , scope ) :
    """Parse node .
 args :
 scope ( Scope ) : Current scope
 raises :
 SyntaxError
 returns :
 self"""

implementation:
self . keyframe , = [ e [ 0 ] if isinstance ( e , tuple ) else e for e in self . tokens if str ( e ) . strip ( ) ]
self . subparse = False
return self


signature:
def file ( self , filename ) :
    """Lex file ."""

implementation:
with open ( filename ) as f :
    self . lexer . input ( f . read ( ) )
return self


signature:
def input ( self , file ) :
    """Load lexer with content from ` file ` which can be a path or a file
 like object ."""

implementation:
if isinstance ( file , string_types ) :
    with open ( file ) as f :
        self . lexer . input ( f . read ( ) )
else :
    self . lexer . input ( file . read ( ) )


signature:
def token ( self ) :
    """Token function . Contains 2 hacks :
 1 . Injects ' ; ' into blocks where the last property
 leaves out the ;
 2 . Strips out whitespace from nonsignificant locations
 to ease parsing ."""

implementation:
if self . next_ :
    t = self . next_
    self . next_ = None
    return t
while True :
    t = self . lexer . token ( )
    if not t :
        return t
    if t . type == 't_ws' and ( self . pretok or ( self . last and self . last . type not in self . significant_ws ) ) :
        continue
    self . pretok = False
    if t . type == 't_bclose' and self . last and self . last . type not in [ 't_bopen' , 't_bclose' ] and self . last . type != 't_semicolon' and not ( hasattr ( t , 'lexer' ) and ( t . lexer . lexstate == 'escapequotes' or t . lexer . lexstate == 'escapeapostrophe' ) ) :
        self . next_ = t
        tok = lex . LexToken ( )
        tok . type = 't_semicolon'
        tok . value = ';'
        tok . lineno = t . lineno
        tok . lexpos = t . lexpos
        self . last = tok
        self . lexer . in_property_decl = False
        return tok
    self . last = t
    break
return t


signature:
def parse ( self , scope ) :
    """Parse node . Block identifiers are stored as
 strings with spaces replaced with ?
 args :
 scope ( Scope ) : Current scope
 raises :
 SyntaxError
 returns :
 self"""

implementation:
names = [ ]
name = [ ]
self . _subp = ( '@media' , '@keyframes' , '@-moz-keyframes' , '@-webkit-keyframes' , '@-ms-keyframes' )
if self . tokens and hasattr ( self . tokens , 'parse' ) :
    self . tokens = list ( utility . flatten ( [ id . split ( ) + [ ',' ] for id in self . tokens . parse ( scope ) . split ( ',' ) ] ) )
    self . tokens . pop ( )
if self . tokens and any ( hasattr ( t , 'parse' ) for t in self . tokens ) :
    tmp_tokens = [ ]
    for t in self . tokens :
        if hasattr ( t , 'parse' ) :
            tmp_tokens . append ( t . parse ( scope ) )
        else :
            tmp_tokens . append ( t )
    self . tokens = list ( utility . flatten ( tmp_tokens ) )
if self . tokens and self . tokens [ 0 ] in self . _subp :
    name = list ( utility . flatten ( self . tokens ) )
    self . subparse = True
else :
    self . subparse = False
    for n in utility . flatten ( self . tokens ) :
        if n == '*' :
            name . append ( '* ' )
        elif n in '>+~' :
            if name and name [ - 1 ] == ' ' :
                name . pop ( )
            name . append ( '?%s?' % n )
        elif n == ',' :
            names . append ( name )
            name = [ ]
        else :
            name . append ( n )
names . append ( name )
parsed = self . root ( scope , names ) if scope else names
# Interpolated selectors need another step , we have to replace variables . Avoid reserved words though
# Example : ' . @ { var } ' results in [ [ ' . ' , ' @ { var } ' ] ]
# But : ' @ media print ' results in [ [ ' @ media ' , ' ' , ' print ' ] ]
def replace_variables ( tokens , scope ) :
    return [ scope . swap ( t ) if ( utility . is_variable ( t ) and not t in reserved . tokens ) else t for t in tokens ]
parsed = [ list ( utility . flatten ( replace_variables ( part , scope ) ) ) for part in parsed ]
self . parsed = [ [ i for i , j in utility . pairwise ( part ) if i != ' ' or ( j and '?' not in j ) ] for part in parsed ]
return self