signature:
def shellcmd ( repo , args ) :
    """Run a shell command within the repo ' s context
 Parameters
 repo : Repository object
 args : Shell command"""

implementation:
with cd ( repo . rootdir ) :
    result = run ( args )
    return result


signature:
def datapackage_exists ( repo ) :
    """Check if the datapackage exists . . ."""

implementation:
datapath = os . path . join ( repo . rootdir , "datapackage.json" )
return os . path . exists ( datapath )


signature:
def delete ( repo , args = [ ] ) :
    """Delete files
 Parameters
 repo : Repository object
 args : Arguments to git command"""

implementation:
# Remove the files
result = generic_repo_cmd ( repo , 'delete' , args )
if result [ 'status' ] != 'success' :
    return status
with cd ( repo . rootdir ) :
    package = repo . package
    resources = package [ 'resources' ]
    cleaned_resources = [ ]
    for r in resources :
        relativepath = r [ 'relativepath' ]
        sha256 = r [ 'sha256' ]
        if relativepath not in [ '' , None ] :
            if not os . path . exists ( relativepath ) : # This file does not exist on disk .
                print ( "Skipping" , relativepath )
                continue
        cleaned_resources . append ( r )
    package [ 'resources' ] = cleaned_resources
    repo . package = package
    with open ( 'datapackage.json' , 'w' ) as fd :
        fd . write ( json . dumps ( repo . package , indent = 4 ) )
    return { 'status' : 'success' , 'message' : '' }


signature:
def bootstrap_datapackage ( repo , force = False , options = None , noinput = False ) :
    """Create the datapackage file . ."""

implementation:
print ( "Bootstrapping datapackage" )
# get the directory
tsprefix = datetime . now ( ) . date ( ) . isoformat ( )
# Initial data package json
package = OrderedDict ( [ ( 'title' , '' ) , ( 'description' , '' ) , ( 'username' , repo . username ) , ( 'reponame' , repo . reponame ) , ( 'name' , str ( repo ) ) , ( 'title' , "" ) , ( 'description' , "" ) , ( 'keywords' , [ ] ) , ( 'resources' , [ ] ) , ( 'creator' , getpass . getuser ( ) ) , ( 'createdat' , datetime . now ( ) . isoformat ( ) ) , ( 'remote-url' , repo . remoteurl ) ] )
if options is not None :
    package [ 'title' ] = options [ 'title' ]
    package [ 'description' ] = options [ 'description' ]
else :
    if noinput :
        raise IncompleteParameters ( "Option field with title and description" )
    for var in [ 'title' , 'description' ] :
        value = ''
        while value in [ '' , None ] :
            value = input ( 'Your Repo ' + var . title ( ) + ": " )
            if len ( value ) == 0 :
                print ( "{} cannot be empty. Please re-enter." . format ( var . title ( ) ) )
        package [ var ] = value
# Now store the package . . .
( handle , filename ) = tempfile . mkstemp ( )
with open ( filename , 'w' ) as fd :
    fd . write ( json . dumps ( package , indent = 4 ) )
repo . package = package
return filename


signature:
def init ( username , reponame , setup , force = False , options = None , noinput = False ) :
    """Initialize an empty repository with datapackage . json
 Parameters
 username : Name of the user
 reponame : Name of the repo
 setup : Specify the ' configuration ' ( git only , git + s3 backend etc )
 force : Force creation of the files
 options : Dictionary with content of dgit . json , if available .
 noinput : Automatic operation with no human interaction"""

implementation:
mgr = plugins_get_mgr ( )
repomgr = mgr . get ( what = 'repomanager' , name = 'git' )
backendmgr = None
if setup == 'git+s3' :
    backendmgr = mgr . get ( what = 'backend' , name = 's3' )
repo = repomgr . init ( username , reponame , force , backendmgr )
# Now bootstrap the datapackage . json metadata file and copy it in . . .
# Insert a gitignore with . dgit directory in the repo . This
# directory will be used to store partial results
( handle , gitignore ) = tempfile . mkstemp ( )
with open ( gitignore , 'w' ) as fd :
    fd . write ( ".dgit" )
# Try to bootstrap . If you cant , cleanup and return
try :
    filename = bootstrap_datapackage ( repo , force , options , noinput )
except Exception as e :
    repomgr . drop ( repo , [ ] )
    os . unlink ( gitignore )
    raise e
repo . run ( 'add_files' , [ { 'relativepath' : 'datapackage.json' , 'localfullpath' : filename , } , { 'relativepath' : '.gitignore' , 'localfullpath' : gitignore , } , ] )
# Cleanup temp files
os . unlink ( filename )
os . unlink ( gitignore )
args = [ '-a' , '-m' , 'Bootstrapped the repo' ]
repo . run ( 'commit' , args )
return repo


signature:
def clone ( url ) :
    """Clone a URL . Examples include :
 - git @ github . com : pingali / dgit . git
 - https : / / github . com : pingali / dgit . git
 - s3 : / / mybucket / git / pingali / dgit . git
 Parameters
 url : URL of the repo"""

implementation:
backend = None
backendmgr = None
if url . startswith ( 's3' ) :
    backendtype = 's3'
elif url . startswith ( "http" ) or url . startswith ( "git" ) :
    backendtype = 'git'
else :
    backendtype = None
mgr = plugins_get_mgr ( )
repomgr = mgr . get ( what = 'repomanager' , name = 'git' )
backendmgr = mgr . get ( what = 'backend' , name = backendtype )
# print ( " Testing { } with backend { } " . format ( url , backendmgr ) )
if backendmgr is not None and not backendmgr . url_is_valid ( url ) :
    raise InvalidParameters ( "Invalid URL" )
key = repomgr . clone ( url , backendmgr )
# Insert a datapackage if it doesnt already exist . . .
repo = repomgr . lookup ( key = key )
if not datapackage_exists ( repo ) :
    filename = bootstrap_datapackage ( repo )
    repo . run ( 'add_files' , [ { 'relativepath' : 'datapackage.json' , 'localfullpath' : filename , } , ] )
    os . unlink ( filename )
    args = [ '-a' , '-m' , 'Bootstrapped the repo' ]
    repo . run ( 'commit' , args )
return repo


signature:
def annotate_metadata_data ( repo , task , patterns = [ "*" ] , size = 0 ) :
    """Update metadata with the content of the files"""

implementation:
mgr = plugins_get_mgr ( )
keys = mgr . search ( 'representation' ) [ 'representation' ]
representations = [ mgr . get_by_key ( 'representation' , k ) for k in keys ]
matching_files = repo . find_matching_files ( patterns )
package = repo . package
rootdir = repo . rootdir
files = package [ 'resources' ]
for f in files :
    relativepath = f [ 'relativepath' ]
    if relativepath in matching_files :
        path = os . path . join ( rootdir , relativepath )
        if task == 'preview' :
            print ( "Adding preview for " , relativepath )
            f [ 'content' ] = open ( path ) . read ( ) [ : size ]
        elif task == 'schema' :
            for r in representations :
                if r . can_process ( path ) :
                    print ( "Adding schema for " , path )
                    f [ 'schema' ] = r . get_schema ( path )
                    break


signature:
def annotate_metadata_code ( repo , files ) :
    """Update metadata with the commit information"""

implementation:
package = repo . package
package [ 'code' ] = [ ]
for p in files :
    matching_files = glob2 . glob ( "**/{}" . format ( p ) )
    for f in matching_files :
        absf = os . path . abspath ( f )
        print ( "Add commit data for {}" . format ( f ) )
        package [ 'code' ] . append ( OrderedDict ( [ ( 'script' , f ) , ( 'permalink' , repo . manager . permalink ( repo , absf ) ) , ( 'mimetypes' , mimetypes . guess_type ( absf ) [ 0 ] ) , ( 'sha256' , compute_sha256 ( absf ) ) ] ) )


signature:
def annotate_metadata_action ( repo ) :
    """Update metadata with the action history"""

implementation:
package = repo . package
print ( "Including history of actions" )
with cd ( repo . rootdir ) :
    filename = ".dgit/log.json"
    if os . path . exists ( filename ) :
        history = open ( filename ) . readlines ( )
        actions = [ ]
        for a in history :
            try :
                a = json . loads ( a )
                for x in [ 'code' ] :
                    if x not in a or a [ x ] == None :
                        a [ x ] = "..."
                actions . append ( a )
            except :
                pass
        package [ 'actions' ] = actions


signature:
def annotate_metadata_platform ( repo ) :
    """Update metadata host information"""

implementation:
print ( "Added platform information" )
package = repo . package
mgr = plugins_get_mgr ( )
repomgr = mgr . get ( what = 'instrumentation' , name = 'platform' )
package [ 'platform' ] = repomgr . get_metadata ( )


signature:
def annotate_metadata_dependencies ( repo ) :
    """Collect information from the dependent repo ' s"""

implementation:
options = repo . options
if 'dependencies' not in options :
    print ( "No dependencies" )
    return [ ]
repos = [ ]
dependent_repos = options [ 'dependencies' ]
for d in dependent_repos :
    if "/" not in d :
        print ( "Invalid dependency specification" )
    ( username , reponame ) = d . split ( "/" )
    try :
        repos . append ( repo . manager . lookup ( username , reponame ) )
    except :
        print ( "Repository does not exist. Please create one" , d )
package = repo . package
package [ 'dependencies' ] = [ ]
for r in repos :
    package [ 'dependencies' ] . append ( { 'username' : r . username , 'reponame' : r . reponame , } )


signature:
def post ( repo , args = [ ] ) :
    """Post to metadata server
 Parameters
 repo : Repository object ( result of lookup )"""

implementation:
mgr = plugins_get_mgr ( )
keys = mgr . search ( what = 'metadata' )
keys = keys [ 'metadata' ]
if len ( keys ) == 0 :
    return
# Incorporate pipeline information . . .
if 'pipeline' in repo . options :
    for name , details in repo . options [ 'pipeline' ] . items ( ) :
        patterns = details [ 'files' ]
        matching_files = repo . find_matching_files ( patterns )
        matching_files . sort ( )
        details [ 'files' ] = matching_files
        for i , f in enumerate ( matching_files ) :
            r = repo . get_resource ( f )
            if 'pipeline' not in r :
                r [ 'pipeline' ] = [ ]
            r [ 'pipeline' ] . append ( name + " [Step {}]" . format ( i ) )
if 'metadata-management' in repo . options :
    print ( "Collecting all the required metadata to post" )
    metadata = repo . options [ 'metadata-management' ]
    # Add data repo history
    if 'include-data-history' in metadata and metadata [ 'include-data-history' ] :
        repo . package [ 'history' ] = get_history ( repo . rootdir )
    # Add action history
    if 'include-action-history' in metadata and metadata [ 'include-action-history' ] :
        annotate_metadata_action ( repo )
    # Add data repo history
    if 'include-preview' in metadata :
        annotate_metadata_data ( repo , task = 'preview' , patterns = metadata [ 'include-preview' ] [ 'files' ] , size = metadata [ 'include-preview' ] [ 'length' ] )
    if ( ( 'include-schema' in metadata ) and metadata [ 'include-schema' ] ) :
        annotate_metadata_data ( repo , task = 'schema' )
    if 'include-code-history' in metadata :
        annotate_metadata_code ( repo , files = metadata [ 'include-code-history' ] )
    if 'include-platform' in metadata :
        annotate_metadata_platform ( repo )
    if 'include-validation' in metadata :
        annotate_metadata_validation ( repo )
    if 'include-dependencies' in metadata :
        annotate_metadata_dependencies ( repo )
    history = repo . package . get ( 'history' , None )
    if ( ( 'include-tab-diffs' in metadata ) and metadata [ 'include-tab-diffs' ] and history is not None ) :
        annotate_metadata_diffs ( repo )
    # Insert options as well
    repo . package [ 'config' ] = repo . options
try :
    for k in keys : # print ( " Key " , k )
        metadatamgr = mgr . get_by_key ( 'metadata' , k )
        url = metadatamgr . url
        o = urlparse ( url )
        print ( "Posting to " , o . netloc )
        response = metadatamgr . post ( repo )
        if isinstance ( response , str ) :
            print ( "Error while posting:" , response )
        elif response . status_code in [ 400 ] :
            content = response . json ( )
            print ( "Error while posting:" )
            for k in content :
                print ( "   " , k , "- " , "," . join ( content [ k ] ) )
except NetworkError as e :
    print ( "Unable to reach metadata server!" )
except NetworkInvalidConfiguration as e :
    print ( "Invalid network configuration in the INI file" )
    print ( e . message )
except Exception as e :
    print ( "Could not post. Unknown error" )
    print ( e )


signature:
def plugins_show ( what = None , name = None , version = None , details = False ) :
    """Show details of available plugins
 Parameters
 what : Class of plugins e . g . , backend
 name : Name of the plugin e . g . , s3
 version : Version of the plugin
 details : Show details be shown ?"""

implementation:
global pluginmgr
return pluginmgr . show ( what , name , version , details )


signature:
def discover_all_plugins ( self ) :
    """Load all plugins from dgit extension"""

implementation:
for v in pkg_resources . iter_entry_points ( 'dgit.plugins' ) :
    m = v . load ( )
    m . setup ( self )


signature:
def register ( self , what , obj ) :
    """Registering a plugin
 Params
 what : Nature of the plugin ( backend , instrumentation , repo )
 obj : Instance of the plugin"""

implementation:
# print ( " Registering pattern " , name , pattern )
name = obj . name
version = obj . version
enable = obj . enable
if enable == 'n' :
    return
key = Key ( name , version )
self . plugins [ what ] [ key ] = obj


signature:
def search ( self , what , name = None , version = None ) :
    """Search for a plugin"""

implementation:
filtered = { }
# The search may for a scan ( what is None ) or
if what is None :
    whats = list ( self . plugins . keys ( ) )
elif what is not None :
    if what not in self . plugins :
        raise Exception ( "Unknown class of plugins" )
    whats = [ what ]
for what in whats :
    if what not in filtered :
        filtered [ what ] = [ ]
    for key in self . plugins [ what ] . keys ( ) :
        ( k_name , k_version ) = key
        if name is not None and k_name != name :
            continue
        if version is not None and k_version != version :
            continue
        if self . plugins [ what ] [ key ] . enable == 'n' :
            continue
        filtered [ what ] . append ( key )
# print ( filtered )
return filtered


signature:
def gather_configs ( self ) :
    """Gather configuration requirements of all plugins"""

implementation:
configs = [ ]
for what in self . order :
    for key in self . plugins [ what ] :
        mgr = self . plugins [ what ] [ key ]
        c = mgr . config ( what = 'get' )
        if c is not None :
            c . update ( { 'description' : mgr . description } )
            # print ( " Gathering configuration from " , c )
            configs . append ( c )
return configs


signature:
def update_configs ( self , config ) :
    """Gather configuration requirements of all plugins"""

implementation:
for what in self . plugins : # backend , repo etc .
    for key in self . plugins [ what ] : # s3 , filesystem etc .
    # print ( " Updating configuration of " , what , key )
        self . plugins [ what ] [ key ] . config ( what = 'set' , params = config )
return


signature:
def run ( self ) :
    """Receives the serial data into the self . _ raw buffer
 : return :"""

implementation:
run_once = True
while ( run_once or self . _threaded ) and self . end is False :
    self . service_tx_queue ( )
    self . parse_messages ( )
    run_once = False
    if self . _threaded :
        time . sleep ( self . _timeout )
if self . _threaded :
    logger . info ( 'bootloader thread complete' )


signature:
def instantiate ( repo , validator_name = None , filename = None , rulesfiles = None ) :
    """Instantiate the validation specification"""

implementation:
default_validators = repo . options . get ( 'validator' , { } )
validators = { }
if validator_name is not None : # Handle the case validator is specified . .
    if validator_name in default_validators :
        validators = { validator_name : default_validators [ validator_name ] }
    else :
        validators = { validator_name : { 'files' : [ ] , 'rules' : { } , 'rules-files' : [ ] } }
else :
    validators = default_validators
# Insert the file names
if filename is not None :
    matching_files = repo . find_matching_files ( [ filename ] )
    if len ( matching_files ) == 0 :
        print ( "Filename could not be found" , filename )
        raise Exception ( "Invalid filename pattern" )
    for v in validators :
        validators [ v ] [ 'files' ] = matching_files
else : # Instantiate the files from the patterns specified
    for v in validators :
        if 'files' not in validators [ v ] :
            validators [ v ] [ 'files' ] = [ ]
        elif len ( validators [ v ] [ 'files' ] ) > 0 :
            matching_files = repo . find_matching_files ( validators [ v ] [ 'files' ] )
            validators [ v ] [ 'files' ] = matching_files
# Insert the rules files . .
if rulesfiles is not None : # Command lines . . .
    matching_files = repo . find_matching_files ( [ rulesfiles ] )
    if len ( matching_files ) == 0 :
        print ( "Could not find matching rules files ({}) for {}" . format ( rulesfiles , v ) )
        raise Exception ( "Invalid rules" )
    for v in validators :
        validators [ v ] [ 'rules-files' ] = matching_files
else : # Instantiate the files from the patterns specified
    for v in validators :
        if 'rules-files' not in validators [ v ] :
            validators [ v ] [ 'rules-files' ] = [ ]
        else :
            rulesfiles = validators [ v ] [ 'rules-files' ]
            matching_files = repo . find_matching_files ( rulesfiles )
            validators [ v ] [ 'rules-files' ] = matching_files
return validators


signature:
def validate ( repo , validator_name = None , filename = None , rulesfiles = None , args = [ ] ) :
    """Validate the content of the files for consistency . Validators can
 look as deeply as needed into the files . dgit treats them all as
 black boxes .
 Parameters
 repo : Repository object
 validator _ name : Name of validator , if any . If none , then all validators specified in dgit . json will be included .
 filename : Pattern that specifies files that must be processed by the validators selected . If none , then the default specification in dgit . json is used .
 rules : Pattern specifying the files that have rules that validators will use
 show : Print the validation results on the terminal
 Returns
 status : A list of dictionaries , each with target file processed , rules file applied , status of the validation and any error message ."""

implementation:
mgr = plugins_get_mgr ( )
# Expand the specification . Now we have full file paths
validator_specs = instantiate ( repo , validator_name , filename , rulesfiles )
# Run the validators with rules files . . .
allresults = [ ]
for v in validator_specs :
    keys = mgr . search ( what = 'validator' , name = v ) [ 'validator' ]
    for k in keys :
        validator = mgr . get_by_key ( 'validator' , k )
        result = validator . evaluate ( repo , validator_specs [ v ] , args )
        allresults . extend ( result )
return allresults


signature:
def url_is_valid ( self , url ) :
    """Check if a URL exists"""

implementation:
# Check if the file system path exists . . .
if url . startswith ( "file://" ) :
    url = url . replace ( "file://" , "" )
return os . path . exists ( url )


signature:
def post ( self , repo ) :
    """Post to the metadata server
 Parameters
 repo"""

implementation:
datapackage = repo . package
url = self . url
token = self . token
headers = { 'Authorization' : 'Token {}' . format ( token ) , 'Content-Type' : 'application/json' }
try :
    r = requests . post ( url , data = json . dumps ( datapackage ) , headers = headers )
    return r
except Exception as e : # print ( e )
# traceback . print _ exc ( )
    raise NetworkError ( )
return ""


signature:
def get_module_class ( class_path ) :
    """imports and returns module class from ` ` path . to . module . Class ` `
 argument"""

implementation:
mod_name , cls_name = class_path . rsplit ( '.' , 1 )
try :
    mod = import_module ( mod_name )
except ImportError as ex :
    raise EvoStreamException ( 'Error importing module %s: ' '"%s"' % ( mod_name , ex ) )
return getattr ( mod , cls_name )


signature:
def find_executable_files ( ) :
    """Find max 5 executables that are responsible for this repo ."""

implementation:
files = glob . glob ( "*" ) + glob . glob ( "*/*" ) + glob . glob ( '*/*/*' )
files = filter ( lambda f : os . path . isfile ( f ) , files )
executable = stat . S_IEXEC | stat . S_IXGRP | stat . S_IXOTH
final = [ ]
for filename in files :
    if os . path . isfile ( filename ) :
        st = os . stat ( filename )
        mode = st . st_mode
        if mode & executable :
            final . append ( filename )
            if len ( final ) > 5 :
                break
return final


signature:
def auto_init ( autofile , force_init = False ) :
    """Initialize a repo - specific configuration file to execute dgit
 Parameters
 autofile : Repo - specific configuration file ( dgit . json )
 force _ init : Flag to force to re - initialization of the configuration file"""

implementation:
if os . path . exists ( autofile ) and not force_init :
    try :
        autooptions = json . loads ( open ( autofile ) . read ( ) )
        return autooptions
    except :
        print ( "Error in dgit.json configuration file" )
        traceback . print_exc ( )
        raise Exception ( "Invalid configuration file" )
config = get_config ( )
pluginmgr = plugins_get_mgr ( )
print ( "Repo configuration file missing or corrupted. Creating one" )
print ( "Let us know a few details about your data repository" )
# Get the username
username = getpass . getuser ( )
revised = input ( "Please specify username [{}]" . format ( username ) )
if revised not in [ "" , None ] :
    username = revised
# Get the reponame
thisdir = os . path . abspath ( os . getcwd ( ) )
reponame = os . path . basename ( thisdir )
revised = input ( "Please specify repo name [{}]" . format ( reponame ) )
if revised not in [ "" , None ] :
    reponame = revised
# Get the default backend URL
keys = pluginmgr . search ( 'backend' )
keys = keys [ 'backend' ]
keys = [ k for k in keys if k [ 0 ] != "local" ]
remoteurl = ""
backend = None
if len ( keys ) > 0 :
    backend = pluginmgr . get_by_key ( 'backend' , keys [ 0 ] )
    candidate = backend . url ( username , reponame )
    revised = input ( "Please specify remote URL [{}]" . format ( candidate ) )
    if revised in [ "" , None ] :
        remoteurl = candidate
    else :
        remoteurl = revised
# Get title . . .
title = ""
while title == "" :
    title = input ( "One line summary of your repo:" )
    if title == "" :
        print ( "The repo requires a one line summary" )
    else :
        break
# Get description
description = ""
while description == "" :
    description = input ( "Detailed description:" )
    if description == "" :
        print ( "The repo requires some text as well" )
    else :
        break
autooptions = OrderedDict ( [ ( "username" , username ) , ( "reponame" , reponame ) , ( "remoteurl" , remoteurl ) , ( "title" , title ) , ( "description" , description ) , ( "working-directory" , "." ) , ( 'track' , OrderedDict ( [ ( 'includes' , [ '*.csv' , '*.tsv' , '*.txt' , '*.json' , '*.xls' , '*.xlsx' , "*.sql" , "*.hql" ] ) , ( 'excludes' , [ '.git' , '.svn' , os . path . basename ( autofile ) ] ) , ] ) ) , ( 'auto-push' , False ) , ( 'pipeline' , OrderedDict ( [ ] ) ) , ( 'import' , OrderedDict ( [ ( 'directory-mapping' , OrderedDict ( [ ( '.' , '' ) ] ) ) ] ) ) , ( 'dependencies' , OrderedDict ( [ ] ) ) ] )
# Gather options from each of the enabled plugins
for p in [ 'validator' , 'transformer' ] :
    keys = pluginmgr . search ( p )
    keys = keys [ p ]
    options = OrderedDict ( )
    for k in keys :
        if k . name in options :
            continue
        mgr = pluginmgr . get_by_key ( p , k )
        options [ k . name ] = mgr . autooptions ( )
    autooptions [ p ] = options
keys = pluginmgr . search ( 'metadata' )
keys = keys [ 'metadata' ]
if len ( keys ) > 0 : # = > Select domains that be included .
    servers = [ ]
    for k in keys :
        server = pluginmgr . get_by_key ( 'metadata' , k )
        server = server . url . split ( "/" ) [ 2 ]
        servers . append ( server )
    # Specify what should be included . Some of these should go ino
    # the metadata modules
    autooptions . update ( OrderedDict ( [ ( 'metadata-management' , OrderedDict ( [ ( 'servers' , servers ) , ( 'include-code-history' , find_executable_files ( ) ) , ( 'include-preview' , OrderedDict ( [ ( 'length' , 512 ) , ( 'files' , [ '*.txt' , '*.csv' , '*.tsv' ] ) ] ) ) , ( 'include-data-history' , True ) , ( 'include-action-history' , True ) , ( 'include-validation' , True ) , ( 'include-dependencies' , True ) , ( 'include-schema' , True ) , ( 'include-tab-diffs' , True ) , ( 'include-platform' , True ) , ] ) ) ] ) )
with open ( autofile , 'w' ) as fd :
    fd . write ( json . dumps ( autooptions , indent = 4 ) )
print ( "" )
print ( "Updated dataset specific config file: {}" . format ( autofile ) )
print ( "Please edit it and rerun dgit auto." )
print ( "Tip: Consider committing dgit.json to the code repository." )
# if platform . system ( ) = = " Linux " :
# subprocess . call ( [ " xdg - open " , autofile ] )
sys . exit ( )


signature:
def auto_get_repo ( autooptions , debug = False ) :
    """Automatically get repo
 Parameters
 autooptions : dgit . json content"""

implementation:
# plugin manager
pluginmgr = plugins_get_mgr ( )
# get the repo manager
repomgr = pluginmgr . get ( what = 'repomanager' , name = 'git' )
repo = None
try :
    if debug :
        print ( "Looking repo" )
    repo = repomgr . lookup ( username = autooptions [ 'username' ] , reponame = autooptions [ 'reponame' ] )
except : # Clone the repo
    try :
        print ( "Checking and cloning if the dataset exists on backend" )
        url = autooptions [ 'remoteurl' ]
        if debug :
            print ( "Doesnt exist. trying to clone: {}" . format ( url ) )
        common_clone ( url )
        repo = repomgr . lookup ( username = autooptions [ 'username' ] , reponame = autooptions [ 'reponame' ] )
        if debug :
            print ( "Cloning successful" )
    except : # traceback . print _ exc ( )
        yes = input ( "Repo doesnt exist. Should I create one? [yN]" )
        if yes == 'y' :
            setup = "git"
            if autooptions [ 'remoteurl' ] . startswith ( 's3://' ) :
                setup = 'git+s3'
            repo = common_init ( username = autooptions [ 'username' ] , reponame = autooptions [ 'reponame' ] , setup = setup , force = True , options = autooptions )
            if debug :
                print ( "Successfully inited repo" )
        else :
            raise Exception ( "Cannot load repo" )
repo . options = autooptions
return repo


signature:
def get_files_to_commit ( autooptions ) :
    """Look through the local directory to pick up files to check"""

implementation:
workingdir = autooptions [ 'working-directory' ]
includes = autooptions [ 'track' ] [ 'includes' ]
excludes = autooptions [ 'track' ] [ 'excludes' ]
# transform glob patterns to regular expressions
# print ( " Includes " , includes )
includes = r'|' . join ( [ fnmatch . translate ( x ) for x in includes ] )
excludes = r'|' . join ( [ fnmatch . translate ( x ) for x in excludes ] ) or r'$.'
matched_files = [ ]
for root , dirs , files in os . walk ( workingdir ) : # print ( " Looking at " , files )
# exclude dirs
# dirs [ : ] = [ os . path . join ( root , d ) for d in dirs ]
    dirs [ : ] = [ d for d in dirs if not re . match ( excludes , d ) ]
    # exclude / include files
    files = [ f for f in files if not re . match ( excludes , f ) ]
    # print ( " Files after excludes " , files )
    # print ( includes )
    files = [ f for f in files if re . match ( includes , f ) ]
    # print ( " Files after includes " , files )
    files = [ os . path . join ( root , f ) for f in files ]
    matched_files . extend ( files )
return matched_files


signature:
def auto_add ( repo , autooptions , files ) :
    """Cleanup the paths and add"""

implementation:
# Get the mappings and keys .
mapping = { "." : "" }
if ( ( 'import' in autooptions ) and ( 'directory-mapping' in autooptions [ 'import' ] ) ) :
    mapping = autooptions [ 'import' ] [ 'directory-mapping' ]
# Apply the longest prefix first . . .
keys = mapping . keys ( )
keys = sorted ( keys , key = lambda k : len ( k ) , reverse = True )
count = 0
params = [ ]
for f in files : # Find the destination
    relativepath = f
    for k in keys :
        v = mapping [ k ]
        if f . startswith ( k + "/" ) : # print ( " Replacing " , k )
            relativepath = f . replace ( k + "/" , v )
            break
    # Now add to repository
    count += files_add ( repo = repo , args = [ f ] , targetdir = os . path . dirname ( relativepath ) )
return count


signature:
def pull_stream ( self , uri , ** kwargs ) :
    """This will try to pull in a stream from an external source . Once a
 stream has been successfully pulled it is assigned a ' local stream
 name ' which can be used to access the stream from the EMS .
 : param uri : The URI of the external stream . Can be RTMP , RTSP or
 unicast / multicast ( d ) mpegts
 : type uri : str
 : param keepAlive : If keepAlive is set to 1 , the server will attempt to
 reestablish connection with a stream source after a connection has
 been lost . The reconnect will be attempted once every second
 ( default : 1 true )
 : type keepAlive : int
 : param localStreamName : If provided , the stream will be given this
 name . Otherwise , a fallback techniques used to determine the stream
 name ( based on the URI )
 : type localStreamName : str
 : param forceTcp : If 1 and if the stream is RTSP , a TCP connection will
 be forced . Otherwise the transport mechanism will be negotiated
 ( UDP or TCP ) ( default : 1 true )
 : type forceTcp : int
 : param tcUrl : When specified , this value will be used to set the TC URL
 in the initial RTMP connect invoke
 : type tcUrl : str
 : param pageUrl : When specified , this value will be used to set the
 originating web page address in the initial RTMP connect invoke
 : type pageUrl : str
 : param swfUrl : When specified , this value will be used to set the
 originating swf URL in the initial RTMP connect invoke
 : type swfUrl : str
 : param rangeStart : For RTSP and RTMP connections . A value from which
 the playback should start expressed in seconds . There are 2 special
 values : - 2 and - 1 . For more information , please read about
 start / len parameters here :
 http : / / livedocs . adobe . com / flashmediaserver / 3.0 / hpdocs / help . html ? content = 00000185 . html
 : type rangeStart : int
 : param rangeEnd : The length in seconds for the playback . - 1 is a
 special value . For more information , please read about start / len
 parameters here :
 http : / / livedocs . adobe . com / flashmediaserver / 3.0 / hpdocs / help . html ? content = 00000185 . html
 : type rangeEnd : int
 : param ttl : Sets the IP _ TTL ( time to live ) option on the socket
 : type ttl : int
 : param tos : Sets the IP _ TOS ( Type of Service ) option on the socket
 : type tos : int
 : param rtcpDetectionInterval : How much time ( in seconds ) should the
 server wait for RTCP packets before declaring the RTSP stream as a
 RTCP - less stream
 : type rtcpDetectionInterval : int
 : param emulateUserAgent : When specified , this value will be used as the
 user agent string . It is meaningful only for RTMP
 : type emulateUserAgent : str
 : param isAudio : If 1 and if the stream is RTP , it indicates that the
 currently pulled stream is an audio source . Otherwise the pulled
 source is assumed as a video source
 : type isAudio : int
 : param audioCodecBytes : The audio codec setup of this RTP stream if it
 is audio . Represented as hex format without ' 0x ' or ' h ' . For
 example : audioCodecBytes = 1190
 : type audioCodecBytes : str
 : param spsBytes : The video SPS bytes of this RTP stream if it is video .
 It should be base 64 encoded .
 : type spsBytes : str
 : param ppsBytes : The video PPS bytes of this RTP stream if it is video .
 It should be base 64 encoded
 : type ppsBytes : str
 : param ssmIp : The source IP from source - specific - multicast . Only usable
 when doing UDP based pull
 : type ssmIp : str
 : param httpProxy : This parameter has two valid values : IP : Port - This
 value combination specifies an RTSP HTTP Proxy from which the RTSP
 stream should be pulled from Self - Specifying " self " as the value
 implies pulling RTSP over HTTP
 : type httpProxy : str
 : link : http : / / docs . evostream . com / ems _ api _ definition / pullstream"""

implementation: return self . protocol . execute ( 'pullStream' , uri = uri , ** kwargs )


signature:
def push_stream ( self , uri , ** kwargs ) :
    """Try to push a local stream to an external destination . The pushed
 stream can only use the RTMP , RTSP or MPEG - TS unicast / multicast
 protocol .
 : param uri : The URI of the external stream . Can be RTMP , RTSP or
 unicast / multicast ( d ) mpegts
 : type uri : str
 : param keepAlive : If keepAlive is set to 1 , the server will attempt to
 reestablish connection with a stream source after a connection has
 been lost . The reconnect will be attempted once every second
 ( default : 1 true )
 : type keepAlive : int
 : param localStreamName : If provided , the stream will be given this
 name . Otherwise , a fallback techniques used to determine the stream
 name ( based on the URI )
 : type localStreamName : str
 : param targetStreamName : The name of the stream at destination . If not
 provided , the target stream name willbe the same as the local
 stream name
 : type targetStreamName : str
 : param targetStreamType : It can be one of following : * * live * * ,
 * * record * * , * * append * * . It is meaningful only for RTMP
 : type targetStreamType : str
 : param tcUrl : When specified , this value will be used to set the TC
 URL in the initial RTMP connect invoke
 : type tcUrl : str
 : param pageUrl : When specified , this value will be used to set the
 originating web page address in the initial RTMP connect invoke
 : type pageUrl : str
 : param swfUrl : When specified , this value will be used to set the
 originating swf URL in the initial RTMP connect invoke
 : type swfUrl : str
 : param ttl : Sets the IP _ TTL ( time to live ) option on the socket
 : type ttl : int
 : param tos : Sets the IP _ TOS ( Type of Service ) option on the socket
 : type tos : int
 : param emulateUserAgent : When specified , this value will be used as the
 user agent string . It is meaningful only for RTMP
 : type emulateUserAgent : str
 : param rtmpAbsoluteTimestamps : Forces the timestamps to be absolute
 when using RTMP .
 : type rtmpAbsoluteTimestamps : int
 : param sendChunkSizeRequest : Sets whether the RTMP stream will or will
 not send a " Set Chunk Length " message . This is significant when
 pushing to Akamai ' s new RTMP HD ingest point where this parameter
 should be set to 0 so that Akamai will not drop the connection .
 : type sendChunkSizeRequest : int
 : param useSourcePts : When value is true , timestamps on source inbound
 RTMP stream are passed directly to the outbound ( pushed ) RTMP
 streams . This affects only pushed Outbound Net RTMP with net RTMP
 source . This parameter overrides the value of the config . lua
 option of the same name .
 : type useSourcePts : int
 : link : http : / / docs . evostream . com / ems _ api _ definition / pushstream"""

implementation: return self . protocol . execute ( 'pushStream' , uri = uri , ** kwargs )


signature:
def create_hls_stream ( self , localStreamNames , targetFolder , ** kwargs ) :
    """Create an HTTP Live Stream ( HLS ) out of an existing H . 264 / AAC stream .
 HLS is used to stream live feeds to iOS devices such as iPhones and
 iPads .
 : param localStreamNames : The stream ( s ) that will be used as the input .
 This is a comma - delimited list of active stream names ( local stream
 names ) .
 : type localStreamNames : str
 : param targetFolder : The folder where all the . ts / . m3u8 files will be
 stored . This folder must be accessible by the HLS clients . It is
 usually in the web - root of the server .
 : type targetFolder : str
 : param keepAlive : If true , the EMS will attempt to reconnect to the
 stream source if the connection is severed .
 : type keepAlive : int
 : param overwriteDestination : If true , it will force overwrite of
 destination files .
 : type overwriteDestination : int
 : param staleRetentionCount : The number of old files kept besides the
 ones listed in the current version of the playlist . Only
 applicable for rolling playlists .
 : type staleRetentionCount : int
 : param createMasterPlaylist : If true , a master playlist will be
 created .
 : type createMasterPlaylist : int
 : param cleanupDestination : If true , all * . ts and * . m3u8 files in the
 target folder will be removed before HLS creation is started .
 : type cleanupDestination : int
 : param bandwidths : The corresponding bandwidths for each stream listed
 in localStreamNames . Again , this can be a comma - delimited list .
 : type bandwidths : int
 : param groupName : The name assigned to the HLS stream or group . If the
 localStreamNames parameter contains only one entry and groupName
 is not specified , groupName will have the value of the input
 stream name .
 : type groupName : str
 : param playlistType : Either appending or rolling .
 : type playlistType : str
 : param playlistLength : The length ( number of elements ) of the playlist .
 Used only when playlistType is rolling . Ignored otherwise .
 : type playlistLength : int
 : param playlistName : The file name of the playlist ( * . m3u8 ) .
 : type playlistName : str
 : param chunkLength : The length ( in seconds ) of each playlist element
 ( * . ts file ) . Minimum value is 1 ( second ) .
 : type chunkLength : int
 : param maxChunkLength : Maximum length ( in seconds ) the EMS will allow
 any single chunk to be . This is primarily in the case of
 chunkOnIDR = true where the EMS will wait for the next key - frame . If
 the maxChunkLength is less than chunkLength , the parameter shall
 be ignored .
 : type maxChunkLength : int
 : param chunkBaseName : The base name used to generate the * . ts chunks .
 : type chunkBaseName : str
 : param chunkOnIDR : If true , chunking is performed ONLY on IDR .
 Otherwise , chunking is performed whenever chunk length is
 achieved .
 : type chunkOnIDR : int
 : param drmType : Type of DRM encryption to use . Options are : none
 ( no encryption ) , evo ( AES Encryption ) , SAMPLE - AES ( Sample - AES ) ,
 verimatrix ( Verimatrix DRM ) . For Verimatrix DRM , the " drm " section
 of the config . lua file must be active and properly configured .
 : type drmType : str
 : param AESKeyCount : Number of keys that will be automatically generated
 and rotated over while encrypting this HLS stream .
 : type AESKeyCount : int
 : param audioOnly : If true , stream will be audio only .
 : type audioOnly : int
 : param hlsResume : If true , HLS will resume in appending segments to
 previously created child playlist even in cases of EMS shutdown or
 cut off stream source .
 : type hlsResume : int
 : param cleanupOnClose : If true , corresponding hls files to a stream
 will be deleted if the said stream is removed or shut down or
 disconnected .
 : type cleanupOnClose : int
 : param useByteRange : If true , will use the EXT - X - BYTERANGE feature of
 HLS ( version 4 and up ) .
 : type useByteRange : int
 : param fileLength : When using useByteRange = 1 , this parameter needs to
 be set too . This will be the size of file before chunking it to
 another file , this replace the chunkLength in case of
 EXT - X - BYTERANGE , since chunkLength will be the byte range chunk .
 : type fileLength : int
 : param useSystemTime : If true , uses UTC in playlist time stamp
 otherwise will use the local server time .
 : type useSystemTime : int
 : param offsetTime :
 : type offsetTime : int
 : param startOffset : A parameter valid only for HLS v . 6 onwards . This
 will indicate the start offset time ( in seconds ) for the playback
 of the playlist .
 : type startOffset : int
 : link : http : / / docs . evostream . com / ems _ api _ definition / createhlsstream"""

implementation: return self . protocol . execute ( 'createhlsstream' , localStreamNames = localStreamNames , targetFolder = targetFolder , ** kwargs )


signature:
def create_hds_stream ( self , localStreamNames , targetFolder , ** kwargs ) :
    """Create an HDS ( HTTP Dynamic Streaming ) stream out of an existing
 H . 264 / AAC stream . HDS is used to stream standard MP4 media over
 regular HTTP connections .
 : param localStreamNames : The stream ( s ) that will be used as the input .
 This is a comma - delimited list of active stream names ( local stream
 names ) .
 : type localStreamNames : str
 : param targetFolder : The folder where all the manifest ( * . f4m ) and
 fragment ( f4v * ) files will be stored . This folder must be
 accessible by the HDS clients . It is usually in the web - root of
 the server .
 : type targetFolder : str
 : param bandwidths : The corresponding bandwidths for each stream listed
 in localStreamNames . Again , this can be a comma - delimited list .
 : type bandwidths : int
 : param chunkBaseName : The base name used to generate the fragments .
 : type chunkBaseName : str
 : param chunkLength : The length ( in seconds ) of fragments to be made .
 Minimum value is 1 ( second ) .
 : type chunkLength : int
 : param chunkOnIDR : If true , chunking is performed ONLY on IDR .
 Otherwise , chunking is performed whenever chunk length is
 achieved .
 : type chunkOnIDR : int
 : param groupName : The name assigned to the HDS stream or group . If the
 localStreamNames parameter contains only one entry and groupName
 is not specified , groupName will have the value of the input
 stream name .
 : type groupName : str
 : param keepAlive : If true , the EMS will attempt to reconnect to the
 stream source if the connection is severed .
 : type keepAlive : int
 : param manifestName : The manifest file name .
 : type manifestName : str
 : param overwriteDestination : If true , it will allow overwrite of
 destination files .
 : type overwriteDestination : int
 : param playlistType : Either appending or rolling .
 : type playlistType : str
 : param playlistLength : The number of fragments before the server
 starts to overwrite the older fragments . Used only when
 playlistType is " rolling " . Ignored otherwise .
 : type playlistLength : int
 : param staleRetentionCount : The number of old files kept besides the
 ones listed in the current version of the playlist . Only
 applicable for rolling playlists .
 : type staleRetentionCount : int
 : param createMasterPlaylist : If true , a master playlist will be
 created .
 : type createMasterPlaylist : int
 : param cleanupDestination : If true , all manifest and fragment files in
 the target folder will be removed before HDS creation is started .
 : type cleanupDestination : int
 : link : http : / / docs . evostream . com / ems _ api _ definition / createhdsstream"""

implementation: return self . protocol . execute ( 'createhdsstream' , localStreamNames = localStreamNames , targetFolder = targetFolder , ** kwargs )


signature:
def create_mss_stream ( self , localStreamNames , targetFolder , ** kwargs ) :
    """Create a Microsoft Smooth Stream ( MSS ) out of an existing H . 264 / AAC
 stream . Smooth Streaming was developed by Microsoft to compete with
 other adaptive streaming technologies .
 : param localStreamNames : The stream ( s ) that will be used as the input .
 This is a comma - delimited list of active stream names ( local
 stream names )
 : type localStreamNames : str
 : param targetFolder : The folder where all the manifest and fragment
 files will be stored . This folder must be accessible by the MSS
 clients . It is usually in the web - root of the server .
 : type targetFolder : str
 : param bandwidths : The corresponding bandwidths for each stream listed
 in ` localStreamNames ` . Again , this can be a comma - delimited list .
 : type bandwidths : int or str
 : param groupName : The name assigned to the MSS stream or group . If the
 ` localStreamNames ` parameter contains only one entry and groupName
 is not specified , groupName will have the value of the input
 stream name .
 : type groupName : str
 : param playlistType : Either ` appending ` or ` rolling `
 : type playlistType : str
 : param playlistLength : The number of fragments before the server
 starts to overwrite the older fragments . Used only when
 ` playlistType ` is ` rolling ` . Ignored otherwise .
 : type playlistLength : int
 : param manifestName : The manifest file name .
 : type manifestName : str
 : param chunkLength : The length ( in seconds ) of fragments to be made .
 : type chunkLength : int
 : param chunkOnIDR : If 1 ( true ) , chunking is performed ONLY on IDR .
 Otherwise , chunking is performed whenever chunk length is
 achieved .
 : type chunkOnIDR : int
 : param keepAlive : If 1 ( true ) , the EMS will attempt to reconnect to
 the stream source if the connection is severed .
 : type keepAlive : int
 : param overwriteDestination : If 1 ( true ) , it will allow overwrite of
 destination files .
 : type overwriteDestination : int
 : param staleRetentionCount : How many old files are kept besides the
 ones present in the current version of the playlist . Only
 applicable for rolling playlists .
 : type staleRetentionCount : int
 : param cleanupDestination : If 1 ( true ) , all manifest and fragment
 files in the target folder will be removed before MSS creation is
 started .
 : type cleanupDestination : int
 : param ismType : Either ismc for serving content to client or isml for
 serving content to smooth server .
 : type ismType : int
 : param isLive : If true , creates a live MSS stream , otherwise set to 0
 for VOD .
 : type isLive : int
 : param publishingPoint : This parameter is needed when ` ismType = isml ` ,
 it is the REST URI where the mss contents will be ingested .
 : type publishingPoint : str
 : param ingestMode : Either ` single ` for a non looping ingest or ` loop `
 for looping an ingest .
 : type ingestMode : str
 : link : http : / / docs . evostream . com / ems _ api _ definition / createmssstream"""

implementation: return self . protocol . execute ( 'createmssstream' , localStreamNames = localStreamNames , targetFolder = targetFolder , ** kwargs )


signature:
def create_dash_stream ( self , localStreamNames , targetFolder , ** kwargs ) :
    """Create Dynamic Adaptive Streaming over HTTP ( DASH ) out of an existing
 H . 264 / AAC stream . DASH was developed by the Moving Picture Experts
 Group ( MPEG ) to establish a standard for HTTP adaptive - bitrate
 streaming that would be accepted by multiple vendors and facilitate
 interoperability .
 : param localStreamNames : The stream ( s ) that will be used as the
 input . This is a comma - delimited list of active stream names
 ( local stream names ) .
 : type localStreamNames : str
 : param targetFolder : The folder where all the manifest and fragment
 files will be stored . This folder must be accessible by the DASH
 clients . It is usually in the web - root of the server .
 : type targetFolder : str
 : param bandwidths : The corresponding bandwidths for each stream listed
 in ` localStreamNames ` . Again , this can be a comma - delimited list .
 : type bandwidths : int or str
 : param groupName : The name assigned to the DASH stream or group . If
 the ` localStreamNames ` parameter contains only one entry and
 ` groupName ` is not specified , ` groupName ` will have the value of
 the input stream name .
 : type groupName : str
 : param playlistType : Either ` appending ` or ` rolling ` .
 : type playlistType : str
 : param playlistLength : The number of fragments before the server
 starts to overwrite the older fragments . Used only when
 ` playlistType ` is ` rolling ` . Ignored otherwise .
 : type playlistLength : int
 : param manifestName : The manifest file name .
 : type manifestName : str
 : param chunkLength : The length ( in seconds ) of fragments to be made .
 : type chunkLength : int
 : param chunkOnIDR : If true , chunking is performed ONLY on IDR .
 Otherwise , chunking is performed whenever chunk length is
 achieved .
 : type chunkOnIDR : int
 : param keepAlive : If true , the EMS will attempt to reconnect to the
 stream source if the connection is severed .
 : type keepAlive : int
 : param overwriteDestination : If true , it will allow overwrite of
 destination files .
 : type overwriteDestination : int
 : param staleRetentionCount : How many old files are kept besides the
 ones present in the current version of the playlist . Only
 applicable for rolling playlists .
 : type staleRetentionCount : int
 : param cleanupDestination : If true , all manifest and fragment files in
 the target folder will be removed before DASH creation is started .
 : type cleanupDestination : int
 : param dynamicProfile : Set this parameter to 1 ( default ) for a live
 DASH , otherwise set it to 0 for a VOD .
 : type dynamicProfile : int
 : link : http : / / docs . evostream . com / ems _ api _ definition / createdashstream"""

implementation: return self . protocol . execute ( 'createdashstream' , localStreamNames = localStreamNames , targetFolder = targetFolder , ** kwargs )


signature:
def record ( self , localStreamName , pathToFile , ** kwargs ) :
    """Records any inbound stream . The record command allows users to record
 a stream that may not yet exist . When a new stream is brought into
 the server , it is checked against a list of streams to be recorded .
 Streams can be recorded as FLV files , MPEG - TS files or as MP4 files .
 : param localStreamName : The name of the stream to be used as input
 for recording .
 : type localStreamName : str
 : param pathToFile : Specify path and file name to write to .
 : type pathToFile : str
 : param type : ` ts ` , ` mp4 ` or ` flv `
 : type type : str
 : param overwrite : If false , when a file already exists for the stream
 name , a new file will be created with the next appropriate number
 appended . If 1 ( true ) , files with the same name will be
 overwritten .
 : type overwrite : int
 : param keepAlive : If 1 ( true ) , the server will restart recording every
 time the stream becomes available again .
 : type keepAlive : int
 : param chunkLength : If non - zero the record command will start a new
 recording file after ChunkLength seconds have elapsed .
 : type chunkLength : int
 : param waitForIDR : This is used if the recording is being chunked .
 When true , new files will only be created on IDR boundaries .
 : type waitForIDR : int
 : param winQtCompat : Mandates 32bit header fields to ensure
 compatibility with Windows QuickTime .
 : type winQtCompat : int
 : param dateFolderStructure : If set to 1 ( true ) , folders will be
 created with names in ` YYYYMMDD ` format . Recorded files will be
 placed inside these folders based on the date they were created .
 : type dateFolderStructure : int
 : link : http : / / docs . evostream . com / ems _ api _ definition / record"""

implementation: return self . protocol . execute ( 'record' , localStreamName = localStreamName , pathToFile = pathToFile , ** kwargs )


signature:
def transcode ( self , source , destinations , ** kwargs ) :
    """Changes the compression characteristics of an audio and / or video
 stream . Allows you to change the resolution of a source stream , change
 the bitrate of a stream , change a VP8 or MPEG2 stream into H . 264 and
 much more . Allow users to create overlays on the final stream as well
 as crop streams .
 : param source : Can be a URI or a local stream name from EMS .
 : type source : str
 : param destinations : The target URI ( s ) or stream name ( s ) of the
 transcoded stream . If only a name is given , it will be pushed
 back to the EMS .
 : type destinations : str
 : param targetStreamNames : The name of the stream ( s ) at destination ( s ) .
 If not specified , and a full URI is provided to destinations ,
 name will have a time stamped value .
 : type targetStreamNames : str
 : param groupName : The group name assigned to this process . If not
 specified , groupName will have a random value .
 : type groupName : str
 : param videoBitrates : Target output video bitrate ( s ) ( in bits / s ,
 append ` k ` to value for kbits / s ) . Accepts the value ` copy ` to
 copy the input bitrate . An empty value passed would mean no video .
 : type videoBitrates : str
 : param videoSizes : Target output video size ( s ) in wxh ( width x height )
 format . IE : 240x480.
 : type videoSizes : str
 : param videoAdvancedParamsProfiles : Name of video profile template
 that will be used .
 : type videoAdvancedParamsProfiles : str
 : param audioBitrates : Target output audio bitrate ( s ) ( in bits / s ,
 append ` k ` to value for kbits / s ) . Accepts the value ` copy ` to
 copy the input bitrate . An empty value passed would mean no audio .
 : type audioBitrates : str
 : param audioChannelsCounts : Target output audio channel ( s ) count ( s ) .
 Valid values are 1 ( mono ) , 2 ( stereo ) , and so on . Actual supported
 channel count is dependent on the number of input audio channels .
 : type audioChannelsCounts : str
 : param audioFrequencies : Target output audio frequency ( ies ) ( in Hz ,
 append ` k ` to value for kHz ) .
 : type audioFrequencies : str
 : param audioAdvancedParamsProfiles : Name of audio profile template
 that will be used .
 : type audioAdvancedParamsProfiles : str
 : param overlays : Location of the overlay source ( s ) to be used . These
 are transparent images ( normally in PNG format ) that have the same
 or smaller size than the video . Image is placed at the top - left
 position of the video .
 : type overlays : str
 : param croppings : Target video cropping position ( s ) and size ( s ) in
 ` left : top : width : height ` format ( e . g . 0:0:200:100 . Positions
 are optional ( 200:100 for a centered cropping of 200 width and 100
 height in pixels ) . Values are limited to the actual size of the
 video .
 : type croppings : str
 : param keepAlive : If keepAlive is set to 1 , the server will restart
 transcoding if it was previously activated .
 : type keepAlive : int
 : param commandFlags : Other commands to the transcode process that are
 not supported by the baseline transcode command .
 : type commandFlags : str
 : link : http : / / docs . evostream . com / ems _ api _ definition / transcode"""

implementation: return self . protocol . execute ( 'transcode' , source = source , destinations = destinations , ** kwargs )


signature:
def add_stream_alias ( self , localStreamName , aliasName , ** kwargs ) :
    """Allows you to create secondary name ( s ) for internal streams . Once an
 alias is created the localstreamname cannot be used to request
 playback of that stream . Once an alias is used ( requested by a client )
 the alias is removed . Aliases are designed to be used to protect / hide
 your source streams .
 : param localStreamName : The original stream name
 : type localStreamName : str
 : param aliasName : The alias alternative to the localStreamName
 : type aliasName : str
 : param expirePeriod : The expiration period for this alias . Negative
 values will be treated as one - shot but no longer than the absolute
 positive value in seconds , 0 means it will not expire , positive
 values mean the alias can be used multiple times but expires after
 this many seconds . The default is - 600 ( one - shot , 10 mins )
 : type expirePeriod : int
 : link : http : / / docs . evostream . com / ems _ api _ definition / addstreamalias"""

implementation: return self . protocol . execute ( 'addStreamAlias' , localStreamName = localStreamName , aliasName = aliasName , ** kwargs )


signature:
def add_group_name_alias ( self , groupName , aliasName ) :
    """Creates secondary name ( s ) for group names . Once an alias is created the
 group name cannot be used to request HTTP playback of that stream . Once
 an alias is used ( requested by a client ) the alias is removed . Aliases
 are designed to be used to protect / hide your source streams .
 : param groupName : The original group name
 : type groupName : str
 : param aliasName : The alias alternative to the group name
 : type aliasName : str
 : link : http : / / docs . evostream . com / ems _ api _ definition / addgroupnamealias"""

implementation: return self . protocol . execute ( 'addGroupNameAlias' , groupName = groupName , aliasName = aliasName )


signature:
def create_ingest_point ( self , privateStreamName , publicStreamName ) :
    """Creates an RTMP ingest point , which mandates that streams pushed into
 the EMS have a target stream name which matches one Ingest Point
 privateStreamName .
 : param privateStreamName : The name that RTMP Target Stream Names must
 match .
 : type privateStreamName : str
 : param publicStreamName : The name that is used to access the stream
 pushed to the privateStreamName . The publicStreamName becomes the
 streams localStreamName .
 : type publicStreamName : str
 : link : http : / / docs . evostream . com / ems _ api _ definition / createingestpoint"""

implementation: return self . protocol . execute ( 'createIngestPoint' , privateStreamName = privateStreamName , publicStreamName = publicStreamName )


signature:
def start_web_rtc ( self , ersip , ersport , roomId ) :
    """Starts a WebRTC signalling client to an ERS ( Evostream Rendezvous
 Server ) .
 : param ersip : IP address ( xx . yy . zz . xx ) of ERS .
 : type ersip : str
 : param ersport : IP port of ERS .
 : type ersport : int
 : param roomId : Unique room Identifier within ERS that will be used by
 client browsers to connect to this EMS .
 : type roomId : str
 : link : http : / / docs . evostream . com / ems _ api _ definition / startwebrtc"""

implementation: return self . protocol . execute ( 'startwebrtc' , ersip = ersip , ersport = ersport , roomId = roomId )


signature:
def instantiate ( repo , name = None , filename = None ) :
    """Instantiate the generator and filename specification"""

implementation:
default_transformers = repo . options . get ( 'transformer' , { } )
# If a name is specified , then lookup the options from dgit . json
# if specfied . Otherwise it is initialized to an empty list of
# files .
transformers = { }
if name is not None : # Handle the case generator is specified . .
    if name in default_transformers :
        transformers = { name : default_transformers [ name ] }
    else :
        transformers = { name : { 'files' : [ ] , } }
else :
    transformers = default_transformers
# Map the filename patterns to list of files
# Instantiate the files from the patterns specified
input_matching_files = None
if filename is not None :
    input_matching_files = repo . find_matching_files ( [ filename ] )
for t in transformers :
    for k in transformers [ t ] :
        if "files" not in k :
            continue
        if k == "files" and input_matching_files is not None : # Use the files specified on the command line . .
            transformers [ t ] [ k ] = input_matching_files
        else : # Try to match the specification
            if transformers [ t ] [ k ] is None or len ( transformers [ t ] [ k ] ) == 0 :
                transformers [ t ] [ k ] = [ ]
            else :
                matching_files = repo . find_matching_files ( transformers [ t ] [ k ] )
                transformers [ t ] [ k ] = matching_files
return transformers


signature:
def transform ( repo , name = None , filename = None , force = False , args = [ ] ) :
    """Materialize queries / other content within the repo .
 Parameters
 repo : Repository object
 name : Name of transformer , if any . If none , then all transformers specified in dgit . json will be included .
 filename : Pattern that specifies files that must be processed by the generators selected . If none , then the default specification in dgit . json is used ."""

implementation:
mgr = plugins_get_mgr ( )
# Expand the specification . Now we have full file paths
specs = instantiate ( repo , name , filename )
# Run the validators with rules files . . .
allresults = [ ]
for s in specs :
    keys = mgr . search ( what = 'transformer' , name = s ) [ 'transformer' ]
    for k in keys :
        t = mgr . get_by_key ( 'transformer' , k )
        result = t . evaluate ( repo , specs [ s ] , force , args )
        allresults . extend ( result )
return allresults


signature:
def _run ( self , cmd ) :
    """Helper function to run commands
 Parameters
 cmd : list
 Arguments to git command"""

implementation:
# This is here in case the . gitconfig is not accessible for
# some reason .
environ = os . environ . copy ( )
environ [ 'GIT_COMMITTER_NAME' ] = self . fullname
environ [ 'GIT_COMMITTER_EMAIL' ] = self . email
environ [ 'GIT_AUTHOR_NAME' ] = self . fullname
environ [ 'GIT_AUTHOR_EMAIL' ] = self . email
cmd = [ pipes . quote ( c ) for c in cmd ]
cmd = " " . join ( [ '/usr/bin/git' ] + cmd )
cmd += "; exit 0"
# print ( " Running cmd " , cmd )
try :
    output = subprocess . check_output ( cmd , stderr = subprocess . STDOUT , shell = True , env = environ )
except subprocess . CalledProcessError as e :
    output = e . output
output = output . decode ( 'utf-8' )
output = output . strip ( )
# print ( " Output of command " , output )
return output


signature:
def _run_generic_command ( self , repo , cmd ) :
    """Run a generic command within the repo . Assumes that you are
 in the repo ' s root directory"""

implementation:
result = None
with cd ( repo . rootdir ) : # Dont use sh . It is not collecting the stdout of all
# child processes .
    output = self . _run ( cmd )
    try :
        result = { 'cmd' : cmd , 'status' : 'success' , 'message' : output , }
    except Exception as e :
        result = { 'cmd' : cmd , 'status' : 'error' , 'message' : str ( e ) }
return result


signature:
def init ( self , username , reponame , force , backend = None ) :
    """Initialize a Git repo
 Parameters
 username , reponame : Repo name is tuple ( name , reponame )
 force : force initialization of the repo even if exists
 backend : backend that must be used for this ( e . g . s3)"""

implementation:
key = self . key ( username , reponame )
# In local filesystem - based server , add a repo
server_repodir = self . server_rootdir ( username , reponame , create = False )
# Force cleanup if needed
if os . path . exists ( server_repodir ) and not force :
    raise RepositoryExists ( )
if os . path . exists ( server_repodir ) :
    shutil . rmtree ( server_repodir )
os . makedirs ( server_repodir )
# Initialize the repo
with cd ( server_repodir ) :
    git . init ( "." , "--bare" )
if backend is not None :
    backend . init_repo ( server_repodir )
# Now clone the filesystem - based repo
repodir = self . rootdir ( username , reponame , create = False )
# Prepare it if needed
if os . path . exists ( repodir ) and not force :
    raise Exception ( "Local repo already exists" )
if os . path . exists ( repodir ) :
    shutil . rmtree ( repodir )
os . makedirs ( repodir )
# Now clone . . .
with cd ( os . path . dirname ( repodir ) ) :
    git . clone ( server_repodir , '--no-hardlinks' )
url = server_repodir
if backend is not None :
    url = backend . url ( username , reponame )
repo = Repo ( username , reponame )
repo . manager = self
repo . remoteurl = url
repo . rootdir = self . rootdir ( username , reponame )
self . add ( repo )
return repo


signature:
def clone ( self , url , backend = None ) :
    """Clone a URL
 Parameters
 url : URL of the repo . Supports s3 : / / , git @ , http : / /"""

implementation:
# s3 : / / bucket / git / username / repo . git
username = self . username
reponame = url . split ( "/" ) [ - 1 ]
# with git
reponame = reponame . replace ( ".git" , "" )
key = ( username , reponame )
# In local filesystem - based server , add a repo
server_repodir = self . server_rootdir ( username , reponame , create = False )
rootdir = self . rootdir ( username , reponame , create = False )
if backend is None : # Backend is standard git repo ( https : / / , git @ . . . )
    with cd ( os . path . dirname ( rootdir ) ) :
        self . _run ( [ 'clone' , '--no-hardlinks' , url ] )
else : # Backend is s3
# Sync if needed .
    if not os . path . exists ( server_repodir ) : # s3 - > . dgit / git / pingali / hello . git - > . dgit / datasets / pingali / hello
        backend . clone_repo ( url , server_repodir )
    # After sync clone ,
    with cd ( os . path . dirname ( rootdir ) ) :
        self . _run ( [ 'clone' , '--no-hardlinks' , server_repodir ] )
# Insert the notes push
if True :
    configfile = os . path . join ( rootdir , '.git' , 'config' )
    content = open ( configfile ) . read ( )
    original = "fetch = +refs/heads/*:refs/remotes/origin/*"
    replacement = """fetch = +refs/heads/*:refs/remotes/origin/*\n        fetch = +refs/notes/*:refs/notes/*"""
    if "notes" not in content :
        content = content . replace ( original , replacement )
        with open ( configfile , 'w' ) as fd :
            fd . write ( content )
    # Pull the notes if any as well . .
    with cd ( rootdir ) :
        self . _run ( [ 'pull' , 'origin' ] )
# Insert the object into the internal table we maintain . . .
r = Repo ( username , reponame )
r . rootdir = rootdir
r . remoteurl = url
r . manager = self
package = os . path . join ( r . rootdir , 'datapackage.json' )
packagedata = open ( package ) . read ( )
r . package = json . JSONDecoder ( object_pairs_hook = collections . OrderedDict ) . decode ( packagedata )
return self . add ( r )


signature:
def delete ( self , repo , args = [ ] ) :
    """Delete files from the repo"""

implementation:
result = None
with cd ( repo . rootdir ) :
    try :
        cmd = [ 'rm' ] + list ( args )
        result = { 'status' : 'success' , 'message' : self . _run ( cmd ) }
    except Exception as e :
        result = { 'status' : 'error' , 'message' : str ( e ) }
    # print ( result )
    return result


signature:
def drop ( self , repo , args = [ ] ) :
    """Cleanup the repo"""

implementation:
# Clean up the rootdir
rootdir = repo . rootdir
if os . path . exists ( rootdir ) :
    print ( "Cleaning repo directory: {}" . format ( rootdir ) )
    shutil . rmtree ( rootdir )
# Cleanup the local version of the repo ( this could be on
# the server etc .
server_repodir = self . server_rootdir_from_repo ( repo , create = False )
if os . path . exists ( server_repodir ) :
    print ( "Cleaning data from local git 'server': {}" . format ( server_repodir ) )
    shutil . rmtree ( server_repodir )
super ( GitRepoManager , self ) . drop ( repo )
return { 'status' : 'success' , 'message' : "successful cleanup" }


signature:
def permalink ( self , repo , path ) :
    """Get the permalink to command that generated the dataset"""

implementation:
if not os . path . exists ( path ) : # print ( " Path does not exist " , path )
    return ( None , None )
# Get this directory
cwd = os . getcwd ( )
# Find the root of the repo and cd into that directory . .
if os . path . isfile ( path ) :
    os . chdir ( os . path . dirname ( path ) )
rootdir = self . _run ( [ "rev-parse" , "--show-toplevel" ] )
if "fatal" in rootdir : # print ( " fatal " , rootdir )
    return ( None , None )
os . chdir ( rootdir )
# print ( " Rootdir = " , rootdir )
# Now find relative path
relpath = os . path . relpath ( path , rootdir )
# print ( " relpath = " , relpath )
# Get the last commit for this file
# 3764cc2600b221ac7d7497de3d0dbcb4cffa2914
sha1 = self . _run ( [ "log" , "-n" , "1" , "--format=format:%H" , relpath ] )
# print ( " sha1 = " , sha1)
# Get the repo URL
# git @ gitlab . com : pingali / simple - regression . git
# https : / / gitlab . com / kanban _ demo / test _ project . git
remoteurl = self . _run ( [ "config" , "--get" , "remote.origin.url" ] )
# print ( " remoteurl = " , remoteurl )
# Go back to the original directory . . .
os . chdir ( cwd )
# Now match it against two possible formats of the remote url
# Examples
# https : / / help . github . com / articles / getting - permanent - links - to - files /
# https : / / github . com / github / hubot / blob / ed25584f5ac2520a6c28547ffd0961c7abd7ea49 / README . md
# https : / / gitlab . com / pingali / simple - regression / blob / 3764cc2600b221ac7d7497de3d0dbcb4cffa2914 / model . py
# https : / / github . com / pingali / dgit / blob / ff91b5d04b2978cad0bf9b006d1b0a16d18a778e / README . rst
# https : / / gitlab . com / kanban _ demo / test _ project / blob / b004677c23b3a31eb7b5588a5194857b2c8b2b95 / README . md
m = re . search ( '^git@([^:\/]+):([^/]+)/([^/]+)' , remoteurl )
if m is None :
    m = re . search ( '^https://([^:/]+)/([^/]+)/([^/]+)' , remoteurl )
if m is not None :
    domain = m . group ( 1 )
    username = m . group ( 2 )
    project = m . group ( 3 )
    if project . endswith ( ".git" ) :
        project = project [ : - 4 ]
    permalink = "https://{}/{}/{}/blob/{}/{}" . format ( domain , username , project , sha1 , relpath )
    # print ( " permalink = " , permalink )
    return ( relpath , permalink )
else :
    return ( None , None )


signature:
def add_files ( self , repo , files ) :
    """Add files to the repo"""

implementation:
rootdir = repo . rootdir
for f in files :
    relativepath = f [ 'relativepath' ]
    sourcepath = f [ 'localfullpath' ]
    if sourcepath is None : # This can happen if the relative path is a URL
        continue
    # Prepare the target path
    targetpath = os . path . join ( rootdir , relativepath )
    try :
        os . makedirs ( os . path . dirname ( targetpath ) )
    except :
        pass
    # print ( sourcepath , " = > " , targetpath )
    print ( "Updating: {}" . format ( relativepath ) )
    shutil . copyfile ( sourcepath , targetpath )
    with cd ( repo . rootdir ) :
        self . _run ( [ 'add' , relativepath ] )


signature:
def config ( self , what = 'get' , params = None ) :
    """Paramers :
 workspace : Directory to store the dataset repositories
 email :"""

implementation:
if what == 'get' :
    return { 'name' : 'git' , 'nature' : 'repomanager' , 'variables' : [ ] , }
elif what == 'set' :
    self . workspace = params [ 'Local' ] [ 'workspace' ]
    self . workspace = os . path . abspath ( self . workspace )
    self . username = params [ 'User' ] [ 'user.name' ]
    self . fullname = params [ 'User' ] [ 'user.fullname' ]
    self . email = params [ 'User' ] [ 'user.email' ]
    repodir = os . path . join ( self . workspace , 'datasets' )
    if not os . path . exists ( repodir ) :
        return
    for username in os . listdir ( repodir ) :
        for reponame in os . listdir ( os . path . join ( repodir , username ) ) :
            if self . is_my_repo ( username , reponame ) :
                r = Repo ( username , reponame )
                r . rootdir = os . path . join ( repodir , username , reponame )
                package = os . path . join ( r . rootdir , 'datapackage.json' )
                if not os . path . exists ( package ) :
                    print ( "datapackage.json does not exist in dataset" )
                    print ( "Skipping: {}/{}" . format ( username , reponame ) )
                    continue
                packagedata = open ( package ) . read ( )
                r . package = json . JSONDecoder ( object_pairs_hook = collections . OrderedDict ) . decode ( packagedata )
                r . manager = self
                self . add ( r )


signature:
def _init_empty ( self ) :
    """Creates the base set of attributes invoice has / needs"""

implementation: self . _jsondata = { "code" : None , "currency" : "EUR" , "subject" : "" , "due_date" : ( datetime . datetime . now ( ) . date ( ) + datetime . timedelta ( days = 14 ) ) . isoformat ( ) , "issue_date" : datetime . datetime . now ( ) . date ( ) . isoformat ( ) , "number" : None , "type" : "outbound" , "receiver" : { "name" : "" , "email" : "" , "street" : "" , "city" : "" , "postcode" : "" , "country" : "" } , "items" : [ ] , }


signature:
def send ( self , send_email = True ) :
    """Marks the invoice as sent in Holvi
 If send _ email is False then the invoice is * not * automatically emailed to the recipient
 and your must take care of sending the invoice yourself ."""

implementation:
url = str ( self . api . base_url + '{code}/status/' ) . format ( code = self . code )
# six . u messes this up
payload = { 'mark_as_sent' : True , 'send_email' : send_email , }
stat = self . api . connection . make_put ( url , payload )


signature:
def to_holvi_dict ( self ) :
    """Convert our Python object to JSON acceptable to Holvi API"""

implementation:
self . _jsondata [ "items" ] = [ ]
for item in self . items :
    self . _jsondata [ "items" ] . append ( item . to_holvi_dict ( ) )
self . _jsondata [ "issue_date" ] = self . issue_date . isoformat ( )
self . _jsondata [ "due_date" ] = self . due_date . isoformat ( )
self . _jsondata [ "receiver" ] = self . receiver . to_holvi_dict ( )
return { k : v for ( k , v ) in self . _jsondata . items ( ) if k in self . _valid_keys }


signature:
def save ( self ) :
    """Saves this invoice to Holvi , returns the created / updated invoice"""

implementation:
if not self . items :
    raise HolviError ( "No items" )
if not self . subject :
    raise HolviError ( "No subject" )
send_json = self . to_holvi_dict ( )
if self . code :
    url = str ( self . api . base_url + '{code}/' ) . format ( code = self . code )
    if not self . code :
        send_patch = { k : v for ( k , v ) in send_json . items ( ) if k in self . _patch_valid_keys }
        send_patch [ "items" ] = [ ]
        for item in self . items :
            send_patch [ "items" ] . append ( item . to_holvi_dict ( True ) )
        stat = self . api . connection . make_patch ( url , send_patch )
    else :
        stat = self . api . connection . make_put ( url , send_json )
    return Invoice ( self . api , stat )
else :
    url = str ( self . api . base_url )
    stat = self . api . connection . make_post ( url , send_json )
    return Invoice ( self . api , stat )


signature:
def get_plugin_source ( module = None , stacklevel = None ) :
    """Returns the : class : ` PluginSource ` for the current module or the given
 module . The module can be provided by name ( in which case an import
 will be attempted ) or as a module object .
 If no plugin source can be discovered , the return value from this method
 is ` None ` .
 This function can be very useful if additional data has been attached
 to the plugin source . For instance this could allow plugins to get
 access to a back reference to the application that created them .
 : param module : optionally the module to locate the plugin source of .
 : param stacklevel : defines how many levels up the module should search
 for before it discovers the plugin frame . The
 default is 0 . This can be useful for writing wrappers
 around this function ."""

implementation:
if module is None :
    frm = sys . _getframe ( ( stacklevel or 0 ) + 1 )
    name = frm . f_globals [ '__name__' ]
    glob = frm . f_globals
elif isinstance ( module , string_types ) :
    frm = sys . _getframe ( 1 )
    name = module
    glob = __import__ ( module , frm . f_globals , frm . f_locals , [ '__dict__' ] ) . __dict__
else :
    name = module . __name__
    glob = module . __dict__
return _discover_space ( name , glob )


signature:
def list_plugins ( self ) :
    """Returns a sorted list of all plugins that are available in this
 plugin source . This can be useful to automatically discover plugins
 that are available and is usually used together with
 : meth : ` load _ plugin ` ."""

implementation:
rv = [ ]
for _ , modname , ispkg in pkgutil . iter_modules ( self . mod . __path__ ) :
    rv . append ( modname )
return sorted ( rv )


signature:
def load_plugin ( self , name ) :
    """This automatically loads a plugin by the given name from the
 current source and returns the module . This is a convenient
 alternative to the import statement and saves you from invoking
 ` ` _ _ import _ _ ` ` or a similar function yourself .
 : param name : the name of the plugin to load ."""

implementation:
if '.' in name :
    raise ImportError ( 'Plugin names cannot contain dots.' )
with self :
    return __import__ ( self . base . package + '.' + name , globals ( ) , { } , [ '__name__' ] )


signature:
def open_resource ( self , plugin , filename ) :
    """This function locates a resource inside the plugin and returns
 a byte stream to the contents of it . If the resource cannot be
 loaded an : exc : ` IOError ` will be raised . Only plugins that are
 real Python packages can contain resources . Plain old Python
 modules do not allow this for obvious reasons .
 . . versionadded : : 0.3
 : param plugin : the name of the plugin to open the resource of .
 : param filename : the name of the file within the plugin to open ."""

implementation:
mod = self . load_plugin ( plugin )
fn = getattr ( mod , '__file__' , None )
if fn is not None :
    if fn . endswith ( ( '.pyc' , '.pyo' ) ) :
        fn = fn [ : - 1 ]
    if os . path . isfile ( fn ) :
        return open ( os . path . join ( os . path . dirname ( fn ) , filename ) , 'rb' )
buf = pkgutil . get_data ( self . mod . __name__ + '.' + plugin , filename )
if buf is None :
    raise IOError ( errno . ENOEXITS , 'Could not find resource' )
return NativeBytesIO ( buf )


signature:
def api_call_action ( func ) :
    """API wrapper documentation"""

implementation:
def _inner ( * args , ** kwargs ) :
    return func ( * args , ** kwargs )
_inner . __name__ = func . __name__
_inner . __doc__ = func . __doc__
return _inner


signature:
def tx ( self , message ) :
    """Transmit a series of bytes
 : param message : a list of bytes to send
 : return : None"""

implementation:
message = message if isinstance ( message , list ) else [ message ]
length = len ( message )
length_high_byte = ( length & 0xff00 ) >> 8
length_low_byte = length & 0x00ff
message_with_length = [ length_low_byte , length_high_byte ] + message
sum1 , sum2 = self . _fletcher16_checksum ( message_with_length )
message_with_length . append ( sum1 )
message_with_length . append ( sum2 )
message = [ self . _START_OF_FRAME ]
for b in message_with_length :
    if b in [ self . _START_OF_FRAME , self . _END_OF_FRAME , self . _ESC ] :
        message . append ( self . _ESC )
        message . append ( b ^ self . _ESC_XOR )
    else :
        message . append ( b )
message . append ( self . _END_OF_FRAME )
self . _port . write ( message )


signature:
def rx ( self ) :
    """Receive a series of bytes that have been verified
 : return : a series of bytes as a tuple or None if empty"""

implementation:
if not self . _threaded :
    self . run ( )
try :
    return tuple ( self . _messages . pop ( 0 ) )
except IndexError :
    return None


signature:
def _parse_raw_data ( self ) :
    """Parses the incoming data and determines if it is valid . Valid
 data gets placed into self . _ messages
 : return : None"""

implementation:
if self . _START_OF_FRAME in self . _raw and self . _END_OF_FRAME in self . _raw :
    while self . _raw [ 0 ] != self . _START_OF_FRAME and len ( self . _raw ) > 0 :
        self . _raw . pop ( 0 )
    if self . _raw [ 0 ] == self . _START_OF_FRAME :
        self . _raw . pop ( 0 )
    eof_index = self . _raw . index ( self . _END_OF_FRAME )
    raw_message = self . _raw [ : eof_index ]
    self . _raw = self . _raw [ eof_index : ]
    logger . debug ( 'raw message: {}' . format ( raw_message ) )
    message = self . _remove_esc_chars ( raw_message )
    logger . debug ( 'message with checksum: {}' . format ( message ) )
    expected_checksum = ( message [ - 1 ] << 8 ) | message [ - 2 ]
    logger . debug ( 'checksum: {}' . format ( expected_checksum ) )
    message = message [ : - 2 ]
    # checksum bytes
    logger . debug ( 'message: {}' . format ( message ) )
    sum1 , sum2 = self . _fletcher16_checksum ( message )
    calculated_checksum = ( sum2 << 8 ) | sum1
    if expected_checksum == calculated_checksum :
        message = message [ 2 : ]
        # remove length
        logger . debug ( 'valid message received: {}' . format ( message ) )
        self . _messages . append ( message )
    else :
        logger . warning ( 'invalid message received: {}, discarding' . format ( message ) )
        logger . debug ( 'expected checksum: {}, calculated checksum: {}' . format ( expected_checksum , calculated_checksum ) )
# remove any extra bytes at the beginning
try :
    while self . _raw [ 0 ] != self . _START_OF_FRAME and len ( self . _raw ) > 0 :
        self . _raw . pop ( 0 )
except IndexError :
    pass


signature:
def _fletcher16_checksum ( self , data ) :
    """Calculates a fletcher16 checksum for the list of bytes
 : param data : a list of bytes that comprise the message
 : return :"""

implementation:
sum1 = 0
sum2 = 0
for i , b in enumerate ( data ) :
    sum1 += b
    sum1 &= 0xff
    # Results wrapped at 16 bits
    sum2 += sum1
    sum2 &= 0xff
logger . debug ( 'sum1: {} sum2: {}' . format ( sum1 , sum2 ) )
return sum1 , sum2


signature:
def _remove_esc_chars ( self , raw_message ) :
    """Removes any escape characters from the message
 : param raw _ message : a list of bytes containing the un - processed data
 : return : a message that has the escaped characters appropriately un - escaped"""

implementation:
message = [ ]
escape_next = False
for c in raw_message :
    if escape_next :
        message . append ( c ^ self . _ESC_XOR )
        escape_next = False
    else :
        if c == self . _ESC :
            escape_next = True
        else :
            message . append ( c )
return message


signature:
def run ( self ) :
    """Receives the serial data into the self . _ raw buffer
 : return :"""

implementation:
run_once = True
while run_once or self . _threaded :
    waiting = self . _port . in_waiting
    if waiting > 0 :
        temp = [ int ( c ) for c in self . _port . read ( waiting ) ]
        self . _raw += temp
    self . _parse_raw_data ( )
    run_once = False
    if self . _threaded :
        time . sleep ( self . _timeout )


signature:
def save ( self ) :
    """Saves this order to Holvi , returns a tuple with the order itself and checkout _ uri"""

implementation:
if self . code :
    raise HolviError ( "Orders cannot be updated" )
send_json = self . to_holvi_dict ( )
send_json . update ( { 'pool' : self . api . connection . pool } )
url = six . u ( self . api . base_url + "order/" )
stat = self . api . connection . make_post ( url , send_json )
code = stat [ "details_uri" ] . split ( "/" ) [ - 2 ]
# Maybe slightly ugly but I don ' t want to basically reimplement all but uri formation of the api method
return ( stat [ "checkout_uri" ] , self . api . get_order ( code ) )


signature:
def untokenize ( tokens ) :
    """Return source code based on tokens .
 This is like tokenize . untokenize ( ) , but it preserves spacing between
 tokens . So if the original soure code had multiple spaces between
 some tokens or if escaped newlines were used , those things will be
 reflected by untokenize ( ) ."""

implementation:
text = ''
previous_line = ''
last_row = 0
last_column = - 1
last_non_whitespace_token_type = None
for ( token_type , token_string , start , end , line ) in tokens :
    if TOKENIZE_HAS_ENCODING and token_type == tokenize . ENCODING :
        continue
    ( start_row , start_column ) = start
    ( end_row , end_column ) = end
    # Preserve escaped newlines .
    if ( last_non_whitespace_token_type != tokenize . COMMENT and start_row > last_row and previous_line . endswith ( ( '\\\n' , '\\\r\n' , '\\\r' ) ) ) :
        text += previous_line [ len ( previous_line . rstrip ( ' \t\n\r\\' ) ) : ]
    # Preserve spacing .
    if start_row > last_row :
        last_column = 0
    if start_column > last_column :
        text += line [ last_column : start_column ]
    text += token_string
    previous_line = line
    last_row = end_row
    last_column = end_column
    if token_type not in WHITESPACE_TOKENS :
        last_non_whitespace_token_type = token_type
return text


signature:
def init ( globalvars = None , show = False ) :
    """Load profile INI"""

implementation:
global config
profileini = getprofileini ( )
if os . path . exists ( profileini ) :
    config = configparser . ConfigParser ( )
    config . read ( profileini )
    mgr = plugins_get_mgr ( )
    mgr . update_configs ( config )
    if show :
        for source in config :
            print ( "[%s] :" % ( source ) )
            for k in config [ source ] :
                print ( "   %s : %s" % ( k , config [ source ] [ k ] ) )
else :
    print ( "Profile does not exist. So creating one" )
    if not show :
        update ( globalvars )
print ( "Complete init" )


signature:
def update ( globalvars ) :
    """Update the profile"""

implementation:
global config
profileini = getprofileini ( )
config = configparser . ConfigParser ( )
config . read ( profileini )
defaults = { }
if globalvars is not None :
    defaults = { a [ 0 ] : a [ 1 ] for a in globalvars }
# Generic variables to be captured . . .
generic_configs = [ { 'name' : 'User' , 'nature' : 'generic' , 'description' : "General information" , 'variables' : [ 'user.email' , 'user.name' , 'user.fullname' ] , 'defaults' : { 'user.email' : { 'value' : defaults . get ( 'user.email' , '' ) , 'description' : "Email address" , 'validator' : EmailValidator ( ) } , 'user.fullname' : { 'value' : defaults . get ( 'user.fullname' , '' ) , 'description' : "Full Name" , 'validator' : NonEmptyValidator ( ) } , 'user.name' : { 'value' : defaults . get ( 'user.name' , getpass . getuser ( ) ) , 'description' : "Name" , 'validator' : NonEmptyValidator ( ) } , } } ]
# Gather configuration requirements from all plugins
mgr = plugins_get_mgr ( )
extra_configs = mgr . gather_configs ( )
allconfigs = generic_configs + extra_configs
# Read the existing config and update the defaults
for c in allconfigs :
    name = c [ 'name' ]
    for v in c [ 'variables' ] :
        try :
            c [ 'defaults' ] [ v ] [ 'value' ] = config [ name ] [ v ]
        except :
            continue
for c in allconfigs :
    print ( "" )
    print ( c [ 'description' ] )
    print ( "==================" )
    if len ( c [ 'variables' ] ) == 0 :
        print ( "Nothing to do. Enabled by default" )
        continue
    name = c [ 'name' ]
    config [ name ] = { }
    config [ name ] [ 'nature' ] = c [ 'nature' ]
    for v in c [ 'variables' ] : # defaults
        value = ''
        description = v + " "
        helptext = ""
        validator = None
        # Look up pre - set values
        if v in c [ 'defaults' ] :
            value = c [ 'defaults' ] [ v ] . get ( 'value' , '' )
            helptext = c [ 'defaults' ] [ v ] . get ( "description" , "" )
            validator = c [ 'defaults' ] [ v ] . get ( 'validator' , None )
        if helptext != "" :
            description += "(" + helptext + ")"
        # Get user input . .
        while True :
            choice = input_with_default ( description , value )
            if validator is not None :
                if validator . is_valid ( choice ) :
                    break
                else :
                    print ( "Invalid input. Expected input is {}" . format ( validator . message ) )
            else :
                break
        config [ name ] [ v ] = choice
        if v == 'enable' and choice == 'n' :
            break
with open ( profileini , 'w' ) as fd :
    config . write ( fd )
print ( "Updated profile file:" , config )


signature:
def init_repo ( self , gitdir ) :
    """Insert hook into the repo"""

implementation:
hooksdir = os . path . join ( gitdir , 'hooks' )
content = postreceive_template % { 'client' : self . client , 'bucket' : self . bucket , 's3cfg' : self . s3cfg , 'prefix' : self . prefix }
postrecv_filename = os . path . join ( hooksdir , 'post-receive' )
with open ( postrecv_filename , 'w' ) as fd :
    fd . write ( content )
self . make_hook_executable ( postrecv_filename )
print ( "Wrote to" , postrecv_filename )


signature:
def compute_sha256 ( filename ) :
    """Try the library . If it doesnt work , use the command line . ."""

implementation:
try :
    h = sha256 ( )
    fd = open ( filename , 'rb' )
    while True :
        buf = fd . read ( 0x1000000 )
        if buf in [ None , "" ] :
            break
        h . update ( buf . encode ( 'utf-8' ) )
    fd . close ( )
    return h . hexdigest ( )
except :
    output = run ( [ "sha256sum" , "-b" , filename ] )
    return output . split ( " " ) [ 0 ]


signature:
def run ( cmd ) :
    """Run a shell command"""

implementation:
cmd = [ pipes . quote ( c ) for c in cmd ]
cmd = " " . join ( cmd )
cmd += "; exit 0"
# print ( " Running { } in { } " . format ( cmd , os . getcwd ( ) ) )
try :
    output = subprocess . check_output ( cmd , stderr = subprocess . STDOUT , shell = True )
except subprocess . CalledProcessError as e :
    output = e . output
output = output . decode ( 'utf-8' )
output = output . strip ( )
return output


signature:
def log_repo_action ( func ) :
    """Log all repo actions to . dgit / log . json"""

implementation:
def _inner ( * args , ** kwargs ) :
    result = func ( * args , ** kwargs )
    log_action ( func , result , * args , ** kwargs )
    return result
_inner . __name__ = func . __name__
_inner . __doc__ = func . __doc__
return _inner


signature:
def get_tree ( gitdir = "." ) :
    """Get the commit history for a given dataset"""

implementation:
cmd = [ "git" , "log" , "--all" , "--branches" , '--pretty=format:{  "commit": "%H",  "abbreviated_commit": "%h",  "tree": "%T",  "abbreviated_tree": "%t",  "parent": "%P",  "abbreviated_parent": "%p",  "refs": "%d",  "encoding": "%e",  "subject": "%s", "sanitized_subject_line": "%f",  "commit_notes": "",  "author": {    "name": "%aN",    "email": "%aE",    "date": "%ai"  },  "commiter": {    "name": "%cN",    "email": "%cE",    "date": "%ci"  }},' ]
output = run ( cmd )
lines = output . split ( "\n" )
content = ""
history = [ ]
for l in lines :
    try :
        revisedcontent = content + l
        if revisedcontent . count ( '"' ) % 2 == 0 :
            j = json . loads ( revisedcontent [ : - 1 ] )
            if "Notes added by" in j [ 'subject' ] :
                content = ""
                continue
            history . append ( j )
            content = ""
        else :
            content = revisedcontent
    except Exception as e :
        print ( "Error while parsing record" )
        print ( revisedcontent )
        content = ""
# Order by time . First commit first . . .
history . reverse ( )
changes = get_change ( )
for i in range ( len ( history ) ) :
    abbrev_commit = history [ i ] [ 'abbreviated_commit' ]
    if abbrev_commit not in changes :
        raise Exception ( "Missing changes for " + abbrev_commit )
    history [ i ] [ 'changes' ] = changes [ abbrev_commit ] [ 'changes' ]
return history


signature:
def get_diffs ( history ) :
    """Look at files and compute the diffs intelligently"""

implementation:
# First get all possible representations
mgr = plugins_get_mgr ( )
keys = mgr . search ( 'representation' ) [ 'representation' ]
representations = [ mgr . get_by_key ( 'representation' , k ) for k in keys ]
for i in range ( len ( history ) ) :
    if i + 1 > len ( history ) - 1 :
        continue
    prev = history [ i ]
    curr = history [ i + 1 ]
    # print ( prev [ ' subject ' ] , " = = > " , curr [ ' subject ' ] )
    # print ( curr [ ' changes ' ] )
    for c in curr [ 'changes' ] :
        path = c [ 'path' ]
        # Skip the metadata file
        if c [ 'path' ] . endswith ( 'datapackage.json' ) :
            continue
        # Find a handler for this kind of file . . .
        handler = None
        for r in representations :
            if r . can_process ( path ) :
                handler = r
                break
        if handler is None :
            continue
        # print ( path , " being handled by " , handler )
        v1_hex = prev [ 'commit' ]
        v2_hex = curr [ 'commit' ]
        temp1 = tempfile . mkdtemp ( prefix = "dgit-diff-" )
        try :
            for h in [ v1_hex , v2_hex ] :
                filename = '{}/{}/checkout.tar' . format ( temp1 , h )
                try :
                    os . makedirs ( os . path . dirname ( filename ) )
                except :
                    pass
                extractcmd = [ 'git' , 'archive' , '-o' , filename , h , path ]
                output = run ( extractcmd )
                if 'fatal' in output :
                    raise Exception ( "File not present in commit" )
                with cd ( os . path . dirname ( filename ) ) :
                    cmd = [ 'tar' , 'xvf' , 'checkout.tar' ]
                    output = run ( cmd )
                    if 'fatal' in output :
                        print ( "Cleaning up - fatal 1" , temp1 )
                        shutil . rmtree ( temp1 )
                        continue
            # Check to make sure that
            path1 = os . path . join ( temp1 , v1_hex , path )
            path2 = os . path . join ( temp1 , v2_hex , path )
            if not os . path . exists ( path1 ) or not os . path . exists ( path2 ) : # print ( " One of the two output files is missing " )
                shutil . rmtree ( temp1 )
                continue
            # print ( path1 , path2)
            # Now call the handler
            diff = handler . get_diff ( path1 , path2 )
            # print ( " Inserting diff " , diff )
            c [ 'diff' ] = diff
        except Exception as e : # traceback . print _ exc ( )
        # print ( " Cleaning up - Exception " , temp1)
            shutil . rmtree ( temp1 )


signature:
def chdir ( self , new_pwd , relative = True ) :
    """Parameters
 new _ pwd : str ,
 Directory to change to
 relative : bool , default True
 If True then the given directory is treated as relative to the
 current directory"""

implementation:
if new_pwd and self . pwd and relative :
    new_pwd = os . path . join ( self . pwd , new_pwd )
self . pwd = new_pwd


signature:
def exec_command ( self , cmd ) :
    """Proceed with caution , if you run a command that causes a prompt and then
 try to read / print the stdout it ' s going to block forever
 Returns
 ( stdin , stdout , stderr )"""

implementation:
if self . pwd is not None :
    cmd = 'cd %s ; %s' % ( self . pwd , cmd )
if self . interactive :
    print ( cmd )
return self . con . exec_command ( cmd )


signature:
def wait ( self , cmd , raise_on_error = True ) :
    """Execute command and wait for it to finish . Proceed with caution because
 if you run a command that causes a prompt this will hang"""

implementation:
_ , stdout , stderr = self . exec_command ( cmd )
stdout . channel . recv_exit_status ( )
output = stdout . read ( )
if self . interactive :
    print ( output )
errors = stderr . read ( )
if self . interactive :
    print ( errors )
if errors and raise_on_error :
    raise ValueError ( errors )
return output


signature:
def sudo ( self , password = None ) :
    """Enter sudo mode"""

implementation:
if self . username == 'root' :
    raise ValueError ( 'Already root user' )
password = self . validate_password ( password )
stdin , stdout , stderr = self . exec_command ( 'sudo su' )
stdin . write ( "%s\n" % password )
stdin . flush ( )
errors = stderr . read ( )
if errors :
    raise ValueError ( errors )


signature:
def apt ( self , package_names , raise_on_error = False ) :
    """Install specified packages using apt - get . - y options are
 automatically used . Waits for command to finish .
 Parameters
 package _ names : list - like of str
 raise _ on _ error : bool , default False
 If True then raise ValueError if stderr is not empty
 debconf often gives tty error"""

implementation:
if isinstance ( package_names , basestring ) :
    package_names = [ package_names ]
cmd = "apt-get install -y %s" % ( ' ' . join ( package_names ) )
return self . wait ( cmd , raise_on_error = raise_on_error )


signature:
def pip ( self , package_names , raise_on_error = True ) :
    """Install specified python packages using pip . - U option added
 Waits for command to finish .
 Parameters
 package _ names : list - like of str
 raise _ on _ error : bool , default True
 If True then raise ValueError if stderr is not empty"""

implementation:
if isinstance ( package_names , basestring ) :
    package_names = [ package_names ]
cmd = "pip install -U %s" % ( ' ' . join ( package_names ) )
return self . wait ( cmd , raise_on_error = raise_on_error )


signature:
def pip_r ( self , requirements , raise_on_error = True ) :
    """Install all requirements contained in the given file path
 Waits for command to finish .
 Parameters
 requirements : str
 Path to requirements . txt
 raise _ on _ error : bool , default True
 If True then raise ValueError if stderr is not empty"""

implementation:
cmd = "pip install -r %s" % requirements
return self . wait ( cmd , raise_on_error = raise_on_error )


signature:
def git ( self , username , repo , alias = None , token = None ) :
    """Parameters
 token : str , default None
 Assumes you have GITHUB _ TOKEN in envvar if None
 https : / / github . com / blog / 1270 - easier - builds - and - deployments - using - git -
 over - https - and - oauth"""

implementation:
if alias is None :
    alias = repo
if token is None :
    token = os . environ . get ( 'GITHUB_TOKEN' )
self . wait ( 'mkdir -p %s' % alias )
old_dir = self . pwd
try :
    self . chdir ( alias , relative = True )
    cmd = 'git init && git pull https://%s@github.com/%s/%s.git'
    # last line to stderr
    return self . wait ( cmd % ( token , username , repo ) , raise_on_error = False )
finally :
    self . chdir ( old_dir , relative = False )