signature:
def _verify ( certificate_or_public_key , signature , data , hash_algorithm ) :
    """Verifies an RSA , DSA or ECDSA signature
 : param certificate _ or _ public _ key :
 A Certificate or PublicKey instance to verify the signature with
 : param signature :
 A byte string of the signature to verify
 : param data :
 A byte string of the data the signature is for
 : param hash _ algorithm :
 A unicode string of " md5 " , " sha1 " , " sha224 " , " sha256 " , " sha384 " or " sha512"
 : raises :
 oscrypto . errors . SignatureError - when the signature is determined to be invalid
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library"""

implementation:
if not isinstance ( certificate_or_public_key , ( Certificate , PublicKey ) ) :
    raise TypeError ( pretty_message ( '''
            certificate_or_public_key must be an instance of the Certificate or
            PublicKey class, not %s
            ''' , type_name ( certificate_or_public_key ) ) )
if not isinstance ( signature , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            signature must be a byte string, not %s
            ''' , type_name ( signature ) ) )
if not isinstance ( data , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            data must be a byte string, not %s
            ''' , type_name ( data ) ) )
valid_hash_algorithms = set ( [ 'md5' , 'sha1' , 'sha224' , 'sha256' , 'sha384' , 'sha512' ] )
if certificate_or_public_key . algorithm == 'rsa' :
    valid_hash_algorithms |= set ( [ 'raw' ] )
if hash_algorithm not in valid_hash_algorithms :
    valid_hash_algorithms_error = '"md5", "sha1", "sha224", "sha256", "sha384", "sha512"'
    if certificate_or_public_key . algorithm == 'rsa' :
        valid_hash_algorithms_error += ', "raw"'
    raise ValueError ( pretty_message ( '''
            hash_algorithm must be one of %s, not %s
            ''' , valid_hash_algorithms_error , repr ( hash_algorithm ) ) )
if certificate_or_public_key . algorithm == 'rsa' and hash_algorithm == 'raw' :
    if len ( data ) > certificate_or_public_key . byte_size - 11 :
        raise ValueError ( pretty_message ( '''
                data must be 11 bytes shorter than the key size when
                hash_algorithm is "raw" - key size is %s bytes, but data
                is %s bytes long
                ''' , certificate_or_public_key . byte_size , len ( data ) ) )
    result = Security . SecKeyRawVerify ( certificate_or_public_key . sec_key_ref , SecurityConst . kSecPaddingPKCS1 , data , len ( data ) , signature , len ( signature ) )
    # errSSLCrypto is returned in some situations on macOS 10.12
    if result == SecurityConst . errSecVerifyFailed or result == SecurityConst . errSSLCrypto :
        raise SignatureError ( 'Signature is invalid' )
    handle_sec_error ( result )
    return
cf_signature = None
cf_data = None
cf_hash_length = None
sec_transform = None
try :
    error_pointer = new ( CoreFoundation , 'CFErrorRef *' )
    cf_signature = CFHelpers . cf_data_from_bytes ( signature )
    sec_transform = Security . SecVerifyTransformCreate ( certificate_or_public_key . sec_key_ref , cf_signature , error_pointer )
    handle_cf_error ( error_pointer )
    hash_constant = { 'md5' : Security . kSecDigestMD5 , 'sha1' : Security . kSecDigestSHA1 , 'sha224' : Security . kSecDigestSHA2 , 'sha256' : Security . kSecDigestSHA2 , 'sha384' : Security . kSecDigestSHA2 , 'sha512' : Security . kSecDigestSHA2 } [ hash_algorithm ]
    Security . SecTransformSetAttribute ( sec_transform , Security . kSecDigestTypeAttribute , hash_constant , error_pointer )
    handle_cf_error ( error_pointer )
    if hash_algorithm in set ( [ 'sha224' , 'sha256' , 'sha384' , 'sha512' ] ) :
        hash_length = { 'sha224' : 224 , 'sha256' : 256 , 'sha384' : 384 , 'sha512' : 512 } [ hash_algorithm ]
        cf_hash_length = CFHelpers . cf_number_from_integer ( hash_length )
        Security . SecTransformSetAttribute ( sec_transform , Security . kSecDigestLengthAttribute , cf_hash_length , error_pointer )
        handle_cf_error ( error_pointer )
    if certificate_or_public_key . algorithm == 'rsa' :
        Security . SecTransformSetAttribute ( sec_transform , Security . kSecPaddingKey , Security . kSecPaddingPKCS1Key , error_pointer )
        handle_cf_error ( error_pointer )
    cf_data = CFHelpers . cf_data_from_bytes ( data )
    Security . SecTransformSetAttribute ( sec_transform , Security . kSecTransformInputAttributeName , cf_data , error_pointer )
    handle_cf_error ( error_pointer )
    res = Security . SecTransformExecute ( sec_transform , error_pointer )
    if not is_null ( error_pointer ) :
        error = unwrap ( error_pointer )
        if not is_null ( error ) :
            raise SignatureError ( 'Signature is invalid' )
    res = bool ( CoreFoundation . CFBooleanGetValue ( res ) )
    if not res :
        raise SignatureError ( 'Signature is invalid' )
finally :
    if sec_transform :
        CoreFoundation . CFRelease ( sec_transform )
    if cf_signature :
        CoreFoundation . CFRelease ( cf_signature )
    if cf_data :
        CoreFoundation . CFRelease ( cf_data )
    if cf_hash_length :
        CoreFoundation . CFRelease ( cf_hash_length )


signature:
def rsa_pss_sign ( private_key , data , hash_algorithm ) :
    """Generates an RSASSA - PSS signature . For the PSS padding the mask gen
 algorithm will be mgf1 using the same hash algorithm as the signature . The
 salt length with be the length of the hash algorithm , and the trailer field
 with be the standard 0xBC byte .
 : param private _ key :
 The PrivateKey to generate the signature with
 : param data :
 A byte string of the data the signature is for
 : param hash _ algorithm :
 A unicode string of " md5 " , " sha1 " , " sha224 " , " sha256 " , " sha384 " or
 " sha512"
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A byte string of the signature"""

implementation:
if not isinstance ( private_key , PrivateKey ) :
    raise TypeError ( pretty_message ( '''
            private_key must be an instance of the PrivateKey class, not %s
            ''' , type_name ( private_key ) ) )
if not isinstance ( data , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            data must be a byte string, not %s
            ''' , type_name ( data ) ) )
if private_key . algorithm != 'rsa' :
    raise ValueError ( 'The key specified is not an RSA private key' )
hash_length = { 'sha1' : 20 , 'sha224' : 28 , 'sha256' : 32 , 'sha384' : 48 , 'sha512' : 64 } . get ( hash_algorithm , 0 )
encoded_data = add_pss_padding ( hash_algorithm , hash_length , private_key . bit_size , data )
key_length = private_key . byte_size
buffer = buffer_from_bytes ( key_length )
output_length = new ( Security , 'size_t *' , key_length )
result = Security . SecKeyDecrypt ( private_key . sec_key_ref , SecurityConst . kSecPaddingNone , encoded_data , len ( encoded_data ) , buffer , output_length )
handle_sec_error ( result )
return bytes_from_buffer ( buffer , deref ( output_length ) )


signature:
def _sign ( private_key , data , hash_algorithm ) :
    """Generates an RSA , DSA or ECDSA signature
 : param private _ key :
 The PrivateKey to generate the signature with
 : param data :
 A byte string of the data the signature is for
 : param hash _ algorithm :
 A unicode string of " md5 " , " sha1 " , " sha224 " , " sha256 " , " sha384 " or
 " sha512"
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A byte string of the signature"""

implementation:
if not isinstance ( private_key , PrivateKey ) :
    raise TypeError ( pretty_message ( '''
            private_key must be an instance of PrivateKey, not %s
            ''' , type_name ( private_key ) ) )
if not isinstance ( data , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            data must be a byte string, not %s
            ''' , type_name ( data ) ) )
valid_hash_algorithms = set ( [ 'md5' , 'sha1' , 'sha224' , 'sha256' , 'sha384' , 'sha512' ] )
if private_key . algorithm == 'rsa' :
    valid_hash_algorithms |= set ( [ 'raw' ] )
if hash_algorithm not in valid_hash_algorithms :
    valid_hash_algorithms_error = '"md5", "sha1", "sha224", "sha256", "sha384", "sha512"'
    if private_key . algorithm == 'rsa' :
        valid_hash_algorithms_error += ', "raw"'
    raise ValueError ( pretty_message ( '''
            hash_algorithm must be one of %s, not %s
            ''' , valid_hash_algorithms_error , repr ( hash_algorithm ) ) )
if private_key . algorithm == 'rsa' and hash_algorithm == 'raw' :
    if len ( data ) > private_key . byte_size - 11 :
        raise ValueError ( pretty_message ( '''
                data must be 11 bytes shorter than the key size when
                hash_algorithm is "raw" - key size is %s bytes, but
                data is %s bytes long
                ''' , private_key . byte_size , len ( data ) ) )
    key_length = private_key . byte_size
    buffer = buffer_from_bytes ( key_length )
    output_length = new ( Security , 'size_t *' , key_length )
    result = Security . SecKeyRawSign ( private_key . sec_key_ref , SecurityConst . kSecPaddingPKCS1 , data , len ( data ) , buffer , output_length )
    handle_sec_error ( result )
    return bytes_from_buffer ( buffer , deref ( output_length ) )
cf_signature = None
cf_data = None
cf_hash_length = None
sec_transform = None
try :
    error_pointer = new ( CoreFoundation , 'CFErrorRef *' )
    sec_transform = Security . SecSignTransformCreate ( private_key . sec_key_ref , error_pointer )
    handle_cf_error ( error_pointer )
    hash_constant = { 'md5' : Security . kSecDigestMD5 , 'sha1' : Security . kSecDigestSHA1 , 'sha224' : Security . kSecDigestSHA2 , 'sha256' : Security . kSecDigestSHA2 , 'sha384' : Security . kSecDigestSHA2 , 'sha512' : Security . kSecDigestSHA2 } [ hash_algorithm ]
    Security . SecTransformSetAttribute ( sec_transform , Security . kSecDigestTypeAttribute , hash_constant , error_pointer )
    handle_cf_error ( error_pointer )
    if hash_algorithm in set ( [ 'sha224' , 'sha256' , 'sha384' , 'sha512' ] ) :
        hash_length = { 'sha224' : 224 , 'sha256' : 256 , 'sha384' : 384 , 'sha512' : 512 } [ hash_algorithm ]
        cf_hash_length = CFHelpers . cf_number_from_integer ( hash_length )
        Security . SecTransformSetAttribute ( sec_transform , Security . kSecDigestLengthAttribute , cf_hash_length , error_pointer )
        handle_cf_error ( error_pointer )
    if private_key . algorithm == 'rsa' :
        Security . SecTransformSetAttribute ( sec_transform , Security . kSecPaddingKey , Security . kSecPaddingPKCS1Key , error_pointer )
        handle_cf_error ( error_pointer )
    cf_data = CFHelpers . cf_data_from_bytes ( data )
    Security . SecTransformSetAttribute ( sec_transform , Security . kSecTransformInputAttributeName , cf_data , error_pointer )
    handle_cf_error ( error_pointer )
    cf_signature = Security . SecTransformExecute ( sec_transform , error_pointer )
    handle_cf_error ( error_pointer )
    return CFHelpers . cf_data_to_bytes ( cf_signature )
finally :
    if sec_transform :
        CoreFoundation . CFRelease ( sec_transform )
    if cf_signature :
        CoreFoundation . CFRelease ( cf_signature )
    if cf_data :
        CoreFoundation . CFRelease ( cf_data )
    if cf_hash_length :
        CoreFoundation . CFRelease ( cf_hash_length )


signature:
def public_key ( self ) :
    """: return :
 The PublicKey object for the public key this certificate contains"""

implementation:
if not self . _public_key and self . sec_certificate_ref :
    sec_public_key_ref_pointer = new ( Security , 'SecKeyRef *' )
    res = Security . SecCertificateCopyPublicKey ( self . sec_certificate_ref , sec_public_key_ref_pointer )
    handle_sec_error ( res )
    sec_public_key_ref = unwrap ( sec_public_key_ref_pointer )
    self . _public_key = PublicKey ( sec_public_key_ref , self . asn1 [ 'tbs_certificate' ] [ 'subject_public_key_info' ] )
return self . _public_key


signature:
def backend ( ) :
    """: return :
 A unicode string of the backend being used : " openssl " , " osx " , " win " ,
 " winlegacy " """

implementation:
if _module_values [ 'backend' ] is not None :
    return _module_values [ 'backend' ]
with _backend_lock :
    if _module_values [ 'backend' ] is not None :
        return _module_values [ 'backend' ]
    if sys . platform == 'win32' : # Windows XP was major version 5 , Vista was 6
        if sys . getwindowsversion ( ) [ 0 ] < 6 :
            _module_values [ 'backend' ] = 'winlegacy'
        else :
            _module_values [ 'backend' ] = 'win'
    elif sys . platform == 'darwin' :
        _module_values [ 'backend' ] = 'osx'
    else :
        _module_values [ 'backend' ] = 'openssl'
    return _module_values [ 'backend' ]


signature:
def use_openssl ( libcrypto_path , libssl_path , trust_list_path = None ) :
    """Forces using OpenSSL dynamic libraries on OS X ( . dylib ) or Windows ( . dll ) ,
 or using a specific dynamic library on Linux / BSD ( . so ) .
 This can also be used to configure oscrypto to use LibreSSL dynamic
 libraries .
 This method must be called before any oscrypto submodules are imported .
 : param libcrypto _ path :
 A unicode string of the file path to the OpenSSL / LibreSSL libcrypto
 dynamic library .
 : param libssl _ path :
 A unicode string of the file path to the OpenSSL / LibreSSL libssl
 dynamic library .
 : param trust _ list _ path :
 An optional unicode string of the path to a file containing
 OpenSSL - compatible CA certificates in PEM format . If this is not
 provided and the platform is OS X or Windows , the system trust roots
 will be exported from the OS and used for all TLS connections .
 : raises :
 ValueError - when one of the paths is not a unicode string
 OSError - when the trust _ list _ path does not exist on the filesystem
 oscrypto . errors . LibraryNotFoundError - when one of the path does not exist on the filesystem
 RuntimeError - when this function is called after another part of oscrypto has been imported"""

implementation:
if not isinstance ( libcrypto_path , str_cls ) :
    raise ValueError ( 'libcrypto_path must be a unicode string, not %s' % type_name ( libcrypto_path ) )
if not isinstance ( libssl_path , str_cls ) :
    raise ValueError ( 'libssl_path must be a unicode string, not %s' % type_name ( libssl_path ) )
if not os . path . exists ( libcrypto_path ) :
    raise LibraryNotFoundError ( 'libcrypto does not exist at %s' % libcrypto_path )
if not os . path . exists ( libssl_path ) :
    raise LibraryNotFoundError ( 'libssl does not exist at %s' % libssl_path )
if trust_list_path is not None :
    if not isinstance ( trust_list_path , str_cls ) :
        raise ValueError ( 'trust_list_path must be a unicode string, not %s' % type_name ( trust_list_path ) )
    if not os . path . exists ( trust_list_path ) :
        raise OSError ( 'trust_list_path does not exist at %s' % trust_list_path )
with _backend_lock :
    if _module_values [ 'backend' ] is not None :
        raise RuntimeError ( 'Another part of oscrypto has already been imported, unable to force use of OpenSSL' )
    _module_values [ 'backend' ] = 'openssl'
    _module_values [ 'backend_config' ] = { 'libcrypto_path' : libcrypto_path , 'libssl_path' : libssl_path , 'trust_list_path' : trust_list_path , }


signature:
def use_winlegacy ( ) :
    """Forces use of the legacy Windows CryptoAPI . This should only be used on
 Windows XP or for testing . It is less full - featured than the Cryptography
 Next Generation ( CNG ) API , and as a result the elliptic curve and PSS
 padding features are implemented in pure Python . This isn ' t ideal , but it
 a shim for end - user client code . No one is going to run a server on Windows
 XP anyway , right ? !
 : raises :
 EnvironmentError - when this function is called on an operating system other than Windows
 RuntimeError - when this function is called after another part of oscrypto has been imported"""

implementation:
if sys . platform != 'win32' :
    plat = platform . system ( ) or sys . platform
    if plat == 'Darwin' :
        plat = 'OS X'
    raise EnvironmentError ( 'The winlegacy backend can only be used on Windows, not %s' % plat )
with _backend_lock :
    if _module_values [ 'backend' ] is not None :
        raise RuntimeError ( 'Another part of oscrypto has already been imported, unable to force use of Windows legacy CryptoAPI' )
    _module_values [ 'backend' ] = 'winlegacy'


signature:
def pkcs12_kdf ( hash_algorithm , password , salt , iterations , key_length , id_ ) :
    """KDF from RFC7292 appendix b . 2 - https : / / tools . ietf . org / html / rfc7292 # page - 19
 : param hash _ algorithm :
 The string name of the hash algorithm to use : " md5 " , " sha1 " , " sha224 " ,
 " sha256 " , " sha384 " , " sha512"
 : param password :
 A byte string of the password to use an input to the KDF
 : param salt :
 A cryptographic random byte string
 : param iterations :
 The numbers of iterations to use when deriving the key
 : param key _ length :
 The length of the desired key in bytes
 : param id _ :
 The ID of the usage - 1 for key , 2 for iv , 3 for mac
 : return :
 The derived key as a byte string"""

implementation:
if not isinstance ( password , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            password must be a byte string, not %s
            ''' , type_name ( password ) ) )
if not isinstance ( salt , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            salt must be a byte string, not %s
            ''' , type_name ( salt ) ) )
if not isinstance ( iterations , int_types ) :
    raise TypeError ( pretty_message ( '''
            iterations must be an integer, not %s
            ''' , type_name ( iterations ) ) )
if iterations < 1 :
    raise ValueError ( pretty_message ( '''
            iterations must be greater than 0 - is %s
            ''' , repr ( iterations ) ) )
if not isinstance ( key_length , int_types ) :
    raise TypeError ( pretty_message ( '''
            key_length must be an integer, not %s
            ''' , type_name ( key_length ) ) )
if key_length < 1 :
    raise ValueError ( pretty_message ( '''
            key_length must be greater than 0 - is %s
            ''' , repr ( key_length ) ) )
if hash_algorithm not in set ( [ 'md5' , 'sha1' , 'sha224' , 'sha256' , 'sha384' , 'sha512' ] ) :
    raise ValueError ( pretty_message ( '''
            hash_algorithm must be one of "md5", "sha1", "sha224", "sha256",
            "sha384", "sha512", not %s
            ''' , repr ( hash_algorithm ) ) )
if id_ not in set ( [ 1 , 2 , 3 ] ) :
    raise ValueError ( pretty_message ( '''
            id_ must be one of 1, 2, 3, not %s
            ''' , repr ( id_ ) ) )
utf16_password = password . decode ( 'utf-8' ) . encode ( 'utf-16be' ) + b'\x00\x00'
algo = getattr ( hashlib , hash_algorithm )
# u and v values are bytes ( not bits as in the RFC )
u = { 'md5' : 16 , 'sha1' : 20 , 'sha224' : 28 , 'sha256' : 32 , 'sha384' : 48 , 'sha512' : 64 } [ hash_algorithm ]
if hash_algorithm in [ 'sha384' , 'sha512' ] :
    v = 128
else :
    v = 64
# Step 1
d = chr_cls ( id_ ) * v
# Step 2
s = b''
if salt != b'' :
    s_len = v * int ( math . ceil ( float ( len ( salt ) ) / v ) )
    while len ( s ) < s_len :
        s += salt
    s = s [ 0 : s_len ]
# Step 3
p = b''
if utf16_password != b'' :
    p_len = v * int ( math . ceil ( float ( len ( utf16_password ) ) / v ) )
    while len ( p ) < p_len :
        p += utf16_password
    p = p [ 0 : p_len ]
# Step 4
i = s + p
# Step 5
c = int ( math . ceil ( float ( key_length ) / u ) )
a = b'\x00' * ( c * u )
for num in range ( 1 , c + 1 ) : # Step 6A
    a2 = algo ( d + i ) . digest ( )
    for _ in range ( 2 , iterations + 1 ) :
        a2 = algo ( a2 ) . digest ( )
    if num < c : # Step 6B
        b = b''
        while len ( b ) < v :
            b += a2
        b = int_from_bytes ( b [ 0 : v ] ) + 1
        # Step 6C
        for num2 in range ( 0 , len ( i ) // v ) :
            start = num2 * v
            end = ( num2 + 1 ) * v
            i_num2 = i [ start : end ]
            i_num2 = int_to_bytes ( int_from_bytes ( i_num2 ) + b )
            # Ensure the new slice is the right size
            i_num2_l = len ( i_num2 )
            if i_num2_l > v :
                i_num2 = i_num2 [ i_num2_l - v : ]
            i = i [ 0 : start ] + i_num2 + i [ end : ]
    # Step 7 ( one peice at a time )
    begin = ( num - 1 ) * u
    to_copy = min ( key_length , u )
    a = a [ 0 : begin ] + a2 [ 0 : to_copy ] + a [ begin + to_copy : ]
return a [ 0 : key_length ]


signature:
def pbkdf2_iteration_calculator ( hash_algorithm , key_length , target_ms = 100 , quiet = False ) :
    """Runs pbkdf2 ( ) twice to determine the approximate number of iterations to
 use to hit a desired time per run . Use this on a production machine to
 dynamically adjust the number of iterations as high as you can .
 : param hash _ algorithm :
 The string name of the hash algorithm to use : " md5 " , " sha1 " , " sha224 " ,
 " sha256 " , " sha384 " , " sha512"
 : param key _ length :
 The length of the desired key in bytes
 : param target _ ms :
 The number of milliseconds the derivation should take
 : param quiet :
 If no output should be printed as attempts are made
 : return :
 An integer number of iterations of PBKDF2 using the specified hash
 that will take at least target _ ms"""

implementation:
if hash_algorithm not in set ( [ 'sha1' , 'sha224' , 'sha256' , 'sha384' , 'sha512' ] ) :
    raise ValueError ( pretty_message ( '''
            hash_algorithm must be one of "sha1", "sha224", "sha256", "sha384",
            "sha512", not %s
            ''' , repr ( hash_algorithm ) ) )
if not isinstance ( key_length , int_types ) :
    raise TypeError ( pretty_message ( '''
            key_length must be an integer, not %s
            ''' , type_name ( key_length ) ) )
if key_length < 1 :
    raise ValueError ( pretty_message ( '''
            key_length must be greater than 0 - is %s
            ''' , repr ( key_length ) ) )
if not isinstance ( target_ms , int_types ) :
    raise TypeError ( pretty_message ( '''
            target_ms must be an integer, not %s
            ''' , type_name ( target_ms ) ) )
if target_ms < 1 :
    raise ValueError ( pretty_message ( '''
            target_ms must be greater than 0 - is %s
            ''' , repr ( target_ms ) ) )
if pbkdf2 . pure_python :
    raise OSError ( pretty_message ( '''
            Only a very slow, pure-python version of PBKDF2 is available,
            making this function useless
            ''' ) )
iterations = 10000
password = 'this is a test' . encode ( 'utf-8' )
salt = rand_bytes ( key_length )
def _measure ( ) :
    start = _get_start ( )
    pbkdf2 ( hash_algorithm , password , salt , iterations , key_length )
    observed_ms = _get_elapsed ( start )
    if not quiet :
        print ( '%s iterations in %sms' % ( iterations , observed_ms ) )
    return 1.0 / target_ms * observed_ms
# Measure the initial guess , then estimate how many iterations it would
# take to reach 1/2 of the target ms and try it to get a good final number
fraction = _measure ( )
iterations = int ( iterations / fraction / 2.0 )
fraction = _measure ( )
iterations = iterations / fraction
# < 20,000 round to 1000
# 20,000-100,000 round to 5,000
# > 100,000 round to 10,000
round_factor = - 3 if iterations < 100000 else - 4
result = int ( round ( iterations , round_factor ) )
if result > 20000 :
    result = ( result // 5000 ) * 5000
return result


signature:
def pbkdf1 ( hash_algorithm , password , salt , iterations , key_length ) :
    """An implementation of PBKDF1 - should only be used for interop with legacy
 systems , not new architectures
 : param hash _ algorithm :
 The string name of the hash algorithm to use : " md2 " , " md5 " , " sha1"
 : param password :
 A byte string of the password to use an input to the KDF
 : param salt :
 A cryptographic random byte string
 : param iterations :
 The numbers of iterations to use when deriving the key
 : param key _ length :
 The length of the desired key in bytes
 : return :
 The derived key as a byte string"""

implementation:
if not isinstance ( password , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            password must be a byte string, not %s
            ''' , ( type_name ( password ) ) ) )
if not isinstance ( salt , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            salt must be a byte string, not %s
            ''' , ( type_name ( salt ) ) ) )
if not isinstance ( iterations , int_types ) :
    raise TypeError ( pretty_message ( '''
            iterations must be an integer, not %s
            ''' , ( type_name ( iterations ) ) ) )
if iterations < 1 :
    raise ValueError ( pretty_message ( '''
            iterations must be greater than 0 - is %s
            ''' , repr ( iterations ) ) )
if not isinstance ( key_length , int_types ) :
    raise TypeError ( pretty_message ( '''
            key_length must be an integer, not %s
            ''' , ( type_name ( key_length ) ) ) )
if key_length < 1 :
    raise ValueError ( pretty_message ( '''
            key_length must be greater than 0 - is %s
            ''' , repr ( key_length ) ) )
if hash_algorithm not in set ( [ 'md2' , 'md5' , 'sha1' ] ) :
    raise ValueError ( pretty_message ( '''
            hash_algorithm must be one of "md2", "md5", "sha1", not %s
            ''' , repr ( hash_algorithm ) ) )
if key_length > 16 and hash_algorithm in set ( [ 'md2' , 'md5' ] ) :
    raise ValueError ( pretty_message ( '''
            key_length can not be longer than 16 for %s - is %s
            ''' , ( hash_algorithm , repr ( key_length ) ) ) )
if key_length > 20 and hash_algorithm == 'sha1' :
    raise ValueError ( pretty_message ( '''
            key_length can not be longer than 20 for sha1 - is %s
            ''' , repr ( key_length ) ) )
algo = getattr ( hashlib , hash_algorithm )
output = algo ( password + salt ) . digest ( )
for _ in range ( 2 , iterations + 1 ) :
    output = algo ( output ) . digest ( )
return output [ : key_length ]


signature:
def handle_error ( result ) :
    """Extracts the last Windows error message into a python unicode string
 : param result :
 A function result , 0 or None indicates failure
 : return :
 A unicode string error message"""

implementation:
if result :
    return
code , error_string = get_error ( )
if code == Advapi32Const . NTE_BAD_SIGNATURE :
    raise SignatureError ( 'Signature is invalid' )
if not isinstance ( error_string , str_cls ) :
    error_string = _try_decode ( error_string )
raise OSError ( error_string )


signature:
def aes_cbc_no_padding_encrypt ( key , data , iv ) :
    """Encrypts plaintext using AES in CBC mode with a 128 , 192 or 256 bit key and
 no padding . This means the ciphertext must be an exact multiple of 16 bytes
 long .
 : param key :
 The encryption key - a byte string either 16 , 24 or 32 bytes long
 : param data :
 The plaintext - a byte string
 : param iv :
 The initialization vector - either a byte string 16 - bytes long or None
 to generate an IV
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A tuple of two byte strings ( iv , ciphertext )"""

implementation:
if len ( key ) not in [ 16 , 24 , 32 ] :
    raise ValueError ( pretty_message ( '''
            key must be either 16, 24 or 32 bytes (128, 192 or 256 bits) long - is %s
            ''' , len ( key ) ) )
if not iv :
    iv = rand_bytes ( 16 )
elif len ( iv ) != 16 :
    raise ValueError ( pretty_message ( '''
            iv must be 16 bytes long - is %s
            ''' , len ( iv ) ) )
if len ( data ) % 16 != 0 :
    raise ValueError ( pretty_message ( '''
            data must be a multiple of 16 bytes long - is %s
            ''' , len ( data ) ) )
return ( iv , _encrypt ( Security . kSecAttrKeyTypeAES , key , data , iv , Security . kSecPaddingNoneKey ) )


signature:
def aes_cbc_no_padding_decrypt ( key , data , iv ) :
    """Decrypts AES ciphertext in CBC mode using a 128 , 192 or 256 bit key and no
 padding .
 : param key :
 The encryption key - a byte string either 16 , 24 or 32 bytes long
 : param data :
 The ciphertext - a byte string
 : param iv :
 The initialization vector - a byte string 16 - bytes long
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A byte string of the plaintext"""

implementation:
if len ( key ) not in [ 16 , 24 , 32 ] :
    raise ValueError ( pretty_message ( '''
            key must be either 16, 24 or 32 bytes (128, 192 or 256 bits) long - is %s
            ''' , len ( key ) ) )
if len ( iv ) != 16 :
    raise ValueError ( pretty_message ( '''
            iv must be 16 bytes long - is %s
            ''' , len ( iv ) ) )
return _decrypt ( Security . kSecAttrKeyTypeAES , key , data , iv , Security . kSecPaddingNoneKey )


signature:
def aes_cbc_pkcs7_encrypt ( key , data , iv ) :
    """Encrypts plaintext using AES in CBC mode with a 128 , 192 or 256 bit key and
 PKCS # 7 padding .
 : param key :
 The encryption key - a byte string either 16 , 24 or 32 bytes long
 : param data :
 The plaintext - a byte string
 : param iv :
 The initialization vector - either a byte string 16 - bytes long or None
 to generate an IV
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A tuple of two byte strings ( iv , ciphertext )"""

implementation:
if len ( key ) not in [ 16 , 24 , 32 ] :
    raise ValueError ( pretty_message ( '''
            key must be either 16, 24 or 32 bytes (128, 192 or 256 bits) long - is %s
            ''' , len ( key ) ) )
if not iv :
    iv = rand_bytes ( 16 )
elif len ( iv ) != 16 :
    raise ValueError ( pretty_message ( '''
            iv must be 16 bytes long - is %s
            ''' , len ( iv ) ) )
return ( iv , _encrypt ( Security . kSecAttrKeyTypeAES , key , data , iv , Security . kSecPaddingPKCS7Key ) )


signature:
def aes_cbc_pkcs7_decrypt ( key , data , iv ) :
    """Decrypts AES ciphertext in CBC mode using a 128 , 192 or 256 bit key
 : param key :
 The encryption key - a byte string either 16 , 24 or 32 bytes long
 : param data :
 The ciphertext - a byte string
 : param iv :
 The initialization vector - a byte string 16 - bytes long
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A byte string of the plaintext"""

implementation:
if len ( key ) not in [ 16 , 24 , 32 ] :
    raise ValueError ( pretty_message ( '''
            key must be either 16, 24 or 32 bytes (128, 192 or 256 bits)
            long - is %s
            ''' , len ( key ) ) )
if len ( iv ) != 16 :
    raise ValueError ( pretty_message ( '''
            iv must be 16 bytes long - is %s
            ''' , len ( iv ) ) )
return _decrypt ( Security . kSecAttrKeyTypeAES , key , data , iv , Security . kSecPaddingPKCS7Key )


signature:
def rc4_encrypt ( key , data ) :
    """Encrypts plaintext using RC4 with a 40-128 bit key
 : param key :
 The encryption key - a byte string 5-16 bytes long
 : param data :
 The plaintext - a byte string
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A byte string of the ciphertext"""

implementation:
if len ( key ) < 5 or len ( key ) > 16 :
    raise ValueError ( pretty_message ( '''
            key must be 5 to 16 bytes (40 to 128 bits) long - is %s
            ''' , len ( key ) ) )
return _encrypt ( Security . kSecAttrKeyTypeRC4 , key , data , None , None )


signature:
def rc4_decrypt ( key , data ) :
    """Decrypts RC4 ciphertext using a 40-128 bit key
 : param key :
 The encryption key - a byte string 5-16 bytes long
 : param data :
 The ciphertext - a byte string
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A byte string of the plaintext"""

implementation:
if len ( key ) < 5 or len ( key ) > 16 :
    raise ValueError ( pretty_message ( '''
            key must be 5 to 16 bytes (40 to 128 bits) long - is %s
            ''' , len ( key ) ) )
return _decrypt ( Security . kSecAttrKeyTypeRC4 , key , data , None , None )


signature:
def rc2_cbc_pkcs5_encrypt ( key , data , iv ) :
    """Encrypts plaintext using RC2 with a 64 bit key
 : param key :
 The encryption key - a byte string 8 bytes long
 : param data :
 The plaintext - a byte string
 : param iv :
 The 8 - byte initialization vector to use - a byte string - set as None
 to generate an appropriate one
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A tuple of two byte strings ( iv , ciphertext )"""

implementation:
if len ( key ) < 5 or len ( key ) > 16 :
    raise ValueError ( pretty_message ( '''
            key must be 5 to 16 bytes (40 to 128 bits) long - is %s
            ''' , len ( key ) ) )
if not iv :
    iv = rand_bytes ( 8 )
elif len ( iv ) != 8 :
    raise ValueError ( pretty_message ( '''
            iv must be 8 bytes long - is %s
            ''' , len ( iv ) ) )
return ( iv , _encrypt ( Security . kSecAttrKeyTypeRC2 , key , data , iv , Security . kSecPaddingPKCS5Key ) )


signature:
def rc2_cbc_pkcs5_decrypt ( key , data , iv ) :
    """Decrypts RC2 ciphertext using a 64 bit key
 : param key :
 The encryption key - a byte string 8 bytes long
 : param data :
 The ciphertext - a byte string
 : param iv :
 The initialization vector used for encryption - a byte string
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A byte string of the plaintext"""

implementation:
if len ( key ) < 5 or len ( key ) > 16 :
    raise ValueError ( pretty_message ( '''
            key must be 5 to 16 bytes (40 to 128 bits) long - is %s
            ''' , len ( key ) ) )
if len ( iv ) != 8 :
    raise ValueError ( pretty_message ( '''
            iv must be 8 bytes long - is %s
            ''' , len ( iv ) ) )
return _decrypt ( Security . kSecAttrKeyTypeRC2 , key , data , iv , Security . kSecPaddingPKCS5Key )


signature:
def tripledes_cbc_pkcs5_encrypt ( key , data , iv ) :
    """Encrypts plaintext using 3DES in either 2 or 3 key mode
 : param key :
 The encryption key - a byte string 16 or 24 bytes long ( 2 or 3 key mode )
 : param data :
 The plaintext - a byte string
 : param iv :
 The 8 - byte initialization vector to use - a byte string - set as None
 to generate an appropriate one
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A tuple of two byte strings ( iv , ciphertext )"""

implementation:
if len ( key ) != 16 and len ( key ) != 24 :
    raise ValueError ( pretty_message ( '''
            key must be 16 bytes (2 key) or 24 bytes (3 key) long - %s
            ''' , len ( key ) ) )
if not iv :
    iv = rand_bytes ( 8 )
elif len ( iv ) != 8 :
    raise ValueError ( pretty_message ( '''
            iv must be 8 bytes long - %s
            ''' , len ( iv ) ) )
# Expand 2 - key to actual 24 byte byte string used by cipher
if len ( key ) == 16 :
    key = key + key [ 0 : 8 ]
return ( iv , _encrypt ( Security . kSecAttrKeyType3DES , key , data , iv , Security . kSecPaddingPKCS5Key ) )


signature:
def tripledes_cbc_pkcs5_decrypt ( key , data , iv ) :
    """Decrypts 3DES ciphertext in either 2 or 3 key mode
 : param key :
 The encryption key - a byte string 16 or 24 bytes long ( 2 or 3 key mode )
 : param data :
 The ciphertext - a byte string
 : param iv :
 The initialization vector used for encryption - a byte string
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A byte string of the plaintext"""

implementation:
if len ( key ) != 16 and len ( key ) != 24 :
    raise ValueError ( pretty_message ( '''
            key must be 16 bytes (2 key) or 24 bytes (3 key) long - is %s
            ''' , len ( key ) ) )
if len ( iv ) != 8 :
    raise ValueError ( pretty_message ( '''
            iv must be 8 bytes long - is %s
            ''' , len ( iv ) ) )
# Expand 2 - key to actual 24 byte byte string used by cipher
if len ( key ) == 16 :
    key = key + key [ 0 : 8 ]
return _decrypt ( Security . kSecAttrKeyType3DES , key , data , iv , Security . kSecPaddingPKCS5Key )


signature:
def des_cbc_pkcs5_encrypt ( key , data , iv ) :
    """Encrypts plaintext using DES with a 56 bit key
 : param key :
 The encryption key - a byte string 8 bytes long ( includes error correction bits )
 : param data :
 The plaintext - a byte string
 : param iv :
 The 8 - byte initialization vector to use - a byte string - set as None
 to generate an appropriate one
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A tuple of two byte strings ( iv , ciphertext )"""

implementation:
if len ( key ) != 8 :
    raise ValueError ( pretty_message ( '''
            key must be 8 bytes (56 bits + 8 parity bits) long - is %s
            ''' , len ( key ) ) )
if not iv :
    iv = rand_bytes ( 8 )
elif len ( iv ) != 8 :
    raise ValueError ( pretty_message ( '''
            iv must be 8 bytes long - is %s
            ''' , len ( iv ) ) )
return ( iv , _encrypt ( Security . kSecAttrKeyTypeDES , key , data , iv , Security . kSecPaddingPKCS5Key ) )


signature:
def des_cbc_pkcs5_decrypt ( key , data , iv ) :
    """Decrypts DES ciphertext using a 56 bit key
 : param key :
 The encryption key - a byte string 8 bytes long ( includes error correction bits )
 : param data :
 The ciphertext - a byte string
 : param iv :
 The initialization vector used for encryption - a byte string
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A byte string of the plaintext"""

implementation:
if len ( key ) != 8 :
    raise ValueError ( pretty_message ( '''
            key must be 8 bytes (56 bits + 8 parity bits) long - is %s
            ''' , len ( key ) ) )
if len ( iv ) != 8 :
    raise ValueError ( pretty_message ( '''
            iv must be 8 bytes long - is %s
            ''' , len ( iv ) ) )
return _decrypt ( Security . kSecAttrKeyTypeDES , key , data , iv , Security . kSecPaddingPKCS5Key )


signature:
def _encrypt ( cipher , key , data , iv , padding ) :
    """Encrypts plaintext
 : param cipher :
 A kSecAttrKeyType * value that specifies the cipher to use
 : param key :
 The encryption key - a byte string 5-16 bytes long
 : param data :
 The plaintext - a byte string
 : param iv :
 The initialization vector - a byte string - unused for RC4
 : param padding :
 The padding mode to use , specified as a kSecPadding * Key value - unused for RC4
 : raises :
 ValueError - when any of the parameters contain an invalid value
 TypeError - when any of the parameters are of the wrong type
 OSError - when an error is returned by the OS crypto library
 : return :
 A byte string of the ciphertext"""

implementation:
if not isinstance ( key , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            key must be a byte string, not %s
            ''' , type_name ( key ) ) )
if not isinstance ( data , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            data must be a byte string, not %s
            ''' , type_name ( data ) ) )
if cipher != Security . kSecAttrKeyTypeRC4 and not isinstance ( iv , byte_cls ) :
    raise TypeError ( pretty_message ( '''
            iv must be a byte string, not %s
            ''' , type_name ( iv ) ) )
if cipher != Security . kSecAttrKeyTypeRC4 and not padding :
    raise ValueError ( 'padding must be specified' )
cf_dict = None
cf_key = None
cf_data = None
cf_iv = None
sec_key = None
sec_transform = None
try :
    cf_dict = CFHelpers . cf_dictionary_from_pairs ( [ ( Security . kSecAttrKeyType , cipher ) ] )
    cf_key = CFHelpers . cf_data_from_bytes ( key )
    cf_data = CFHelpers . cf_data_from_bytes ( data )
    error_pointer = new ( CoreFoundation , 'CFErrorRef *' )
    sec_key = Security . SecKeyCreateFromData ( cf_dict , cf_key , error_pointer )
    handle_cf_error ( error_pointer )
    sec_transform = Security . SecEncryptTransformCreate ( sec_key , error_pointer )
    handle_cf_error ( error_pointer )
    if cipher != Security . kSecAttrKeyTypeRC4 :
        Security . SecTransformSetAttribute ( sec_transform , Security . kSecModeCBCKey , null ( ) , error_pointer )
        handle_cf_error ( error_pointer )
        Security . SecTransformSetAttribute ( sec_transform , Security . kSecPaddingKey , padding , error_pointer )
        handle_cf_error ( error_pointer )
        cf_iv = CFHelpers . cf_data_from_bytes ( iv )
        Security . SecTransformSetAttribute ( sec_transform , Security . kSecIVKey , cf_iv , error_pointer )
        handle_cf_error ( error_pointer )
    Security . SecTransformSetAttribute ( sec_transform , Security . kSecTransformInputAttributeName , cf_data , error_pointer )
    handle_cf_error ( error_pointer )
    ciphertext = Security . SecTransformExecute ( sec_transform , error_pointer )
    handle_cf_error ( error_pointer )
    return CFHelpers . cf_data_to_bytes ( ciphertext )
finally :
    if cf_dict :
        CoreFoundation . CFRelease ( cf_dict )
    if cf_key :
        CoreFoundation . CFRelease ( cf_key )
    if cf_data :
        CoreFoundation . CFRelease ( cf_data )
    if cf_iv :
        CoreFoundation . CFRelease ( cf_iv )
    if sec_key :
        CoreFoundation . CFRelease ( sec_key )
    if sec_transform :
        CoreFoundation . CFRelease ( sec_transform )


signature:
def version ( self ) :
    """Create a new version under this service ."""

implementation:
ver = Version ( )
ver . conn = self . conn
ver . attrs = { # Parent params
'service_id' : self . attrs [ 'id' ] , }
ver . save ( )
return ver


signature:
def vcl ( self , name , content ) :
    """Create a new VCL under this version ."""

implementation:
vcl = VCL ( )
vcl . conn = self . conn
vcl . attrs = { # Parent params
'service_id' : self . attrs [ 'service_id' ] , 'version' : self . attrs [ 'number' ] , # New instance params
'name' : name , 'content' : content , }
vcl . save ( )
return vcl


signature:
def to_dict ( self ) :
    """Converts the column to a dictionary representation accepted
 by the Citrination server .
 : return : Dictionary with basic options , plus any column type specific
 options held under the " options " key
 : rtype : dict"""

implementation: return { "type" : self . type , "name" : self . name , "group_by_key" : self . group_by_key , "role" : self . role , "units" : self . units , "options" : self . build_options ( ) }


signature:
def add_descriptor ( self , descriptor , role = 'ignore' , group_by_key = False ) :
    """Add a descriptor column .
 : param descriptor : A Descriptor instance ( e . g . , RealDescriptor , InorganicDescriptor , etc . )
 : param role : Specify a role ( input , output , latentVariable , or ignore )
 : param group _ by _ key : Whether or not to group by this key during cross validation"""

implementation:
descriptor . validate ( )
if descriptor . key in self . configuration [ "roles" ] :
    raise ValueError ( "Cannot add a descriptor with the same name twice" )
self . configuration [ 'descriptors' ] . append ( descriptor . as_dict ( ) )
self . configuration [ "roles" ] [ descriptor . key ] = role
if group_by_key :
    self . configuration [ "group_by" ] . append ( descriptor . key )


signature:
def _get ( self , route , headers = None , failure_message = None ) :
    """Execute a post request and return the result
 : param headers :
 : return :"""

implementation:
headers = self . _get_headers ( headers )
response_lambda = ( lambda : requests . get ( self . _get_qualified_route ( route ) , headers = headers , verify = False , proxies = self . proxies ) )
response = check_for_rate_limiting ( response_lambda ( ) , response_lambda )
return self . _handle_response ( response , failure_message )


signature:
def _post ( self , route , data , headers = None , failure_message = None ) :
    """Execute a post request and return the result
 : param data :
 : param headers :
 : return :"""

implementation:
headers = self . _get_headers ( headers )
response_lambda = ( lambda : requests . post ( self . _get_qualified_route ( route ) , headers = headers , data = data , verify = False , proxies = self . proxies ) )
response = check_for_rate_limiting ( response_lambda ( ) , response_lambda )
return self . _handle_response ( response , failure_message )


signature:
def _put ( self , route , data , headers = None , failure_message = None ) :
    """Execute a put request and return the result
 : param data :
 : param headers :
 : return :"""

implementation:
headers = self . _get_headers ( headers )
response_lambda = ( lambda : requests . put ( self . _get_qualified_route ( route ) , headers = headers , data = data , verify = False , proxies = self . proxies ) )
response = check_for_rate_limiting ( response_lambda ( ) , response_lambda )
return self . _handle_response ( response , failure_message )


signature:
def _patch ( self , route , data , headers = None , failure_message = None ) :
    """Execute a patch request and return the result"""

implementation:
headers = self . _get_headers ( headers )
response_lambda = ( lambda : requests . patch ( self . _get_qualified_route ( route ) , headers = headers , data = data , verify = False , proxies = self . proxies ) )
response = check_for_rate_limiting ( response_lambda ( ) , response_lambda )
return self . _handle_response ( response , failure_message )


signature:
def _delete ( self , route , headers = None , failure_message = None ) :
    """Execute a delete request and return the result
 : param headers :
 : return :"""

implementation:
headers = self . _get_headers ( headers )
response_lambda = ( lambda : requests . delete ( self . _get_qualified_route ( route ) , headers = headers , verify = False , proxies = self . proxies ) )
response = check_for_rate_limiting ( response_lambda ( ) , response_lambda )
return self . _handle_response ( response , failure_message )


signature:
def _validate_search_query ( self , returning_query ) :
    """Checks to see that the query will not exceed the max query depth
 : param returning _ query : The PIF system or Dataset query to execute .
 : type returning _ query : : class : ` PifSystemReturningQuery ` or : class : ` DatasetReturningQuery `"""

implementation:
start_index = returning_query . from_index or 0
size = returning_query . size or 0
if start_index < 0 :
    raise CitrinationClientError ( "start_index cannot be negative. Please enter a value greater than or equal to zero" )
if size < 0 :
    raise CitrinationClientError ( "Size cannot be negative. Please enter a value greater than or equal to zero" )
if start_index + size > MAX_QUERY_DEPTH :
    raise CitrinationClientError ( "Citrination does not support pagination past the {0}th result. Please reduce either the from_index and/or size such that their sum is below {0}" . format ( MAX_QUERY_DEPTH ) )


signature:
def pif_search ( self , pif_system_returning_query ) :
    """Run a PIF query against Citrination .
 : param pif _ system _ returning _ query : The PIF system query to execute .
 : type pif _ system _ returning _ query : : class : ` PifSystemReturningQuery `
 : return : : class : ` PifSearchResult ` object with the results of the query .
 : rtype : : class : ` PifSearchResult `"""

implementation:
self . _validate_search_query ( pif_system_returning_query )
return self . _execute_search_query ( pif_system_returning_query , PifSearchResult )


signature:
def dataset_search ( self , dataset_returning_query ) :
    """Run a dataset query against Citrination .
 : param dataset _ returning _ query : : class : ` DatasetReturningQuery ` to execute .
 : type dataset _ returning _ query : : class : ` DatasetReturningQuery `
 : return : Dataset search result object with the results of the query .
 : rtype : : class : ` DatasetSearchResult `"""

implementation:
self . _validate_search_query ( dataset_returning_query )
return self . _execute_search_query ( dataset_returning_query , DatasetSearchResult )


signature:
def _execute_search_query ( self , returning_query , result_class ) :
    """Run a PIF query against Citrination .
 : param returning _ query : : class : ` BaseReturningQuery ` to execute .
 : param result _ class : The class of the result to return .
 : return : ` ` result _ class ` ` object with the results of the query ."""

implementation:
if returning_query . from_index :
    from_index = returning_query . from_index
else :
    from_index = 0
if returning_query . size != None :
    size = min ( returning_query . size , client_config . max_query_size )
else :
    size = client_config . max_query_size
if ( size == client_config . max_query_size and size != returning_query . size ) :
    self . _warn ( "Query size greater than max system size - only {} results will be returned" . format ( size ) )
time = 0.0 ;
hits = [ ] ;
while True :
    sub_query = deepcopy ( returning_query )
    sub_query . from_index = from_index + len ( hits )
    partial_results = self . _search_internal ( sub_query , result_class )
    total = partial_results . total_num_hits
    time += partial_results . took
    if partial_results . hits is not None :
        hits . extend ( partial_results . hits )
    if len ( hits ) >= size or len ( hits ) >= total or sub_query . from_index >= total :
        break
return result_class ( hits = hits , total_num_hits = total , took = time )


signature:
def pif_multi_search ( self , multi_query ) :
    """Run each in a list of PIF queries against Citrination .
 : param multi _ query : : class : ` MultiQuery ` object to execute .
 : return : : class : ` PifMultiSearchResult ` object with the results of the query ."""

implementation:
failure_message = "Error while making PIF multi search request"
response_dict = self . _get_success_json ( self . _post ( routes . pif_multi_search , data = json . dumps ( multi_query , cls = QueryEncoder ) , failure_message = failure_message ) )
return PifMultiSearchResult ( ** keys_to_snake_case ( response_dict [ 'results' ] ) )


signature:
def generate_simple_chemical_query ( self , name = None , chemical_formula = None , property_name = None , property_value = None , property_min = None , property_max = None , property_units = None , reference_doi = None , include_datasets = [ ] , exclude_datasets = [ ] , from_index = None , size = None ) :
    """This method generates a : class : ` PifSystemReturningQuery ` object using the
 supplied arguments . All arguments that accept lists have logical OR ' s on the queries that they generate .
 This means that , for example , simple _ chemical _ search ( name = [ ' A ' , ' B ' ] ) will match records that have name
 equal to ' A ' or ' B ' .
 Results will be pulled into the extracted field of the : class : ` PifSearchHit ` objects that are returned . The
 name will appear under the key " name " , chemical formula under " chemical _ formula " , property name under
 " property _ name " , value of the property under " property _ value " , units of the property under " property _ units " ,
 and reference DOI under " reference _ doi " .
 This method is only meant for execution of very simple queries . More complex queries must use the search method
 that accepts a : class : ` PifSystemReturningQuery ` object .
 : param name : One or more strings with the names of the chemical system to match .
 : type name : str or list of str
 : param chemical _ formula : One or more strings with the chemical formulas to match .
 : type chemical _ formula : str or list of str
 : param property _ name : One or more strings with the names of the property to match .
 : type property _ name : str or list of str
 : param property _ value : One or more strings or numbers with the exact values to match .
 : type property _ value : str or int or float or list of str or int or float
 : param property _ min : A single string or number with the minimum value to match .
 : type property _ min : str or int or float
 : param property _ max : A single string or number with the maximum value to match .
 : type property _ max : str or int or float
 : param property _ units : One or more strings with the property units to match .
 : type property _ units : str or list of str
 : param reference _ doi : One or more strings with the DOI to match .
 : type reference _ doin : str or list of str
 : param include _ datasets : One or more integers with dataset IDs to match .
 : type include _ datasets : int or list of int
 : param exclude _ datasets : One or more integers with dataset IDs that must not match .
 : type exclude _ datasets : int or list of int
 : param from _ index : Index of the first record to match .
 : type from _ index : int
 : param size : Total number of records to return .
 : type size : int
 : return : A query to to be submitted with the pif _ search method
 : rtype : : class : ` PifSystemReturningQuery `"""

implementation:
pif_system_query = PifSystemQuery ( )
pif_system_query . names = FieldQuery ( extract_as = 'name' , filter = [ Filter ( equal = i ) for i in self . _get_list ( name ) ] )
pif_system_query . chemical_formula = ChemicalFieldQuery ( extract_as = 'chemical_formula' , filter = [ ChemicalFilter ( equal = i ) for i in self . _get_list ( chemical_formula ) ] )
pif_system_query . references = ReferenceQuery ( doi = FieldQuery ( extract_as = 'reference_doi' , filter = [ Filter ( equal = i ) for i in self . _get_list ( reference_doi ) ] ) )
# Generate the parts of the property query
property_name_query = FieldQuery ( extract_as = 'property_name' , filter = [ Filter ( equal = i ) for i in self . _get_list ( property_name ) ] )
property_units_query = FieldQuery ( extract_as = 'property_units' , filter = [ Filter ( equal = i ) for i in self . _get_list ( property_units ) ] )
property_value_query = FieldQuery ( extract_as = 'property_value' , filter = [ ] )
for i in self . _get_list ( property_value ) :
    property_value_query . filter . append ( Filter ( equal = i ) )
if property_min is not None or property_max is not None :
    property_value_query . filter . append ( Filter ( min = property_min , max = property_max ) )
# Generate the full property query
pif_system_query . properties = PropertyQuery ( name = property_name_query , value = property_value_query , units = property_units_query )
# Generate the dataset query
dataset_query = list ( )
if include_datasets :
    dataset_query . append ( DatasetQuery ( logic = 'MUST' , id = [ Filter ( equal = i ) for i in include_datasets ] ) )
if exclude_datasets :
    dataset_query . append ( DatasetQuery ( logic = 'MUST_NOT' , id = [ Filter ( equal = i ) for i in exclude_datasets ] ) )
# Run the query
pif_system_returning_query = PifSystemReturningQuery ( query = DataQuery ( system = pif_system_query , dataset = dataset_query ) , from_index = from_index , size = size , score_relevance = True )
return pif_system_returning_query


signature:
def check_for_rate_limiting ( response , response_lambda , timeout = 1 , attempts = 0 ) :
    """Takes an initial response , and a way to repeat the request that produced it and retries the request with an increasing sleep period between requests if rate limiting resposne codes are encountered .
 If more than 3 attempts are made , a RateLimitingException is raised
 : param response : A response from Citrination
 : type response : requests . Response
 : param response _ lambda : a callable that runs the request that returned the
 response
 : type response _ lambda : function
 : param timeout : the time to wait before retrying
 : type timeout : int
 : param attempts : the number of the retry being executed
 : type attempts : int"""

implementation:
if attempts >= 3 :
    raise RateLimitingException ( )
if response . status_code == 429 :
    sleep ( timeout )
    new_timeout = timeout + 1
    new_attempts = attempts + 1
    return check_for_rate_limiting ( response_lambda ( timeout , attempts ) , response_lambda , timeout = new_timeout , attempts = new_attempts )
return response


signature:
def create ( self , configuration , name , description ) :
    """Creates a data view from the search template and ml template given
 : param configuration : Information to construct the data view from ( eg descriptors , datasets etc )
 : param name : Name of the data view
 : param description : Description for the data view
 : return : The data view id"""

implementation:
data = { "configuration" : configuration , "name" : name , "description" : description }
failure_message = "Dataview creation failed"
result = self . _get_success_json ( self . _post_json ( 'v1/data_views' , data , failure_message = failure_message ) )
data_view_id = result [ 'data' ] [ 'id' ]
return data_view_id


signature:
def update ( self , id , configuration , name , description ) :
    """Updates an existing data view from the search template and ml template given
 : param id : Identifier for the data view . This returned from the create method .
 : param configuration : Information to construct the data view from ( eg descriptors , datasets etc )
 : param name : Name of the data view
 : param description : Description for the data view"""

implementation:
data = { "configuration" : configuration , "name" : name , "description" : description }
failure_message = "Dataview creation failed"
self . _patch_json ( 'v1/data_views/' + id , data , failure_message = failure_message )


signature:
def get ( self , data_view_id ) :
    """Gets basic information about a view
 : param data _ view _ id : Identifier of the data view
 : return : Metadata about the view as JSON"""

implementation:
failure_message = "Dataview get failed"
return self . _get_success_json ( self . _get ( 'v1/data_views/' + data_view_id , None , failure_message = failure_message ) ) [ 'data' ] [ 'data_view' ]


signature:
def get_data_view_service_status ( self , data_view_id ) :
    """Retrieves the status for all of the services associated with a data view :
 - predict
 - experimental _ design
 - data _ reports
 - model _ reports
 : param data _ view _ id : The ID number of the data view to which the
 run belongs , as a string
 : type data _ view _ id : str
 : return : A : class : ` DataViewStatus `
 : rtype : DataViewStatus"""

implementation:
url = "data_views/{}/status" . format ( data_view_id )
response = self . _get ( url ) . json ( )
result = response [ "data" ] [ "status" ]
return DataViewStatus ( predict = ServiceStatus . from_response_dict ( result [ "predict" ] ) , experimental_design = ServiceStatus . from_response_dict ( result [ "experimental_design" ] ) , data_reports = ServiceStatus . from_response_dict ( result [ "data_reports" ] ) , model_reports = ServiceStatus . from_response_dict ( result [ "model_reports" ] ) )


signature:
def create_ml_configuration_from_datasets ( self , dataset_ids ) :
    """Creates an ml configuration from dataset _ ids and extract _ as _ keys
 : param dataset _ ids : Array of dataset identifiers to make search template from
 : return : An identifier used to request the status of the builder job ( get _ ml _ configuration _ status )"""

implementation:
available_columns = self . search_template_client . get_available_columns ( dataset_ids )
# Create a search template from dataset ids
search_template = self . search_template_client . create ( dataset_ids , available_columns )
return self . create_ml_configuration ( search_template , available_columns , dataset_ids )


signature:
def create_ml_configuration ( self , search_template , extract_as_keys , dataset_ids ) :
    """This method will spawn a server job to create a default ML configuration based on a search template and
 the extract as keys .
 This function will submit the request to build , and wait for the configuration to finish before returning .
 : param search _ template : A search template defining the query ( properties , datasets etc )
 : param extract _ as _ keys : Array of extract - as keys defining the descriptors
 : param dataset _ ids : Array of dataset identifiers to make search template from
 : return : An identifier used to request the status of the builder job ( get _ ml _ configuration _ status )"""

implementation:
data = { "search_template" : search_template , "extract_as_keys" : extract_as_keys }
failure_message = "ML Configuration creation failed"
config_job_id = self . _get_success_json ( self . _post_json ( 'v1/descriptors/builders/simple/default/trigger' , data , failure_message = failure_message ) ) [ 'data' ] [ 'result' ] [ 'uid' ]
while True :
    config_status = self . __get_ml_configuration_status ( config_job_id )
    print ( 'Configuration status: ' , config_status )
    if config_status [ 'status' ] == 'Finished' :
        ml_config = self . __convert_response_to_configuration ( config_status [ 'result' ] , dataset_ids )
        return ml_config
    time . sleep ( 5 )


signature:
def __convert_response_to_configuration ( self , result_blob , dataset_ids ) :
    """Utility function to turn the result object from the configuration builder endpoint into something that
 can be used directly as a configuration .
 : param result _ blob : Nested dicts representing the possible descriptors
 : param dataset _ ids : Array of dataset identifiers to make search template from
 : return : An object suitable to be used as a parameter to data view create"""

implementation:
builder = DataViewBuilder ( )
builder . dataset_ids ( dataset_ids )
for i , ( k , v ) in enumerate ( result_blob [ 'descriptors' ] . items ( ) ) :
    try :
        descriptor = self . __snake_case ( v [ 0 ] )
        print ( json . dumps ( descriptor ) )
        descriptor [ 'descriptor_key' ] = k
        builder . add_raw_descriptor ( descriptor )
    except IndexError :
        pass
for i , ( k , v ) in enumerate ( result_blob [ 'types' ] . items ( ) ) :
    builder . set_role ( k , v . lower ( ) )
return builder . build ( )


signature:
def __snake_case ( self , descriptor ) :
    """Utility method to convert camelcase to snake
 : param descriptor : The dictionary to convert"""

implementation:
newdict = { }
for i , ( k , v ) in enumerate ( descriptor . items ( ) ) :
    newkey = ""
    for j , c in enumerate ( k ) :
        if c . isupper ( ) :
            if len ( newkey ) != 0 :
                newkey += '_'
            newkey += c . lower ( )
        else :
            newkey += c
    newdict [ newkey ] = v
return newdict


signature:
def __get_ml_configuration_status ( self , job_id ) :
    """After invoking the create _ ml _ configuration async method , you can use this method to
 check on the status of the builder job .
 : param job _ id : The identifier returned from create _ ml _ configuration
 : return : Job status"""

implementation:
failure_message = "Get status on ml configuration failed"
response = self . _get_success_json ( self . _get ( 'v1/descriptors/builders/simple/default/' + job_id + '/status' , None , failure_message = failure_message ) ) [ 'data' ]
return response


signature:
def tsne ( self , data_view_id ) :
    """Get the t - SNE projection , including responses and tags .
 : param data _ view _ id : The ID of the data view to retrieve TSNE from
 : type data _ view _ id : int
 : return : The TSNE analysis
 : rtype : : class : ` Tsne `"""

implementation:
analysis = self . _data_analysis ( data_view_id )
projections = analysis [ 'projections' ]
tsne = Tsne ( )
for k , v in projections . items ( ) :
    projection = Projection ( xs = v [ 'x' ] , ys = v [ 'y' ] , responses = v [ 'label' ] , tags = v [ 'inputs' ] , uids = v [ 'uid' ] )
    tsne . add_projection ( k , projection )
return tsne


signature:
def predict ( self , data_view_id , candidates , method = "scalar" , use_prior = True ) :
    """Predict endpoint . This simply wraps the async methods ( submit and poll for status / results ) .
 : param data _ view _ id : The ID of the data view to use for prediction
 : type data _ view _ id : str
 : param candidates : A list of candidates to make predictions on
 : type candidates : list of dicts
 : param method : Method for propagating predictions through model graphs . " scalar " uses linearized uncertainty
 propagation , whereas " scalar _ from _ distribution " still returns scalar predictions but uses sampling to
 propagate uncertainty without a linear approximation .
 : type method : str ( " scalar " or " scalar _ from _ distribution " )
 : param use _ prior : Whether to apply prior values implied by the property descriptors
 : type use _ prior : bool
 : return : The results of the prediction
 : rtype : list of : class : ` PredictionResult `"""

implementation:
uid = self . submit_predict_request ( data_view_id , candidates , method , use_prior )
while self . check_predict_status ( data_view_id , uid ) [ 'status' ] not in [ "Finished" , "Failed" , "Killed" ] :
    time . sleep ( 1 )
result = self . check_predict_status ( data_view_id , uid )
if result [ "status" ] == "Finished" :
    paired = zip ( result [ "results" ] [ "candidates" ] , result [ "results" ] [ "loss" ] )
    prediction_result_format = [ { k : ( p [ 0 ] [ k ] , p [ 1 ] [ k ] ) for k in p [ 0 ] . keys ( ) } for p in paired ]
    return list ( map ( lambda c : _get_prediction_result_from_candidate ( c ) , prediction_result_format ) )
else :
    raise RuntimeError ( "Prediction failed: UID={}, result={}" . format ( uid , result [ "status" ] ) )


signature:
def retrain ( self , dataview_id ) :
    """Start a model retraining
 : param dataview _ id : The ID of the views
 : return :"""

implementation:
url = 'data_views/{}/retrain' . format ( dataview_id )
response = self . _post_json ( url , data = { } )
if response . status_code != requests . codes . ok :
    raise RuntimeError ( 'Retrain requested ' + str ( response . status_code ) + ' response: ' + str ( response . message ) )
return True


signature:
def _data_analysis ( self , data_view_id ) :
    """Data analysis endpoint .
 : param data _ view _ id : The model identifier ( id number for data views )
 : type data _ view _ id : str
 : return : dictionary containing information about the data , e . g . dCorr and tsne"""

implementation:
failure_message = "Error while retrieving data analysis for data view {}" . format ( data_view_id )
return self . _get_success_json ( self . _get ( routes . data_analysis ( data_view_id ) , failure_message = failure_message ) )


signature:
def submit_predict_request ( self , data_view_id , candidates , prediction_source = 'scalar' , use_prior = True ) :
    """Submits an async prediction request .
 : param data _ view _ id : The id returned from create
 : param candidates : Array of candidates
 : param prediction _ source : ' scalar ' or ' scalar _ from _ distribution '
 : param use _ prior : True to use prior prediction , otherwise False
 : return : Predict request Id ( used to check status )"""

implementation:
data = { "prediction_source" : prediction_source , "use_prior" : use_prior , "candidates" : candidates }
failure_message = "Configuration creation failed"
post_url = 'v1/data_views/' + str ( data_view_id ) + '/predict/submit'
return self . _get_success_json ( self . _post_json ( post_url , data , failure_message = failure_message ) ) [ 'data' ] [ 'uid' ]


signature:
def check_predict_status ( self , view_id , predict_request_id ) :
    """Returns a string indicating the status of the prediction job
 : param view _ id : The data view id returned from data view create
 : param predict _ request _ id : The id returned from predict
 : return : Status data , also includes results if state is finished"""

implementation:
failure_message = "Get status on predict failed"
bare_response = self . _get_success_json ( self . _get ( 'v1/data_views/' + str ( view_id ) + '/predict/' + str ( predict_request_id ) + '/status' , None , failure_message = failure_message ) )
result = bare_response [ "data" ]
# result . update ( { " message " : bare _ response [ " message " ] } )
return result


signature:
def submit_design_run ( self , data_view_id , num_candidates , effort , target = None , constraints = [ ] , sampler = "Default" ) :
    """Submits a new experimental design run .
 : param data _ view _ id : The ID number of the data view to which the
 run belongs , as a string
 : type data _ view _ id : str
 : param num _ candidates : The number of candidates to return
 : type num _ candidates : int
 : param target : An : class : ` ` Target ` ` instance representing
 the design run optimization target
 : type target : : class : ` ` Target ` `
 : param constraints : An array of design constraints ( instances of
 objects which extend : class : ` ` BaseConstraint ` ` )
 : type constraints : list of : class : ` ` BaseConstraint ` `
 : param sampler : The name of the sampler to use during the design run :
 either " Default " or " This view "
 : type sampler : str
 : return : A : class : ` DesignRun ` instance containing the UID of the
 new run"""

implementation:
if effort > 30 :
    raise CitrinationClientError ( "Parameter effort must be less than 30 to trigger a design run" )
if target is not None :
    target = target . to_dict ( )
constraint_dicts = [ c . to_dict ( ) for c in constraints ]
body = { "num_candidates" : num_candidates , "target" : target , "effort" : effort , "constraints" : constraint_dicts , "sampler" : sampler }
url = routes . submit_data_view_design ( data_view_id )
response = self . _post_json ( url , body ) . json ( )
return DesignRun ( response [ "data" ] [ "design_run" ] [ "uid" ] )


signature:
def get_design_run_status ( self , data_view_id , run_uuid ) :
    """Retrieves the status of an in progress or completed design run
 : param data _ view _ id : The ID number of the data view to which the
 run belongs , as a string
 : type data _ view _ id : str
 : param run _ uuid : The UUID of the design run to retrieve status for
 : type run _ uuid : str
 : return : A : class : ` ProcessStatus ` object"""

implementation:
url = routes . get_data_view_design_status ( data_view_id , run_uuid )
response = self . _get ( url ) . json ( )
status = response [ "data" ]
return ProcessStatus ( result = status . get ( "result" ) , progress = status . get ( "progress" ) , status = status . get ( "status" ) , messages = status . get ( "messages" ) )


signature:
def get_design_run_results ( self , data_view_id , run_uuid ) :
    """Retrieves the results of an existing designrun
 : param data _ view _ id : The ID number of the data view to which the
 run belongs , as a string
 : type data _ view _ id : str
 : param run _ uuid : The UUID of the design run to retrieve results from
 : type run _ uuid : str
 : return : A : class : ` DesignResults ` object"""

implementation:
url = routes . get_data_view_design_results ( data_view_id , run_uuid )
response = self . _get ( url ) . json ( )
result = response [ "data" ]
return DesignResults ( best_materials = result . get ( "best_material_results" ) , next_experiments = result . get ( "next_experiment_results" ) )


signature:
def get_data_view ( self , data_view_id ) :
    """Retrieves a summary of information for a given data view
 - view id
 - name
 - description
 - columns
 : param data _ view _ id : The ID number of the data view to which the
 run belongs , as a string
 : type data _ view _ id : str"""

implementation:
url = routes . get_data_view ( data_view_id )
response = self . _get ( url ) . json ( )
result = response [ "data" ] [ "data_view" ]
datasets_list = [ ]
for dataset in result [ "datasets" ] :
    datasets_list . append ( Dataset ( name = dataset [ "name" ] , id = dataset [ "id" ] , description = dataset [ "description" ] ) )
columns_list = [ ]
for column in result [ "columns" ] :
    columns_list . append ( ColumnFactory . from_dict ( column ) )
return DataView ( view_id = data_view_id , name = result [ "name" ] , description = result [ "description" ] , datasets = datasets_list , columns = columns_list , )


signature:
def kill_design_run ( self , data_view_id , run_uuid ) :
    """Kills an in progress experimental design run
 : param data _ view _ id : The ID number of the data view to which the
 run belongs , as a string
 : type data _ view _ id : str
 : param run _ uuid : The UUID of the design run to kill
 : type run _ uuid : str
 : return : The UUID of the design run"""

implementation:
url = routes . kill_data_view_design_run ( data_view_id , run_uuid )
response = self . _delete ( url ) . json ( )
return response [ "data" ] [ "uid" ]


signature:
def load_file_as_yaml ( path ) :
    """Given a filepath , loads the file as a dictionary from YAML
 : param path : The path to a YAML file"""

implementation:
with open ( path , "r" ) as f :
    raw_yaml = f . read ( )
    parsed_dict = yaml . load ( raw_yaml )
return parsed_dict


signature:
def get_credentials_from_file ( filepath ) :
    """Extracts credentials from the yaml formatted credential filepath
 passed in . Uses the default profile if the CITRINATION _ PROFILE env var
 is not set , otherwise looks for a profile with that name in the credentials file .
 : param filepath : The path of the credentials file"""

implementation:
try :
    creds = load_file_as_yaml ( filepath )
except Exception :
    creds = { }
profile_name = os . environ . get ( citr_env_vars . CITRINATION_PROFILE )
if profile_name is None or len ( profile_name ) == 0 :
    profile_name = DEFAULT_CITRINATION_PROFILE
api_key = None
site = None
try :
    profile = creds [ profile_name ]
    api_key = profile [ CREDENTIALS_API_KEY_KEY ]
    site = profile [ CREDENTIALS_SITE_KEY ]
except KeyError :
    pass
return ( api_key , site )


signature:
def get_preferred_credentials ( api_key , site , cred_file = DEFAULT_CITRINATION_CREDENTIALS_FILE ) :
    """Given an API key , a site url and a credentials file path , runs through a prioritized list of credential sources to find credentials .
 Specifically , this method ranks credential priority as follows :
 1 . Those passed in as the first two parameters to this method
 2 . Those found in the environment as variables
 3 . Those found in the credentials file at the profile specified
 by the profile environment variable
 4 . Those found in the default stanza in the credentials file
 : param api _ key : A Citrination API Key or None
 : param site : A Citrination site URL or None
 : param cred _ file : The path to a credentials file"""

implementation:
profile_api_key , profile_site = get_credentials_from_file ( cred_file )
if api_key is None :
    api_key = os . environ . get ( citr_env_vars . CITRINATION_API_KEY )
if api_key is None or len ( api_key ) == 0 :
    api_key = profile_api_key
if site is None :
    site = os . environ . get ( citr_env_vars . CITRINATION_SITE )
if site is None or len ( site ) == 0 :
    site = profile_site
if site is None :
    site = "https://citrination.com"
return api_key , site


signature:
def upload ( self , dataset_id , source_path , dest_path = None ) :
    """Upload a file , specifying source and dest paths a file ( acts as the scp command ) . asdfasdf
 : param source _ path : The path to the file on the source host asdf
 : type source _ path : str
 : param dest _ path : The path to the file where the contents of the upload will be written ( on the dest host )
 : type dest _ path : str
 : return : The result of the upload process
 : rtype : : class : ` UploadResult `"""

implementation:
upload_result = UploadResult ( )
source_path = str ( source_path )
if not dest_path :
    dest_path = source_path
else :
    dest_path = str ( dest_path )
if os . path . isdir ( source_path ) :
    for path , subdirs , files in os . walk ( source_path ) :
        relative_path = os . path . relpath ( path , source_path )
        current_dest_prefix = dest_path
        if relative_path is not "." :
            current_dest_prefix = os . path . join ( current_dest_prefix , relative_path )
        for name in files :
            current_dest_path = os . path . join ( current_dest_prefix , name )
            current_source_path = os . path . join ( path , name )
            try :
                if self . upload ( dataset_id , current_source_path , current_dest_path ) . successful ( ) :
                    upload_result . add_success ( current_source_path )
                else :
                    upload_result . add_failure ( current_source_path , "Upload failure" )
            except ( CitrinationClientError , ValueError ) as e :
                upload_result . add_failure ( current_source_path , str ( e ) )
    return upload_result
elif os . path . isfile ( source_path ) :
    file_data = { "dest_path" : str ( dest_path ) , "src_path" : str ( source_path ) }
    j = self . _get_success_json ( self . _post_json ( routes . upload_to_dataset ( dataset_id ) , data = file_data ) )
    s3url = _get_s3_presigned_url ( j )
    with open ( source_path , 'rb' ) as f :
        if os . stat ( source_path ) . st_size == 0 : # Upload a null character as a placeholder for
        # the empty file since Citrination does not support
        # truly empty files
            data = "\0"
        else :
            data = f
        r = requests . put ( s3url , data = data , headers = j [ "required_headers" ] )
        if r . status_code == 200 :
            data = { 's3object' : j [ 'url' ] [ 'path' ] , 's3bucket' : j [ 'bucket' ] }
            self . _post_json ( routes . update_file ( j [ 'file_id' ] ) , data = data )
            upload_result . add_success ( source_path )
            return upload_result
        else :
            raise CitrinationClientError ( "Failure to upload {} to Citrination" . format ( source_path ) )
else :
    raise ValueError ( "No file at specified path {}" . format ( source_path ) )


signature:
def list_files ( self , dataset_id , glob = "." , is_dir = False ) :
    """List matched filenames in a dataset on Citrination .
 : param dataset _ id : The ID of the dataset to search for files .
 : type dataset _ id : int
 : param glob : A pattern which will be matched against files in the dataset .
 : type glob : str
 : param is _ dir : A boolean indicating whether or not the pattern should match against the beginning of paths in the dataset .
 : type is _ dir : bool
 : return : A list of filepaths in the dataset matching the provided glob .
 : rtype : list of strings"""

implementation:
data = { "list" : { "glob" : glob , "isDir" : is_dir } }
return self . _get_success_json ( self . _post_json ( routes . list_files ( dataset_id ) , data , failure_message = "Failed to list files for dataset {}" . format ( dataset_id ) ) ) [ 'files' ]


signature:
def matched_file_count ( self , dataset_id , glob = "." , is_dir = False ) :
    """Returns the number of files matching a pattern in a dataset .
 : param dataset _ id : The ID of the dataset to search for files .
 : type dataset _ id : int
 : param glob : A pattern which will be matched against files in the dataset .
 : type glob : str
 : param is _ dir : A boolean indicating whether or not the pattern should match against the beginning of paths in the dataset .
 : type is _ dir : bool
 : return : The number of matching files
 : rtype : int"""

implementation:
list_result = self . list_files ( dataset_id , glob , is_dir )
return len ( list_result )


signature:
def get_ingest_status ( self , dataset_id ) :
    """Returns the current status of dataset ingestion . If any file uploaded to a dataset is in an error / failure state
 this endpoint will return error / failure . If any files are still processing , will return processing .
 : param dataset _ id : Dataset identifier
 : return : Status of dataset ingestion as a string"""

implementation:
failure_message = "Failed to create dataset ingest status for dataset {}" . format ( dataset_id )
response = self . _get_success_json ( self . _get ( 'v1/datasets/' + str ( dataset_id ) + '/ingest-status' , failure_message = failure_message ) ) [ 'data' ]
if 'status' in response :
    return response [ 'status' ]
return ''


signature:
def get_dataset_files ( self , dataset_id , glob = "." , is_dir = False , version_number = None ) :
    """Retrieves URLs for the files matched by a glob or a path to a directory
 in a given dataset .
 : param dataset _ id : The id of the dataset to retrieve files from
 : type dataset _ id : int
 : param glob : A regex used to select one or more files in the dataset
 : type glob : str
 : param is _ dir : Whether or not the supplied pattern should be treated as a directory to search in
 : type is _ dir : bool
 : param version _ number : The version number of the dataset to retrieve files from
 : type version _ number : int
 : return : A list of dataset files whose paths match the provided pattern .
 : rtype : list of : class : ` DatasetFile `"""

implementation:
if version_number is None :
    latest = True
else :
    latest = False
data = { "download_request" : { "glob" : glob , "isDir" : is_dir , "latest" : latest } }
failure_message = "Failed to get matched files in dataset {}" . format ( dataset_id )
versions = self . _get_success_json ( self . _post_json ( routes . matched_files ( dataset_id ) , data , failure_message = failure_message ) ) [ 'versions' ]
# if you don ' t provide a version number , only the latest
# will be included in the response body
if version_number is None :
    version = versions [ 0 ]
else :
    try :
        version = list ( filter ( lambda v : v [ 'number' ] == version_number , versions ) ) [ 0 ]
    except IndexError :
        raise ResourceNotFoundException ( )
return list ( map ( lambda f : DatasetFile ( path = f [ 'filename' ] , url = f [ 'url' ] ) , version [ 'files' ] ) )


signature:
def get_dataset_file ( self , dataset_id , file_path , version = None ) :
    """Retrieves a dataset file matching a provided file path
 : param dataset _ id : The id of the dataset to retrieve file from
 : type dataset _ id : int
 : param file _ path : The file path within the dataset
 : type file _ path : str
 : param version : The dataset version to look for the file in . If nothing is supplied , the latest dataset version will be searched
 : type version : int
 : return : A dataset file matching the filepath provided
 : rtype : : class : ` DatasetFile `"""

implementation: return self . get_dataset_files ( dataset_id , "^{}$" . format ( file_path ) , version_number = version ) [ 0 ]


signature:
def download_files ( self , dataset_files , destination = '.' ) :
    """Downloads file ( s ) to a local destination .
 : param dataset _ files :
 : type dataset _ files : list of : class : ` DatasetFile `
 : param destination : The path to the desired local download destination
 : type destination : str
 : param chunk : Whether or not to chunk the file . Default True
 : type chunk : bool"""

implementation:
if not isinstance ( dataset_files , list ) :
    dataset_files = [ dataset_files ]
for f in dataset_files :
    filename = f . path . lstrip ( '/' )
    local_path = os . path . join ( destination , filename )
    if not os . path . isdir ( os . path . dirname ( local_path ) ) :
        os . makedirs ( os . path . dirname ( local_path ) )
    r = requests . get ( f . url , stream = True )
    with open ( local_path , 'wb' ) as output_file :
        shutil . copyfileobj ( r . raw , output_file )


signature:
def get_pif ( self , dataset_id , uid , dataset_version = None ) :
    """Retrieves a PIF from a given dataset .
 : param dataset _ id : The id of the dataset to retrieve PIF from
 : type dataset _ id : int
 : param uid : The uid of the PIF to retrieve
 : type uid : str
 : param dataset _ version : The dataset version to look for the PIF in . If nothing is supplied , the latest dataset version will be searched
 : type dataset _ version : int
 : return : A : class : ` Pif ` object
 : rtype : : class : ` Pif `"""

implementation:
failure_message = "An error occurred retrieving PIF {}" . format ( uid )
if dataset_version == None :
    response = self . _get ( routes . pif_dataset_uid ( dataset_id , uid ) , failure_message = failure_message )
else :
    response = self . _get ( routes . pif_dataset_version_uid ( dataset_id , uid , dataset_version ) , failure_message = failure_message )
return pif . loads ( response . content . decode ( "utf-8" ) )


signature:
def create_dataset ( self , name = None , description = None , public = False ) :
    """Create a new data set .
 : param name : name of the dataset
 : type name : str
 : param description : description for the dataset
 : type description : str
 : param public : A boolean indicating whether or not the dataset should be public .
 : type public : bool
 : return : The newly created dataset .
 : rtype : : class : ` Dataset `"""

implementation:
data = { "public" : _convert_bool_to_public_value ( public ) }
if name :
    data [ "name" ] = name
if description :
    data [ "description" ] = description
dataset = { "dataset" : data }
failure_message = "Unable to create dataset"
result = self . _get_success_json ( self . _post_json ( routes . create_dataset ( ) , dataset , failure_message = failure_message ) )
return _dataset_from_response_dict ( result )


signature:
def update_dataset ( self , dataset_id , name = None , description = None , public = None ) :
    """Update a data set .
 : param dataset _ id : The ID of the dataset to update
 : type dataset _ id : int
 : param name : name of the dataset
 : type name : str
 : param description : description for the dataset
 : type description : str
 : param public : A boolean indicating whether or not the dataset should
 be public .
 : type public : bool
 : return : The updated dataset .
 : rtype : : class : ` Dataset `"""

implementation:
data = { "public" : _convert_bool_to_public_value ( public ) }
if name :
    data [ "name" ] = name
if description :
    data [ "description" ] = description
dataset = { "dataset" : data }
failure_message = "Failed to update dataset {}" . format ( dataset_id )
response = self . _get_success_json ( self . _post_json ( routes . update_dataset ( dataset_id ) , data = dataset , failure_message = failure_message ) )
return _dataset_from_response_dict ( response )


signature:
def create_dataset_version ( self , dataset_id ) :
    """Create a new data set version .
 : param dataset _ id : The ID of the dataset for which the version must be bumped .
 : type dataset _ id : int
 : return : The new dataset version .
 : rtype : : class : ` DatasetVersion `"""

implementation:
failure_message = "Failed to create dataset version for dataset {}" . format ( dataset_id )
number = self . _get_success_json ( self . _post_json ( routes . create_dataset_version ( dataset_id ) , data = { } , failure_message = failure_message ) ) [ 'dataset_scoped_id' ]
return DatasetVersion ( number = number )


signature:
def get_available_columns ( self , dataset_ids ) :
    """Retrieves the set of columns from the combination of dataset ids given
 : param dataset _ ids : The id of the dataset to retrieve columns from
 : type dataset _ ids : list of int
 : return : A list of column names from the dataset ids given .
 : rtype : list of str"""

implementation:
if not isinstance ( dataset_ids , list ) :
    dataset_ids = [ dataset_ids ]
data = { "dataset_ids" : dataset_ids }
failure_message = "Failed to get available columns in dataset(s) {}" . format ( dataset_ids )
return self . _get_success_json ( self . _post_json ( 'v1/datasets/get-available-columns' , data , failure_message = failure_message ) ) [ 'data' ]


signature:
def __generate_search_template ( self , dataset_ids ) :
    """Generates a default search templates from the available columns in the dataset ids given .
 : param dataset _ ids : The id of the dataset to retrieve files from
 : type dataset _ ids : list of int
 : return : A search template based on the columns in the datasets given"""

implementation:
data = { "dataset_ids" : dataset_ids }
failure_message = "Failed to generate a search template from columns in dataset(s) {}" . format ( dataset_ids )
return self . _get_success_json ( self . _post_json ( 'v1/search_templates/builders/from-dataset-ids' , data , failure_message = failure_message ) ) [ 'data' ]


signature:
def __prune_search_template ( self , extract_as_keys , search_template ) :
    """Returns a new search template , but the new template has only the extract _ as _ keys given .
 : param extract _ as _ keys : List of extract as keys to keep
 : param search _ template : The search template to prune
 : return : New search template with pruned columns"""

implementation:
data = { "extract_as_keys" : extract_as_keys , "search_template" : search_template }
failure_message = "Failed to prune a search template"
return self . _get_success_json ( self . _post_json ( 'v1/search_templates/prune-to-extract-as' , data , failure_message = failure_message ) ) [ 'data' ]


signature:
def default ( self , obj ) :
    """Convert an object to a form ready to dump to json .
 : param obj : Object being serialized . The type of this object must be one of the following : None ; a single object derived from the Pio class ; or a list of objects , each derived from the Pio class .
 : return : List of dictionaries , each representing a physical information object , ready to be serialized ."""

implementation:
if obj is None :
    return [ ]
elif isinstance ( obj , list ) :
    return [ i . as_dictionary ( ) for i in obj ]
elif isinstance ( obj , dict ) :
    return self . _keys_to_camel_case ( obj )
else :
    return obj . as_dictionary ( )


signature:
def _keys_to_camel_case ( self , obj ) :
    """Make a copy of a dictionary with all keys converted to camel case . This is just calls to _ camel _ case on each of the keys in the dictionary and returns a new dictionary .
 : param obj : Dictionary to convert keys to camel case .
 : return : Dictionary with the input values and all keys in camel case"""

implementation: return dict ( ( to_camel_case ( key ) , value ) for ( key , value ) in obj . items ( ) )


signature:
def validate ( self , ml_template ) :
    """Runs the template against the validation endpoint , returns a message indicating status of the templte
 : param ml _ template : Template to validate
 : return : OK or error message if validation failed"""

implementation:
data = { "ml_template" : ml_template }
failure_message = "ML template validation invoke failed"
res = self . _get_success_json ( self . _post_json ( 'ml_templates/validate' , data , failure_message = failure_message ) ) [ 'data' ]
if res [ 'valid' ] :
    return 'OK'
return res [ 'reason' ]


signature:
def _validate_course_key ( course_key ) :
    """Validation helper"""

implementation:
if not validators . course_key_is_valid ( course_key ) :
    exceptions . raise_exception ( "CourseKey" , course_key , exceptions . InvalidCourseKeyException )


signature:
def _validate_organization_data ( organization_data ) :
    """Validation helper"""

implementation:
if not validators . organization_data_is_valid ( organization_data ) :
    exceptions . raise_exception ( "Organization" , organization_data , exceptions . InvalidOrganizationException )


signature:
def add_organization_course ( organization_data , course_key ) :
    """Adds a organization - course link to the system"""

implementation:
_validate_course_key ( course_key )
_validate_organization_data ( organization_data )
data . create_organization_course ( organization = organization_data , course_key = course_key )


signature:
def remove_organization_course ( organization , course_key ) :
    """Removes the specfied course from the specified organization"""

implementation:
_validate_organization_data ( organization )
_validate_course_key ( course_key )
return data . delete_organization_course ( course_key = course_key , organization = organization )


signature:
def raise_exception ( entity_type , entity , exception ) :
    """Exception helper"""

implementation: raise exception ( u'The {} you have provided is not valid: {}' . format ( entity_type , entity ) . encode ( 'utf-8' ) )


signature:
def course_key_is_valid ( course_key ) :
    """Course key object validation"""

implementation:
if course_key is None :
    return False
try :
    CourseKey . from_string ( text_type ( course_key ) )
except ( InvalidKeyError , UnicodeDecodeError ) :
    return False
return True


signature:
def organization_data_is_valid ( organization_data ) :
    """Organization data validation"""

implementation:
if organization_data is None :
    return False
if 'id' in organization_data and not organization_data . get ( 'id' ) :
    return False
if 'name' in organization_data and not organization_data . get ( 'name' ) :
    return False
return True


signature:
def _activate_organization ( organization ) :
    """Activates an inactivated ( soft - deleted ) organization as well as any inactive relationships"""

implementation:
[ _activate_organization_course_relationship ( record ) for record in internal . OrganizationCourse . objects . filter ( organization_id = organization . id , active = False ) ]
[ _activate_record ( record ) for record in internal . Organization . objects . filter ( id = organization . id , active = False ) ]


signature:
def _inactivate_organization ( organization ) :
    """Inactivates an activated organization as well as any active relationships"""

implementation:
[ _inactivate_organization_course_relationship ( record ) for record in internal . OrganizationCourse . objects . filter ( organization_id = organization . id , active = True ) ]
[ _inactivate_record ( record ) for record in internal . Organization . objects . filter ( id = organization . id , active = True ) ]


signature:
def _activate_organization_course_relationship ( relationship ) : # pylint : disable = invalid - name
    """Activates an inactive organization - course relationship"""

implementation:
# If the relationship doesn ' t exist or the organization isn ' t active we ' ll want to raise an error
relationship = internal . OrganizationCourse . objects . get ( id = relationship . id , active = False , organization__active = True )
_activate_record ( relationship )


signature:
def _inactivate_organization_course_relationship ( relationship ) : # pylint : disable = invalid - name
    """Inactivates an active organization - course relationship"""

implementation:
relationship = internal . OrganizationCourse . objects . get ( id = relationship . id , active = True )
_inactivate_record ( relationship )


signature:
def create_organization ( organization ) :
    """Inserts a new organization into app / local state given the following dictionary :
 ' name ' : string ,
 ' description ' : string
 Returns an updated dictionary including a new ' id ' : integer field / value"""

implementation:
# Trust , but verify . . .
if not organization . get ( 'name' ) :
    exceptions . raise_exception ( "organization" , organization , exceptions . InvalidOrganizationException )
organization_obj = serializers . deserialize_organization ( organization )
try :
    organization = internal . Organization . objects . get ( name = organization_obj . name , )
    # If the organization exists , but was inactivated , we can simply turn it back on
    if not organization . active :
        _activate_organization ( organization_obj )
except internal . Organization . DoesNotExist :
    organization = internal . Organization . objects . create ( name = organization_obj . name , short_name = organization_obj . short_name , description = organization_obj . description , logo = organization_obj . logo , active = True )
return serializers . serialize_organization ( organization )


signature:
def update_organization ( organization ) :
    """Updates an existing organization in app / local state
 Returns a dictionary representation of the object"""

implementation:
organization_obj = serializers . deserialize_organization ( organization )
try :
    organization = internal . Organization . objects . get ( id = organization_obj . id )
    organization . name = organization_obj . name
    organization . short_name = organization_obj . short_name
    organization . description = organization_obj . description
    organization . logo = organization_obj . logo
    organization . active = organization_obj . active
except internal . Organization . DoesNotExist :
    exceptions . raise_exception ( "organization" , organization , exceptions . InvalidOrganizationException )
return serializers . serialize_organization ( organization )


signature:
def fetch_organization ( organization_id ) :
    """Retrieves a specific organization from app / local state
 Returns a dictionary representation of the object"""

implementation:
organization = { 'id' : organization_id }
if not organization_id :
    exceptions . raise_exception ( "organization" , organization , exceptions . InvalidOrganizationException )
organizations = serializers . serialize_organizations ( internal . Organization . objects . filter ( id = organization_id , active = True ) )
if not organizations :
    exceptions . raise_exception ( "organization" , organization , exceptions . InvalidOrganizationException )
return organizations [ 0 ]


signature:
def fetch_organization_by_short_name ( organization_short_name ) :
    """Retrieves a specific organization from app / local state by short name
 Returns a dictionary representation of the object"""

implementation:
organization = { 'short_name' : organization_short_name }
if not organization_short_name :
    exceptions . raise_exception ( "organization" , organization , exceptions . InvalidOrganizationException )
organizations = serializers . serialize_organizations ( internal . Organization . objects . filter ( active = True , short_name = organization_short_name ) )
if not organizations :
    exceptions . raise_exception ( "organization" , organization , exceptions . InvalidOrganizationException )
return organizations [ 0 ]