signature:
def cds_score ( self , x_te , y_te ) :
    """Computes the cds statistic from variable 1 to variable 2
 Args :
 x _ te ( numpy . ndarray ) : Variable 1
 y _ te ( numpy . ndarray ) : Variable 2
 Returns :
 float : CDS fit score"""

implementation:
if type ( x_te ) == np . ndarray :
    x_te , y_te = pd . Series ( x_te . reshape ( - 1 ) ) , pd . Series ( y_te . reshape ( - 1 ) )
xd , yd = discretized_sequences ( x_te , y_te , self . ffactor , self . maxdev )
cx = Counter ( xd )
cy = Counter ( yd )
yrange = sorted ( cy . keys ( ) )
ny = len ( yrange )
py = np . array ( [ cy [ i ] for i in yrange ] , dtype = float )
py = py / py . sum ( )
pyx = [ ]
for a in cx :
    if cx [ a ] > self . minc :
        yx = y_te [ xd == a ]
        # if not numerical ( ty ) :
        # cyx = Counter ( yx )
        # pyxa = np . array ( [ cyx [ i ] for i in yrange ] , dtype = float )
        # pyxa . sort ( )
        if count_unique ( y_te ) > len_discretized_values ( y_te , "Numerical" , self . ffactor , self . maxdev ) :
            yx = ( yx - np . mean ( yx ) ) / np . std ( y_te )
            yx = discretized_sequence ( yx , "Numerical" , self . ffactor , self . maxdev , norm = False )
            cyx = Counter ( yx . astype ( int ) )
            pyxa = np . array ( [ cyx [ i ] for i in discretized_values ( y_te , "Numerical" , self . ffactor , self . maxdev ) ] , dtype = float )
        else :
            cyx = Counter ( yx )
            pyxa = [ cyx [ i ] for i in yrange ]
            pyxax = np . array ( [ 0 ] * ( ny - 1 ) + pyxa + [ 0 ] * ( ny - 1 ) , dtype = float )
            xcorr = [ sum ( py * pyxax [ i : i + ny ] ) for i in range ( 2 * ny - 1 ) ]
            imax = xcorr . index ( max ( xcorr ) )
            pyxa = np . array ( [ 0 ] * ( 2 * ny - 2 - imax ) + pyxa + [ 0 ] * imax , dtype = float )
        assert pyxa . sum ( ) == cx [ a ]
        pyxa = pyxa / pyxa . sum ( )
        pyx . append ( pyxa )
if len ( pyx ) == 0 :
    return 0
pyx = np . array ( pyx )
pyx = pyx - pyx . mean ( axis = 0 )
return np . std ( pyx )


signature:
def predict_proba ( self , a , b , ** kwargs ) :
    """Prediction method for pairwise causal inference using the ANM model .
 Args :
 a ( numpy . ndarray ) : Variable 1
 b ( numpy . ndarray ) : Variable 2
 Returns :
 float : Causation score ( Value : 1 if a - > b and - 1 if b - > a )"""

implementation:
a = scale ( a ) . reshape ( ( - 1 , 1 ) )
b = scale ( b ) . reshape ( ( - 1 , 1 ) )
return self . anm_score ( b , a ) - self . anm_score ( a , b )


signature:
def anm_score ( self , x , y ) :
    """Compute the fitness score of the ANM model in the x - > y direction .
 Args :
 a ( numpy . ndarray ) : Variable seen as cause
 b ( numpy . ndarray ) : Variable seen as effect
 Returns :
 float : ANM fit score"""

implementation:
gp = GaussianProcessRegressor ( ) . fit ( x , y )
y_predict = gp . predict ( x )
indepscore = normalized_hsic ( y_predict - y , x )
return indepscore


signature:
def orient_undirected_graph ( self , data , graph , ** kwargs ) :
    """Run PC on an undirected graph .
 Args :
 data ( pandas . DataFrame ) : DataFrame containing the data
 graph ( networkx . Graph ) : Skeleton of the graph to orient
 Returns :
 networkx . DiGraph : Solution given by PC on the given skeleton ."""

implementation:
# Building setup w / arguments .
self . arguments [ '{CITEST}' ] = self . dir_CI_test [ self . CI_test ]
self . arguments [ '{METHOD_INDEP}' ] = self . dir_method_indep [ self . method_indep ]
self . arguments [ '{DIRECTED}' ] = 'TRUE'
self . arguments [ '{ALPHA}' ] = str ( self . alpha )
self . arguments [ '{NJOBS}' ] = str ( self . nb_jobs )
self . arguments [ '{VERBOSE}' ] = str ( self . verbose ) . upper ( )
fe = DataFrame ( nx . adj_matrix ( graph , weight = None ) . todense ( ) )
fg = DataFrame ( 1 - fe . values )
results = self . _run_pc ( data , fixedEdges = fe , fixedGaps = fg , verbose = self . verbose )
return nx . relabel_nodes ( nx . DiGraph ( results ) , { idx : i for idx , i in enumerate ( data . columns ) } )


signature:
def create_graph_from_data ( self , data , ** kwargs ) :
    """Run the PC algorithm .
 Args :
 data ( pandas . DataFrame ) : DataFrame containing the data
 Returns :
 networkx . DiGraph : Solution given by PC on the given data ."""

implementation:
# Building setup w / arguments .
self . arguments [ '{CITEST}' ] = self . dir_CI_test [ self . CI_test ]
self . arguments [ '{METHOD_INDEP}' ] = self . dir_method_indep [ self . method_indep ]
self . arguments [ '{DIRECTED}' ] = 'TRUE'
self . arguments [ '{ALPHA}' ] = str ( self . alpha )
self . arguments [ '{NJOBS}' ] = str ( self . nb_jobs )
self . arguments [ '{VERBOSE}' ] = str ( self . verbose ) . upper ( )
results = self . _run_pc ( data , verbose = self . verbose )
return nx . relabel_nodes ( nx . DiGraph ( results ) , { idx : i for idx , i in enumerate ( data . columns ) } )


signature:
def _run_pc ( self , data , fixedEdges = None , fixedGaps = None , verbose = True ) :
    """Setting up and running pc with all arguments ."""

implementation:
# Checking coherence of arguments
# print ( self . arguments )
if ( self . arguments [ '{CITEST}' ] == self . dir_CI_test [ 'hsic' ] and self . arguments [ '{METHOD_INDEP}' ] == self . dir_method_indep [ 'corr' ] ) :
    warnings . warn ( 'Selected method for indep is unfit for the hsic test,' ' setting the hsic.gamma method.' )
    self . arguments [ '{METHOD_INDEP}' ] = self . dir_method_indep [ 'hsic_gamma' ]
elif ( self . arguments [ '{CITEST}' ] == self . dir_CI_test [ 'gaussian' ] and self . arguments [ '{METHOD_INDEP}' ] != self . dir_method_indep [ 'corr' ] ) :
    warnings . warn ( 'Selected method for indep is unfit for the selected test,' ' setting the classic correlation-based method.' )
    self . arguments [ '{METHOD_INDEP}' ] = self . dir_method_indep [ 'corr' ]
# Run PC
id = str ( uuid . uuid4 ( ) )
os . makedirs ( '/tmp/cdt_pc' + id + '/' )
self . arguments [ '{FOLDER}' ] = '/tmp/cdt_pc' + id + '/'
def retrieve_result ( ) :
    return read_csv ( '/tmp/cdt_pc' + id + '/result.csv' , delimiter = ',' ) . values
try :
    data . to_csv ( '/tmp/cdt_pc' + id + '/data.csv' , header = False , index = False )
    if fixedGaps is not None and fixedEdges is not None :
        fixedGaps . to_csv ( '/tmp/cdt_pc' + id + '/fixedgaps.csv' , index = False , header = False )
        fixedEdges . to_csv ( '/tmp/cdt_pc' + id + '/fixededges.csv' , index = False , header = False )
        self . arguments [ '{SKELETON}' ] = 'TRUE'
    else :
        self . arguments [ '{SKELETON}' ] = 'FALSE'
    pc_result = launch_R_script ( "{}/R_templates/pc.R" . format ( os . path . dirname ( os . path . realpath ( __file__ ) ) ) , self . arguments , output_function = retrieve_result , verbose = verbose )
# Cleanup
except Exception as e :
    rmtree ( '/tmp/cdt_pc' + id + '' )
    raise e
except KeyboardInterrupt :
    rmtree ( '/tmp/cdt_pc' + id + '/' )
    raise KeyboardInterrupt
rmtree ( '/tmp/cdt_pc' + id + '' )
return pc_result


signature:
def b_fit_score ( self , x , y ) :
    """Computes the cds statistic from variable 1 to variable 2
 Args :
 a ( numpy . ndarray ) : Variable 1
 b ( numpy . ndarray ) : Variable 2
 Returns :
 float : BF fit score"""

implementation:
x = np . reshape ( scale ( x ) , ( - 1 , 1 ) )
y = np . reshape ( scale ( y ) , ( - 1 , 1 ) )
gp = GaussianProcessRegressor ( ) . fit ( x , y )
y_predict = gp . predict ( x )
error = mean_squared_error ( y_predict , y )
return error


signature:
def predict ( self , data , alpha = 0.01 , max_iter = 2000 , ** kwargs ) :
    """Predict the graph skeleton .
 Args :
 data ( pandas . DataFrame ) : observational data
 alpha ( float ) : regularization parameter
 max _ iter ( int ) : maximum number of iterations
 Returns :
 networkx . Graph : Graph skeleton"""

implementation:
edge_model = GraphLasso ( alpha = alpha , max_iter = max_iter )
edge_model . fit ( data . values )
return nx . relabel_nodes ( nx . DiGraph ( edge_model . get_precision ( ) ) , { idx : i for idx , i in enumerate ( data . columns ) } )


signature:
def predict_features ( self , df_features , df_target , idx = 0 , ** kwargs ) :
    """For one variable , predict its neighbouring nodes .
 Args :
 df _ features ( pandas . DataFrame ) :
 df _ target ( pandas . Series ) :
 idx ( int ) : ( optional ) for printing purposes
 kwargs ( dict ) : additional options for algorithms
 Returns :
 list : scores of each feature relatively to the target
 . . warning : :
 Not implemented . Implemented by the algorithms ."""

implementation:
y = np . transpose ( df_target . values )
X = np . transpose ( df_features . values )
path , beta , A , lam = hsiclasso ( X , y )
return beta


signature:
def autoset_settings ( set_var ) :
    """Autoset GPU parameters using CUDA _ VISIBLE _ DEVICES variables .
 Return default config if variable not set .
 : param set _ var : Variable to set . Must be of type ConfigSettings"""

implementation:
try :
    devices = ast . literal_eval ( os . environ [ "CUDA_VISIBLE_DEVICES" ] )
    if type ( devices ) != list and type ( devices ) != tuple :
        devices = [ devices ]
    if len ( devices ) != 0 :
        set_var . GPU = len ( devices )
        set_var . NB_JOBS = len ( devices )
        warnings . warn ( "Detecting CUDA devices : {}" . format ( devices ) )
except KeyError :
    set_var . GPU = check_cuda_devices ( )
    set_var . NB_JOBS = set_var . GPU
    warnings . warn ( "Detecting {} CUDA devices." . format ( set_var . GPU ) )
    if not set_var . GPU :
        warnings . warn ( "No GPU automatically detected. Setting SETTINGS.GPU to 0, " + "and SETTINGS.NB_JOBS to cpu_count." )
        set_var . GPU = 0
        set_var . NB_JOBS = multiprocessing . cpu_count ( )
return set_var


signature:
def check_cuda_devices ( ) :
    """Output some information on CUDA - enabled devices on your computer ,
 including current memory usage . Modified to only get number of devices .
 It ' s a port of https : / / gist . github . com / f0k / 0d6431e3faa60bffc788f8b4daa029b1
 from C to Python with ctypes , so it can run without compiling
 anything . Note that this is a direct translation with no attempt to
 make the code Pythonic . It ' s meant as a general demonstration on how
 to obtain CUDA device information from Python without resorting to
 nvidia - smi or a compiled Python extension .
 . . note : : Author : Jan Schlüter , https : / / gist . github . com / 63a664160d016a491b2cbea15913d549 . git"""

implementation:
import ctypes
# Some constants taken from cuda . h
CUDA_SUCCESS = 0
libnames = ( 'libcuda.so' , 'libcuda.dylib' , 'cuda.dll' )
for libname in libnames :
    try :
        cuda = ctypes . CDLL ( libname )
    except OSError :
        continue
    else :
        break
else : # raise OSError ( " could not load any of : " + ' ' . join ( libnames ) )
    return 0
nGpus = ctypes . c_int ( )
error_str = ctypes . c_char_p ( )
result = cuda . cuInit ( 0 )
if result != CUDA_SUCCESS :
    cuda . cuGetErrorString ( result , ctypes . byref ( error_str ) )
    # print ( " cuInit failed with error code % d : % s " % ( result , error _ str . value . decode ( ) ) )
    return 0
result = cuda . cuDeviceGetCount ( ctypes . byref ( nGpus ) )
if result != CUDA_SUCCESS :
    cuda . cuGetErrorString ( result , ctypes . byref ( error_str ) )
    # print ( " cuDeviceGetCount failed with error code % d : % s " % ( result , error _ str . value . decode ( ) ) )
    return 0
# print ( " Found % d device ( s ) . " % nGpus . value )
return nGpus . value


signature:
def get_default ( self , * args , ** kwargs ) :
    """Get the default parameters as defined in the Settings instance .
 This function proceeds to seamlessly retrieve the argument to pass
 through , depending on either it was overidden or not : If no argument
 was overridden in a function of the toolbox , the default argument will
 be set to ` ` None ` ` , and this function will retrieve the default
 parameters as defined by the ` ` cdt . SETTINGS ` ` ' s attributes .
 It has two modes of processing :
 1 . * * kwargs for retrieving a single argument : ` ` get _ default ( argument _ name = value ) ` ` .
 2 . * args through a list of tuples of the shape ` ` ( ' argument _ name ' , value ) ` ` to retrieve multiple values at once ."""

implementation:
def retrieve_param ( i ) :
    try :
        return self . __getattribute__ ( i )
    except AttributeError :
        if i == "device" :
            return self . default_device
        else :
            return self . __getattribute__ ( i . upper ( ) )
if len ( args ) == 0 :
    if len ( kwargs ) == 1 and kwargs [ list ( kwargs . keys ( ) ) [ 0 ] ] is not None :
        return kwargs [ list ( kwargs . keys ( ) ) [ 0 ] ]
    elif len ( kwargs ) == 1 :
        return retrieve_param ( list ( kwargs . keys ( ) ) [ 0 ] )
    else :
        raise TypeError ( "As dict is unordered, it is impossible to give" "the parameters in the correct order." )
else :
    out = [ ]
    for i in args :
        if i [ 1 ] is None :
            out . append ( retrieve_param ( i [ 0 ] ) )
        else :
            out . append ( i [ 1 ] )
    return out


signature:
def read_adjacency_matrix ( filename , directed = True , ** kwargs ) :
    """Read a file ( containing an adjacency matrix ) and convert it into a
 directed or undirected networkx graph .
 : param filename : file to read or DataFrame containing the data
 : type filename : str or pandas . DataFrame
 : param directed : Return directed graph
 : type directed : bool
 : param kwargs : extra parameters to be passed to pandas . read _ csv
 : return : networkx graph containing the graph .
 : rtype : * * networkx . DiGraph * * or * * networkx . Graph * * depending on the
 ` ` directed ` ` parameter ."""

implementation:
if isinstance ( filename , str ) :
    data = read_csv ( filename , ** kwargs )
elif isinstance ( filename , DataFrame ) :
    data = filename
else :
    raise TypeError ( "Type not supported." )
if directed :
    return nx . relabel_nodes ( nx . DiGraph ( data . values ) , { idx : i for idx , i in enumerate ( data . columns ) } )
else :
    return nx . relabel_nodes ( nx . Graph ( data . values ) , { idx : i for idx , i in enumerate ( data . columns ) } )


signature:
def read_list_edges ( filename , directed = True , ** kwargs ) :
    """Read a file ( containing list of edges ) and convert it into a directed
 or undirected networkx graph .
 : param filename : file to read or DataFrame containing the data
 : type filename : str or pandas . DataFrame
 : param directed : Return directed graph
 : type directed : bool
 : param kwargs : extra parameters to be passed to pandas . read _ csv
 : return : networkx graph containing the graph .
 : rtype : * * networkx . DiGraph * * or * * networkx . Graph * * depending on the
 ` ` directed ` ` parameter ."""

implementation:
if isinstance ( filename , str ) :
    data = read_csv ( filename , ** kwargs )
elif isinstance ( filename , DataFrame ) :
    data = filename
else :
    raise TypeError ( "Type not supported." )
if directed :
    graph = nx . DiGraph ( )
else :
    graph = nx . Graph ( )
for idx , row in data . iterrows ( ) :
    try :
        score = row [ "Score" ]
    except KeyError :
        score = 1
    graph . add_edge ( row [ 'Cause' ] , row [ "Effect" ] , weight = score )
return graph


signature:
def forward ( self , pred , target ) :
    """Compute the loss model .
 : param pred : predicted Variable
 : param target : Target Variable
 : return : Loss"""

implementation:
loss = th . FloatTensor ( [ 0 ] )
for i in range ( 1 , self . moments ) :
    mk_pred = th . mean ( th . pow ( pred , i ) , 0 )
    mk_tar = th . mean ( th . pow ( target , i ) , 0 )
    loss . add_ ( th . mean ( ( mk_pred - mk_tar ) ** 2 ) )
# L2
return loss


signature:
def predict ( self , a , b ) :
    """Compute the test statistic
 Args :
 a ( array - like ) : Variable 1
 b ( array - like ) : Variable 2
 Returns :
 float : test statistic"""

implementation:
a = np . array ( a ) . reshape ( ( - 1 , 1 ) )
b = np . array ( b ) . reshape ( ( - 1 , 1 ) )
return ( mutual_info_regression ( a , b . reshape ( ( - 1 , ) ) ) + mutual_info_regression ( b , a . reshape ( ( - 1 , ) ) ) ) / 2


implementation:
a = np . array ( a ) . reshape ( ( - 1 , 1 ) )
b = np . array ( b ) . reshape ( ( - 1 , 1 ) )
return sp . kendalltau ( a , b ) [ 0 ]


signature:
def predict ( self , a , b , sig = [ - 1 , - 1 ] , maxpnt = 500 ) :
    """Compute the test statistic
 Args :
 a ( array - like ) : Variable 1
 b ( array - like ) : Variable 2
 sig ( list ) : [ 0 ] ( resp [ 1 ] ) is kernel size for a ( resp b ) ( set to median distance if - 1)
 maxpnt ( int ) : maximum number of points used , for computational time
 Returns :
 float : test statistic"""

implementation:
a = ( a - np . mean ( a ) ) / np . std ( a )
b = ( b - np . mean ( b ) ) / np . std ( b )
return FastHsicTestGamma ( a , b , sig , maxpnt )


signature:
def predict ( self , x , * args , ** kwargs ) :
    """Generic predict method , chooses which subfunction to use for a more
 suited .
 Depending on the type of ` x ` and of ` * args ` , this function process to execute
 different functions in the priority order :
 1 . If ` ` args [ 0 ] ` ` is a ` ` networkx . ( Di ) Graph ` ` , then ` ` self . orient _ graph ` ` is executed .
 2 . If ` ` args [ 0 ] ` ` exists , then ` ` self . predict _ proba ` ` is executed .
 3 . If ` ` x ` ` is a ` ` pandas . DataFrame ` ` , then ` ` self . predict _ dataset ` ` is executed .
 4 . If ` ` x ` ` is a ` ` pandas . Series ` ` , then ` ` self . predict _ proba ` ` is executed .
 Args :
 x ( numpy . array or pandas . DataFrame or pandas . Series ) : First variable or dataset .
 args ( numpy . array or networkx . Graph ) : graph or second variable .
 Returns :
 pandas . Dataframe or networkx . Digraph : predictions output"""

implementation:
if len ( args ) > 0 :
    if type ( args [ 0 ] ) == nx . Graph or type ( args [ 0 ] ) == nx . DiGraph :
        return self . orient_graph ( x , * args , ** kwargs )
    else :
        return self . predict_proba ( x , * args , ** kwargs )
elif type ( x ) == DataFrame :
    return self . predict_dataset ( x , * args , ** kwargs )
elif type ( x ) == Series :
    return self . predict_proba ( x . iloc [ 0 ] , x . iloc [ 1 ] , * args , ** kwargs )


signature:
def predict_dataset ( self , x , ** kwargs ) :
    """Generic dataset prediction function .
 Runs the score independently on all pairs .
 Args :
 x ( pandas . DataFrame ) : a CEPC format Dataframe .
 kwargs ( dict ) : additional arguments for the algorithms
 Returns :
 pandas . DataFrame : a Dataframe with the predictions ."""

implementation:
printout = kwargs . get ( "printout" , None )
pred = [ ]
res = [ ]
x . columns = [ "A" , "B" ]
for idx , row in x . iterrows ( ) :
    a = scale ( row [ 'A' ] . reshape ( ( len ( row [ 'A' ] ) , 1 ) ) )
    b = scale ( row [ 'B' ] . reshape ( ( len ( row [ 'B' ] ) , 1 ) ) )
    pred . append ( self . predict_proba ( a , b , idx = idx ) )
    if printout is not None :
        res . append ( [ row [ 'SampleID' ] , pred [ - 1 ] ] )
        DataFrame ( res , columns = [ 'SampleID' , 'Predictions' ] ) . to_csv ( printout , index = False )
return pred


signature:
def orient_graph ( self , df_data , graph , nb_runs = 6 , printout = None , ** kwargs ) :
    """Orient an undirected graph using the pairwise method defined by the subclass .
 The pairwise method is ran on every undirected edge .
 Args :
 df _ data ( pandas . DataFrame ) : Data
 umg ( networkx . Graph ) : Graph to orient
 nb _ runs ( int ) : number of times to rerun for each pair ( bootstrap )
 printout ( str ) : ( optional ) Path to file where to save temporary results
 Returns :
 networkx . DiGraph : a directed graph , which might contain cycles
 . . warning :
 Requirement : Name of the nodes in the graph correspond to name of
 the variables in df _ data"""

implementation:
if type ( graph ) == nx . DiGraph :
    edges = [ a for a in list ( graph . edges ( ) ) if ( a [ 1 ] , a [ 0 ] ) in list ( graph . edges ( ) ) ]
    oriented_edges = [ a for a in list ( graph . edges ( ) ) if ( a [ 1 ] , a [ 0 ] ) not in list ( graph . edges ( ) ) ]
    for a in edges :
        if ( a [ 1 ] , a [ 0 ] ) in list ( graph . edges ( ) ) :
            edges . remove ( a )
    output = nx . DiGraph ( )
    for i in oriented_edges :
        output . add_edge ( * i )
elif type ( graph ) == nx . Graph :
    edges = list ( graph . edges ( ) )
    output = nx . DiGraph ( )
else :
    raise TypeError ( "Data type not understood." )
res = [ ]
for idx , ( a , b ) in enumerate ( edges ) :
    weight = self . predict_proba ( df_data [ a ] . values . reshape ( ( - 1 , 1 ) ) , df_data [ b ] . values . reshape ( ( - 1 , 1 ) ) , idx = idx , nb_runs = nb_runs , ** kwargs )
    if weight > 0 : # a causes b
        output . add_edge ( a , b , weight = weight )
    else :
        output . add_edge ( b , a , weight = abs ( weight ) )
    if printout is not None :
        res . append ( [ str ( a ) + '-' + str ( b ) , weight ] )
        DataFrame ( res , columns = [ 'SampleID' , 'Predictions' ] ) . to_csv ( printout , index = False )
for node in list ( df_data . columns . values ) :
    if node not in output . nodes ( ) :
        output . add_node ( node )
return output


signature:
def orient_undirected_graph ( self , data , graph ) :
    """Run the algorithm on an undirected graph .
 Args :
 data ( pandas . DataFrame ) : DataFrame containing the data
 graph ( networkx . Graph ) : Skeleton of the graph to orient
 Returns :
 networkx . DiGraph : Solution on the given skeleton ."""

implementation:
# Building setup w / arguments .
self . arguments [ '{VERBOSE}' ] = str ( self . verbose ) . upper ( )
self . arguments [ '{SCORE}' ] = self . score
self . arguments [ '{BETA}' ] = str ( self . beta )
self . arguments [ '{OPTIM}' ] = str ( self . optim ) . upper ( )
self . arguments [ '{ALPHA}' ] = str ( self . alpha )
whitelist = DataFrame ( list ( nx . edges ( graph ) ) , columns = [ "from" , "to" ] )
blacklist = DataFrame ( list ( nx . edges ( nx . DiGraph ( DataFrame ( - nx . adj_matrix ( graph , weight = None ) . to_dense ( ) + 1 , columns = list ( graph . nodes ( ) ) , index = list ( graph . nodes ( ) ) ) ) ) ) , columns = [ "from" , "to" ] )
results = self . _run_bnlearn ( data , whitelist = whitelist , blacklist = blacklist , verbose = self . verbose )
return nx . relabel_nodes ( nx . DiGraph ( results ) , { idx : i for idx , i in enumerate ( data . columns ) } )


signature:
def orient_directed_graph ( self , data , graph ) :
    """Run the algorithm on a directed _ graph .
 Args :
 data ( pandas . DataFrame ) : DataFrame containing the data
 graph ( networkx . DiGraph ) : Skeleton of the graph to orient
 Returns :
 networkx . DiGraph : Solution on the given skeleton .
 . . warning : :
 The algorithm is ran on the skeleton of the given graph ."""

implementation:
warnings . warn ( "The algorithm is ran on the skeleton of the given graph." )
return self . orient_undirected_graph ( data , nx . Graph ( graph ) )


signature:
def create_graph_from_data ( self , data ) :
    """Run the algorithm on data .
 Args :
 data ( pandas . DataFrame ) : DataFrame containing the data
 Returns :
 networkx . DiGraph : Solution given by the algorithm ."""

implementation:
# Building setup w / arguments .
self . arguments [ '{SCORE}' ] = self . score
self . arguments [ '{VERBOSE}' ] = str ( self . verbose ) . upper ( )
self . arguments [ '{BETA}' ] = str ( self . beta )
self . arguments [ '{OPTIM}' ] = str ( self . optim ) . upper ( )
self . arguments [ '{ALPHA}' ] = str ( self . alpha )
results = self . _run_bnlearn ( data , verbose = self . verbose )
graph = nx . DiGraph ( )
graph . add_edges_from ( results )
return graph


signature:
def computeGaussKernel ( x ) :
    """Compute the gaussian kernel on a 1D vector ."""

implementation:
xnorm = np . power ( euclidean_distances ( x , x ) , 2 )
return np . exp ( - xnorm / ( 2.0 ) )


signature:
def gmm_cause ( points , k = 4 , p1 = 2 , p2 = 2 ) :
    """Init a root cause with a Gaussian Mixture Model w / a spherical covariance type ."""

implementation:
g = GMM ( k , covariance_type = "spherical" )
g . fit ( np . random . randn ( 300 , 1 ) )
g . means_ = p1 * np . random . randn ( k , 1 )
g . covars_ = np . power ( abs ( p2 * np . random . randn ( k , 1 ) + 1 ) , 2 )
g . weights_ = abs ( np . random . rand ( k ) )
g . weights_ = g . weights_ / sum ( g . weights_ )
return g . sample ( points ) [ 0 ] . reshape ( - 1 )


signature:
def normal_noise ( points ) :
    """Init a noise variable ."""

implementation: return np . random . rand ( 1 ) * np . random . randn ( points , 1 ) + random . sample ( [ 2 , - 2 ] , 1 )


signature:
def uniform_noise ( points ) :
    """Init a uniform noise variable ."""

implementation: return np . random . rand ( 1 ) * np . random . uniform ( points , 1 ) + random . sample ( [ 2 , - 2 ] , 1 )


signature:
def mechanism ( self , x ) :
    """Mechanism function ."""

implementation:
result = np . zeros ( ( self . points , 1 ) )
for i in range ( self . points ) :
    result [ i , 0 ] = self . a * self . b * ( x [ i ] + self . c ) / ( 1 + abs ( self . b * ( x [ i ] + self . c ) ) )
return result + self . noise


signature:
def mechanism ( self , causes ) :
    """Mechanism function ."""

implementation:
result = np . zeros ( ( self . points , 1 ) )
for i in range ( self . points ) :
    pre_add_effect = 0
    for c in range ( causes . shape [ 1 ] ) :
        pre_add_effect += causes [ i , c ]
    pre_add_effect += self . noise [ i ]
    result [ i , 0 ] = self . a * self . b * ( pre_add_effect + self . c ) / ( 1 + abs ( self . b * ( pre_add_effect + self . c ) ) )
return result


signature:
def mechanism ( self , x , par ) :
    """Mechanism function ."""

implementation:
list_coeff = self . polycause [ par ]
result = np . zeros ( ( self . points , 1 ) )
for i in range ( self . points ) :
    for j in range ( self . d + 1 ) :
        result [ i , 0 ] += list_coeff [ j ] * np . power ( x [ i ] , j )
    result [ i , 0 ] = min ( result [ i , 0 ] , 1 )
    result [ i , 0 ] = max ( result [ i , 0 ] , - 1 )
return result


implementation:
self . nb_step += 1
x = np . reshape ( x , ( x . shape [ 0 ] , 1 ) )
if ( self . nb_step < 5 ) :
    cov = computeGaussKernel ( x )
    mean = np . zeros ( ( 1 , self . points ) ) [ 0 , : ]
    y = np . random . multivariate_normal ( mean , cov )
elif ( self . nb_step == 5 ) :
    cov = computeGaussKernel ( x )
    mean = np . zeros ( ( 1 , self . points ) ) [ 0 , : ]
    y = np . random . multivariate_normal ( mean , cov )
    self . gpr = GaussianProcessRegressor ( )
    self . gpr . fit ( x , y )
    y = self . gpr . predict ( x )
else :
    y = self . gpr . predict ( x )
return y


implementation:
layers = [ ]
layers . append ( th . nn . modules . Linear ( self . n_causes + 1 , self . nh ) )
layers . append ( th . nn . Tanh ( ) )
layers . append ( th . nn . modules . Linear ( self . nh , 1 ) )
self . layers = th . nn . Sequential ( * layers )
data = x . astype ( 'float32' )
data = th . from_numpy ( data )
return np . reshape ( self . layers ( data ) . data , ( x . shape [ 0 ] , ) )


signature:
def predict_dataset ( self , df ) :
    """Runs Jarfo independently on all pairs .
 Args :
 x ( pandas . DataFrame ) : a CEPC format Dataframe .
 kwargs ( dict ) : additional arguments for the algorithms
 Returns :
 pandas . DataFrame : a Dataframe with the predictions ."""

implementation:
if len ( list ( df . columns ) ) == 2 :
    df . columns = [ "A" , "B" ]
if self . model is None :
    raise AssertionError ( "Model has not been trained before predictions" )
df2 = DataFrame ( )
for idx , row in df . iterrows ( ) :
    df2 = df2 . append ( row , ignore_index = True )
    df2 = df2 . append ( { 'A' : row [ "B" ] , 'B' : row [ "A" ] } , ignore_index = True )
return predict . predict ( deepcopy ( df2 ) , deepcopy ( self . model ) ) [ : : 2 ]


signature:
def predict_proba ( self , a , b , idx = 0 , ** kwargs ) :
    """Use Jarfo to predict the causal direction of a pair of vars .
 Args :
 a ( numpy . ndarray ) : Variable 1
 b ( numpy . ndarray ) : Variable 2
 idx ( int ) : ( optional ) index number for printing purposes
 Returns :
 float : Causation score ( Value : 1 if a - > b and - 1 if b - > a )"""

implementation: return self . predict_dataset ( DataFrame ( [ [ a , b ] ] , columns = [ 'A' , 'B' ] ) )


signature:
def network_deconvolution ( mat , ** kwargs ) :
    """Python implementation / translation of network deconvolution by MIT - KELLIS LAB .
 . . note : :
 code author : gidonro [ Github username ] ( https : / / github . com / gidonro / Network - Deconvolution )
 LICENSE : MIT - KELLIS LAB
 AUTHORS :
 Algorithm was programmed by Soheil Feizi .
 Paper authors are S . Feizi , D . Marbach , M . M ? © dard and M . Kellis
 Python implementation : Gideon Rosenthal
 For more details , see the following paper :
 Network Deconvolution as a General Method to Distinguish
 Direct Dependencies over Networks
 By : Soheil Feizi , Daniel Marbach , Muriel Médard and Manolis Kellis
 Nature Biotechnology
 Args :
 mat ( numpy . ndarray ) : matrix , if it is a square matrix , the program assumes
 it is a relevance matrix where mat ( i , j ) represents the similarity content
 between nodes i and j . Elements of matrix should be
 non - negative .
 beta ( float ) : Scaling parameter , the program maps the largest absolute eigenvalue
 of the direct dependency matrix to beta . It should be
 between 0 and 1.
 alpha ( float ) : fraction of edges of the observed dependency matrix to be kept in
 deconvolution process .
 control ( int ) : if 0 , displaying direct weights for observed
 interactions , if 1 , displaying direct weights for both observed and
 non - observed interactions .
 Returns :
 mat _ nd ( numpy . ndarray ) : Output deconvolved matrix ( direct dependency matrix ) . Its components
 represent direct edge weights of observed interactions .
 Choosing top direct interactions ( a cut - off ) depends on the application and
 is not implemented in this code .
 . . note : :
 To apply ND on regulatory networks , follow steps explained in Supplementary notes
 1.4.1 and 2.1 and 2.3 of the paper .
 In this implementation , input matrices are made symmetric ."""

implementation:
alpha = kwargs . get ( 'alpha' , 1 )
beta = kwargs . get ( 'beta' , 0.99 )
control = kwargs . get ( 'control' , 0 )
# ToDO : ASSERTS
try :
    assert beta < 1 or beta > 0
    assert alpha <= 1 or alpha > 0
except AssertionError :
    raise ValueError ( "alpha must be in ]0, 1] and beta in [0, 1]" )
# Processing the input matrix , diagonal values are filtered
np . fill_diagonal ( mat , 0 )
# Thresholding the input matrix
y = stat . mquantiles ( mat [ : ] , prob = [ 1 - alpha ] )
th = mat >= y
mat_th = mat * th
# Making the matrix symetric if already not
mat_th = ( mat_th + mat_th . T ) / 2
# Eigen decomposition
Dv , U = LA . eigh ( mat_th )
D = np . diag ( ( Dv ) )
lam_n = np . abs ( np . min ( np . min ( np . diag ( D ) ) , 0 ) )
lam_p = np . abs ( np . max ( np . max ( np . diag ( D ) ) , 0 ) )
m1 = lam_p * ( 1 - beta ) / beta
m2 = lam_n * ( 1 + beta ) / beta
m = max ( m1 , m2 )
# network deconvolution
for i in range ( D . shape [ 0 ] ) :
    D [ i , i ] = ( D [ i , i ] ) / ( m + D [ i , i ] )
mat_new1 = np . dot ( U , np . dot ( D , LA . inv ( U ) ) )
# Displying direct weights
if control == 0 :
    ind_edges = ( mat_th > 0 ) * 1.0
    ind_nonedges = ( mat_th == 0 ) * 1.0
    m1 = np . max ( np . max ( mat * ind_nonedges ) )
    m2 = np . min ( np . min ( mat_new1 ) )
    mat_new2 = ( mat_new1 + np . max ( m1 - m2 , 0 ) ) * ind_edges + ( mat * ind_nonedges )
else :
    m2 = np . min ( np . min ( mat_new1 ) )
    mat_new2 = ( mat_new1 + np . max ( - m2 , 0 ) )
# linearly mapping the deconvolved matrix to be between 0 and 1
m1 = np . min ( np . min ( mat_new2 ) )
m2 = np . max ( np . max ( mat_new2 ) )
mat_nd = ( mat_new2 - m1 ) / ( m2 - m1 )
return mat_nd


signature:
def clr ( M , ** kwargs ) :
    """Implementation of the Context Likelihood or Relatedness Network algorithm .
 Args :
 mat ( numpy . ndarray ) : matrix , if it is a square matrix , the program assumes
 it is a relevance matrix where mat ( i , j ) represents the similarity content
 between nodes i and j . Elements of matrix should be
 non - negative .
 Returns :
 mat _ nd ( numpy . ndarray ) : Output deconvolved matrix ( direct dependency matrix ) . Its components
 represent direct edge weights of observed interactions .
 . . note : :
 Ref : Jeremiah J . Faith , Boris Hayete , Joshua T . Thaden , Ilaria Mogno , Jamey
 Wierzbowski , Guillaume Cottarel , Simon Kasif , James J . Collins , and Timothy
 S . Gardner . Large - scale mapping and validation of escherichia coli
 transcriptional regulation from a compendium of expression profiles .
 PLoS Biology , 2007"""

implementation:
R = np . zeros ( M . shape )
Id = [ [ 0 , 0 ] for i in range ( M . shape [ 0 ] ) ]
for i in range ( M . shape [ 0 ] ) :
    mu_i = np . mean ( M [ i , : ] )
    sigma_i = np . std ( M [ i , : ] )
    Id [ i ] = [ mu_i , sigma_i ]
for i in range ( M . shape [ 0 ] ) :
    for j in range ( i + 1 , M . shape [ 0 ] ) :
        z_i = np . max ( [ 0 , ( M [ i , j ] - Id [ i ] [ 0 ] ) / Id [ i ] [ 0 ] ] )
        z_j = np . max ( [ 0 , ( M [ i , j ] - Id [ j ] [ 0 ] ) / Id [ j ] [ 0 ] ] )
        R [ i , j ] = np . sqrt ( z_i ** 2 + z_j ** 2 )
        R [ j , i ] = R [ i , j ]
# Symmetric
return R


signature:
def aracne ( m , ** kwargs ) :
    """Implementation of the ARACNE algorithm .
 Args :
 mat ( numpy . ndarray ) : matrix , if it is a square matrix , the program assumes
 it is a relevance matrix where mat ( i , j ) represents the similarity content
 between nodes i and j . Elements of matrix should be
 non - negative .
 Returns :
 mat _ nd ( numpy . ndarray ) : Output deconvolved matrix ( direct dependency matrix ) . Its components
 represent direct edge weights of observed interactions .
 . . note : :
 Ref : ARACNE : An Algorithm for the Reconstruction of Gene Regulatory Networks in a Mammalian Cellular Context
 Adam A Margolin , Ilya Nemenman , Katia Basso , Chris Wiggins , Gustavo Stolovitzky , Riccardo Dalla Favera and Andrea Califano
 DOI : https : / / doi . org / 10.1186/1471-2105-7 - S1 - S7"""

implementation:
I0 = kwargs . get ( 'I0' , 0.0 )
# No default thresholding
W0 = kwargs . get ( 'W0' , 0.05 )
# thresholding
m = np . where ( m > I0 , m , 0 )
# Finding triplets and filtering them
for i in range ( m . shape [ 0 ] - 2 ) :
    for j in range ( i + 1 , m . shape [ 0 ] - 1 ) :
        for k in range ( j + 1 , m . shape [ 0 ] ) :
            triplet = [ m [ i , j ] , m [ j , k ] , m [ i , k ] ]
            min_index , min_value = min ( enumerate ( triplet ) , key = operator . itemgetter ( 1 ) )
            if 0 < min_value < W0 :
                if min_index == 0 :
                    m [ i , j ] = m [ j , i ] = 0.
                elif min_index == 1 :
                    m [ j , k ] = m [ k , j ] = 0.
                else :
                    m [ i , k ] = m [ k , i ] = 0.
return m


signature:
def remove_indirect_links ( g , alg = "aracne" , ** kwargs ) :
    """Apply deconvolution to a networkx graph .
 Args :
 g ( networkx . Graph ) : Graph to apply deconvolution to
 alg ( str ) : Algorithm to use ( ' aracne ' , ' clr ' , ' nd ' )
 kwargs ( dict ) : extra options for algorithms
 Returns :
 networkx . Graph : graph with undirected links removed ."""

implementation:
alg = { "aracne" : aracne , "nd" : network_deconvolution , "clr" : clr } [ alg ]
mat = np . array ( nx . adjacency_matrix ( g ) . todense ( ) )
return nx . relabel_nodes ( nx . DiGraph ( alg ( mat , ** kwargs ) ) , { idx : i for idx , i in enumerate ( list ( g . nodes ( ) ) ) } )


signature:
def dagify_min_edge ( g ) :
    """Input a graph and output a DAG .
 The heuristic is to reverse the edge with the lowest score of the cycle
 if possible , else remove it .
 Args :
 g ( networkx . DiGraph ) : Graph to modify to output a DAG
 Returns :
 networkx . DiGraph : DAG made out of the input graph ."""

implementation:
while not nx . is_directed_acyclic_graph ( g ) :
    cycle = next ( nx . simple_cycles ( g ) )
    scores = [ ]
    edges = [ ]
    for i , j in zip ( cycle [ : 1 ] , cycle [ : 1 ] ) :
        edges . append ( ( i , j ) )
        scores . append ( g [ i ] [ j ] [ 'weight' ] )
    i , j = edges [ scores . index ( min ( scores ) ) ]
    gc = deepcopy ( g )
    gc . remove_edge ( i , j )
    gc . add_edge ( j , i )
    if len ( list ( nx . simple_cycles ( gc ) ) ) < len ( list ( nx . simple_cycles ( g ) ) ) :
        g . add_edge ( j , i , weight = min ( scores ) )
    g . remove_edge ( i , j )
return g


signature:
def weighted_mean_and_std ( values , weights ) :
    """Returns the weighted average and standard deviation .
 values , weights - - numpy ndarrays with the same shape ."""

implementation:
average = np . average ( values , weights = weights , axis = 0 )
variance = np . dot ( weights , ( values - average ) ** 2 ) / weights . sum ( )
# Fast and numerically precise
return ( average , np . sqrt ( variance ) )


signature:
def GNN_instance ( x , idx = 0 , device = None , nh = 20 , ** kwargs ) :
    """Run an instance of GNN , testing causal direction .
 : param m : data corresponding to the config : ( N , 2 ) data , [ : , 0 ] cause and [ : , 1 ] effect
 : param pair _ idx : print purposes
 : param run : numner of the run ( for GPU dispatch )
 : param device : device on with the algorithm is going to be run on .
 : return :"""

implementation:
device = SETTINGS . get_default ( device = device )
xy = scale ( x ) . astype ( 'float32' )
inputx = th . FloatTensor ( xy [ : , [ 0 ] ] ) . to ( device )
target = th . FloatTensor ( xy [ : , [ 1 ] ] ) . to ( device )
GNNXY = GNN_model ( x . shape [ 0 ] , device = device , nh = nh ) . to ( device )
GNNYX = GNN_model ( x . shape [ 0 ] , device = device , nh = nh ) . to ( device )
GNNXY . reset_parameters ( )
GNNYX . reset_parameters ( )
XY = GNNXY . run ( inputx , target , ** kwargs )
YX = GNNYX . run ( target , inputx , ** kwargs )
return [ XY , YX ]


signature:
def forward ( self , x ) :
    """Pass data through the net structure .
 : param x : input data : shape ( : , 1)
 : type x : torch . Variable
 : return : output of the shallow net
 : rtype : torch . Variable"""

implementation:
self . noise . normal_ ( )
return self . layers ( th . cat ( [ x , self . noise ] , 1 ) )


signature:
def run ( self , x , y , lr = 0.01 , train_epochs = 1000 , test_epochs = 1000 , idx = 0 , verbose = None , ** kwargs ) :
    """Run the GNN on a pair x , y of FloatTensor data ."""

implementation:
verbose = SETTINGS . get_default ( verbose = verbose )
optim = th . optim . Adam ( self . parameters ( ) , lr = lr )
running_loss = 0
teloss = 0
for i in range ( train_epochs + test_epochs ) :
    optim . zero_grad ( )
    pred = self . forward ( x )
    loss = self . criterion ( pred , y )
    running_loss += loss . item ( )
    if i < train_epochs :
        loss . backward ( )
        optim . step ( )
    else :
        teloss += running_loss
    # print statistics
    if verbose and not i % 300 :
        print ( 'Idx:{}; epoch:{}; score:{}' . format ( idx , i , running_loss / 300 ) )
        running_loss = 0.0
return teloss / test_epochs


signature:
def predict_proba ( self , a , b , nb_runs = 6 , nb_jobs = None , gpu = None , idx = 0 , verbose = None , ttest_threshold = 0.01 , nb_max_runs = 16 , train_epochs = 1000 , test_epochs = 1000 ) :
    """Run multiple times GNN to estimate the causal direction .
 Args :
 a ( np . ndarray ) : Variable 1
 b ( np . ndarray ) : Variable 2
 nb _ runs ( int ) : number of runs to execute per batch ( before testing for significance with t - test ) .
 nb _ jobs ( int ) : number of runs to execute in parallel . ( Initialized with ` ` cdt . SETTINGS . NB _ JOBS ` ` )
 gpu ( bool ) : use gpu ( Initialized with ` ` cdt . SETTINGS . GPU ` ` )
 idx ( int ) : ( optional ) index of the pair , for printing purposes
 verbose ( bool ) : verbosity ( Initialized with ` ` cdt . SETTINGS . verbose ` ` )
 ttest _ threshold ( float ) : threshold to stop the boostraps before ` ` nb _ max _ runs ` ` if the difference is significant
 nb _ max _ runs ( int ) : Max number of bootstraps
 train _ epochs ( int ) : Number of epochs during which the model is going to be trained
 test _ epochs ( int ) : Number of epochs during which the model is going to be tested
 Returns :
 float : Causal score of the pair ( Value : 1 if a - > b and - 1 if b - > a )"""

implementation:
Nb_jobs , verbose , gpu = SETTINGS . get_default ( ( 'nb_jobs' , nb_jobs ) , ( 'verbose' , verbose ) , ( 'gpu' , gpu ) )
x = np . stack ( [ a . ravel ( ) , b . ravel ( ) ] , 1 )
ttest_criterion = TTestCriterion ( max_iter = nb_max_runs , runs_per_iter = nb_runs , threshold = ttest_threshold )
AB = [ ]
BA = [ ]
while ttest_criterion . loop ( AB , BA ) :
    if nb_jobs != 1 :
        result_pair = Parallel ( n_jobs = nb_jobs ) ( delayed ( GNN_instance ) ( x , idx = idx , device = 'cuda:{}' . format ( run % gpu ) if gpu else 'cpu' , verbose = verbose , train_epochs = train_epochs , test_epochs = test_epochs ) for run in range ( ttest_criterion . iter , ttest_criterion . iter + nb_runs ) )
    else :
        result_pair = [ GNN_instance ( x , idx = idx , device = 'cuda:0' if gpu else 'cpu' , verbose = verbose , train_epochs = train_epochs , test_epochs = test_epochs ) for run in range ( ttest_criterion . iter , ttest_criterion . iter + nb_runs ) ]
    AB . extend ( [ runpair [ 0 ] for runpair in result_pair ] )
    BA . extend ( [ runpair [ 1 ] for runpair in result_pair ] )
if verbose :
    print ( "P-value after {} runs : {}" . format ( ttest_criterion . iter , ttest_criterion . p_value ) )
score_AB = np . mean ( AB )
score_BA = np . mean ( BA )
return ( score_BA - score_AB ) / ( score_BA + score_AB )


signature:
def init_variables ( self , verbose = False ) :
    """Redefine the causes of the graph ."""

implementation:
# Resetting adjacency matrix
for i in range ( self . nodes ) :
    for j in np . random . choice ( range ( self . nodes ) , np . random . randint ( 0 , self . parents_max + 1 ) , replace = False ) :
        if i != j :
            self . adjacency_matrix [ j , i ] = 1
try :
    assert any ( [ sum ( self . adjacency_matrix [ : , i ] ) == self . parents_max for i in range ( self . nodes ) ] )
    self . g = nx . DiGraph ( self . adjacency_matrix )
    assert list ( nx . simple_cycles ( self . g ) )
    assert any ( len ( i ) == 2 for i in nx . simple_cycles ( self . g ) )
except AssertionError :
    if verbose :
        print ( "Regenerating, graph non valid..." )
    self . init_variables ( )
if verbose :
    print ( "Matrix generated ! \
              Number of cycles: {}" . format ( len ( list ( nx . simple_cycles ( self . g ) ) ) ) )
for i in range ( self . nodes ) :
    self . data . iloc [ : , i ] = scale ( self . initial_generator ( self . points ) )
# Mechanisms
self . cfunctions = [ self . mechanism ( int ( sum ( self . adjacency_matrix [ : , i ] ) ) , self . points , self . noise , noise_coeff = self . noise_coeff ) for i in range ( self . nodes ) ]


signature:
def generate ( self , nb_steps = 100 , averaging = 50 , rescale = True ) :
    """Generate data from an FCM containing cycles ."""

implementation:
if self . cfunctions is None :
    self . init_variables ( )
new_df = pd . DataFrame ( )
causes = [ [ c for c in np . nonzero ( self . adjacency_matrix [ : , j ] ) [ 0 ] ] for j in range ( self . nodes ) ]
values = [ [ ] for i in range ( self . nodes ) ]
for i in range ( nb_steps ) :
    for j in range ( self . nodes ) :
        new_df [ "V" + str ( j ) ] = self . cfunctions [ j ] ( self . data . iloc [ : , causes [ j ] ] . values ) [ : , 0 ]
        if rescale :
            new_df [ "V" + str ( j ) ] = scale ( new_df [ "V" + str ( j ) ] )
        if i > nb_steps - averaging :
            values [ j ] . append ( new_df [ "V" + str ( j ) ] )
    self . data = new_df
self . data = pd . DataFrame ( np . array ( [ np . mean ( values [ i ] , axis = 0 ) for i in range ( self . nodes ) ] ) . transpose ( ) , columns = [ "V{}" . format ( j ) for j in range ( self . nodes ) ] )
return self . g , self . data


signature:
def create_graph_from_data ( self , data , ** kwargs ) :
    """Apply causal discovery on observational data using CAM .
 Args :
 data ( pandas . DataFrame ) : DataFrame containing the data
 Returns :
 networkx . DiGraph : Solution given by the CAM algorithm ."""

implementation:
# Building setup w / arguments .
self . arguments [ '{SCORE}' ] = self . scores [ self . score ]
self . arguments [ '{CUTOFF}' ] = str ( self . cutoff )
self . arguments [ '{VARSEL}' ] = str ( self . variablesel ) . upper ( )
self . arguments [ '{SELMETHOD}' ] = self . var_selection [ self . selmethod ]
self . arguments [ '{PRUNING}' ] = str ( self . pruning ) . upper ( )
self . arguments [ '{PRUNMETHOD}' ] = self . var_selection [ self . prunmethod ]
self . arguments [ '{NJOBS}' ] = str ( self . nb_jobs )
self . arguments [ '{VERBOSE}' ] = str ( self . verbose ) . upper ( )
results = self . _run_cam ( data , verbose = self . verbose )
return nx . relabel_nodes ( nx . DiGraph ( results ) , { idx : i for idx , i in enumerate ( data . columns ) } )


signature:
def predict_features ( self , df_features , df_target , idx = 0 , ** kwargs ) :
    """For one variable , predict its neighbouring nodes .
 Args :
 df _ features ( pandas . DataFrame ) :
 df _ target ( pandas . Series ) :
 idx ( int ) : ( optional ) for printing purposes
 kwargs ( dict ) : additional options for algorithms
 Returns :
 list : scores of each feature relatively to the target"""

implementation:
estimator = SVR ( kernel = 'linear' )
selector = RFECV ( estimator , step = 1 )
selector = selector . fit ( df_features . values , df_target . values [ : , 0 ] )
return selector . grid_scores_


signature:
def predict_features ( self , df_features , df_target , idx = 0 , C = .1 , ** kwargs ) :
    """For one variable , predict its neighbouring nodes .
 Args :
 df _ features ( pandas . DataFrame ) :
 df _ target ( pandas . Series ) :
 idx ( int ) : ( optional ) for printing purposes
 kwargs ( dict ) : additional options for algorithms
 C ( float ) : Penalty parameter of the error term
 Returns :
 list : scores of each feature relatively to the target"""

implementation:
lsvc = LinearSVR ( C = C ) . fit ( df_features . values , df_target . values )
return np . abs ( lsvc . coef_ )


implementation:
X = df_features . values
y = df_target . values
regressor = DecisionTreeRegressor ( )
regressor . fit ( X , y )
return regressor . feature_importances_


implementation:
X = df_features . values
y = df_target . values
clf = ard ( compute_score = True )
clf . fit ( X , y . ravel ( ) )
return np . abs ( clf . coef_ )


implementation:
X = df_features . values
y = df_target . values [ : , 0 ]
rr = ReliefF ( )
rr . fit ( X , y )
return rr . feature_importances_


signature:
def forward ( self , x ) :
    """Passing data through the network .
 : param x : 2d tensor containing both ( x , y ) Variables
 : return : output of the net"""

implementation:
features = self . conv ( x ) . mean ( dim = 2 )
return self . dense ( features )


signature:
def fit ( self , x_tr , y_tr , epochs = 50 , batchsize = 32 , learning_rate = 0.01 , verbose = None , device = None ) :
    """Fit the NCC model .
 Args :
 x _ tr ( pd . DataFrame ) : CEPC format dataframe containing the pairs
 y _ tr ( pd . DataFrame or np . ndarray ) : labels associated to the pairs
 epochs ( int ) : number of train epochs
 learning _ rate ( float ) : learning rate of Adam
 verbose ( bool ) : verbosity ( defaults to ` ` cdt . SETTINGS . verbose ` ` )
 device ( str ) : cuda or cpu device ( defaults to ` ` cdt . SETTINGS . default _ device ` ` )"""

implementation:
if batchsize > len ( x_tr ) :
    batchsize = len ( x_tr )
verbose , device = SETTINGS . get_default ( ( 'verbose' , verbose ) , ( 'device' , device ) )
self . model = NCC_model ( )
opt = th . optim . Adam ( self . model . parameters ( ) , lr = learning_rate )
criterion = th . nn . BCEWithLogitsLoss ( )
y = y_tr . values if isinstance ( y_tr , pd . DataFrame ) else y_tr
y = th . Tensor ( y ) / 2 + .5
# print ( y )
self . model = self . model . to ( device )
y = y . to ( device )
dataset = [ ]
for i , ( idx , row ) in enumerate ( x_tr . iterrows ( ) ) :
    a = row [ 'A' ] . reshape ( ( len ( row [ 'A' ] ) , 1 ) )
    b = row [ 'B' ] . reshape ( ( len ( row [ 'B' ] ) , 1 ) )
    m = np . hstack ( ( a , b ) )
    m = m . astype ( 'float32' )
    m = th . from_numpy ( m ) . t ( ) . unsqueeze ( 0 )
    dataset . append ( m )
dataset = [ m . to ( device ) for m in dataset ]
acc = [ 0 ]
da = th . utils . data . DataLoader ( Dataset ( dataset , y ) , batch_size = batchsize , shuffle = True )
data_per_epoch = ( len ( dataset ) // batchsize )
with trange ( epochs , desc = "Epochs" , disable = not verbose ) as te :
    for epoch in te :
        with trange ( data_per_epoch , desc = "Batches of {}" . format ( batchsize ) , disable = not ( verbose and batchsize == len ( dataset ) ) ) as t :
            output = [ ]
            labels = [ ]
            for ( batch , label ) , i in zip ( da , t ) :
                opt . zero_grad ( )
                # print ( batch . shape , labels . shape )
                out = th . stack ( [ self . model ( m ) for m in batch ] , 0 ) . squeeze ( 2 )
                loss = criterion ( out , label )
                loss . backward ( )
                t . set_postfix ( loss = loss . item ( ) )
                opt . step ( )
                output . append ( out )
                labels . append ( label )
            acc = th . where ( th . cat ( output , 0 ) > .5 , th . ones ( len ( output ) ) , th . zeros ( len ( output ) ) ) - th . cat ( labels , 0 )
            te . set_postfix ( Acc = 1 - acc . abs ( ) . mean ( ) . item ( ) )


signature:
def predict_proba ( self , a , b , device = None ) :
    """Infer causal directions using the trained NCC pairwise model .
 Args :
 a ( numpy . ndarray ) : Variable 1
 b ( numpy . ndarray ) : Variable 2
 device ( str ) : Device to run the algorithm on ( defaults to ` ` cdt . SETTINGS . default _ device ` ` )
 Returns :
 float : Causation score ( Value : 1 if a - > b and - 1 if b - > a )"""

implementation:
device = SETTINGS . get_default ( device = device )
if self . model is None :
    print ( 'Model has to be trained before doing any predictions' )
    raise ValueError
if len ( np . array ( a ) . shape ) == 1 :
    a = np . array ( a ) . reshape ( ( - 1 , 1 ) )
    b = np . array ( b ) . reshape ( ( - 1 , 1 ) )
m = np . hstack ( ( a , b ) )
m = scale ( m )
m = m . astype ( 'float32' )
m = th . from_numpy ( m ) . t ( ) . unsqueeze ( 0 )
if th . cuda . is_available ( ) :
    m = m . cuda ( )
return ( self . model ( m ) . data . cpu ( ) . numpy ( ) - .5 ) * 2


signature:
def predict_dataset ( self , df , device = None , verbose = None ) :
    """Args :
 x _ tr ( pd . DataFrame ) : CEPC format dataframe containing the pairs
 y _ tr ( pd . DataFrame or np . ndarray ) : labels associated to the pairs
 epochs ( int ) : number of train epochs
 learning rate ( float ) : learning rate of Adam
 verbose ( bool ) : verbosity ( defaults to ` ` cdt . SETTINGS . verbose ` ` )
 device ( str ) : cuda or cpu device ( defaults to ` ` cdt . SETTINGS . default _ device ` ` )
 Returns :
 pandas . DataFrame : dataframe containing the predicted causation coefficients"""

implementation:
verbose , device = SETTINGS . get_default ( ( 'verbose' , verbose ) , ( 'device' , device ) )
dataset = [ ]
for i , ( idx , row ) in enumerate ( df . iterrows ( ) ) :
    a = row [ 'A' ] . reshape ( ( len ( row [ 'A' ] ) , 1 ) )
    b = row [ 'B' ] . reshape ( ( len ( row [ 'B' ] ) , 1 ) )
    m = np . hstack ( ( a , b ) )
    m = m . astype ( 'float32' )
    m = th . from_numpy ( m ) . t ( ) . unsqueeze ( 0 )
    dataset . append ( m )
dataset = [ m . to ( device ) for m in dataset ]
return pd . DataFrame ( ( th . cat ( [ self . model ( m ) for m , t in zip ( dataset , trange ( len ( dataset ) ) , disable = not verbose ) ] , 0 ) . data . cpu ( ) . numpy ( ) - .5 ) * 2 )


signature:
def phrase_to_filename ( self , phrase ) :
    """Convert phrase to normilized file name ."""

implementation:
# remove non - word characters
name = re . sub ( r"[^\w\s\.]" , '' , phrase . strip ( ) . lower ( ) )
# replace whitespace with underscores
name = re . sub ( r"\s+" , '_' , name )
return name + '.png'


signature:
def seed_url ( self ) :
    """A URL that can be used to open the page .
 The URL is formatted from : py : attr : ` URL _ TEMPLATE ` , which is then
 appended to : py : attr : ` base _ url ` unless the template results in an
 absolute URL .
 : return : URL that can be used to open the page .
 : rtype : str"""

implementation:
url = self . base_url
if self . URL_TEMPLATE is not None :
    url = urlparse . urljoin ( self . base_url , self . URL_TEMPLATE . format ( ** self . url_kwargs ) )
if not url :
    return None
url_parts = list ( urlparse . urlparse ( url ) )
query = urlparse . parse_qsl ( url_parts [ 4 ] )
for k , v in self . url_kwargs . items ( ) :
    if v is None :
        continue
    if "{{{}}}" . format ( k ) not in str ( self . URL_TEMPLATE ) :
        for i in iterable ( v ) :
            query . append ( ( k , i ) )
url_parts [ 4 ] = urlencode ( query )
return urlparse . urlunparse ( url_parts )


signature:
def open ( self ) :
    """Open the page .
 Navigates to : py : attr : ` seed _ url ` and calls : py : func : ` wait _ for _ page _ to _ load ` .
 : return : The current page object .
 : rtype : : py : class : ` Page `
 : raises : UsageError"""

implementation:
if self . seed_url :
    self . driver_adapter . open ( self . seed_url )
    self . wait_for_page_to_load ( )
    return self
raise UsageError ( "Set a base URL or URL_TEMPLATE to open this page." )


signature:
def wait_for_page_to_load ( self ) :
    """Wait for the page to load ."""

implementation:
self . wait . until ( lambda _ : self . loaded )
self . pm . hook . pypom_after_wait_for_page_to_load ( page = self )
return self


signature:
def register ( ) :
    """Register the Selenium specific driver implementation .
 This register call is performed by the init module if
 selenium is available ."""

implementation: registerDriver ( ISelenium , Selenium , class_implements = [ Firefox , Chrome , Ie , Edge , Opera , Safari , BlackBerry , PhantomJS , Android , Remote , EventFiringWebDriver , ] , )


signature:
def root ( self ) :
    """Root element for the page region .
 Page regions should define a root element either by passing this on
 instantiation or by defining a : py : attr : ` _ root _ locator ` attribute . To
 reduce the chances of hitting : py : class : ` ~ selenium . common . exceptions . StaleElementReferenceException `
 or similar you should use : py : attr : ` _ root _ locator ` , as this is looked up every
 time the : py : attr : ` root ` property is accessed ."""

implementation:
if self . _root is None and self . _root_locator is not None :
    return self . page . find_element ( * self . _root_locator )
return self . _root


signature:
def wait_for_region_to_load ( self ) :
    """Wait for the page region to load ."""

implementation:
self . wait . until ( lambda _ : self . loaded )
self . pm . hook . pypom_after_wait_for_region_to_load ( region = self )
return self


signature:
def find_element ( self , strategy , locator ) :
    """Finds an element on the page .
 : param strategy : Location strategy to use . See : py : class : ` ~ selenium . webdriver . common . by . By ` or : py : attr : ` ~ pypom . splinter _ driver . ALLOWED _ STRATEGIES ` .
 : param locator : Location of target element .
 : type strategy : str
 : type locator : str
 : return : An element .
 : rytpe : : py : class : ` ~ selenium . webdriver . remote . webelement . WebElement ` or : py : class : ` ~ splinter . driver . webdriver . WebDriverElement `"""

implementation: return self . driver_adapter . find_element ( strategy , locator , root = self . root )


signature:
def find_elements ( self , strategy , locator ) :
    """Finds elements on the page .
 : param strategy : Location strategy to use . See : py : class : ` ~ selenium . webdriver . common . by . By ` or : py : attr : ` ~ pypom . splinter _ driver . ALLOWED _ STRATEGIES ` .
 : param locator : Location of target elements .
 : type strategy : str
 : type locator : str
 : return : List of : py : class : ` ~ selenium . webdriver . remote . webelement . WebElement ` or : py : class : ` ~ splinter . element _ list . ElementList `
 : rtype : list"""

implementation: return self . driver_adapter . find_elements ( strategy , locator , root = self . root )


signature:
def is_element_present ( self , strategy , locator ) :
    """Checks whether an element is present .
 : param strategy : Location strategy to use . See : py : class : ` ~ selenium . webdriver . common . by . By ` or : py : attr : ` ~ pypom . splinter _ driver . ALLOWED _ STRATEGIES ` .
 : param locator : Location of target element .
 : type strategy : str
 : type locator : str
 : return : ` ` True ` ` if element is present , else ` ` False ` ` .
 : rtype : bool"""

implementation: return self . driver_adapter . is_element_present ( strategy , locator , root = self . root )


signature:
def is_element_displayed ( self , strategy , locator ) :
    """Checks whether an element is displayed .
 : param strategy : Location strategy to use . See : py : class : ` ~ selenium . webdriver . common . by . By ` or : py : attr : ` ~ pypom . splinter _ driver . ALLOWED _ STRATEGIES ` .
 : param locator : Location of target element .
 : type strategy : str
 : type locator : str
 : return : ` ` True ` ` if element is displayed , else ` ` False ` ` .
 : rtype : bool"""

implementation: return self . driver_adapter . is_element_displayed ( strategy , locator , root = self . root )


signature:
def registerDriver ( iface , driver , class_implements = [ ] ) :
    """Register driver adapter used by page object"""

implementation:
for class_item in class_implements :
    classImplements ( class_item , iface )
component . provideAdapter ( factory = driver , adapts = [ iface ] , provides = IDriver )


signature:
def isHcl ( s ) :
    '''Detects whether a string is JSON or HCL
 : param s : String that may contain HCL or JSON
 : returns : True if HCL , False if JSON , raises ValueError
 if neither'''

implementation:
for c in s :
    if c . isspace ( ) :
        continue
    if c == '{' :
        return False
    else :
        return True
raise ValueError ( "No HCL object could be decoded" )


signature:
def loads ( s ) :
    '''Deserializes a string and converts it to a dictionary . The contents
 of the string must either be JSON or HCL .
 : returns : Dictionary'''

implementation:
s = u ( s )
if isHcl ( s ) :
    return HclParser ( ) . parse ( s )
else :
    return json . loads ( s )


signature:
def t_hexnumber ( self , t ) :
    r'- ? 0 [ xX ] [ 0-9a - fA - F ] +'

implementation:
t . value = int ( t . value , base = 16 )
t . type = 'NUMBER'
return t


signature:
def t_tabbedheredoc ( self , t ) :
    r'< < - \ S + \ r ? \ n'

implementation:
t . lexer . is_tabbed = True
self . _init_heredoc ( t )
t . lexer . begin ( 'tabbedheredoc' )


signature:
def t_heredoc ( self , t ) :
    r'< < \ S + \ r ? \ n'

implementation:
t . lexer . is_tabbed = False
self . _init_heredoc ( t )
t . lexer . begin ( 'heredoc' )


signature:
def objectlist_flat ( self , lt , replace ) :
    '''Similar to the dict constructor , but handles dups
 HCL is unclear on what one should do when duplicate keys are
 encountered . These comments aren ' t clear either :
 from decoder . go : if we ' re at the root or we ' re directly within
 a list , decode into dicts , otherwise lists
 from object . go : there ' s a flattened list structure'''

implementation:
d = { }
for k , v in lt :
    if k in d . keys ( ) and not replace :
        if type ( d [ k ] ) is list :
            d [ k ] . append ( v )
        else :
            d [ k ] = [ d [ k ] , v ]
    else :
        if isinstance ( v , dict ) :
            dd = d . setdefault ( k , { } )
            for kk , vv in iteritems ( v ) :
                if type ( dd ) == list :
                    dd . append ( { kk : vv } )
                elif kk in dd . keys ( ) :
                    if hasattr ( vv , 'items' ) :
                        for k2 , v2 in iteritems ( vv ) :
                            dd [ kk ] [ k2 ] = v2
                    else :
                        d [ k ] = [ dd , { kk : vv } ]
                else :
                    dd [ kk ] = vv
        else :
            d [ k ] = v
return d


signature:
def p_top ( self , p ) :
    "top : objectlist"

implementation:
if DEBUG :
    self . print_p ( p )
p [ 0 ] = self . objectlist_flat ( p [ 1 ] , True )


signature:
def p_objectlist_1 ( self , p ) :
    "objectlist : objectlist objectitem"

implementation:
if DEBUG :
    self . print_p ( p )
p [ 0 ] = p [ 1 ] + [ p [ 2 ] ]


signature:
def p_objectlist_2 ( self , p ) :
    "objectlist : objectlist COMMA objectitem"

implementation:
if DEBUG :
    self . print_p ( p )
p [ 0 ] = p [ 1 ] + [ p [ 3 ] ]


signature:
def p_object_0 ( self , p ) :
    "object : LEFTBRACE objectlist RIGHTBRACE"

implementation:
if DEBUG :
    self . print_p ( p )
p [ 0 ] = self . objectlist_flat ( p [ 2 ] , False )


signature:
def p_object_1 ( self , p ) :
    "object : LEFTBRACE objectlist COMMA RIGHTBRACE"


signature:
def p_objectitem_0 ( self , p ) :
    '''objectitem : objectkey EQUAL number
 | objectkey EQUAL BOOL
 | objectkey EQUAL STRING
 | objectkey EQUAL object
 | objectkey EQUAL list'''

implementation:
if DEBUG :
    self . print_p ( p )
p [ 0 ] = ( p [ 1 ] , p [ 3 ] )


signature:
def p_block_0 ( self , p ) :
    "block : blockId object"

implementation:
if DEBUG :
    self . print_p ( p )
p [ 0 ] = ( p [ 1 ] , p [ 2 ] )


signature:
def p_block_1 ( self , p ) :
    "block : blockId block"


signature:
def p_listitems_1 ( self , p ) :
    "listitems : listitems COMMA listitem"


signature:
def p_number_1 ( self , p ) :
    "number : float"

implementation:
if DEBUG :
    self . print_p ( p )
p [ 0 ] = float ( p [ 1 ] )


signature:
def p_number_2 ( self , p ) :
    "number : int exp"

implementation:
if DEBUG :
    self . print_p ( p )
p [ 0 ] = float ( "{0}{1}" . format ( p [ 1 ] , p [ 2 ] ) )


signature:
def p_number_3 ( self , p ) :
    "number : float exp"


signature:
def p_exp_0 ( self , p ) :
    "exp : EPLUS NUMBER"

implementation:
if DEBUG :
    self . print_p ( p )
p [ 0 ] = "e{0}" . format ( p [ 2 ] )


signature:
def p_exp_1 ( self , p ) :
    "exp : EMINUS NUMBER"

implementation:
if DEBUG :
    self . print_p ( p )
p [ 0 ] = "e-{0}" . format ( p [ 2 ] )


signature:
def _pre_install ( ) :
    '''Initialize the parse table at install time'''

implementation:
# Generate the parsetab . dat file at setup time
dat = join ( setup_dir , 'src' , 'hcl' , 'parsetab.dat' )
if exists ( dat ) :
    os . unlink ( dat )
sys . path . insert ( 0 , join ( setup_dir , 'src' ) )
import hcl
from hcl . parser import HclParser
parser = HclParser ( )


signature:
def append ( self , linenumber , raw_text , cells ) :
    """Add another row of data from a test suite"""

implementation: self . rows . append ( Row ( linenumber , raw_text , cells ) )


signature:
def steps ( self ) :
    """Return a list of steps ( statements that are not settings or comments )"""

implementation:
steps = [ ]
for statement in self . statements :
    if ( ( not statement . is_comment ( ) ) and ( not statement . is_setting ( ) ) ) :
        steps . append ( statement )
return steps


signature:
def is_comment ( self ) :
    '''Return True if the first non - empty cell starts with " # "'''

implementation:
for cell in self [ : ] :
    if cell == "" :
        continue
    # this is the first non - empty cell . Check whether it is
    # a comment or not .
    if cell . lstrip ( ) . startswith ( "#" ) :
        return True
    else :
        return False
return False


signature:
def RobotFactory ( path , parent = None ) :
    '''Return an instance of SuiteFile , ResourceFile , SuiteFolder
 Exactly which is returned depends on whether it ' s a file or
 folder , and if a file , the contents of the file . If there is a
 testcase table , this will return an instance of SuiteFile ,
 otherwise it will return an instance of ResourceFile .'''

implementation:
if os . path . isdir ( path ) :
    return SuiteFolder ( path , parent )
else :
    rf = RobotFile ( path , parent )
    for table in rf . tables :
        if isinstance ( table , TestcaseTable ) :
            rf . __class__ = SuiteFile
            return rf
    rf . __class__ = ResourceFile
    return rf


signature:
def walk ( self , * types ) :
    '''Iterator which visits all suites and suite files ,
 yielding test cases and keywords'''

implementation:
requested = types if len ( types ) > 0 else [ SuiteFile , ResourceFile , SuiteFolder , Testcase , Keyword ]
for thing in self . robot_files :
    if thing . __class__ in requested :
        yield thing
    if isinstance ( thing , SuiteFolder ) :
        for child in thing . walk ( ) :
            if child . __class__ in requested :
                yield child
    else :
        for child in thing . walk ( * types ) :
            yield child


signature:
def robot_files ( self ) :
    '''Return a list of all folders , and test suite files ( . txt , . robot )'''

implementation:
result = [ ]
for name in os . listdir ( self . path ) :
    fullpath = os . path . join ( self . path , name )
    if os . path . isdir ( fullpath ) :
        result . append ( RobotFactory ( fullpath , parent = self ) )
    else :
        if ( ( name . endswith ( ".txt" ) or name . endswith ( ".robot" ) ) and ( name not in ( "__init__.txt" , "__init__.robot" ) ) ) :
            result . append ( RobotFactory ( fullpath , parent = self ) )
return result


signature:
def walk ( self , * types ) :
    '''Iterator which can return all test cases and / or keywords
 You can specify with objects to return as parameters ; if
 no parameters are given , both tests and keywords will
 be returned .
 For example , to get only test cases , you could call it
 like this :
 robot _ file = RobotFactory ( . . . )
 for testcase in robot _ file . walk ( Testcase ) : . . .'''

implementation:
requested = types if len ( types ) > 0 else [ Testcase , Keyword ]
if Testcase in requested :
    for testcase in self . testcases :
        yield testcase
if Keyword in requested :
    for keyword in self . keywords :
        yield keyword


signature:
def _load ( self , path ) :
    '''The general idea is to do a quick parse , creating a list of
 tables . Each table is nothing more than a list of rows , with
 each row being a list of cells . Additional parsing such as
 combining rows into statements is done on demand . This first
 pass is solely to read in the plain text and organize it by table .'''

implementation:
self . tables = [ ]
current_table = DefaultTable ( self )
with Utf8Reader ( path ) as f : # N . B . the caller should be catching errors
    self . raw_text = f . read ( )
    f . _file . seek ( 0 )
    # bleh ; wish this wasn ' t a private property
    matcher = Matcher ( re . IGNORECASE )
    for linenumber , raw_text in enumerate ( f . readlines ( ) ) :
        linenumber += 1 ;
        # start counting at 1 rather than zero
        # this mimics what the robot TSV reader does - -
        # it replaces non - breaking spaces with regular spaces ,
        # and then strips trailing whitespace
        raw_text = raw_text . replace ( u'\xA0' , ' ' )
        raw_text = raw_text . rstrip ( )
        # FIXME : I ' m keeping line numbers but throwing away
        # where each cell starts . I should be preserving that
        # ( though to be fair , robot is throwing that away so
        # I ' ll have to write my own splitter if I want to save
        # the character position )
        cells = TxtReader . split_row ( raw_text )
        _heading_regex = r'^\s*\*+\s*(.*?)[ *]*$'
        if matcher ( _heading_regex , cells [ 0 ] ) : # we ' ve found the start of a new table
            table_name = matcher . group ( 1 )
            current_table = tableFactory ( self , linenumber , table_name , raw_text )
            self . tables . append ( current_table )
        else :
            current_table . append ( Row ( linenumber , raw_text , cells ) )


signature:
def type ( self ) :
    '''Return ' suite ' or ' resource ' or None
 This will return ' suite ' if a testcase table is found ;
 It will return ' resource ' if at least one robot table
 is found . If no tables are found it will return None'''

implementation:
robot_tables = [ table for table in self . tables if not isinstance ( table , UnknownTable ) ]
if len ( robot_tables ) == 0 :
    return None
for table in self . tables :
    if isinstance ( table , TestcaseTable ) :
        return "suite"
return "resource"


signature:
def keywords ( self ) :
    '''Generator which returns all keywords in the suite'''

implementation:
for table in self . tables :
    if isinstance ( table , KeywordTable ) :
        for keyword in table . keywords :
            yield keyword


signature:
def dump ( self ) :
    '''Regurgitate the tables and rows'''

implementation:
for table in self . tables :
    print ( "*** %s ***" % table . name )
    table . dump ( )


signature:
def settings ( self ) :
    '''Generator which returns all of the statements in all of the settings tables'''

implementation:
for table in self . tables :
    if isinstance ( table , SettingTable ) :
        for statement in table . statements :
            yield statement


signature:
def variables ( self ) :
    '''Generator which returns all of the statements in all of the variables tables'''

implementation:
for table in self . tables :
    if isinstance ( table , VariableTable ) : # FIXME : settings have statements , variables have rows WTF ? : - (
        for statement in table . rows :
            if statement [ 0 ] != "" :
                yield statement


signature:
def statements ( self ) :
    '''Return a list of statements
 This is done by joining together any rows that
 have continuations'''

implementation:
# FIXME : no need to do this every time ; we should cache the
# result
if len ( self . rows ) == 0 :
    return [ ]
current_statement = Statement ( self . rows [ 0 ] )
current_statement . startline = self . rows [ 0 ] . linenumber
current_statement . endline = self . rows [ 0 ] . linenumber
statements = [ ]
for row in self . rows [ 1 : ] :
    if len ( row ) > 0 and row [ 0 ] == "..." : # we found a continuation
        current_statement += row [ 1 : ]
        current_statement . endline = row . linenumber
    else :
        if len ( current_statement ) > 0 : # append current statement to the list of statements . . .
            statements . append ( current_statement )
        # start a new statement
        current_statement = Statement ( row )
        current_statement . startline = row . linenumber
        current_statement . endline = row . linenumber
if len ( current_statement ) > 0 :
    statements . append ( current_statement )
# trim trailing blank statements
while ( len ( statements [ - 1 ] ) == 0 or ( ( len ( statements [ - 1 ] ) == 1 ) and len ( statements [ - 1 ] [ 0 ] ) == 0 ) ) :
    statements . pop ( )
return statements


signature:
def append ( self , row ) :
    '''The idea is , we recognize when we have a new testcase by
 checking the first cell . If it ' s not empty and not a comment ,
 we have a new test case .'''

implementation:
if len ( row ) == 0 : # blank line . Should we throw it away , or append a BlankLine object ?
    return
if ( row [ 0 ] != "" and ( not row [ 0 ] . lstrip ( ) . startswith ( "#" ) ) ) : # we have a new child table
    self . _children . append ( self . _childClass ( self . parent , row . linenumber , row [ 0 ] ) )
    if len ( row . cells ) > 1 : # It appears the first row - - which contains the test case or
    # keyword name - - also has the first logical row of cells .
    # We ' ll create a Row , but we ' ll make the first cell empty instead
    # of leaving the name in it , since other code always assumes the
    # first cell is empty .
    # To be honest , I ' m not sure this is the Right Thing To Do , but
    # I ' m too lazy to audit the code to see if it matters if we keep
    # the first cell intact . Sorry if this ends up causing you grief
    # some day . . .
        row [ 0 ] = ""
        self . _children [ - 1 ] . append ( row . linenumber , row . raw_text , row . cells )
elif len ( self . _children ) == 0 : # something before the first test case
# For now , append it to self . comments ; eventually we should flag
# an error if it ' s NOT a comment
    self . comments . append ( row )
else : # another row for the testcase
    if len ( row . cells ) > 0 :
        self . _children [ - 1 ] . append ( row . linenumber , row . raw_text , row . cells )


signature:
def report ( self , obj , message , linenum , char_offset = 0 ) :
    """Report an error or warning"""

implementation: self . controller . report ( linenumber = linenum , filename = obj . path , severity = self . severity , message = message , rulename = self . __class__ . __name__ , char = char_offset )


signature:
def doc ( self ) :
    '''Algorithm from https : / / www . python . org / dev / peps / pep - 0257/'''

implementation:
if not self . __doc__ :
    return ""
lines = self . __doc__ . expandtabs ( ) . splitlines ( )
# Determine minimum indentation ( first line doesn ' t count ) :
indent = sys . maxsize
for line in lines [ 1 : ] :
    stripped = line . lstrip ( )
    if stripped :
        indent = min ( indent , len ( line ) - len ( stripped ) )
# Remove indentation ( first line is special ) :
trimmed = [ lines [ 0 ] . strip ( ) ]
if indent < sys . maxsize :
    for line in lines [ 1 : ] :
        trimmed . append ( line [ indent : ] . rstrip ( ) )
# Strip off trailing and leading blank lines :
while trimmed and not trimmed [ - 1 ] :
    trimmed . pop ( )
while trimmed and not trimmed [ 0 ] :
    trimmed . pop ( 0 )
# Return a single string :
return '\n' . join ( trimmed )


signature:
def run ( self , args ) :
    """Parse command line arguments , and run rflint"""

implementation:
self . args = self . parse_and_process_args ( args )
if self . args . version :
    print ( __version__ )
    return 0
if self . args . rulefile :
    for filename in self . args . rulefile :
        self . _load_rule_file ( filename )
if self . args . list :
    self . list_rules ( )
    return 0
if self . args . describe :
    self . _describe_rules ( self . args . args )
    return 0
self . counts = { ERROR : 0 , WARNING : 0 , "other" : 0 }
for filename in self . args . args :
    if not ( os . path . exists ( filename ) ) :
        sys . stderr . write ( "rflint: %s: No such file or directory\n" % filename )
        continue
    if os . path . isdir ( filename ) :
        self . _process_folder ( filename )
    else :
        self . _process_file ( filename )
if self . counts [ ERROR ] > 0 :
    return self . counts [ ERROR ] if self . counts [ ERROR ] < 254 else 255
return 0


signature:
def list_rules ( self ) :
    """Print a list of all rules"""

implementation:
for rule in sorted ( self . all_rules , key = lambda rule : rule . name ) :
    print ( rule )
    if self . args . verbose :
        for line in rule . doc . split ( "\n" ) :
            print ( "    " , line )


signature:
def report ( self , linenumber , filename , severity , message , rulename , char ) :
    """Report a rule violation"""

implementation:
if self . _print_filename is not None : # we print the filename only once . self . _ print _ filename
# will get reset each time a new file is processed .
    print ( "+ " + self . _print_filename )
    self . _print_filename = None
if severity in ( WARNING , ERROR ) :
    self . counts [ severity ] += 1
else :
    self . counts [ "other" ] += 1
print ( self . args . format . format ( linenumber = linenumber , filename = filename , severity = severity , message = message . encode ( 'utf-8' ) , rulename = rulename , char = char ) )


signature:
def _get_rules ( self , cls ) :
    """Returns a list of rules of a given class
 Rules are treated as singletons - we only instantiate each
 rule once ."""

implementation:
result = [ ]
for rule_class in cls . __subclasses__ ( ) :
    rule_name = rule_class . __name__ . lower ( )
    if rule_name not in self . _rules :
        rule = rule_class ( self )
        self . _rules [ rule_name ] = rule
    result . append ( self . _rules [ rule_name ] )
return result


signature:
def _load_rule_file ( self , filename ) :
    """Import the given rule file"""

implementation:
if not ( os . path . exists ( filename ) ) :
    sys . stderr . write ( "rflint: %s: No such file or directory\n" % filename )
    return
try :
    basename = os . path . basename ( filename )
    ( name , ext ) = os . path . splitext ( basename )
    imp . load_source ( name , filename )
except Exception as e :
    sys . stderr . write ( "rflint: %s: exception while loading: %s\n" % ( filename , str ( e ) ) )


signature:
def parse_and_process_args ( self , args ) :
    """Handle the parsing of command line arguments ."""

implementation:
parser = argparse . ArgumentParser ( prog = "python -m rflint" , description = "A style checker for robot framework plain text files." , formatter_class = argparse . RawDescriptionHelpFormatter , epilog = ( "You can use 'all' in place of RULENAME to refer to all rules. \n" "\n" "For example: '--ignore all --warn DuplicateTestNames' will ignore all\n" "rules except DuplicateTestNames.\n" "\n" "FORMAT is a string that performs a substitution on the following \n" "patterns: {severity}, {linenumber}, {char}, {message}, and {rulename}.\n" "\n" "For example: --format 'line: {linenumber}: message: {message}'. \n" "\n" "ARGUMENTFILE is a filename with contents that match the format of \n" "standard robot framework argument files\n" "\n" "If you give a directory as an argument, all files in the directory\n" "with the suffix .txt, .robot or .tsv will be processed. With the \n" "--recursive option, subfolders within the directory will also be\n" "processed." ) )
parser . add_argument ( "--error" , "-e" , metavar = "RULENAME" , action = SetErrorAction , help = "Assign a severity of ERROR to the given RULENAME" )
parser . add_argument ( "--ignore" , "-i" , metavar = "RULENAME" , action = SetIgnoreAction , help = "Ignore the given RULENAME" )
parser . add_argument ( "--warning" , "-w" , metavar = "RULENAME" , action = SetWarningAction , help = "Assign a severity of WARNING for the given RULENAME" )
parser . add_argument ( "--list" , "-l" , action = "store_true" , help = "show a list of known rules and exit" )
parser . add_argument ( "--describe" , "-d" , action = "store_true" , help = "describe the given rules" )
parser . add_argument ( "--no-filenames" , action = "store_false" , dest = "print_filenames" , default = True , help = "suppress the printing of filenames" )
parser . add_argument ( "--format" , "-f" , help = "Define the output format" , default = '{severity}: {linenumber}, {char}: {message} ({rulename})' )
parser . add_argument ( "--version" , action = "store_true" , default = False , help = "Display version number and exit" )
parser . add_argument ( "--verbose" , "-v" , action = "store_true" , default = False , help = "Give verbose output" )
parser . add_argument ( "--configure" , "-c" , action = ConfigureAction , help = "Configure a rule" )
parser . add_argument ( "--recursive" , "-r" , action = "store_true" , default = False , help = "Recursively scan subfolders in a directory" )
parser . add_argument ( "--rulefile" , "-R" , action = RulefileAction , help = "import additional rules from the given RULEFILE" )
parser . add_argument ( "--argumentfile" , "-A" , action = ArgfileLoader , help = "read arguments from the given file" )
parser . add_argument ( 'args' , metavar = "file" , nargs = argparse . REMAINDER )
# create a custom namespace , in which we can store a reference to
# our rules . This lets the custom argument actions access the list
# of rules
ns = argparse . Namespace ( )
setattr ( ns , "app" , self )
args = parser . parse_args ( args , ns )
Rule . output_format = args . format
return args


signature:
def from_resolver ( cls , spec_resolver ) :
    """Creates a customized Draft4ExtendedValidator .
 : param spec _ resolver : resolver for the spec
 : type resolver : : class : ` jsonschema . RefResolver `"""

implementation:
spec_validators = cls . _get_spec_validators ( spec_resolver )
return validators . extend ( Draft4Validator , spec_validators )


signature:
def create ( self , spec_resolver ) :
    """Creates json documents validator from spec resolver .
 : param spec _ resolver : reference resolver .
 : return : RefResolver for spec with cached remote $ refs used during
 validation .
 : rtype : : class : ` jsonschema . RefResolver `"""

implementation:
validator_cls = self . spec_validator_factory . from_resolver ( spec_resolver )
return validator_cls ( self . schema , resolver = self . schema_resolver )


signature:
def construct_mapping ( self , node , deep = False ) :
    """While yaml supports integer keys , these are not valid in
 json , and will break jsonschema . This method coerces all keys
 to strings ."""

implementation:
mapping = super ( ExtendedSafeConstructor , self ) . construct_mapping ( node , deep )
return { ( str ( key ) if isinstance ( key , int ) else key ) : mapping [ key ] for key in mapping }


signature:
def read_yaml_file ( path , loader = ExtendedSafeLoader ) :
    """Open a file , read it and return its contents ."""

implementation:
with open ( path ) as fh :
    return load ( fh , loader )


signature:
def from_spec_resolver ( cls , spec_resolver ) :
    """Creates validators generator for the spec resolver .
 : param spec _ resolver : resolver for the spec
 : type instance _ resolver : : class : ` jsonschema . RefResolver `"""

implementation:
deref = DerefValidatorDecorator ( spec_resolver )
for key , validator_callable in iteritems ( cls . validators ) :
    yield key , deref ( validator_callable )


signature:
def grade ( grader_data , submission ) :
    """Grades a specified submission using specified models
 grader _ data - A dictionary :
 ' model ' : trained model ,
 ' extractor ' : trained feature extractor ,
 ' prompt ' : prompt for the question ,
 ' algorithm ' : algorithm for the question ,
 submission - The student submission ( string )"""

implementation:
# Initialize result dictionary
results = { 'errors' : [ ] , 'tests' : [ ] , 'score' : 0 , 'feedback' : "" , 'success' : False , 'confidence' : 0 }
has_error = False
grader_set = EssaySet ( essaytype = "test" )
feedback = { }
model , extractor = get_classifier_and_ext ( grader_data )
# This is to preserve legacy functionality
if 'algorithm' not in grader_data :
    grader_data [ 'algorithm' ] = util_functions . AlgorithmTypes . classification
try : # Try to add essay to essay set object
    grader_set . add_essay ( str ( submission ) , 0 )
    grader_set . update_prompt ( str ( grader_data [ 'prompt' ] ) )
except Exception :
    error_message = "Essay could not be added to essay set:{0}" . format ( submission )
    log . exception ( error_message )
    results [ 'errors' ] . append ( error_message )
    has_error = True
# Try to extract features from submission and assign score via the model
try :
    grader_feats = extractor . gen_feats ( grader_set )
    feedback = extractor . gen_feedback ( grader_set , grader_feats ) [ 0 ]
    results [ 'score' ] = int ( model . predict ( grader_feats ) [ 0 ] )
except Exception :
    error_message = "Could not extract features and score essay."
    log . exception ( error_message )
    results [ 'errors' ] . append ( error_message )
    has_error = True
# Try to determine confidence level
try :
    results [ 'confidence' ] = get_confidence_value ( grader_data [ 'algorithm' ] , model , grader_feats , results [ 'score' ] , grader_data [ 'score' ] )
except Exception : # If there is an error getting confidence , it is not a show - stopper , so just log
    log . exception ( "Problem generating confidence value" )
if not has_error : # If the essay is just a copy of the prompt , return a 0 as the score
    if ( 'too_similar_to_prompt' in feedback and feedback [ 'too_similar_to_prompt' ] ) :
        results [ 'score' ] = 0
        results [ 'correct' ] = False
    results [ 'success' ] = True
    # Generate short form output - - number of problem areas identified in feedback
    # Add feedback to results if available
    results [ 'feedback' ] = { }
    if 'topicality' in feedback and 'prompt_overlap' in feedback :
        results [ 'feedback' ] . update ( { 'topicality' : feedback [ 'topicality' ] , 'prompt-overlap' : feedback [ 'prompt_overlap' ] , } )
    results [ 'feedback' ] . update ( { 'spelling' : feedback [ 'spelling' ] , 'grammar' : feedback [ 'grammar' ] , 'markup-text' : feedback [ 'markup_text' ] , } )
else : # If error , success is False .
    results [ 'success' ] = False
return results


signature:
def grade_generic ( grader_data , numeric_features , textual_features ) :
    """Grades a set of numeric and textual features using a generic model
 grader _ data - - dictionary containing :
 ' algorithm ' - Type of algorithm to use to score
 numeric _ features - list of numeric features to predict on
 textual _ features - list of textual feature to predict on"""

implementation:
results = { 'errors' : [ ] , 'tests' : [ ] , 'score' : 0 , 'success' : False , 'confidence' : 0 }
has_error = False
# Try to find and load the model file
grader_set = predictor_set . PredictorSet ( essaytype = "test" )
model , extractor = get_classifier_and_ext ( grader_data )
# Try to add essays to essay set object
try :
    grader_set . add_row ( numeric_features , textual_features , 0 )
except Exception :
    error_msg = "Row could not be added to predictor set:{0} {1}" . format ( numeric_features , textual_features )
    log . exception ( error_msg )
    results [ 'errors' ] . append ( error_msg )
    has_error = True
# Try to extract features from submission and assign score via the model
try :
    grader_feats = extractor . gen_feats ( grader_set )
    results [ 'score' ] = model . predict ( grader_feats ) [ 0 ]
except Exception :
    error_msg = "Could not extract features and score essay."
    log . exception ( error_msg )
    results [ 'errors' ] . append ( error_msg )
    has_error = True
# Try to determine confidence level
try :
    results [ 'confidence' ] = get_confidence_value ( grader_data [ 'algorithm' ] , model , grader_feats , results [ 'score' ] )
except Exception : # If there is an error getting confidence , it is not a show - stopper , so just log
    log . exception ( "Problem generating confidence value" )
if not has_error :
    results [ 'success' ] = True
return results


signature:
def get_confidence_value ( algorithm , model , grader_feats , score , scores ) :
    """Determines a confidence in a certain score , given proper input parameters
 algorithm - from util _ functions . AlgorithmTypes
 model - a trained model
 grader _ feats - a row of features used by the model for classification / regression
 score - The score assigned to the submission by a prior model"""

implementation:
min_score = min ( numpy . asarray ( scores ) )
max_score = max ( numpy . asarray ( scores ) )
if algorithm == util_functions . AlgorithmTypes . classification and hasattr ( model , "predict_proba" ) : # If classification , predict with probability , which gives you a matrix of confidences per score point
    raw_confidence = model . predict_proba ( grader_feats ) [ 0 , ( float ( score ) - float ( min_score ) ) ]
    # TODO : Normalize confidence somehow here
    confidence = raw_confidence
elif hasattr ( model , "predict" ) :
    raw_confidence = model . predict ( grader_feats ) [ 0 ]
    confidence = max ( float ( raw_confidence ) - math . floor ( float ( raw_confidence ) ) , math . ceil ( float ( raw_confidence ) ) - float ( raw_confidence ) )
else :
    confidence = 0
return confidence


signature:
def create_model_path ( model_path ) :
    """Creates a path to model files
 model _ path - string"""

implementation:
if not model_path . startswith ( "/" ) and not model_path . startswith ( "models/" ) :
    model_path = "/" + model_path
if not model_path . startswith ( "models" ) :
    model_path = "models" + model_path
if not model_path . endswith ( ".p" ) :
    model_path += ".p"
return model_path


signature:
def sub_chars ( string ) :
    """Strips illegal characters from a string . Used to sanitize input essays .
 Removes all non - punctuation , digit , or letter characters .
 Returns sanitized string .
 string - string"""

implementation:
# Define replacement patterns
sub_pat = r"[^A-Za-z\.\?!,';:]"
char_pat = r"\."
com_pat = r","
ques_pat = r"\?"
excl_pat = r"!"
sem_pat = r";"
col_pat = r":"
whitespace_pat = r"\s{1,}"
# Replace text . Ordering is very important !
nstring = re . sub ( sub_pat , " " , string )
nstring = re . sub ( char_pat , " ." , nstring )
nstring = re . sub ( com_pat , " ," , nstring )
nstring = re . sub ( ques_pat , " ?" , nstring )
nstring = re . sub ( excl_pat , " !" , nstring )
nstring = re . sub ( sem_pat , " ;" , nstring )
nstring = re . sub ( col_pat , " :" , nstring )
nstring = re . sub ( whitespace_pat , " " , nstring )
return nstring


signature:
def spell_correct ( string ) :
    """Uses aspell to spell correct an input string .
 Requires aspell to be installed and added to the path .
 Returns the spell corrected string if aspell is found , original string if not .
 string - string"""

implementation:
# Create a temp file so that aspell could be used
# By default , tempfile will delete this file when the file handle is closed .
f = tempfile . NamedTemporaryFile ( mode = 'w' )
f . write ( string )
f . flush ( )
f_path = os . path . abspath ( f . name )
try :
    p = os . popen ( aspell_path + " -a < " + f_path + " --sug-mode=ultra" )
    # Aspell returns a list of incorrect words with the above flags
    incorrect = p . readlines ( )
    p . close ( )
except Exception :
    log . exception ( "aspell process failed; could not spell check" )
    # Return original string if aspell fails
    return string , 0 , string
finally :
    f . close ( )
incorrect_words = list ( )
correct_spelling = list ( )
for i in range ( 1 , len ( incorrect ) ) :
    if ( len ( incorrect [ i ] ) > 10 ) : # Reformat aspell output to make sense
        match = re . search ( ":" , incorrect [ i ] )
        if hasattr ( match , "start" ) :
            begstring = incorrect [ i ] [ 2 : match . start ( ) ]
            begmatch = re . search ( " " , begstring )
            begword = begstring [ 0 : begmatch . start ( ) ]
            sugstring = incorrect [ i ] [ match . start ( ) + 2 : ]
            sugmatch = re . search ( "," , sugstring )
            if hasattr ( sugmatch , "start" ) :
                sug = sugstring [ 0 : sugmatch . start ( ) ]
                incorrect_words . append ( begword )
                correct_spelling . append ( sug )
# Create markup based on spelling errors
newstring = string
markup_string = string
already_subbed = [ ]
for i in range ( 0 , len ( incorrect_words ) ) :
    sub_pat = r"\b" + incorrect_words [ i ] + r"\b"
    sub_comp = re . compile ( sub_pat )
    newstring = re . sub ( sub_comp , correct_spelling [ i ] , newstring )
    if incorrect_words [ i ] not in already_subbed :
        markup_string = re . sub ( sub_comp , '<bs>' + incorrect_words [ i ] + "</bs>" , markup_string )
        already_subbed . append ( incorrect_words [ i ] )
return newstring , len ( incorrect_words ) , markup_string


signature:
def ngrams ( tokens , min_n , max_n ) :
    """Generates ngrams ( word sequences of fixed length ) from an input token sequence .
 tokens is a list of words .
 min _ n is the minimum length of an ngram to return .
 max _ n is the maximum length of an ngram to return .
 returns a list of ngrams ( words separated by a space )"""

implementation:
all_ngrams = list ( )
n_tokens = len ( tokens )
for i in xrange ( n_tokens ) :
    for j in xrange ( i + min_n , min ( n_tokens , i + max_n ) + 1 ) :
        all_ngrams . append ( " " . join ( tokens [ i : j ] ) )
return all_ngrams


signature:
def f7 ( seq ) :
    """Makes a list unique"""

implementation:
seen = set ( )
seen_add = seen . add
return [ x for x in seq if x not in seen and not seen_add ( x ) ]


signature:
def count_list ( the_list ) :
    """Generates a count of the number of times each unique item appears in a list"""

implementation:
count = the_list . count
result = [ ( item , count ( item ) ) for item in set ( the_list ) ]
result . sort ( )
return result


signature:
def regenerate_good_tokens ( string ) :
    """Given an input string , part of speech tags the string , then generates a list of
 ngrams that appear in the string .
 Used to define grammatically correct part of speech tag sequences .
 Returns a list of part of speech tag sequences ."""

implementation:
toks = nltk . word_tokenize ( string )
pos_string = nltk . pos_tag ( toks )
pos_seq = [ tag [ 1 ] for tag in pos_string ]
pos_ngrams = ngrams ( pos_seq , 2 , 4 )
sel_pos_ngrams = f7 ( pos_ngrams )
return sel_pos_ngrams


signature:
def get_vocab ( text , score , max_feats = 750 , max_feats2 = 200 ) :
    """Uses a fisher test to find words that are significant in that they separate
 high scoring essays from low scoring essays .
 text is a list of input essays .
 score is a list of scores , with score [ n ] corresponding to text [ n ]
 max _ feats is the maximum number of features to consider in the first pass
 max _ feats2 is the maximum number of features to consider in the second ( final ) pass
 Returns a list of words that constitute the significant vocabulary"""

implementation:
dict = CountVectorizer ( ngram_range = ( 1 , 2 ) , max_features = max_feats )
dict_mat = dict . fit_transform ( text )
set_score = numpy . asarray ( score , dtype = numpy . int )
med_score = numpy . median ( set_score )
new_score = set_score
if ( med_score == 0 ) :
    med_score = 1
new_score [ set_score < med_score ] = 0
new_score [ set_score >= med_score ] = 1
fish_vals = [ ]
for col_num in range ( 0 , dict_mat . shape [ 1 ] ) :
    loop_vec = dict_mat . getcol ( col_num ) . toarray ( )
    good_loop_vec = loop_vec [ new_score == 1 ]
    bad_loop_vec = loop_vec [ new_score == 0 ]
    good_loop_present = len ( good_loop_vec [ good_loop_vec > 0 ] )
    good_loop_missing = len ( good_loop_vec [ good_loop_vec == 0 ] )
    bad_loop_present = len ( bad_loop_vec [ bad_loop_vec > 0 ] )
    bad_loop_missing = len ( bad_loop_vec [ bad_loop_vec == 0 ] )
    fish_val = pvalue ( good_loop_present , bad_loop_present , good_loop_missing , bad_loop_missing ) . two_tail
    fish_vals . append ( fish_val )
cutoff = 1
if ( len ( fish_vals ) > max_feats2 ) :
    cutoff = sorted ( fish_vals ) [ max_feats2 ]
good_cols = numpy . asarray ( [ num for num in range ( 0 , dict_mat . shape [ 1 ] ) if fish_vals [ num ] <= cutoff ] )
getVar = lambda searchList , ind : [ searchList [ i ] for i in ind ]
vocab = getVar ( dict . get_feature_names ( ) , good_cols )
return vocab


signature:
def edit_distance ( s1 , s2 ) :
    """Calculates string edit distance between string 1 and string 2.
 Deletion , insertion , substitution , and transposition all increase edit distance ."""

implementation:
d = { }
lenstr1 = len ( s1 )
lenstr2 = len ( s2 )
for i in xrange ( - 1 , lenstr1 + 1 ) :
    d [ ( i , - 1 ) ] = i + 1
for j in xrange ( - 1 , lenstr2 + 1 ) :
    d [ ( - 1 , j ) ] = j + 1
for i in xrange ( lenstr1 ) :
    for j in xrange ( lenstr2 ) :
        if s1 [ i ] == s2 [ j ] :
            cost = 0
        else :
            cost = 1
        d [ ( i , j ) ] = min ( d [ ( i - 1 , j ) ] + 1 , # deletion
        d [ ( i , j - 1 ) ] + 1 , # insertion
        d [ ( i - 1 , j - 1 ) ] + cost , # substitution
        )
        if i and j and s1 [ i ] == s2 [ j - 1 ] and s1 [ i - 1 ] == s2 [ j ] :
            d [ ( i , j ) ] = min ( d [ ( i , j ) ] , d [ i - 2 , j - 2 ] + cost )
# transposition
return d [ lenstr1 - 1 , lenstr2 - 1 ]


signature:
def gen_cv_preds ( clf , arr , sel_score , num_chunks = 3 ) :
    """Generates cross validated predictions using an input classifier and data .
 clf is a classifier that implements that implements the fit and predict methods .
 arr is the input data array ( X )
 sel _ score is the target list ( y ) . y [ n ] corresponds to X [ n , : ]
 num _ chunks is the number of cross validation folds to use
 Returns an array of the predictions where prediction [ n ] corresponds to X [ n , : ]"""

implementation:
cv_len = int ( math . floor ( len ( sel_score ) / num_chunks ) )
chunks = [ ]
for i in range ( 0 , num_chunks ) :
    range_min = i * cv_len
    range_max = ( ( i + 1 ) * cv_len )
    if i == num_chunks - 1 :
        range_max = len ( sel_score )
    chunks . append ( range ( range_min , range_max ) )
preds = [ ]
set_score = numpy . asarray ( sel_score , dtype = numpy . int )
chunk_vec = numpy . asarray ( range ( 0 , len ( chunks ) ) )
for i in xrange ( 0 , len ( chunks ) ) :
    loop_inds = list ( chain . from_iterable ( [ chunks [ int ( z ) ] for z , m in enumerate ( range ( 0 , len ( chunks ) ) ) if int ( z ) != i ] ) )
    sim_fit = clf . fit ( arr [ loop_inds ] , set_score [ loop_inds ] )
    preds . append ( list ( sim_fit . predict ( arr [ chunks [ i ] ] ) ) )
all_preds = list ( chain ( * preds ) )
return ( all_preds )


signature:
def gen_model ( clf , arr , sel_score ) :
    """Fits a classifier to data and a target score
 clf is an input classifier that implements the fit method .
 arr is a data array ( X )
 sel _ score is the target list ( y ) where y [ n ] corresponds to X [ n , : ]
 sim _ fit is not a useful return value . Instead the clf is the useful output ."""

implementation:
set_score = numpy . asarray ( sel_score , dtype = numpy . int )
sim_fit = clf . fit ( arr , set_score )
return ( sim_fit )


signature:
def gen_preds ( clf , arr ) :
    """Generates predictions on a novel data array using a fit classifier
 clf is a classifier that has already been fit
 arr is a data array identical in dimension to the array clf was trained on
 Returns the array of predictions ."""

implementation:
if ( hasattr ( clf , "predict_proba" ) ) :
    ret = clf . predict ( arr )
    # pred _ score = preds . argmax ( 1 ) + min ( x . _ score )
else :
    ret = clf . predict ( arr )
return ret


signature:
def calc_list_average ( l ) :
    """Calculates the average value of a list of numbers
 Returns a float"""

implementation:
total = 0.0
for value in l :
    total += value
return total / len ( l )