add python files from official repo

example-scripts

@@ -1 +0,0 @@
-Subproject commit 838bfd1788feaf40362d6bedb3e4683832a9dbb1

example_model_advanced.py

@@ -0,0 +1,296 @@
+import pandas as pd
+from lightgbm import LGBMRegressor
+import gc
+from numerapi import NumerAPI
+from pathlib import Path
+from utils import (
+    save_model,
+    load_model,
+    neutralize,
+    get_biggest_change_features,
+    get_time_series_cross_val_splits,
+    validation_metrics,
+    load_model_config,
+    save_model_config,
+    save_prediction,
+    TARGET_COL,
+)
+
+
+EXAMPLE_PREDS_COL = "example_preds"
+ERA_COL = "era"
+# params we'll use to train all of our models.
+# Ideal params would be more like 20000, 0.001, 6, 2**6, 0.1, but this is slow enough as it is
+model_params = {"n_estimators": 2000,
+                "learning_rate": 0.01,
+                "max_depth": 5,
+                "num_leaves": 2 ** 5,
+                "colsample_bytree": 0.1}
+
+# the amount of downsampling we'll use to speed up cross validation and full train.
+# a value of 1 means no downsampling
+# a value of 10 means use every 10th row
+downsample_cross_val = 20
+downsample_full_train = 2
+
+# if model_selection_loop=True get OOS performance for training_data
+# and use that to select best model
+# if model_selection_loop=False, just predict on tournament data using existing models and model config
+model_selection_loop = True
+model_config_name = "advanced_example_model"
+
+napi = NumerAPI()
+
+current_round = napi.get_current_round()
+
+Path("./v4").mkdir(parents=False, exist_ok=True)
+napi.download_dataset("v4/train.parquet")
+napi.download_dataset("v4/features.json")
+
+
+print("Entering model selection loop.  This may take awhile.")
+if model_selection_loop:
+    model_config = {}
+    print('reading training_data')
+    training_data = pd.read_parquet('v4/train.parquet')
+
+    # keep track of some prediction columns
+    ensemble_cols = set()
+    pred_cols = set()
+
+    # pick some targets to use
+    possible_targets = [c for c in training_data.columns if c.startswith("target_")]
+    # randomly pick a handful of targets
+    # this can be vastly improved
+    targets = ["target", "target_nomi_v4_60", "target_jerome_v4_20"]
+
+    # all the possible features to train on
+    feature_cols = [c for c in training_data if c.startswith("feature_")]
+
+    """ do cross val to get out of sample training preds"""
+    cv = 3
+    train_test_zip = get_time_series_cross_val_splits(training_data, cv=cv, embargo=12)
+    # get out of sample training preds via embargoed time series cross validation
+    # optionally downsample training data to speed up this section.
+    print("entering time series cross validation loop")
+    for split, train_test_split in enumerate(train_test_zip):
+        gc.collect()
+        print(f"doing split {split+1} out of {cv}")
+        train_split, test_split = train_test_split
+        train_split_index = training_data[ERA_COL].isin(train_split)
+        test_split_index = training_data[ERA_COL].isin(test_split)
+        downsampled_train_split_index = train_split_index[train_split_index].index[::downsample_cross_val]
+
+        # getting the per era correlation of each feature vs the primary target across the training split
+        print("getting feature correlations over time and identifying riskiest features")
+        all_feature_corrs_split = training_data.loc[downsampled_train_split_index, :].groupby(ERA_COL).apply(
+            lambda d: d[feature_cols].corrwith(d[TARGET_COL]))
+        # find the riskiest features by comparing their correlation vs the target in half 1 and half 2 of training data
+        # there are probably more clever ways to do this
+        riskiest_features_split = get_biggest_change_features(all_feature_corrs_split, 50)
+
+        print(f"entering model training loop for split {split+1}")
+        for target in targets:
+            model_name = f"model_{target}"
+            print(f"model: {model_name}")
+
+            # train a model on the training split (and save it for future use)
+            split_model_name = f"model_{target}_split{split+1}cv{cv}downsample{downsample_cross_val}"
+            split_model = load_model(split_model_name)
+            if not split_model:
+                print(f"training model: {model_name}")
+                split_model = LGBMRegressor(**model_params)
+                split_model.fit(training_data.loc[downsampled_train_split_index, feature_cols],
+                                training_data.loc[downsampled_train_split_index,
+                                                  [target]])
+                save_model(split_model, split_model_name)
+            # now we can predict on the test part of the split
+            model_expected_features = split_model.booster_.feature_name()
+            if set(model_expected_features) != set(feature_cols):
+                print(f"New features are available! Might want to retrain model {split_model_name}.")
+            print(f"predicting {model_name}")
+            training_data.loc[test_split_index, f"preds_{model_name}"] = \
+                split_model.predict(training_data.loc[test_split_index, model_expected_features])
+
+            # do neutralization
+            print("doing neutralization to riskiest features")
+            training_data.loc[test_split_index, f"preds_{model_name}_neutral_riskiest_50"] = neutralize(
+                df=training_data.loc[test_split_index, :],
+                columns=[f"preds_{model_name}"],
+                neutralizers=riskiest_features_split,
+                proportion=1.0,
+                normalize=True,
+                era_col=ERA_COL)[f"preds_{model_name}"]
+
+            # remember that we made all of these different pred columns
+            pred_cols.add(f"preds_{model_name}")
+            pred_cols.add(f"preds_{model_name}_neutral_riskiest_50")
+
+        print("creating ensembles")
+        # ranking per era for all of our pred cols so we can combine safely on the same scales
+        training_data[list(pred_cols)] = training_data.groupby(ERA_COL).apply(
+            lambda d: d[list(pred_cols)].rank(pct=True))
+        # do ensembles
+        training_data["ensemble_neutral_riskiest_50"] = sum(
+            [training_data[pred_col] for pred_col in pred_cols if pred_col.endswith("neutral_riskiest_50")]).rank(
+            pct=True)
+        training_data["ensemble_not_neutral"] = sum(
+            [training_data[pred_col] for pred_col in pred_cols if "neutral" not in pred_col]).rank(pct=True)
+        training_data["ensemble_all"] = sum([training_data[pred_col] for pred_col in pred_cols]).rank(pct=True)
+
+        ensemble_cols.add("ensemble_neutral_riskiest_50")
+        ensemble_cols.add("ensemble_not_neutral")
+        ensemble_cols.add("ensemble_all")
+
+    """ Now get some stats and pick our favorite model"""
+    print("gathering validation metrics for out of sample training results")
+    all_model_cols = list(pred_cols) + list(ensemble_cols)
+    # use example_col preds_model_target as an estimates since no example preds provided for training
+    # fast_mode=True so that we skip some of the stats that are slower to calculate
+    training_stats = validation_metrics(training_data, all_model_cols, example_col="preds_model_target",
+                                        fast_mode=True, target_col=TARGET_COL)
+    print(training_stats[["mean", "sharpe"]].sort_values(by="sharpe", ascending=False).to_markdown())
+
+    # pick the model that has the highest correlation sharpe
+    best_pred_col = training_stats.sort_values(by="sharpe", ascending=False).head(1).index[0]
+    print(f"selecting model {best_pred_col} as our highest sharpe model in validation")
+
+    """ Now do a full train"""
+    print("entering full training section")
+    # getting the per era correlation of each feature vs the target across all of training data
+    print("getting feature correlations with target and identifying riskiest features")
+    all_feature_corrs = training_data.groupby(ERA_COL).apply(
+        lambda d: d[feature_cols].corrwith(d[TARGET_COL]))
+    # find the riskiest features by comparing their correlation vs the target in half 1 and half 2 of training data
+    riskiest_features = get_biggest_change_features(all_feature_corrs, 50)
+
+    for target in targets:
+        gc.collect()
+        model_name = f"model_{target}_downsample{downsample_full_train}"
+        model = load_model(model_name)
+        if not model:
+            print(f"training {model_name}")
+            model = LGBMRegressor(**model_params)
+            # train on all of train, predict on val, predict on tournament
+            model.fit(training_data.iloc[::downsample_full_train].loc[:, feature_cols],
+                      training_data.iloc[::downsample_full_train][target])
+            save_model(model, model_name)
+        gc.collect()
+
+    model_config["feature_cols"] = feature_cols
+    model_config["targets"] = targets
+    model_config["best_pred_col"] = best_pred_col
+    model_config["riskiest_features"] = riskiest_features
+    print(f"saving model config for {model_config_name}")
+    save_model_config(model_config, model_config_name)
+else:
+    # load model config from previous model selection loop
+    print(f"loading model config for {model_config_name}")
+    model_config = load_model_config(model_config_name)
+    feature_cols = model_config["feature_cols"]
+    targets = model_config["targets"]
+    best_pred_col = model_config["best_pred_col"]
+    riskiest_features = model_config["riskiest_features"]
+
+
+""" Things that we always do even if we've already trained """
+gc.collect()
+
+print("reading tournament_data")
+live_data = pd.read_parquet('v4/live.parquet')
+print("reading validation_data")
+validation_data = pd.read_parquet('v4/validation.parquet')
+print("reading example_predictions")
+example_preds = pd.read_parquet('v4/live_example_preds.parquet')
+print("reading example_validaton_predictions")
+validation_example_preds = pd.read_parquet('v4/validation_example_preds.parquet')
+# set the example predictions
+validation_data[EXAMPLE_PREDS_COL] = validation_example_preds["prediction"]
+
+# check for nans and fill nans
+print("checking for nans in the tournament data")
+if live_data.loc[:, feature_cols].isna().sum().sum():
+    cols_w_nan = live_data.loc[:, feature_cols].isna().sum()
+    total_rows = len(live_data)
+    print(f"Number of nans per column this week: {cols_w_nan[cols_w_nan > 0]}")
+    print(f"out of {total_rows} total rows")
+    print(f"filling nans with 0.5")
+    live_data.loc[:, feature_cols] = live_data.loc[:, feature_cols].fillna(0.5)
+
+else:
+    print("No nans in the features this week!")
+
+
+pred_cols = set()
+ensemble_cols = set()
+for target in targets:
+    gc.collect()
+    model_name = f"model_{target}_downsample{downsample_full_train}"
+    print(f"loading {model_name}")
+    model = load_model(model_name)
+    if not model:
+        raise ValueError(f"{model_name} is not trained yet!")
+
+    model_expected_features = model.booster_.feature_name()
+    if set(model_expected_features) != set(feature_cols):
+        print(f"New features are available! Might want to retrain model {model_name}.")
+    print(f"predicting tournament and validation for {model_name}")
+    validation_data.loc[:, f"preds_{model_name}"] = model.predict(validation_data.loc[:, model_expected_features])
+    live_data.loc[:, f"preds_{model_name}"] = model.predict(live_data.loc[:, model_expected_features])
+
+    # do different neutralizations
+    # neutralize our predictions to the riskiest features only
+    print("neutralizing to riskiest_50 for validation and tournament")
+    validation_data[f"preds_{model_name}_neutral_riskiest_50"] = neutralize(df=validation_data,
+                                                                            columns=[f"preds_{model_name}"],
+                                                                            neutralizers=riskiest_features,
+                                                                            proportion=1.0,
+                                                                            normalize=True,
+                                                                            era_col=ERA_COL)[f"preds_{model_name}"]
+    live_data[f"preds_{model_name}_neutral_riskiest_50"] = neutralize(df=live_data,
+                                                                            columns=[f"preds_{model_name}"],
+                                                                            neutralizers=riskiest_features,
+                                                                            proportion=1.0,
+                                                                            normalize=True,
+                                                                            era_col=ERA_COL)[f"preds_{model_name}"]
+
+    pred_cols.add(f"preds_{model_name}")
+    pred_cols.add(f"preds_{model_name}_neutral_riskiest_50")
+
+
+# rank per era for each prediction column so that we can combine safely
+validation_data[list(pred_cols)] = validation_data.groupby(ERA_COL).apply(lambda d: d[list(pred_cols)].rank(pct=True))
+live_data[list(pred_cols)] = live_data.groupby(ERA_COL).apply(lambda d: d[list(pred_cols)].rank(pct=True))
+# make ensembles for val and tournament
+print('creating ensembles for tournament and validation')
+validation_data["ensemble_neutral_riskiest_50"] = sum(
+    [validation_data[pred_col] for pred_col in pred_cols if pred_col.endswith("neutral_riskiest_50")]).rank(
+    pct=True)
+live_data["ensemble_neutral_riskiest_50"] = sum(
+    [live_data[pred_col] for pred_col in pred_cols if pred_col.endswith("neutral_riskiest_50")]).rank(
+    pct=True)
+ensemble_cols.add("ensemble_neutral_riskiest_50")
+
+validation_data["ensemble_not_neutral"] = sum(
+    [validation_data[pred_col] for pred_col in pred_cols if "neutral" not in pred_col]).rank(pct=True)
+live_data["ensemble_not_neutral"] = sum(
+    [live_data[pred_col] for pred_col in pred_cols if "neutral" not in pred_col]).rank(pct=True)
+ensemble_cols.add("ensemble_not_neutral")
+
+validation_data["ensemble_all"] = sum([validation_data[pred_col] for pred_col in pred_cols]).rank(pct=True)
+live_data["ensemble_all"] = sum([live_data[pred_col] for pred_col in pred_cols]).rank(pct=True)
+
+ensemble_cols.add("ensemble_all")
+
+gc.collect()
+print("getting final validation stats")
+# get our final validation stats for our chosen model
+validation_stats = validation_metrics(validation_data, list(pred_cols)+list(ensemble_cols), example_col=EXAMPLE_PREDS_COL,
+                                      fast_mode=False, target_col=TARGET_COL)
+print(validation_stats.to_markdown())
+
+# rename best model to prediction and rank from 0 to 1 to meet diagnostic/submission file requirements
+validation_data["prediction"] = validation_data[best_pred_col].rank(pct=True)
+live_data["prediction"] = live_data[best_pred_col].rank(pct=True)
+save_prediction(validation_data["prediction"], f"validation_predictions_{current_round}")
+save_prediction(live_data["prediction"], f"live_data_{current_round}")

utils.py

@@ -0,0 +1,312 @@
+import numpy as np
+import pandas as pd
+import scipy
+from halo import Halo
+from pathlib import Path
+import json
+from scipy.stats import skew
+
+ERA_COL = "era"
+TARGET_COL = "target_nomi_v4_20"
+DATA_TYPE_COL = "data_type"
+EXAMPLE_PREDS_COL = "example_preds"
+
+spinner = Halo(text='', spinner='dots')
+
+MODEL_FOLDER = "models"
+MODEL_CONFIGS_FOLDER = "model_configs"
+PREDICTION_FILES_FOLDER = "prediction_files"
+
+
+def save_prediction(df, name):
+    try:
+        Path(PREDICTION_FILES_FOLDER).mkdir(exist_ok=True, parents=True)
+    except Exception as ex:
+        pass
+    df.to_csv(f"{PREDICTION_FILES_FOLDER}/{name}.csv", index=True)
+
+
+def save_model(model, name):
+    try:
+        Path(MODEL_FOLDER).mkdir(exist_ok=True, parents=True)
+    except Exception as ex:
+        pass
+    pd.to_pickle(model, f"{MODEL_FOLDER}/{name}.pkl")
+
+
+def load_model(name):
+    path = Path(f"{MODEL_FOLDER}/{name}.pkl")
+    if path.is_file():
+        model = pd.read_pickle(f"{MODEL_FOLDER}/{name}.pkl")
+    else:
+        model = False
+    return model
+
+
+def save_model_config(model_config, model_name):
+    try:
+        Path(MODEL_CONFIGS_FOLDER).mkdir(exist_ok=True, parents=True)
+    except Exception as ex:
+        pass
+    with open(f"{MODEL_CONFIGS_FOLDER}/{model_name}.json", 'w') as fp:
+        json.dump(model_config, fp)
+
+
+def load_model_config(model_name):
+    path_str = f"{MODEL_CONFIGS_FOLDER}/{model_name}.json"
+    path = Path(path_str)
+    if path.is_file():
+        with open(path_str, 'r') as fp:
+            model_config = json.load(fp)
+    else:
+        model_config = False
+    return model_config
+
+
+def get_biggest_change_features(corrs, n):
+    all_eras = corrs.index.sort_values()
+    h1_eras = all_eras[:len(all_eras) // 2]
+    h2_eras = all_eras[len(all_eras) // 2:]
+
+    h1_corr_means = corrs.loc[h1_eras, :].mean()
+    h2_corr_means = corrs.loc[h2_eras, :].mean()
+
+    corr_diffs = h2_corr_means - h1_corr_means
+    worst_n = corr_diffs.abs().sort_values(ascending=False).head(n).index.tolist()
+    return worst_n
+
+
+def get_time_series_cross_val_splits(data, cv=3, embargo=12):
+    all_train_eras = data[ERA_COL].unique()
+    len_split = len(all_train_eras) // cv
+    test_splits = [all_train_eras[i * len_split:(i + 1) * len_split] for i in range(cv)]
+    # fix the last test split to have all the last eras, in case the number of eras wasn't divisible by cv
+    remainder = len(all_train_eras) % cv
+    if remainder != 0:
+        test_splits[-1] = np.append(test_splits[-1], all_train_eras[-remainder:])
+
+    train_splits = []
+    for test_split in test_splits:
+        test_split_max = int(np.max(test_split))
+        test_split_min = int(np.min(test_split))
+        # get all of the eras that aren't in the test split
+        train_split_not_embargoed = [e for e in all_train_eras if not (test_split_min <= int(e) <= test_split_max)]
+        # embargo the train split so we have no leakage.
+        # one era is length 5, so we need to embargo by target_length/5 eras.
+        # To be consistent for all targets, let's embargo everything by 60/5 == 12 eras.
+        train_split = [e for e in train_split_not_embargoed if
+                       abs(int(e) - test_split_max) > embargo and abs(int(e) - test_split_min) > embargo]
+        train_splits.append(train_split)
+
+    # convenient way to iterate over train and test splits
+    train_test_zip = zip(train_splits, test_splits)
+    return train_test_zip
+
+
+def neutralize(df,
+               columns,
+               neutralizers=None,
+               proportion=1.0,
+               normalize=True,
+               era_col="era"):
+    if neutralizers is None:
+        neutralizers = []
+    unique_eras = df[era_col].unique()
+    computed = []
+    for u in unique_eras:
+        df_era = df[df[era_col] == u]
+        scores = df_era[columns].values
+        if normalize:
+            scores2 = []
+            for x in scores.T:
+                x = (scipy.stats.rankdata(x, method='ordinal') - .5) / len(x)
+                x = scipy.stats.norm.ppf(x)
+                scores2.append(x)
+            scores = np.array(scores2).T
+        exposures = df_era[neutralizers].values
+
+        scores -= proportion * exposures.dot(
+            np.linalg.pinv(exposures.astype(np.float32), rcond=1e-6).dot(scores.astype(np.float32)))
+
+        scores /= scores.std(ddof=0)
+
+        computed.append(scores)
+
+    return pd.DataFrame(np.concatenate(computed),
+                        columns=columns,
+                        index=df.index)
+
+
+def neutralize_series(series, by, proportion=1.0):
+    scores = series.values.reshape(-1, 1)
+    exposures = by.values.reshape(-1, 1)
+
+    # this line makes series neutral to a constant column so that it's centered and for sure gets corr 0 with exposures
+    exposures = np.hstack(
+        (exposures,
+         np.array([np.mean(series)] * len(exposures)).reshape(-1, 1)))
+
+    correction = proportion * (exposures.dot(
+        np.linalg.lstsq(exposures, scores, rcond=None)[0]))
+    corrected_scores = scores - correction
+    neutralized = pd.Series(corrected_scores.ravel(), index=series.index)
+    return neutralized
+
+
+def unif(df):
+    x = (df.rank(method="first") - 0.5) / len(df)
+    return pd.Series(x, index=df.index)
+
+
+def get_feature_neutral_mean(df, prediction_col, target_col, features_for_neutralization=None):
+    if features_for_neutralization is None:
+        features_for_neutralization = [c for c in df.columns if c.startswith("feature")]
+    df.loc[:, "neutral_sub"] = neutralize(df, [prediction_col],
+                                          features_for_neutralization)[prediction_col]
+    scores = df.groupby("era").apply(
+        lambda x: (unif(x["neutral_sub"]).corr(x[target_col]))).mean()
+    return np.mean(scores)
+
+def get_feature_neutral_mean_tb_era(df, prediction_col, target_col, tb, features_for_neutralization=None):
+    if features_for_neutralization is None:
+        features_for_neutralization = [c for c in df.columns if c.startswith("feature")]
+    temp_df = df.reset_index(drop=True).copy()        # Reset index due to use of argsort later
+    temp_df.loc[:, "neutral_sub"] = neutralize(temp_df, [prediction_col],
+                                          features_for_neutralization)[prediction_col]
+    temp_df_argsort = temp_df.loc[:, 'neutral_sub'].argsort()
+    temp_df_tb_idx = pd.concat([temp_df_argsort.iloc[:tb],
+                           temp_df_argsort.iloc[-tb:]])
+    temp_df_tb = temp_df.loc[temp_df_tb_idx]
+    tb_fnc = unif(temp_df_tb['neutral_sub']).corr(temp_df_tb[target_col])
+    return tb_fnc
+
+
+def fast_score_by_date(df, columns, target, tb=None, era_col="era"):
+    unique_eras = df[era_col].unique()
+    computed = []
+    for u in unique_eras:
+        df_era = df[df[era_col] == u]
+        era_pred = np.float64(df_era[columns].values.T)
+        era_target = np.float64(df_era[target].values.T)
+
+        if tb is None:
+            ccs = np.corrcoef(era_target, era_pred)[0, 1:]
+        else:
+            tbidx = np.argsort(era_pred, axis=1)
+            tbidx = np.concatenate([tbidx[:, :tb], tbidx[:, -tb:]], axis=1)
+            ccs = [np.corrcoef(era_target[tmpidx], tmppred[tmpidx])[0, 1] for tmpidx, tmppred in zip(tbidx, era_pred)]
+            ccs = np.array(ccs)
+
+        computed.append(ccs)
+
+    return pd.DataFrame(np.array(computed), columns=columns, index=df[era_col].unique())
+
+def exposure_dissimilarity_per_era(df, prediction_col, example_col, feature_cols=None):
+    if feature_cols is None:
+        feature_cols = [c for c in df.columns if c.startswith("feature")]
+    u = df.loc[:, feature_cols].corrwith(df[prediction_col])
+    e = df.loc[:, feature_cols].corrwith(df[example_col])
+    return (1 - (np.dot(u,e)/np.dot(e,e)))
+
+def validation_metrics(validation_data, pred_cols, example_col, fast_mode=False,
+                        target_col=TARGET_COL, features_for_neutralization=None):
+    validation_stats = pd.DataFrame()
+    feature_cols = [c for c in validation_data if c.startswith("feature_")]
+    for pred_col in pred_cols:
+        # Check the per-era correlations on the validation set (out of sample)
+        validation_correlations = validation_data.groupby(ERA_COL).apply(
+            lambda d: unif(d[pred_col]).corr(d[target_col]))
+
+        mean = validation_correlations.mean()
+        std = validation_correlations.std(ddof=0)
+        sharpe = mean / std
+
+        validation_stats.loc["mean", pred_col] = mean
+        validation_stats.loc["std", pred_col] = std
+        validation_stats.loc["sharpe", pred_col] = sharpe
+
+        rolling_max = (validation_correlations + 1).cumprod().rolling(window=9000,  # arbitrarily large
+                                                                      min_periods=1).max()
+        daily_value = (validation_correlations + 1).cumprod()
+        max_drawdown = -((rolling_max - daily_value) / rolling_max).max()
+        validation_stats.loc["max_drawdown", pred_col] = max_drawdown
+
+        payout_scores = validation_correlations.clip(-0.25, 0.25)
+        payout_daily_value = (payout_scores + 1).cumprod()
+
+        apy = (
+                      (
+                              (payout_daily_value.dropna().iloc[-1])
+                              ** (1 / len(payout_scores))
+                      )
+                      ** 49  # 52 weeks of compounding minus 3 for stake compounding lag
+                      - 1
+              ) * 100
+
+        validation_stats.loc["apy", pred_col] = apy
+
+        if not fast_mode:
+            # Check the feature exposure of your validation predictions
+            max_per_era = validation_data.groupby(ERA_COL).apply(
+                lambda d: d[feature_cols].corrwith(d[pred_col]).abs().max())
+            max_feature_exposure = max_per_era.mean()
+            validation_stats.loc["max_feature_exposure", pred_col] = max_feature_exposure
+
+            # Check feature neutral mean
+            feature_neutral_mean = get_feature_neutral_mean(validation_data, pred_col,
+                                                                target_col, features_for_neutralization)
+            validation_stats.loc["feature_neutral_mean", pred_col] = feature_neutral_mean
+
+            # Check TB200 feature neutral mean
+            tb200_feature_neutral_mean_era = validation_data.groupby(ERA_COL).apply(lambda df: \
+                                            get_feature_neutral_mean_tb_era(df, pred_col,
+                                                                            target_col, 200,
+                                                                            features_for_neutralization))
+            validation_stats.loc["tb200_feature_neutral_mean", pred_col] = tb200_feature_neutral_mean_era.mean()
+
+            # Check top and bottom 200 metrics (TB200)
+            tb200_validation_correlations = fast_score_by_date(
+                validation_data,
+                [pred_col],
+                target_col,
+                tb=200,
+                era_col=ERA_COL
+            )
+
+            tb200_mean = tb200_validation_correlations.mean()[pred_col]
+            tb200_std = tb200_validation_correlations.std(ddof=0)[pred_col]
+            tb200_sharpe = tb200_mean / tb200_std
+
+            validation_stats.loc["tb200_mean", pred_col] = tb200_mean
+            validation_stats.loc["tb200_std", pred_col] = tb200_std
+            validation_stats.loc["tb200_sharpe", pred_col] = tb200_sharpe
+
+        # MMC over validation
+        mmc_scores = []
+        corr_scores = []
+        for _, x in validation_data.groupby(ERA_COL):
+            series = neutralize_series(unif(x[pred_col]), (x[example_col]))
+            mmc_scores.append(np.cov(series, x[target_col])[0, 1] / (0.29 ** 2))
+            corr_scores.append(unif(x[pred_col]).corr(x[target_col]))
+
+        val_mmc_mean = np.mean(mmc_scores)
+        val_mmc_std = np.std(mmc_scores)
+        corr_plus_mmcs = [c + m for c, m in zip(corr_scores, mmc_scores)]
+        corr_plus_mmc_sharpe = np.mean(corr_plus_mmcs) / np.std(corr_plus_mmcs)
+
+        validation_stats.loc["mmc_mean", pred_col] = val_mmc_mean
+        validation_stats.loc["corr_plus_mmc_sharpe", pred_col] = corr_plus_mmc_sharpe
+
+        # Check correlation with example predictions
+        per_era_corrs = validation_data.groupby(ERA_COL).apply(lambda d: unif(d[pred_col]).corr(unif(d[example_col])))
+        corr_with_example_preds = per_era_corrs.mean()
+        validation_stats.loc["corr_with_example_preds", pred_col] = corr_with_example_preds
+
+        #Check exposure dissimilarity per era
+        tdf = validation_data.groupby(ERA_COL).apply(lambda df: \
+                                                exposure_dissimilarity_per_era(df, pred_col,
+                                                example_col, feature_cols))
+        validation_stats.loc["exposure_dissimilarity_mean", pred_col] = tdf.mean()
+
+    # .transpose so that stats are columns and the model_name is the row
+    return validation_stats.transpose()