Convert python script into streamlit app - first steps

app.py

@@ -3,9 +3,11 @@ import streamlit as st
 st.title('Numerai Example Script')
 
 
-# content below from 
+# content below adapted from
 # https://github.com/numerai/example-scripts/blob/master/example_model.py
-# 
+#
+
+IS_RUNNING_IN_HUGGING_FACE = False
 
 import pandas as pd
 from lightgbm import LGBMRegressor
@@ -27,144 +29,208 @@ from utils import (
 )
 
 
-# download all the things
-
 napi = NumerAPI()
-
 current_round = napi.get_current_round()
 
 # Tournament data changes every week so we specify the round in their name. Training
-# and validation data only change periodically, so no need to download them every time.
-print('Downloading dataset files...')
+# and validation data only change periodically, so no need to download
+# them every time.
 
 Path("./v4").mkdir(parents=False, exist_ok=True)
-napi.download_dataset("v4/train.parquet")
-napi.download_dataset("v4/validation.parquet")
-napi.download_dataset("v4/live.parquet", f"v4/live_{current_round}.parquet")
-napi.download_dataset("v4/validation_example_preds.parquet")
-napi.download_dataset("v4/features.json")
-
-print('Reading minimal training data')
-# read the feature metadata and get a feature set (or all the features)
-with open("v4/features.json", "r") as f:
-    feature_metadata = json.load(f)
-# features = list(feature_metadata["feature_stats"].keys()) # get all the features
-# features = feature_metadata["feature_sets"]["small"] # get the small feature set
-features = feature_metadata["feature_sets"]["medium"] # get the medium feature set
-# read in just those features along with era and target columns
-read_columns = features + [ERA_COL, DATA_TYPE_COL, TARGET_COL]
-
-# note: sometimes when trying to read the downloaded data you get an error about invalid magic parquet bytes...
-# if so, delete the file and rerun the napi.download_dataset to fix the corrupted file
-training_data = pd.read_parquet('v4/train.parquet',
+
+
+@st.cache
+def download_dataset():
+    print('download_dataset')
+
+    if IS_RUNNING_IN_HUGGING_FACE:
+        from datasets import load_dataset_builder
+        ds_builder = load_dataset_builder("Numerati/numerai-datasets")
+    else:
+        napi.download_dataset("v4/train.parquet")
+        napi.download_dataset("v4/validation.parquet")
+        napi.download_dataset("v4/validation_example_preds.parquet")
+        napi.download_dataset("v4/features.json")
+
+    napi.download_dataset("v4/live.parquet", f"v4/live_{current_round}.parquet")
+    print('done download_dataset')
+
+@st.cache
+def load_dataset(feature_set: str):
+    print('load_dataset with feature_set', feature_set)
+    # read the feature metadata and get a feature set (or all the features)
+    with open("v4/features.json", "r") as f:
+        feature_metadata = json.load(f)
+    # features = list(feature_metadata["feature_stats"].keys()) # get all the features
+    # features = feature_metadata["feature_sets"]["small"] # get the small
+    # feature set
+    features = feature_metadata["feature_sets"][feature_set]  # get the medium feature set
+    # read in just those features along with era and target columns
+    read_columns = features + [ERA_COL, DATA_TYPE_COL, TARGET_COL]
+
+    # note: sometimes when trying to read the downloaded data you get an error about invalid magic parquet bytes...
+    # if so, delete the file and rerun the napi.download_dataset to fix the
+    # corrupted file
+    training_data = pd.read_parquet('v4/train.parquet',
+                                    columns=read_columns)
+    validation_data = pd.read_parquet('v4/validation.parquet',
+                                      columns=read_columns)
+    live_data = pd.read_parquet(f'v4/live_{current_round}.parquet',
                                 columns=read_columns)
-validation_data = pd.read_parquet('v4/validation.parquet',
-                                  columns=read_columns)
-live_data = pd.read_parquet(f'v4/live_{current_round}.parquet',
-                                  columns=read_columns)
 
+    # pare down the number of eras to every 4th era
+    # every_4th_era = training_data[ERA_COL].unique()[::4]
+    # training_data = training_data[training_data[ERA_COL].isin(every_4th_era)]
 
-# pare down the number of eras to every 4th era
-# every_4th_era = training_data[ERA_COL].unique()[::4]
-# training_data = training_data[training_data[ERA_COL].isin(every_4th_era)]
+    # getting the per era correlation of each feature vs the target
+    all_feature_corrs = training_data.groupby(ERA_COL).apply(
+        lambda era: era[features].corrwith(era[TARGET_COL])
+    )
 
-# getting the per era correlation of each feature vs the target
-all_feature_corrs = training_data.groupby(ERA_COL).apply(
-    lambda era: era[features].corrwith(era[TARGET_COL])
-)
+    # find the riskiest features by comparing their correlation vs
+    # the target in each half of training data; we'll use these later
+    riskiest_features = get_biggest_change_features(all_feature_corrs, 50)
 
-# find the riskiest features by comparing their correlation vs
-# the target in each half of training data; we'll use these later
-riskiest_features = get_biggest_change_features(all_feature_corrs, 50)
+    # "garbage collection" (gc) gets rid of unused data and frees up memory
+    gc.collect()
+    print('done  with feature_set', feature_set)
+    return training_data, validation_data, live_data, features, riskiest_features
 
-# "garbage collection" (gc) gets rid of unused data and frees up memory
-gc.collect()
+feature_set = st.selectbox(
+    'Which feature set should be used?',
+    ('small', 'medium', 'fncv3_features', 'v2_equivalent_features', 'v3_equivalent_features'))
 
-model_name = f"model_target"
-print(f"Checking for existing model '{model_name}'")
-model = load_model(model_name)
-if not model:
-    print(f"model not found, creating new one")
-    params = {"n_estimators": 2000,
-              "learning_rate": 0.01,
-              "max_depth": 5,
-              "num_leaves": 2 ** 5,
-              "colsample_bytree": 0.1}
-
-    model = LGBMRegressor(**params)
-
-    # train on all of train and save the model so we don't have to train next time
-    model.fit(training_data.filter(like='feature_', axis='columns'),
-              training_data[TARGET_COL])
-    print(f"saving new model: {model_name}")
-    save_model(model, model_name)
-
-gc.collect()
-
-nans_per_col = live_data[live_data["data_type"] == "live"][features].isna().sum()
-
-# check for nans and fill nans
-if nans_per_col.any():
-    total_rows = len(live_data[live_data["data_type"] == "live"])
-    print(f"Number of nans per column this week: {nans_per_col[nans_per_col > 0]}")
-    print(f"out of {total_rows} total rows")
-    print(f"filling nans with 0.5")
-    live_data.loc[:, features] = live_data.loc[:, features].fillna(0.5)
-
-else:
-    print("No nans in the features this week!")
-
-
-# double check the feature that the model expects vs what is available to prevent our
-# pipeline from failing if Numerai adds more data and we don't have time to retrain!
-model_expected_features = model.booster_.feature_name()
-if set(model_expected_features) != set(features):
-    print(f"New features are available! Might want to retrain model {model_name}.")
-validation_data.loc[:, f"preds_{model_name}"] = model.predict(
-    validation_data.loc[:, model_expected_features])
-live_data.loc[:, f"preds_{model_name}"] = model.predict(
-    live_data.loc[:, model_expected_features])
-
-gc.collect()
-
-# neutralize our predictions to the riskiest features
-validation_data[f"preds_{model_name}_neutral_riskiest_50"] = neutralize(
-    df=validation_data,
-    columns=[f"preds_{model_name}"],
-    neutralizers=riskiest_features,
-    proportion=1.0,
-    normalize=True,
-    era_col=ERA_COL
-)
+data_load_state = st.text('Loading data...')
+download_dataset()
+training_data, validation_data, live_data, features, riskiest_features = load_dataset(feature_set)
+data_load_state.text('Loading data...done!')
 
-live_data[f"preds_{model_name}_neutral_riskiest_50"] = neutralize(
-    df=live_data,
-    columns=[f"preds_{model_name}"],
-    neutralizers=riskiest_features,
-    proportion=1.0,
-    normalize=True,
-    era_col=ERA_COL
-)
+st.subheader('Raw data')
+st.write(training_data.head())
 
-model_to_submit = f"preds_{model_name}_neutral_riskiest_50"
+st.subheader('Model Configuration')
 
-# rename best model to "prediction" and rank from 0 to 1 to meet upload requirements
-validation_data["prediction"] = validation_data[model_to_submit].rank(pct=True)
-live_data["prediction"] = live_data[model_to_submit].rank(pct=True)
-validation_data["prediction"].to_csv(f"validation_predictions_{current_round}.csv")
-live_data["prediction"].to_csv(f"live_predictions_{current_round}.csv")
+n_estimators = st.slider('n_estimators', 100, 10000, 2000)
+learning_rate = st.slider('learning_rate', 0.0001, 0.1, 0.01)
+max_depth = st.slider('max_depth', 2, 20, 5)
 
-validation_preds = pd.read_parquet('v4/validation_example_preds.parquet')
-validation_data[EXAMPLE_PREDS_COL] = validation_preds["prediction"]
+params = {"n_estimators": n_estimators,
+          "learning_rate": learning_rate,
+          "max_depth": max_depth,
+          "num_leaves": 2 ** 5,
+          "colsample_bytree": 0.1
+          }
 
-# get some stats about each of our models to compare...
-# fast_mode=True so that we skip some of the stats that are slower to calculate
-validation_stats = validation_metrics(validation_data, [model_to_submit, f"preds_{model_name}"], example_col=EXAMPLE_PREDS_COL, fast_mode=True, target_col=TARGET_COL)
-print(validation_stats[["mean", "sharpe"]].to_markdown())
+model_name = f"model_target"
 
-print(f'''
-Done! Next steps:
-    1. Go to numer.ai/tournament (make sure you have an account)
-    2. Submit validation_predictions_{current_round}.csv to the diagnostics tool
-    3. Submit tournament_predictions_{current_round}.csv to the "Upload Predictions" button
-''')
+@st.cache
+def get_model_and_fit(model_name, *params):
+    print('get_model_and_fit')
+    model = load_model(model_name)
+    if not model:
+        with st.spinner('Wait model training...'):
+            print(f"model not found, creating new one")
+
+            model = LGBMRegressor(**params)
+
+            # train on all of train and save the model so we don't have to
+            # train next time
+            model.fit(training_data.filter(like='feature_', axis='columns'),
+                      training_data[TARGET_COL])
+            print(f"saving new model: {model_name}")
+            save_model(model, model_name)
+        st.success('Done model training!')
+
+    gc.collect()
+    print('done get_model_and_fit')
+
+
+
+has_model_preds = False
+
+@st.cache
+def get_model_preds(model_name, *params):
+    print('get_model_preds')
+    model = load_model(model_name)
+
+    has_model_preds = False
+    nans_per_col = live_data[live_data["data_type"]
+                             == "live"][features].isna().sum()
+
+    # check for nans and fill nans
+    if nans_per_col.any():
+        total_rows = len(live_data[live_data["data_type"] == "live"])
+        print(f"Number of nans per column this week: {nans_per_col[nans_per_col > 0]}")
+        print(f"out of {total_rows} total rows")
+        print(f"filling nans with 0.5")
+        live_data.loc[:, features] = live_data.loc[:, features].fillna(0.5)
+
+    else:
+        print("No nans in the features this week!")
+
+
+    # double check the feature that the model expects vs what is available to prevent our
+    # pipeline from failing if Numerai adds more data and we don't have time
+    # to retrain!
+    model_expected_features = model.booster_.feature_name()
+    if set(model_expected_features) != set(features):
+        print(f"New features are available! Might want to retrain model {model_name}.")
+    validation_data.loc[:, f"preds_{model_name}"] = model.predict(
+        validation_data.loc[:, model_expected_features])
+    live_data.loc[:, f"preds_{model_name}"] = model.predict(
+        live_data.loc[:, model_expected_features])
+
+    gc.collect()
+
+    # neutralize our predictions to the riskiest features
+    validation_data[f"preds_{model_name}_neutral_riskiest_50"] = neutralize(
+        df=validation_data,
+        columns=[f"preds_{model_name}"],
+        neutralizers=riskiest_features,
+        proportion=1.0,
+        normalize=True,
+        era_col=ERA_COL
+    )
+
+    live_data[f"preds_{model_name}_neutral_riskiest_50"] = neutralize(
+        df=live_data,
+        columns=[f"preds_{model_name}"],
+        neutralizers=riskiest_features,
+        proportion=1.0,
+        normalize=True,
+        era_col=ERA_COL
+    )
+
+    model_to_submit = f"preds_{model_name}_neutral_riskiest_50"
+
+    # rename best model to "prediction" and rank from 0 to 1 to meet upload
+    # requirements
+    validation_data["prediction"] = validation_data[model_to_submit].rank(pct=True)
+    live_data["prediction"] = live_data[model_to_submit].rank(pct=True)
+    validation_prediction_fname = f"validation_predictions_{current_round}.csv"
+    validation_data["prediction"].to_csv(validation_prediction_fname)
+    live_data["prediction"].to_csv(f"live_predictions_{current_round}.csv")
+
+    validation_preds = pd.read_parquet('v4/validation_example_preds.parquet')
+    validation_data[EXAMPLE_PREDS_COL] = validation_preds["prediction"]
+
+    # get some stats about each of our models to compare...
+    # fast_mode=True so that we skip some of the stats that are slower to calculate
+    print('start validation_metrics')
+    validation_stats = validation_metrics(validation_data, [model_to_submit, f"preds_{model_name}"], example_col=EXAMPLE_PREDS_COL, fast_mode=True, target_col=TARGET_COL)
+    st.markdown(validation_stats[["mean", "sharpe"]].to_markdown())
+
+    # st.write(f'''
+    # Done! Next steps:
+    #     1. Go to numer.ai/tournament (make sure you have an account)
+    #     2. Submit validation_predictions_{current_round}.csv to the diagnostics tool
+    #     3. Submit tournament_predictions_{current_round}.csv to the "Upload Predictions" button
+    # ''')
+    has_model_preds = True
+
+
+st.button('Start model training', on_click=get_model_and_fit, args=[model_name, params])
+st.button('Start model evaluation', on_click=get_model_preds, args=[model_name, params])
+
+if has_model_preds:
+    st.download_button('Validation data for diagnostics tool', validation_data["prediction"], validation_prediction_fname)