import random
import time
import math
import json
import pprint
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
sns.set_style('darkgrid')
sns.set_context('paper')
sns.set(font_scale=1.5)
import warnings
warnings.filterwarnings('ignore')
import h2o
h2o.init()
DATA_LOCATION = "../../data/"
MODELS_LOCATION = "../../models/"
Checking whether there is an H2O instance running at http://localhost:54321 . connected. Warning: Your H2O cluster version is too old (5 months and 20 days)! Please download and install the latest version from http://h2o.ai/download/
H2O_cluster_uptime: | 08 secs |
H2O_cluster_timezone: | Etc/UTC |
H2O_data_parsing_timezone: | UTC |
H2O_cluster_version: | 3.30.0.4 |
H2O_cluster_version_age: | 5 months and 20 days !!! |
H2O_cluster_name: | H2O_from_python_azureuser_leufpx |
H2O_cluster_total_nodes: | 1 |
H2O_cluster_free_memory: | 6.108 Gb |
H2O_cluster_total_cores: | 4 |
H2O_cluster_allowed_cores: | 4 |
H2O_cluster_status: | locked, healthy |
H2O_connection_url: | http://localhost:54321 |
H2O_connection_proxy: | {"http": null, "https": null} |
H2O_internal_security: | False |
H2O_API_Extensions: | Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4 |
Python_version: | 3.6.9 final |
# Create a util function to extract training time in minutes
def millis_to_minutes(millis):
minutes=(millis/(1000*60))%60
# minutes = math.ceil(float(minutes))
return minutes
top_dl= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_dl/DeepLearning_model_python_1605423034668_341")
top_drf= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_drf/DRF_model_python_1605423034668_386")
top_gbm= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_gbm/GBM_model_python_1605423034668_39")
top_glm= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_glm/GLM_model_python_1605423034668_20")
top_nb= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_nb/NaiveBayes_model_python_1605423034668_1")
top_xgb= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_xgb/XGBoost_model_python_1605423034668_274")
# top_ensemble_ALL_MODELS_METALEARNER_auto= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_ALL_MODELS_METALEARNER_auto/stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_auto")
top_ensemble_ALL_MODELS_METALEARNER_deeplearning= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_ALL_MODELS_METALEARNER_deeplearning/stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_deeplearning")
top_ensemble_ALL_MODELS_METALEARNER_drf= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_ALL_MODELS_METALEARNER_drf/stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_drf")
top_ensemble_ALL_MODELS_METALEARNER_gbm= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_ALL_MODELS_METALEARNER_gbm/stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_gbm")
top_ensemble_ALL_MODELS_METALEARNER_glm= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_ALL_MODELS_METALEARNER_glm/stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_glm")
top_ensemble_ALL_MODELS_METALEARNER_naivebayes= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_ALL_MODELS_METALEARNER_naivebayes/stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_naivebayes")
top_ensemble_ALL_MODELS_METALEARNER_xgboost= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_ALL_MODELS_METALEARNER_xgboost/stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_xgboost")
# top_ensemble_CHECKPOINT_MODELS_METALEARNER_auto= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_MODELS_METALEARNER_auto/stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_auto")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_deeplearning= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_MODELS_METALEARNER_deeplearning/stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_drf= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_MODELS_METALEARNER_drf/stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_drf")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_gbm= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_MODELS_METALEARNER_gbm/stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_gbm")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_glm= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_MODELS_METALEARNER_glm/stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_glm")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_naivebayes= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_MODELS_METALEARNER_naivebayes/stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_xgboost= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_MODELS_METALEARNER_xgboost/stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost")
# top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_auto= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_auto/stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_auto")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_deeplearning= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_deeplearning/stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_drf= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_drf/stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_gbm= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_gbm/stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_glm= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_glm/stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_naivebayes= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_naivebayes/stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_xgboost= h2o.load_model(MODELS_LOCATION + "ALL_FEATURES/FINAL/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_xgboost/stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost")
# save the model
# model_path = h2o.save_model(model= my_pca, path="../models/my_pca_model", force=True)
model_path = "../../models/PCA300/PCA_model_python_1603962989759_1_k300"
# load the model
pca300 = h2o.load_model(model_path)
a4_dims = (15, 10)
plt.figure(
# figsize=(10,10),
dpi=100)
fig, ax = plt.subplots(figsize=a4_dims)
ax.set_xticks([49,99,149, 199, 249, 299])
sns.lineplot(ax=ax, data= pca300_df.loc['Cumulative Proportion'])
plt.ylabel('Cumulative Proportion of Variance')
plt.xlabel('Number of Components')
top_dl_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_dl/DeepLearning_model_python_1605362597414_3340")
top_drf_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_drf/DRF_model_python_1605362597414_2415")
top_gbm_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_gbm/GBM_model_python_1605362597414_1525")
top_glm_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_glm/GLM_model_python_1605362597414_1506")
top_nb_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_nb/NaiveBayes_model_python_1605362597414_1487")
top_xgb_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_xgb/XGBoost_model_python_1605362597414_1768")
# top_ensemble_ALL_MODELS_METALEARNER_auto_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_ALL_MODELS_METALEARNER_auto/stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_auto")
top_ensemble_ALL_MODELS_METALEARNER_deeplearning_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_ALL_MODELS_METALEARNER_deeplearning/stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_deeplearning")
top_ensemble_ALL_MODELS_METALEARNER_drf_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_ALL_MODELS_METALEARNER_drf/stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_drf")
top_ensemble_ALL_MODELS_METALEARNER_gbm_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_ALL_MODELS_METALEARNER_gbm/stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_gbm")
top_ensemble_ALL_MODELS_METALEARNER_glm_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_ALL_MODELS_METALEARNER_glm/stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_glm")
top_ensemble_ALL_MODELS_METALEARNER_naivebayes_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_ALL_MODELS_METALEARNER_naivebayes/stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_naivebayes")
top_ensemble_ALL_MODELS_METALEARNER_xgboost_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_ALL_MODELS_METALEARNER_xgboost/stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_xgboost")
# top_ensemble_CHECKPOINT_MODELS_METALEARNER_auto_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_MODELS_METALEARNER_auto/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_auto")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_deeplearning_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_MODELS_METALEARNER_deeplearning/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_drf_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_MODELS_METALEARNER_drf/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_drf")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_gbm_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_MODELS_METALEARNER_gbm/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_gbm")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_glm_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_MODELS_METALEARNER_glm/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_glm")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_naivebayes_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_MODELS_METALEARNER_naivebayes/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes")
top_ensemble_CHECKPOINT_MODELS_METALEARNER_xgboost_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_MODELS_METALEARNER_xgboost/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost")
# top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_auto_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_auto/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_auto")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_deeplearning_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_deeplearning/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_drf_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_drf/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_gbm_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_gbm/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_glm_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_glm/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_naivebayes_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_naivebayes/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes")
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_xgboost_pca= h2o.load_model(MODELS_LOCATION + "PCA300/top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_xgboost/stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost")
all_final_models = [
top_dl,
top_dl_pca,
top_drf,
top_drf_pca,
top_gbm,
top_gbm_pca,
top_glm,
top_glm_pca,
top_nb,
top_nb_pca,
top_xgb,
top_xgb_pca,
top_ensemble_ALL_MODELS_METALEARNER_deeplearning,
top_ensemble_ALL_MODELS_METALEARNER_deeplearning_pca,
top_ensemble_ALL_MODELS_METALEARNER_drf,
top_ensemble_ALL_MODELS_METALEARNER_drf_pca,
top_ensemble_ALL_MODELS_METALEARNER_gbm,
top_ensemble_ALL_MODELS_METALEARNER_gbm_pca,
top_ensemble_ALL_MODELS_METALEARNER_glm,
top_ensemble_ALL_MODELS_METALEARNER_glm_pca,
top_ensemble_ALL_MODELS_METALEARNER_naivebayes,
top_ensemble_ALL_MODELS_METALEARNER_naivebayes_pca,
top_ensemble_ALL_MODELS_METALEARNER_xgboost,
top_ensemble_ALL_MODELS_METALEARNER_xgboost_pca,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_deeplearning,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_deeplearning_pca,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_drf,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_drf_pca,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_gbm,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_gbm_pca,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_glm,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_glm_pca,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_naivebayes,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_naivebayes_pca,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_xgboost,
top_ensemble_CHECKPOINT_MODELS_METALEARNER_xgboost_pca,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_deeplearning,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_deeplearning_pca,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_drf,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_drf_pca,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_gbm,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_gbm_pca,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_glm,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_glm_pca,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_naivebayes,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_naivebayes_pca ,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_xgboost,
top_ensemble_CHECKPOINT_nogbm_MODELS_METALEARNER_xgboost_pca,
]
import time
for a_mdl in all_final_models:
xval_perf = a_mdl.model_performance(xval=True)
time.sleep(3)
valid_perf = a_mdl.model_performance(valid=True)
print('Model ID: ', a_mdl.model_id)
print('XVal Accuracy: ', xval_perf.accuracy()[0][1])
print('Training time (ms): ', a_mdl.run_time)
print('XVal AUC: ', xval_perf.auc())
time.sleep(3)
xval_perf.plot()
# print('Validation data AUC: ', valid_perf.auc())
# print('Validation data Accuracy: ', valid_perf.accuracy()[0][1])
time.sleep(3)
# valid_perf.plot()
print("-----------------------------")
Model ID: DeepLearning_model_python_1605423034668_341 XVal Accuracy: 0.8511856091578087 Training time (ms): 470361 XVal AUC: 0.9027482269503546
----------------------------- Model ID: DeepLearning_model_python_1605362597414_3340 XVal Accuracy: 0.7710547833197057 Training time (ms): 92889 XVal AUC: 0.8209884751773049
----------------------------- Model ID: DRF_model_python_1605423034668_386 XVal Accuracy: 0.856909239574816 Training time (ms): 1900902 XVal AUC: 0.9106057919621748
----------------------------- Model ID: DRF_model_python_1605362597414_2415 XVal Accuracy: 0.8348323793949305 Training time (ms): 4970 XVal AUC: 0.8844089834515367
----------------------------- Model ID: GBM_model_python_1605423034668_39 XVal Accuracy: 0.8479149632052331 Training time (ms): 71120 XVal AUC: 0.9054151891252956
----------------------------- Model ID: GBM_model_python_1605362597414_1525 XVal Accuracy: 0.8274734260016353 Training time (ms): 22756 XVal AUC: 0.8782254728132387
----------------------------- Model ID: GLM_model_python_1605423034668_20 XVal Accuracy: 0.8577269010629599 Training time (ms): 6048 XVal AUC: 0.9118498817966902
----------------------------- Model ID: GLM_model_python_1605362597414_1506 XVal Accuracy: 0.8086672117743254 Training time (ms): 216 XVal AUC: 0.8693484042553191
----------------------------- Model ID: NaiveBayes_model_python_1605423034668_1 XVal Accuracy: 0.7334423548650858 Training time (ms): 5211 XVal AUC: 0.7146276595744682
----------------------------- Model ID: NaiveBayes_model_python_1605362597414_1487 XVal Accuracy: 0.6745707277187244 Training time (ms): 130 XVal AUC: 0.62177304964539
----------------------------- Model ID: XGBoost_model_python_1605423034668_274 XVal Accuracy: 0.866721177432543 Training time (ms): 9638 XVal AUC: 0.9238918439716312
----------------------------- Model ID: XGBoost_model_python_1605362597414_1768 XVal Accuracy: 0.83892068683565 Training time (ms): 4348 XVal AUC: 0.8880614657210402
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_deeplearning XVal Accuracy: 0.8086672117743254 Training time (ms): 54195 XVal AUC: 0.8633983451536642
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_deeplearning XVal Accuracy: 0.7391659852820932 Training time (ms): 280253 XVal AUC: 0.7148921394799055
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_drf XVal Accuracy: 0.8683565004088307 Training time (ms): 16388 XVal AUC: 0.9260268912529551
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_drf XVal Accuracy: 0.8242027800490597 Training time (ms): 7132 XVal AUC: 0.880774231678487
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_gbm XVal Accuracy: 0.866721177432543 Training time (ms): 14372 XVal AUC: 0.9249010047281323
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_gbm XVal Accuracy: 0.8086672117743254 Training time (ms): 12611 XVal AUC: 0.8498921394799054
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_glm XVal Accuracy: 0.874897792313982 Training time (ms): 13136 XVal AUC: 0.9231072695035462
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_glm XVal Accuracy: 0.8405560098119379 Training time (ms): 685 XVal AUC: 0.9017479314420803
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_naivebayes XVal Accuracy: 0.8634505314799673 Training time (ms): 12785 XVal AUC: 0.9184530141843972
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_naivebayes XVal Accuracy: 0.8454619787408013 Training time (ms): 655 XVal AUC: 0.8984471040189125
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_xgboost XVal Accuracy: 0.8503679476696647 Training time (ms): 17681 XVal AUC: 0.9159352836879433
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_xgboost XVal Accuracy: 0.8250204415372036 Training time (ms): 5528 XVal AUC: 0.8839967494089835
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning XVal Accuracy: 0.7481602616516762 Training time (ms): 47602 XVal AUC: 0.8082505910165484
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning XVal Accuracy: 0.7269010629599346 Training time (ms): 279786 XVal AUC: 0.7530082742316785 ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_drf XVal Accuracy: 0.8659035159443991 Training time (ms): 11379 XVal AUC: 0.9191282505910165 ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_drf XVal Accuracy: 0.821749795584628 Training time (ms): 6638 XVal AUC: 0.8804920212765958 ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_gbm XVal Accuracy: 0.866721177432543 Training time (ms): 11276 XVal AUC: 0.9223847517730496 ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_gbm XVal Accuracy: 0.8062142273098937 Training time (ms): 11894 XVal AUC: 0.854057328605201 ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_glm XVal Accuracy: 0.8691741618969746 Training time (ms): 9850 XVal AUC: 0.9235534869976358
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_glm XVal Accuracy: 0.8413736713000818 Training time (ms): 597 XVal AUC: 0.8965174349881797
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes XVal Accuracy: 0.8618152085036794 Training time (ms): 9295 XVal AUC: 0.9221498226950354
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes XVal Accuracy: 0.8405560098119379 Training time (ms): 601 XVal AUC: 0.8956368203309693
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost XVal Accuracy: 0.8528209321340965 Training time (ms): 14069 XVal AUC: 0.9068676122931443
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost XVal Accuracy: 0.8233851185609158 Training time (ms): 5439 XVal AUC: 0.8786244089834515
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning XVal Accuracy: 0.8282910874897792 Training time (ms): 45733 XVal AUC: 0.8788741134751773
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning XVal Accuracy: 0.7031888798037612 Training time (ms): 338640 XVal AUC: 0.6323758865248227
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf XVal Accuracy: 0.8708094848732625 Training time (ms): 10068 XVal AUC: 0.9200059101654847
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf XVal Accuracy: 0.8160261651676206 Training time (ms): 6518 XVal AUC: 0.8663593380614658
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm XVal Accuracy: 0.8618152085036794 Training time (ms): 10280 XVal AUC: 0.9218543144208038
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm XVal Accuracy: 0.7923139820114473 Training time (ms): 11571 XVal AUC: 0.8412086288416076
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm XVal Accuracy: 0.866721177432543 Training time (ms): 8128 XVal AUC: 0.9242213356973995
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm XVal Accuracy: 0.8413736713000818 Training time (ms): 589 XVal AUC: 0.8967715721040189
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes XVal Accuracy: 0.8683565004088307 Training time (ms): 7497 XVal AUC: 0.9242686170212766
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes XVal Accuracy: 0.83892068683565 Training time (ms): 468 XVal AUC: 0.8972680260047281
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost XVal Accuracy: 0.8479149632052331 Training time (ms): 11570 XVal AUC: 0.9019355791962175
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost XVal Accuracy: 0.8143908421913328 Training time (ms): 6247 XVal AUC: 0.8744400118203309
-----------------------------
for a_mdl in all_final_models:
xval_perf = a_mdl.model_performance(xval=True)
time.sleep(3)
valid_perf = a_mdl.model_performance(valid=True)
print('Model ID: ', a_mdl.model_id)
# print('XVal Accuracy: ', xval_perf.accuracy()[0][1])
# print('Training time (ms): ', a_mdl.run_time)
# print('XVal AUC: ', xval_perf.auc())
time.sleep(1)
# xval_perf.plot()
print('Validation data AUC: ', valid_perf.auc())
print('Validation data Accuracy: ', valid_perf.accuracy()[0][1])
time.sleep(3)
valid_perf.plot()
print("-----------------------------")
Model ID: DeepLearning_model_python_1605423034668_341 Validation data AUC: 0.6180890841107924 Validation data Accuracy: 0.6786427145708582
----------------------------- Model ID: DeepLearning_model_python_1605362597414_3340 Validation data AUC: 0.5544977674519911 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: DRF_model_python_1605423034668_386 Validation data AUC: 0.6008821287254511 Validation data Accuracy: 0.6806387225548902
----------------------------- Model ID: DRF_model_python_1605362597414_2415 Validation data AUC: 0.5821051294151813 Validation data Accuracy: 0.6866267465069861
----------------------------- Model ID: GBM_model_python_1605423034668_39 Validation data AUC: 0.684212436925981 Validation data Accuracy: 0.7005988023952096
----------------------------- Model ID: GBM_model_python_1605362597414_1525 Validation data AUC: 0.5569935020147384 Validation data Accuracy: 0.6826347305389222
----------------------------- Model ID: GLM_model_python_1605423034668_20 Validation data AUC: 0.7188169310632736 Validation data Accuracy: 0.6906187624750499
----------------------------- Model ID: GLM_model_python_1605362597414_1506 Validation data AUC: 0.6266381094130032 Validation data Accuracy: 0.6786427145708582
----------------------------- Model ID: NaiveBayes_model_python_1605423034668_1 Validation data AUC: 0.6037862562166479 Validation data Accuracy: 0.6946107784431138
----------------------------- Model ID: NaiveBayes_model_python_1605362597414_1487 Validation data AUC: 0.5736650088938905 Validation data Accuracy: 0.6786427145708582
----------------------------- Model ID: XGBoost_model_python_1605423034668_274 Validation data AUC: 0.7019820670127418 Validation data Accuracy: 0.6926147704590818
----------------------------- Model ID: XGBoost_model_python_1605362597414_1768 Validation data AUC: 0.6006189421715613 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_deeplearning Validation data AUC: 0.7236450430173885 Validation data Accuracy: 0.6906187624750499
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_deeplearning Validation data AUC: 0.6143227937706466 Validation data Accuracy: 0.6766467065868264
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_drf Validation data AUC: 0.699050713326315 Validation data Accuracy: 0.6846307385229541
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_drf Validation data AUC: 0.5751079972410789 Validation data Accuracy: 0.6766467065868264
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_gbm Validation data AUC: 0.6927342360329619 Validation data Accuracy: 0.6906187624750499
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_gbm Validation data AUC: 0.5934584528260791 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_glm Validation data AUC: 0.7093875921152939 Validation data Accuracy: 0.6926147704590818
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_glm Validation data AUC: 0.6213834537336189 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_naivebayes Validation data AUC: 0.6918085453951428 Validation data Accuracy: 0.6946107784431138
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_naivebayes Validation data AUC: 0.6091044396849021 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_xgboost Validation data AUC: 0.650896649362907 Validation data Accuracy: 0.6826347305389222
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_xgboost Validation data AUC: 0.5999473626892221 Validation data Accuracy: 0.6786427145708582
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning Validation data AUC: 0.6794206265655063 Validation data Accuracy: 0.6946107784431138
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning Validation data AUC: 0.5992485570116528 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_drf Validation data AUC: 0.6710349584346753 Validation data Accuracy: 0.6866267465069861
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_drf Validation data AUC: 0.5761970450502777 Validation data Accuracy: 0.6806387225548902
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_gbm Validation data AUC: 0.6637927905035031 Validation data Accuracy: 0.6786427145708582
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_gbm Validation data AUC: 0.5840744908701492 Validation data Accuracy: 0.6806387225548902
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_glm Validation data AUC: 0.6710712600283153 Validation data Accuracy: 0.6946107784431138
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_glm Validation data AUC: 0.6081696736486731 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes Validation data AUC: 0.676806911823429 Validation data Accuracy: 0.6946107784431138
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes Validation data AUC: 0.5926507423675899 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost Validation data AUC: 0.6314480705702981 Validation data Accuracy: 0.6806387225548902
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost Validation data AUC: 0.5822231095945112 Validation data Accuracy: 0.6826347305389222
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning Validation data AUC: 0.6776055468835083 Validation data Accuracy: 0.6926147704590818
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning Validation data AUC: 0.6149943732529858 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf Validation data AUC: 0.6450157911932334 Validation data Accuracy: 0.6806387225548902
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf Validation data AUC: 0.577776164373616 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm Validation data AUC: 0.6578665553417795 Validation data Accuracy: 0.6826347305389222
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm Validation data AUC: 0.5681834682542564 Validation data Accuracy: 0.6786427145708582
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm Validation data AUC: 0.6919991287617526 Validation data Accuracy: 0.6966067864271457
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm Validation data AUC: 0.6095309834101716 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes Validation data AUC: 0.6688477874178677 Validation data Accuracy: 0.6846307385229541
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes Validation data AUC: 0.6125530910806984 Validation data Accuracy: 0.6746506986027944
----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost Validation data AUC: 0.6451428467709732 Validation data Accuracy: 0.6786427145708582
----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost Validation data AUC: 0.5808164228409627 Validation data Accuracy: 0.6746506986027944
-----------------------------
import time
all_final_model_metrics = []
for a_mdl in all_final_models:
xval_perf = a_mdl.model_performance(xval=True)
valid_perf = a_mdl.model_performance(valid=True)
perf_dict = {}
perf_dict['model_id'] = a_mdl.model_id
perf_dict['XVal Accuracy'] = xval_perf.accuracy()[0][1]
perf_dict['Training time (ms)'] = a_mdl.run_time
perf_dict['XVal AUC'] = xval_perf.auc()
perf_dict['Validation data AUC'] = valid_perf.auc()
perf_dict['Validation data Accuracy'] = valid_perf.accuracy()[0][1]
all_final_model_metrics.append([a_mdl, perf_dict])
print('Model ID: ', a_mdl.model_id)
print("-----------------------------")
Model ID: DeepLearning_model_python_1605423034668_341 ----------------------------- Model ID: DeepLearning_model_python_1605362597414_3340 ----------------------------- Model ID: DRF_model_python_1605423034668_386 ----------------------------- Model ID: DRF_model_python_1605362597414_2415 ----------------------------- Model ID: GBM_model_python_1605423034668_39 ----------------------------- Model ID: GBM_model_python_1605362597414_1525 ----------------------------- Model ID: GLM_model_python_1605423034668_20 ----------------------------- Model ID: GLM_model_python_1605362597414_1506 ----------------------------- Model ID: NaiveBayes_model_python_1605423034668_1 ----------------------------- Model ID: NaiveBayes_model_python_1605362597414_1487 ----------------------------- Model ID: XGBoost_model_python_1605423034668_274 ----------------------------- Model ID: XGBoost_model_python_1605362597414_1768 ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_deeplearning ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_deeplearning ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_drf ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_drf ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_gbm ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_gbm ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_glm ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_glm ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_naivebayes ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_naivebayes ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_xgboost ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_xgboost ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_drf ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_drf ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_gbm ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_gbm ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_glm ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_glm ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes ----------------------------- Model ID: stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost ----------------------------- Model ID: stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost -----------------------------
for mdl_tuple in all_final_model_metrics:
perf_dict = mdl_tuple[1]
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(perf_dict)
{ 'Training time (ms)': 470361, 'Validation data AUC': 0.6180890841107924, 'Validation data Accuracy': 0.6786427145708582, 'XVal AUC': 0.9027482269503546, 'XVal Accuracy': 0.8511856091578087, 'model_id': 'DeepLearning_model_python_1605423034668_341'} { 'Training time (ms)': 92889, 'Validation data AUC': 0.5544977674519911, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.8209884751773049, 'XVal Accuracy': 0.7710547833197057, 'model_id': 'DeepLearning_model_python_1605362597414_3340'} { 'Training time (ms)': 1900902, 'Validation data AUC': 0.6008821287254511, 'Validation data Accuracy': 0.6806387225548902, 'XVal AUC': 0.9106057919621748, 'XVal Accuracy': 0.856909239574816, 'model_id': 'DRF_model_python_1605423034668_386'} { 'Training time (ms)': 4970, 'Validation data AUC': 0.5821051294151813, 'Validation data Accuracy': 0.6866267465069861, 'XVal AUC': 0.8844089834515367, 'XVal Accuracy': 0.8348323793949305, 'model_id': 'DRF_model_python_1605362597414_2415'} { 'Training time (ms)': 71120, 'Validation data AUC': 0.684212436925981, 'Validation data Accuracy': 0.7005988023952096, 'XVal AUC': 0.9054151891252956, 'XVal Accuracy': 0.8479149632052331, 'model_id': 'GBM_model_python_1605423034668_39'} { 'Training time (ms)': 22756, 'Validation data AUC': 0.5569935020147384, 'Validation data Accuracy': 0.6826347305389222, 'XVal AUC': 0.8782254728132387, 'XVal Accuracy': 0.8274734260016353, 'model_id': 'GBM_model_python_1605362597414_1525'} { 'Training time (ms)': 6048, 'Validation data AUC': 0.7188169310632736, 'Validation data Accuracy': 0.6906187624750499, 'XVal AUC': 0.9118498817966902, 'XVal Accuracy': 0.8577269010629599, 'model_id': 'GLM_model_python_1605423034668_20'} { 'Training time (ms)': 216, 'Validation data AUC': 0.6266381094130032, 'Validation data Accuracy': 0.6786427145708582, 'XVal AUC': 0.8693484042553191, 'XVal Accuracy': 0.8086672117743254, 'model_id': 'GLM_model_python_1605362597414_1506'} { 'Training time (ms)': 5211, 'Validation data AUC': 0.6037862562166479, 'Validation data Accuracy': 0.6946107784431138, 'XVal AUC': 0.7146276595744682, 'XVal Accuracy': 0.7334423548650858, 'model_id': 'NaiveBayes_model_python_1605423034668_1'} { 'Training time (ms)': 130, 'Validation data AUC': 0.5736650088938905, 'Validation data Accuracy': 0.6786427145708582, 'XVal AUC': 0.62177304964539, 'XVal Accuracy': 0.6745707277187244, 'model_id': 'NaiveBayes_model_python_1605362597414_1487'} { 'Training time (ms)': 9638, 'Validation data AUC': 0.7019820670127418, 'Validation data Accuracy': 0.6926147704590818, 'XVal AUC': 0.9238918439716312, 'XVal Accuracy': 0.866721177432543, 'model_id': 'XGBoost_model_python_1605423034668_274'} { 'Training time (ms)': 4348, 'Validation data AUC': 0.6006189421715613, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.8880614657210402, 'XVal Accuracy': 0.83892068683565, 'model_id': 'XGBoost_model_python_1605362597414_1768'} { 'Training time (ms)': 54195, 'Validation data AUC': 0.7236450430173885, 'Validation data Accuracy': 0.6906187624750499, 'XVal AUC': 0.8633983451536642, 'XVal Accuracy': 0.8086672117743254, 'model_id': 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_deeplearning'} { 'Training time (ms)': 280253, 'Validation data AUC': 0.6143227937706466, 'Validation data Accuracy': 0.6766467065868264, 'XVal AUC': 0.7148921394799055, 'XVal Accuracy': 0.7391659852820932, 'model_id': 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_deeplearning'} { 'Training time (ms)': 16388, 'Validation data AUC': 0.699050713326315, 'Validation data Accuracy': 0.6846307385229541, 'XVal AUC': 0.9260268912529551, 'XVal Accuracy': 0.8683565004088307, 'model_id': 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_drf'} { 'Training time (ms)': 7132, 'Validation data AUC': 0.5751079972410789, 'Validation data Accuracy': 0.6766467065868264, 'XVal AUC': 0.880774231678487, 'XVal Accuracy': 0.8242027800490597, 'model_id': 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_drf'} { 'Training time (ms)': 14372, 'Validation data AUC': 0.6927342360329619, 'Validation data Accuracy': 0.6906187624750499, 'XVal AUC': 0.9249010047281323, 'XVal Accuracy': 0.866721177432543, 'model_id': 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_gbm'} { 'Training time (ms)': 12611, 'Validation data AUC': 0.5934584528260791, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.8498921394799054, 'XVal Accuracy': 0.8086672117743254, 'model_id': 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_gbm'} { 'Training time (ms)': 13136, 'Validation data AUC': 0.7093875921152939, 'Validation data Accuracy': 0.6926147704590818, 'XVal AUC': 0.9231072695035462, 'XVal Accuracy': 0.874897792313982, 'model_id': 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_glm'} { 'Training time (ms)': 685, 'Validation data AUC': 0.6213834537336189, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.9017479314420803, 'XVal Accuracy': 0.8405560098119379, 'model_id': 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_glm'} { 'Training time (ms)': 12785, 'Validation data AUC': 0.6918085453951428, 'Validation data Accuracy': 0.6946107784431138, 'XVal AUC': 0.9184530141843972, 'XVal Accuracy': 0.8634505314799673, 'model_id': 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_naivebayes'} { 'Training time (ms)': 655, 'Validation data AUC': 0.6091044396849021, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.8984471040189125, 'XVal Accuracy': 0.8454619787408013, 'model_id': 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_naivebayes'} { 'Training time (ms)': 17681, 'Validation data AUC': 0.650896649362907, 'Validation data Accuracy': 0.6826347305389222, 'XVal AUC': 0.9159352836879433, 'XVal Accuracy': 0.8503679476696647, 'model_id': 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_xgboost'} { 'Training time (ms)': 5528, 'Validation data AUC': 0.5999473626892221, 'Validation data Accuracy': 0.6786427145708582, 'XVal AUC': 0.8839967494089835, 'XVal Accuracy': 0.8250204415372036, 'model_id': 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_xgboost'} { 'Training time (ms)': 47602, 'Validation data AUC': 0.6794206265655063, 'Validation data Accuracy': 0.6946107784431138, 'XVal AUC': 0.8082505910165484, 'XVal Accuracy': 0.7481602616516762, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning'} { 'Training time (ms)': 279786, 'Validation data AUC': 0.5992485570116528, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.7530082742316785, 'XVal Accuracy': 0.7269010629599346, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning'} { 'Training time (ms)': 11379, 'Validation data AUC': 0.6710349584346753, 'Validation data Accuracy': 0.6866267465069861, 'XVal AUC': 0.9191282505910165, 'XVal Accuracy': 0.8659035159443991, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_drf'} { 'Training time (ms)': 6638, 'Validation data AUC': 0.5761970450502777, 'Validation data Accuracy': 0.6806387225548902, 'XVal AUC': 0.8804920212765958, 'XVal Accuracy': 0.821749795584628, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_drf'} { 'Training time (ms)': 11276, 'Validation data AUC': 0.6637927905035031, 'Validation data Accuracy': 0.6786427145708582, 'XVal AUC': 0.9223847517730496, 'XVal Accuracy': 0.866721177432543, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_gbm'} { 'Training time (ms)': 11894, 'Validation data AUC': 0.5840744908701492, 'Validation data Accuracy': 0.6806387225548902, 'XVal AUC': 0.854057328605201, 'XVal Accuracy': 0.8062142273098937, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_gbm'} { 'Training time (ms)': 9850, 'Validation data AUC': 0.6710712600283153, 'Validation data Accuracy': 0.6946107784431138, 'XVal AUC': 0.9235534869976358, 'XVal Accuracy': 0.8691741618969746, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_glm'} { 'Training time (ms)': 597, 'Validation data AUC': 0.6081696736486731, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.8965174349881797, 'XVal Accuracy': 0.8413736713000818, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_glm'} { 'Training time (ms)': 9295, 'Validation data AUC': 0.676806911823429, 'Validation data Accuracy': 0.6946107784431138, 'XVal AUC': 0.9221498226950354, 'XVal Accuracy': 0.8618152085036794, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes'} { 'Training time (ms)': 601, 'Validation data AUC': 0.5926507423675899, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.8956368203309693, 'XVal Accuracy': 0.8405560098119379, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes'} { 'Training time (ms)': 14069, 'Validation data AUC': 0.6314480705702981, 'Validation data Accuracy': 0.6806387225548902, 'XVal AUC': 0.9068676122931443, 'XVal Accuracy': 0.8528209321340965, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost'} { 'Training time (ms)': 5439, 'Validation data AUC': 0.5822231095945112, 'Validation data Accuracy': 0.6826347305389222, 'XVal AUC': 0.8786244089834515, 'XVal Accuracy': 0.8233851185609158, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost'} { 'Training time (ms)': 45733, 'Validation data AUC': 0.6776055468835083, 'Validation data Accuracy': 0.6926147704590818, 'XVal AUC': 0.8788741134751773, 'XVal Accuracy': 0.8282910874897792, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning'} { 'Training time (ms)': 338640, 'Validation data AUC': 0.6149943732529858, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.6323758865248227, 'XVal Accuracy': 0.7031888798037612, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning'} { 'Training time (ms)': 10068, 'Validation data AUC': 0.6450157911932334, 'Validation data Accuracy': 0.6806387225548902, 'XVal AUC': 0.9200059101654847, 'XVal Accuracy': 0.8708094848732625, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf'} { 'Training time (ms)': 6518, 'Validation data AUC': 0.577776164373616, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.8663593380614658, 'XVal Accuracy': 0.8160261651676206, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf'} { 'Training time (ms)': 10280, 'Validation data AUC': 0.6578665553417795, 'Validation data Accuracy': 0.6826347305389222, 'XVal AUC': 0.9218543144208038, 'XVal Accuracy': 0.8618152085036794, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm'} { 'Training time (ms)': 11571, 'Validation data AUC': 0.5681834682542564, 'Validation data Accuracy': 0.6786427145708582, 'XVal AUC': 0.8412086288416076, 'XVal Accuracy': 0.7923139820114473, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm'} { 'Training time (ms)': 8128, 'Validation data AUC': 0.6919991287617526, 'Validation data Accuracy': 0.6966067864271457, 'XVal AUC': 0.9242213356973995, 'XVal Accuracy': 0.866721177432543, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm'} { 'Training time (ms)': 589, 'Validation data AUC': 0.6095309834101716, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.8967715721040189, 'XVal Accuracy': 0.8413736713000818, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm'} { 'Training time (ms)': 7497, 'Validation data AUC': 0.6688477874178677, 'Validation data Accuracy': 0.6846307385229541, 'XVal AUC': 0.9242686170212766, 'XVal Accuracy': 0.8683565004088307, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes'} { 'Training time (ms)': 468, 'Validation data AUC': 0.6125530910806984, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.8972680260047281, 'XVal Accuracy': 0.83892068683565, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes'} { 'Training time (ms)': 11570, 'Validation data AUC': 0.6451428467709732, 'Validation data Accuracy': 0.6786427145708582, 'XVal AUC': 0.9019355791962175, 'XVal Accuracy': 0.8479149632052331, 'model_id': 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost'} { 'Training time (ms)': 6247, 'Validation data AUC': 0.5808164228409627, 'Validation data Accuracy': 0.6746506986027944, 'XVal AUC': 0.8744400118203309, 'XVal Accuracy': 0.8143908421913328, 'model_id': 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost'}
# TODO: MOVE THIS TO EDA
# libraries
import numpy as np
import matplotlib.pyplot as plt
plt.figure(
figsize=(10,10),
dpi=100)
# set width of bar
barWidth = 0.25
# set height of bar
bars1 = [0, 586, 12, 1, 1, 621, 1]
import json
import pprint
pca300_auc_list = []
for mdl_tuple in all_final_model_metrics[1:12:2]:
perf_dict = mdl_tuple[1]
pca300_auc_list.append(perf_dict['XVal AUC'])
pp.pprint(perf_dict['model_id'])
bars2 = [56, 222, 9, 0, 96, 116, 0]
# Set position of bar on X axis
r1 = np.arange(len(bars1))
r2 = [x + barWidth for x in r1]
# Make the plot
plt.bar(r1, bars1, width=barWidth, edgecolor='white', label='TBP')
plt.bar(r2, bars2, width=barWidth, edgecolor='white', label='HTBC')
# Add xticks on the middle of the group bars
# plt.xlabel('Cohort Drug Resistance Profile', fontweight='bold')
plt.xticks([r - barWidth for r in range(len(bars1))], ['Lineage-1', 'Lineage-2', 'Lineage-2; Lineage-3', 'Lineage-2; Lineage-4', 'Lineage-3', 'Lineage-4', 'Lineage-BOV'])
plt.xticks(rotation=45)
plt.ylabel('Number of samples')
# Create legend & Show graphic
plt.legend()
plt.show()
# NOTE: Performance of Top Models
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
import pandas as pd
import json
import pprint
fig = plt.figure(
figsize=(18,15),
dpi=100)
ax = fig.add_subplot(1, 1, 1)
# set width of bar
barWidth = 0.25
all_features_auc_list = []
for mdl_tuple in all_final_model_metrics[0:12:2]:
perf_dict = mdl_tuple[1]
all_features_auc_list.append(perf_dict['XVal AUC'])
pp.pprint(perf_dict['model_id'])
bars1 = all_features_auc_list
pca300_auc_list = []
for mdl_tuple in all_final_model_metrics[1:12:2]:
perf_dict = mdl_tuple[1]
pca300_auc_list.append(perf_dict['XVal AUC'])
pp.pprint(perf_dict['model_id'])
bars2 =pca300_auc_list
# HTBC-Resistant
# bars3 = [21, 156, 0, 7, 87, 67]
# HTBC-Sensitive
# bars4 = [35, 66, 0, 2, 9, 49]
# # Heights of bars1 + bars2
# bars12_h = np.add(bars1, bars2).tolist()
# bars34_h = np.add(bars3, bars4).tolist()
# Set position of bar on X axis
r1 = np.arange(len(bars1))
r2 = [x + barWidth for x in r1]
# Create brown bars
plt.bar(r1, bars1, edgecolor='white', width=barWidth, label='All Features')
# Create green bars (middle), on top of the firs ones
plt.bar(r1, bars2, bottom=0, edgecolor='white', width=barWidth, label='PCA300 Features')
# # Create brown bars
# plt.bar(r2, bars3, edgecolor='white', width=barWidth, label='HTBC-Resistant')
# # Create green bars (middle), on top of the firs ones
# plt.bar(r2, bars4, bottom=bars3, edgecolor='white', width=barWidth, label='HTBC-Sensitive')
# Custom X axis
plt.xticks([r1 for r1 in range(len(bars1))], ['DL', 'DRF', 'GBM', 'GLM', 'NB', 'XGB'])
# plt.xticks(rotation=45)
plt.ylabel('Area Under the ROC curve')
# plt.xlabel('Class of Algorithms')
ax.set_ylim(0.55, 1)
# Show graphic
plt.legend()
plt.show()
'DeepLearning_model_python_1605423034668_341' 'DRF_model_python_1605423034668_386' 'GBM_model_python_1605423034668_39' 'GLM_model_python_1605423034668_20' 'NaiveBayes_model_python_1605423034668_1' 'XGBoost_model_python_1605423034668_274' 'DeepLearning_model_python_1605362597414_3340' 'DRF_model_python_1605362597414_2415' 'GBM_model_python_1605362597414_1525' 'GLM_model_python_1605362597414_1506' 'NaiveBayes_model_python_1605362597414_1487' 'XGBoost_model_python_1605362597414_1768'
# NOTE: Performance of Top Models
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
import pandas as pd
import json
import pprint
fig = plt.figure(
figsize=(18,15),
dpi=100)
ax = fig.add_subplot(1, 1, 1)
# set width of bar
barWidth = 0.25
all_features_accuracy_list = []
for mdl_tuple in all_final_model_metrics[0:12:2]:
perf_dict = mdl_tuple[1]
all_features_accuracy_list.append(perf_dict['XVal Accuracy'])
pp.pprint(perf_dict['model_id'])
bars1 = all_features_accuracy_list
pca300_accuracy_list = []
for mdl_tuple in all_final_model_metrics[1:12:2]:
perf_dict = mdl_tuple[1]
pca300_accuracy_list.append(perf_dict['XVal Accuracy'])
pp.pprint(perf_dict['model_id'])
bars2 =pca300_accuracy_list
# Set position of bar on X axis
r1 = np.arange(len(bars1))
r2 = [x + barWidth for x in r1]
# Create brown bars
plt.bar(r1, bars1, edgecolor='white', width=barWidth, label='All Features')
# Create green bars (middle), on top of the firs ones
plt.bar(r1, bars2, bottom=0, edgecolor='white', width=barWidth, label='PCA300 Features')
# Custom X axis
plt.xticks([r1 for r1 in range(len(bars1))], ['DL', 'DRF', 'GBM', 'GLM', 'NB', 'XGB'])
# plt.xticks(rotation=45)
plt.ylabel('Accuracy')
# plt.xlabel('Class of Algorithms')
ax.set_ylim(0.6, 1)
# Show graphic
plt.legend()
plt.show()
'DeepLearning_model_python_1605423034668_341' 'DRF_model_python_1605423034668_386' 'GBM_model_python_1605423034668_39' 'GLM_model_python_1605423034668_20' 'NaiveBayes_model_python_1605423034668_1' 'XGBoost_model_python_1605423034668_274' 'DeepLearning_model_python_1605362597414_3340' 'DRF_model_python_1605362597414_2415' 'GBM_model_python_1605362597414_1525' 'GLM_model_python_1605362597414_1506' 'NaiveBayes_model_python_1605362597414_1487' 'XGBoost_model_python_1605362597414_1768'
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
import pandas as pd
fig = plt.figure(
figsize=(18,15),
dpi=100)
ax = fig.add_subplot(1, 1, 1)
# set width of bar
barWidth = 0.25
bars1 = list(map(lambda x: millis_to_minutes(x),
[5211, 6048, 71120, 1900902, 9638, 470361]
))
bars2 = list(map(lambda x: millis_to_minutes(x),
[130, 216, 22756, 4970, 4348, 92889]
))
# Set position of bar on X axis
r1 = np.arange(len(bars1))
r2 = [x + barWidth for x in r1]
# Create brown bars
plt.bar(r1, bars1, edgecolor='white', width=barWidth, label='All Features')
# Create green bars (middle), on top of the firs ones
plt.bar(r1, bars2, bottom=0, edgecolor='white', width=barWidth, label='PCA300 Features')
# Custom X axis
plt.xticks([r1 for r1 in range(len(bars1))], ['DL', 'DRF', 'GBM', 'GLM', 'NB', 'XGB'])
# plt.xticks(rotation=45)
plt.ylabel('Model Training Time (minutes)')
# plt.xlabel('Class of Algorithms')
# ax.set_ylim(0.5, 1)
# Show graphic
plt.legend()
plt.show()
# libraries
import numpy as np
import matplotlib.pyplot as plt
plt.figure(
figsize=(10,10),
dpi=100)
# set width of bar
barWidth = 0.25
all_features_time_list = []
for mdl_tuple in all_final_model_metrics[0:12:2]:
perf_dict = mdl_tuple[1]
all_features_time_list.append(perf_dict['Training time (ms)'])
pp.pprint(perf_dict['model_id'])
bars1 = all_features_time_list
pca300_time_list = []
for mdl_tuple in all_final_model_metrics[1:12:2]:
perf_dict = mdl_tuple[1]
pca300_time_list.append(perf_dict['Training time (ms)'])
pp.pprint(perf_dict['model_id'])
bars2 =pca300_time_list
bars1 = list(map(lambda x: millis_to_minutes(x),
all_features_time_list
))
bars2 = list(map(lambda x: millis_to_minutes(x),
pca300_time_list
))
# Set position of bar on X axis
r1 = np.arange(len(bars1))
r2 = [x + barWidth for x in r1]
r3 = [x + barWidth for x in r2]
# Make the plot
plt.bar(r1, bars1, width=barWidth, edgecolor='white', label='All Features')
plt.bar(r2, bars2, width=barWidth, edgecolor='white', label='PCA300 Features')
# Add xticks on the middle of the group bars
# plt.xlabel('Cohort Drug Resistance Profile', fontweight='bold')
plt.xticks([r + 0.10 for r in range(len(bars1))], ['DL', 'DRF', 'GBM', 'GLM', 'NB', 'XGB'])
# plt.xticks(rotation=45)
plt.ylabel('Model Training Time (minutes)')
# pyplot.xlabel('Statistics for samples with 500+ mutations')
# Create legend & Show graphic
plt.legend()
plt.show()
'DeepLearning_model_python_1605423034668_341' 'DRF_model_python_1605423034668_386' 'GBM_model_python_1605423034668_39' 'GLM_model_python_1605423034668_20' 'NaiveBayes_model_python_1605423034668_1' 'XGBoost_model_python_1605423034668_274' 'DeepLearning_model_python_1605362597414_3340' 'DRF_model_python_1605362597414_2415' 'GBM_model_python_1605362597414_1525' 'GLM_model_python_1605362597414_1506' 'NaiveBayes_model_python_1605362597414_1487' 'XGBoost_model_python_1605362597414_1768'
# NOTE: Performance of Top Models
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
import pandas as pd
fig = plt.figure(
figsize=(15,15),
dpi=100)
ax = fig.add_subplot(1, 1, 1)
# set width of bar
barWidth = 0.25
bars1 = []
for mdl_tuple in all_final_model_metrics[12:24:2]:
perf_dict = mdl_tuple[1]
bars1.append(perf_dict['XVal AUC'])
pp.pprint(perf_dict['model_id'])
bars2 = []
for mdl_tuple in all_final_model_metrics[13:24:2]:
perf_dict = mdl_tuple[1]
bars2.append(perf_dict['XVal AUC'])
pp.pprint(perf_dict['model_id'])
#---------------------
# NOTE: Performance of Stacked Models with Checkpoint Base Learners
bars3 = []
for mdl_tuple in all_final_model_metrics[24:36:2]:
perf_dict = mdl_tuple[1]
bars3.append(perf_dict['XVal AUC'])
pp.pprint(perf_dict['model_id'])
bars4 = []
for mdl_tuple in all_final_model_metrics[25:36:2]:
perf_dict = mdl_tuple[1]
bars4.append(perf_dict['XVal AUC'])
pp.pprint(perf_dict['model_id'])
#---------------------
# NOTE: Performance of Stacked Models with Minimal Checkpoint Base Learners
bars5 = []
for mdl_tuple in all_final_model_metrics[36:48:2]:
perf_dict = mdl_tuple[1]
bars5.append(perf_dict['XVal AUC'])
pp.pprint(perf_dict['model_id'])
bars6 = []
for mdl_tuple in all_final_model_metrics[37:48:2]:
perf_dict = mdl_tuple[1]
bars6.append(perf_dict['XVal AUC'])
pp.pprint(perf_dict['model_id'])
#---------------------
# Set position of bar on X axis
r1 = np.arange(len(bars3))
r2 = [x + barWidth for x in r1]
r3 = [x + barWidth for x in r2]
plt.bar(r1, bars1, edgecolor='white', width=barWidth, label='Base_1 - All Features')
plt.bar(r1, bars2, bottom=0, edgecolor='white', width=barWidth, label='Base_1 - PCA300 Features')
plt.bar(r2, bars3, edgecolor='white', width=barWidth, label='Base_2 - All Features')
plt.bar(r2, bars4, bottom=0, edgecolor='white', width=barWidth, label='Base_2 - PCA300 Features')
plt.bar(r3, bars5, edgecolor='white', width=barWidth, label='Base_3 - All Features')
plt.bar(r3, bars6, bottom=0, edgecolor='white', width=barWidth, label='Base_3 - PCA300 Features')
plt.xticks([r1 + barWidth for r1 in range(len(bars3))], ['DL', 'DRF', 'GBM', 'GLM', 'NB', 'XGB'], fontsize= 20)
ax.set_ylim(0.6, 1)
plt.ylabel('Area Under the ROC curve - Training Dataset', fontsize= 20)
# Show graphic
plt.legend(fontsize= 14)
plt.show()
'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_xgboost' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost'
# NOTE: Performance of Top Models
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
import pandas as pd
fig = plt.figure(
figsize=(15,15),
dpi=100)
ax = fig.add_subplot(1, 1, 1)
# set width of bar
barWidth = 0.25
bars1 = []
for mdl_tuple in all_final_model_metrics[12:24:2]:
perf_dict = mdl_tuple[1]
bars1.append(perf_dict['XVal Accuracy'])
pp.pprint(perf_dict['model_id'])
bars2 = []
for mdl_tuple in all_final_model_metrics[13:24:2]:
perf_dict = mdl_tuple[1]
bars2.append(perf_dict['XVal Accuracy'])
pp.pprint(perf_dict['model_id'])
#---------------------
# NOTE: Performance of Stacked Models with Checkpoint Base Learners
bars3 = []
for mdl_tuple in all_final_model_metrics[24:36:2]:
perf_dict = mdl_tuple[1]
bars3.append(perf_dict['XVal Accuracy'])
pp.pprint(perf_dict['model_id'])
bars4 = []
for mdl_tuple in all_final_model_metrics[25:36:2]:
perf_dict = mdl_tuple[1]
bars4.append(perf_dict['XVal Accuracy'])
pp.pprint(perf_dict['model_id'])
#---------------------
#---------------------
# NOTE: Performance of Stacked Models with Minimal Checkpoint Base Learners
bars5 = []
for mdl_tuple in all_final_model_metrics[36:48:2]:
perf_dict = mdl_tuple[1]
bars5.append(perf_dict['XVal Accuracy'])
pp.pprint(perf_dict['model_id'])
bars6 = []
for mdl_tuple in all_final_model_metrics[37:48:2]:
perf_dict = mdl_tuple[1]
bars6.append(perf_dict['XVal Accuracy'])
pp.pprint(perf_dict['model_id'])
#---------------------
# Set position of bar on X axis
r1 = np.arange(len(bars3))
r2 = [x + barWidth for x in r1]
r3 = [x + barWidth for x in r2]
plt.bar(r1, bars1, edgecolor='white', width=barWidth, label='Base_1 - All Features')
plt.bar(r1, bars2, bottom=0, edgecolor='white', width=barWidth, label='Base_1 - PCA300 Features')
plt.bar(r2, bars3, edgecolor='white', width=barWidth, label='Base_2 - All Features')
plt.bar(r2, bars4, bottom=0, edgecolor='white', width=barWidth, label='Base_2 - PCA300 Features')
plt.bar(r3, bars5, edgecolor='white', width=barWidth, label='Base_3 - All Features')
plt.bar(r3, bars6, bottom=0, edgecolor='white', width=barWidth, label='Base_3 - PCA300 Features')
# Custom X axis
plt.xticks([r1 + barWidth for r1 in range(len(bars3))], ['DL', 'DRF', 'GBM', 'GLM', 'NB', 'XGB'], fontsize= 20)
ax.set_ylim(0.65, 1)
plt.ylabel('Accuracy - Training Dataset', fontsize= 20)
# Show graphic
plt.legend(fontsize= 14)
plt.show()
'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_xgboost' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost'
# NOTE: Performance of Top Models
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
import pandas as pd
METRIC = 'Validation data AUC'
fig = plt.figure(
figsize=(15,15),
dpi=100)
ax = fig.add_subplot(1, 1, 1)
# set width of bar
barWidth = 0.25
bars1 = []
for mdl_tuple in all_final_model_metrics[12:24:2]:
perf_dict = mdl_tuple[1]
bars1.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
bars2 = []
for mdl_tuple in all_final_model_metrics[13:24:2]:
perf_dict = mdl_tuple[1]
bars2.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
#---------------------
# NOTE: Performance of Stacked Models with Checkpoint Base Learners
bars3 = []
for mdl_tuple in all_final_model_metrics[24:36:2]:
perf_dict = mdl_tuple[1]
bars3.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
bars4 = []
for mdl_tuple in all_final_model_metrics[25:36:2]:
perf_dict = mdl_tuple[1]
bars4.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
#---------------------
# NOTE: Performance of Stacked Models with Minimal Checkpoint Base Learners
bars5 = []
for mdl_tuple in all_final_model_metrics[36:48:2]:
perf_dict = mdl_tuple[1]
bars5.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
bars6 = []
for mdl_tuple in all_final_model_metrics[37:48:2]:
perf_dict = mdl_tuple[1]
bars6.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
#---------------------
# Set position of bar on X axis
r1 = np.arange(len(bars3))
r2 = [x + barWidth for x in r1]
r3 = [x + barWidth for x in r2]
plt.bar(r1, bars1, edgecolor='white', width=barWidth, label='Base_1 - All Features')
plt.bar(r1, bars2, bottom=0, edgecolor='white', width=barWidth, label='Base_1 - PCA300 Features')
plt.bar(r2, bars3, edgecolor='white', width=barWidth, label='Base_2 - All Features')
plt.bar(r2, bars4, bottom=0, edgecolor='white', width=barWidth, label='Base_2 - PCA300 Features')
plt.bar(r3, bars5, edgecolor='white', width=barWidth, label='Base_3 - All Features')
plt.bar(r3, bars6, bottom=0, edgecolor='white', width=barWidth, label='Base_3 - PCA300 Features')
plt.xticks([r1 + barWidth for r1 in range(len(bars3))], ['DL', 'DRF', 'GBM', 'GLM', 'NB', 'XGB'], fontsize= 20)
ax.set_ylim(0.5, 1)
plt.ylabel('Area Under the ROC curve - Validation Dataset', fontsize= 20)
# Show graphic
plt.legend(fontsize= 14)
plt.show()
'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_xgboost' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost'
# NOTE: Performance of Top Models
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
import pandas as pd
METRIC = 'Validation data Accuracy'
fig = plt.figure(
figsize=(15,15),
dpi=100)
ax = fig.add_subplot(1, 1, 1)
# set width of bar
barWidth = 0.25
bars1 = []
for mdl_tuple in all_final_model_metrics[12:24:2]:
perf_dict = mdl_tuple[1]
bars1.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
bars2 = []
for mdl_tuple in all_final_model_metrics[13:24:2]:
perf_dict = mdl_tuple[1]
bars2.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
#---------------------
# NOTE: Performance of Stacked Models with Checkpoint Base Learners
bars3 = []
for mdl_tuple in all_final_model_metrics[24:36:2]:
perf_dict = mdl_tuple[1]
bars3.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
bars4 = []
for mdl_tuple in all_final_model_metrics[25:36:2]:
perf_dict = mdl_tuple[1]
bars4.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
#---------------------
#---------------------
# NOTE: Performance of Stacked Models with Minimal Checkpoint Base Learners
bars5 = []
for mdl_tuple in all_final_model_metrics[36:48:2]:
perf_dict = mdl_tuple[1]
bars5.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
bars6 = []
for mdl_tuple in all_final_model_metrics[37:48:2]:
perf_dict = mdl_tuple[1]
bars6.append(perf_dict[METRIC])
pp.pprint(perf_dict['model_id'])
#---------------------
# Set position of bar on X axis
r1 = np.arange(len(bars3))
r2 = [x + barWidth for x in r1]
r3 = [x + barWidth for x in r2]
plt.bar(r1, bars1, edgecolor='white', width=barWidth, label='Base_1 - All Features')
plt.bar(r1, bars2, bottom=0, edgecolor='white', width=barWidth, label='Base_1 - PCA300 Features')
plt.bar(r2, bars3, edgecolor='white', width=barWidth, label='Base_2 - All Features')
plt.bar(r2, bars4, bottom=0, edgecolor='white', width=barWidth, label='Base_2 - PCA300 Features')
plt.bar(r3, bars5, edgecolor='white', width=barWidth, label='Base_3 - All Features')
plt.bar(r3, bars6, bottom=0, edgecolor='white', width=barWidth, label='Base_3 - PCA300 Features')
# Custom X axis
plt.xticks([r1 + barWidth for r1 in range(len(bars3))], ['DL', 'DRF', 'GBM', 'GLM', 'NB', 'XGB'], fontsize= 20)
ax.set_ylim(0.50, 1)
plt.ylabel('Accuracy - Va Dataset', fontsize= 20)
# Show graphic
plt.legend(fontsize= 14)
plt.show()
'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_ALL_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_ALL_MODELS_metalearner_xgboost' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_MODELS_metalearner_xgboost' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes' 'stacked_ensemble_ALL_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_deeplearning' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_drf' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_gbm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_glm' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_naivebayes' 'stacked_ensemble_PCA300_FEATURES_CHECKPOINT_nogbm_MODELS_metalearner_xgboost'