using ScikitLearn.GridSearch: GridSearchCV, RandomizedSearchCV
################################ Hyperparameter Optimization #################################
function hyperparameter_optimization_MLP(df, train_pmids)
  hidden_layers = []
  for x = 1:100
    i = rand(20:100:4000)
    j = rand(20:100:4000)
    k = rand(20:100:4000)
    vectori = (i,)
    vectorj = (i, j,)
    vectork = (i, j, k)
    push!(hidden_layers, vectori)
    push!(hidden_layers, vectorj)
    push!(hidden_layers, vectork)
  end
  train_features, train_labels, train_genes = get_features_labels(df, train_pmids) #gets train_features and train_labels for the training dataset
  clf = MLPClassifier() #classifier that is being optimized
  param_grid_MLP = Dict(:hidden_layer_sizes=> vcat([(20,)], hidden_layers), :solver => [ "adam", "sgd", "lbgfs"], :learning_rate=> ["constant", "invscaling", "adaptive"],
  :alpha => collect(0.0001:0.0001:0.05), :activation => ["identity","logistic", "tanh", "relu"], :learning_rate_init => collect(0.001:0.001:10.0)) # gives a grid of potential hyperparameter values
  n_iter_search = 1000 #Number of sample combinations to test from
  random_search = RandomizedSearchCV(clf, param_grid_MLP, n_iter=n_iter_search, random_state=MersenneTwister(42))
  fit!(random_search, train_features, train_labels)
  opt_param_MLP = random_search.best_params_
end


function hyperparameter_optimization_RF(df, train_pmids)
  train_features, train_labels, train_genes = get_features_labels(df, train_pmids)
  clf = RandomForestClassifier()
  max_feature_range = vcat(["auto", "sqrt"], collect(0.2:0.05:0.9))
  param_grid_RF = Dict(:criterion => ["gini", "entropy"], :max_features=> max_feature_range, :min_impurity_decrease=>collect(0.000000001:0.000000001:0.01),  :min_samples_split => [2, 3, 4, 8],
  :min_samples_leaf=> [1, 2, 4, 8, 10])
  n_iter_search = 1000
  random_search = RandomizedSearchCV(clf, param_grid_RF, n_iter=n_iter_search, random_state=MersenneTwister(42))
  fit!(random_search, train_features, train_labels)
  opt_param_RF = random_search.best_params_
end

function hyperparameter_optimization_LR(df, train_pmids)
  train_features, train_labels, train_genes = get_features_labels(df, train_pmids)
  clf = LogisticRegression()
  c_par= vcat(collect(0.001:0.05:1.001), collect(10:10:100))
  param_grid_LR = Dict(:C => c_par, :solver => ["saga", "sag", "liblinear"], :max_iter =>collect(50:1000), :multi_class => ["ovr", "multinomial", "auto"])
  n_iter_search = 1000
  random_search = RandomizedSearchCV(clf, param_grid_LR, n_iter=n_iter_search, random_state=MersenneTwister(42))
  fit!(random_search, train_features, train_labels)
  opt_param_LR = random_search.best_params_
end