# imports
import time
import datetime
import os
import pathlib as path
import math
import random
import warnings
import json
import itertools as it
import gc
from datetime import datetime
import matplotlib.pyplot as plt
import basic_funcs as bf

import numpy as np
from numpy.linalg import inv
import pandas as pd

import scipy as sp
from scipy.stats import norm
from scipy.optimize import curve_fit
from scipy.spatial.transform import Rotation as R
from scipy.signal import savgol_filter

import statsmodels.api as sm
import statsmodels.formula.api as smf

from statsmodels.stats.anova import AnovaRM
from IPython.display import display, HTML

#warnings.filterwarnings('ignore')
np.set_printoptions(suppress=True)
pd.set_option('display.float_format', lambda x: '%.3f' % x)

# final funcs
def print_df(df):
    display(HTML(df.to_html()))

def df_from_file(d_file):
    df = bf.df_from_log(d_file)
    df = bf.add_recode_resp(df, "Response", {"Left": 0, "Right": 1})
    df = bf.add_recode_resp(df, "TargetHoriPos", {-3: -1.37, -2: -0.92, -1: -0.46, 0: 0, 1: 0.46, 2: 0.92, 3: 1.37}) # here
    
    # add dfile name as col
    version = d_file.split("\\")[-1].split(".")[0].split("_")[1:6]
    for idx, ele in enumerate(version):
        if idx == 0:
            version_str = ele 
        else:
            version_str = version_str + "_" + ele
    
    df["Version"] = [version_str] * df.shape[0]
    
    # special serial dependency stuff
    # df = serial_dependency_cols(df, "TargetSide", "TrialGain")
    # df = gain_categorize(df, "SD_Gain")
    # df = df.drop([df.index[0]]) # remove 1st trial since SD
    
    file_col = [d_file] * df.shape[0]
    df = df.assign(Filename = file_col)
    
    return df

def df_multi_files(f_list):
    df = df_from_file(f_list[0]) #bf.df_from_log(f_list[0])
    
    if len(f_list) > 1:
        for f_p in f_list[1:]:
            df_app = df_from_file(f_p) #bf.df_from_log(f_p)
            df = pd.concat([df, df_app]) # df = df.append(df_app, ignore_index=True)
        
    return df

def check_for_multi_session(d_files):
    ret_dict = {}
    
    uni_ids = []
    
    for d_file in d_files:
        f_name = d_file.split("\\")[-1]
        top_id = f_name.split("_")[0]
        uni_ids.append(top_id)
        
    uni_ids = list(set(uni_ids))
    
    for subj in uni_ids:
        ret_dict[subj] = []
            
        for d_file in d_files:
            f_name = d_file.split("\\")[-1]
            i_id = f_name.split("_")[0]

            if subj == i_id:
                ret_dict[subj].append(d_file)
            
    return ret_dict

def serial_dependency_cols(df, side_col, gain_col):
    # gains
    ada_gains = list(df[gain_col])
    ada_gains.insert(0, False) # shift lists 1 forward + mark first trial as unusable
    del ada_gains[-1] # remove last element
    df = df.assign(SD_Gain = ada_gains)
    
    # sides (goal: check if prior same side as trial n)
    # same side as prior or not, absolute side irrelevant
    sd_sides = []
    ada_sides = list(df[side_col])
    
    for idx, side in enumerate(ada_sides):
        if idx == 0:
            sd_sides.append(False)
        elif side == ada_sides[idx - 1]:
            sd_sides.append("Congruent")
        elif side != ada_sides[idx - 1]:
            sd_sides.append("Incongruent")
            
    df = df.assign(SD_Side = sd_sides)
    
    return df

def gain_categorize(df, gain_col):
    ada_gains = list(df[gain_col])
    cat_col = [False]
    
    for gain in ada_gains[1:]:
        if gain == 0:
            cat_col.append("Baseline")
        elif gain < 0:
            cat_col.append("Negative")
        elif gain > 0:
            cat_col.append("Positive")
            
    df = df.assign(SD_Gain = cat_col)
    
    return df

def prob_stim_lvl(df,lvl_col,av_col):
    stim_lvls = np.sort(list(set(list(df[lvl_col]))))
    
    lvl_arr = []
    resp_arr = []
    points_arr = []
    
    for lvl in stim_lvls:
        lvl_arr.append(lvl)
        arr = []
        counter = 0
        
        for idx,row in df.iterrows():
            if row[lvl_col] == lvl:
                arr.append(row[av_col])
                counter = counter + 1
                
        resp_arr.append(bf.mean(arr))
        points_arr.append(counter)
                
    ret_dict = {"Lvl_Arr": lvl_arr, 
                "Resp_Arr": resp_arr, 
                "Num_Points": points_arr}
          
    return ret_dict

def parabola_func(x, a, b, c):
    return a * np.power(x, 2) + b*x + c # np.exp(x * 2)

def calc_r2(y, y_fit):
    ss_tot = np.sum([np.power(np.mean(y) - y_val, 2) for y_val in y])
    ss_res = np.sum([np.power(y_fit_val - y_val, 2) for y_val, y_fit_val in zip(y, y_fit)])
    r_2 = 1 - (ss_res / ss_tot)
    
    return r_2

def fxn():
    warnings.warn("deprecated", RuntimeWarning) #DeprecationWarning
    
def get_parabola_vals(a, b, c):
    axis = - (b / 2*a)
    focal_length = 1 / (4*a)
    vertex = [-(b / (2 * a)), ((4 * a * c) - np.power(b, 2)) / 4 * a]
    focus = [-(b / (2 * a)), ((4 * a * c) - np.power(b,2) + 1) / 4 * a]
    directrix = (4 * a * c - np.power(b, 2) - 1) / (4 * a)
    
    return [axis, focal_length, vertex, focus, directrix]

def parabola_pses(a,b,c):
    # a * np.power(x, 2) + b*x + c
    p = b / a
    q = (c - 0.5) / a

    x1 = -(p / 2) + np.sqrt(np.power((p / 2), 2) - q)
    x2 = -(p / 2) - np.sqrt(np.power((p / 2), 2) - q)
    
    return [x1,x2]

def balance_data(df, sub_col, cols):
    col_lvls = [list(set(list(df[col]))) for col in cols]
    combis_l = [x for x in it.product(*col_lvls)]
    
    subs = np.sort(list(set(df[sub_col])))
    inc_subs = []
    for sub in subs:
        sub_df = df[df[sub_col] == sub]
        
        if sub_df.shape[0] == len(combis_l):
            inc_subs.append(sub)
        
    return df[df[sub_col].isin(inc_subs)]

# head funcs
def get_time_col(df):
    cols = list(df.columns)
    return [col for col in cols if "Time" in col][0]

def rot_from_str_rot(vector):
    vec_arr = vector.split(" ")
    vec_vals = [x.split("=")[1] for x in vec_arr]
    pos_vec = [float(x) for x in vec_vals]
    return pos_vec

# pf funcs

def balance_data(df, sub_col, cols):
    col_lvls = [list(set(list(df[col]))) for col in cols]
    combis_l = [x for x in it.product(*col_lvls)]
    
    subs = np.sort(list(set(df[sub_col])))
    inc_subs = []
    for sub in subs:
        sub_df = df[df[sub_col] == sub]
        
        if sub_df.shape[0] == len(combis_l):
            inc_subs.append(sub)
        
    return df[df[sub_col].isin(inc_subs)]

def pos_from_matrix(list_saved_as_str):
    list_mat = str_mat_to_mat(list_saved_as_str)
    conv_mat = convert_steam_vr_matrix(list_mat)
    inv_mat = inv(conv_mat)
    
    matrix33 = [x[0:3] for idx, x in enumerate(inv_mat) if idx != 3] # remove scaling value + 4th row
    np_mat = np.matrix(matrix33) #* 100 # ue4 units for 1
    
    # offset to tracking pos 0,0,0 unknown, therefore only an estimate
    # doesnt seem to be possible to get exact UE coordinates outside of UE
    # and in UE it does not seem to be possible to retrieve the coordinates used in game with more than 90hz
    estimate = [-inv_mat[3][2], inv_mat[3][0], inv_mat[3][1]]
    estimate = [x * 100 for x in estimate] # 100 for world scale, 100 ue units = 1m
    
    return estimate

# analysis funcs
def dep_ttest(comp_str, arr1, arr2, sided, num_corr, b_print):
    t, p = sp.stats.ttest_rel(arr1, arr2, alternative=sided) #greater
    
    if b_print:
        print("{}: p = {}, corrected: {}".format(comp_str, np.round(p, 3), np.round(p * num_corr, 3)))
        
    return t, p, len(arr1) - 1

def one_sample_ttest(comp_str, arr, value, sided, num_corr, b_print):
    t, p = sp.stats.ttest_1samp(arr, 0, axis=0, alternative=sided) # greater
    
    if b_print:
        print("{}: t({}) = {}, p = {}, corrected: {}".format(comp_str, len(arr) - 1, np.round(t, 2), np.round(p, 3), np.round(p * 5, 3)))
        
    return t, p, len(arr) - 1

# excluded subs
    # cs19 (half bad, complete exclusion)
    # at23 (1 too bad, maybe stay in)

# write out which files were finally used for analysis
all_files = np.concatenate((d_files, d_timing_files, d_eye_files, d_head_files, d_head_app_files), axis=0) #  +  +  +  + 
# np.sort(all_files)

all_files = [x.split("\\")[-1] for x in all_files]

# pd.DataFrame({"Files": np.sort(all_files)}).to_csv(r"D:\HeadRotationObjectLocalization\files_to_use.csv")
# all_files

# remove trials which did not fixate at correct times
# remove these files from pf data
# recalc pfs

### to dos
##### exclude trials based on head data
##### exclude trials based on eye data
#### pf analysis
#### descriptive stuff for head

#### used AHR1 as template

# funcs
def preprocess_head_files(d_timing_files, data_p):
    keys = ["Trial", "PreStart", "Start", "End", "FirstStimulusOnset", "FirstStimulusOffset", "SecondStimulusOnset", "SecondStimulusOffset"]
    
    for idx, file_path in enumerate(d_timing_files):
        file_path_comps = file_path.split("\\")
        
        # print(file_path_comps[-1].split("_"))
        # ['AD21', 'Eye', 'Movement', 'NoGain', 'PermanentTargets', 'Gabor', '19', '6', '2023', '10', '16', 'timings.csv']
        # ['ST07', 'Eye', 'Movement', 'NoGain', 'PermanentTargets', 'Gabor', '26', '6', '2023', '10', '35', 'timings.csv']
        
        # try:
        sub, run, move, gain, target, gabor, day, month, year, hour, minute, end = file_path_comps[-1].split("_")
        
        file_name = data_p.joinpath("{}_{}_{}_{}_{}_{}.{}.{}_timings.pickle".format(sub, run, move, target, gabor, day, month, year))
        file_name = str(file_name).replace("Raw", "Preprocessed")#.replace("csv", "pickle")
        
        if not os.path.exists(file_name):
            df_timing = pd.read_csv(file_path, index_col=0)
            
            if df_timing.shape[0] > 0:
                df_timing_preprocessed = pd.DataFrame()
                time_c = "StimulusTime"
                
                trial_num = 1
                trials = []
                for idx, row in df_timing.iterrows():
                    trials.append(trial_num)
                    
                    if row["StimulusType"] == "SecondStimulus" and row["StimulusEvent"] == "StimulusOffset":
                        trial_num = trial_num + 1
                    
                df_timing["Trial"] = trials
                
                # print_df(df_timing)
                
                last_trial = max(trials)
                trials = list(range(1, last_trial + 1))
                
                for trial in trials:
                    trial_dict = {}
                    for key in keys:
                        trial_dict[key] = []
                        
                    trial_df = df_timing.query('Trial == @trial')
                    
                    # print_df(trial_df)
                
                    try:
                        start = trial_df.query('StimulusType == "FirstStimulus" & StimulusEvent == "StimulusOnset"')[time_c].iloc[-1] # select last as trial could have been restarted
                        pre_start = start - (200 * 1_000_000)
                        first_onset = start
                        first_offset = trial_df.query('StimulusType == "FirstStimulus" & StimulusEvent == "StimulusOffset"')[time_c].iloc[-1]
                        second_onset = trial_df.query('StimulusType == "SecondStimulus" & StimulusEvent == "StimulusOnset"')[time_c].iloc[-1]
                        end = trial_df.query('StimulusType == "SecondStimulus" & StimulusEvent == "StimulusOffset"')[time_c].iloc[-1]
                        second_offset = end
                        
                        # if trial_df.query('StimulusType == "FirstStimulus" & StimulusEvent == "StimulusOnset"').shape[0] > 1:
                        #     print(trial)
                        #     print_df(trial_df.query('StimulusType == "FirstStimulus" & StimulusEvent == "StimulusOnset"'))
                        
                        vals = [trial, pre_start, start, end, first_onset, first_offset, second_onset, second_offset]
                        
                        for key, val in zip(keys, vals):
                            trial_dict[key].extend([val])
                            
                        df_timing_preprocessed = pd.concat([df_timing_preprocessed, pd.DataFrame(trial_dict)])
                    except:
                        print(sub, run, move, target, gabor, trial)
            else:
                print(sub, run, move, target, gabor, "no trials")
                
            if df_timing.shape[0] > 0:
                df_timing_preprocessed = df_timing_preprocessed.reset_index(drop=True)
                df_timing_preprocessed.to_pickle(file_name)
                # print_df(df_timing_preprocessed)
                    
        # except:
        #     print("file info: {}".format(file_path_comps[-1].split("_")))

# preprocess timing files 
timings = True 

if timings:
    preprocess_head_files(d_timing_files, data_p.parent.joinpath("preprocessed"))
else:
    print("Timing preprocessing done")

# head tracking data to yaw
def str_mat_to_mat(str_mat):
    return json.loads(str_mat)

def convert_steam_vr_matrix(pose):
    return np.array((
        (pose[0][0], pose[1][0], pose[2][0], 0.0),
        (pose[0][1], pose[1][1], pose[2][1], 0.0),
        (pose[0][2], pose[1][2], pose[2][2], 0.0),
        (pose[0][3], pose[1][3], pose[2][3], 1.0),
    ), dtype=np.float32)

def vals_from_matrix(list_saved_as_str):
    list_mat = str_mat_to_mat(list_saved_as_str)
    
    # https://github.com/cmbruns/pyopenvr/blob/6c153f986d46b61a3e136c144011695841b56502/src/samples/glfw/hellovr_glfw.py#L880
    conv_mat = convert_steam_vr_matrix(list_mat)
    inv_mat = inv(conv_mat)
    
    matrix33 = [x[0:3] for idx, x in enumerate(inv_mat) if idx != 3] # remove scaling value + 4th row
    np_mat = np.matrix(matrix33) #* 100 # ue4 units for 1
    
    rot = R.from_matrix(np_mat)
    pitch, yaw, roll = rot.as_euler('xyz', degrees=True)
    
    return pitch, yaw * -1, roll

def vals_from_str_rot(str_rot):
    return [float(x.split("=")[1]) for x in str_rot.split(" ")]

# funcs
def preprocess_head_data(timing_files, thread_files, app_files):
    for idx, timing_file in enumerate(timing_files):
        sub, run, move, target, gabor, date, end = timing_file.split("\\")[-1].split("_")
        file_name = data_p.joinpath("{}_{}_{}_{}_{}_{}_head.pickle".format(sub, run, move, target, gabor, date))
        file_name = str(file_name).replace("Raw", "preprocessed").replace("csv", "pickle")
        
        print("{} / {}: {}". format(idx + 1, len(timing_files), timing_file.split("\\")[-1]))
        print(file_name)
        
        if not os.path.exists(file_name):
            date = date.replace(".", "_")
            poss_thread_files = [x for x in thread_files if "{}_{}_{}_NoGain_{}_{}".format(sub, run, move, target, gabor) in x]
            poss_app_files = [x for x in app_files if "{}_{}_{}_NoGain_{}_{}".format(sub, run, move, target, gabor) in x]
            
            can_use_app_file = False
            if len(poss_app_files) > 0:
                can_use_app_file = True
                print("Can use app file")
            else:
                print("Cannot use app file")
            
            timing_df = pd.read_pickle(timing_file)
            full_head_df = pd.DataFrame()
            
            trials_with_app = 0
            trials_with_thread = 0
            
            for idx, row in timing_df.iterrows():
                trial_start, trial_end = row["PreStart"], row["End"]
                head_trial_df = pd.DataFrame()
                
                app_provides_data = False
                if can_use_app_file:
                    head_df = pd.read_csv(poss_app_files[0], index_col=0, sep=";", engine="python")
                    head_trial_df = head_df.query('Time >= @trial_start & Time <= @trial_end')
                    
                    if head_trial_df.shape[0] > 0:
                        new_cols = {}
                        col_names = ["Pitch", "Yaw", "Roll"]
                        for col in col_names:
                            new_cols[col] = []
                            
                        for idx, row in head_trial_df.iterrows():
                            for key, val in zip(col_names, vals_from_matrix(row["Matrix"])):
                                new_cols[key].append(val)
                        
                        head_trial_df = head_trial_df.assign(**new_cols)
                        head_trial_df = head_trial_df[["Time", "Pitch", "Yaw", "Roll"]]
                        app_provides_data = True 
                        trials_with_app = trials_with_app + 1
                        
                if not app_provides_data or not can_use_app_file:
                    head_df = pd.read_csv(poss_thread_files[0], index_col=0)
                    head_trial_df = head_df.query('Time >= @trial_start & Time <= @trial_end')
                    
                    new_cols = {}
                    col_names = ["Pitch", "Yaw", "Roll"]
                    for col in col_names:
                        new_cols[col] = []
                        
                    for idx, row in head_trial_df.iterrows():
                        for key, val in zip(col_names, vals_from_str_rot(row["Rotator"])):
                            new_cols[key].append(val)
                    
                    head_trial_df = head_trial_df.assign(**new_cols)
                    head_trial_df = head_trial_df[["Time", "Pitch", "Yaw", "Roll"]]
                    trials_with_thread = trials_with_thread + 1
                    
                full_head_df = pd.concat([full_head_df, head_trial_df])
                
            print("Thread trials: {}, App trials: {}".format(trials_with_thread, trials_with_app))
            full_head_df.reset_index(drop=True).to_pickle(file_name)

# eye angle calc funcs
def unreal_pos_to_single_cols(df, col): # new func to get abs position
    new_cols = {}
    keys = ["X", "Y", "Z"]
    keys = [col + "_" + x for x in keys]
    for key in keys:
        new_cols[key] = []
    
    for idx, row in df.iterrows():
        val = row[col]
        vals = val.split(" ")
        
        for key, val in zip(keys, vals):
            new_cols[key].append(val.split("=")[1])
           
    df = df.assign(**new_cols)
    
    d_type_dict = {}
    for key in keys:
        d_type_dict[key] = "float64"
    df = df.astype(d_type_dict)
    
    return df

def angle_of_vectors(vec1, vec2, rel_idx):
    # dont use z
    # vec1, vec2 = vec1[0:2], vec2[0:2]
    
    scalar_product = np.dot(vec1, vec2)
    amount_vec1 = math.sqrt(np.sum([math.pow(x, 2) for x in vec1])) # betrag
    amount_vec2  = math.sqrt(np.sum([math.pow(x, 2) for x in vec2]))
    
    if (amount_vec1 * amount_vec2) != 0:
        angle = math.degrees(math.acos(scalar_product / (amount_vec1 * amount_vec2)))
    else:
        angle = np.nan
        
    if vec1[rel_idx] < vec2[rel_idx]: # angle below / left are negative
        angle = angle * (-1)
        
    return angle

def str_vec_to_array(str_vec):
    ini_vec = [float(x.split("=")[1]) for x in str_vec.split(" ")]#[0] * coef
    real_vec = [x * 250 for x in ini_vec] # (camera_rot.RotateVector(direction) * 250) + camera_pos;
    vec_x, vec_y, vec_z = real_vec
    straight_ahead = [250, 0, 0]
    
    x = angle_of_vectors([vec_x, 0, 0], straight_ahead, 0)
    y = angle_of_vectors([vec_x, vec_y, 0], straight_ahead, 1)
    z = angle_of_vectors([vec_x, 0, vec_z], straight_ahead, 2)
    
    return [x,y,z]

def string_vec_to_eye_pos(df, col, new_col_name):
    new_cols = {}
    keys = [new_col_name + "_{}".format(x) for x in ["X", "Y", "Z"]]
    for key in keys:
        new_cols[key] = []
        
    for idx, row in df.iterrows():
        vals = str_vec_to_array(row[col])
        
        for key, val in zip(keys, vals):
            new_cols[key].append(val)
            
    df = df.assign(**new_cols)
    return df

# string_vec_to_eye_pos(run_df, "Direction", "EyeAngle")

# funcs
def preprocess_eye_data(timing_files, eye_files):
    problematic_runs = []
    
    for idx, timing_file in enumerate(timing_files):
        sub, run, move, target, gabor, date, end = timing_file.split("\\")[-1].split("_")
        file_name = data_p.joinpath("{}_{}_{}_{}_{}_{}_eye.pickle".format(sub, run, move, target, gabor, date))
        file_name = str(file_name).replace("Raw", "preprocessed").replace("csv", "pickle")
        
        print("{} / {}: {}". format(idx + 1, len(timing_files), timing_file.split("\\")[-1]))
        
        if not os.path.exists(file_name):
            poss_eye_files = [x for x in eye_files if "{}_{}_{}_NoGain_{}_{}".format(sub, run, move, target, gabor) in x]
            # print(poss_eye_files)
            
            if len(poss_eye_files) > 0:
                # print("Eye file found")
                timing_df = pd.read_pickle(timing_file)
                eye_df = pd.read_csv(poss_eye_files[0])
                full_eye_df = pd.DataFrame()
                problematic_trials = []
                
                for idx, row in timing_df.iterrows():
                    trial_start, trial_end = row["PreStart"], row["End"]
                    
                    eye_trial_df = eye_df.query('Time >= @trial_start & Time <= @trial_end')
                    # eye_trial_df = unreal_pos_to_single_cols(eye_trial_df, "Direction")
                    eye_trial_df = string_vec_to_eye_pos(eye_trial_df, "Direction", "EyeAngle")
                    eye_trial_df = eye_trial_df[["Time", "EyeAngle_Y", "EyeAngle_Z"]]
                    
                    # trial_dur = (eye_trial_df["Time"].iloc[-1] - eye_trial_df["Time"].iloc[0]) / 1_000_000
                    # print("Trial duration: {}".format(trial_dur))
                    
                    if eye_trial_df.shape[0] > 0:
                        full_eye_df = pd.concat([full_eye_df, eye_trial_df])
                    else:
                        problematic_trials.append(idx + 1)
                        
                if len(problematic_trials) > 0:
                    print("Number of trials without eye data: {}".format(len(problematic_trials)))
                    
                full_eye_df.reset_index(drop=True).to_pickle(file_name)
            else:
                print("No eye file found")
                problematic_runs.append("{}_{}_{}".format(sub, run, date))
        else:
            print("Preprocessed file found")
            
    print("Problematic Runs: {}".format(problematic_runs))

# test for inconsistencies in transformation
def angle_of_vectors(vec1, vec2, rel_idx):
    # dont use z
    # vec1, vec2 = vec1[0:2], vec2[0:2]
    
    scalar_product = np.dot(vec1, vec2)
    amount_vec1 = math.sqrt(np.sum([math.pow(x, 2) for x in vec1])) # betrag
    amount_vec2  = math.sqrt(np.sum([math.pow(x, 2) for x in vec2]))
    
    if (amount_vec1 * amount_vec2) != 0:
        angle = math.degrees(math.acos(scalar_product / (amount_vec1 * amount_vec2)))
    else:
        angle = np.nan
        
    if vec1[rel_idx] < vec2[rel_idx]: # angle below / left are negative
        angle = angle * (-1)
        
    return angle

def str_vec_to_array(str_vec, time):
    ini_vec = [float(x.split("=")[1]) for x in str_vec.split(" ")]#[0] * coef
    real_vec = [x * 250 for x in ini_vec] # (camera_rot.RotateVector(direction) * 250) + camera_pos;
    vec_x, vec_y, vec_z = real_vec
    straight_ahead = [250, 0, 0]
    
    x = angle_of_vectors([vec_x, 0, 0], straight_ahead, 0)
    y = angle_of_vectors([vec_x, vec_y, 0], straight_ahead, 1)
    z = angle_of_vectors([vec_x, 0, vec_z], straight_ahead, 2)
    
    # print(time, ini_vec, [x,y,z])
    
    return [x,y,z]

def string_vec_to_eye_pos(df, col, new_col_name):
    new_cols = {}
    keys = [new_col_name + "_{}".format(x) for x in ["X", "Y", "Z"]]
    for key in keys:
        new_cols[key] = []
        
    for idx, row in df.iterrows():
        time = (row["Time"] - df.iloc[0]["Time"]) / 1_000_000
        vals = str_vec_to_array(row[col], time)
        
        for key, val in zip(keys, vals):
            new_cols[key].append(val)
            
    df = df.assign(**new_cols)
    return df 

if False:
    infos = [
        ["ALKA_MovementGain_", 31],
        ["ALKA_Movement_", 65],
        ["ALKA_MovementGain_", 62],
        ["ALKA_MovementGain_", 57],
        ["ALKA_MovementGainHigh_", 10]
    ]

    for info in infos:
        cond, trial = info

        timing_file = [x for x in timing_files if cond in x][0]
        eye_file = [x for x in d_eye_files if cond in x][0]

        timing_df = pd.read_pickle(timing_file)
        eye_df = pd.read_csv(eye_file)

        for idx, row in timing_df.iterrows():
            if row["Trial"] == trial:
                trial_start, trial_end = row["PreStart"], row["End"]
                eye_trial_df = eye_df.query('Time >= @trial_start & Time <= @trial_end')
                eye_trial_df = string_vec_to_eye_pos(eye_trial_df, "Direction", "EyeAngle")
                eye_trial_df = eye_trial_df.assign(Time = [(x-eye_trial_df.iloc[0]["Time"]) / 1_000_000 for x in eye_trial_df["Time"]])
                # eye_trial_df = eye_trial_df.query('Time > 100 & Time < 250')

        positions = [float(x.split(" ")[0].split("=")[1]) for x in eye_trial_df["Direction"]]
        eye_trial_df = eye_trial_df.assign(ExcludeX = positions)

        post_exc_positions = [float(x.split(" ")[0].split("=")[1]) for x in eye_trial_df["Direction"]]
        plt.scatter(eye_trial_df["Time"], [x/10 for x in post_exc_positions], color="tab:green")
        plt.scatter(eye_trial_df["Time"], eye_trial_df["EyeAngle_Y"], color="tab:blue")

        val = np.mean(eye_trial_df["ExcludeX"]) - 10
        eye_trial_df = eye_trial_df.query('ExcludeX >= @val')
        plt.scatter(eye_trial_df["Time"], eye_trial_df["EyeAngle_Y"], color="tab:red")
        # plt.show()
        plt.close()

        # plt.plot(eye_trial_df["Time"], eye_trial_df["EyeAngle_Y"], color="tab:blue", lw=1)
        # eye_trial_df

# weird eye data is in fact due to weird output from eyetracker, not transformation fault
if False:
    '''
    216.332, 20.263, -91.944]	0.0, 5.3510664023392085, -23.026142566186646]
    216.459, 21.24, -91.584]	0.0, 5.604197259628303, -22.933266151667787]
    216.459, 21.24, -91.584]	0.0, 5.604197259628303, -22.933266151667787]
    1.077, 4.478, 34.244]	0.0, 76.47668624442127, 88.19859769360522]
    1.077, 4.478, 34.244]	0.0, 76.47668624442127, 88.19859769360522]
    1.066, 4.511, 34.258]	0.0, 76.70427299139874, 88.21771266346411]
    243.442, 33.533, -19.686]	0.0, 7.842871921914408, -4.623178192824711]
    243.442, 33.533, -19.686]	0.0, 7.842871921914408, -4.623178192824711]
    243.393, 34.572, -19.31]	0.0, 8.084320401776449, -4.536157267143796]
    '''

# test problem runs for movement detection pp
def find_correct_files(sub, run, move, target, gabor, d_files, timing_files, head_pp_data, eye_pp_data):
    d_file = [x for x in d_files if "{}_{}_{}_NoGain_{}_{}".format(sub, run, move, target, gabor) in x]
    timing_file = [x for x in timing_files if "{}_{}_{}_{}_{}".format(sub, run, move, target, gabor) in x]
    head_pp_d = [x for x in head_pp_data if "{}_{}_{}_{}_{}".format(sub, run, move, target, gabor) in x]
    eye_pp_d = [x for x in eye_pp_data if "{}_{}_{}_{}_{}".format(sub, run, move, target, gabor) in x]

    for idx, x in enumerate([d_file, timing_file, head_pp_d, eye_pp_d]):
        if len(x) > 1:
            print(idx, "several files found")
            print(np.array(x))

    return d_file[0], timing_file[0], head_pp_d[0], eye_pp_d[0]

if False:
    scripts_p = path.Path.cwd()
    stats_p = scripts_p.parents[1]
    data_p = stats_p.joinpath("effmann 2", "data", "Raw") # C:\Users\serda\Desktop\CodeProjects\UE\ada_head_rot_eyetracking\Z_Stats\effmann 2\data\Raw
    plot_p = stats_p.joinpath("effmann 2","plots")
    dfs_p = stats_p.joinpath("effmann 2","dfs")

    d_files = np.sort(bf.get_data(data_p,"csv","", ["timings", "head","eye"]))
    d_timing_files = np.sort(bf.get_data(data_p,"csv","timings.csv", []))
    d_eye_files = np.sort(bf.get_data(data_p,"csv","eye.csv", []))
    d_head_files = np.sort(bf.get_data(data_p,"csv","head.csv", []))
    d_head_app_files = np.sort(bf.get_data(data_p,"csv","head_app.csv", []))

    timing_files = np.sort(bf.get_data(stats_p.joinpath("effmann 2", "data","preprocessed"),"pickle","timings", []))
    head_pp_data = np.sort(bf.get_data(stats_p.joinpath("effmann 2", "data","preprocessed"),"pickle","head", ["eye", "timings"]))
    eye_pp_data = np.sort(bf.get_data(stats_p.joinpath("effmann 2", "data","preprocessed"),"pickle","eye", ["head", "timings"]))

    timing_file = r"C:\Users\serda\Desktop\CodeProjects\UE\ada_head_rot_eyetracking\Z_Stats\effmann 2\data\preprocessed\FS28_HeadEye_Movement_TemporaryTargets_Gabor_20.6.2023_timings.pickle" #[x for x in timing_files if "AD21_Eye_Movement_PermanentTargets_Gabor" in x][0]
    test = pd.read_pickle(timing_file)

    for t_file in timing_files[:1]:
        timing_df = pd.read_pickle(timing_file)
        sub, run, move, target, gabor, date, end = timing_file.split("\\")[-1].split("_")

        d_file, timing_file, head_pp_d, eye_pp_d = find_correct_files(sub, run, move, target, gabor, d_files, timing_files, head_pp_data, eye_pp_data)
        beh_df = pd.read_csv(d_file, index_col=0)
        head_df = pd.read_pickle(head_pp_d)
        eye_df = pd.read_pickle(eye_pp_d)

        # test_p = plot_p.joinpath("test_tracking_data")

        # for idx, row in timing_df.iterrows():
        #     trial, pre, start, end, f_onset, f_offset, s_onset, s_offset = row 
        #     f_onset, f_offset, s_onset, s_offset = [(x - start) / 1_000_000 for x in [f_onset, f_offset, s_onset, s_offset]]
        #     beh_trial = beh_df.query('TrialNum == @trial').iloc[0]
            
        #     trial_eye = eye_df.query('Time >= @pre & Time <= @end')
        #     trial_eye = trial_eye.assign(Time = [(x - start) / 1_000_000 for x in trial_eye["Time"]])
            
        #     trial_head = head_df.query('Time >= @pre & Time <= @end')
        #     trial_head = trial_head.assign(Time = [(x - start) / 1_000_000 for x in trial_head["Time"]])
            
        #     plt.plot(trial_head["Time"], trial_head["Yaw"])
        #     plt.xlabel("Time")
        #     plt.xlabel("°")
        #     plt.ylim([-20,20])
        #     plt.savefig(test_p.joinpath("{}.png".format(trial)), facecolor="white")
        #     plt.close()
        
    test.head(2)

# functions for trial movement
def find_correct_files(sub, run, date, d_files, timing_files, head_pp_data, eye_pp_data):
    d_file = [x for x in d_files if "{}_{}_{}".format(sub, run, date) in x]
    timing_file = [x for x in timing_files if "{}_{}_{}".format(sub, run, date) in x]
    head_pp_d = [x for x in head_pp_data if "{}_{}_{}".format(sub, run, date) in x]
    eye_pp_d = [x for x in eye_pp_data if "{}_{}_{}".format(sub, run, date) in x]
    
    for idx, x in enumerate([d_file, timing_file, head_pp_d, eye_pp_d]):
        if len(x) > 1:
            print(idx, "several files found")
            print(np.array(x))
    
    return d_file[0], timing_file[0], head_pp_d[0], eye_pp_d[0]

def plot_trial_movements(beh, timing, head, eye, trials):
    sub, run = beh.iloc[0]["SubjectID"], beh.iloc[0]["Version"]
    
    if trials == "all":
        rows = timing
    else:
        rows = timing.query('') #.iloc[0:num_trials]
    
    for idx, row in rows.iterrows():
        # get trial info
        trial, pre, start, end, f_onset, f_offset, s_onset, s_offset = row 
        # print("Trial: {}".format(int(trial)))
        
        trial_head = head.query('Time >= @pre & Time <= @end')
        trial_eye = eye.query('Time >= @pre & Time <= @end')
        
        stimuli_times = [(x - start) / 1_000_000 for x in [f_onset, f_offset, s_onset, s_offset]]
        beh_trial = beh.query('TrialNum == @trial').iloc[0]
        obj_positions = get_trial_object_positions(beh_trial["TargetHoriPos"], beh_trial["TargetVertPos"])
        stim_positions = obj_positions["Stimulus"]
        fix_positions = obj_positions["Fixation_Cross"]
        
        for axis in ["Y", "Z"]:
            save_p = plot_p.joinpath("trial_movements", "{}_{}_{}_{}.png".format(sub, run, int(trial), axis))
            
            if not os.path.exists(save_p):
                head_col = "Yaw"
                upper_border = "Left_Border"
                lower_border = "Right_Border"
                
                if axis == "Z":
                    head_col = "Pitch"
                    upper_border = "Upper_Border"
                    lower_border = "Lower_Border"
                
                # create plot
                fig, ax = plt.subplots(1,1, figsize=[6,5])
                ax.spines['top'].set_visible(False)
                ax.spines['right'].set_visible(False)
                ax.set_ylabel(axis)
                
                # head eye plotting
                head_times = [(x - start) / 1_000_000 for x in trial_head["Time"]]
                ax.plot(head_times, trial_head[head_col], color="tab:blue", label="Head")
                ax.plot([(x - start) / 1_000_000 for x in trial_eye["Time"]], trial_eye["EyeAngle_{}".format(axis)], color="tab:red", label="Eye")
                
                # stimuli plotting
                ax.plot(stimuli_times[0:2], [stim_positions["Probe"][upper_border], stim_positions["Probe"][upper_border]], color="b", label="Probe")
                ax.plot(stimuli_times[0:2], [stim_positions["Probe"][lower_border], stim_positions["Probe"][lower_border]], color="b")
                
                ax.plot(stimuli_times[2:], [stim_positions["Reference"][upper_border], stim_positions["Reference"][upper_border]], color="b", label="Reference")
                ax.plot(stimuli_times[2:], [stim_positions["Reference"][lower_border], stim_positions["Reference"][lower_border]], color="b")
                
                # fixation cross
                ax.plot(stimuli_times[0:2], [fix_positions["Left_Cross"][upper_border]] * 2, color="k", label="Left Fixation Cross")
                ax.plot(stimuli_times[0:2], [fix_positions["Left_Cross"][lower_border]] * 2, color="k")
                
                ax.plot(stimuli_times[2:], [fix_positions["Right_Cross"][upper_border]] * 2, color="k", label="Right Fixation Cross")
                ax.plot(stimuli_times[2:], [fix_positions["Right_Cross"][lower_border]] * 2, color="k")
                
                ax.legend(bbox_to_anchor=(1, 0.15, 0.5, 0.5), frameon=False)
                
                plt.savefig(save_p, dpi=150, facecolor="white", bbox_inches="tight")
                plt.close()
            
def plot_movements(print_b):
    files_to_process = timing_files
    
    for file_idx, file in enumerate(files_to_process):
        print("{} / {}: {}".format(file_idx + 1, len(files_to_process), datetime.now()))
        timing = pd.read_pickle(file)
        sub, run, date = timing_files[file_idx].split("\\")[-1].split("_")[0:3]
        d_file, timing_file, head_pp_d, eye_pp_d = find_correct_files(sub, run, date, d_files, timing_files, head_pp_data, eye_pp_data)
        
        beh = pd.read_csv(d_file, index_col=0)
        head = pd.read_pickle(head_pp_d)
        eye = pd.read_pickle(eye_pp_d)
        
        if print_b:
            print(
                d_file, "\n",
                timing_file, "\n",
                head_pp_d, "\n",
                eye_pp_d
            )

        plot_trial_movements(beh, timing, head, eye, "all")

# single thread variant
if False:
    plot_movements(False)

# parallel processing approach
def pp_plot_file_trials(file, timing_files, d_files, head_pp_data, eye_pp_data, print_b, plot_p):
    import math
    import os
    import pandas as pd
    import numpy as np
    import matplotlib.pyplot as plt
    
    def angle_of_vectors(vec1, vec2, rel_idx):
        # dont use z
        # vec1, vec2 = vec1[0:2], vec2[0:2]
        
        scalar_product = np.dot(vec1, vec2)
        amount_vec1 = math.sqrt(np.sum([math.pow(x, 2) for x in vec1])) # betrag
        amount_vec2  = math.sqrt(np.sum([math.pow(x, 2) for x in vec2]))
        
        if (amount_vec1 * amount_vec2) != 0:
            angle = math.degrees(math.acos(scalar_product / (amount_vec1 * amount_vec2)))
        else:
            angle = np.nan
            
        if vec1[rel_idx] < vec2[rel_idx]: # angle below / left are negative
            angle = angle * (-1)
            
        return angle
    
    def get_trial_object_positions(stimulus_shift, probe_vert_pos):
        fixation_size = 7.5 # 7.5 cm from center to outer edges
        fixation_pos = 10 # -10 and 10 degree (horizontal axis)
        stimulus_size = 10 # 10 cm from center to outer edges
        ref_vert_pos = probe_vert_pos * -1
        straight_vec = [250, 0, 0]

        object_positions = {
            "Fixation_Cross": {
                "Left_Cross": {
                    "Left_Border": angle_of_vectors(straight_vec, [250, fixation_size, 0], 1) - fixation_pos,
                    "Right_Border": (angle_of_vectors(straight_vec, [250, fixation_size, 0], 1) + fixation_pos) * -1,
                    "Upper_Border": angle_of_vectors(straight_vec, [250, 0, -1 * fixation_size], 2),
                    "Lower_Border": angle_of_vectors(straight_vec, [250, 0, -1 * fixation_size], 2) * -1
                },
                "Right_Cross": {
                    "Left_Border": angle_of_vectors(straight_vec, [250, fixation_size, 0], 1) + fixation_pos,
                    "Right_Border": angle_of_vectors(straight_vec, [250, fixation_size, 0], 0) + fixation_pos,
                    "Upper_Border": angle_of_vectors(straight_vec, [250, 0, -1 * fixation_size], 2),
                    "Lower_Border": angle_of_vectors(straight_vec, [250, 0, -1 * fixation_size], 2) * -1
                }
            },
            "Stimulus": {
                "Probe": {
                    "Left_Border": angle_of_vectors(straight_vec, [250, stimulus_size + stimulus_shift, 0], 1),
                    "Right_Border": (angle_of_vectors(straight_vec, [250, stimulus_size + stimulus_shift, 0], 1)) * -1,
                    "Upper_Border": angle_of_vectors(straight_vec, [250, 0, probe_vert_pos + stimulus_size], 2),
                    "Lower_Border": angle_of_vectors(straight_vec, [250, 0, probe_vert_pos - stimulus_size], 2)# * -1
                },
                "Reference": {
                    "Left_Border": angle_of_vectors(straight_vec, [250, stimulus_size + stimulus_shift, 0], 1),
                    "Right_Border": (angle_of_vectors(straight_vec, [250, stimulus_size + stimulus_shift, 0], 1)) * -1,
                    "Upper_Border": angle_of_vectors(straight_vec, [250, 0, ref_vert_pos + stimulus_size], 2),
                    "Lower_Border": angle_of_vectors(straight_vec, [250, 0, ref_vert_pos - stimulus_size], 2)# * -1
                }
            }
        }

        return object_positions
    
    def find_correct_files(sub, run, date, d_files, timing_files, head_pp_data, eye_pp_data):
        d_file = [x for x in d_files if "{}_{}_{}".format(sub, run, date) in x]
        timing_file = [x for x in timing_files if "{}_{}_{}".format(sub, run, date) in x]
        head_pp_d = [x for x in head_pp_data if "{}_{}_{}".format(sub, run, date) in x]
        eye_pp_d = [x for x in eye_pp_data if "{}_{}_{}".format(sub, run, date) in x]
        
        for idx, x in enumerate([d_file, timing_file, head_pp_d, eye_pp_d]):
            if len(x) > 1:
                print(idx, "several files found")
                print(np.array(x))
        
        return d_file[0], timing_file[0], head_pp_d[0], eye_pp_d[0]
    
    def plot_trial_movements(beh, timing, head, eye, num_trials):
        sub, run = beh.iloc[0]["SubjectID"], beh.iloc[0]["Version"]
        print("Started for: {}".format([sub, run]))
    
        if num_trials == "all":
            rows = timing
        else:
            rows = timing.iloc[0:num_trials]
        
        for idx, row in rows.iterrows():
            # get trial info
            trial, pre, start, end, f_onset, f_offset, s_onset, s_offset = row 
            # print("Trial: {}".format(int(trial)))
            
            trial_head = head.query('Time >= @pre & Time <= @end')
            trial_eye = eye.query('Time >= @pre & Time <= @end')
            
            stimuli_times = [(x - start) / 1_000_000 for x in [f_onset, f_offset, s_onset, s_offset]]
            beh_trial = beh.query('TrialNum == @trial').iloc[0]
            obj_positions = get_trial_object_positions(beh_trial["TargetHoriPos"], beh_trial["TargetVertPos"])
            stim_positions = obj_positions["Stimulus"]
            fix_positions = obj_positions["Fixation_Cross"]
            
            for axis in ["Y", "Z"]:
                save_p = plot_p.joinpath("trial_movements_pp", "{}_{}_{}_{}.png".format(sub, run, int(trial), axis))
                
                if not os.path.exists(save_p):
                    head_col = "Yaw"
                    upper_border = "Left_Border"
                    lower_border = "Right_Border"
                    
                    if axis == "Z":
                        head_col = "Pitch"
                        upper_border = "Upper_Border"
                        lower_border = "Lower_Border"
                    
                    # create plot
                    fig, ax = plt.subplots(1,1, figsize=[6,5])
                    ax.spines['top'].set_visible(False)
                    ax.spines['right'].set_visible(False)
                    ax.set_ylabel(axis)
                    
                    # head eye plotting
                    head_times = [(x - start) / 1_000_000 for x in trial_head["Time"]]
                    ax.plot(head_times, trial_head[head_col], color="tab:blue", label="Head")
                    ax.plot([(x - start) / 1_000_000 for x in trial_eye["Time"]], trial_eye["EyeAngle_{}".format(axis)], color="tab:red", label="Eye")
                    
                    # stimuli plotting
                    ax.plot(stimuli_times[0:2], [stim_positions["Probe"][upper_border], stim_positions["Probe"][upper_border]], color="b", label="Probe")
                    ax.plot(stimuli_times[0:2], [stim_positions["Probe"][lower_border], stim_positions["Probe"][lower_border]], color="b")
                    
                    ax.plot(stimuli_times[2:], [stim_positions["Reference"][upper_border], stim_positions["Reference"][upper_border]], color="b", label="Reference")
                    ax.plot(stimuli_times[2:], [stim_positions["Reference"][lower_border], stim_positions["Reference"][lower_border]], color="b")
                    
                    # fixation cross
                    ax.plot(stimuli_times[0:2], [fix_positions["Left_Cross"][upper_border]] * 2, color="k", label="Left Fixation Cross")
                    ax.plot(stimuli_times[0:2], [fix_positions["Left_Cross"][lower_border]] * 2, color="k")
                    
                    ax.plot(stimuli_times[2:], [fix_positions["Right_Cross"][upper_border]] * 2, color="k", label="Right Fixation Cross")
                    ax.plot(stimuli_times[2:], [fix_positions["Right_Cross"][lower_border]] * 2, color="k")
                    
                    ax.legend(bbox_to_anchor=(1, 0.15, 0.5, 0.5), frameon=False)
                    
                    safe_save_fig(fig, save_p)
                    # plt.savefig(save_p, dpi=150, facecolor="white", bbox_inches="tight")
                    plt.close()
                    
        print("Finished for: {}".format([sub, run]))
        
    def safe_save_fig(fig, path):
        fig.savefig(path, dpi=150, facecolor="white", bbox_inches="tight")
    
    timing = pd.read_pickle(file)
    sub, run, date = file.split("\\")[-1].split("_")[0:3]
    d_file, timing_file, head_pp_d, eye_pp_d = find_correct_files(sub, run, date, d_files, timing_files, head_pp_data, eye_pp_data)
    
    beh = pd.read_csv(d_file, index_col=0)
    head = pd.read_pickle(head_pp_d)
    eye = pd.read_pickle(eye_pp_d)
    
    if print_b:
        print(
            d_file, "\n",
            timing_file, "\n",
            head_pp_d, "\n",
            eye_pp_d
        )

    plot_trial_movements(beh, timing, head, eye, "all")
    
###########################################################

if False:
    from multiprocessing import Pool
    import time

    # in_arrs = [[file, timing_files, d_files, head_pp_data, eye_pp_data, False, plot_p] for file in timing_files]
    in_arrs = [[timing_files[0], timing_files, d_files, head_pp_data, eye_pp_data, False, plot_p]]

    start_t = time.perf_counter()
    with Pool() as pool:
        results = pool.map(pp_plot_file_trials, in_arrs)

        for filename, duration in results:
            print(f"{filename} completed in {duration:.2f}s")

    end_t = time.perf_counter()
    total_duration = end_t - start_t
    print(f"took {total_duration:.2f}s total")


# files_to_process = timing_files#[:5]
    
# from multiprocessing.pool import ThreadPool as Pool
# with Pool(processes=len(files_to_process)) as pool:
#     for file in files_to_process:
#         result = pool.apply_async(func=pp_plot_file_trials, 
#                                 args=(file, timing_files, d_files, head_pp_data, eye_pp_data, False, plot_p,))
#         # print(result.get())
#     pool.close()
#     pool.join()