///Licensed under Creative Commons Attribution 4.0 License, Written by Vladislav Nachev
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace Exploitation
{
    public class DecisionRules : Decision
    {
        /// <summary>
        /// Implements Action selection methods, in which the next flower to be visited is selected (from the vector of remembered values for each flower).
        /// Remembered values are between 0 (bad) and 1 (good).
        /// Two main action selection methods are either softmax or epsilon-greedy (where epsilon is the lapse rate from Perception.Lapse).
        /// </summary>
        static SimpleRNG rand = new SimpleRNG();
        

        public DecisionRules(int ruleNo, Memory memory)
            : base(ruleNo, memory)
        {
            this.decisionType = ruleNo; //decisionType currently does nothing
        }

        /// <summary>
        /// Main action selection function
        /// </summary>
        /// <param name="time">the current time</param>
        /// <returns>the next feeder to be visited based on decision rule</returns>
        public override Ressource RessourceGoal(long time)
        {            
            double[] PSum = new double[memory.Ressources.Count()];       // Dictionary of ressource positions and their current attractivity
            double sumOfAttractions = 0;
            int t = 0;
            int memCapacity = 1000; // determindes the size of the window previous rewards are evaluated over (only if dynaverageUtdate is used)
            // with a value of 1000 events and fewer than 1000 generated events, events are never forgotten 

            double temp; // temperature parameter for softmax selection
            
            // Devaluation of options with time
            Dictionary<Position, List<MemoryStruct>> memories = memory.CompleteMemory();  //dump memory contents to a variable
            // memoryDecay(time, memories, 0); //memory decays with time (currently the function does nothing)
            if (memory.CountMemories() > 0)
                
            {
                switch ((memory as EvaluationFamilies).MemoryRule)
                {
                    case 3:
                    // MemoryRule == 3 means dynamic averageValue update, that is, a floating average of all experienced instantaneous values
                    // if the window size is very large, that amounts to the long-term average 
                        Position key = memory.LastRessourceKey();
                        (memory as EvaluationFamilies).RValue[memories[key].First<MemoryStruct>().RessourceObj.ID] =
                            memory.DynAverageUtDate(key, memCapacity);
                    break;
                }
                    
            }          

            avoidRessource(time, memories, 0.02);
            //avoidance function for the last visited feeder to avoid revisits, 
            //the Avoidancedecay parameter determines how long a feeder is avoided for, high values equal no avoidance.
            //The default value of 0.02 is overridden if MemoryRule = 1 or 2, see below



            // using Value vector, rescale into probability to next visit, by
            // dividing Value over the sum of all values
            // first step get the sum of values (using softmax or epsilon-greedy action selection)

            switch ((memory as EvaluationFamilies).PerceptionType)
            {
                case 6: case 7: case 8:
                temp = (memory as EvaluationFamilies).Lapse;
                // temperature parameter of softmax rule determines how reliant is the decision on memory,
                // very high temp = random walk; low temp = perfect preferencee for max
                sumOfAttractions = SoftmaxSum((memory as EvaluationFamilies).DecayedValue, temp);//total sum of remembered values for softmax rule
                break;

                default:
                    sumOfAttractions = (memory as EvaluationFamilies).DecayedValue.Sum(); //for epsilon-greedy models lapse rate = epsilon
                    if (sumOfAttractions == 0 || rand.GetUniform() < (memory as EvaluationFamilies).Lapse)
                    {
                        //  if sumOfAttractions = 0, make visit at random, to prevent divide by zero error
                        // also if the visit is a lapse, it is made at random
                        t = rand.GetInt(0, memory.Ressources.Count() - 1);
                        return memory.Ressources[t];
                    }
                    break;
            }



            foreach (Ressource feeder in memory.Ressources)
            {
                switch ((memory as EvaluationFamilies).PerceptionType)
                {
                    case 6: case 7: case 8:
                        temp = (memory as EvaluationFamilies).Lapse;
                        // temperature parameter of softmax rule determines how reliant is the decision on memory,
                        // very high temp = random walk; low temp = perfect preferencee for max

                        (memory as EvaluationFamilies).PVisit[feeder.ID] = Math.Exp((memory as EvaluationFamilies).
                    DecayedValue[feeder.ID] / temp) / sumOfAttractions; //softmax action selection
                        break;
                    default:
                        (memory as EvaluationFamilies).PVisit[feeder.ID] = (memory as EvaluationFamilies).DecayedValue[feeder.ID] / sumOfAttractions;
                        // epsilon-greedy action selection
                        break;
                }
                
       
                if (feeder.ID < 1) PSum[feeder.ID] = (memory as EvaluationFamilies).PVisit[feeder.ID]; 
                else PSum[feeder.ID] = PSum[feeder.ID - 1] + (memory as EvaluationFamilies).PVisit[feeder.ID];
                // the vector PSum contains the cumulative probabilities
            }
            PSum[memory.Ressources.Count() - 1] = 1; // set upper bound to 1. this step is necessary
            // because due to rounding errors, simple summation sometimes equals less or more than 1

            // choose a random number and see which feeder it points to. the higher the Value, 
            // the higher the probability to be chosen. intervals are not sorted!
            t = 0;
            double r = rand.GetUniform();
            if (r < PSum[0]) return memory.Ressources[t]; else t = 1;
            for (int i = 1; i < memory.Ressources.Count(); i++)
            {
                if (r >= PSum[i - 1] && r < PSum[i]) break; else t = t + 1;
            }
            return memory.Ressources[t];
        }

        /// <summary>
        /// A forced visit alternation function
        /// </summary>
        /// <param name="time"></param>
        /// <param name="altvisit">alternation visit number</param>
        /// <param name="altSequence">sequence of feeder numbers to be visited in alternation</param>
        /// <returns>next feeder from alternation sequence</returns>
        public override Ressource RessourceAltGoal(long time, int altvisit, int[] altSequence)
        {
            Dictionary<Position, List<MemoryStruct>> memories = memory.CompleteMemory();  //dump memory contents to a variable
            memoryDecay(time, memories, 0); //memory decays with time (currently the function does nothing) 
            int memCapacity = 1000; // determindes the size of the window previous rewards are evaluated over (only if dynaverageutdate is used)
            // used to be 1000 to never forget an event

            if (((memory as EvaluationFamilies).MemoryRule == 3) && memory.CountMemories() > 0)
            // memoryryle == 3 means dynamic averageValue update, that is, a floating average of all experienced instantaneous values
            // if the window size is very large, this amounts to the long-term average, as in the paper
            {
                Position key = memory.LastRessourceKey();
                (memory as EvaluationFamilies).RValue[memories[key].First<MemoryStruct>().RessourceObj.ID] = memory.DynAverageUtDate(key, memCapacity);
            }

            avoidRessource(time, memories, 0.02); //default avoidance. overridden if MemoryType = 1 or 2, see below

            int t=0;
            t = ((altvisit + 1) % altSequence.Length) - 1;

            if (t < 0) t = altSequence.Length - 1;
            return memory.Ressources[altSequence[t]-1];
        }


        /// <summary>
        /// Exponential memory decay function
        /// </summary>
        /// <param name="time">current time</param>
        /// <param name="memories">memory record to retrieve time interval since last visit</param>
        /// <param name="memDecay" > the decay parameter determines how quickly memory decays over time </param>
        public void memoryDecay(long time, Dictionary<Position, List<MemoryStruct>> memories, double memDecay)
        {
            
            if (memories.Count > 0)
            {
                
                foreach (Position key in memories.Keys)
                {
                    long timeLastVisit = memories[key].Last<MemoryStruct>().Time;
                    long intervalLastVisit = time - timeLastVisit;                          // interval since last visit

                    (memory as EvaluationFamilies).RValue[memories[key].First<MemoryStruct>().RessourceObj.ID] =
                        (memory as EvaluationFamilies).RValue[memories[key].First<MemoryStruct>().RessourceObj.ID] *
                        (Math.Exp(-memDecay * intervalLastVisit));
                }
            }
        }

        /// <summary>
        /// Avoidance function for feeders that were just visited
        /// </summary>
        /// <param name="time">current time</param>
        /// <param name="memories">memory record to retrieve time interval since last visit</param>
        /// <param name="avoidanceDecay" > the avoidance decay determines how quickly the avoidance effect dissipates.
        /// if it is very large (>2) no avoidance takes place</param>
        public void avoidRessource(long time, Dictionary<Position, List<MemoryStruct>> memories, double avoidanceDecay)
        {

            foreach (Position key in memories.Keys)
            { 
                    long timeLastVisit = memories[key].Last<MemoryStruct>().Time;
                    long intervalLastVisit = time - timeLastVisit;                          // interval since last visit
                    
                    switch ((memory as EvaluationFamilies).MemoryRule)
                {
                    case 1:
                        avoidanceDecay = (1 - Math.Exp(-0.005 * memories[key].Last<MemoryStruct>().Reward[1])); //avoidance depends on the
                        //volume of the last visited option. smaller volume => slower decay (avoid for longer),
                        //larger volume => faster decay (next visit comes sooner)
                    break;

                    case 2:
                        avoidanceDecay = 100; //this value equals no avoidance
                    break;               
                    
                }
                    
                    (memory as EvaluationFamilies).DecayedValue[memories[key].First<MemoryStruct>().RessourceObj.ID] =
                                  (memory as EvaluationFamilies).RValue[memories[key].First<MemoryStruct>().RessourceObj.ID] *
                                  (1 - Math.Exp(-avoidanceDecay  * intervalLastVisit));
            }

        }

        /// <summary>
        /// SoftMax action selection function 
        /// </summary>
        /// <param name="input">vector of decayed remembered values for all options</param>
        /// <param name="tau">"temperature" parameter, determines the exploration/exploitation ratio</param>
        /// <returns>the sum of weighted remembered values</returns>
        public double SoftmaxSum(double[] input, double tau)
        {
            double sum = 0;

            for (int i = 0; i < input.Count(); i++)
            {
                sum = sum + Math.Exp(input[i]/tau);
            }
                return sum;
        }

    }
}

