/**
 * This applet demonstrates SARSA(lambda)-learning for a particular
 * grid world problem. It isn't designed to be general or reusable.
<p>
 * Copyright (C) 2003-2006 <A HREF="http://www.cs.ubc.ca/spider/poole/">David Poole</A>. 
<p>
 * This program gives SARSA(labmda)-learning code. The GUI is in <A HREF="SarsaGUI.java">SarsaGUI.java</A>. The controller code is at <A HREF="SarsaCore.java">SarseCotroller.java</A>. It used the environment <A HREF="Q_Env.java">Q_Env.java</A>.
<p>
 This program is free software; you can redistribute it and/or
 modify it under the terms of the GNU General Public License
 as published by the Free Software Foundation; either version 2
 of the License, or (at your option) any later version.
<p>
 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
<p>
 You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.


 * @author David Poole  poole@cs.ubc.ca
 * @version 0.4 2006-12-06 */

public class SarsaController
{
    /**
       The environment in which the controller operates/
    */
    Q_Env environment;
    /**
       The Q values: Q[xpos,ypos,action]
     */
    public double qvalues[][][] = new double[10][10][4];
    /**
       The the eligibility trace for state (xpos,ypos) and action
     */
    public double e[][][] = new double[10][10][4];
    public double discount = 0.9;
    public double alpha = 0.4;
    public double lambda = 0.9;
    public double greedyProb = 0.8;

    public boolean tracing=false;

    public int prevX=0;
    public int prevY=0;
    public int prevAction=0;
    public double reward=0;

    public boolean replacingTrace=false;

    SarsaController(Q_Env env)
    {
	environment = env;
    }

    /**
     * resets the Q-values
     *
     * sets all of the Q-values to initVal, 
     * and the eligibility to 0
     * @param initVal   the initial value to set all values to
     */
    public void doreset(double initVal)
    {
	for (int xval=0 ; xval <10; xval++) {
	    for (int yval=0; yval < 10; yval++) {
		for (int i=0; i<4; i++) {
		    qvalues[xval][yval][i]=initVal;
		    e[xval][yval][i]=0;
		}
	    }
	}
    }

    /**
     * does one step
     *
     * carries out the action, and sets the discount and the alpha value
     *
     * @param nextAction  the next action that the agent does
     * @param newDiscount  the discount to use
     * @param newAlpha the new alpha value to use
     * @param newLambda the new lambda value to use
     */
    public void dostep(int nextAction, 
		       double newDiscount, 
		       double newAlpha, 
		       double newLambda)  
    { 
	discount = newDiscount;
	alpha = newAlpha;
	lambda = newLambda;
	dostep(nextAction);
    }

    /**
     * does one step
     *
     * carries out the action
     *
     * @param nextAction  the next action that the agent does
     */

    public void dostep(int nextAction) 
    {
	// update Q values

	double newDatum=reward+discount*qvalues[environment.currX][environment.currY][nextAction];
	double delta=newDatum - qvalues[prevX][prevY][prevAction];
	if (replacingTrace) // replacing trace
	    {
                for (int oact=0; oact<4; oact++)
		    {
			if (oact != prevAction)
			    e[prevX][prevY][oact]=0.0;
		    }
		e[prevX][prevY][prevAction]=1.0;
	    }
	else 
	    {
		e[prevX][prevY][prevAction]++;
	    }
		
	if (tracing) {
	    System.out.println("S=("+prevX+","+prevY+") A="+prevAction+" R="+reward+" S'=("+
			   environment.currX+","+environment.currY+") A'="+nextAction);
	    System.out.println("     Qold="+qvalues[prevX][prevY][prevAction]+" newDatum="+newDatum);
	}
	double alphaDelta = alpha * delta;
	double gammaLambda = discount*lambda;
	for (int allX=0; allX<10; allX++) {
	    for (int allY=0; allY < 10; allY++) {
		for (int allA=0; allA<4; allA++) {
		    qvalues[allX][allY][allA] +=
			alphaDelta * e[allX][allY][allA];
		    e[allX][allY][allA] *= gammaLambda;
		}
	    }
	}


	// Store state and action
        prevX = environment.currX;
	prevY = environment.currY;
	prevAction = nextAction;

	// do the action and store the reward
	reward = environment.dostep(nextAction);

    }

    /**
     * determines the value of a location
     *
     * the value is the maximum, for all actions, of the q-value
     *
     * @param xval the x-coordinate
     * @param yval the y-coordinate
     * @return the value of the (xval,yval) position
     */
    public double value(int xval, int yval) {
	double val=qvalues[xval][yval][3];
	for (int i=2; i>=0; i--) {
	    if(qvalues[xval][yval][i]>val) {
		val=qvalues[xval][yval][i];
	    }
	}
	return val;
    }

    /**
     * does count number of steps
     *
     * whether each step is greedy or random is determine by greedyProb
     * @param count  the number of steps to do
     * @param newGreedyProb  the probability that is step is chosen greedily
     * @param newDiscount  the discount to use
     * @param newAlpha the new alpha value to use
     * @param newLambda the new lambda value to use
     */

    public void doSteps(int count, 
			double newGreedyProb,
			double newDiscount, 
			double newAlpha, 
			double newLambda)
    {
	greedyProb = newGreedyProb;
	discount = newDiscount;
	alpha = newAlpha;
	lambda = newLambda;
	for(int i=0; i<count; i++)
	    {
		double rand = Math.random();
		if (rand<greedyProb)
		    {// act greedily
			int startDir = (int) (Math.random() * 4);
			double bestVal = qvalues[environment.currX][environment.currY][startDir];
			int bestDir = startDir;
			for (int dir=1; dir<4; dir++)
			    {
				startDir = (startDir+1)%4;
				if (qvalues[environment.currX][environment.currY][startDir] > bestVal)
				    {
					bestVal = qvalues[environment.currX][environment.currY][startDir];
					bestDir = startDir;
				    }
			    }
			dostep(bestDir);
		    }
		else
		    { // act randomly
			dostep((int) (Math.random() * 4));
		    }
	    }
    }


}