%% Script file to run the N-armed bandit using pursuit strategy
% Initializations
NumMachines=10;
ActQ=randn(NumMachines,1);
NumPlay=200;

Initialbeta=0.01;
Endingbeta=0.001;
beta=Initialbeta;

NumPlayed=zeros(NumMachines,1);
ValPlayed=zeros(NumMachines,1);

EstQ=zeros(NumMachines,1);
Probs=(1/NumMachines)*ones(10,1);

%% Main Loop
for i=1:NumPlay
    %Pick a machine to play:
    [a,Probs]=winstay(EstQ,Probs,beta);
    %Play the machine and update EstQ, tau
    Payoff=randn+ActQ(a);
    NumPlayed(a)=NumPlayed(a)+1;
    ValPlayed(a)=ValPlayed(a)+Payoff;
    EstQ(a)=ValPlayed(a)/NumPlayed(a);
    beta=Initialbeta*(Endingbeta/Initialbeta)^(i/NumPlay);
end

%% Display outpute
[v,winningmachine]=max(ActQ);
fprintf('The winning machine is %d\n',winningmachine);
fprintf('The number times played is %d\n',NumPlayed(winningmachine));

plot(1:10,ActQ,'k',1:10,EstQ,'r')
legend('Actual','Estimated');