#pragma once
#include <iostream>
#include "unordered_map_serialization.h"
#include "QState.h"

#define ALPHA 0.0000001 // Learning rate 0.0000001
#define GAMMA 0.02 // Future discount 0.02
#define EPSILON 1 // Probability 1 means exploitation only.

#define ACTIONS 2
#define EXIT -1
#define ATTACK 0
#define FLEE 1
#define EXPLORE 2
#define CARRYON 3

class QTable{
private:
	friend std::ostream& operator<<(std::ostream&, const QTable&);
	friend class boost::serialization::access;

	boost::unordered_map<QState, double, StateActionHash> Q;
	std::vector<double> weights;

	template<class Archive>
	void save(Archive & ar, const unsigned int version) const
	{
		boost::serialization::save(ar, Q, 0);
	}
	template<class Archive>
	void load(Archive & ar, const unsigned int version)
	{
		boost::serialization::load(ar, Q, 0);
	}
	BOOST_SERIALIZATION_SPLIT_MEMBER()

public:
	QTable();
	void insert(QState qState, double value);
	double update(State state, int action, State nextState, double reward, bool isTerminalState);
	void updateWeights(State state, int action, State nextState, double reward, bool isTerminalState);
	double getMaxValue(State state, bool terminalState);
	double getQValue(QState qState);
	double getQValue(State state, int action);
	QState getFirstQState();
	boost::unordered_map<QState, double, StateActionHash> getQ();
	std::vector<double> getWeights();
	void setWeights(std::vector<double> w);
	std::vector<double> extractFeatures(State state, int action);

	// Quantitative features
	int feature0(State state, int action); // hp of our own units in state s.
	int feature1(State state, int action); // hp of the enemy units in state s.
	int feature2(State state, int action); // dps of our own units in state s.
	int feature3(State state, int action); // dps of the enemy units in state s.
	int feature4(State state, int action); // number of allies in state s.
	int feature5(State state, int action); // number of enemies in state s.
	int feature6(State state, int action); // distance to closest enemy in state s.
	int feature7(State state, int action); // weapon's cool down frames.
	// Boolean features
	int feature8(State state, int action); // action a takes us most likely closer to an enemy.
	int feature9(State state, int action); // action a takes us most likely away from an enemy.
	int feature10(State state, int action); // action a will most likely hurt the enemy.
	int feature11(State state, int action); // action a will most likely kill the enemy.
	int feature12(State state, int action); // action a will most likely cause us damage from the enemy.
	int feature13(State state, int action); // action a will most likely cause us death from the enemy.
	int feature14(State state, int action); // Whether the unit is under attack or not.
};