{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zmPfMTgR7BRs"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "G0vUYuq0-wtg"
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.models import load_model\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import Adam\n",
    "import math\n",
    "import numpy as np\n",
    "import random\n",
    "from collections import deque"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ch7zKs0T7YpV"
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('^GSPC_2011.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "B4SJB15M8x1-",
    "outputId": "f95cec1e-2bfd-43b3-cdea-8aebed5fd8ec"
   },
   "outputs": [],
   "source": [
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TeMtghAF8jy-",
    "outputId": "d0e868f1-3bdb-43f7-a6c6-a86e66f01753"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "\n",
    "class QLearningStockTrader:\n",
    "    def __init__(self, data, epsilon=0.1, alpha=0.1, gamma=0.9):\n",
    "        self.data = data\n",
    "        self.epsilon = epsilon\n",
    "        self.alpha = alpha\n",
    "        self.gamma = gamma\n",
    "        self.actions = [0, 1, 2]  # 0: Hold, 1: Buy, 2: Sell\n",
    "        self.q_table = {}  # Initialize Q-table\n",
    "\n",
    "    def get_state(self, index):\n",
    "        state = tuple(self.data.iloc[index][['Open', 'Close', 'High', 'Low', 'Volume']].values)\n",
    "        return state\n",
    "\n",
    "    def get_action(self, state):\n",
    "        if random.uniform(0, 1) < self.epsilon:\n",
    "            return random.choice(self.actions)\n",
    "        else:\n",
    "            return np.argmax(self.q_table.get(state, [0, 0, 0]))\n",
    "\n",
    "    def get_reward(self, action, current_price, next_price):\n",
    "        if action == 1:\n",
    "            return next_price - current_price\n",
    "        elif action == 2:\n",
    "            return current_price - next_price\n",
    "        return 0\n",
    "\n",
    "    def update_q_value(self, state, action, reward, next_state):\n",
    "\n",
    "        current_q_table_value = self.q_table.get(state, [0, 0, 0])[action]\n",
    "\n",
    "        max_future_q = np.max(self.q_table.get(next_state, [0, 0, 0]))\n",
    "\n",
    "        new_q = current_q_table_value +(reward + self.gamma * max_future_q - current_q_table_value)\n",
    "        if state not in self.q_table:\n",
    "            self.q_table[state] = [0, 0, 0]\n",
    "        self.q_table[state][action] = new_q\n",
    "\n",
    "    def train(self, episodes=100):\n",
    "        for episode in range(episodes):\n",
    "            total_reward = 0\n",
    "            for i in range(len(self.data) - 1):\n",
    "                current_state = self.get_state(i) #Choosing Random State\n",
    "                action = self.get_action(current_state) #Choosing Random Action based on current state\n",
    "                current_price = self.data.iloc[i]['Close'] # Price of stock is closing price of stock on the particular day\n",
    "                next_price = self.data.iloc[i + 1]['Close']\n",
    "                reward = self.get_reward(action, current_price, next_price)\n",
    "                next_state = self.get_state(i + 1)\n",
    "                self.update_q_value(current_state, action, reward, next_state)\n",
    "                total_reward += reward\n",
    "\n",
    "            print(f\"Episode {episode + 1}: Total Reward: {total_reward}\")\n",
    "        return self.q_table\n",
    "\n",
    "    def test(self):\n",
    "\n",
    "        total_profit = 0\n",
    "        for i in range(len(self.data) - 1):\n",
    "            current_state = self.get_state(i)\n",
    "            action = self.get_action(current_state)\n",
    "            current_price = self.data.iloc[i]['Close']\n",
    "            next_price = self.data.iloc[i + 1]['Close']\n",
    "            reward = self.get_reward(action, current_price, next_price)\n",
    "            total_profit += reward\n",
    "        print(f\"Total Profit after testing: {total_profit}\")\n",
    "\n",
    "\n",
    "\n",
    "data = pd.read_csv('^GSPC_2011.csv')\n",
    "\n",
    "\n",
    "\n",
    "agent = QLearningStockTrader(data)\n",
    "\n",
    "\n",
    "q_table = agent.train(episodes=50)\n",
    "\n",
    "\n",
    "agent.test()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3GWq7Q2Qduxi",
    "outputId": "62a8faf5-e1b3-463a-db93-f7c52ec012a8"
   },
   "outputs": [],
   "source": [
    "for i in q_table.keys():\n",
    "    print(q_table[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "v1xyITnmUJ05",
    "outputId": "bece0b1d-0682-4853-8c05-7bb4801e31de"
   },
   "outputs": [],
   "source": [
    "taken_actions=[]\n",
    "for i in q_table.keys():\n",
    "   for j in q_table[i]:\n",
    "        taken_actions.append(actions[np.argmax(q_table[i])])\n",
    "for i in range(253):\n",
    "    if taken_actions[i]==0:\n",
    "      print(i,\"Hold\")\n",
    "    elif taken_actions[i]==1:\n",
    "       print(i,\"Buy\")\n",
    "    else:\n",
    "        print(i,\"Sell\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "I_G0NBtgc3Fp"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}