V9_Goku.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/abdulazeezb/1_SparkFundsInvestmentAnalysis/blob/master/V9_Goku.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Checking GPU Version"
      ],
      "metadata": {
        "id": "NAjscLW7Sk9V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Not connected to a GPU')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "metadata": {
        "id": "t8nxH9ZvSaY6",
        "outputId": "182526a7-4a80-4f82-8059-2acee22cdc1a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mon May  1 12:15:48 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   42C    P8    11W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from psutil import virtual_memory\n",
        "ram_gb = virtual_memory().total / 1e9\n",
        "print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n",
        "\n",
        "if ram_gb < 20:\n",
        "  print('Not using a high-RAM runtime')\n",
        "else:\n",
        "  print('You are using a high-RAM runtime!')"
      ],
      "metadata": {
        "id": "KEPzY7qCS2Lc",
        "outputId": "f3151c28-167c-4240-8245-518a75bdb5ec",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Your runtime has 27.3 gigabytes of available RAM\n",
            "\n",
            "You are using a high-RAM runtime!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QMMqmdiYMkvi"
      },
      "source": [
        "## Part 1: Getting Started - Install Python Packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "23TOba33L4qf",
        "outputId": "588c2ac5-2933-4a08-be76-f86976d7e2ef"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting wrds\n",
            "  Downloading wrds-3.1.6-py3-none-any.whl (12 kB)\n",
            "Collecting sqlalchemy<2\n",
            "  Downloading SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m50.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting psycopg2-binary\n",
            "  Downloading psycopg2_binary-2.9.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m106.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from wrds) (1.5.3)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from wrds) (1.10.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from wrds) (1.22.4)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from sqlalchemy<2->wrds) (2.0.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->wrds) (2022.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->wrds) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->wrds) (1.16.0)\n",
            "Installing collected packages: sqlalchemy, psycopg2-binary, wrds\n",
            "  Attempting uninstall: sqlalchemy\n",
            "    Found existing installation: SQLAlchemy 2.0.10\n",
            "    Uninstalling SQLAlchemy-2.0.10:\n",
            "      Successfully uninstalled SQLAlchemy-2.0.10\n",
            "Successfully installed psycopg2-binary-2.9.6 sqlalchemy-1.4.48 wrds-3.1.6\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting swig\n",
            "  Downloading swig-4.1.1-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m51.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: swig\n",
            "Successfully installed swig-4.1.1\n",
            "⏬ Downloading https://github.com/conda-forge/miniforge/releases/download/23.1.0-1/Mambaforge-23.1.0-1-Linux-x86_64.sh...\n",
            "📦 Installing...\n",
            "📌 Adjusting configuration...\n",
            "🩹 Patching environment...\n",
            "⏲ Done in 0:00:09\n",
            "🔁 Restarting kernel...\n",
            "Selecting previously unselected package libgl1-mesa-glx:amd64.\n",
            "(Reading database ... 122518 files and directories currently installed.)\n",
            "Preparing to unpack .../libgl1-mesa-glx_21.2.6-0ubuntu0.1~20.04.2_amd64.deb ...\n",
            "Unpacking libgl1-mesa-glx:amd64 (21.2.6-0ubuntu0.1~20.04.2) ...\n",
            "Selecting previously unselected package swig4.0.\n",
            "Preparing to unpack .../swig4.0_4.0.1-5build1_amd64.deb ...\n",
            "Unpacking swig4.0 (4.0.1-5build1) ...\n",
            "Selecting previously unselected package swig.\n",
            "Preparing to unpack .../swig_4.0.1-5build1_all.deb ...\n",
            "Unpacking swig (4.0.1-5build1) ...\n",
            "Setting up libgl1-mesa-glx:amd64 (21.2.6-0ubuntu0.1~20.04.2) ...\n",
            "Setting up swig4.0 (4.0.1-5build1) ...\n",
            "Setting up swig (4.0.1-5build1) ...\n",
            "Processing triggers for man-db (2.9.1-1) ...\n"
          ]
        }
      ],
      "source": [
        "## install finrl library\n",
        "!pip install wrds\n",
        "!pip install swig\n",
        "!pip install -q condacolab\n",
        "import condacolab\n",
        "condacolab.install()\n",
        "!apt-get update -y -qq && apt-get install -y -qq cmake libopenmpi-dev python3-dev zlib1g-dev libgl1-mesa-glx swig"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/AI4Finance-Foundation/FinRL.git"
      ],
      "metadata": {
        "id": "haJsmd15TCdH",
        "outputId": "c572b46e-a989-431d-f68e-df9a90fc2cb1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/AI4Finance-Foundation/FinRL.git\n",
            "  Cloning https://github.com/AI4Finance-Foundation/FinRL.git to /tmp/pip-req-build-h6jhwi36\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/AI4Finance-Foundation/FinRL.git /tmp/pip-req-build-h6jhwi36\n",
            "  Resolved https://github.com/AI4Finance-Foundation/FinRL.git to commit 58a8ec268c71941ecab9666afbf5db87562ba3b3\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting elegantrl@ git+https://github.com/AI4Finance-Foundation/ElegantRL.git#egg=elegantrl\n",
            "  Cloning https://github.com/AI4Finance-Foundation/ElegantRL.git to /tmp/pip-install-wj363nwm/elegantrl_4855eabf3c6a4978bbfbaa0a644b8e46\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/AI4Finance-Foundation/ElegantRL.git /tmp/pip-install-wj363nwm/elegantrl_4855eabf3c6a4978bbfbaa0a644b8e46\n",
            "  Resolved https://github.com/AI4Finance-Foundation/ElegantRL.git to commit b974a806e6235f59055c954418e54640fa549331\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2\n",
            "  Cloning https://github.com/quantopian/pyfolio.git to /tmp/pip-install-wj363nwm/pyfolio_1a00986807c24dcaa3a34cd7d24439ca\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/quantopian/pyfolio.git /tmp/pip-install-wj363nwm/pyfolio_1a00986807c24dcaa3a34cd7d24439ca\n",
            "  Resolved https://github.com/quantopian/pyfolio.git to commit 4b901f6d73aa02ceb6d04b7d83502e5c6f2e81aa\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pandas>=1.1.5\n",
            "  Downloading pandas-2.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m98.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting numpy>=1.17.3\n",
            "  Downloading numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.3/17.3 MB\u001b[0m \u001b[31m80.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ccxt>=1.66.32\n",
            "  Downloading ccxt-3.0.85-py2.py3-none-any.whl (3.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m113.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting wrds>=3.1.6\n",
            "  Using cached wrds-3.1.6-py3-none-any.whl (12 kB)\n",
            "Collecting lz4\n",
            "  Downloading lz4-4.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m89.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting stable-baselines3<2.0.0,>=1.6.2\n",
            "  Downloading stable_baselines3-1.8.0-py3-none-any.whl (174 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m174.5/174.5 kB\u001b[0m \u001b[31m25.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting exchange_calendars==3.6.3\n",
            "  Downloading exchange_calendars-3.6.3.tar.gz (152 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.8/152.8 kB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting yfinance\n",
            "  Downloading yfinance-0.2.18-py2.py3-none-any.whl (60 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.3/60.3 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting alpaca_trade_api>=2.1.0\n",
            "  Downloading alpaca_trade_api-3.0.0-py3-none-any.whl (33 kB)\n",
            "Collecting matplotlib\n",
            "  Downloading matplotlib-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.6/11.6 MB\u001b[0m \u001b[31m102.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gym>=0.17\n",
            "  Downloading gym-0.26.2.tar.gz (721 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m721.7/721.7 kB\u001b[0m \u001b[31m61.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting ray[default,tune]>=2.0.0\n",
            "  Downloading ray-2.4.0-cp310-cp310-manylinux2014_x86_64.whl (58.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.6/58.6 MB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting stockstats>=0.4.0\n",
            "  Downloading stockstats-0.5.2-py2.py3-none-any.whl (20 kB)\n",
            "Collecting importlib-metadata==4.13.0\n",
            "  Downloading importlib_metadata-4.13.0-py3-none-any.whl (23 kB)\n",
            "Collecting jqdatasdk\n",
            "  Downloading jqdatasdk-1.8.11-py3-none-any.whl (158 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.2/158.2 kB\u001b[0m \u001b[31m25.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting scikit-learn>=0.21.0\n",
            "  Downloading scikit_learn-1.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (9.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.6/9.6 MB\u001b[0m \u001b[31m115.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gputil\n",
            "  Downloading GPUtil-1.4.0.tar.gz (5.5 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting tensorboardX\n",
            "  Downloading tensorboardX-2.6-py2.py3-none-any.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyluach\n",
            "  Downloading pyluach-2.2.0-py3-none-any.whl (25 kB)\n",
            "Collecting python-dateutil\n",
            "  Downloading python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m247.7/247.7 kB\u001b[0m \u001b[31m34.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pytz\n",
            "  Downloading pytz-2023.3-py2.py3-none-any.whl (502 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m502.3/502.3 kB\u001b[0m \u001b[31m55.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: toolz in /usr/local/lib/python3.10/site-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (0.12.0)\n",
            "Collecting korean_lunar_calendar\n",
            "  Downloading korean_lunar_calendar-0.3.1-py3-none-any.whl (9.0 kB)\n",
            "Collecting zipp>=0.5\n",
            "  Downloading zipp-3.15.0-py3-none-any.whl (6.8 kB)\n",
            "Collecting websockets<11,>=9.0\n",
            "  Downloading websockets-10.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (106 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.8/106.8 kB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiohttp==3.8.1\n",
            "  Downloading aiohttp-3.8.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m78.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting deprecation==2.1.0\n",
            "  Downloading deprecation-2.1.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting websocket-client<2,>=0.56.0\n",
            "  Downloading websocket_client-1.5.1-py3-none-any.whl (55 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.9/55.9 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting msgpack==1.0.3\n",
            "  Downloading msgpack-1.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (323 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m323.7/323.7 kB\u001b[0m \u001b[31m41.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests<3,>2 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (2.28.2)\n",
            "Requirement already satisfied: urllib3<2,>1.24 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (1.26.15)\n",
            "Collecting PyYAML==6.0\n",
            "  Downloading PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (682 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m682.2/682.2 kB\u001b[0m \u001b[31m60.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m37.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m23.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting charset-normalizer<3.0,>=2.0\n",
            "  Downloading charset_normalizer-2.1.1-py3-none-any.whl (39 kB)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting attrs>=17.3.0\n",
            "  Downloading attrs-23.1.0-py3-none-any.whl (61 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.2/61.2 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting packaging\n",
            "  Downloading packaging-23.1-py3-none-any.whl (48 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.9/48.9 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiodns>=1.1.1\n",
            "  Downloading aiodns-3.0.0-py3-none-any.whl (5.0 kB)\n",
            "Requirement already satisfied: certifi>=2018.1.18 in /usr/local/lib/python3.10/site-packages (from ccxt>=1.66.32->finrl==0.3.5) (2022.12.7)\n",
            "Requirement already satisfied: setuptools>=60.9.0 in /usr/local/lib/python3.10/site-packages (from ccxt>=1.66.32->finrl==0.3.5) (65.6.3)\n",
            "Requirement already satisfied: cryptography>=2.6.1 in /usr/local/lib/python3.10/site-packages (from ccxt>=1.66.32->finrl==0.3.5) (40.0.1)\n",
            "Collecting gym-notices>=0.0.4\n",
            "  Downloading gym_notices-0.0.8-py3-none-any.whl (3.0 kB)\n",
            "Collecting cloudpickle>=1.2.0\n",
            "  Downloading cloudpickle-2.2.1-py3-none-any.whl (25 kB)\n",
            "Collecting tzdata>=2022.1\n",
            "  Downloading tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.8/341.8 kB\u001b[0m \u001b[31m47.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting click>=7.0\n",
            "  Downloading click-8.1.3-py3-none-any.whl (96 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m96.6/96.6 kB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting filelock\n",
            "  Downloading filelock-3.12.0-py3-none-any.whl (10 kB)\n",
            "Collecting grpcio<=1.51.3,>=1.42.0\n",
            "  Downloading grpcio-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.8/4.8 MB\u001b[0m \u001b[31m104.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting virtualenv<20.21.1,>=20.0.24\n",
            "  Downloading virtualenv-20.21.0-py3-none-any.whl (8.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.7/8.7 MB\u001b[0m \u001b[31m112.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting protobuf!=3.19.5,>=3.15.3\n",
            "  Downloading protobuf-4.22.3-cp37-abi3-manylinux2014_x86_64.whl (302 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.4/302.4 kB\u001b[0m \u001b[31m40.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jsonschema\n",
            "  Downloading jsonschema-4.17.3-py3-none-any.whl (90 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.4/90.4 kB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tabulate\n",
            "  Downloading tabulate-0.9.0-py3-none-any.whl (35 kB)\n",
            "Collecting gpustat>=1.0.0\n",
            "  Downloading gpustat-1.1.tar.gz (97 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.9/97.9 kB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting aiohttp-cors\n",
            "  Downloading aiohttp_cors-0.7.0-py3-none-any.whl (27 kB)\n",
            "Collecting smart-open\n",
            "  Downloading smart_open-6.3.0-py3-none-any.whl (56 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.8/56.8 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting prometheus-client>=0.7.1\n",
            "  Downloading prometheus_client-0.16.0-py3-none-any.whl (122 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m122.5/122.5 kB\u001b[0m \u001b[31m21.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pydantic\n",
            "  Downloading pydantic-1.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m85.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting py-spy>=0.2.0\n",
            "  Downloading py_spy-0.3.14-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl (3.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m84.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting colorful\n",
            "  Downloading colorful-0.5.5-py2.py3-none-any.whl (201 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m201.4/201.4 kB\u001b[0m \u001b[31m30.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opencensus\n",
            "  Downloading opencensus-0.11.2-py2.py3-none-any.whl (128 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m128.2/128.2 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting joblib>=1.1.1\n",
            "  Downloading joblib-1.2.0-py3-none-any.whl (297 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m298.0/298.0 kB\u001b[0m \u001b[31m38.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting scipy>=1.3.2\n",
            "  Downloading scipy-1.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m34.4/34.4 MB\u001b[0m \u001b[31m49.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting threadpoolctl>=2.0.0\n",
            "  Downloading threadpoolctl-3.1.0-py3-none-any.whl (14 kB)\n",
            "Collecting gym>=0.17\n",
            "  Downloading gym-0.21.0.tar.gz (1.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m96.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting torch>=1.11\n",
            "  Downloading torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl (619.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m619.9/619.9 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting protobuf!=3.19.5,>=3.15.3\n",
            "  Downloading protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m83.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sqlalchemy<2\n",
            "  Using cached SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)\n",
            "Collecting psycopg2-binary\n",
            "  Using cached psycopg2_binary-2.9.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
            "Collecting six\n",
            "  Downloading six-1.16.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting thriftpy2>=0.3.9\n",
            "  Downloading thriftpy2-0.4.16.tar.gz (643 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m643.4/643.4 kB\u001b[0m \u001b[31m70.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pymysql>=0.7.6\n",
            "  Downloading PyMySQL-1.0.3-py3-none-any.whl (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pillow>=6.2.0\n",
            "  Downloading Pillow-9.5.0-cp310-cp310-manylinux_2_28_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m119.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting cycler>=0.10\n",
            "  Downloading cycler-0.11.0-py3-none-any.whl (6.4 kB)\n",
            "Collecting contourpy>=1.0.1\n",
            "  Downloading contourpy-1.0.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (300 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m300.3/300.3 kB\u001b[0m \u001b[31m43.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyparsing>=2.3.1\n",
            "  Downloading pyparsing-3.0.9-py3-none-any.whl (98 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.3/98.3 kB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting kiwisolver>=1.0.1\n",
            "  Downloading kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m99.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fonttools>=4.22.0\n",
            "  Downloading fonttools-4.39.3-py3-none-any.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m78.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ipython>=3.2.3\n",
            "  Downloading ipython-8.13.1-py3-none-any.whl (797 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m797.6/797.6 kB\u001b[0m \u001b[31m72.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting seaborn>=0.7.1\n",
            "  Downloading seaborn-0.12.2-py3-none-any.whl (293 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m293.3/293.3 kB\u001b[0m \u001b[31m39.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting empyrical>=0.5.0\n",
            "  Downloading empyrical-0.5.5.tar.gz (52 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.8/52.8 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting lxml>=4.9.1\n",
            "  Downloading lxml-4.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (7.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m128.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting html5lib>=1.1\n",
            "  Downloading html5lib-1.1-py2.py3-none-any.whl (112 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.2/112.2 kB\u001b[0m \u001b[31m18.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting appdirs>=1.4.4\n",
            "  Downloading appdirs-1.4.4-py2.py3-none-any.whl (9.6 kB)\n",
            "Collecting beautifulsoup4>=4.11.1\n",
            "  Downloading beautifulsoup4-4.12.2-py3-none-any.whl (142 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.0/143.0 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multitasking>=0.0.7\n",
            "  Downloading multitasking-0.0.11-py3-none-any.whl (8.5 kB)\n",
            "Collecting frozendict>=2.3.4\n",
            "  Downloading frozendict-2.3.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pycares>=4.0.0\n",
            "  Downloading pycares-4.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (288 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m288.7/288.7 kB\u001b[0m \u001b[31m41.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting soupsieve>1.2\n",
            "  Downloading soupsieve-2.4.1-py3-none-any.whl (36 kB)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/site-packages (from cryptography>=2.6.1->ccxt>=1.66.32->finrl==0.3.5) (1.15.1)\n",
            "Collecting pandas-datareader>=0.2\n",
            "  Downloading pandas_datareader-0.10.0-py3-none-any.whl (109 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m109.5/109.5 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting blessed>=1.17.1\n",
            "  Downloading blessed-1.20.0-py2.py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.4/58.4 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-ml-py>=11.450.129\n",
            "  Downloading nvidia_ml_py-11.525.112-py3-none-any.whl (35 kB)\n",
            "Collecting psutil>=5.6.0\n",
            "  Downloading psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (282 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m282.1/282.1 kB\u001b[0m \u001b[31m37.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting webencodings\n",
            "  Downloading webencodings-0.5.1-py2.py3-none-any.whl (11 kB)\n",
            "Collecting matplotlib-inline\n",
            "  Downloading matplotlib_inline-0.1.6-py3-none-any.whl (9.4 kB)\n",
            "Collecting stack-data\n",
            "  Downloading stack_data-0.6.2-py3-none-any.whl (24 kB)\n",
            "Collecting decorator\n",
            "  Downloading decorator-5.1.1-py3-none-any.whl (9.1 kB)\n",
            "Collecting pexpect>4.3\n",
            "  Downloading pexpect-4.8.0-py2.py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.0/59.0 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting backcall\n",
            "  Downloading backcall-0.2.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30\n",
            "  Downloading prompt_toolkit-3.0.38-py3-none-any.whl (385 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m385.8/385.8 kB\u001b[0m \u001b[31m43.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jedi>=0.16\n",
            "  Downloading jedi-0.18.2-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m91.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting traitlets>=5\n",
            "  Downloading traitlets-5.9.0-py3-none-any.whl (117 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.4/117.4 kB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pickleshare\n",
            "  Downloading pickleshare-0.7.5-py2.py3-none-any.whl (6.9 kB)\n",
            "Collecting pygments>=2.4.0\n",
            "  Downloading Pygments-2.15.1-py3-none-any.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m83.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests<3,>2->alpaca_trade_api>=2.1.0->finrl==0.3.5) (3.4)\n",
            "Collecting greenlet!=0.4.17\n",
            "  Downloading greenlet-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (613 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m613.7/613.7 kB\u001b[0m \u001b[31m60.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ply<4.0,>=3.4\n",
            "  Downloading ply-3.11-py2.py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.6/49.6 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu11==10.9.0.58\n",
            "  Downloading nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux1_x86_64.whl (168.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m168.4/168.4 MB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cublas-cu11==11.10.3.66\n",
            "  Downloading nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl (317.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.1/317.1 MB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jinja2\n",
            "  Downloading Jinja2-3.1.2-py3-none-any.whl (133 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m133.1/133.1 kB\u001b[0m \u001b[31m19.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-nvrtc-cu11==11.7.99\n",
            "  Downloading nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl (21.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.0/21.0 MB\u001b[0m \u001b[31m67.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu11==2.14.3\n",
            "  Downloading nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl (177.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.1/177.1 MB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-extensions\n",
            "  Using cached typing_extensions-4.5.0-py3-none-any.whl (27 kB)\n",
            "Collecting nvidia-cusolver-cu11==11.4.0.1\n",
            "  Downloading nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl (102.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.6/102.6 MB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu11==11.7.99\n",
            "  Downloading nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl (849 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m849.3/849.3 kB\u001b[0m \u001b[31m68.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu11==8.5.0.96\n",
            "  Downloading nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl (557.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m557.1/557.1 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting networkx\n",
            "  Downloading networkx-3.1-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m63.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting triton==2.0.0\n",
            "  Downloading triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (63.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.3/63.3 MB\u001b[0m \u001b[31m23.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sympy\n",
            "  Downloading sympy-1.11.1-py3-none-any.whl (6.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.5/6.5 MB\u001b[0m \u001b[31m119.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu11==11.7.91\n",
            "  Downloading nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl (98 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.6/98.6 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu11==11.7.101\n",
            "  Downloading nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl (11.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.8/11.8 MB\u001b[0m \u001b[31m93.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu11==10.2.10.91\n",
            "  Downloading nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl (54.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.6/54.6 MB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu11==11.7.4.91\n",
            "  Downloading nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl (173.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m173.2/173.2 MB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wheel in /usr/local/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (0.40.0)\n",
            "Collecting cmake\n",
            "  Downloading cmake-3.26.3-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (24.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.0/24.0 MB\u001b[0m \u001b[31m70.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting lit\n",
            "  Downloading lit-16.0.2.tar.gz (137 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.9/137.9 kB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting platformdirs<4,>=2.4\n",
            "  Downloading platformdirs-3.5.0-py3-none-any.whl (15 kB)\n",
            "Collecting distlib<1,>=0.3.6\n",
            "  Downloading distlib-0.3.6-py2.py3-none-any.whl (468 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.5/468.5 kB\u001b[0m \u001b[31m51.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gym[box2d]\n",
            "  Downloading gym-0.26.1.tar.gz (719 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m719.9/719.9 kB\u001b[0m \u001b[31m67.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.26.0.tar.gz (710 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m710.3/710.3 kB\u001b[0m \u001b[31m63.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.25.2.tar.gz (734 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m734.5/734.5 kB\u001b[0m \u001b[31m58.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.25.1.tar.gz (732 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m732.2/732.2 kB\u001b[0m \u001b[31m58.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.25.0.tar.gz (720 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m720.4/720.4 kB\u001b[0m \u001b[31m62.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.24.1.tar.gz (696 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m696.4/696.4 kB\u001b[0m \u001b[31m56.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.24.0.tar.gz (694 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m694.4/694.4 kB\u001b[0m \u001b[31m61.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.23.1.tar.gz (626 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m626.2/626.2 kB\u001b[0m \u001b[31m51.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.23.0.tar.gz (624 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m624.4/624.4 kB\u001b[0m \u001b[31m57.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.22.0.tar.gz (631 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m631.1/631.1 kB\u001b[0m \u001b[31m53.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting box2d-py==2.3.5\n",
            "  Downloading box2d-py-2.3.5.tar.gz (374 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m374.4/374.4 kB\u001b[0m \u001b[31m45.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pyglet>=1.4.0\n",
            "  Downloading pyglet-2.0.5-py3-none-any.whl (831 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m831.3/831.3 kB\u001b[0m \u001b[31m68.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0\n",
            "  Downloading pyrsistent-0.19.3-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opencensus-context>=0.1.3\n",
            "  Downloading opencensus_context-0.1.3-py2.py3-none-any.whl (5.1 kB)\n",
            "Collecting google-api-core<3.0.0,>=1.0.0\n",
            "  Downloading google_api_core-2.11.0-py3-none-any.whl (120 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m120.3/120.3 kB\u001b[0m \u001b[31m18.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting wcwidth>=0.1.4\n",
            "  Downloading wcwidth-0.2.6-py2.py3-none-any.whl (29 kB)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=2.6.1->ccxt>=1.66.32->finrl==0.3.5) (2.21)\n",
            "Collecting google-auth<3.0dev,>=2.14.1\n",
            "  Downloading google_auth-2.17.3-py2.py3-none-any.whl (178 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.2/178.2 kB\u001b[0m \u001b[31m24.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting googleapis-common-protos<2.0dev,>=1.56.2\n",
            "  Downloading googleapis_common_protos-1.59.0-py2.py3-none-any.whl (223 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m223.6/223.6 kB\u001b[0m \u001b[31m31.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting parso<0.9.0,>=0.8.0\n",
            "  Downloading parso-0.8.3-py2.py3-none-any.whl (100 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100.8/100.8 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ptyprocess>=0.5\n",
            "  Downloading ptyprocess-0.7.0-py2.py3-none-any.whl (13 kB)\n",
            "Collecting MarkupSafe>=2.0\n",
            "  Downloading MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\n",
            "Collecting executing>=1.2.0\n",
            "  Downloading executing-1.2.0-py2.py3-none-any.whl (24 kB)\n",
            "Collecting pure-eval\n",
            "  Downloading pure_eval-0.2.2-py3-none-any.whl (11 kB)\n",
            "Collecting asttokens>=2.1.0\n",
            "  Downloading asttokens-2.2.1-py2.py3-none-any.whl (26 kB)\n",
            "Collecting mpmath>=0.19\n",
            "  Downloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m59.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting cachetools<6.0,>=2.0.0\n",
            "  Downloading cachetools-5.3.0-py3-none-any.whl (9.3 kB)\n",
            "Collecting rsa<5,>=3.1.4\n",
            "  Downloading rsa-4.9-py3-none-any.whl (34 kB)\n",
            "Collecting pyasn1-modules>=0.2.1\n",
            "  Downloading pyasn1_modules-0.3.0-py2.py3-none-any.whl (181 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m181.3/181.3 kB\u001b[0m \u001b[31m24.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyasn1<0.6.0,>=0.4.6\n",
            "  Downloading pyasn1-0.5.0-py2.py3-none-any.whl (83 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m83.9/83.9 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: finrl, exchange_calendars, gym, elegantrl, gputil, pyfolio, empyrical, gpustat, thriftpy2, box2d-py, lit\n",
            "  Building wheel for finrl (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for finrl: filename=finrl-0.3.5-py3-none-any.whl size=4668722 sha256=b5d9990feddfce4dac1109677bd1616fcf78abb54bcf1b30c280638a7696b8d0\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-pl5j81lr/wheels/72/3b/1a/0fc805a8cc65ecd5bfe4f74a3c586b6075678b8ba53fd8f749\n",
            "  Building wheel for exchange_calendars (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for exchange_calendars: filename=exchange_calendars-3.6.3-py3-none-any.whl size=182636 sha256=9c4788a9a73fe8f026fa94f48f3387c74732b2bf84529d1829c693d4b4094b33\n",
            "  Stored in directory: /root/.cache/pip/wheels/93/ca/8f/1e1c90cc79fb3ca9b5413ff58e3fccf3baf2182c994c6dfd37\n",
            "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "  \n",
            "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py bdist_wheel\u001b[0m did not run successfully.\n",
            "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "  \u001b[31m╰─>\u001b[0m See above for output.\n",
            "  \n",
            "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "  Building wheel for gym (setup.py) ... \u001b[?25lerror\n",
            "\u001b[31m  ERROR: Failed building wheel for gym\u001b[0m\u001b[31m\n",
            "\u001b[0m\u001b[?25h  Running setup.py clean for gym\n",
            "  Building wheel for elegantrl (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for elegantrl: filename=elegantrl-0.3.6-py3-none-any.whl size=195067 sha256=0e465747a460c5f03c355315b3c9103c547757514189fdbf9d0aa03184316980\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-pl5j81lr/wheels/c0/51/a5/b05f165548221bc570f7223babd33e2992fa873cdcebe2d229\n",
            "  Building wheel for gputil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gputil: filename=GPUtil-1.4.0-py3-none-any.whl size=7408 sha256=3129cb0062f9460ffe2ce8f2611b9f603ab38917f444e8e1a5e080e1f8cd8157\n",
            "  Stored in directory: /root/.cache/pip/wheels/a9/8a/bd/81082387151853ab8b6b3ef33426e98f5cbfebc3c397a9d4d0\n",
            "  Building wheel for pyfolio (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyfolio: filename=pyfolio-0.9.2+75.g4b901f6-py3-none-any.whl size=75775 sha256=60f2932870cb35d21b8877070af622a501367c15533396200f58a49f9efcd59e\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-pl5j81lr/wheels/c4/1d/91/8ab5d1c88a11b06a63dcd6a69ea81547e2247123232949bb26\n",
            "  Building wheel for empyrical (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for empyrical: filename=empyrical-0.5.5-py3-none-any.whl size=39779 sha256=72316305ab0051cf1bc5476f9de95c32cd315f5c4625f6eb4455be5214a72b01\n",
            "  Stored in directory: /root/.cache/pip/wheels/0e/2e/f2/d6d2d9a1eb8fbbd9949bb5d4c00f753e3b74e5bd7ed10b1d36\n",
            "  Building wheel for gpustat (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gpustat: filename=gpustat-1.1-py3-none-any.whl size=26298 sha256=21959f61120d0d109a9df0adf64d48ba73b9eb46715f695e8a891b5369b34f75\n",
            "  Stored in directory: /root/.cache/pip/wheels/ee/d0/2c/1e02440645c2318ba03aea99993a44a9108dc8f74de0bd370b\n",
            "  Building wheel for thriftpy2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for thriftpy2: filename=thriftpy2-0.4.16-cp310-cp310-linux_x86_64.whl size=522576 sha256=32572ff4b8eb399560d77acb096f044d18b1589634ee5c3b964fe3194394d5e3\n",
            "  Stored in directory: /root/.cache/pip/wheels/95/f5/1d/fe404692e1c8aaea45220c322d1d0f32c9fd40eb0e2bdd571e\n",
            "  Building wheel for box2d-py (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for box2d-py: filename=box2d_py-2.3.5-cp310-cp310-linux_x86_64.whl size=494649 sha256=2aa8cacf8f18351bb871be1de510643910c34aec28d5cdce2a9ccccda6950925\n",
            "  Stored in directory: /root/.cache/pip/wheels/db/8f/6a/eaaadf056fba10a98d986f6dce954e6201ba3126926fc5ad9e\n",
            "  Building wheel for lit (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for lit: filename=lit-16.0.2-py3-none-any.whl size=88190 sha256=11cfa31c73a8363f9eeb62e3f0b39cf2f0c8710e07e6eceda80fd460afb7d230\n",
            "  Stored in directory: /root/.cache/pip/wheels/4d/31/6f/140862d5c69ddd665b3ed9485f4c93aa9c84cb34b359cba3ce\n",
            "Successfully built finrl exchange_calendars elegantrl gputil pyfolio empyrical gpustat thriftpy2 box2d-py lit\n",
            "Failed to build gym\n",
            "Installing collected packages: webencodings, wcwidth, pytz, pyglet, py-spy, pure-eval, ptyprocess, ply, pickleshare, opencensus-context, nvidia-ml-py, multitasking, msgpack, mpmath, lit, korean_lunar_calendar, gputil, executing, distlib, colorful, cmake, box2d-py, backcall, appdirs, zipp, websockets, websocket-client, tzdata, typing-extensions, traitlets, threadpoolctl, tabulate, sympy, soupsieve, smart-open, six, PyYAML, pyrsistent, pyparsing, pymysql, pyluach, pygments, pyasn1, psycopg2-binary, psutil, protobuf, prompt-toolkit, prometheus-client, platformdirs, pillow, pexpect, parso, packaging, nvidia-nvtx-cu11, nvidia-nccl-cu11, nvidia-cusparse-cu11, nvidia-curand-cu11, nvidia-cufft-cu11, nvidia-cuda-runtime-cu11, nvidia-cuda-nvrtc-cu11, nvidia-cuda-cupti-cu11, nvidia-cublas-cu11, numpy, networkx, multidict, MarkupSafe, lz4, lxml, kiwisolver, joblib, grpcio, greenlet, frozenlist, frozendict, fonttools, filelock, decorator, cycler, cloudpickle, click, charset-normalizer, cachetools, attrs, async-timeout, yarl, virtualenv, thriftpy2, tensorboardX, sqlalchemy, scipy, rsa, python-dateutil, pydantic, pycares, pyasn1-modules, nvidia-cusolver-cu11, nvidia-cudnn-cu11, matplotlib-inline, jsonschema, jinja2, jedi, importlib-metadata, html5lib, gym, googleapis-common-protos, deprecation, contourpy, blessed, beautifulsoup4, asttokens, aiosignal, stack-data, scikit-learn, ray, pandas, matplotlib, gpustat, google-auth, aiohttp, aiodns, yfinance, wrds, stockstats, seaborn, pandas-datareader, jqdatasdk, ipython, google-api-core, exchange_calendars, ccxt, alpaca_trade_api, aiohttp-cors, opencensus, empyrical, pyfolio, triton, torch, stable-baselines3, elegantrl, finrl\n",
            "  Attempting uninstall: charset-normalizer\n",
            "    Found existing installation: charset-normalizer 3.1.0\n",
            "    Uninstalling charset-normalizer-3.1.0:\n",
            "      Successfully uninstalled charset-normalizer-3.1.0\n",
            "  Running setup.py install for gym ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[33m  DEPRECATION: gym was installed using the legacy 'setup.py install' method, because a wheel could not be built for it. pip 23.1 will enforce this behaviour change. A possible replacement is to fix the wheel build issue reported above. Discussion can be found at https://github.com/pypa/pip/issues/8368\u001b[0m\u001b[33m\n",
            "\u001b[0mSuccessfully installed MarkupSafe-2.1.2 PyYAML-6.0 aiodns-3.0.0 aiohttp-3.8.1 aiohttp-cors-0.7.0 aiosignal-1.3.1 alpaca_trade_api-3.0.0 appdirs-1.4.4 asttokens-2.2.1 async-timeout-4.0.2 attrs-23.1.0 backcall-0.2.0 beautifulsoup4-4.12.2 blessed-1.20.0 box2d-py-2.3.5 cachetools-5.3.0 ccxt-3.0.85 charset-normalizer-2.1.1 click-8.1.3 cloudpickle-2.2.1 cmake-3.26.3 colorful-0.5.5 contourpy-1.0.7 cycler-0.11.0 decorator-5.1.1 deprecation-2.1.0 distlib-0.3.6 elegantrl-0.3.6 empyrical-0.5.5 exchange_calendars-3.6.3 executing-1.2.0 filelock-3.12.0 finrl-0.3.5 fonttools-4.39.3 frozendict-2.3.8 frozenlist-1.3.3 google-api-core-2.11.0 google-auth-2.17.3 googleapis-common-protos-1.59.0 gpustat-1.1 gputil-1.4.0 greenlet-2.0.2 grpcio-1.51.3 gym-0.21.0 html5lib-1.1 importlib-metadata-4.13.0 ipython-8.13.1 jedi-0.18.2 jinja2-3.1.2 joblib-1.2.0 jqdatasdk-1.8.11 jsonschema-4.17.3 kiwisolver-1.4.4 korean_lunar_calendar-0.3.1 lit-16.0.2 lxml-4.9.2 lz4-4.3.2 matplotlib-3.7.1 matplotlib-inline-0.1.6 mpmath-1.3.0 msgpack-1.0.3 multidict-6.0.4 multitasking-0.0.11 networkx-3.1 numpy-1.24.3 nvidia-cublas-cu11-11.10.3.66 nvidia-cuda-cupti-cu11-11.7.101 nvidia-cuda-nvrtc-cu11-11.7.99 nvidia-cuda-runtime-cu11-11.7.99 nvidia-cudnn-cu11-8.5.0.96 nvidia-cufft-cu11-10.9.0.58 nvidia-curand-cu11-10.2.10.91 nvidia-cusolver-cu11-11.4.0.1 nvidia-cusparse-cu11-11.7.4.91 nvidia-ml-py-11.525.112 nvidia-nccl-cu11-2.14.3 nvidia-nvtx-cu11-11.7.91 opencensus-0.11.2 opencensus-context-0.1.3 packaging-23.1 pandas-2.0.1 pandas-datareader-0.10.0 parso-0.8.3 pexpect-4.8.0 pickleshare-0.7.5 pillow-9.5.0 platformdirs-3.5.0 ply-3.11 prometheus-client-0.16.0 prompt-toolkit-3.0.38 protobuf-3.20.3 psutil-5.9.5 psycopg2-binary-2.9.6 ptyprocess-0.7.0 pure-eval-0.2.2 py-spy-0.3.14 pyasn1-0.5.0 pyasn1-modules-0.3.0 pycares-4.3.0 pydantic-1.10.7 pyfolio-0.9.2+75.g4b901f6 pyglet-2.0.5 pygments-2.15.1 pyluach-2.2.0 pymysql-1.0.3 pyparsing-3.0.9 pyrsistent-0.19.3 python-dateutil-2.8.2 pytz-2023.3 ray-2.4.0 rsa-4.9 scikit-learn-1.2.2 scipy-1.10.1 seaborn-0.12.2 six-1.16.0 smart-open-6.3.0 soupsieve-2.4.1 sqlalchemy-1.4.48 stable-baselines3-1.8.0 stack-data-0.6.2 stockstats-0.5.2 sympy-1.11.1 tabulate-0.9.0 tensorboardX-2.6 threadpoolctl-3.1.0 thriftpy2-0.4.16 torch-2.0.0 traitlets-5.9.0 triton-2.0.0 typing-extensions-4.5.0 tzdata-2023.3 virtualenv-20.21.0 wcwidth-0.2.6 webencodings-0.5.1 websocket-client-1.5.1 websockets-10.4 wrds-3.1.6 yarl-1.9.2 yfinance-0.2.18 zipp-3.15.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.2 Check if the additional packages needed are present, if not install them"
      ],
      "metadata": {
        "id": "DMaIlOcVTJfp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# !pip install trading_calendars\n",
        "# !pip install alpaca_trade_api\n",
        "# !pip install ccxt\n",
        "# !pip install jqdatasdk\n",
        "# !pip install wrds\n",
        "\n",
        "# !pip install lz4\n",
        "# !pip install ray[tune]\n",
        "# !pip install tensorboardX\n",
        "# !pip install gputil\n",
        "\n",
        "#%%capture\n",
        "if True:\n",
        "    # installing packages\n",
        "    !pip install pyfolio-reloaded  #original pyfolio no longer maintained\n",
        "    !pip install optuna\n",
        "    !pip install -U \"ray[rllib]\"\n",
        "    !pip install plotly\n",
        "    !pip install ipywidgets\n",
        "    !pip install -U kaleido   # enables saving plots to file\n",
        "    \n",
        "    "
      ],
      "metadata": {
        "id": "UfmXkH1rTFM7",
        "outputId": "8efc8841-de8b-4436-d2f7-6228db2b50cd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pyfolio-reloaded\n",
            "  Downloading pyfolio_reloaded-0.9.5-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m43.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: ipython>=3.2.3 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (8.13.1)\n",
            "Requirement already satisfied: scikit-learn>=0.16.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (1.2.2)\n",
            "Requirement already satisfied: numpy>=1.11.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (1.24.3)\n",
            "Collecting empyrical-reloaded>=0.5.8\n",
            "  Downloading empyrical_reloaded-0.5.9-py3-none-any.whl (32 kB)\n",
            "Requirement already satisfied: scipy>=0.14.0 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (1.10.1)\n",
            "Requirement already satisfied: pandas>=0.18.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (2.0.1)\n",
            "Requirement already satisfied: pytz>=2014.10 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (2023.3)\n",
            "Requirement already satisfied: matplotlib>=1.4.0 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (3.7.1)\n",
            "Requirement already satisfied: seaborn>=0.7.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (0.12.2)\n",
            "Requirement already satisfied: pandas-datareader>=0.4 in /usr/local/lib/python3.10/site-packages (from empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.10.0)\n",
            "Collecting bottleneck>=1.3.0\n",
            "  Downloading Bottleneck-1.3.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (354 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m354.0/354.0 kB\u001b[0m \u001b[31m39.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: yfinance>=0.1.63 in /usr/local/lib/python3.10/site-packages (from empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.2.18)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (3.0.38)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.2.0)\n",
            "Requirement already satisfied: pygments>=2.4.0 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (2.15.1)\n",
            "Requirement already satisfied: traitlets>=5 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (5.9.0)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.18.2)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (5.1.1)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.1.6)\n",
            "Requirement already satisfied: stack-data in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.6.2)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (4.8.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.7.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (23.1)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (1.0.7)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (2.8.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (1.4.4)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (3.0.9)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (9.5.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (4.39.3)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (0.11.0)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/site-packages (from pandas>=0.18.1->pyfolio-reloaded) (2023.3)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=0.16.1->pyfolio-reloaded) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=0.16.1->pyfolio-reloaded) (3.1.0)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.10/site-packages (from jedi>=0.16->ipython>=3.2.3->pyfolio-reloaded) (0.8.3)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/site-packages (from pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (4.9.2)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/site-packages (from pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.28.2)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/site-packages (from pexpect>4.3->ipython>=3.2.3->pyfolio-reloaded) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=3.2.3->pyfolio-reloaded) (0.2.6)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=1.4.0->pyfolio-reloaded) (1.16.0)\n",
            "Requirement already satisfied: html5lib>=1.1 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.1)\n",
            "Requirement already satisfied: beautifulsoup4>=4.11.1 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (4.12.2)\n",
            "Requirement already satisfied: cryptography>=3.3.2 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (40.0.1)\n",
            "Requirement already satisfied: appdirs>=1.4.4 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.4.4)\n",
            "Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.0.11)\n",
            "Requirement already satisfied: frozendict>=2.3.4 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.3.8)\n",
            "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio-reloaded) (2.2.1)\n",
            "Requirement already satisfied: pure-eval in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio-reloaded) (0.2.2)\n",
            "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio-reloaded) (1.2.0)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/site-packages (from beautifulsoup4>=4.11.1->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.4.1)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/site-packages (from cryptography>=3.3.2->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.15.1)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/site-packages (from html5lib>=1.1->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.5.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.26.15)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.1.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2022.12.7)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=3.3.2->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.21)\n",
            "Installing collected packages: bottleneck, empyrical-reloaded, pyfolio-reloaded\n",
            "Successfully installed bottleneck-1.3.7 empyrical-reloaded-0.5.9 pyfolio-reloaded-0.9.5\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting optuna\n",
            "  Downloading optuna-3.1.1-py3-none-any.whl (365 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m365.7/365.7 kB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting cmaes>=0.9.1\n",
            "  Downloading cmaes-0.9.1-py3-none-any.whl (21 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/site-packages (from optuna) (1.24.3)\n",
            "Requirement already satisfied: sqlalchemy>=1.3.0 in /usr/local/lib/python3.10/site-packages (from optuna) (1.4.48)\n",
            "Collecting colorlog\n",
            "  Downloading colorlog-6.7.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting alembic>=1.5.0\n",
            "  Downloading alembic-1.10.4-py3-none-any.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.9/212.9 kB\u001b[0m \u001b[31m28.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from optuna) (23.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/site-packages (from optuna) (4.65.0)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/site-packages (from optuna) (6.0)\n",
            "Collecting Mako\n",
            "  Downloading Mako-1.2.4-py3-none-any.whl (78 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.7/78.7 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=4 in /usr/local/lib/python3.10/site-packages (from alembic>=1.5.0->optuna) (4.5.0)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/site-packages (from sqlalchemy>=1.3.0->optuna) (2.0.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.10/site-packages (from Mako->alembic>=1.5.0->optuna) (2.1.2)\n",
            "Installing collected packages: Mako, colorlog, cmaes, alembic, optuna\n",
            "Successfully installed Mako-1.2.4 alembic-1.10.4 cmaes-0.9.1 colorlog-6.7.0 optuna-3.1.1\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: ray[rllib] in /usr/local/lib/python3.10/site-packages (2.4.0)\n",
            "Requirement already satisfied: frozenlist in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.3.3)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (4.17.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (3.12.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (2.28.2)\n",
            "Requirement already satisfied: aiosignal in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.3.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (23.1)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (8.1.3)\n",
            "Requirement already satisfied: attrs in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (23.1.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (6.0)\n",
            "Requirement already satisfied: grpcio<=1.51.3,>=1.42.0 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.51.3)\n",
            "Requirement already satisfied: numpy>=1.19.3 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.24.3)\n",
            "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (3.20.3)\n",
            "Requirement already satisfied: virtualenv<20.21.1,>=20.0.24 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (20.21.0)\n",
            "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.0.3)\n",
            "Requirement already satisfied: tensorboardX>=1.9 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (2.6)\n",
            "Collecting dm-tree\n",
            "  Downloading dm_tree-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (152 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.8/152.8 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gymnasium==0.26.3\n",
            "  Downloading Gymnasium-0.26.3-py3-none-any.whl (836 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m836.9/836.9 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (2.0.1)\n",
            "Requirement already satisfied: lz4 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (4.3.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.10.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (0.9.0)\n",
            "Collecting rich\n",
            "  Downloading rich-13.3.5-py3-none-any.whl (238 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m238.7/238.7 kB\u001b[0m \u001b[31m29.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typer\n",
            "  Downloading typer-0.8.0-py3-none-any.whl (38 kB)\n",
            "Collecting scikit-image\n",
            "  Downloading scikit_image-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.2/13.2 MB\u001b[0m \u001b[31m61.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gymnasium-notices>=0.0.1\n",
            "  Downloading gymnasium_notices-0.0.1-py3-none-any.whl (2.8 kB)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/site-packages (from gymnasium==0.26.3->ray[rllib]) (2.2.1)\n",
            "Requirement already satisfied: platformdirs<4,>=2.4 in /usr/local/lib/python3.10/site-packages (from virtualenv<20.21.1,>=20.0.24->ray[rllib]) (3.5.0)\n",
            "Requirement already satisfied: distlib<1,>=0.3.6 in /usr/local/lib/python3.10/site-packages (from virtualenv<20.21.1,>=20.0.24->ray[rllib]) (0.3.6)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/site-packages (from jsonschema->ray[rllib]) (0.19.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/site-packages (from pandas->ray[rllib]) (2.8.2)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/site-packages (from pandas->ray[rllib]) (2023.3)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/site-packages (from pandas->ray[rllib]) (2023.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (2.1.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (1.26.15)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (3.4)\n",
            "Collecting markdown-it-py<3.0.0,>=2.2.0\n",
            "  Downloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.5/84.5 kB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/site-packages (from rich->ray[rllib]) (2.15.1)\n",
            "Requirement already satisfied: networkx>=2.8 in /usr/local/lib/python3.10/site-packages (from scikit-image->ray[rllib]) (3.1)\n",
            "Collecting imageio>=2.4.1\n",
            "  Downloading imageio-2.28.1-py3-none-any.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m67.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tifffile>=2019.7.26\n",
            "  Downloading tifffile-2023.4.12-py3-none-any.whl (219 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m219.4/219.4 kB\u001b[0m \u001b[31m31.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting lazy_loader>=0.1\n",
            "  Downloading lazy_loader-0.2-py3-none-any.whl (8.6 kB)\n",
            "Requirement already satisfied: pillow>=9.0.1 in /usr/local/lib/python3.10/site-packages (from scikit-image->ray[rllib]) (9.5.0)\n",
            "Collecting PyWavelets>=1.1.1\n",
            "  Downloading PyWavelets-1.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m112.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting mdurl~=0.1\n",
            "  Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->ray[rllib]) (1.16.0)\n",
            "Installing collected packages: gymnasium-notices, dm-tree, typer, tifffile, PyWavelets, mdurl, lazy_loader, imageio, gymnasium, scikit-image, markdown-it-py, rich\n",
            "Successfully installed PyWavelets-1.4.1 dm-tree-0.1.8 gymnasium-0.26.3 gymnasium-notices-0.0.1 imageio-2.28.1 lazy_loader-0.2 markdown-it-py-2.2.0 mdurl-0.1.2 rich-13.3.5 scikit-image-0.20.0 tifffile-2023.4.12 typer-0.8.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting plotly\n",
            "  Downloading plotly-5.14.1-py2.py3-none-any.whl (15.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m76.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from plotly) (23.1)\n",
            "Collecting tenacity>=6.2.0\n",
            "  Downloading tenacity-8.2.2-py3-none-any.whl (24 kB)\n",
            "Installing collected packages: tenacity, plotly\n",
            "Successfully installed plotly-5.14.1 tenacity-8.2.2\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting ipywidgets\n",
            "  Downloading ipywidgets-8.0.6-py3-none-any.whl (138 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.3/138.3 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ipykernel>=4.5.1\n",
            "  Downloading ipykernel-6.22.0-py3-none-any.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m150.0/150.0 kB\u001b[0m \u001b[31m22.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: ipython>=6.1.0 in /usr/local/lib/python3.10/site-packages (from ipywidgets) (8.13.1)\n",
            "Collecting widgetsnbextension~=4.0.7\n",
            "  Downloading widgetsnbextension-4.0.7-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m79.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.10/site-packages (from ipywidgets) (5.9.0)\n",
            "Collecting jupyterlab-widgets~=3.0.7\n",
            "  Downloading jupyterlab_widgets-3.0.7-py3-none-any.whl (198 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m198.2/198.2 kB\u001b[0m \u001b[31m28.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nest-asyncio\n",
            "  Downloading nest_asyncio-1.5.6-py3-none-any.whl (5.2 kB)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (23.1)\n",
            "Collecting debugpy>=1.6.5\n",
            "  Downloading debugpy-1.6.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m97.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.6)\n",
            "Collecting pyzmq>=20\n",
            "  Downloading pyzmq-25.0.2-cp310-cp310-manylinux_2_28_x86_64.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jupyter-core!=5.0.*,>=4.12\n",
            "  Downloading jupyter_core-5.3.0-py3-none-any.whl (93 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.2/93.2 kB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (5.9.5)\n",
            "Collecting tornado>=6.1\n",
            "  Downloading tornado-6.3.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (426 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m426.8/426.8 kB\u001b[0m \u001b[31m54.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting comm>=0.1.1\n",
            "  Downloading comm-0.1.3-py3-none-any.whl (6.6 kB)\n",
            "Collecting jupyter-client>=6.1.12\n",
            "  Downloading jupyter_client-8.2.0-py3-none-any.whl (103 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m103.2/103.2 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pygments>=2.4.0 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (2.15.1)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.7.5)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.18.2)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (4.8.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n",
            "Requirement already satisfied: stack-data in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.2)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.38)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.0)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.10/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (2.8.2)\n",
            "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/site-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel>=4.5.1->ipywidgets) (3.5.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets) (0.2.6)\n",
            "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (1.2.0)\n",
            "Requirement already satisfied: pure-eval in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\n",
            "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.2.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\n",
            "Installing collected packages: widgetsnbextension, tornado, pyzmq, nest-asyncio, jupyterlab-widgets, jupyter-core, debugpy, comm, jupyter-client, ipykernel, ipywidgets\n",
            "Successfully installed comm-0.1.3 debugpy-1.6.7 ipykernel-6.22.0 ipywidgets-8.0.6 jupyter-client-8.2.0 jupyter-core-5.3.0 jupyterlab-widgets-3.0.7 nest-asyncio-1.5.6 pyzmq-25.0.2 tornado-6.3.1 widgetsnbextension-4.0.7\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting kaleido\n",
            "  Downloading kaleido-0.2.1-py2.py3-none-manylinux1_x86_64.whl (79.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.9/79.9 MB\u001b[0m \u001b[31m15.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: kaleido\n",
            "Successfully installed kaleido-0.2.1\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pandas==1.5.3\n",
        "!pip install gymnasium"
      ],
      "metadata": {
        "id": "9Zkmtvr-TFn_",
        "outputId": "432ff578-a0a2-4fda-a514-8ae351e33991",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pandas==1.5.3 in /usr/local/lib/python3.9/site-packages (1.5.3)\n",
            "Requirement already satisfied: numpy>=1.20.3 in /usr/local/lib/python3.9/site-packages (from pandas==1.5.3) (1.24.3)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/site-packages (from pandas==1.5.3) (2023.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/site-packages (from pandas==1.5.3) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/site-packages (from python-dateutil>=2.8.1->pandas==1.5.3) (1.16.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gymnasium in /usr/local/lib/python3.9/site-packages (0.26.3)\n",
            "Requirement already satisfied: importlib-metadata>=4.8.0 in /usr/local/lib/python3.9/site-packages (from gymnasium) (4.13.0)\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.9/site-packages (from gymnasium) (1.24.3)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.9/site-packages (from gymnasium) (2.2.1)\n",
            "Requirement already satisfied: gymnasium-notices>=0.0.1 in /usr/local/lib/python3.9/site-packages (from gymnasium) (0.0.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/site-packages (from importlib-metadata>=4.8.0->gymnasium) (3.15.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append(\"../FinRL-Library\")"
      ],
      "metadata": {
        "id": "cMn1VNKCR0ld"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.3 Import packages"
      ],
      "metadata": {
        "id": "BbOMJnGdTRYX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Importing the libraries\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "# matplotlib.use('Agg')\n",
        "import datetime\n",
        "import optuna\n",
        "from pathlib import Path\n",
        "from google.colab import files\n",
        "%matplotlib inline\n",
        "from finrl import config\n",
        "from finrl import config_tickers\n",
        "from optuna.integration import PyTorchLightningPruningCallback\n",
        "from finrl.meta.preprocessor.yahoodownloader import YahooDownloader\n",
        "from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
        "from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
        "from finrl.meta.env_stock_trading.env_stocktrading_np import StockTradingEnv as StockTradingEnv_numpy\n",
        "from finrl.agents.stablebaselines3.models import DRLAgent\n",
        "from finrl.agents.rllib.models import DRLAgent as DRLAgent_rllib\n",
        "from finrl.meta.data_processor import DataProcessor\n",
        "import joblib\n",
        "from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "import ray\n",
        "from pprint import pprint\n",
        "import kaleido\n",
        "\n",
        "\n",
        "\n",
        "import itertools\n",
        "\n",
        "import torch\n",
        "if torch.cuda.is_available():\n",
        "  device = torch.device(\"cuda\")\n",
        "else:\n",
        "  device = torch.device(\"cpu\")\n",
        "print(f'Torch device: {device}')"
      ],
      "metadata": {
        "id": "tG5qIkVHTNv8",
        "outputId": "21cafa00-a443-4a73-c400-f4f3743b4037",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Torch device: cuda\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/site-packages/pyfolio/pos.py:25: UserWarning: Module \"zipline.assets\" not found; multipliers will not be applied to position notionals.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "config_tickers.DOW_30_TICKER"
      ],
      "metadata": {
        "id": "DB11ygQLVLUD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "da849234-7155-4a88-eb56-18197c3b2c62"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['AXP',\n",
              " 'AMGN',\n",
              " 'AAPL',\n",
              " 'BA',\n",
              " 'CAT',\n",
              " 'CSCO',\n",
              " 'CVX',\n",
              " 'GS',\n",
              " 'HD',\n",
              " 'HON',\n",
              " 'IBM',\n",
              " 'INTC',\n",
              " 'JNJ',\n",
              " 'KO',\n",
              " 'JPM',\n",
              " 'MCD',\n",
              " 'MMM',\n",
              " 'MRK',\n",
              " 'MSFT',\n",
              " 'NKE',\n",
              " 'PG',\n",
              " 'TRV',\n",
              " 'UNH',\n",
              " 'CRM',\n",
              " 'VZ',\n",
              " 'V',\n",
              " 'WBA',\n",
              " 'WMT',\n",
              " 'DIS',\n",
              " 'DOW']"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "fMaPm8MsUo0R"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "if not os.path.exists(\"./\" + config.DATA_SAVE_DIR):\n",
        "    os.makedirs(\"./\" + config.DATA_SAVE_DIR)\n",
        "if not os.path.exists(\"./\" + config.TRAINED_MODEL_DIR):\n",
        "    os.makedirs(\"./\" + config.TRAINED_MODEL_DIR)\n",
        "if not os.path.exists(\"./\" + config.TENSORBOARD_LOG_DIR):\n",
        "    os.makedirs(\"./\" + config.TENSORBOARD_LOG_DIR)\n",
        "if not os.path.exists(\"./\" + config.RESULTS_DIR):\n",
        "    os.makedirs(\"./\" + config.RESULTS_DIR)"
      ],
      "metadata": {
        "id": "FAZpwTH3VVi-"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Collecting data and preprocessing"
      ],
      "metadata": {
        "id": "cqJ8ngFWVcNG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#config_tickers.DOW_30_TICKER = [\"PYPL\"]"
      ],
      "metadata": {
        "id": "4M4UwcP4Ap_o"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Custom ticker list dataframe download\n",
        "#TODO save df to avoid download\n",
        "path_pf = '/content/ticker_data.csv'\n",
        "if Path(path_pf).is_file():\n",
        "  print('Reading ticker data')\n",
        "  df = pd.read_csv(path_pf)\n",
        "  \n",
        "else:\n",
        "  print('Downloading ticker data')\n",
        "  ticker_list = config_tickers.DOW_30_TICKER\n",
        "  df = YahooDownloader(start_date = '2009-01-01',\n",
        "                     end_date = '2023-04-30',\n",
        "                     ticker_list = ticker_list).fetch_data()\n",
        "  df.to_csv('ticker_data.csv')"
      ],
      "metadata": {
        "id": "7BJNlsAuVZ-o",
        "outputId": "57ef2e93-8ec7-4777-8ffb-11ccda75f1c9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading ticker data\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (105581, 8)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_processed_full(processed):\n",
        "  list_ticker = processed[\"tic\"].unique().tolist()\n",
        "  list_date = list(pd.date_range(processed['date'].min(),processed['date'].max()).astype(str))\n",
        "  combination = list(itertools.product(list_date,list_ticker))\n",
        "\n",
        "  processed_full = pd.DataFrame(combination,columns=[\"date\",\"tic\"]).merge(processed,on=[\"date\",\"tic\"],how=\"left\")\n",
        "  processed_full = processed_full[processed_full['date'].isin(processed['date'])]\n",
        "  processed_full = processed_full.sort_values(['date','tic'])\n",
        "\n",
        "  processed_full = processed_full.fillna(0)\n",
        "  processed_full.sort_values(['date','tic'],ignore_index=True).head(5)\n",
        "\n",
        "  processed_full.to_csv('processed_full.csv')\n",
        "  return processed_full"
      ],
      "metadata": {
        "id": "RD6wwtGSVZVU"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#You can add technical indicators and turbulence factor to dataframe\n",
        "#Just set the use_technical_indicator=True, use_vix=True and use_turbulence=True\n",
        "def create_techind():\n",
        "  fe = FeatureEngineer(\n",
        "                    use_technical_indicator=True,\n",
        "                    tech_indicator_list = config.INDICATORS,\n",
        "                    use_vix=True,\n",
        "                    use_turbulence=True,\n",
        "                    user_defined_feature = False)\n",
        "\n",
        "  processed = fe.preprocess_data(df)\n",
        "  return processed"
      ],
      "metadata": {
        "id": "UNqEALmbVoJo"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Load price and technical indicator data from file if available\n",
        "path_pf = '/content/processed_full.csv'\n",
        "if Path(path_pf).is_file():\n",
        "  print('Reading processed_full data')\n",
        "  processed_full = pd.read_csv(path_pf)\n",
        "\n",
        "else:\n",
        "  print('Creating processed_full file')\n",
        "  processed=create_techind()\n",
        "  processed_full=create_processed_full(processed)"
      ],
      "metadata": {
        "id": "Qr1Qd-lMVpzg",
        "outputId": "bea575a3-8dd3-4452-e7d7-8d72c6dc65ab",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Creating processed_full file\n",
            "Successfully added technical indicators\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (3604, 8)\n",
            "Successfully added vix\n",
            "Successfully added turbulence index\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "date_col = \"date\"\n",
        "tic_col = \"tic\"\n",
        "\n",
        "init_train_trade_data = processed_full.sort_values([date_col, tic_col])\n",
        "\n",
        "init_train_trade_data = processed_full.fillna(0)\n",
        "\n",
        "init_train_data = data_split(\n",
        "    init_train_trade_data, '2020-01-01', '2020-05-01')\n",
        "init_trade_data = data_split(\n",
        "    init_train_trade_data, '2021-05-01','2021-10-01')\n",
        "\n",
        "print(f'Number of training samples: {len(init_train_data)}')\n",
        "print(f'Number of testing samples: {len(init_train_trade_data)}')"
      ],
      "metadata": {
        "id": "KFs4AjWIVr2s",
        "outputId": "c7808113-2e86-4852-ca1a-f38837b0f4a2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 151,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of training samples: 2407\n",
            "Number of testing samples: 104516\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "init_trade_data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "id": "GI2qgPGFTerU",
        "outputId": "d04a21f3-c377-4274-95ab-7648f0f3737b"
      },
      "execution_count": 152,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         date   tic        open        high         low       close   \n",
              "0  2021-05-03  AAPL  132.039993  134.070007  131.830002  130.963562  \\\n",
              "0  2021-05-03  AMGN  240.669998  247.020004  240.550003  230.412506   \n",
              "0  2021-05-03   AXP  154.589996  156.050003  154.009995  150.957840   \n",
              "0  2021-05-03    BA  234.110001  237.100006  233.809998  235.190002   \n",
              "0  2021-05-03   CAT  230.000000  230.929993  227.210007  218.290497   \n",
              "\n",
              "       volume  day      macd     boll_ub     boll_lb     rsi_30      cci_30   \n",
              "0  75135100.0  0.0  1.805508  135.539551  126.477489  53.856531   50.478494  \\\n",
              "0   3587700.0  0.0 -0.410710  248.223088  221.790985  50.842042  -81.617841   \n",
              "0   2726200.0  0.0  2.754006  151.027352  139.179342  62.209650  175.796005   \n",
              "0   9887800.0  0.0 -2.593339  260.290298  227.831702  49.445883 -102.331978   \n",
              "0   3182900.0  0.0  1.167004  223.005760  216.688279  56.081915  -17.678507   \n",
              "\n",
              "       dx_30  close_30_sma  close_60_sma        vix  turbulence  \n",
              "0  20.583396    127.527258    126.185904  18.309999   20.080482  \n",
              "0   5.785497    234.734414    226.585526  18.309999   20.080482  \n",
              "0  22.874057    142.745663    138.440772  18.309999   20.080482  \n",
              "0  17.827725    245.819000    237.101333  18.309999   20.080482  \n",
              "0   2.780977    218.974400    211.601990  18.309999   20.080482  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f584714a-e41b-48aa-a896-fb224ac28bde\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>tic</th>\n",
              "      <th>open</th>\n",
              "      <th>high</th>\n",
              "      <th>low</th>\n",
              "      <th>close</th>\n",
              "      <th>volume</th>\n",
              "      <th>day</th>\n",
              "      <th>macd</th>\n",
              "      <th>boll_ub</th>\n",
              "      <th>boll_lb</th>\n",
              "      <th>rsi_30</th>\n",
              "      <th>cci_30</th>\n",
              "      <th>dx_30</th>\n",
              "      <th>close_30_sma</th>\n",
              "      <th>close_60_sma</th>\n",
              "      <th>vix</th>\n",
              "      <th>turbulence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AAPL</td>\n",
              "      <td>132.039993</td>\n",
              "      <td>134.070007</td>\n",
              "      <td>131.830002</td>\n",
              "      <td>130.963562</td>\n",
              "      <td>75135100.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.805508</td>\n",
              "      <td>135.539551</td>\n",
              "      <td>126.477489</td>\n",
              "      <td>53.856531</td>\n",
              "      <td>50.478494</td>\n",
              "      <td>20.583396</td>\n",
              "      <td>127.527258</td>\n",
              "      <td>126.185904</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080482</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AMGN</td>\n",
              "      <td>240.669998</td>\n",
              "      <td>247.020004</td>\n",
              "      <td>240.550003</td>\n",
              "      <td>230.412506</td>\n",
              "      <td>3587700.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>-0.410710</td>\n",
              "      <td>248.223088</td>\n",
              "      <td>221.790985</td>\n",
              "      <td>50.842042</td>\n",
              "      <td>-81.617841</td>\n",
              "      <td>5.785497</td>\n",
              "      <td>234.734414</td>\n",
              "      <td>226.585526</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080482</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AXP</td>\n",
              "      <td>154.589996</td>\n",
              "      <td>156.050003</td>\n",
              "      <td>154.009995</td>\n",
              "      <td>150.957840</td>\n",
              "      <td>2726200.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.754006</td>\n",
              "      <td>151.027352</td>\n",
              "      <td>139.179342</td>\n",
              "      <td>62.209650</td>\n",
              "      <td>175.796005</td>\n",
              "      <td>22.874057</td>\n",
              "      <td>142.745663</td>\n",
              "      <td>138.440772</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080482</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>BA</td>\n",
              "      <td>234.110001</td>\n",
              "      <td>237.100006</td>\n",
              "      <td>233.809998</td>\n",
              "      <td>235.190002</td>\n",
              "      <td>9887800.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>-2.593339</td>\n",
              "      <td>260.290298</td>\n",
              "      <td>227.831702</td>\n",
              "      <td>49.445883</td>\n",
              "      <td>-102.331978</td>\n",
              "      <td>17.827725</td>\n",
              "      <td>245.819000</td>\n",
              "      <td>237.101333</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080482</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>CAT</td>\n",
              "      <td>230.000000</td>\n",
              "      <td>230.929993</td>\n",
              "      <td>227.210007</td>\n",
              "      <td>218.290497</td>\n",
              "      <td>3182900.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.167004</td>\n",
              "      <td>223.005760</td>\n",
              "      <td>216.688279</td>\n",
              "      <td>56.081915</td>\n",
              "      <td>-17.678507</td>\n",
              "      <td>2.780977</td>\n",
              "      <td>218.974400</td>\n",
              "      <td>211.601990</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080482</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f584714a-e41b-48aa-a896-fb224ac28bde')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f584714a-e41b-48aa-a896-fb224ac28bde button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f584714a-e41b-48aa-a896-fb224ac28bde');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 152
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Creating Environment"
      ],
      "metadata": {
        "id": "L3VlkfBMW8e7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from __future__ import annotations\n",
        "\n",
        "from typing import List\n",
        "\n",
        "import gym\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from gym import spaces\n",
        "from gym.utils import seeding\n",
        "from stable_baselines3.common.vec_env import DummyVecEnv\n",
        "\n",
        "matplotlib.use(\"Agg\")\n",
        "\n",
        "# from stable_baselines3.common.logger import Logger, KVWriter, CSVOutputFormat\n",
        "\n",
        "\n",
        "class GokuEnv(gym.Env):\n",
        "    \"\"\"A stock trading environment for OpenAI gym\"\"\"\n",
        "\n",
        "    metadata = {\"render.modes\": [\"human\"]}\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        df: pd.DataFrame,\n",
        "        stock_dim: int,\n",
        "        hmax: int,\n",
        "        initial_amount: int,\n",
        "        num_stock_shares: list[int],\n",
        "        buy_cost_pct: list[float],\n",
        "        sell_cost_pct: list[float],\n",
        "        reward_scaling: float,\n",
        "        state_space: int,\n",
        "        action_space: int,\n",
        "        tech_indicator_list: list[str],\n",
        "        turbulence_threshold=None,\n",
        "        risk_indicator_col=\"turbulence\",\n",
        "        make_plots: bool = False,\n",
        "        print_verbosity=10,\n",
        "        day=0,\n",
        "        initial=True,\n",
        "        previous_state=[],\n",
        "        model_name=\"\",\n",
        "        mode=\"\",\n",
        "        iteration=\"\",\n",
        "    ):\n",
        "        self.day = day\n",
        "        self.df = df\n",
        "        self.stock_dim = stock_dim\n",
        "        self.hmax = hmax\n",
        "        self.num_stock_shares = num_stock_shares\n",
        "        self.initial_amount = initial_amount  # get the initial cash\n",
        "        self.buy_cost_pct = buy_cost_pct\n",
        "        self.sell_cost_pct = sell_cost_pct\n",
        "        self.reward_scaling = reward_scaling\n",
        "        self.state_space = state_space\n",
        "        self.action_space = action_space\n",
        "        self.tech_indicator_list = tech_indicator_list\n",
        "        self.action_space = spaces.Box(low=-1, high=1, shape=(self.action_space,))\n",
        "        self.observation_space = spaces.Box(\n",
        "            low=-np.inf, high=np.inf, shape=(self.state_space,)\n",
        "        )\n",
        "        self.data = self.df.loc[self.day, :]\n",
        "        self.terminal = False\n",
        "        self.make_plots = make_plots\n",
        "        self.print_verbosity = print_verbosity\n",
        "        self.turbulence_threshold = turbulence_threshold\n",
        "        self.risk_indicator_col = risk_indicator_col\n",
        "        self.initial = initial\n",
        "        self.previous_state = previous_state\n",
        "        self.model_name = model_name\n",
        "        self.mode = mode\n",
        "        self.iteration = iteration\n",
        "        # initalize state\n",
        "        self.state = self._initiate_state()\n",
        "\n",
        "        # initialize reward\n",
        "        self.reward = 0\n",
        "        self.turbulence = 0\n",
        "        self.cost = 0\n",
        "        self.trades = 0\n",
        "        self.episode = 0\n",
        "        # memorize all the total balance change\n",
        "        self.asset_memory = [\n",
        "            self.initial_amount\n",
        "            + np.sum(\n",
        "                np.array(self.num_stock_shares)\n",
        "                * np.array(self.state[1 : 1 + self.stock_dim])\n",
        "            )\n",
        "        ]  # the initial total asset is calculated by cash + sum (num_share_stock_i * price_stock_i)\n",
        "        self.rewards_memory = []\n",
        "        self.actions_memory = []\n",
        "        self.state_memory = (\n",
        "            []\n",
        "        )  # we need sometimes to preserve the state in the middle of trading process\n",
        "        self.date_memory = [self._get_date()]\n",
        "        #         self.logger = Logger('results',[CSVOutputFormat])\n",
        "        # self.reset()\n",
        "        self._seed()\n",
        "        self.total_price = np.array([0.0] * 29)\n",
        "        self.avg_price = np.array([0.0] * 29)\n",
        "        self.total_stockss = np.array([0.0] * 29)\n",
        "\n",
        "    def _sell_stock(self, index, action):\n",
        "        def _do_sell_normal():\n",
        "            if (\n",
        "                self.state[index + 2 * self.stock_dim + 1] != True\n",
        "            ):  # check if the stock is able to sell, for simlicity we just add it in techical index\n",
        "                # if self.state[index + 1] > 0: # if we use price<0 to denote a stock is unable to trade in that day, the total asset calculation may be wrong for the price is unreasonable\n",
        "                # Sell only if the price is > 0 (no missing data in this particular date)\n",
        "                # perform sell action based on the sign of the action\n",
        "                if self.state[index + self.stock_dim + 1] > 0:\n",
        "                    # Sell only if current asset is > 0\n",
        "                    sell_num_shares = min(\n",
        "                        abs(action), self.state[index + self.stock_dim + 1]\n",
        "                    )\n",
        "                    sell_amount = (\n",
        "                        self.state[index + 1]\n",
        "                        * sell_num_shares\n",
        "                        * (1 - self.sell_cost_pct[index])\n",
        "                    )\n",
        "                    # update balance\n",
        "                    self.state[0] += sell_amount\n",
        "\n",
        "                    self.state[index + self.stock_dim + 1] -= sell_num_shares\n",
        "                    self.cost += (\n",
        "                        self.state[index + 1]\n",
        "                        * sell_num_shares\n",
        "                        * self.sell_cost_pct[index]\n",
        "                    )\n",
        "                    self.trades += 1\n",
        "                    #if sell_num_shares >0:\n",
        "                    #  print(\"stocks Sold sell_num_shares\", sell_num_shares)\n",
        "                else:\n",
        "                    sell_num_shares = 0\n",
        "            else:\n",
        "                sell_num_shares = 0\n",
        "            return sell_num_shares\n",
        "\n",
        "        # perform sell action based on the sign of the action\n",
        "        if self.turbulence_threshold is not None:\n",
        "            if self.turbulence >= self.turbulence_threshold:\n",
        "                if self.state[index + 1] > 0:\n",
        "                    # Sell only if the price is > 0 (no missing data in this particular date)\n",
        "                    # if turbulence goes over threshold, just clear out all positions\n",
        "                    if self.state[index + self.stock_dim + 1] > 0:\n",
        "                        # Sell only if current asset is > 0\n",
        "                        sell_num_shares = self.state[index + self.stock_dim + 1]\n",
        "                        sell_amount = (\n",
        "                            self.state[index + 1]\n",
        "                            * sell_num_shares\n",
        "                            * (1 - self.sell_cost_pct[index])\n",
        "                        )\n",
        "                        # update balance\n",
        "                        self.state[0] += sell_amount\n",
        "                        self.state[index + self.stock_dim + 1] = 0\n",
        "                        self.cost += (\n",
        "                            self.state[index + 1]\n",
        "                            * sell_num_shares\n",
        "                            * self.sell_cost_pct[index]\n",
        "                        )\n",
        "                        self.trades += 1\n",
        "                    else:\n",
        "                        sell_num_shares = 0\n",
        "                else:\n",
        "                    sell_num_shares = 0\n",
        "            else:\n",
        "                sell_num_shares = _do_sell_normal()\n",
        "        else:\n",
        "            sell_num_shares = _do_sell_normal()\n",
        "        #print(\"stocks Sold sell_num_shares\", sell_num_shares)\n",
        "        return sell_num_shares\n",
        "\n",
        "    def _buy_stock(self, index, action):\n",
        "        def _do_buy():\n",
        "            if (self.state[index + 2 * self.stock_dim + 1] != True):  # check if the stock is able to buy\n",
        "                # if self.state[index + 1] >0:\n",
        "                # Buy only if the price is > 0 (no missing data in this particular date)\n",
        "                available_amount = self.state[0] // (self.state[index + 1] * (1 + self.buy_cost_pct[index]))\n",
        "                # when buying stocks, we should consider the cost of trading when calculating available_amount, or we may be have cash<0\n",
        "                # print('available_amount:{}'.format(available_amount))\n",
        "                # update balance\n",
        "                buy_num_shares = min(available_amount, action)\n",
        "                buy_amount = (self.state[index + 1] * buy_num_shares * (1 + self.buy_cost_pct[index]))\n",
        "                self.state[0] -= buy_amount\n",
        "                self.state[index + self.stock_dim + 1] += buy_num_shares\n",
        "                self.cost += (self.state[index + 1] * buy_num_shares * self.buy_cost_pct[index])\n",
        "                self.trades += 1\n",
        "            else:\n",
        "                buy_num_shares = 0\n",
        "\n",
        "            return buy_num_shares\n",
        "\n",
        "        # perform buy action based on the sign of the action\n",
        "        if self.turbulence_threshold is None:\n",
        "            buy_num_shares = _do_buy()\n",
        "        else:\n",
        "            if self.turbulence < self.turbulence_threshold:\n",
        "                buy_num_shares = _do_buy()\n",
        "            else:\n",
        "                buy_num_shares = 0\n",
        "                pass\n",
        "\n",
        "        return buy_num_shares\n",
        "\n",
        "    def _make_plot(self):\n",
        "        plt.plot(self.asset_memory, \"r\")\n",
        "        plt.savefig(f\"results/account_value_trade_{self.episode}.png\")\n",
        "        plt.close()\n",
        "\n",
        "    def step(self, actions):\n",
        "        self.terminal = self.day >= len(self.df.index.unique()) - 1\n",
        "        if self.terminal:\n",
        "            # print(f\"Episode: {self.episode}\")\n",
        "            if self.make_plots:\n",
        "                self._make_plot()\n",
        "            end_total_asset = self.state[0] + sum(np.array(self.state[1 : (self.stock_dim + 1)]) * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            df_total_value = pd.DataFrame(self.asset_memory)\n",
        "            tot_reward = (self.state[0] + \n",
        "                          sum(np.array(self.state[1 : (self.stock_dim + 1)]) \n",
        "                          * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])) \n",
        "                          - self.asset_memory[0])  # initial_amount is only cash part of our initial asset\n",
        "            df_total_value.columns = [\"account_value\"]\n",
        "            df_total_value[\"date\"] = self.date_memory\n",
        "            df_total_value[\"daily_return\"] = df_total_value[\"account_value\"].pct_change(1)\n",
        "            if df_total_value[\"daily_return\"].std() != 0:\n",
        "                sharpe = ((252**0.5) \n",
        "                * df_total_value[\"daily_return\"].mean() \n",
        "                / df_total_value[\"daily_return\"].std())\n",
        "            df_rewards = pd.DataFrame(self.rewards_memory)\n",
        "            df_rewards.columns = [\"account_rewards\"]\n",
        "            df_rewards[\"date\"] = self.date_memory[:-1]\n",
        "            \n",
        "            if self.episode % self.print_verbosity == 0:\n",
        "                print(f\"day: {self.day}, episode: {self.episode}\")\n",
        "                print(f\"begin_total_asset: {self.asset_memory[0]:0.2f}\")\n",
        "                print(f\"end_total_asset: {end_total_asset:0.2f}\")\n",
        "                print(f\"total_reward: {tot_reward:0.2f}\")\n",
        "                print(f\"total_cost: {self.cost:0.2f}\")\n",
        "                print(f\"total_trades: {self.trades}\")\n",
        "                if df_total_value[\"daily_return\"].std() != 0:\n",
        "                    print(f\"Sharpe: {sharpe:0.3f}\")\n",
        "                print(\"=================================\")\n",
        "\n",
        "            if (self.model_name != \"\") and (self.mode != \"\"):\n",
        "                df_actions = self.save_action_memory()\n",
        "                df_actions.to_csv(\"results/actions_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration))\n",
        "                df_total_value.to_csv(\"results/account_value_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration),index=False,)\n",
        "                df_rewards.to_csv(\"results/account_rewards_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration),index=False,)\n",
        "                plt.plot(self.asset_memory, \"r\")\n",
        "                plt.savefig(\"results/account_value_{}_{}_{}.png\".format(self.mode, self.model_name, self.iteration))\n",
        "                plt.close()\n",
        "            return self.state, self.reward, self.terminal, {}\n",
        "\n",
        "        else:\n",
        "            actions = actions * self.hmax  # actions initially is scaled between 0 to 1\n",
        "            actions = actions.astype(int)  # convert into integer because we can't by fraction of shares\n",
        "\n",
        "\n",
        "            if self.turbulence_threshold is not None:\n",
        "                if self.turbulence >= self.turbulence_threshold:\n",
        "                    actions = np.array([-self.hmax] * self.stock_dim)\n",
        "            \n",
        "            # Mandatory Selling Profits and Losses\n",
        "            for i in range(0,len(actions)):\n",
        "              # self.total_price\n",
        "              # self.total_stockss\n",
        "              # self.avg_price\n",
        "              #print(\"Current Price\")\n",
        "              #print(np.array(self.state[1 : (self.stock_dim + 1)])[i])\n",
        "              current_price  = np.array(self.state[1 : (self.stock_dim + 1)])[i]\n",
        "              avg_price = self.avg_price[i]\n",
        "              if current_price > ( avg_price * 0.3  + avg_price)  and avg_price > 0:\n",
        "                actions[i] = self.total_stockss[i] * -1\n",
        "                #print(\"Updated actions\")\n",
        "                #self.total_price[i] = 0.0\n",
        "                print(round(current_price) , \"|\", \" avg_price\", round(avg_price), \"Profit\", current_price -  avg_price)\n",
        "              elif current_price < (avg_price - avg_price * 0.15)  and avg_price > 0:\n",
        "                actions[i] = self.total_stockss[i] * -1\n",
        "                print(round(current_price) , \"|\", \" avg_price\", round(avg_price),  \"loss\", current_price -  avg_price )\n",
        "                #self.total_price[i] = 0.0\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "            begin_total_asset = self.state[0] + sum(np.array(self.state[1 : (self.stock_dim + 1)])* np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            # print(\"begin_total_asset:{}\".format(begin_total_asset))\n",
        "            #print(\"*\" * 20 )\n",
        "            #print(np.array(self.state[1 : (self.stock_dim + 1)]))\n",
        "            \n",
        "            #print(np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            #print(\"*\" * 20 )\n",
        "            \n",
        "            #print(\"AVG PRices\")  \n",
        "            #print(self.avg_price)\n",
        "            #print(np.where(np.array(self.state[1 : (self.stock_dim + 1)]) > (self.avg_price + self.avg_price*.15))) \n",
        "            #print(self.avg_price)\n",
        "            #print(self.total_stockss)\n",
        "            #print(\"*\"*20)\n",
        "\n",
        "            # if sum(self.total_stockss) !=0:\n",
        "            #   for i in np.where(np.array(self.state[1 : (self.stock_dim + 1)]) > (self.avg_price + self.avg_price*.15)):\n",
        "            #     for a in i:\n",
        "            #       actions[a] = actions[a]*-1\n",
        "\n",
        "            # for a in actions:\n",
        "            #   print(a)\n",
        "\n",
        "            argsort_actions = np.argsort(actions)\n",
        "            sell_index = argsort_actions[: np.where(actions < 0)[0].shape[0]]\n",
        "            buy_index = argsort_actions[::-1][: np.where(actions > 0)[0].shape[0]]\n",
        "\n",
        "            for index in sell_index:\n",
        "              actions[index] = self._sell_stock(index, actions[index]) * (-1)\n",
        "\n",
        "            for index in buy_index:\n",
        "                actions[index] = self._buy_stock(index, actions[index])\n",
        "\n",
        "\n",
        "            self.actions_memory.append(actions)\n",
        "\n",
        "            # state: s -> s+1\n",
        "            self.day += 1\n",
        "            self.data = self.df.loc[self.day, :]\n",
        "            if self.turbulence_threshold is not None:\n",
        "                if len(self.df.tic.unique()) == 1:\n",
        "                    self.turbulence = self.data[self.risk_indicator_col]\n",
        "                elif len(self.df.tic.unique()) > 1:\n",
        "                    self.turbulence = self.data[self.risk_indicator_col].values[0]\n",
        "\n",
        "\n",
        "\n",
        "            \n",
        "            \n",
        "            recent_price = self.state[1 : (self.stock_dim + 1)]\n",
        "            recent_buy_sell = actions\n",
        "            \n",
        "            self.total_price = np.add(np.multiply(recent_price,\n",
        "                                           recent_buy_sell,\n",
        "                                           out=np.zeros_like(recent_price),\n",
        "                                           where=recent_buy_sell!=0.0),\n",
        "                                      self.total_price,\n",
        "                                      out=np.zeros_like(self.total_price),\n",
        "                                           where=recent_buy_sell!=0.0 )\n",
        "            \n",
        "            #self.total_price = np.where(self.total_price<0.0, 0.0, self.total_price)\n",
        "            self.total_stockss = actions + self.total_stockss\n",
        "            \n",
        "            self.avg_price = np.divide(self.total_price,\n",
        "                                       self.total_stockss,\n",
        "                                       out=np.zeros_like(self.total_price),\n",
        "                                       where=recent_buy_sell!=0.0)\n",
        "          \n",
        "            # print(\"Total Stocks\")\n",
        "            # print(self.total_stockss)\n",
        "            # print(\"Price\")\n",
        "            # print(recent_price)\n",
        "\n",
        "            # print(\"Avg Price\")\n",
        "            # print(self.avg_price)\n",
        "            # print(\"Total Price\")\n",
        "            # print(self.total_price)\n",
        "\n",
        "\n",
        "            self.state = self._update_state()\n",
        "\n",
        "            end_total_asset = self.state[0] + sum(\n",
        "                np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "                * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "            )\n",
        "            self.asset_memory.append(end_total_asset)\n",
        "            self.date_memory.append(self._get_date())\n",
        "            self.reward = end_total_asset - begin_total_asset\n",
        "            self.rewards_memory.append(self.reward)\n",
        "            self.reward = self.reward * self.reward_scaling\n",
        "            self.state_memory.append(\n",
        "                self.state\n",
        "            )  # add current state in state_recorder for each step\n",
        "\n",
        "        return self.state, self.reward, self.terminal, {}\n",
        "\n",
        "    def reset(self):\n",
        "        # initiate state\n",
        "        self.state = self._initiate_state()\n",
        "\n",
        "        if self.initial:\n",
        "            self.asset_memory = [\n",
        "                self.initial_amount\n",
        "                + np.sum(\n",
        "                    np.array(self.num_stock_shares)\n",
        "                    * np.array(self.state[1 : 1 + self.stock_dim])\n",
        "                )\n",
        "            ]\n",
        "        else:\n",
        "            previous_total_asset = self.previous_state[0] + sum(\n",
        "                np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "                * np.array(\n",
        "                    self.previous_state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]\n",
        "                )\n",
        "            )\n",
        "            self.asset_memory = [previous_total_asset]\n",
        "\n",
        "        self.day = 0\n",
        "        self.data = self.df.loc[self.day, :]\n",
        "        self.turbulence = 0\n",
        "        self.cost = 0\n",
        "        self.trades = 0\n",
        "        self.terminal = False\n",
        "        # self.iteration=self.iteration\n",
        "        self.rewards_memory = []\n",
        "        self.actions_memory = []\n",
        "        self.date_memory = [self._get_date()]\n",
        "        if self.total_price[5] > 0.0:\n",
        "          dd = pd.DataFrame(data = self.avg_price)\n",
        "          dd.to_csv(\"avg\")\n",
        "          dd = pd.DataFrame(data = self.total_price)\n",
        "          dd.to_csv(\"tp\")\n",
        "        \n",
        "        self.episode += 1\n",
        "        self.total_price = np.array([0.0] * 29)\n",
        "        self.avg_price = np.array([0.0] * 29)\n",
        "        self.total_stockss = np.array([0.0] * 29)\n",
        "        print(\"Resteting Account\")\n",
        "        return self.state\n",
        "\n",
        "    def render(self, mode=\"human\", close=False):\n",
        "        return self.state\n",
        "\n",
        "    def _initiate_state(self):\n",
        "        if self.initial:\n",
        "            # For Initial State\n",
        "            if len(self.df.tic.unique()) > 1:\n",
        "                # for multiple stock\n",
        "                state = (\n",
        "                    [self.initial_amount]\n",
        "                    + self.data.close.values.tolist()\n",
        "                    + self.num_stock_shares\n",
        "                    + sum(\n",
        "                        (\n",
        "                            self.data[tech].values.tolist()\n",
        "                            for tech in self.tech_indicator_list\n",
        "                        ),\n",
        "                        [],\n",
        "                    )\n",
        "                )  # append initial stocks_share to initial state, instead of all zero\n",
        "            else:\n",
        "                # for single stock\n",
        "                state = (\n",
        "                    [self.initial_amount]\n",
        "                    + [self.data.close]\n",
        "                    + [0] * self.stock_dim\n",
        "                    + sum(([self.data[tech]] for tech in self.tech_indicator_list), [])\n",
        "                )\n",
        "        else:\n",
        "            # Using Previous State\n",
        "            if len(self.df.tic.unique()) > 1:\n",
        "                # for multiple stock\n",
        "                state = (\n",
        "                    [self.previous_state[0]]\n",
        "                    + self.data.close.values.tolist()\n",
        "                    + self.previous_state[\n",
        "                        (self.stock_dim + 1) : (self.stock_dim * 2 + 1)\n",
        "                    ]\n",
        "                    + sum(\n",
        "                        (\n",
        "                            self.data[tech].values.tolist()\n",
        "                            for tech in self.tech_indicator_list\n",
        "                        ),\n",
        "                        [],\n",
        "                    )\n",
        "                )\n",
        "            else:\n",
        "                # for single stock\n",
        "                state = (\n",
        "                    [self.previous_state[0]]\n",
        "                    + [self.data.close]\n",
        "                    + self.previous_state[\n",
        "                        (self.stock_dim + 1) : (self.stock_dim * 2 + 1)\n",
        "                    ]\n",
        "                    + sum(([self.data[tech]] for tech in self.tech_indicator_list), [])\n",
        "                )\n",
        "        return state\n",
        "\n",
        "    def _update_state(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            # for multiple stock\n",
        "            state = (\n",
        "                [self.state[0]]\n",
        "                + self.data.close.values.tolist()\n",
        "                + list(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "                + sum((self.data[tech].values.tolist() for tech in self.tech_indicator_list),[],))\n",
        "\n",
        "        else:\n",
        "            # for single stock\n",
        "            state = ([self.state[0]] + [self.data.close] \n",
        "                     + list(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "                     + sum(([self.data[tech]] for tech in self.tech_indicator_list), []))\n",
        "\n",
        "        return state\n",
        "\n",
        "    def _get_date(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            date = self.data.date.unique()[0]\n",
        "        else:\n",
        "            date = self.data.date\n",
        "        return date\n",
        "\n",
        "\n",
        "    def save_asset_memory(self):\n",
        "        date_list = self.date_memory\n",
        "        asset_list = self.asset_memory\n",
        "        # print(len(date_list))\n",
        "        # print(len(asset_list))\n",
        "        df_account_value = pd.DataFrame({\"date\": date_list, \"account_value\": asset_list})\n",
        "        return df_account_value\n",
        "\n",
        "    def save_action_memory(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            # date and close price length must match actions length\n",
        "            date_list = self.date_memory[:-1]\n",
        "            df_date = pd.DataFrame(date_list)\n",
        "            df_date.columns = [\"date\"]\n",
        "\n",
        "            action_list = self.actions_memory\n",
        "            df_actions = pd.DataFrame(action_list)\n",
        "            df_actions.columns = self.data.tic.values\n",
        "            df_actions.index = df_date.date\n",
        "            # df_actions = pd.DataFrame({'date':date_list,'actions':action_list})\n",
        "        else:\n",
        "            date_list = self.date_memory[:-1]\n",
        "            action_list = self.actions_memory\n",
        "            df_actions = pd.DataFrame({\"date\": date_list, \"actions\": action_list})\n",
        "        return df_actions\n",
        "\n",
        "    def _seed(self, seed=None):\n",
        "        self.np_random, seed = seeding.np_random(seed)\n",
        "        return [seed]\n",
        "\n",
        "    def get_sb_env(self):\n",
        "        e = DummyVecEnv([lambda: self])\n",
        "        obs = e.reset()\n",
        "        return e, obs\n"
      ],
      "metadata": {
        "id": "4p02b2cx_dj5"
      },
      "execution_count": 215,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stock_dimension = len(init_train_data.tic.unique())\n",
        "state_space = 1 + 2 * stock_dimension + len(config.INDICATORS) * stock_dimension\n",
        "print(f\"Stock Dimension: {stock_dimension}, State Space: {state_space}\")\n",
        "buy_cost_list = sell_cost_list = [0.001] * stock_dimension\n",
        "num_stock_shares = [0] * stock_dimension"
      ],
      "metadata": {
        "id": "rg7DTZbVV10n",
        "outputId": "5d8abfb4-d792-41f0-cf6c-df5afa4c3aa3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 216,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Stock Dimension: 29, State Space: 291\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Defining the environment kwargs\n",
        "\n",
        "initial_amount = 1000000\n",
        "env_kwargs = {\n",
        "    \"hmax\": 100,\n",
        "    \"initial_amount\": initial_amount,\n",
        "    \"num_stock_shares\": num_stock_shares,\n",
        "    \"buy_cost_pct\": buy_cost_list,\n",
        "    \"sell_cost_pct\": sell_cost_list,\n",
        "    \"state_space\": state_space,\n",
        "    \"stock_dim\": stock_dimension,\n",
        "    \"tech_indicator_list\": config.INDICATORS,\n",
        "    \"action_space\": stock_dimension,\n",
        "    \"reward_scaling\": 1e-4,\n",
        "}"
      ],
      "metadata": {
        "id": "zODPmc5hV42J"
      },
      "execution_count": 217,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Instantiate the training gym compatible environment\n",
        "e_train_gym = GokuEnv(df = init_train_data, **env_kwargs)\n",
        "env_train, _ = e_train_gym.get_sb_env()\n",
        "agent = DRLAgent(env = env_train)"
      ],
      "metadata": {
        "id": "7n2QYGP5V58p",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5971298f-1e41-45b1-bb4e-591454e8f6aa"
      },
      "execution_count": 218,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Resteting Account\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Instantiate the trading environment\n",
        "e_trade_gym = GokuEnv(df = init_trade_data, turbulence_threshold = None, **env_kwargs)"
      ],
      "metadata": {
        "id": "xoBM74XEX1oI"
      },
      "execution_count": 219,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Trade performance code\n",
        "The following code calculates trade performance metrics, which are then used as an objective for optimizing hyperparameter values.\n",
        "\n",
        "There are several available metrics. In this tutorial, the default choice is the ratio of average value of winning to losing trades."
      ],
      "metadata": {
        "id": "Dq6zJ14ZYsTK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "#Main method\n",
        "# Calculates Trade Performance for Objective\n",
        "# Called from objective method\n",
        "# Returns selected trade perf metric(s)\n",
        "# Requires actions and associated prices\n",
        "\n",
        "def calc_trade_perf_metric(df_actions, \n",
        "                           df_prices_trade,\n",
        "                           tp_metric,\n",
        "                           dbg=False):\n",
        "  \n",
        "    df_actions_p, df_prices_p, tics = prep_data(df_actions.copy(),\n",
        "                                                df_prices_trade.copy())\n",
        "    # actions predicted by trained model on trade data\n",
        "    df_actions_p.to_csv('df_actions.csv') \n",
        "\n",
        "    \n",
        "    # Confirms that actions, prices and tics are consistent\n",
        "    df_actions_s, df_prices_s, tics_prtfl = \\\n",
        "        sync_tickers(df_actions_p.copy(),df_prices_p.copy(),tics)\n",
        "    \n",
        "    # copy to ensure that tics from portfolio remains unchanged\n",
        "    tics = tics_prtfl.copy()\n",
        "    \n",
        "    # Analysis is performed on each portfolio ticker\n",
        "    perf_data= collect_performance_data(df_actions_s, df_prices_s, tics)\n",
        "    # profit/loss for each ticker\n",
        "    pnl_all = calc_pnl_all(perf_data, tics)\n",
        "    # values for trade performance metrics\n",
        "    perf_results = calc_trade_perf(pnl_all)\n",
        "    df = pd.DataFrame.from_dict(perf_results, orient='index')\n",
        "    \n",
        "    # calculate and return trade metric value as objective\n",
        "    m = calc_trade_metric(df,tp_metric)\n",
        "    print(f'Ratio Avg Win/Avg Loss: {m}')\n",
        "    k = str(len(tpm_hist)+1)\n",
        "    # save metric value\n",
        "    tpm_hist[k] = m\n",
        "    return m\n",
        "\n",
        "\n",
        "# Supporting methods\n",
        "def calc_trade_metric(df,metric='avgwl'):\n",
        "    '''# trades', '# wins', '# losses', 'wins total value', 'wins avg value',\n",
        "       'losses total value', 'losses avg value'''\n",
        "    # For this tutorial, the only metric available is the ratio of \n",
        "    #  average values of winning to losing trades. Others are in development.\n",
        "    \n",
        "    # some test cases produce no losing trades.\n",
        "    # The code below assigns a value as a multiple of the highest value during\n",
        "    # previous hp optimization runs. If the first run experiences no losses,\n",
        "    # a fixed value is assigned for the ratio\n",
        "    tpm_mult = 1.0\n",
        "    avgwl_no_losses = 25\n",
        "    if metric == 'avgwl':\n",
        "        if sum(df['# losses']) == 0:\n",
        "          try:\n",
        "            return max(tpm_hist.values())*tpm_mult\n",
        "          except ValueError:\n",
        "            return avgwl_no_losses\n",
        "        avg_w = sum(df['wins total value'])/sum(df['# wins'])\n",
        "        avg_l = sum(df['losses total value'])/sum(df['# losses'])\n",
        "        m = abs(avg_w/avg_l)\n",
        "\n",
        "    return m\n",
        "\n",
        "\n",
        "def prep_data(df_actions,\n",
        "              df_prices_trade):\n",
        "    \n",
        "    df=df_prices_trade[['date','close','tic']]\n",
        "    df['Date'] = pd.to_datetime(df['date'])\n",
        "    df = df.set_index('Date')\n",
        "    # set indices on both df to datetime\n",
        "    idx = pd.to_datetime(df_actions.index, infer_datetime_format=True)\n",
        "    df_actions.index=idx\n",
        "    tics = np.unique(df.tic)\n",
        "    n_tics = len(tics)\n",
        "    print(f'Number of tickers: {n_tics}')\n",
        "    print(f'Tickers: {tics}')\n",
        "    dategr = df.groupby('tic')\n",
        "    p_d={t:dategr.get_group(t).loc[:,'close'] for t in tics}\n",
        "    df_prices = pd.DataFrame.from_dict(p_d)\n",
        "    df_prices.index = df_prices.index.normalize()\n",
        "    return df_actions, df_prices, tics\n",
        "\n",
        "\n",
        "# prepares for integrating action and price files\n",
        "def link_prices_actions(df_a,\n",
        "                        df_p):\n",
        "    cols_a = [t + '_a' for t in df_a.columns]\n",
        "    df_a.columns = cols_a\n",
        "    cols_p = [t + '_p' for t in df_p.columns]\n",
        "    df_p.columns = cols_p\n",
        "    return df_a, df_p\n",
        "\n",
        "\n",
        "def sync_tickers(df_actions,df_tickers_p,tickers):\n",
        "    # Some DOW30 components may not be included in portfolio\n",
        "    # passed tickers includes all DOW30 components\n",
        "    # actions and ticker files may have different length indices\n",
        "    if len(df_actions) != len(df_tickers_p):\n",
        "      msng_dates = set(df_actions.index)^set(df_tickers_p.index)\n",
        "      try:\n",
        "        #assumption is prices has one additional timestamp (row)\n",
        "        df_tickers_p.drop(msng_dates,inplace=True)\n",
        "      except:\n",
        "        df_actions.drop(msng_dates,inplace=True)\n",
        "    df_actions, df_tickers_p = link_prices_actions(df_actions,df_tickers_p)\n",
        "    # identify any DOW components not in portfolio\n",
        "    t_not_in_a = [t for t in tickers if t + '_a' not in list(df_actions.columns)]\n",
        "  \n",
        "    # remove t_not_in_a from df_tickers_p\n",
        "    drop_cols = [t + '_p' for t in t_not_in_a]\n",
        "    df_tickers_p.drop(columns=drop_cols,inplace=True)\n",
        "    \n",
        "    # Tickers in portfolio\n",
        "    tickers_prtfl = [c.split('_')[0] for c in df_actions.columns]\n",
        "    return df_actions,df_tickers_p, tickers_prtfl\n",
        "\n",
        "def collect_performance_data(dfa,dfp,tics, dbg=False):\n",
        "    \n",
        "    perf_data = {}\n",
        "    # In current version, files columns include secondary identifier\n",
        "    for t in tics:\n",
        "        # actions: purchase/sale of DOW equities\n",
        "        acts = dfa['_'.join([t,'a'])].values\n",
        "        # ticker prices\n",
        "        prices = dfp['_'.join([t,'p'])].values\n",
        "        # market value of purchases/sales\n",
        "        tvals_init = np.multiply(acts,prices)\n",
        "        d={'actions':acts, 'prices':prices,'init_values':tvals_init}\n",
        "        perf_data[t]=d\n",
        "\n",
        "    return perf_data\n",
        "\n",
        "\n",
        "def calc_pnl_all(perf_dict, tics_all):\n",
        "    # calculate profit/loss for each ticker\n",
        "    print(f'Calculating profit/loss for each ticker')\n",
        "    pnl_all = {}\n",
        "    for tic in tics_all:\n",
        "        pnl_t = []\n",
        "        tic_data = perf_dict[tic]\n",
        "        init_values = tic_data['init_values']\n",
        "        acts = tic_data['actions']\n",
        "        prices = tic_data['prices']\n",
        "        cs = np.cumsum(acts)\n",
        "        args_s = [i + 1 for i in range(len(cs) - 1) if cs[i + 1] < cs[i]]\n",
        "        # tic actions with no sales\n",
        "        if not args_s:\n",
        "            pnl = complete_calc_buyonly(acts, prices, init_values)\n",
        "            pnl_all[tic] = pnl\n",
        "            continue\n",
        "        # copy acts: acts_rev will be revised based on closing/reducing init positions\n",
        "        pnl_all = execute_position_sales(tic,acts,prices,args_s,pnl_all)\n",
        "\n",
        "    return pnl_all\n",
        "\n",
        "\n",
        "def complete_calc_buyonly(actions, prices, init_values):\n",
        "    # calculate final pnl for each ticker assuming no sales\n",
        "    fnl_price = prices[-1]\n",
        "    final_values = np.multiply(fnl_price, actions)\n",
        "    pnl = np.subtract(final_values, init_values)\n",
        "    return pnl\n",
        "\n",
        "\n",
        "def execute_position_sales(tic,acts,prices,args_s,pnl_all):\n",
        "  # calculate final pnl for each ticker with sales\n",
        "    pnl_t = []\n",
        "    acts_rev = acts.copy()\n",
        "    # location of sales transactions\n",
        "    for s in args_s:  # s is scaler\n",
        "        # price_s = [prices[s]]\n",
        "        act_s = [acts_rev[s]]\n",
        "        args_b = [i for i in range(s) if acts_rev[i] > 0]\n",
        "        prcs_init_trades = prices[args_b]\n",
        "        acts_init_trades = acts_rev[args_b]\n",
        "  \n",
        "        # update actions for sales\n",
        "        # reduce/eliminate init values through trades\n",
        "        # always start with earliest purchase that has not been closed through sale\n",
        "        # selectors for purchase and sales trades\n",
        "        # find earliest remaining purchase\n",
        "        arg_sel = min(args_b)\n",
        "        # sel_s = len(acts_trades) - 1\n",
        "\n",
        "        # closing part/all of earliest init trade not yet closed\n",
        "        # sales actions are negative\n",
        "        # in this test case, abs_val of init and sales share counts are same\n",
        "        # zero-out sales actions\n",
        "        # market value of sale\n",
        "        # max number of shares to be closed: may be less than # originally purchased\n",
        "        acts_shares = min(abs(act_s.pop()), acts_rev[arg_sel])\n",
        "\n",
        "        # mv of shares when purchased\n",
        "        mv_p = abs(acts_shares * prices[arg_sel])\n",
        "        # mv of sold shares\n",
        "        mv_s = abs(acts_shares * prices[s])\n",
        "\n",
        "        # calc pnl\n",
        "        pnl = mv_s - mv_p\n",
        "        # reduce init share count\n",
        "        # close all/part of init purchase\n",
        "        acts_rev[arg_sel] -= acts_shares\n",
        "        acts_rev[s] += acts_shares\n",
        "        # calculate pnl for trade\n",
        "        # value of associated purchase\n",
        "        \n",
        "        # find earliest non-zero positive act in acts_revs\n",
        "        pnl_t.append(pnl)\n",
        "    \n",
        "    pnl_op = calc_pnl_for_open_positions(acts_rev, prices)\n",
        "    #pnl_op is list\n",
        "    # add pnl_op results (if any) to pnl_t (both lists)\n",
        "    pnl_t.extend(pnl_op)\n",
        "    #print(f'Total pnl for {tic}: {np.sum(pnl_t)}')\n",
        "    pnl_all[tic] = np.array(pnl_t)\n",
        "    return pnl_all\n",
        "\n",
        "\n",
        "def calc_pnl_for_open_positions(acts,prices):\n",
        "    # identify any positive share values after accounting for sales\n",
        "    pnl = []\n",
        "    fp = prices[-1] # last price\n",
        "    open_pos_arg = np.argwhere(acts>0)\n",
        "    if len(open_pos_arg)==0:return pnl # no open positions\n",
        "\n",
        "    mkt_vals_open = np.multiply(acts[open_pos_arg], prices[open_pos_arg])\n",
        "    # mkt val at end of testing period\n",
        "    # treat as trades for purposes of calculating pnl at end of testing period\n",
        "    mkt_vals_final = np.multiply(fp, acts[open_pos_arg])\n",
        "    pnl_a = np.subtract(mkt_vals_final, mkt_vals_open)\n",
        "    #convert to list\n",
        "    pnl = [i[0] for i in pnl_a.tolist()]\n",
        "    #print(f'Market value of open positions at end of testing {pnl}')\n",
        "    return pnl\n",
        "\n",
        "\n",
        "def calc_trade_perf(pnl_d):\n",
        "    # calculate trade performance metrics\n",
        "    perf_results = {}\n",
        "    for t,pnl in pnl_d.items():\n",
        "        wins = pnl[pnl>0]  # total val\n",
        "        losses = pnl[pnl<0]\n",
        "        n_wins = len(wins)\n",
        "        n_losses = len(losses)\n",
        "        n_trades = n_wins + n_losses\n",
        "        wins_val = np.sum(wins)\n",
        "        losses_val = np.sum(losses)\n",
        "        wins_avg = 0 if n_wins==0 else np.mean(wins)\n",
        "        #print(f'{t} n_wins: {n_wins} n_losses: {n_losses}')\n",
        "        losses_avg = 0 if n_losses==0 else np.mean(losses)\n",
        "        d = {'# trades':n_trades,'# wins':n_wins,'# losses':n_losses,\n",
        "             'wins total value':wins_val, 'wins avg value':wins_avg,\n",
        "             'losses total value':losses_val, 'losses avg value':losses_avg,}\n",
        "        perf_results[t] = d\n",
        "    return perf_results"
      ],
      "metadata": {
        "id": "QydaFexDX5BQ"
      },
      "execution_count": 220,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tuning hyperparameters using Optuna"
      ],
      "metadata": {
        "id": "c_AT1dsuZApj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params(trial:optuna.Trial):\n",
        "  # Size of the replay buffer\n",
        "  buffer_size = trial.suggest_categorical(\"buffer_size\", [int(1e4), int(1e5), int(1e6)])\n",
        "  learning_rate = trial.suggest_loguniform(\"learning_rate\", 1e-5, 1)\n",
        "  batch_size = trial.suggest_categorical(\"batch_size\", [32, 64, 128, 256, 512])\n",
        "  \n",
        "  return {\"buffer_size\": buffer_size,\n",
        "          \"learning_rate\":learning_rate,\n",
        "          \"batch_size\":batch_size}"
      ],
      "metadata": {
        "id": "fKGkKHV7Y9AA"
      },
      "execution_count": 221,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Set Variables\n",
        "## Fixed\n",
        "tpm_hist = {}  # record tp metric values for trials\n",
        "tp_metric = 'avgwl'  # specified trade_param_metric: ratio avg value win/loss\n",
        "## Settable by User\n",
        "n_trials = 50  # number of HP optimization runs\n",
        "total_timesteps = 2000 # per HP optimization run\n",
        "## Logging callback params\n",
        "lc_threshold=1e-5\n",
        "lc_patience=15\n",
        "lc_trial_number=5"
      ],
      "metadata": {
        "id": "N6blMWpz-pR0"
      },
      "execution_count": 222,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "OPTIONAL CODE FOR SAMPLING HYPERPARAMETERS\n",
        "\n",
        "Replace current call in function objective with\n",
        "\n",
        "hyperparameters = sample_ddpg_params_all(trial)"
      ],
      "metadata": {
        "id": "fHET-odKZShg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params_all(trial:optuna.Trial,\n",
        "                           # fixed values from previous study\n",
        "                           learning_rate=0.0103,\n",
        "                           batch_size=128,\n",
        "                           buffer_size=int(1e6)):\n",
        "\n",
        "    gamma = trial.suggest_categorical(\"gamma\", [0.94, 0.96, 0.98])\n",
        "    # Polyak coeff\n",
        "    tau = trial.suggest_categorical(\"tau\", [0.08, 0.1, 0.12])\n",
        "\n",
        "    train_freq = trial.suggest_categorical(\"train_freq\", [512,768,1024])\n",
        "    gradient_steps = train_freq\n",
        "    \n",
        "    noise_type = trial.suggest_categorical(\"noise_type\", [\"ornstein-uhlenbeck\", \"normal\", None])\n",
        "    noise_std = trial.suggest_categorical(\"noise_std\", [.1,.2,.3] )\n",
        "\n",
        "    # NOTE: Add \"verybig\" to net_arch when tuning HER (see TD3)\n",
        "    net_arch = trial.suggest_categorical(\"net_arch\", [\"small\", \"big\"])\n",
        "    # activation_fn = trial.suggest_categorical('activation_fn', [nn.Tanh, nn.ReLU, nn.ELU, nn.LeakyReLU])\n",
        "\n",
        "    net_arch = {\n",
        "        \"small\": [64, 64],\n",
        "        \"medium\": [256, 256],\n",
        "        \"big\": [512, 512],\n",
        "    }[net_arch]\n",
        "  \n",
        "    hyperparams = {\n",
        "        \"batch_size\": batch_size,\n",
        "        \"buffer_size\": buffer_size,\n",
        "        \"gamma\": gamma,\n",
        "        \"gradient_steps\": gradient_steps,\n",
        "        \"learning_rate\": learning_rate,\n",
        "        \"tau\": tau,\n",
        "        \"train_freq\": train_freq,\n",
        "        #\"noise_std\": noise_std,\n",
        "        #\"noise_type\": noise_type,\n",
        "        \n",
        "        \"policy_kwargs\": dict(net_arch=net_arch)\n",
        "    }\n",
        "    return hyperparams"
      ],
      "metadata": {
        "id": "htLdZHKTZKPJ"
      },
      "execution_count": 223,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params_all(trial:optuna.Trial,\n",
        "                           # fixed values from previous study\n",
        "                           learning_rate=0.0103,\n",
        "                           batch_size=128,\n",
        "                           buffer_size=int(1e6)):\n",
        "\n",
        "    gamma = trial.suggest_categorical(\"gamma\", [0.94, 0.96, 0.98])\n",
        "    # Polyak coeff\n",
        "    tau = trial.suggest_categorical(\"tau\", [0.08, 0.1, 0.12])\n",
        "\n",
        "    train_freq = trial.suggest_categorical(\"train_freq\", [512,768,1024])\n",
        "    gradient_steps = train_freq\n",
        "    \n",
        "    noise_type = trial.suggest_categorical(\"noise_type\", [\"ornstein-uhlenbeck\", \"normal\", None])\n",
        "    noise_std = trial.suggest_categorical(\"noise_std\", [.1,.2,.3] )\n",
        "\n",
        "    # NOTE: Add \"verybig\" to net_arch when tuning HER (see TD3)\n",
        "    net_arch = trial.suggest_categorical(\"net_arch\", [\"small\", \"big\"])\n",
        "    # activation_fn = trial.suggest_categorical('activation_fn', [nn.Tanh, nn.ReLU, nn.ELU, nn.LeakyReLU])\n",
        "\n",
        "    net_arch = {\n",
        "        \"small\": [64, 64],\n",
        "        \"medium\": [256, 256],\n",
        "        \"big\": [512, 512],\n",
        "    }[net_arch]\n",
        "  \n",
        "    hyperparams = {\n",
        "        \"batch_size\": batch_size,\n",
        "        \"buffer_size\": buffer_size,\n",
        "        \"gamma\": gamma,\n",
        "        \"gradient_steps\": gradient_steps,\n",
        "        \"learning_rate\": learning_rate,\n",
        "        \"tau\": tau,\n",
        "        \"train_freq\": train_freq,\n",
        "        #\"noise_std\": noise_std,\n",
        "        #\"noise_type\": noise_type,\n",
        "        \n",
        "        \"policy_kwargs\": dict(net_arch=net_arch)\n",
        "    }\n",
        "    return hyperparams"
      ],
      "metadata": {
        "id": "dJYdh-E4ZQKP"
      },
      "execution_count": 224,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Callbacks\n",
        "\n",
        "\n",
        "1. The callback will terminate if the improvement margin is below certain point\n",
        "2. It will terminate after certain number of trial_number are reached, not before that\n",
        "3. It will hold its patience to reach the threshold"
      ],
      "metadata": {
        "id": "mWEoJMPLZY4o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class LoggingCallback:\n",
        "    def __init__(self,threshold,trial_number,patience):\n",
        "      '''\n",
        "      threshold:int tolerance for increase in objective\n",
        "      trial_number: int Prune after minimum number of trials\n",
        "      patience: int patience for the threshold\n",
        "      '''\n",
        "      self.threshold = threshold\n",
        "      self.trial_number  = trial_number\n",
        "      self.patience = patience\n",
        "      print(f'Callback threshold {self.threshold}, \\\n",
        "            trial_number {self.trial_number}, \\\n",
        "            patience {self.patience}')\n",
        "      self.cb_list = [] #Trials list for which threshold is reached\n",
        "    def __call__(self,study:optuna.study, frozen_trial:optuna.Trial):\n",
        "      #Setting the best value in the current trial\n",
        "      study.set_user_attr(\"previous_best_value\", study.best_value)\n",
        "      \n",
        "      #Checking if the minimum number of trials have pass\n",
        "      if frozen_trial.number >self.trial_number:\n",
        "          previous_best_value = study.user_attrs.get(\"previous_best_value\",None)\n",
        "          #Checking if the previous and current objective values have the same sign\n",
        "          if previous_best_value * study.best_value >=0:\n",
        "              #Checking for the threshold condition\n",
        "              if abs(previous_best_value-study.best_value) < self.threshold: \n",
        "                  self.cb_list.append(frozen_trial.number)\n",
        "                  #If threshold is achieved for the patience amount of time\n",
        "                  if len(self.cb_list)>self.patience:\n",
        "                      print('The study stops now...')\n",
        "                      print('With number',frozen_trial.number ,'and value ',frozen_trial.value)\n",
        "                      print('The previous and current best values are {} and {} respectively'\n",
        "                              .format(previous_best_value, study.best_value))\n",
        "                      study.stop()"
      ],
      "metadata": {
        "id": "He7GTZZUZWP8"
      },
      "execution_count": 225,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import clear_output\n",
        "import sys   \n",
        "\n",
        "os.makedirs(\"models\",exist_ok=True)\n",
        "\n",
        "def objective(trial:optuna.Trial):\n",
        "  #Trial will suggest a set of hyperparamters from the specified range\n",
        "\n",
        "  # Optional to optimize larger set of parameters\n",
        "  # hyperparameters = sample_ddpg_params_all(trial)\n",
        "  \n",
        "  # Optimize buffer size, batch size, learning rate\n",
        "  hyperparameters = sample_ddpg_params_all(trial)\n",
        "  print(f'Hyperparameters from objective: {hyperparameters.keys()}')\n",
        "  policy_kwargs = None  # default\n",
        "  if 'policy_kwargs' in hyperparameters.keys():\n",
        "    policy_kwargs = hyperparameters['policy_kwargs']\n",
        "    del hyperparameters['policy_kwargs']\n",
        "    print(f'Policy keyword arguments {policy_kwargs}')\n",
        "  model_ddpg = agent.get_model(\"ddpg\",\n",
        "                               policy_kwargs = policy_kwargs,\n",
        "                               model_kwargs = hyperparameters )\n",
        "  \n",
        "  #You can increase it for better comparison\n",
        "  trained_ddpg = agent.train_model(model=model_ddpg,\n",
        "                                   tb_log_name=\"ddpg\",\n",
        "                                   total_timesteps=total_timesteps)\n",
        "  trained_ddpg.save('models/ddpg_{}.pth'.format(trial.number))\n",
        "  clear_output(wait=True)\n",
        "  \n",
        "  #For the given hyperparamters, determine the account value in the trading period\n",
        "  df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
        "    model=trained_ddpg, \n",
        "    environment = e_trade_gym)\n",
        " \n",
        "  # Calculate trade performance metric\n",
        "  # Currently ratio of average win and loss market values\n",
        "  tpm = calc_trade_perf_metric(df_actions,init_trade_data,tp_metric)\n",
        "  return tpm\n",
        "\n",
        "#Create a study object and specify the direction as 'maximize'\n",
        "#As you want to maximize sharpe\n",
        "#Pruner stops not promising iterations\n",
        "#Use a pruner, else you will get error related to divergence of model\n",
        "#You can also use Multivariate samplere\n",
        "#sampler = optuna.samplers.TPESampler(multivarite=True,seed=42)\n",
        "sampler = optuna.samplers.TPESampler()\n",
        "\n",
        "study = optuna.create_study(study_name=\"ddpg_study\",direction='maximize',\n",
        "                            sampler = sampler, pruner=optuna.pruners.HyperbandPruner())\n",
        "\n",
        "logging_callback = LoggingCallback(threshold=lc_threshold,\n",
        "                                   patience=lc_patience,\n",
        "                                   trial_number=lc_trial_number)\n",
        "#You can increase the n_trials for a better search space scanning\n",
        "study.optimize(objective, n_trials=n_trials,catch=(ValueError,),callbacks=[logging_callback])"
      ],
      "metadata": {
        "id": "b6EiwiiIZpRU",
        "outputId": "518192b7-adaa-4c37-94b9-dc8723ad4f00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 963
        }
      },
      "execution_count": 226,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2023-05-01 20:55:55,697]\u001b[0m A new study created in memory with name: ddpg_study\u001b[0m\n",
            "\u001b[33m[W 2023-05-01 20:55:55,740]\u001b[0m Trial 0 failed with parameters: {'gamma': 0.98, 'tau': 0.08, 'train_freq': 512, 'noise_type': 'ornstein-uhlenbeck', 'noise_std': 0.2, 'net_arch': 'big'} because of the following error: ValueError('The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()').\u001b[0m\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\", line 200, in _run_trial\n",
            "    value_or_values = func(trial)\n",
            "  File \"<ipython-input-226-66f3543408ef>\", line 25, in objective\n",
            "    trained_ddpg = agent.train_model(model=model_ddpg,\n",
            "  File \"/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\", line 103, in train_model\n",
            "    model = model.learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\", line 123, in learn\n",
            "    return super().learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\", line 216, in learn\n",
            "    return super().learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\", line 311, in learn\n",
            "    rollout = self.collect_rollouts(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\", line 543, in collect_rollouts\n",
            "    new_obs, rewards, dones, infos = env.step(actions)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/vec_env/base_vec_env.py\", line 163, in step\n",
            "    return self.step_wait()\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/vec_env/dummy_vec_env.py\", line 54, in step_wait\n",
            "    obs, self.buf_rews[env_idx], self.buf_dones[env_idx], self.buf_infos[env_idx] = self.envs[env_idx].step(\n",
            "  File \"<ipython-input-215-b0471d00a0e9>\", line 346, in step\n",
            "    where=recent_buy_sell!=0.0 and recent_buy_sell )\n",
            "ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()\n",
            "\u001b[33m[W 2023-05-01 20:55:55,742]\u001b[0m Trial 0 failed with value None.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Callback threshold 1e-05,             trial_number 5,             patience 15\n",
            "Hyperparameters from objective: dict_keys(['batch_size', 'buffer_size', 'gamma', 'gradient_steps', 'learning_rate', 'tau', 'train_freq', 'policy_kwargs'])\n",
            "Policy keyword arguments {'net_arch': [512, 512]}\n",
            "{'batch_size': 128, 'buffer_size': 1000000, 'gamma': 0.98, 'gradient_steps': 512, 'learning_rate': 0.0103, 'tau': 0.08, 'train_freq': 512}\n",
            "Using cuda device\n",
            "Resteting Account\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-226-66f3543408ef>\u001b[0m in \u001b[0;36m<cell line: 56>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     54\u001b[0m                                    trial_number=lc_trial_number)\n\u001b[1;32m     55\u001b[0m \u001b[0;31m#You can increase the n_trials for a better search space scanning\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_trials\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcatch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mValueError\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlogging_callback\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/study.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m    423\u001b[0m         \"\"\"\n\u001b[1;32m    424\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 425\u001b[0;31m         _optimize(\n\u001b[0m\u001b[1;32m    426\u001b[0m             \u001b[0mstudy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m             \u001b[0mfunc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m     64\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m             _optimize_sequential(\n\u001b[0m\u001b[1;32m     67\u001b[0m                 \u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m                 \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m    172\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    173\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 174\u001b[0;31m                 \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfrozen_trial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    175\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mprogress_bar\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-225-184607d05961>\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, study, frozen_trial)\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0moptuna\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfrozen_trial\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0moptuna\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m       \u001b[0;31m#Setting the best value in the current trial\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m       \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_user_attr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"previous_best_value\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_value\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m       \u001b[0;31m#Checking if the minimum number of trials have pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/study.py\u001b[0m in \u001b[0;36mbest_value\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    128\u001b[0m         \"\"\"\n\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m         \u001b[0mbest_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_trial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mbest_value\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/study.py\u001b[0m in \u001b[0;36mbest_trial\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    157\u001b[0m             )\n\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 159\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_best_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_study_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    160\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/storages/_in_memory.py\u001b[0m in \u001b[0;36mget_best_trial\u001b[0;34m(self, study_id)\u001b[0m\n\u001b[1;32m    248\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    249\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbest_trial_id\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 250\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"No trials are completed yet.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    251\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_studies\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstudy_id\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdirections\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m                 raise RuntimeError(\n",
            "\u001b[0;31mValueError\u001b[0m: No trials are completed yet."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "joblib.dump(study, \"final_ddpg_study__.pkl\")"
      ],
      "metadata": {
        "id": "1yIXC6W2ZqPP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Get the best hyperparamters\n",
        "print('Hyperparameters after tuning',study.best_params)\n",
        "print('Hyperparameters before tuning',config.DDPG_PARAMS)"
      ],
      "metadata": {
        "id": "gXzfenQTdqq_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ac510c77-8ba6-43fc-ae7a-396bb7496184"
      },
      "execution_count": 107,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hyperparameters after tuning {'gamma': 0.98, 'tau': 0.08, 'train_freq': 1024, 'noise_type': None, 'noise_std': 0.3, 'net_arch': 'big'}\n",
            "Hyperparameters before tuning {'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "study.best_trial"
      ],
      "metadata": {
        "id": "VQcuj-yidu0O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cfbbc138-4f22-4ecd-d187-c0a88c6167c3"
      },
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.9235269323597551], datetime_start=datetime.datetime(2023, 5, 1, 15, 32, 52, 652114), datetime_complete=datetime.datetime(2023, 5, 1, 15, 33, 11, 635757), params={'gamma': 0.98, 'tau': 0.08, 'train_freq': 1024, 'noise_type': None, 'noise_std': 0.3, 'net_arch': 'big'}, user_attrs={}, system_attrs={}, intermediate_values={}, distributions={'gamma': CategoricalDistribution(choices=(0.94, 0.96, 0.98)), 'tau': CategoricalDistribution(choices=(0.08, 0.1, 0.12)), 'train_freq': CategoricalDistribution(choices=(512, 768, 1024)), 'noise_type': CategoricalDistribution(choices=('ornstein-uhlenbeck', 'normal', None)), 'noise_std': CategoricalDistribution(choices=(0.1, 0.2, 0.3)), 'net_arch': CategoricalDistribution(choices=('small', 'big'))}, trial_id=2, value=None)"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from stable_baselines3 import DDPG\n",
        "tuned_model_ddpg = DDPG.load('models/ddpg_{}.pth'.format(study.best_trial.number),env=env_train)"
      ],
      "metadata": {
        "id": "ZjYe9lOIdw3i"
      },
      "execution_count": 109,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Trading period account value with tuned model\n",
        "df_account_value_tuned, df_actions_tuned = DRLAgent.DRL_prediction(\n",
        "    model=tuned_model_ddpg, \n",
        "    environment = e_trade_gym)"
      ],
      "metadata": {
        "id": "3bnbx87vdzHZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "20164428-254c-4ca4-ac8a-aba8b32da06b"
      },
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hit end!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_account_value_tuned[104:105][\"account_value\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G1Y1jjMLf0qF",
        "outputId": "dced00bf-cb09-401f-8c19-31a21926747e"
      },
      "execution_count": 111,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "104    1000000.0\n",
              "Name: account_value, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 111
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Backtesting with our pruned model\n",
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all_tuned = backtest_stats(account_value=df_account_value_tuned)\n",
        "perf_stats_all_tuned = pd.DataFrame(perf_stats_all_tuned)\n",
        "perf_stats_all_tuned.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_tuned_\"+now+'.csv')"
      ],
      "metadata": {
        "id": "FJrv144od3G_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "af8d6ecb-c1c8-484b-ad93-13b6bc8dd544"
      },
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return          0.0\n",
            "Cumulative returns     0.0\n",
            "Annual volatility      0.0\n",
            "Sharpe ratio           NaN\n",
            "Calmar ratio           NaN\n",
            "Stability              0.0\n",
            "Max drawdown           0.0\n",
            "Omega ratio            NaN\n",
            "Sortino ratio          NaN\n",
            "Skew                   NaN\n",
            "Kurtosis               NaN\n",
            "Tail ratio             NaN\n",
            "Daily value at risk    0.0\n",
            "dtype: float64\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/site-packages/empyrical/stats.py:1412: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  return np.abs(np.percentile(returns, 95)) / np.abs(np.percentile(returns, 5))\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Now train with not tuned hyperaparameters\n",
        "#Default config.ddpg_PARAMS\n",
        "non_tuned_model_ddpg = agent.get_model(\"ddpg\",model_kwargs = config.DDPG_PARAMS )\n",
        "trained_ddpg = agent.train_model(model=non_tuned_model_ddpg, \n",
        "                             tb_log_name='ddpg',\n",
        "                             total_timesteps=50000)"
      ],
      "metadata": {
        "id": "asWsWzemd5dk",
        "outputId": "5bfa4cdf-d304-46d8-adae-99c5dc6f4ab5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n",
            "Using cuda device\n",
            "------------------------------------\n",
            "| time/              |             |\n",
            "|    episodes        | 4           |\n",
            "|    fps             | 129         |\n",
            "|    time_elapsed    | 10          |\n",
            "|    total_timesteps | 1340        |\n",
            "| train/             |             |\n",
            "|    actor_loss      | -71         |\n",
            "|    critic_loss     | 2.97e+03    |\n",
            "|    learning_rate   | 0.001       |\n",
            "|    n_updates       | 1005        |\n",
            "|    reward          | -0.60931635 |\n",
            "------------------------------------\n",
            "day: 334, episode: 160\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 1143478.33\n",
            "total_reward: 143478.33\n",
            "total_cost: 999.00\n",
            "total_trades: 4008\n",
            "Sharpe: 0.458\n",
            "=================================\n",
            "------------------------------------\n",
            "| time/              |             |\n",
            "|    episodes        | 8           |\n",
            "|    fps             | 119         |\n",
            "|    time_elapsed    | 22          |\n",
            "|    total_timesteps | 2680        |\n",
            "| train/             |             |\n",
            "|    actor_loss      | -15.4       |\n",
            "|    critic_loss     | 2.79e+03    |\n",
            "|    learning_rate   | 0.001       |\n",
            "|    n_updates       | 2345        |\n",
            "|    reward          | -0.60931635 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| time/              |             |\n",
            "|    episodes        | 12          |\n",
            "|    fps             | 115         |\n",
            "|    time_elapsed    | 34          |\n",
            "|    total_timesteps | 4020        |\n",
            "| train/             |             |\n",
            "|    actor_loss      | -7.04       |\n",
            "|    critic_loss     | 44.5        |\n",
            "|    learning_rate   | 0.001       |\n",
            "|    n_updates       | 3685        |\n",
            "|    reward          | -0.60931635 |\n",
            "------------------------------------\n",
            "day: 334, episode: 170\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 1143478.33\n",
            "total_reward: 143478.33\n",
            "total_cost: 999.00\n",
            "total_trades: 4008\n",
            "Sharpe: 0.458\n",
            "=================================\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-81-3b1b0264cc27>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#Default config.ddpg_PARAMS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mnon_tuned_model_ddpg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ddpg\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmodel_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDDPG_PARAMS\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m trained_ddpg = agent.train_model(model=non_tuned_model_ddpg, \n\u001b[0m\u001b[1;32m      5\u001b[0m                              \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ddpg'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m                              total_timesteps=50000)\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(self, model, tb_log_name, total_timesteps)\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m         model = model.learn(\n\u001b[0m\u001b[1;32m    104\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     ) -> SelfDDPG:\n\u001b[0;32m--> 123\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    124\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m     ) -> SelfTD3:\n\u001b[0;32m--> 216\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    217\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    328\u001b[0m                 \u001b[0;31m# Special case when the user passes `gradient_steps=0`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    329\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgradient_steps\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 330\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgradient_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    331\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    332\u001b[0m         \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_training_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, gradient_steps, batch_size)\u001b[0m\n\u001b[1;32m    192\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m                 \u001b[0mactor_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 194\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m                 \u001b[0mpolyak_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_target\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    278\u001b[0m                                                f\"but got {result}.\")\n\u001b[1;32m    279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m                 \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer_step_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36m_use_grad\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'differentiable'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprev_grad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    139\u001b[0m                 state_steps)\n\u001b[1;32m    140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m             adam(\n\u001b[0m\u001b[1;32m    142\u001b[0m                 \u001b[0mparams_with_grad\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m                 \u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, foreach, capturable, differentiable, fused, grad_scale, found_inf, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize)\u001b[0m\n\u001b[1;32m    279\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_single_tensor_adam\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m     func(params,\n\u001b[0m\u001b[1;32m    282\u001b[0m          \u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m          \u001b[0mexp_avgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36m_multi_tensor_adam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, grad_scale, found_inf, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize, capturable, differentiable)\u001b[0m\n\u001b[1;32m    444\u001b[0m         \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    445\u001b[0m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_foreach_mul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_exp_avgs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 446\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_foreach_add_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_exp_avgs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice_grads\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    447\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_foreach_mul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_exp_avg_sqs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
        "    model=trained_ddpg, \n",
        "    environment = e_trade_gym)"
      ],
      "metadata": {
        "id": "mv1zgwOgd9FY",
        "outputId": "3fd9a94a-98d4-41c0-8870-74b8271b6494",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-35-38368194789e>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m df_account_value, df_actions = DRLAgent.DRL_prediction(\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrained_ddpg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     environment = e_trade_gym)\n",
            "\u001b[0;31mNameError\u001b[0m: name 'trained_ddpg' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Backtesting for not tuned hyperparamters\n",
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all = backtest_stats(account_value=df_account_value)\n",
        "perf_stats_all = pd.DataFrame(perf_stats_all)\n",
        "# perf_stats_all.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_\"+now+'.csv')"
      ],
      "metadata": {
        "id": "DuZL52qDeEz6",
        "outputId": "f4e9cd19-27cd-4dba-956e-353df0dabffa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return          0.247826\n",
            "Cumulative returns     0.368416\n",
            "Annual volatility      0.167226\n",
            "Sharpe ratio           1.412093\n",
            "Calmar ratio           2.531845\n",
            "Stability              0.897149\n",
            "Max drawdown          -0.097883\n",
            "Omega ratio            1.280406\n",
            "Sortino ratio          2.010344\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.033938\n",
            "Daily value at risk   -0.020131\n",
            "dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#You can see with trial, our sharpe ratio is increasing\n",
        "#Certainly you can afford more number of trials for further optimization\n",
        "from optuna.visualization import plot_optimization_history\n",
        "plot_optimization_history(study)"
      ],
      "metadata": {
        "id": "5aSsEmpZeI5C",
        "outputId": "4e4acb29-dc3a-4b5b-fabe-736db19aea9d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.20.0.min.js\"></script>                <div id=\"6afac010-8c60-495c-b9f7-1749179d9521\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"6afac010-8c60-495c-b9f7-1749179d9521\")) {                    Plotly.newPlot(                        \"6afac010-8c60-495c-b9f7-1749179d9521\",                        [{\"mode\":\"markers\",\"name\":\"Objective Value\",\"x\":[0,1,2,3,4],\"y\":[25.0,5.458908059384231,9.181461931558035,26.496666465769017,7.580685200661261],\"type\":\"scatter\"},{\"name\":\"Best Value\",\"x\":[0,1,2,3,4],\"y\":[25.0,25.0,25.0,26.496666465769017,26.496666465769017],\"type\":\"scatter\"}],                        {\"title\":{\"text\":\"Optimization History Plot\"},\"xaxis\":{\"title\":{\"text\":\"Trial\"}},\"yaxis\":{\"title\":{\"text\":\"Objective Value\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('6afac010-8c60-495c-b9f7-1749179d9521');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from optuna.visualization import plot_contour\n",
        "from optuna.visualization import plot_edf\n",
        "from optuna.visualization import plot_intermediate_values\n",
        "from optuna.visualization import plot_optimization_history\n",
        "from optuna.visualization import plot_parallel_coordinate\n",
        "from optuna.visualization import plot_param_importances\n",
        "from optuna.visualization import plot_slice"
      ],
      "metadata": {
        "id": "K9zcey_BeJMe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Hyperparamters importance\n",
        "#Ent_coef is the most important\n",
        "plot_param_importances(study)"
      ],
      "metadata": {
        "id": "J6WmB5DneOgh",
        "outputId": "9510c118-7601-45ac-adba-0226c3c76277",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.20.0.min.js\"></script>                <div id=\"79c56b7b-1dac-42d7-a7e1-58bd16ed9c0e\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"79c56b7b-1dac-42d7-a7e1-58bd16ed9c0e\")) {                    Plotly.newPlot(                        \"79c56b7b-1dac-42d7-a7e1-58bd16ed9c0e\",                        [{\"cliponaxis\":false,\"hovertemplate\":[\"gamma (CategoricalDistribution): 0.051402280785507216<extra></extra>\",\"noise_std (CategoricalDistribution): 0.10120568535574027<extra></extra>\",\"noise_type (CategoricalDistribution): 0.16624172227685763<extra></extra>\",\"net_arch (CategoricalDistribution): 0.20489031403705613<extra></extra>\",\"train_freq (CategoricalDistribution): 0.22282841915371404<extra></extra>\",\"tau (CategoricalDistribution): 0.25343157839112473<extra></extra>\"],\"marker\":{\"color\":\"rgb(66,146,198)\"},\"orientation\":\"h\",\"text\":[\"0.05\",\"0.10\",\"0.17\",\"0.20\",\"0.22\",\"0.25\"],\"textposition\":\"outside\",\"x\":[0.051402280785507216,0.10120568535574027,0.16624172227685763,0.20489031403705613,0.22282841915371404,0.25343157839112473],\"y\":[\"gamma\",\"noise_std\",\"noise_type\",\"net_arch\",\"train_freq\",\"tau\"],\"type\":\"bar\"}],                        {\"showlegend\":false,\"title\":{\"text\":\"Hyperparameter Importances\"},\"xaxis\":{\"title\":{\"text\":\"Importance for Objective Value\"}},\"yaxis\":{\"title\":{\"text\":\"Hyperparameter\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('79c56b7b-1dac-42d7-a7e1-58bd16ed9c0e');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def main():\n",
        "    import warnings\n",
        "    warnings.filterwarnings(\"ignore\")\n",
        "    import pandas as pd\n",
        "    import numpy as np\n",
        "    import matplotlib\n",
        "    import matplotlib.pyplot as plt\n",
        "    # matplotlib.use('Agg')\n",
        "    import datetime\n",
        "\n",
        "\n",
        "    from finrl.config_tickers import DOW_30_TICKER\n",
        "    from finrl.meta.preprocessor.yahoodownloader import YahooDownloader\n",
        "    from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
        "    from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
        "    from finrl.agents.stablebaselines3.models import DRLAgent, DRLEnsembleAgent\n",
        "    from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "\n",
        "    from pprint import pprint\n",
        "\n",
        "    import sys\n",
        "    sys.path.append(\"../FinRL-Library\")\n",
        "\n",
        "    import itertools\n",
        "\n",
        "    import os\n",
        "    from finrl.main import check_and_make_directories\n",
        "    from finrl.config import (\n",
        "        DATA_SAVE_DIR,\n",
        "        TRAINED_MODEL_DIR,\n",
        "        TENSORBOARD_LOG_DIR,\n",
        "        RESULTS_DIR,\n",
        "        INDICATORS,\n",
        "        TRAIN_START_DATE,\n",
        "        TRAIN_END_DATE,\n",
        "        TEST_START_DATE,\n",
        "        TEST_END_DATE,\n",
        "        TRADE_START_DATE,\n",
        "        TRADE_END_DATE,\n",
        "    )\n",
        "\n",
        "    check_and_make_directories([DATA_SAVE_DIR, TRAINED_MODEL_DIR, TENSORBOARD_LOG_DIR, RESULTS_DIR])\n",
        "    print(DOW_30_TICKER)\n",
        "    TRAIN_START_DATE = '2009-04-01'\n",
        "    TRAIN_END_DATE = '2021-01-01'\n",
        "    TEST_START_DATE = '2021-01-01'\n",
        "    TEST_END_DATE = '2022-06-01'\n",
        "\n",
        "    df = YahooDownloader(start_date=TRAIN_START_DATE,\n",
        "                         end_date=TEST_END_DATE,\n",
        "                         ticker_list=DOW_30_TICKER).fetch_data()\n",
        "\n",
        "    df.sort_values(['date', 'tic']).head()\n",
        "\n",
        "    fe = FeatureEngineer(use_technical_indicator=True,\n",
        "                         tech_indicator_list=INDICATORS,\n",
        "                         use_turbulence=True,\n",
        "                         user_defined_feature=False)\n",
        "\n",
        "    processed = fe.preprocess_data(df)\n",
        "    processed = processed.copy()\n",
        "    processed = processed.fillna(0)\n",
        "    processed = processed.replace(np.inf, 0)\n",
        "\n",
        "    stock_dimension = len(processed.tic.unique())\n",
        "    state_space = 1 + 2 * stock_dimension + len(INDICATORS) * stock_dimension\n",
        "    print(f\"Stock Dimension: {stock_dimension}, State Space: {state_space}\")\n",
        "\n",
        "    env_kwargs = {\n",
        "        \"hmax\": 100,\n",
        "        \"initial_amount\": 1000000,\n",
        "        \"buy_cost_pct\": 0.001,\n",
        "        \"sell_cost_pct\": 0.001,\n",
        "        \"state_space\": state_space,\n",
        "        \"stock_dim\": stock_dimension,\n",
        "        \"tech_indicator_list\": INDICATORS,\n",
        "        \"action_space\": stock_dimension,\n",
        "        \"reward_scaling\": 1e-4,\n",
        "        \"print_verbosity\": 5\n",
        "\n",
        "    }\n",
        "\n",
        "    rebalance_window = 63  # rebalance_window is the number of days to retrain the model\n",
        "    validation_window = 63  # validation_window is the number of days to do validation and trading (e.g. if validation_window=63, then both validation and trading period will be 63 days)\n",
        "\n",
        "    ensemble_agent = DRLEnsembleAgent(df=processed,\n",
        "                                      train_period=(TRAIN_START_DATE, TRAIN_END_DATE),\n",
        "                                      val_test_period=(TEST_START_DATE, TEST_END_DATE),\n",
        "                                      rebalance_window=rebalance_window,\n",
        "                                      validation_window=validation_window,\n",
        "                                      **env_kwargs)\n",
        "\n",
        "    A2C_model_kwargs = {\n",
        "        'n_steps': 5,\n",
        "        'ent_coef': 0.005,\n",
        "        'learning_rate': 0.0007\n",
        "    }\n",
        "\n",
        "    PPO_model_kwargs = {\n",
        "        \"ent_coef\": 0.01,\n",
        "        \"n_steps\": 2048,\n",
        "        \"learning_rate\": 0.00025,\n",
        "        \"batch_size\": 128\n",
        "    }\n",
        "\n",
        "    DDPG_model_kwargs = {\n",
        "        # \"action_noise\":\"ornstein_uhlenbeck\",\n",
        "        \"buffer_size\": 10_000,\n",
        "        \"learning_rate\": 0.0005,\n",
        "        \"batch_size\": 64\n",
        "    }\n",
        "\n",
        "    timesteps_dict = {'a2c': 10_000,\n",
        "                      'ppo': 10_000,\n",
        "                      'ddpg': 10_000\n",
        "                      }\n",
        "    df_summary = ensemble_agent.run_ensemble_strategy(A2C_model_kwargs,\n",
        "                                                      PPO_model_kwargs,\n",
        "                                                      DDPG_model_kwargs,\n",
        "                                                      timesteps_dict)\n",
        "\n",
        "    unique_trade_date = processed[(processed.date > TEST_START_DATE) & (processed.date <= TEST_END_DATE)].date.unique()\n",
        "\n",
        "    df_trade_date = pd.DataFrame({'datadate': unique_trade_date})\n",
        "\n",
        "    df_account_value = pd.DataFrame()\n",
        "    for i in range(rebalance_window + validation_window, len(unique_trade_date) + 1, rebalance_window):\n",
        "        temp = pd.read_csv('results/account_value_trade_{}_{}.csv'.format('ensemble', i))\n",
        "        df_account_value = df_account_value.append(temp, ignore_index=True)\n",
        "    sharpe = (252 ** 0.5) * df_account_value.account_value.pct_change(\n",
        "        1).mean() / df_account_value.account_value.pct_change(1).std()\n",
        "    print('Sharpe Ratio: ', sharpe)\n",
        "    df_account_value = df_account_value.join(df_trade_date[validation_window:].reset_index(drop=True))\n",
        "\n",
        "    df_account_value.account_value.plot()\n",
        "\n",
        "    print(\"==============Get Backtest Results===========\")\n",
        "    now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "    perf_stats_all = backtest_stats(account_value=df_account_value)\n",
        "    perf_stats_all = pd.DataFrame(perf_stats_all)\n",
        "\n",
        "    # baseline stats\n",
        "    print(\"==============Get Baseline Stats===========\")\n",
        "    baseline_df = get_baseline(\n",
        "        ticker=\"^DJI\",\n",
        "        start=df_account_value.loc[0, 'date'],\n",
        "        end=df_account_value.loc[len(df_account_value) - 1, 'date'])\n",
        "\n",
        "    stats = backtest_stats(baseline_df, value_col_name='close')\n",
        "\n",
        "    print(\"==============Compare to DJIA===========\")\n",
        "\n",
        "    # S&P 500: ^GSPC\n",
        "    # Dow Jones Index: ^DJI\n",
        "    # NASDAQ 100: ^NDX\n",
        "    backtest_plot(df_account_value,\n",
        "                  baseline_ticker='^DJI',\n",
        "                  baseline_start=df_account_value.loc[0, 'date'],\n",
        "                  baseline_end=df_account_value.loc[len(df_account_value) - 1, 'date'])\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    main()"
      ],
      "metadata": {
        "id": "s_xcVpBSeOyE",
        "outputId": "52b3d82a-1f8d-4af4-a405-6b5b9778781d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['AXP', 'AMGN', 'AAPL', 'BA', 'CAT', 'CSCO', 'CVX', 'GS', 'HD', 'HON', 'IBM', 'INTC', 'JNJ', 'KO', 'JPM', 'MCD', 'MMM', 'MRK', 'MSFT', 'NKE', 'PG', 'TRV', 'UNH', 'CRM', 'VZ', 'V', 'WBA', 'WMT', 'DIS', 'DOW']\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (96942, 8)\n",
            "Successfully added technical indicators\n",
            "Successfully added turbulence index\n",
            "Stock Dimension: 29, State Space: 291\n",
            "============Start Ensemble Strategy============\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-01-04\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_126_1\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 100        |\n",
            "|    time_elapsed       | 4          |\n",
            "|    total_timesteps    | 500        |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | 0.00295    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 99         |\n",
            "|    policy_loss        | -71.2      |\n",
            "|    reward             | 0.78650117 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 12.2       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 200        |\n",
            "|    time_elapsed       | 9          |\n",
            "|    total_timesteps    | 1000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | 0.00111    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 199        |\n",
            "|    policy_loss        | 6.59       |\n",
            "|    reward             | 0.66569686 |\n",
            "|    std                | 0.998      |\n",
            "|    value_loss         | 1.03       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 102       |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 14        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.1     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -123      |\n",
            "|    reward             | 2.9206326 |\n",
            "|    std                | 0.998     |\n",
            "|    value_loss         | 15.8      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 102        |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 19         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | -6.35      |\n",
            "|    reward             | -1.8242201 |\n",
            "|    std                | 0.999      |\n",
            "|    value_loss         | 3.27       |\n",
            "--------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                 |               |\n",
            "|    fps                | 102           |\n",
            "|    iterations         | 500           |\n",
            "|    time_elapsed       | 24            |\n",
            "|    total_timesteps    | 2500          |\n",
            "| train/                |               |\n",
            "|    entropy_loss       | -41.2         |\n",
            "|    explained_variance | -1.19e-07     |\n",
            "|    learning_rate      | 0.0007        |\n",
            "|    n_updates          | 499           |\n",
            "|    policy_loss        | 16            |\n",
            "|    reward             | -0.0061751665 |\n",
            "|    std                | 1             |\n",
            "|    value_loss         | 0.657         |\n",
            "-----------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 102       |\n",
            "|    iterations         | 600       |\n",
            "|    time_elapsed       | 29        |\n",
            "|    total_timesteps    | 3000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 599       |\n",
            "|    policy_loss        | -33.6     |\n",
            "|    reward             | 0.5337091 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.837     |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 102        |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 34         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -1.97      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -76.4      |\n",
            "|    reward             | -1.3246601 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 3.85       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 800        |\n",
            "|    time_elapsed       | 39         |\n",
            "|    total_timesteps    | 4000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.0632     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 799        |\n",
            "|    policy_loss        | 44.6       |\n",
            "|    reward             | -1.6959082 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.53       |\n",
            "--------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 900      |\n",
            "|    time_elapsed       | 44       |\n",
            "|    total_timesteps    | 4500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41.2    |\n",
            "|    explained_variance | 0.0211   |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 899      |\n",
            "|    policy_loss        | -33.2    |\n",
            "|    reward             | 3.994608 |\n",
            "|    std                | 1        |\n",
            "|    value_loss         | 3.53     |\n",
            "------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1000        |\n",
            "|    time_elapsed       | 49          |\n",
            "|    total_timesteps    | 5000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | 1.79e-07    |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 999         |\n",
            "|    policy_loss        | 36.3        |\n",
            "|    reward             | -0.20298508 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 1.68        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1100        |\n",
            "|    time_elapsed       | 54          |\n",
            "|    total_timesteps    | 5500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | 0.0114      |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1099        |\n",
            "|    policy_loss        | 393         |\n",
            "|    reward             | -0.92543656 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 120         |\n",
            "---------------------------------------\n",
            "----------------------------------------\n",
            "| time/                 |              |\n",
            "|    fps                | 101          |\n",
            "|    iterations         | 1200         |\n",
            "|    time_elapsed       | 59           |\n",
            "|    total_timesteps    | 6000         |\n",
            "| train/                |              |\n",
            "|    entropy_loss       | -41.2        |\n",
            "|    explained_variance | 0.0236       |\n",
            "|    learning_rate      | 0.0007       |\n",
            "|    n_updates          | 1199         |\n",
            "|    policy_loss        | 151          |\n",
            "|    reward             | -0.099366836 |\n",
            "|    std                | 1            |\n",
            "|    value_loss         | 19.4         |\n",
            "----------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1300        |\n",
            "|    time_elapsed       | 64          |\n",
            "|    total_timesteps    | 6500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | -0.000106   |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1299        |\n",
            "|    policy_loss        | -47.6       |\n",
            "|    reward             | -0.39146426 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 3.15        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1400        |\n",
            "|    time_elapsed       | 69          |\n",
            "|    total_timesteps    | 7000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1399        |\n",
            "|    policy_loss        | 31.8        |\n",
            "|    reward             | -0.25985184 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 1.84        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1500      |\n",
            "|    time_elapsed       | 74        |\n",
            "|    total_timesteps    | 7500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1499      |\n",
            "|    policy_loss        | -30.8     |\n",
            "|    reward             | 3.1766434 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 4.93      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 1600       |\n",
            "|    time_elapsed       | 78         |\n",
            "|    total_timesteps    | 8000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.00673    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1599       |\n",
            "|    policy_loss        | 133        |\n",
            "|    reward             | -2.0720153 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 10.9       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 83        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 5.96e-08  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -84.2     |\n",
            "|    reward             | 3.8356218 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 7         |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 1800       |\n",
            "|    time_elapsed       | 88         |\n",
            "|    total_timesteps    | 9000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.063      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1799       |\n",
            "|    policy_loss        | 42         |\n",
            "|    reward             | 0.97460705 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 2.23       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1900      |\n",
            "|    time_elapsed       | 93        |\n",
            "|    total_timesteps    | 9500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0.0122    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1899      |\n",
            "|    policy_loss        | -84.7     |\n",
            "|    reward             | 2.2624621 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 8.34      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 98         |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | 45.9       |\n",
            "|    reward             | -1.0189797 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.77       |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-01-04 to  2021-04-06\n",
            "A2C Sharpe Ratio:  0.26168708794170054\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_126_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 111       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 18        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.8877938 |\n",
            "----------------------------------\n",
            "day: 2959, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3881161.61\n",
            "total_reward: 2881161.61\n",
            "total_cost: 376867.37\n",
            "total_trades: 82873\n",
            "Sharpe: 0.776\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 109         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 37          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016203282 |\n",
            "|    clip_fraction        | 0.192       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0104     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4           |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0191     |\n",
            "|    reward               | 1.0358981   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 11.3        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 109         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 56          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012528071 |\n",
            "|    clip_fraction        | 0.126       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00217    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 57          |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0176     |\n",
            "|    reward               | 0.07821791  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 55.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 107         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 76          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012139827 |\n",
            "|    clip_fraction        | 0.146       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00454    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 13.7        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.018      |\n",
            "|    reward               | -1.5925834  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 43.4        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 107         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 94          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019309906 |\n",
            "|    clip_fraction        | 0.17        |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | -0.0161     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.07        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0225     |\n",
            "|    reward               | 0.0615912   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 11.3        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-01-04 to  2021-04-06\n",
            "PPO Sharpe Ratio:  0.3016310407987196\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_126_1\n",
            "day: 2959, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4424085.63\n",
            "total_reward: 3424085.63\n",
            "total_cost: 1037.34\n",
            "total_trades: 50241\n",
            "Sharpe: 0.797\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 78        |\n",
            "|    time_elapsed    | 151       |\n",
            "|    total_timesteps | 11840     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | -60.3     |\n",
            "|    critic_loss     | 441       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 8880      |\n",
            "|    reward          | 3.0541244 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-01-04 to  2021-04-06\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-04-06\n",
            "======Trading from:  2021-04-06 to  2021-07-06\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-04-06\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_189_1\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 99         |\n",
            "|    iterations         | 100        |\n",
            "|    time_elapsed       | 5          |\n",
            "|    total_timesteps    | 500        |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.446     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 99         |\n",
            "|    policy_loss        | -85.9      |\n",
            "|    reward             | -0.6058027 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 9.16       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 9         |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | -19.4     |\n",
            "|    reward             | 1.8647146 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 0.253     |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 300      |\n",
            "|    time_elapsed       | 14       |\n",
            "|    total_timesteps    | 1500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41      |\n",
            "|    explained_variance | 0.0122   |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 299      |\n",
            "|    policy_loss        | -179     |\n",
            "|    reward             | 3.291392 |\n",
            "|    std                | 0.996    |\n",
            "|    value_loss         | 22.1     |\n",
            "------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 400       |\n",
            "|    time_elapsed       | 19        |\n",
            "|    total_timesteps    | 2000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0.0507    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 399       |\n",
            "|    policy_loss        | -11.1     |\n",
            "|    reward             | 0.3743614 |\n",
            "|    std                | 0.996     |\n",
            "|    value_loss         | 1.09      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 500        |\n",
            "|    time_elapsed       | 24         |\n",
            "|    total_timesteps    | 2500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 499        |\n",
            "|    policy_loss        | 169        |\n",
            "|    reward             | -0.3291204 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 16         |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 29         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0.054      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | 905        |\n",
            "|    reward             | -1.0711567 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 521        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 34         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.0433    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -32.9      |\n",
            "|    reward             | -2.9666545 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 0.946      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 800        |\n",
            "|    time_elapsed       | 39         |\n",
            "|    total_timesteps    | 4000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 799        |\n",
            "|    policy_loss        | -3.25      |\n",
            "|    reward             | -0.5585839 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 1.05       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 900       |\n",
            "|    time_elapsed       | 44        |\n",
            "|    total_timesteps    | 4500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | -0.66     |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 899       |\n",
            "|    policy_loss        | 306       |\n",
            "|    reward             | 2.3080251 |\n",
            "|    std                | 0.992     |\n",
            "|    value_loss         | 70.3      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1000      |\n",
            "|    time_elapsed       | 49        |\n",
            "|    total_timesteps    | 5000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 999       |\n",
            "|    policy_loss        | -108      |\n",
            "|    reward             | 1.1761196 |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 10.8      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1100      |\n",
            "|    time_elapsed       | 54        |\n",
            "|    total_timesteps    | 5500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1099      |\n",
            "|    policy_loss        | -117      |\n",
            "|    reward             | 3.2501192 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 10.7      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1200      |\n",
            "|    time_elapsed       | 59        |\n",
            "|    total_timesteps    | 6000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1199      |\n",
            "|    policy_loss        | 362       |\n",
            "|    reward             | 6.932239  |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 84.6      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 64         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | 44.8       |\n",
            "|    reward             | -1.5383602 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 2.08       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1400      |\n",
            "|    time_elapsed       | 69        |\n",
            "|    total_timesteps    | 7000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1399      |\n",
            "|    policy_loss        | -143      |\n",
            "|    reward             | 0.7461217 |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 17        |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 74         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | -145       |\n",
            "|    reward             | -1.6722031 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 14.8       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 79        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 1.19e-07  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | 4.35      |\n",
            "|    reward             | 3.1992574 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 0.916     |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 84        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | -0.00566  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -1.16e+03 |\n",
            "|    reward             | 11.774156 |\n",
            "|    std                | 0.993     |\n",
            "|    value_loss         | 936       |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 1800     |\n",
            "|    time_elapsed       | 89       |\n",
            "|    total_timesteps    | 9000     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -40.9    |\n",
            "|    explained_variance | -0.00464 |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1799     |\n",
            "|    policy_loss        | -185     |\n",
            "|    reward             | 4.000574 |\n",
            "|    std                | 0.993    |\n",
            "|    value_loss         | 41.6     |\n",
            "------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1900       |\n",
            "|    time_elapsed       | 94         |\n",
            "|    total_timesteps    | 9500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -40.9      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1899       |\n",
            "|    policy_loss        | 12.7       |\n",
            "|    reward             | -1.3208354 |\n",
            "|    std                | 0.991      |\n",
            "|    value_loss         | 0.306      |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 2000      |\n",
            "|    time_elapsed       | 99        |\n",
            "|    total_timesteps    | 10000     |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1999      |\n",
            "|    policy_loss        | -10.6     |\n",
            "|    reward             | 0.7043803 |\n",
            "|    std                | 0.992     |\n",
            "|    value_loss         | 0.431     |\n",
            "-------------------------------------\n",
            "======A2C Validation from:  2021-04-06 to  2021-07-06\n",
            "A2C Sharpe Ratio:  0.23121777505990648\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_189_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 104       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 19        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.1126131 |\n",
            "----------------------------------\n",
            "day: 3022, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3636477.12\n",
            "total_reward: 2636477.12\n",
            "total_cost: 386575.15\n",
            "total_trades: 84625\n",
            "Sharpe: 0.731\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 38          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015871149 |\n",
            "|    clip_fraction        | 0.234       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0129     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 3.2         |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0282     |\n",
            "|    reward               | 0.88015693  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 10.1        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 58          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019049045 |\n",
            "|    clip_fraction        | 0.166       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | 0.00262     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 32.1        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.023      |\n",
            "|    reward               | 0.49640504  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 53.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 77          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015561214 |\n",
            "|    clip_fraction        | 0.161       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0133     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 31.9        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.0201     |\n",
            "|    reward               | -0.7624933  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 45.6        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 96          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014756277 |\n",
            "|    clip_fraction        | 0.141       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00517    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 6.56        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0204     |\n",
            "|    reward               | 1.2296445   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 14.8        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-04-06 to  2021-07-06\n",
            "PPO Sharpe Ratio:  0.010011405217654536\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_189_1\n",
            "day: 3022, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 5009064.34\n",
            "total_reward: 4009064.34\n",
            "total_cost: 1282.47\n",
            "total_trades: 51375\n",
            "Sharpe: 0.880\n",
            "=================================\n",
            "---------------------------------\n",
            "| time/              |          |\n",
            "|    episodes        | 4        |\n",
            "|    fps             | 76       |\n",
            "|    time_elapsed    | 157      |\n",
            "|    total_timesteps | 12092    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -65.5    |\n",
            "|    critic_loss     | 3.14e+03 |\n",
            "|    learning_rate   | 0.0005   |\n",
            "|    n_updates       | 9069     |\n",
            "|    reward          | 8.301459 |\n",
            "---------------------------------\n",
            "======DDPG Validation from:  2021-04-06 to  2021-07-06\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-07-06\n",
            "======Trading from:  2021-07-06 to  2021-10-04\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-07-06\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_252_1\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 100         |\n",
            "|    time_elapsed       | 5           |\n",
            "|    total_timesteps    | 500         |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | -0.0351     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 99          |\n",
            "|    policy_loss        | -28.6       |\n",
            "|    reward             | -0.15436654 |\n",
            "|    std                | 0.998       |\n",
            "|    value_loss         | 5.15        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 97        |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 10        |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | 50.5      |\n",
            "|    reward             | 1.3676119 |\n",
            "|    std                | 0.996     |\n",
            "|    value_loss         | 2.12      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 15        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -179      |\n",
            "|    reward             | 5.1952586 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 22.2      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 20         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | 79.1       |\n",
            "|    reward             | -0.3978011 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 7.82       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 500        |\n",
            "|    time_elapsed       | 25         |\n",
            "|    total_timesteps    | 2500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -40.9      |\n",
            "|    explained_variance | 0.19       |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 499        |\n",
            "|    policy_loss        | -8.81      |\n",
            "|    reward             | 0.19138733 |\n",
            "|    std                | 0.993      |\n",
            "|    value_loss         | 6.38       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 30         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.00925   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | -498       |\n",
            "|    reward             | -11.956305 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 222        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 35         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.169     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -68.7      |\n",
            "|    reward             | 0.39965674 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 3.86       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 800         |\n",
            "|    time_elapsed       | 40          |\n",
            "|    total_timesteps    | 4000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41         |\n",
            "|    explained_variance | -0.0865     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 799         |\n",
            "|    policy_loss        | -164        |\n",
            "|    reward             | -0.33571205 |\n",
            "|    std                | 0.996       |\n",
            "|    value_loss         | 16.1        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 900         |\n",
            "|    time_elapsed       | 45          |\n",
            "|    total_timesteps    | 4500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41         |\n",
            "|    explained_variance | 0.159       |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 899         |\n",
            "|    policy_loss        | 46.5        |\n",
            "|    reward             | -0.19355251 |\n",
            "|    std                | 0.996       |\n",
            "|    value_loss         | 1.99        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1000        |\n",
            "|    time_elapsed       | 50          |\n",
            "|    total_timesteps    | 5000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | 0.0618      |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 999         |\n",
            "|    policy_loss        | -15.1       |\n",
            "|    reward             | -0.48996487 |\n",
            "|    std                | 0.997       |\n",
            "|    value_loss         | 0.677       |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1100      |\n",
            "|    time_elapsed       | 55        |\n",
            "|    total_timesteps    | 5500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | -0.192    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1099      |\n",
            "|    policy_loss        | -233      |\n",
            "|    reward             | 1.1524396 |\n",
            "|    std                | 0.997     |\n",
            "|    value_loss         | 35.2      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1200      |\n",
            "|    time_elapsed       | 60        |\n",
            "|    total_timesteps    | 6000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.1     |\n",
            "|    explained_variance | -0.214    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1199      |\n",
            "|    policy_loss        | 14.9      |\n",
            "|    reward             | -2.838138 |\n",
            "|    std                | 0.998     |\n",
            "|    value_loss         | 3.95      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 65         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.167      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | 43.7       |\n",
            "|    reward             | -1.0078373 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.86       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1400      |\n",
            "|    time_elapsed       | 70        |\n",
            "|    total_timesteps    | 7000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -0.0727   |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1399      |\n",
            "|    policy_loss        | -17.9     |\n",
            "|    reward             | 2.1590974 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 1.71      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 75         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | 42.5       |\n",
            "|    reward             | 0.47163484 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.17       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 80        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0.000262  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | -73.2     |\n",
            "|    reward             | 0.8027305 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 19.5      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 85        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -184      |\n",
            "|    reward             | 0.3686392 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 29.3      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1800      |\n",
            "|    time_elapsed       | 90        |\n",
            "|    total_timesteps    | 9000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1799      |\n",
            "|    policy_loss        | 116       |\n",
            "|    reward             | 1.6162095 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 21.6      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1900      |\n",
            "|    time_elapsed       | 96        |\n",
            "|    total_timesteps    | 9500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -0.00951  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1899      |\n",
            "|    policy_loss        | -1.74     |\n",
            "|    reward             | 1.1433389 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.195     |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 101        |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -0.0113    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | -38.9      |\n",
            "|    reward             | 0.97623837 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 0.806      |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-07-06 to  2021-10-04\n",
            "A2C Sharpe Ratio:  -0.0528373614471615\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_252_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 101       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 20        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.2801732 |\n",
            "----------------------------------\n",
            "day: 3085, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4550937.53\n",
            "total_reward: 3550937.53\n",
            "total_cost: 408098.28\n",
            "total_trades: 86397\n",
            "Sharpe: 0.873\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 39          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.018978704 |\n",
            "|    clip_fraction        | 0.233       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00994    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.48        |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0284     |\n",
            "|    reward               | 0.49718028  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 10.8        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 103         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 59          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012346724 |\n",
            "|    clip_fraction        | 0.155       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00278    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 19.6        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0154     |\n",
            "|    reward               | -0.8982904  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 50.6        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 103         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 78          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014507987 |\n",
            "|    clip_fraction        | 0.123       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.012      |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 43.8        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.016      |\n",
            "|    reward               | -0.0212465  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 74.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 104         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 98          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016703494 |\n",
            "|    clip_fraction        | 0.186       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | -0.0496     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4.09        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0285     |\n",
            "|    reward               | 1.0109037   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 14.2        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-07-06 to  2021-10-04\n",
            "PPO Sharpe Ratio:  -0.05262682094979255\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_252_1\n",
            "day: 3085, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4501430.61\n",
            "total_reward: 3501430.61\n",
            "total_cost: 1082.97\n",
            "total_trades: 61741\n",
            "Sharpe: 0.737\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 76        |\n",
            "|    time_elapsed    | 161       |\n",
            "|    total_timesteps | 12344     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | -3.66     |\n",
            "|    critic_loss     | 176       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 9258      |\n",
            "|    reward          | 2.1645079 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-07-06 to  2021-10-04\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-10-04\n",
            "======Trading from:  2021-10-04 to  2022-01-03\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-10-04\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_315_1\n",
            "----------------------------------------\n",
            "| time/                 |              |\n",
            "|    fps                | 98           |\n",
            "|    iterations         | 100          |\n",
            "|    time_elapsed       | 5            |\n",
            "|    total_timesteps    | 500          |\n",
            "| train/                |              |\n",
            "|    entropy_loss       | -41.3        |\n",
            "|    explained_variance | 0            |\n",
            "|    learning_rate      | 0.0007       |\n",
            "|    n_updates          | 99           |\n",
            "|    policy_loss        | -51.2        |\n",
            "|    reward             | -0.096114956 |\n",
            "|    std                | 1.01         |\n",
            "|    value_loss         | 4.56         |\n",
            "----------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 10        |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 5.96e-08  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | -36.9     |\n",
            "|    reward             | 2.9646184 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.756     |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 97        |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 15        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -236      |\n",
            "|    reward             | 1.1201329 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 36.4      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 20         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | -53.9      |\n",
            "|    reward             | 0.03644575 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 2.41       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 500       |\n",
            "|    time_elapsed       | 25        |\n",
            "|    total_timesteps    | 2500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 499       |\n",
            "|    policy_loss        | 34.1      |\n",
            "|    reward             | 0.3948357 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 1.08      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 30         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.3      |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | -618       |\n",
            "|    reward             | -12.502208 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 338        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 35         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -0.0164    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -105       |\n",
            "|    reward             | 0.30092302 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 10.8       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 800         |\n",
            "|    time_elapsed       | 40          |\n",
            "|    total_timesteps    | 4000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.3       |\n",
            "|    explained_variance | 1.19e-07    |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 799         |\n",
            "|    policy_loss        | 74.6        |\n",
            "|    reward             | 0.009111182 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 4.73        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 900       |\n",
            "|    time_elapsed       | 45        |\n",
            "|    total_timesteps    | 4500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | -0.0124   |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 899       |\n",
            "|    policy_loss        | -131      |\n",
            "|    reward             | 2.0478728 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 16.5      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1000       |\n",
            "|    time_elapsed       | 50         |\n",
            "|    total_timesteps    | 5000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.3      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 999        |\n",
            "|    policy_loss        | -34.9      |\n",
            "|    reward             | 0.04952446 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 2.24       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1100        |\n",
            "|    time_elapsed       | 55          |\n",
            "|    total_timesteps    | 5500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.3       |\n",
            "|    explained_variance | -0.0188     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1099        |\n",
            "|    policy_loss        | 344         |\n",
            "|    reward             | -0.11844914 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 64.7        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1200        |\n",
            "|    time_elapsed       | 60          |\n",
            "|    total_timesteps    | 6000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.4       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1199        |\n",
            "|    policy_loss        | 58.9        |\n",
            "|    reward             | -0.28185675 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 6.99        |\n",
            "---------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 66         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0.549      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | -11.7      |\n",
            "|    reward             | -0.7957832 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.135      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1400       |\n",
            "|    time_elapsed       | 71         |\n",
            "|    total_timesteps    | 7000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 1.19e-07   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1399       |\n",
            "|    policy_loss        | -29.7      |\n",
            "|    reward             | 0.16117185 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.859      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 76         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 1.79e-07   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | 15.7       |\n",
            "|    reward             | 0.17504984 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.528      |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 81        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.4     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | 48.7      |\n",
            "|    reward             | 3.1540215 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 6.42      |\n",
            "-------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1700        |\n",
            "|    time_elapsed       | 86          |\n",
            "|    total_timesteps    | 8500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.4       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1699        |\n",
            "|    policy_loss        | 22.9        |\n",
            "|    reward             | -0.09754947 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 0.677       |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1800      |\n",
            "|    time_elapsed       | 91        |\n",
            "|    total_timesteps    | 9000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.4     |\n",
            "|    explained_variance | -0.171    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1799      |\n",
            "|    policy_loss        | 99        |\n",
            "|    reward             | -1.175583 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 9.98      |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 98       |\n",
            "|    iterations         | 1900     |\n",
            "|    time_elapsed       | 96       |\n",
            "|    total_timesteps    | 9500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41.4    |\n",
            "|    explained_variance | 0        |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1899     |\n",
            "|    policy_loss        | -24.6    |\n",
            "|    reward             | 0.362812 |\n",
            "|    std                | 1.01     |\n",
            "|    value_loss         | 0.367    |\n",
            "------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 101        |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | -146       |\n",
            "|    reward             | -1.7740997 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 13.5       |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-10-04 to  2022-01-03\n",
            "A2C Sharpe Ratio:  0.3744625013790845\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_315_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 106       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 19        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 0.7561091 |\n",
            "----------------------------------\n",
            "day: 3148, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4259003.47\n",
            "total_reward: 3259003.47\n",
            "total_cost: 426068.70\n",
            "total_trades: 87901\n",
            "Sharpe: 0.790\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 101         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 40          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016951367 |\n",
            "|    clip_fraction        | 0.229       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.000697   |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.25        |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0307     |\n",
            "|    reward               | 0.9373541   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 9.91        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 60          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017283382 |\n",
            "|    clip_fraction        | 0.176       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | 0.00633     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 33.4        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0207     |\n",
            "|    reward               | 7.712371    |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 56.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 79          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010827523 |\n",
            "|    clip_fraction        | 0.0962      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | 0.00682     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 73.4        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.018      |\n",
            "|    reward               | -0.83698237 |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 134         |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 99          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.013603406 |\n",
            "|    clip_fraction        | 0.172       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.4       |\n",
            "|    explained_variance   | 0.0243      |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4.64        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0243     |\n",
            "|    reward               | 0.5904208   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 12.7        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-10-04 to  2022-01-03\n",
            "PPO Sharpe Ratio:  0.0593551480877188\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_315_1\n",
            "day: 3148, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4617783.42\n",
            "total_reward: 3617783.42\n",
            "total_cost: 1110.92\n",
            "total_trades: 49019\n",
            "Sharpe: 0.745\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 75        |\n",
            "|    time_elapsed    | 167       |\n",
            "|    total_timesteps | 12596     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | 36.1      |\n",
            "|    critic_loss     | 656       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 9447      |\n",
            "|    reward          | 5.4972343 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-10-04 to  2022-01-03\n",
            "======Best Model Retraining from:  2009-04-01 to  2022-01-03\n",
            "======Trading from:  2022-01-03 to  2022-04-04\n",
            "Ensemble Strategy took:  25.210217595100403  minutes\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-30-4153cda81a48>\u001b[0m in \u001b[0;36m<cell line: 163>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    162\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 164\u001b[0;31m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-30-4153cda81a48>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m    127\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrebalance_window\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mvalidation_window\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munique_trade_date\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrebalance_window\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m         \u001b[0mtemp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'results/account_value_trade_{}_{}.csv'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ensemble'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m         \u001b[0mdf_account_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf_account_value\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m     sharpe = (252 ** 0.5) * df_account_value.account_value.pct_change(\n\u001b[1;32m    131\u001b[0m         1).mean() / df_account_value.account_value.pct_change(1).std()\n",
            "\u001b[0;32m/usr/local/lib/python3.9/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   5987\u001b[0m         ):\n\u001b[1;32m   5988\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5989\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5990\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5991\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'append'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "current_price  = 110\n",
        "avg_price = 130\n",
        "print(( current_price * 0.3 ) + current_price)\n",
        "if current_price > ( avg_price * 0.3  + avg_price)  and avg_price > 0:\n",
        "  #actions[i] = self.total_stockss[i] * -1\n",
        "  #print(\"Updated actions\")\n",
        "  print(\"Current Price\", current_price , \" \", \" avg_price\", \" \", avg_price, \"Profit\", current_price -  avg_price)\n",
        "elif current_price < (avg_price - avg_price * 0.15)  and avg_price > 0:\n",
        "  #actions[i] = self.total_stockss[i] * -1\n",
        "  print(\"Current Price\", current_price , \" \", \" avg_price\", \" \", avg_price, \"loss\", current_price -  avg_price )"
      ],
      "metadata": {
        "id": "R6wjk6Qwg-ar",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "65204f2e-8f07-4ca7-8505-b1dfcad5f769"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "143.0\n",
            "Current Price 110    avg_price   130 loss -20\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "avg_price < ( current_price * 0.3  + current_price)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lZm_y4awCWO9",
        "outputId": "89222edb-bc50-4871-c67d-ee4300ed49db"
      },
      "execution_count": 153,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 153
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "current_price - current_price * 0.15"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wr3yRoNwCvV7",
        "outputId": "60d7ebbb-13fb-4b17-ece3-b008217d2987"
      },
      "execution_count": 163,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "93.5"
            ]
          },
          "metadata": {},
          "execution_count": 163
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JtmhGYiRC07B"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "Making the most of your colab subscription",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}