{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Train a PPO agent on Accenta's environment\n\nThis example shows how to train a PPO agent.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Import required packages\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from rlenv.envs.wall.core import AccentaEnv\n\nimport torch as th\nfrom stable_baselines3 import PPO\nfrom rlagent.data import ppo_trained_model_example_path"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Make the environment\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "env = AccentaEnv()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Make the agent's policy\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# Custom actor (pi) and value function (vf) networks\n# of two layers of size 32 each with Relu activation function\n# https://stable-baselines3.readthedocs.io/en/master/guide/custom_policy.html#custom-network-architecture\npolicy_kwargs = dict(activation_fn=th.nn.ReLU,\n                     net_arch=[dict(pi=[32, 32], vf=[32, 32])])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Make the agent\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "model = PPO(\"MlpPolicy\", env, policy_kwargs=policy_kwargs, verbose=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Train the agent\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "model.learn(total_timesteps=10000)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Save the trained agent\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "model.save(ppo_trained_model_example_path())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Reload the trained agent\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "del model\nmodel = PPO.load(ppo_trained_model_example_path(), env=env)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.15"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}