Merge pull request #1 from pythonlessons/develop

Develop

Author: pythonlessons

CHANGELOG.md

@@ -1,3 +1,17 @@
+## [0.4.0] - 2024-01-02
+### Added:
+- Created `indicators` file, where I added `BolingerBands`, `RSI`, `PSAR`, `SMA` indicators
+- Added `SharpeRatio` and `MaxDrawdown` metrics to `metrics`
+- Included indicators handling into `data_feeder.PdDataFeeder` object
+- Included indicators handling into `state.State` object
+
+### Changed:
+- Changed `finrock` package dependency from `0.0.4` to `0.4.1`
+- Refactored `render.PygameRender` object to handle indicators rendering (getting very messy)
+- Updated `scalers.MinMaxScaler` to handle indicators scaling
+- Updated `trading_env.TradingEnv` to raise an error with `np.nan` data and skip `None` states
+
+
 ## [0.3.0] - 2023-12-05
 ### Added:
 - Added `DifferentActions` and `AccountValue` as metrics. Metrics are the main way to evaluate the performance of the agent.

README.md

@@ -3,7 +3,7 @@ Reinforcement Learning package for Finance
 
 # Environment Structure:
 <p align="center">
-  <img src="Tutorials\Documents\03_FinRock.jpg">
+  <img src="Tutorials\Documents\04_FinRock.jpg">
 </p>
 
 ### Install requirements:
@@ -30,10 +30,21 @@ experiments/testing_ppo_sinusoid.py
 
 ### Environment Render:
 <p align="center">
-  <img src="Tutorials\Documents\03_FinRock_render.png">
+  <img src="Tutorials\Documents\04_FinRock_render.png">
 </p>
 
 ## Links to YouTube videos:
 - [Introduction to FinRock package](https://youtu.be/xU_YJB7vilA)
 - [Complete Trading Simulation Backbone](https://youtu.be/1z5geob8Yho)
-- [Training RL agent on Sinusoid data](https://youtu.be/JkA4BuYvWyE)
+- [Training RL agent on Sinusoid data](https://youtu.be/JkA4BuYvWyE)
+- [Included metrics and indicators into environment](https://youtu.be/bGpBEnKzIdo)
+
+# TODO:
+- [ ] Train model on `continuous` actions (control allocation percentage)
+- [ ] Add more indicators
+- [ ] Add more metrics
+- [ ] Add more reward functions
+- [ ] Add more scalers
+- [ ] Train RL agent on real data
+- [ ] Add more RL algorithms
+- [ ] Refactor rendering, maybe move to browser?

Tutorials/04_Indicators_and_Metrics.md

@@ -0,0 +1,43 @@
+# Complete Trading Simulation Backbone
+
+### Environment Structure:
+<p align="center">
+  <img src="Documents\04_FinRock.jpg">
+</p>
+
+### Link to YouTube video:
+https://youtu.be/bGpBEnKzIdo
+
+### Link to tutorial code:
+https://github.com/pythonlessons/FinRock/tree/0.4.0
+
+### Download tutorial code:
+https://github.com/pythonlessons/FinRock/archive/refs/tags/0.4.0.zip
+
+
+### Install requirements:
+```
+pip install -r requirements.txt
+pip install pygame
+pip install .
+```
+
+### Create sinusoid data:
+```
+python bin/create_sinusoid_data.py
+```
+
+### Train RL (PPO) agent on discrete actions:
+```
+experiments/training_ppo_sinusoid.py
+```
+
+### Test trained agent (Change path to the saved model):
+```
+experiments/testing_ppo_sinusoid.py
+```
+
+### Environment Render:
+<p align="center">
+  <img src="Documents\04_FinRock_render.png">
+</p>

Tutorials/Documents/04_FinRock.jpg

@@ -6,11 +6,21 @@
 from finrock.render import PygameRender
 from finrock.scalers import MinMaxScaler
 from finrock.reward import simpleReward
+from finrock.indicators import BolingerBands, SMA, RSI, PSAR
 
 df = pd.read_csv('Datasets/random_sinusoid.csv')
 
-pd_data_feeder = PdDataFeeder(df)
-
+pd_data_feeder = PdDataFeeder(
+    df = df,
+    indicators = [
+        BolingerBands(data=df, period=20, std=2),
+        RSI(data=df, period=14),
+        PSAR(data=df),
+        SMA(data=df, period=7),
+        SMA(data=df, period=25),
+        SMA(data=df, period=99),
+    ]
+)
 
 env = TradingEnv(
     data_feeder = pd_data_feeder,
@@ -21,10 +31,10 @@
     reward_function = simpleReward
 )
 action_space = env.action_space
+input_shape = env.observation_space.shape
 
 pygameRender = PygameRender(frame_rate=60)
 
-
 state, info = env.reset()
 pygameRender.render(info)
 rewards = 0.0

Tutorials/Documents/04_FinRock_render.png

@@ -10,13 +10,24 @@
 from finrock.render import PygameRender
 from finrock.scalers import MinMaxScaler
 from finrock.reward import simpleReward
-from finrock.metrics import DifferentActions, AccountValue
+from finrock.metrics import DifferentActions, AccountValue, MaxDrawdown, SharpeRatio
+from finrock.indicators import BolingerBands, RSI, PSAR, SMA
 
 
 df = pd.read_csv('Datasets/random_sinusoid.csv')
 df = df[-1000:]
 
-pd_data_feeder = PdDataFeeder(df)
+pd_data_feeder = PdDataFeeder(
+    df,
+    indicators = [
+        BolingerBands(data=df, period=20, std=2),
+        RSI(data=df, period=14),
+        PSAR(data=df),
+        SMA(data=df, period=7),
+        SMA(data=df, period=25),
+        SMA(data=df, period=99),
+    ]
+    )
 
 env = TradingEnv(
     data_feeder = pd_data_feeder,
@@ -28,14 +39,16 @@
     metrics = [
         DifferentActions(),
         AccountValue(),
+        MaxDrawdown(),
+        SharpeRatio(),
     ]
 )
 
 action_space = env.action_space
 input_shape = env.observation_space.shape
 pygameRender = PygameRender(frame_rate=120)
 
-agent = tf.keras.models.load_model('runs/1701698276/ppo_sinusoid_actor.h5')
+agent = tf.keras.models.load_model('runs/1702982487/ppo_sinusoid_actor.h5')
 
 state, info = env.reset()
 pygameRender.render(info)
@@ -51,7 +64,9 @@
     pygameRender.render(info)
 
     if terminated or truncated:
-        print(rewards, info["metrics"]['account_value'])
+        print(rewards)
+        for metric, value in info['metrics'].items():
+            print(metric, value)
         state, info = env.reset()
         rewards = 0.0
         pygameRender.reset()

experiments/playing_random_sinusoid.py

@@ -11,7 +11,8 @@
 from finrock.trading_env import TradingEnv
 from finrock.scalers import MinMaxScaler
 from finrock.reward import simpleReward
-from finrock.metrics import DifferentActions, AccountValue
+from finrock.metrics import DifferentActions, AccountValue, MaxDrawdown, SharpeRatio
+from finrock.indicators import BolingerBands, RSI, PSAR, SMA
 
 from rockrl.utils.misc import MeanAverage
 from rockrl.utils.memory import Memory
@@ -20,8 +21,17 @@
 df = pd.read_csv('Datasets/random_sinusoid.csv')
 df = df[:-1000] # leave 1000 for testing
 
-pd_data_feeder = PdDataFeeder(df)
-
+pd_data_feeder = PdDataFeeder(
+    df,
+    indicators = [
+        BolingerBands(data=df, period=20, std=2),
+        RSI(data=df, period=14),
+        PSAR(data=df),
+        SMA(data=df, period=7),
+        SMA(data=df, period=25),
+        SMA(data=df, period=99),
+    ]
+)
 
 env = TradingEnv(
     data_feeder = pd_data_feeder,
@@ -33,6 +43,8 @@
     metrics = [
         DifferentActions(),
         AccountValue(),
+        MaxDrawdown(),
+        SharpeRatio(),
     ]
 )
 
@@ -63,15 +75,14 @@
 agent = PPOAgent(
     actor = actor_model,
     critic = critic_model,
-    optimizer=tf.keras.optimizers.Adam(learning_rate=0.0002),
+    optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),
     batch_size=512,
     lamda=0.95,
     kl_coeff=0.5,
     c2=0.01,
     writer_comment='ppo_sinusoid',
 )
 
-
 memory = Memory()
 meanAverage = MeanAverage(best_mean_score_episode=1000)
 state, info = env.reset()
@@ -98,6 +109,5 @@
         memory.reset()
         state, info = env.reset()
 
-
         if agent.epoch >= 10000:
             break

experiments/testing_ppo_sinusoid.py

@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "0.4.0"

experiments/training_ppo_sinusoid.py

@@ -1,17 +1,21 @@
 import pandas as pd
-
 from finrock.state import State
+from finrock.indicators import Indicator
+
 
 class PdDataFeeder:
     def __init__(
             self, 
             df: pd.DataFrame,
+            indicators: list = [],
             min: float = None,
             max: float = None
             ) -> None:
         self._df = df
         self._min = min
         self._max = max
+        self._indicators = indicators
+        self._cache = {}
 
         assert isinstance(self._df, pd.DataFrame) == True, "df must be a pandas.DataFrame"
         assert 'timestamp' in self._df.columns, "df must have 'timestamp' column"
@@ -20,6 +24,9 @@ def __init__(
         assert 'low' in self._df.columns, "df must have 'low' column"
         assert 'close' in self._df.columns, "df must have 'close' column"
 
+        assert isinstance(self._indicators, list) == True, "indicators must be an iterable"
+        assert all(isinstance(indicator, Indicator) for indicator in self._indicators) == True, "indicators must be a list of Indicator objects"
+
     @property
     def min(self) -> float:
         return self._min or self._df['low'].min()
@@ -32,16 +39,30 @@ def __len__(self) -> int:
         return len(self._df)
 
     def __getitem__(self, idx: int, args=None) -> State:
-        data = self._df.iloc[idx]
+        # Use cache to speed up training
+        if idx in self._cache:
+            return self._cache[idx]
+
+        indicators = []
+        for indicator in self._indicators:
+            results = indicator(idx)
+            if results is None:
+                self._cache[idx] = None
+                return None
+            
+            indicators.append(results)
 
+        data = self._df.iloc[idx]
         state = State(
             timestamp=data['timestamp'],
             open=data['open'],
             high=data['high'],
             low=data['low'],
             close=data['close'],
-            volume=data.get('volume', 0.0)
+            volume=data.get('volume', 0.0),
+            indicators=indicators
         )
+        self._cache[idx] = state
 
         return state