From 5f4cc39abe35594f834fb08b35da19e531d8d6b1 Mon Sep 17 00:00:00 2001 From: Sean Luo <41530060+lu0x1a0@users.noreply.github.com> Date: Tue, 4 Jan 2022 23:32:50 +1100 Subject: [PATCH] can start qt in ipynb --- logeg.ipynb => OldNoteBook/logeg.ipynb | 0 .../visualize.ipynb | 0 README.md | 16 +- indicatorplayground.ipynb | 242 ++++++++++++++++++ newqt.py | 12 +- 5 files changed, 265 insertions(+), 5 deletions(-) rename logeg.ipynb => OldNoteBook/logeg.ipynb (100%) rename visualize.ipynb => OldNoteBook/visualize.ipynb (100%) create mode 100644 indicatorplayground.ipynb diff --git a/logeg.ipynb b/OldNoteBook/logeg.ipynb similarity index 100% rename from logeg.ipynb rename to OldNoteBook/logeg.ipynb diff --git a/visualize.ipynb b/OldNoteBook/visualize.ipynb similarity index 100% rename from visualize.ipynb rename to OldNoteBook/visualize.ipynb diff --git a/README.md b/README.md index ebe8889..948819f 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,18 @@ * Display Live Streaming Data from IG and aggregate to historical data * make qtplot accept jupyter data +## Current Bug: +streaming data saved some very weird residual? +i.e. +``` +,time,Open,High,Low,Close +0,2022-01-04 10:56:00,1.1296300000000001,1.12964,1.12952,1.1295250000000001 +1,2022-01-04 10:57:00,1.1295350000000002,1.12962,1.12947,1.12948 +2,2022-01-04 10:58:00,1.12949,1.12962,1.12946,1.12948 +3,2022-01-04 10:59:00,1.12949,1.1295549999999999,1.1294650000000002,1.1294650000000002 +4,2022-01-04 11:00:00,1.129475,1.129505,1.129235,1.1292900000000001 +``` + # Requirements install conda environment from the environment.yml file via ``` @@ -26,13 +38,15 @@ Run newqt.py, in MainWindow class change between Test1plot (minutes) and Test2pl # Predictions: 1 step Predition of test data for Dema_9-rolling_Standard Deviation_100 on Hourly data -details inside nnstruct.ipynb +details inside 1stepprediction.ipynb ![MSE Loss](./Images/MSELoss.png) ![BCE Loss](./Images/BCELoss.png) # Preliminary 4 step prediction on scaled D1EMA100 on Hourly Data, 300 epochs +details inside d1multistep notebook + D1 means price[hour = i]-price[hour = (i-1)] ![4 step](./Images/nsteppredict.png) diff --git a/indicatorplayground.ipynb b/indicatorplayground.ipynb new file mode 100644 index 0000000..06d27d1 --- /dev/null +++ b/indicatorplayground.ipynb @@ -0,0 +1,242 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "source": [ + "import sys\n", + "from PyQt5.QtWidgets import QApplication, QMainWindow, QHBoxLayout, QVBoxLayout, QWidget\n", + "\n", + "from QTVisual.PlotsHolder import PGFigureLayoutWrap\n", + "from QTVisual.Plot import PlotPanel\n", + "#from QTVisual.utils import plotloader\n", + "\n", + "def getOHLC_pickle(pklpath):\n", + " import pickle\n", + " with open(pklpath,'rb') as f:\n", + " data = pickle.load(f)\n", + " return data\n", + "def getOHLC_raw(to_pickle=False):\n", + " from DataManipulation.DataHandler import mergeMonths, DemaMinDayMultinom\n", + " import pandas as pd\n", + " from torch.utils.data import ConcatDataset\n", + " #data = mergePeriod(1,4,beg=\"DAT_ASCII_EURUSD_M1_2021\",dir=\"./eurusd2021/\",dump=False)\n", + " data2021 = mergeMonths(start=1, end=9,beg=\"DAT_ASCII_EURUSD_M1_2021\",dir = './Data/eurusd2021/', dump=False)\n", + "\n", + " dir = \"./Data/\"\n", + " beg = \"DAT_ASCII_EURUSD_M1_\"\n", + "\n", + " years = range(2010,2021)\n", + " dataY = []\n", + " for y in years:\n", + " dataY.append(pd.read_csv(dir+beg+str(y)+'.csv',sep=';',names = ['Open','High','Low','Close','Volume']))\n", + " dataY.append(data2021)\n", + " data = pd.concat(dataY)\n", + " data.index.name = \"Date\"\n", + " data.index = pd.to_datetime(data.index)\n", + " if to_pickle:\n", + " import pickle\n", + " with open('OHLC.pkl','wb') as f:\n", + " pickle.dump(data,f,protocol=pickle.HIGHEST_PROTOCOL)\n", + " return data\n", + "\n", + "\n", + "\n", + "class MainWindow(QMainWindow):\n", + "\n", + " def __init__(self, picklepath = None, dfs = None):\n", + " super(MainWindow, self).__init__()\n", + "\n", + " self.setWindowTitle(\"My App\")\n", + " self.layout = QHBoxLayout()\n", + " \n", + " self.leftbar = QVBoxLayout()\n", + " \n", + " if picklepath is None:\n", + " self.plotpanels,self.data = Test2Plots()\n", + " elif dfs is None:\n", + " self.dfs = getOHLC_pickle(picklepath)\n", + " self.data = self.dfs[0]\n", + " self.plotpanels = [PlotPanel(df) for df in self.dfs]\n", + " else:\n", + " self.dfs = dfs\n", + " self.data = self.dfs[0]\n", + " self.plotpanels = [PlotPanel(df) for df in self.dfs]\n", + " #print(self.data.tail(100))\n", + " self.rightmain = PGFigureLayoutWrap(self.plotpanels, len(self.data))\n", + " self.layout.addLayout(self.leftbar)\n", + " self.layout.addLayout(self.rightmain)\n", + "\n", + " widget = QWidget()\n", + " widget.setLayout(self.layout)\n", + " self.setCentralWidget(widget)\n", + "def Test1Plots():\n", + " from DataManipulation.indicators import DEMA,MYEMA,D1\n", + " from time import time\n", + " t = time()\n", + " data = getOHLC_pickle(\"EURUSD_M_2010_2021.pkl\")\n", + " # 9min\n", + " demaperiod = [9]\n", + " # 100 min, 200 min, 5 h, 40 h, 80 h, 100 h, 300 h, 100 d, 200 d\n", + " # 1.6d, 3.3d, 4.16d, 12.5d, \n", + " emaperiod = [100,200,300,2400,4800, 100*60,300*60,2400*60,4800*60]\n", + " for p in demaperiod:\n", + " data['dema'+str(p)] = DEMA(data['Close'],p)\n", + " for p in emaperiod:\n", + " data['ema'+str(p)] = MYEMA(data['Close'],p)\n", + " D1demaperiod = [9]\n", + " D1emaperiod = [100,200,300,2400,4800]\n", + " for p in D1demaperiod:\n", + " data['D1dema'+str(p)] = D1(data['dema'+str(p)])\n", + " for p in D1emaperiod:\n", + " data['D1ema'+str(p)] = D1(data['ema'+str(p)])\n", + "\n", + " stdPeriod = [9,100,300,100*60]\n", + " stdInd = ['D1dema9']#,'D1ema100','D1ema300']\n", + " for p in stdPeriod:\n", + " for i in stdInd:\n", + " data[i+\"_std\"+str(p)] = data[i].rolling(p).std()\n", + " print(data.columns)\n", + " print(time()-t)\n", + " t = time()\n", + " plotpanels = [\n", + " PlotPanel(\n", + " data[['Close']+['dema'+str(p) for p in demaperiod]+['ema'+str(p) for p in emaperiod]],\n", + " ),\n", + " PlotPanel(\n", + " data[['D1dema'+str(p) for p in D1demaperiod]+['D1ema'+str(p) for p in D1emaperiod]]\n", + " ),\n", + " PlotPanel(\n", + " data[['D1dema9_std'+str(p) for p in stdPeriod]]\n", + " )\n", + " ]\n", + " return plotpanels,data\n", + "\n", + "def Test2Plots():\n", + " from DataManipulation.indicators import DEMA,MYEMA,D1\n", + " from time import time\n", + " t = time()\n", + " mdata = getOHLC_pickle(\"EURUSD_M_2010_2021.pkl\")\n", + " data = mdata.resample('1H').agg({'Open': 'first', \n", + " 'High': 'max', \n", + " 'Low': 'min', \n", + " 'Close': 'last'}).dropna()\n", + " # 9min\n", + " demaperiod = [9]\n", + " # 100 min, 200 min, 5 h, 40 h, 80 h, 100 h, 300 h, 100 d, 200 d\n", + " # 1.6d, 3.3d, 4.16d, 12.5d, \n", + " emaperiod = [100,200,300,2400,4800, 100*60,300*60,2400*60,4800*60]\n", + " for p in demaperiod:\n", + " data['dema'+str(p)] = DEMA(data['Close'],p)\n", + " for p in emaperiod:\n", + " data['ema'+str(p)] = MYEMA(data['Close'],p)\n", + " D1demaperiod = [9]\n", + " D1emaperiod = [100,200,300,2400,4800]\n", + " for p in D1demaperiod:\n", + " data['D1dema'+str(p)] = D1(data['dema'+str(p)])\n", + " for p in D1emaperiod:\n", + " data['D1ema'+str(p)] = D1(data['ema'+str(p)])\n", + "\n", + " stdPeriod = [9,100,300,100*60]\n", + " stdInd = ['D1dema9','dema9']#,'D1ema100','D1ema300']\n", + " for p in stdPeriod:\n", + " for i in stdInd:\n", + " data[i+\"_std\"+str(p)] = data[i].rolling(p).std()\n", + " \n", + " print(data.columns)\n", + " print(time()-t)\n", + " t = time()\n", + " plotpanels = [\n", + " PlotPanel(\n", + " data[['Close']+['dema'+str(p) for p in demaperiod]+['ema'+str(p) for p in emaperiod]],\n", + " ),\n", + " PlotPanel(\n", + " data[['D1dema'+str(p) for p in D1demaperiod]+['D1ema'+str(p) for p in D1emaperiod]]\n", + " ),\n", + " PlotPanel(\n", + " data[['D1dema9_std'+str(p) for p in stdPeriod]]\n", + " ),\n", + " PlotPanel(\n", + " data[['dema9_std'+str(p) for p in stdPeriod]]\n", + " )\n", + " ]\n", + " return plotpanels,data\n", + "\n", + "\n", + "app = QApplication(sys.argv)\n", + "\n", + "window = MainWindow()\n", + "window.show()\n", + "\n", + "app.exec() " + ], + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "QStandardPaths: XDG_RUNTIME_DIR not set, defaulting to '/tmp/runtime-sean'\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Index(['Open', 'High', 'Low', 'Close', 'dema9', 'ema100', 'ema200', 'ema300',\n", + " 'ema2400', 'ema4800', 'ema6000', 'ema18000', 'ema144000', 'ema288000',\n", + " 'D1dema9', 'D1ema100', 'D1ema200', 'D1ema300', 'D1ema2400', 'D1ema4800',\n", + " 'D1dema9_std9', 'dema9_std9', 'D1dema9_std100', 'dema9_std100',\n", + " 'D1dema9_std300', 'dema9_std300', 'D1dema9_std6000', 'dema9_std6000'],\n", + " dtype='object')\n", + "5.127691984176636\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/home/sean/miniconda3/envs/qttrade/lib/python3.9/site-packages/pyqtgraph/graphicsItems/PlotCurveItem.py:153: RuntimeWarning: All-NaN slice encountered\n", + " b = (np.nanmin(d), np.nanmax(d))\n", + "libGL error: No matching fbConfigs or visuals found\n", + "libGL error: failed to load driver: swrast\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "0" + ] + }, + "metadata": {}, + "execution_count": 2 + } + ], + "metadata": {} + } + ], + "metadata": { + "orig_nbformat": 4, + "language_info": { + "name": "python", + "version": "3.9.7", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3.9.7 64-bit ('qttrade': conda)" + }, + "interpreter": { + "hash": "a67e17c8fc420817cbdbd0caa16d6c1c40b59f1d67d5d3254ca561366ced5b2e" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/newqt.py b/newqt.py index 703025d..463fc26 100644 --- a/newqt.py +++ b/newqt.py @@ -3,7 +3,7 @@ from QTVisual.PlotsHolder import PGFigureLayoutWrap from QTVisual.Plot import PlotPanel -from utils import plotloader +#from utils import plotloader def getOHLC_pickle(pklpath): import pickle @@ -38,7 +38,7 @@ def getOHLC_raw(to_pickle=False): class MainWindow(QMainWindow): - def __init__(self, picklepath = None): + def __init__(self, picklepath = None, dfs = None): super(MainWindow, self).__init__() self.setWindowTitle("My App") @@ -48,10 +48,14 @@ def __init__(self, picklepath = None): if picklepath is None: self.plotpanels,self.data = Test2Plots() - else: + elif dfs is None: self.dfs = getOHLC_pickle(picklepath) self.data = self.dfs[0] - self.plotpanels = plotloader(self.dfs) + self.plotpanels = [PlotPanel(df) for df in self.dfs] + else: + self.dfs = dfs + self.data = self.dfs[0] + self.plotpanels = [PlotPanel(df) for df in self.dfs] #print(self.data.tail(100)) self.rightmain = PGFigureLayoutWrap(self.plotpanels, len(self.data)) self.layout.addLayout(self.leftbar)