Initial commit

.env.example

@@ -0,0 +1,4 @@
+THREADS=
+SESSION_ID=
+SKIP_IF_EXISTS=true
+HEADLESS=false

.eslintignore

@@ -0,0 +1,4 @@
+node_modules
+build
+dist
+.out

.eslintrc

@@ -0,0 +1,16 @@
+{
+  "parser": "@typescript-eslint/parser",
+  "extends": [
+    "@nersent/eslint-config-ts"
+  ],
+  "settings": {
+    "import/resolver": {
+      "typescript": {}
+    }
+  },
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module",
+    "project": ["./tsconfig.json"]
+  }
+}

.gitignore

@@ -0,0 +1,6 @@
+node_modules
+build
+dist
+.out
+.env
+.runtime

.prettierignore

@@ -0,0 +1,3 @@
+build
+coverage
+.accounts copy

README.md

@@ -0,0 +1,42 @@
+# Tradingview Downloader
+
+## Prerequisites
+
+- [Node.js](https://nodejs.org/en/) min v16
+
+## How to install
+
+### 1. Install Yarn
+
+Open shell and type:
+
+```bash
+npm install yarn --global
+```
+
+### 2. Install Node.js dependencies
+
+Open cmd at the root of this project
+
+```bash
+yarn
+```
+
+### 3. Update `.env`
+
+    a) Sign in to Tradingview
+    b) Go to random chart page
+    c) Click lock icon in chrome next to address bar
+    d) Click cookies
+    e) Select tradingview.com > cookies
+    f) Click item with name `sessionid`
+    g) Copy value from `content`. **DO NOT SHARE THIS VALUE WITH ANYONE**
+    h) Copy `.env.example` file to `.env` and change respective values
+
+### Set what asset you want to download
+
+Go to `src/list.ts` and edit `TRADINGVIEW_DOWNLOAD_LIST` array
+
+- `name` is your custom name for this asset like BTC_OLCHV_4H
+- `chartId` is a value for custom layout id from `https://tradingview.com/chart/{chartId}/?symbol=${symbolName}
+- `symbol` is a symbol name of given asset

package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@masterclass/tradingview-downloader",
+  "version": "1.0.0",
+  "main": "index.js",
+  "private": true,
+  "author": "Mikołaj Palkiewicz <[email protected]>",
+  "prettier": "@nersent/prettier-config",
+  "scripts": {
+    "dev": "cross-env NODE_ENV=development tsc-watch --onSuccess \"node build/src/index.js\""
+  },
+  "devDependencies": {
+    "@nersent/eslint-config-ts": "^0.1.0",
+    "@nersent/prettier-config": "^0.1.0",
+    "@types/node": "^18.11.12",
+    "@types/progress": "^2.0.5",
+    "@typescript-eslint/eslint-plugin": "^5.18.0",
+    "@typescript-eslint/parser": "^5.18.0",
+    "cross-env": "^7.0.3",
+    "env-cmd": "^10.1.0",
+    "eslint": "^8.13.0",
+    "eslint-config-next": "12.1.2",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-import-resolver-typescript": "^2.5.0",
+    "eslint-plugin-import": "^2.25.4",
+    "eslint-plugin-jsx-a11y": "6.5.1",
+    "eslint-plugin-prettier": "^4.0.0",
+    "prettier": "^2.6.2",
+    "ts-node": "^10.4.0",
+    "tsc-watch": "^4.6.0",
+    "typescript": "^4.7.0-dev.20220225"
+  },
+  "dependencies": {
+    "@mathieuc/tradingview": "^3.3.3",
+    "@nersent/event-emitter": "^1.0.0-dev.1",
+    "@nersent/lambda-fun": "^1.0.0-dev.2",
+    "chalk": "^4.1.2",
+    "concurrently": "^7.3.0",
+    "csv-reader": "^1.0.10",
+    "csv-writer": "^1.6.0",
+    "dotenv": "^16.0.3",
+    "execa": "^5.1.0",
+    "playwright": "^1.29.1",
+    "progress": "^2.0.3",
+    "rimraf": "^3.0.2",
+    "tslib": "^2.4.1"
+  }
+}

src/app.ts

@@ -0,0 +1,67 @@
+import { resolve } from "path";
+
+import { ETA } from "@nersent/lambda-fun";
+import chalk from "chalk";
+
+import { Config } from "./config/get-config";
+import {
+  TRADINGVIEW_DOWNLOAD_LIST,
+  formatTradingviewDownloadPath,
+} from "./list";
+import { TradingviewClient } from "./tradingview/tradingview-client";
+import { stringifyTradingviewChartTimeFrame } from "./tradingview/tradingview-utils";
+
+export const downloadList = async (
+  tradingviewClient: TradingviewClient,
+): Promise<void> => {
+  const downloadCount = TRADINGVIEW_DOWNLOAD_LIST.reduce((count, item) => {
+    return count + (Array.isArray(item.timeframe) ? item.timeframe.length : 1);
+  }, 0);
+  console.log(chalk.yellowBright(`💸 Downloading ${downloadCount} items`));
+
+  const startTime = Date.now();
+  const downloadEta = new ETA().setTotal(downloadCount).setCurrent(0);
+  downloadEta.start();
+
+  tradingviewClient.on("skipDownload", (ctx) => {
+    console.log(
+      chalk.green(
+        `✔️  Skipped ${ctx.name} | ${stringifyTradingviewChartTimeFrame(
+          ctx.timeframe,
+        )}`,
+      ),
+    );
+  });
+
+  tradingviewClient.on("download", (ctx) => {
+    console.log(
+      chalk.magentaBright(
+        `⬇️  Downloading ${ctx.name} | ${stringifyTradingviewChartTimeFrame(
+          ctx.timeframe,
+        )} | ETA: ${downloadEta.getCurrent()}s`,
+      ),
+    );
+  });
+
+  tradingviewClient.on("downloaded", (ctx) => {
+    console.log(
+      chalk.greenBright(
+        `✔️  Downloaded ${ctx.name} | ${stringifyTradingviewChartTimeFrame(
+          ctx.timeframe,
+        )} | ${ctx.path}`,
+      ),
+    );
+    downloadEta.setCurrent(downloadEta.getCurrent() + 1);
+  });
+
+  await Promise.all(
+    TRADINGVIEW_DOWNLOAD_LIST.map(async (item) => {
+      await tradingviewClient.download(item, {
+        path: (ctx) => resolve(".out", formatTradingviewDownloadPath(ctx)),
+      });
+    }),
+  );
+
+  const endTime = Date.now() - startTime;
+  console.log(`✅ Done after ${endTime / 1000}s`);
+};

src/config/get-config.ts

@@ -0,0 +1,32 @@
+import { cpus } from "os";
+
+import { config } from "dotenv";
+
+import { throwIfNullish } from "../utils/assert";
+
+config();
+
+export interface Config {
+  threads: number;
+  sessionId: string;
+  skipIfExists: boolean;
+  headless: boolean;
+}
+
+export const getConfig = (): Config => {
+  return {
+    threads:
+      process.env["THREADS"] != null
+        ? parseInt(process.env["THREADS"])
+        : cpus().length,
+    sessionId: throwIfNullish(process.env["SESSION_ID"]),
+    skipIfExists:
+      process.env["SKIP_IF_EXISTS"] != null
+        ? process.env["SKIP_IF_EXISTS"] === "true"
+        : true,
+    headless:
+      process.env["HEADLESS"] != null
+        ? process.env["HEADLESS"] === "true"
+        : false,
+  };
+};

src/global-types.ts

@@ -0,0 +1 @@
+declare module "@mathieuc/tradingview";

src/index.ts

@@ -0,0 +1,48 @@
+import { resolve } from "path";
+
+import chalk from "chalk";
+
+import { downloadList } from "./app";
+import { getConfig } from "./config/get-config";
+import { TradingviewClient } from "./tradingview/tradingview-client";
+
+process.on("uncaughtException", (e) => {
+  console.log(`uncaughtException at ${new Date().toLocaleString()}`);
+  console.log(e);
+});
+
+process.on("unhandledRejection", (e) => {
+  console.log(`unhandledRejection at ${new Date().toLocaleString()}`);
+  console.log(e);
+});
+
+process.on("exit", () => {
+  console.log(`Process exited at ${new Date().toLocaleString()}`);
+});
+
+const main = async (): Promise<void> => {
+  const config = getConfig();
+  const runtimeDataPath = resolve(".runtime");
+  const tradingviewClient = new TradingviewClient({
+    sessionId: config.sessionId,
+    headless: config.headless,
+    cachePath: resolve(runtimeDataPath, "cache"),
+    userDataPath: resolve(runtimeDataPath, "userdata"),
+    threads: config.threads,
+    skipIfExists: config.skipIfExists,
+  });
+
+  console.log(
+    chalk.blue(
+      `⚡ Initializing | Threads: ${config.threads} | Headless: ${config.headless}`,
+    ),
+  );
+  await tradingviewClient.init();
+  console.log(chalk.blueBright("✔️  Initialized"));
+
+  await downloadList(tradingviewClient);
+
+  await tradingviewClient.close();
+};
+
+main();

src/list.ts

@@ -0,0 +1,26 @@
+import { TradingviewDownloadFormatPathContext } from "./tradingview/tradingview-client";
+import { TradingviewDownloadRequest } from "./tradingview/tradingview-downloader-types";
+import { TradingviewCharts } from "./tradingview/tradingview-public-charts";
+import { TradingviewChartTimeframe } from "./tradingview/tradingview-types";
+import { stringifyTradingviewChartTimeFrame } from "./tradingview/tradingview-utils";
+
+export const formatTradingviewDownloadPath = (
+  ctx: TradingviewDownloadFormatPathContext,
+): string => {
+  return `${ctx.name}/${stringifyTradingviewChartTimeFrame(ctx.timeframe)}.csv`;
+};
+
+export const TRADINGVIEW_DOWNLOAD_LIST: TradingviewDownloadRequest[] = [
+  {
+    name: "BTCUSDT",
+    chartId: TradingviewCharts.OHLCV,
+    symbol: "BITSTAMP%3ABTCUSD",
+    timeframe: [TradingviewChartTimeframe.ONE_HOUR],
+  },
+  {
+    name: "ETHUSDT",
+    chartId: TradingviewCharts.OHLCV,
+    symbol: "BITSTAMP%3AETHUSD",
+    timeframe: [TradingviewChartTimeframe.FOUR_HOURS],
+  },
+];

src/tradingview/tradingview-client-thread.ts

@@ -0,0 +1,60 @@
+import { Thread } from "@nersent/lambda-fun";
+import { BrowserContext, Page } from "playwright";
+
+export interface TradingViewClientThreadOptions {
+  sessionId: string;
+}
+
+export class TradingViewClientThread<T> extends Thread<T> {
+  private page: Page | undefined = undefined;
+
+  constructor(
+    id: string,
+    private readonly browserCtx: BrowserContext,
+    private readonly options: TradingViewClientThreadOptions,
+  ) {
+    super(id);
+  }
+
+  public override async init(): Promise<void> {
+    super.init();
+  }
+
+  public async createPage(): Promise<Page> {
+    if (this.page != null && !this.page.isClosed()) {
+      await this.page.close();
+    }
+    this.page = await this.browserCtx.newPage();
+    await this.page.setViewportSize({
+      width: 1024,
+      height: 1024,
+    });
+    await this.page.context().addCookies([
+      {
+        name: "sessionid",
+        value: this.options.sessionId,
+        domain: ".tradingview.com",
+        path: "/",
+      },
+    ]);
+    return this.page;
+  }
+
+  public getBrowserContext(): BrowserContext {
+    return this.browserCtx;
+  }
+
+  public async getPage(): Promise<Page> {
+    if (this.page != null) return this.page;
+    return await this.createPage();
+  }
+
+  public getPageOrFail(): Page {
+    if (this.page == null) throw new Error("Page is not initialized");
+    return this.page;
+  }
+
+  public override canRun(): boolean {
+    return super.canRun();
+  }
+}

src/tradingview/tradingview-client-types.ts

@@ -0,0 +1,27 @@
+export interface TradingviewMarketEntry {
+  id: string;
+  exchange: string;
+  fullExchange: string;
+  screener: string;
+  symbol: string;
+  description: string;
+  type: string;
+  getTA: any;
+}
+
+export interface TradingviewIndicator {
+  id: string;
+  version: string;
+  name: string;
+  author: TradingviewIndicatorAuthor;
+  image: string;
+  access: string;
+  source: string;
+  type: string;
+  get: (...args: any[]) => any;
+}
+
+export interface TradingviewIndicatorAuthor {
+  id: number;
+  username: string;
+}