feat: use keep alive connections (#166)

* chore: replace cross-fetch by node-fetch

* feat: use keep-alive connections

package.json

@@ -40,11 +40,11 @@
     "compression": "^1.7.4",
     "connection-string": "^4.4.0",
     "cors": "^2.8.5",
-    "cross-fetch": "^4.0.0",
     "express": "^4.18.2",
     "graphql": "^16.7.1",
     "multiformats": "^9",
-    "mysql": "^2.18.1"
+    "mysql": "^2.18.1",
+    "node-fetch": "^2.7.0"
   },
   "devDependencies": {
     "@snapshot-labs/eslint-config": "^0.1.0-beta.9",
@@ -56,6 +56,7 @@
     "@types/jest": "^29.5.3",
     "@types/mysql": "^2.15.21",
     "@types/node": "^20.4.8",
+    "@types/node-fetch": "^2.6.4",
     "@types/supertest": "^2.0.12",
     "@typescript-eslint/eslint-plugin": "^5.60.1",
     "copyfiles": "^2.4.1",

src/helpers/snapshot.ts

@@ -1,6 +1,6 @@
 import { gql, ApolloClient, InMemoryCache, createHttpLink } from '@apollo/client/core';
 import { setContext } from '@apollo/client/link/context';
-import fetch from 'cross-fetch';
+import { fetchWithKeepAlive } from './utils';
 
 export type Proposal = {
   id: string;
@@ -27,7 +27,7 @@ export type Space = {
 
 const httpLink = createHttpLink({
   uri: `${process.env.HUB_URL || 'https://hub.snapshot.org'}/graphql`,
-  fetch
+  fetch: fetchWithKeepAlive
 });
 
 const authLink = setContext((_, { headers }) => {

src/helpers/utils.ts

@@ -1,3 +1,5 @@
+import http from 'node:http';
+import https from 'node:https';
 import FileStorageEngine from '../lib/storage/file';
 import AwsStorageEngine from '../lib/storage/aws';
 import type { Response } from 'express';
@@ -43,3 +45,15 @@ export function storageEngine(subDir?: string) {
     return new FileStorageEngine(subDir);
   }
 }
+
+const agentOptions = { keepAlive: true };
+const httpAgent = new http.Agent(agentOptions);
+const httpsAgent = new https.Agent(agentOptions);
+
+function agent(url: string) {
+  return new URL(url).protocol === 'http:' ? httpAgent : httpsAgent;
+}
+
+export const fetchWithKeepAlive = (uri: any, options: any = {}) => {
+  return fetch(uri, { agent: agent(uri), ...options });
+};

src/lib/metrics/digitalOcean.ts

@@ -1,5 +1,5 @@
-import fetch from 'cross-fetch';
 import { capture } from '@snapshot-labs/snapshot-sentry';
+import { fetchWithKeepAlive } from '../../helpers/utils';
 
 const BASE_URL = 'https://api.digitalocean.com/v2';
 const TTL = 60 * 1000; // 1 minute
@@ -77,7 +77,8 @@ export default class DigitalOcean {
       this.expirationTime = now + TTL;
     }
 
-    const response = await fetch(`${BASE_URL}/${path}`, {
+    const url = `${BASE_URL}/${path}`;
+    const response = await fetchWithKeepAlive(url, {
       headers: { Authorization: `Bearer ${this.key}` }
     });
 

src/lib/nftClaimer/utils.ts

@@ -1,5 +1,4 @@
 import { gql, ApolloClient, InMemoryCache, HttpLink } from '@apollo/client/core';
-import fetch from 'cross-fetch';
 import snapshot from '@snapshot-labs/snapshot.js';
 import { CID } from 'multiformats/cid';
 import { Wallet } from '@ethersproject/wallet';
@@ -8,6 +7,7 @@ import { getAddress, isAddress } from '@ethersproject/address';
 import { BigNumber } from '@ethersproject/bignumber';
 import { capture } from '@snapshot-labs/snapshot-sentry';
 import type { Proposal, Space } from '../../helpers/snapshot';
+import { fetchWithKeepAlive } from '../../helpers/utils';
 
 const requiredEnvKeys = [
   'NFT_CLAIMER_PRIVATE_KEY',
@@ -90,7 +90,7 @@ export async function getProposalContract(spaceId: string) {
 }
 
 const client = new ApolloClient({
-  link: new HttpLink({ uri: process.env.NFT_CLAIMER_SUBGRAPH_URL, fetch }),
+  link: new HttpLink({ uri: process.env.NFT_CLAIMER_SUBGRAPH_URL, fetch: fetchWithKeepAlive }),
   cache: new InMemoryCache({
     addTypename: false
   }),

src/sentryTunnel.ts

@@ -1,9 +1,8 @@
 import express from 'express';
-import fetch from 'cross-fetch';
 import bodyParser from 'body-parser';
-import { URL } from 'url';
-import { rpcError } from './helpers/utils';
 import { capture } from '@snapshot-labs/snapshot-sentry';
+import { URL } from 'url';
+import { rpcError, fetchWithKeepAlive } from './helpers/utils';
 
 const router = express.Router();
 
@@ -17,8 +16,8 @@ router.post('/sentry', bodyParser.raw({ type: () => true, limit: '4mb' }), async
 
     const dnsUri = new URL(dsn);
     const sentryApiUrl = `https://${dnsUri.hostname}/api${dnsUri.pathname}/envelope/`;
-    const response = await fetch(sentryApiUrl, {
-      method: 'post',
+    const response = await fetchWithKeepAlive(sentryApiUrl, {
+      method: 'POST',
       body: req.body
     });
 

test/integration/helpers/utils.test.ts

@@ -0,0 +1,18 @@
+import { fetchWithKeepAlive } from '../../../src/helpers/utils';
+import fetch from 'node-fetch';
+
+describe('utils.ts', () => {
+  describe('fetchWithKeepAlive', () => {
+    const TEST_URL = 'https://snapshot.org';
+
+    it('set the custom agent with keep-alive', async () => {
+      const response = await fetchWithKeepAlive(TEST_URL);
+      expect(response.headers.get('connection')).toEqual('keep-alive');
+    });
+
+    it('does not use a keep-alive connection with default node-fetch', async () => {
+      const response = await fetch(TEST_URL);
+      expect(response.headers.get('connection')).toEqual('close');
+    });
+  });
+});

yarn.lock

@@ -2314,6 +2314,14 @@
   dependencies:
     "@types/node" "*"
 
+"@types/node-fetch@^2.6.4":
+  version "2.6.4"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
+  integrity sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==
+  dependencies:
+    "@types/node" "*"
+    form-data "^3.0.0"
+
 "@types/node@*", "@types/node@^20.4.8":
   version "20.4.8"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-20.4.8.tgz#b5dda19adaa473a9bf0ab5cbd8f30ec7d43f5c85"
@@ -3136,13 +3144,6 @@ cross-fetch@^3.1.6:
   dependencies:
     node-fetch "^2.6.12"
 
-cross-fetch@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-4.0.0.tgz#f037aef1580bb3a1a35164ea2a848ba81b445983"
-  integrity sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==
-  dependencies:
-    node-fetch "^2.6.12"
-
 cross-spawn@^7.0.2, cross-spawn@^7.0.3:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
@@ -3724,6 +3725,15 @@ follow-redirects@^1.14.9:
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
   integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
 
+form-data@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f"
+  integrity sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.8"
+    mime-types "^2.1.12"
+
 form-data@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
@@ -4902,6 +4912,13 @@ node-fetch@^2.6.12:
   dependencies:
     whatwg-url "^5.0.0"
 
+node-fetch@^2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
+  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-int64@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"