Merge pull request #44 from 21jake/feature/http-scraper

feature/http-scraper
21jake · Mar 11, 2023 · 0687d47 · 0687d47
2 parents 38be866 + ebd5855
commit 0687d47
Show file tree

Hide file tree

Showing 6 changed files with 323 additions and 227 deletions.
diff --git a/backend/src/database/migrations/1678075998716-createUnitTestRecords.ts b/backend/src/database/migrations/1678075998716-createUnitTestRecords.ts
@@ -1,67 +1,15 @@
-import { Like, MigrationInterface, QueryRunner } from 'typeorm';
-import { appEnv } from '../../configs/config';
-import { Batch } from '../../entities/batch.entity';
-import { Keyword } from '../../entities/keyword.entity';
-import { User } from '../../entities/user.entity';
-import { batch, keywords, testKeywordPref, user } from '../unit-test.data';
+import { MigrationInterface, QueryRunner } from 'typeorm';
 
-// UNCOMMENT THIS BEFORE RUNNING TEST CASES. 
-
-/**
 export class createUnitTestRecords1678075998716 implements MigrationInterface {
-  name = 'createUnitTestRecords1678075998716'
+  name = 'createUnitTestRecords1678075998716';
 
   // This migration is only for testing purposes and only runs in the local database
 
   public async up(qr: QueryRunner): Promise<void> {
-    if (appEnv.DATABASE_NAME !== 'local') return;
-
-    const conn = qr.connection;
-    const userRepo = conn.getRepository(User);
-    const keywordRepo = conn.getRepository(Keyword);
-    const batchRepo = conn.getRepository(Batch);
-
-    const testUser = await userRepo.save(user);
-    const testBatch = new Batch();
-    testBatch.originalName = batch.originalName;
-    testBatch.fileName = batch.fileName;
-    testBatch.uploader = testUser;
-    await batchRepo.save(testBatch);
-
-    const insertKwPromises = keywords.map((kw) => {
-      const keyword = new Keyword();
-      keyword.name = kw.name;
-      keyword.fileName = kw.fileName;
-      keyword.success = kw.success;
-      keyword.proxy = kw.proxy;
-      keyword.error = kw.error;
-      keyword.totalLinks = Number(kw.totalLinks);
-      keyword.totalAds = Number(kw.totalAds);
-      keyword.totalResults = Number(kw.totalResults);
-      keyword.searchTime = kw.searchTime;
-      keyword.batch = testBatch;
-      return keywordRepo.save(keyword);
-    });
-
-    const kws = await Promise.all(insertKwPromises);
-
-    console.log({ testBatch, testUser, kws });
+    console.log('This migration is no longer needed');
   }
 
   public async down(qr: QueryRunner): Promise<void> {
-    if (appEnv.DATABASE_NAME !== 'local') return;
-
-    const conn = qr.connection;
-    const userRepo = conn.getRepository(User);
-    const keywordRepo = conn.getRepository(Keyword);
-    const batchRepo = conn.getRepository(Batch);
-
-    await userRepo.delete({ username: user.username });
-    await batchRepo.delete({ originalName: batch.originalName });
-
-    // delete where name Like 'testKeywordPref%'
-    await keywordRepo.delete({ name: Like(testKeywordPref) });
+    console.log('This migration is no longer needed');
   }
-  
 }
- */
diff --git a/backend/src/database/unit-test.data.ts b/backend/src/database/unit-test.data.ts
diff --git a/backend/src/services/cron.service.ts b/backend/src/services/cron.service.ts
@@ -16,8 +16,6 @@ export class CronService {
     private keywordRepository: Repository<Keyword>,
 
     private scraperService: ScraperService,
-
-    private fileService: FileService,
   ) {}
 
   @Cron(CronExpression.EVERY_MINUTE)