Redis python (#32)

* Redis Feature: Fixes #31

* Closes #31

Ensured it was able to function within a docker container

commits/repository_links_commits.json

@@ -1 +1 @@
-{"last_saved_sha_internship": "", "last_saved_sha_newgrad": ""}
+{"last_saved_sha_internship": "", "last_saved_sha_newgrad": ""}

docker-compose.yml

@@ -6,8 +6,18 @@ services:
     container_name: ColorStackBot
     environment:
       DISCORD_TOKEN: ${DISCORD_TOKEN}
-      CHANNEL_ID: ${CHANNEL_ID}
       GIT_TOKEN: ${GIT_TOKEN}
     volumes:
       - ./commits:/app/commits
       - ./logs:/app/logs
+
+  redis:
+    image: redis:latest
+    container_name: JobsRedis
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis-data:/data
+
+volumes:
+  redis-data:

requirements.txt

@@ -3,4 +3,5 @@ pytest==7.4.0
 pygithub==2.2.0
 python-dotenv==1.0.1
 pytest-asyncio==0.23.4
-oracledb==2.0.1
+oracledb==2.0.1
+redis==5.2.0

src/DiscordBot.py

@@ -10,13 +10,16 @@
 - A Discord bot token with the necessary permissions.
 - A GitHub personal access token with the necessary permissions.
 """
+
 import asyncio
 import logging
 import os
 from datetime import datetime
 from logging.handlers import RotatingFileHandler
 
 import discord
+import redis
+
 from DatabaseConnector import DatabaseConnector
 from discord.ext import commands, tasks
 from dotenv import load_dotenv
@@ -52,7 +55,9 @@
 
 
 @tasks.loop(seconds=60)
-async def scheduled_task(internship_github: GitHubUtilities, newgrad_github: GitHubUtilities, job_utilities: JobsUtilities):
+async def scheduled_task(
+    internship_github: GitHubUtilities, newgrad_github: GitHubUtilities, job_utilities: JobsUtilities
+):
     """
     A scheduled task that runs every 60 seconds to check for new commits in the GitHub repository.
 
@@ -68,24 +73,25 @@ async def scheduled_task(internship_github: GitHubUtilities, newgrad_github: Git
             internship_repo = internship_github.createGitHubConnection()
             internship_sha = internship_github.getSavedSha(internship_repo, False)
             newgrad_repo = newgrad_github.createGitHubConnection()
-            newgrad_sha = newgrad_github.getSavedSha(newgrad_repo, True) 
+            newgrad_sha = newgrad_github.getSavedSha(newgrad_repo, True)
+            redis_client = redis.Redis(host="redis", port=6379, db=0)
 
-            # Process all internship 
+            # Process all internship
             if internship_github.isNewCommit(internship_repo, internship_sha):
                 logger.info("New internship commit has been found. Finding new jobs...")
-                internship_github.setComparison(internship_repo, False)  
+                internship_github.setComparison(internship_repo, False)
 
                 # Get the channels to send the job postings
                 db = DatabaseConnector()
                 channel_ids = db.getChannels()
 
                 if internship_github.is_coop:
                     job_postings = internship_github.getCommitChanges("README-Off-Season.md")
-                    await job_utilities.getJobs(bot, channel_ids[:20], job_postings, "Co-Op")
+                    await job_utilities.getJobs(bot, redis_client, channel_ids[:20], job_postings, "Co-Op")
 
                 if internship_github.is_summer:
                     job_postings = internship_github.getCommitChanges("README.md")
-                    await job_utilities.getJobs(bot, channel_ids[:20], job_postings, "Summer")
+                    await job_utilities.getJobs(bot, redis_client, channel_ids[:20], job_postings, "Summer")
 
                 sha_commit = internship_github.getLastCommit(internship_repo)
                 internship_github.setNewCommit(sha_commit, False)
@@ -101,13 +107,13 @@ async def scheduled_task(internship_github: GitHubUtilities, newgrad_github: Git
             # Process all new gradjobs
             if newgrad_github.isNewCommit(newgrad_repo, newgrad_sha):
                 logger.info("New grad commit has been found. Finding new jobs...")
-                newgrad_github.setComparison(newgrad_repo, True)  
+                newgrad_github.setComparison(newgrad_repo, True)
 
                 # Get the channels to send the job postings
                 db = DatabaseConnector()
                 channel_ids = db.getChannels()
                 job_postings = newgrad_github.getCommitChanges("README.md")
-                await job_utilities.getJobs(bot, channel_ids[:20], job_postings, "New Grad")
+                await job_utilities.getJobs(bot, redis_client, channel_ids[:20], job_postings, "New Grad")
 
                 sha_commit = newgrad_github.getLastCommit(newgrad_repo)
                 newgrad_github.setNewCommit(sha_commit, True)
@@ -124,6 +130,7 @@ async def scheduled_task(internship_github: GitHubUtilities, newgrad_github: Git
             logger.error("An error occurred in the scheduled task.", exc_info=True)
             await bot.close()
         finally:
+            redis_client.close()
             end_time = datetime.now()
             execution_time = end_time - start_time
             logger.info(f"Task execution time: {execution_time}")
@@ -183,8 +190,10 @@ async def on_ready():
     """
     logger.info(f"Logged in as {bot.user.name}")
 
-    github_internship = GitHubUtilities(token=GITHUB_TOKEN, repo_name="SimplifyJobs/Summer2025-Internships", isSummer=True, isCoop=True)
-    github_newgrad = GitHubUtilities(token=GITHUB_TOKEN, repo_name="SimplifyJobs/New-Grad-Positions") 
+    github_internship = GitHubUtilities(
+        token=GITHUB_TOKEN, repo_name="SimplifyJobs/Summer2025-Internships", isSummer=True, isCoop=True
+    )
+    github_newgrad = GitHubUtilities(token=GITHUB_TOKEN, repo_name="SimplifyJobs/New-Grad-Positions")
     job_utilities = JobsUtilities()
     scheduled_task.start(github_internship, github_newgrad, job_utilities)  # Start the loop
 

src/JobsUtilities.py

@@ -12,6 +12,7 @@
 import asyncio
 import logging
 import re
+import redis
 from collections.abc import Iterable
 from datetime import datetime, timedelta
 
@@ -62,6 +63,7 @@ def saveCompanyName(self, company_name: str) -> None:
     async def getJobs(
         self,
         bot: discord.ext.commands.Bot,
+        redis_client: redis.client.Redis,
         channels: list[int],
         job_postings: Iterable[str],
         term: str,
@@ -93,6 +95,11 @@ async def getJobs(
                 if job_link in self.job_cache:
                     continue
 
+                # Verify it hasn't been posted
+                if redis_client.exists(job_link):
+                    logging.info("It already exits within redis database: ", job_link)
+                    continue
+
                 self.job_cache.add(job_link)  # Save the job link
 
                 # If the company name is not present, we need to use the previous company name
@@ -178,6 +185,10 @@ async def getJobs(
                 post += f"**👉 Job Link:** <{job_link}>\n" f"{'-' * 153}"
                 self.total_jobs += 1
 
+                # Add the job link to redis database
+                redis_client.set(job_link, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+                logging.info("Added the job link to redis!")
+
                 # Send the job posting to the Discord channel
                 coroutines = (bot.get_channel(channel).send(post) for channel in channels if bot.get_channel(channel))
                 await asyncio.gather(*coroutines)