From 58b71eabdd425c9a7045ebbe27ae23dc3a6e93af Mon Sep 17 00:00:00 2001 From: David Salazar <71240305+DavidSalazar123@users.noreply.github.com> Date: Fri, 25 Oct 2024 09:45:23 -0400 Subject: [PATCH] Redis python (#32) * Redis Feature: Fixes #31 * Closes #31 Ensured it was able to function within a docker container --- commits/repository_links_commits.json | 2 +- docker-compose.yml | 12 ++++++++++- requirements.txt | 3 ++- src/DiscordBot.py | 29 ++++++++++++++++++--------- src/JobsUtilities.py | 11 ++++++++++ 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/commits/repository_links_commits.json b/commits/repository_links_commits.json index 3e87492..5bca863 100644 --- a/commits/repository_links_commits.json +++ b/commits/repository_links_commits.json @@ -1 +1 @@ -{"last_saved_sha_internship": "", "last_saved_sha_newgrad": ""} \ No newline at end of file +{"last_saved_sha_internship": "", "last_saved_sha_newgrad": ""} diff --git a/docker-compose.yml b/docker-compose.yml index 4da720b..29d7f08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,18 @@ services: container_name: ColorStackBot environment: DISCORD_TOKEN: ${DISCORD_TOKEN} - CHANNEL_ID: ${CHANNEL_ID} GIT_TOKEN: ${GIT_TOKEN} volumes: - ./commits:/app/commits - ./logs:/app/logs + + redis: + image: redis:latest + container_name: JobsRedis + ports: + - "6379:6379" + volumes: + - redis-data:/data + +volumes: + redis-data: diff --git a/requirements.txt b/requirements.txt index 04ddc09..00c4cb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ pytest==7.4.0 pygithub==2.2.0 python-dotenv==1.0.1 pytest-asyncio==0.23.4 -oracledb==2.0.1 \ No newline at end of file +oracledb==2.0.1 +redis==5.2.0 diff --git a/src/DiscordBot.py b/src/DiscordBot.py index f8fd527..b334d0f 100644 --- a/src/DiscordBot.py +++ b/src/DiscordBot.py @@ -10,6 +10,7 @@ - A Discord bot token with the necessary permissions. - A GitHub personal access token with the necessary permissions. """ + import asyncio import logging import os @@ -17,6 +18,8 @@ from logging.handlers import RotatingFileHandler import discord +import redis + from DatabaseConnector import DatabaseConnector from discord.ext import commands, tasks from dotenv import load_dotenv @@ -52,7 +55,9 @@ @tasks.loop(seconds=60) -async def scheduled_task(internship_github: GitHubUtilities, newgrad_github: GitHubUtilities, job_utilities: JobsUtilities): +async def scheduled_task( + internship_github: GitHubUtilities, newgrad_github: GitHubUtilities, job_utilities: JobsUtilities +): """ A scheduled task that runs every 60 seconds to check for new commits in the GitHub repository. @@ -68,12 +73,13 @@ async def scheduled_task(internship_github: GitHubUtilities, newgrad_github: Git internship_repo = internship_github.createGitHubConnection() internship_sha = internship_github.getSavedSha(internship_repo, False) newgrad_repo = newgrad_github.createGitHubConnection() - newgrad_sha = newgrad_github.getSavedSha(newgrad_repo, True) + newgrad_sha = newgrad_github.getSavedSha(newgrad_repo, True) + redis_client = redis.Redis(host="redis", port=6379, db=0) - # Process all internship + # Process all internship if internship_github.isNewCommit(internship_repo, internship_sha): logger.info("New internship commit has been found. Finding new jobs...") - internship_github.setComparison(internship_repo, False) + internship_github.setComparison(internship_repo, False) # Get the channels to send the job postings db = DatabaseConnector() @@ -81,11 +87,11 @@ async def scheduled_task(internship_github: GitHubUtilities, newgrad_github: Git if internship_github.is_coop: job_postings = internship_github.getCommitChanges("README-Off-Season.md") - await job_utilities.getJobs(bot, channel_ids[:20], job_postings, "Co-Op") + await job_utilities.getJobs(bot, redis_client, channel_ids[:20], job_postings, "Co-Op") if internship_github.is_summer: job_postings = internship_github.getCommitChanges("README.md") - await job_utilities.getJobs(bot, channel_ids[:20], job_postings, "Summer") + await job_utilities.getJobs(bot, redis_client, channel_ids[:20], job_postings, "Summer") sha_commit = internship_github.getLastCommit(internship_repo) internship_github.setNewCommit(sha_commit, False) @@ -101,13 +107,13 @@ async def scheduled_task(internship_github: GitHubUtilities, newgrad_github: Git # Process all new gradjobs if newgrad_github.isNewCommit(newgrad_repo, newgrad_sha): logger.info("New grad commit has been found. Finding new jobs...") - newgrad_github.setComparison(newgrad_repo, True) + newgrad_github.setComparison(newgrad_repo, True) # Get the channels to send the job postings db = DatabaseConnector() channel_ids = db.getChannels() job_postings = newgrad_github.getCommitChanges("README.md") - await job_utilities.getJobs(bot, channel_ids[:20], job_postings, "New Grad") + await job_utilities.getJobs(bot, redis_client, channel_ids[:20], job_postings, "New Grad") sha_commit = newgrad_github.getLastCommit(newgrad_repo) newgrad_github.setNewCommit(sha_commit, True) @@ -124,6 +130,7 @@ async def scheduled_task(internship_github: GitHubUtilities, newgrad_github: Git logger.error("An error occurred in the scheduled task.", exc_info=True) await bot.close() finally: + redis_client.close() end_time = datetime.now() execution_time = end_time - start_time logger.info(f"Task execution time: {execution_time}") @@ -183,8 +190,10 @@ async def on_ready(): """ logger.info(f"Logged in as {bot.user.name}") - github_internship = GitHubUtilities(token=GITHUB_TOKEN, repo_name="SimplifyJobs/Summer2025-Internships", isSummer=True, isCoop=True) - github_newgrad = GitHubUtilities(token=GITHUB_TOKEN, repo_name="SimplifyJobs/New-Grad-Positions") + github_internship = GitHubUtilities( + token=GITHUB_TOKEN, repo_name="SimplifyJobs/Summer2025-Internships", isSummer=True, isCoop=True + ) + github_newgrad = GitHubUtilities(token=GITHUB_TOKEN, repo_name="SimplifyJobs/New-Grad-Positions") job_utilities = JobsUtilities() scheduled_task.start(github_internship, github_newgrad, job_utilities) # Start the loop diff --git a/src/JobsUtilities.py b/src/JobsUtilities.py index 7a60280..bb72ea4 100644 --- a/src/JobsUtilities.py +++ b/src/JobsUtilities.py @@ -12,6 +12,7 @@ import asyncio import logging import re +import redis from collections.abc import Iterable from datetime import datetime, timedelta @@ -62,6 +63,7 @@ def saveCompanyName(self, company_name: str) -> None: async def getJobs( self, bot: discord.ext.commands.Bot, + redis_client: redis.client.Redis, channels: list[int], job_postings: Iterable[str], term: str, @@ -93,6 +95,11 @@ async def getJobs( if job_link in self.job_cache: continue + # Verify it hasn't been posted + if redis_client.exists(job_link): + logging.info("It already exits within redis database: ", job_link) + continue + self.job_cache.add(job_link) # Save the job link # If the company name is not present, we need to use the previous company name @@ -178,6 +185,10 @@ async def getJobs( post += f"**👉 Job Link:** <{job_link}>\n" f"{'-' * 153}" self.total_jobs += 1 + # Add the job link to redis database + redis_client.set(job_link, datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + logging.info("Added the job link to redis!") + # Send the job posting to the Discord channel coroutines = (bot.get_channel(channel).send(post) for channel in channels if bot.get_channel(channel)) await asyncio.gather(*coroutines)