diff --git a/api/.env.example b/api/.env.example index f893ff28..64a0ef49 100644 --- a/api/.env.example +++ b/api/.env.example @@ -1,4 +1,4 @@ -DB_URL=postgres://postgres:postgres@localhost:5432/postgres +DB_URL="postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable" NODE_ENV=development PORT=8080 JWT_SECRET='' diff --git a/api/Makefile b/api/Makefile index 65fa59f6..5840f5bc 100644 --- a/api/Makefile +++ b/api/Makefile @@ -29,17 +29,3 @@ migrate: undo-migrate: yarn run migrate:undo - -# move along, this is for mocking -create-mock-network: - docker network create queuer || true - -start-mock-deps: - docker run --network=queuer --name transcription-db -e POSTGRES_PASSWORD=postgres -p 5432:5432 -d postgres - docker run --network=queuer --name transcription-redis -p 6379:6379 -d redis - -build-mock: - docker build -t transcription . - -run-mock: - docker run --network=queuer transcription diff --git a/api/app/controllers/review.controller.ts b/api/app/controllers/review.controller.ts index f419ec0c..471053b4 100644 --- a/api/app/controllers/review.controller.ts +++ b/api/app/controllers/review.controller.ts @@ -2,20 +2,8 @@ import { Request, Response } from "express"; import { Op } from "sequelize"; import { Review, Transaction, Transcript, User } from "../db/models"; -import { - DB_QUERY_LIMIT, - DB_START_PAGE, - QUERY_REVIEW_STATUS, - HOUR_END_OF_DAY, - MINUTE_END_OF_DAY, - SECOND_END_OF_DAY, - MILLISECOND_END_OF_DAY, -} from "../utils/constants"; -import { - buildIsActiveCondition, - buildIsInActiveCondition, - buildIsPendingCondition, -} from "../utils/review.inference"; +import { DB_QUERY_LIMIT, DB_START_PAGE } from "../utils/constants"; +import { buildCondition, buildReviewResponse } from "../utils/review.inference"; import { parseMdToJSON } from "../helpers/transcript"; import axios from "axios"; import { BaseParsedMdContent, TranscriptAttributes } from "../types/transcript"; @@ -95,10 +83,10 @@ export async function create(req: Request, res: Response) { // Retrieve all reviews from the database. export async function findAll(req: Request, res: Response) { - let queryStatus = req.query.status; + const queryStatus = req.query.status as string | undefined; const userId = Number(req.body.userId); - const page: number = Number(req.query.page) || 1; - const limit: number = Number(req.query.limit) || 5; + const page: number = Number(req.query.page) || DB_START_PAGE; + const limit: number = Number(req.query.limit) || DB_QUERY_LIMIT; const offset: number = (page - 1) * limit; const user = await User.findOne({ @@ -114,59 +102,26 @@ export async function findAll(req: Request, res: Response) { return; } - let groupedCondition = {}; - const currentTime = new Date().getTime(); - - const userIdCondition = { userId: { [Op.eq]: user.id } }; - - // add condition if query exists - if (Boolean(user.id)) { - groupedCondition = { ...groupedCondition, ...userIdCondition }; - } - if (queryStatus) { - switch (queryStatus) { - case QUERY_REVIEW_STATUS.ACTIVE: - const activeCondition = buildIsActiveCondition(currentTime); - groupedCondition = { ...groupedCondition, ...activeCondition }; - break; - case QUERY_REVIEW_STATUS.PENDING: - const pendingCondition = buildIsPendingCondition(); - groupedCondition = { ...groupedCondition, ...pendingCondition }; - break; - case QUERY_REVIEW_STATUS.INACTIVE: - const inActiveCondition = buildIsInActiveCondition(currentTime); - groupedCondition = { ...groupedCondition, ...inActiveCondition }; - break; - default: - break; - } - } + const { condition } = buildCondition({ + status: queryStatus, + userId: user.id, + }); try { const totalItems = await Review.count({ - where: groupedCondition, + where: condition, }); - const totalPages = Math.ceil(totalItems / limit); - const hasNextPage = page < totalPages; - const hasPreviousPage = page > 1; const data = await Review.findAll({ - where: groupedCondition, - limit: limit, - offset: offset, + where: condition, + limit, + offset, order: [["createdAt", "DESC"]], include: { model: Transcript }, }); - const response = { - totalItems: totalItems, - itemsPerPage: limit, - totalPages: totalPages, - currentPage: page, - hasNextPage, - hasPreviousPage, - data, - }; + const response = buildReviewResponse(data, page, limit, totalItems); + res.status(200).send(response); } catch (error) { console.log(error); @@ -279,124 +234,28 @@ export const getAllReviewsForAdmin = async (req: Request, res: Response) => { const transcriptId = Number(req.query.transcriptId); const userId = Number(req.query.userId); const mergedAt = req.query.mergedAt as string; + const submittedAt = req.query.submittedAt as string; const status = req.query.status as string; const userSearch = req.query.user as string; const page: number = Number(req.query.page) || DB_START_PAGE; const limit: number = Number(req.query.limit) || DB_QUERY_LIMIT; - - const condition: { - [key: string | number]: any; - } = {}; - - const userCondition: { - [Op.or]?: { - email?: { [Op.iLike]: string }; - githubUsername?: { [Op.iLike]: string }; - }[]; - } = {}; - - if (status) { - const currentTime = new Date().getTime(); - switch (status) { - case QUERY_REVIEW_STATUS.ACTIVE: - const activeCondition = buildIsActiveCondition(currentTime); - condition[Op.and as unknown as keyof typeof Op] = activeCondition; - break; - - case "expired": - const expiredCondition = buildIsInActiveCondition(currentTime); - condition[Op.and as unknown as keyof typeof Op] = expiredCondition; - break; - - case QUERY_REVIEW_STATUS.PENDING: - const pendingCondition = buildIsPendingCondition(); - condition[Op.and as unknown as keyof typeof Op] = pendingCondition; - break; - - default: - break; - } - } - - // Check if the mergedAt parameter is provided in the query - if (Boolean(mergedAt)) { - // Convert the mergedAt string to a Date object - const date = new Date(mergedAt as string); - - // Calculate the start of the day (00:00:00.000) for the mergedAt date - const startOfDay = new Date( - date.getFullYear(), - date.getMonth(), - date.getDate() - ); - - // Calculate the end of the day (23:59:59.999) for the mergedAt date - const endOfDay = new Date( - date.getFullYear(), - date.getMonth(), - date.getDate(), - HOUR_END_OF_DAY, - MINUTE_END_OF_DAY, - SECOND_END_OF_DAY, - MILLISECOND_END_OF_DAY - ); - - // Set the condition for mergedAt to filter records within the specified day - condition.mergedAt = { - [Op.gte]: startOfDay, - [Op.lte]: endOfDay, - }; - } - - if (Boolean(transcriptId)) { - condition.transcriptId = { [Op.eq]: transcriptId }; - } - if (Boolean(userId)) { - condition.userId = { [Op.eq]: userId }; - } - - // Check if the mergedAt parameter is provided in the query for all time zone support - if (Boolean(mergedAt)) { - // Convert the mergedAt string to a Date object - const date = new Date(mergedAt as string); - - // Calculate the start of the day (00:00:00.000) for the mergedAt date - const startOfDay = new Date( - date.getFullYear(), - date.getMonth(), - date.getDate() - ); - - // Calculate the end of the day (23:59:59.999) for the mergedAt date - const endOfDay = new Date( - date.getFullYear(), - date.getMonth(), - date.getDate(), - HOUR_END_OF_DAY, - MINUTE_END_OF_DAY, - SECOND_END_OF_DAY, - MILLISECOND_END_OF_DAY - ); - - // Set the condition for mergedAt to filter records within the specified day - condition.mergedAt = { - [Op.gte]: startOfDay, - [Op.lte]: endOfDay, - }; - } - - if (userSearch) { - const searchCondition = { [Op.iLike]: `%${userSearch.toLowerCase()}%` }; - userCondition[Op.or] = [ - { email: searchCondition }, - { githubUsername: searchCondition }, - ]; - } + const offset = Math.max(0, (page - 1) * limit); + + const { condition, userCondition } = buildCondition({ + status, + transcriptId, + userId, + mergedAt, + userSearch, + submittedAt, + }); try { const reviews = await Review.findAll({ where: condition, order: [["createdAt", "DESC"]], + offset, + limit, include: [ { model: Transcript, required: true, attributes: { exclude: ["id"] } }, { @@ -421,19 +280,7 @@ export const getAllReviewsForAdmin = async (req: Request, res: Response) => { ], }); - const totalPages = Math.ceil(reviewCount / limit); - const hasNextPage = page < totalPages; - const hasPreviousPage = page > 1; - - const response = { - totalItems: reviewCount, - totalPages, - currentPage: page, - itemsPerPage: limit, - hasNextPage, - hasPreviousPage, - data: reviews, - }; + const response = buildReviewResponse(reviews, page, limit, reviewCount); res.status(200).json(response); } catch (error) { diff --git a/api/app/controllers/transcript.controller.ts b/api/app/controllers/transcript.controller.ts index 7cd3cd71..3572c6bc 100644 --- a/api/app/controllers/transcript.controller.ts +++ b/api/app/controllers/transcript.controller.ts @@ -9,7 +9,13 @@ import { buildIsPendingCondition, getTotalWords, } from "../utils/review.inference"; -import { DB_QUERY_LIMIT, MAXPENDINGREVIEWS } from "../utils/constants"; +import { + DB_QUERY_LIMIT, + DB_START_PAGE, + MAX_PENDING_REVIEWS, + MERGED_REVIEWS_THRESHOLD, +} from "../utils/constants"; + import { generateUniqueHash } from "../helpers/transcript"; import { redis } from "../db"; import { @@ -96,7 +102,7 @@ export async function create(req: Request, res: Response) { // Retrieve all unarchived and queued transcripts from the database. export async function findAll(req: Request, res: Response) { - const page: number = Number(req.query.page) || 1; + const page: number = Number(req.query.page) || DB_START_PAGE; const limit: number = Number(req.query.limit) || DB_QUERY_LIMIT; const offset: number = (page - 1) * limit; let condition = { @@ -135,6 +141,7 @@ export async function findAll(req: Request, res: Response) { totalItems, itemsPerPage: limit, totalPages, + currentPage: Number(page), hasNextPage, hasPreviousPage, data: cachedTranscripts, @@ -194,7 +201,7 @@ export async function findAll(req: Request, res: Response) { totalItems: totalItems, itemsPerPage: limit, totalPages: totalPages, - currentPage: page, + currentPage: Number(page), hasNextPage, hasPreviousPage, data, @@ -347,7 +354,7 @@ export async function claim(req: Request, res: Response) { const transcriptId = req.params.id; const uid = req.body.claimedBy; - const branchUrl = req.body.branchUrl + const branchUrl = req.body.branchUrl; const currentTime = new Date().getTime(); const activeCondition = buildIsActiveCondition(currentTime); const pendingCondition = buildIsPendingCondition(); @@ -366,12 +373,28 @@ export async function claim(req: Request, res: Response) { return; } + // if user has successfully reviewed fewer than 3 transcripts + // allow to claim only 1 transcript and return if user has already has a pending review + // if user has successfully reviewed 3 or more transcripts, allow to have 6 pending reviews + const successfulReviews = await Review.findAll({ + where: { ...userCondition, mergedAt: { [Op.ne]: null } }, + }); const pendingReview = await Review.findAll({ where: { ...userCondition, ...pendingCondition }, }); - if (pendingReview.length >= MAXPENDINGREVIEWS) { + if ( + successfulReviews.length <= MERGED_REVIEWS_THRESHOLD && + pendingReview.length + ) { + res.status(500).send({ + message: + "You have a pending review, finish it first before claiming another!", + }); + return; + } + if (pendingReview.length >= MAX_PENDING_REVIEWS) { res.status(500).send({ - message: "User has too many pending reviews, clear some and try again!", + message: `You have ${pendingReview.length} pending reviews, clear some and try again!`, }); return; } @@ -382,7 +405,7 @@ export async function claim(req: Request, res: Response) { }; if (branchUrl) { - review.branchUrl = branchUrl + review.branchUrl = branchUrl; } try { diff --git a/api/app/controllers/webhook.controller.ts b/api/app/controllers/webhook.controller.ts index 38b620f7..b1dcbdcb 100644 --- a/api/app/controllers/webhook.controller.ts +++ b/api/app/controllers/webhook.controller.ts @@ -297,4 +297,4 @@ export async function handlePushEvent(req: Request, res: Response) { return handleError(error, res); } return res.sendStatus(200); -} \ No newline at end of file +} diff --git a/api/app/routes/review.routes.ts b/api/app/routes/review.routes.ts index b218da55..abd5f891 100644 --- a/api/app/routes/review.routes.ts +++ b/api/app/routes/review.routes.ts @@ -208,7 +208,7 @@ export function reviewRoutes(app: Express) { * name: status * schema: * type: string - * enum: [expired, pending, active] + * enum: [expired, pending, active, merged] * description: Filter reviews based on status * - in: query * name: transcriptId @@ -231,6 +231,11 @@ export function reviewRoutes(app: Express) { * type: string * description: Filter reviews based on mergedAt * - in: query + * name: submittedAt + * schema: + * type: string + * description: Filter reviews based on submittedAt + * - in: query * name: page * schema: * $ref: '#/components/schemas/Pagination' diff --git a/api/app/types/review.ts b/api/app/types/review.ts index 21e33029..a643c7bb 100644 --- a/api/app/types/review.ts +++ b/api/app/types/review.ts @@ -8,3 +8,13 @@ export interface ReviewAttributes { pr_url?: string | null; branchUrl?: string | null; } + + +export interface BuildConditionArgs { + status?: string; + transcriptId?: number; + userId?: number; + mergedAt?: string; + userSearch?: string; + submittedAt?: string; +} diff --git a/api/app/utils/constants.ts b/api/app/utils/constants.ts index 5e4a601c..1fb90f57 100644 --- a/api/app/utils/constants.ts +++ b/api/app/utils/constants.ts @@ -8,6 +8,7 @@ const QUERY_REVIEW_STATUS = { ACTIVE: "active", PENDING: "pending", INACTIVE: "inactive", + MERGED: "merged", } as const; const SATS_REWARD_RATE_PER_WORD = 0.5; @@ -16,25 +17,24 @@ const expiresInHours = 24; const currentTime = Math.floor(Date.now() / 1000); -const JWTEXPIRYTIMEINHOURS = currentTime + (expiresInHours * 60 * 60); - +const JWTEXPIRYTIMEINHOURS = currentTime + expiresInHours * 60 * 60; // This is a random number that is used to note the number of pages to be cached const PAGE_COUNT = 100; - const EXPIRYTIMEINHOURS = 24; -const MAXPENDINGREVIEWS = 3; -const INVOICEEXPIRYTIME = 5 * 60 * 1000 -const FEE_LIMIT_SAT=100 -const INVOICE_TIME_OUT = 60 +const MAX_PENDING_REVIEWS = 6; +const MERGED_REVIEWS_THRESHOLD = 3; +const INVOICEEXPIRYTIME = 5 * 60 * 1000; +const FEE_LIMIT_SAT = 100; +const INVOICE_TIME_OUT = 60; const PICO_BTC_TO_SATS = 10000; const DB_QUERY_LIMIT = 10; const DB_TXN_QUERY_LIMIT = 20; -const DB_START_PAGE = 0; +const DB_START_PAGE = 1; -const PUBLIC_PROFILE_REVIEW_LIMIT = 5 +const PUBLIC_PROFILE_REVIEW_LIMIT = 5; const LOG_LEVEL = process.env.LOG_LEVEL || "info"; @@ -50,7 +50,7 @@ export { QUERY_REVIEW_STATUS, SATS_REWARD_RATE_PER_WORD, EXPIRYTIMEINHOURS, - MAXPENDINGREVIEWS, + MAX_PENDING_REVIEWS, JWTEXPIRYTIMEINHOURS, INVOICEEXPIRYTIME, FEE_LIMIT_SAT, @@ -66,9 +66,6 @@ export { SECOND_END_OF_DAY, MILLISECOND_END_OF_DAY, PAGE_COUNT, - DELAY_IN_BETWEEN_REQUESTS + DELAY_IN_BETWEEN_REQUESTS, + MERGED_REVIEWS_THRESHOLD, }; - - - - diff --git a/api/app/utils/review.inference.ts b/api/app/utils/review.inference.ts index d90707d7..c42c5015 100644 --- a/api/app/utils/review.inference.ts +++ b/api/app/utils/review.inference.ts @@ -3,7 +3,9 @@ import { Op } from "sequelize"; import { TranscriptAttributes } from "../types/transcript"; import { wordCount } from "./functions"; -import { EXPIRYTIMEINHOURS } from "./constants"; +import { EXPIRYTIMEINHOURS, HOUR_END_OF_DAY, MILLISECOND_END_OF_DAY, MINUTE_END_OF_DAY, QUERY_REVIEW_STATUS, SECOND_END_OF_DAY } from "./constants"; +import { Review } from "../db/models"; +import { BuildConditionArgs } from "../types/review"; const unixEpochTimeInMilliseconds = getUnixTimeFromHours(EXPIRYTIMEINHOURS); @@ -26,19 +28,16 @@ const buildIsPendingCondition = () => { }; }; -const buildIsInActiveCondition = (currentTime: number) => { +const buildIsExpiredAndArchivedCondition = (currentTime: number) => { const timeStringAt24HoursPrior = new Date( currentTime - unixEpochTimeInMilliseconds ).toISOString(); return { - [Op.or]: { - mergedAt: { [Op.not]: null }, // has been merged + [Op.and]: { + mergedAt: { [Op.eq]: null }, // has not been merged archivedAt: { [Op.not]: null }, // has been archived - // inactive conditions when review has expired - [Op.and]: { - createdAt: { [Op.lt]: timeStringAt24HoursPrior }, // expired - submittedAt: { [Op.eq]: null }, // has not been submitted - }, + submittedAt: { [Op.eq]: null }, // has not been submitted + createdAt: { [Op.lt]: timeStringAt24HoursPrior }, // expired }, }; }; @@ -57,6 +56,26 @@ const buildIsExpiredAndNotArchivedCondition = (currentTime: number) => { }; }; +// This condition is used to get all expired reviews, whether they are archived or not. +// Because we don't want to ignore expired reviews that has not yet been archived by the +// daily cron job. +const buildIsExpiredCondition = (currentTime: number) => { + const expiredAndArchivedCondition = buildIsExpiredAndArchivedCondition(currentTime); + const expiredAndNotArchivedCondition = buildIsExpiredAndNotArchivedCondition(currentTime); + return { + [Op.or]: [expiredAndArchivedCondition, expiredAndNotArchivedCondition], + }; +}; + +const buildIsMergedCondition = () => { + const mergedQuery = { + [Op.and]: [ //ensuring all conditions are met + { mergedAt: { [Op.not]: null } }, // has been merged + ] + }; + return mergedQuery; +} + function getUnixTimeFromHours(hours: number) { const millisecondsInHour = 60 * 60 * 1000; const unixTimeInMilliseconds = hours * millisecondsInHour; @@ -143,12 +162,131 @@ async function calculateWordDiff(data: TranscriptAttributes) { return { totalDiff, totalWords, addedWords, removedWords }; } + +export const buildCondition = ({ + status, + transcriptId, + userId, + mergedAt, + userSearch, + submittedAt, +}: BuildConditionArgs) => { + const condition: { [key: string | number]: any } = {}; + const userCondition: { [Op.or]?: { email?: { [Op.iLike]: string }; githubUsername?: { [Op.iLike]: string } }[] } = {}; + + if (status) { + const currentTime = new Date().getTime(); + switch (status) { + case QUERY_REVIEW_STATUS.ACTIVE: + const activeCondition = buildIsActiveCondition(currentTime); + condition[Op.and as unknown as keyof typeof Op] = activeCondition; + break; + + case 'expired': + const expiredCondition = buildIsExpiredCondition(currentTime); + condition[Op.and as unknown as keyof typeof Op] = expiredCondition; + break; + + case QUERY_REVIEW_STATUS.PENDING: + const pendingCondition = buildIsPendingCondition(); + condition[Op.and as unknown as keyof typeof Op] = pendingCondition; + break; + + case QUERY_REVIEW_STATUS.MERGED: + const mergedCondition = buildIsMergedCondition(); + condition[Op.and as unknown as keyof typeof Op] = mergedCondition; + break; + + default: + break; + } + } + + if (mergedAt) { + const date = new Date(mergedAt); + const startOfDay = new Date(date.getFullYear(), date.getMonth(), date.getDate()); + const endOfDay = new Date( + date.getFullYear(), + date.getMonth(), + date.getDate(), + HOUR_END_OF_DAY, + MINUTE_END_OF_DAY, + SECOND_END_OF_DAY, + MILLISECOND_END_OF_DAY + ); + + condition.mergedAt = { + [Op.gte]: startOfDay, + [Op.lte]: endOfDay, + }; + } + + if (submittedAt) { + const date = new Date(submittedAt); + const startOfDay = new Date(date.getFullYear(), date.getMonth(), date.getDate()); + const endOfDay = new Date( + date.getFullYear(), + date.getMonth(), + date.getDate(), + HOUR_END_OF_DAY, + MINUTE_END_OF_DAY, + SECOND_END_OF_DAY, + MILLISECOND_END_OF_DAY + ); + + condition.submittedAt = { + [Op.gte]: startOfDay, + [Op.lte]: endOfDay, + }; + } + + if (transcriptId) { + condition.transcriptId = { [Op.eq]: transcriptId }; + } + + if (userId) { + condition.userId = { [Op.eq]: userId }; + } + + if (userSearch) { + const searchCondition = { [Op.iLike]: `%${userSearch.toLowerCase()}%` }; + userCondition[Op.or] = [ + { email: searchCondition }, + { githubUsername: searchCondition }, + ]; + } + + return { condition, userCondition }; +}; + +export const buildReviewResponse = ( + reviews: Review[], + page: number, + limit: number, + totalItems: number +) => { + const totalPages = Math.ceil(totalItems / limit); + const hasNextPage = page < totalPages; + const hasPreviousPage = page > 1; + + return { + totalItems, + totalPages, + currentPage: page, + itemsPerPage: limit, + hasNextPage, + hasPreviousPage, + data: reviews, + }; +}; + export { getUnixTimeFromHours, buildIsActiveCondition, buildIsPendingCondition, - buildIsInActiveCondition, + buildIsExpiredAndArchivedCondition, buildIsExpiredAndNotArchivedCondition, calculateWordDiff, getTotalWords, + buildIsMergedCondition, }; diff --git a/docker-compose.yml b/docker-compose.yml index d5c4c1a5..6578992c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,41 @@ -version: "3.9" +version: '3.9' services: - api: + nodeapi: build: ./api ports: - "8080:8080" + environment: + - POSTGRES_HOST=transcription-api-postgres-1 + - REDIS_HOST=transcription-api-redis-1 + - NODE_ENV=development + depends_on: + - postgres + - redis + restart: unless-stopped + volumes: + - .:/app + - /app/node_modules - nginx: - build: ./nginx - container_name: nginx + postgres: + image: postgres:latest + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres ports: - - "80:80" + - "5432:5432" # Map PostgreSQL port to host volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf - depends_on: - - api \ No newline at end of file + - postgres_data:/var/lib/postgresql/data + + redis: + image: redis:latest + ports: + - "6379:6379" # Map Redis port to host + volumes: + - redis_data:/data + +volumes: + postgres_data: + redis_data: +