Skip to content

Commit

Permalink
Feat: updated document chunk size to 1300 from 1500
Browse files Browse the repository at this point in the history
  • Loading branch information
noisrucer committed Feb 16, 2024
1 parent 098ee21 commit aacf33e
Show file tree
Hide file tree
Showing 25 changed files with 109 additions and 126 deletions.
10 changes: 5 additions & 5 deletions reminder/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,15 @@ def load_config() -> AppConfig:
access_key=os.environ["PICKTOSS_AWS_ACCESS_KEY"], secret_key=os.environ["PICKTOSS_AWS_SECRET_KEY"]
)

s3_config = S3Config(region_name="ap-northeast-1", bucket_name=os.environ['PICKTOSS_S3_BUCKET_NAME'])
s3_config = S3Config(region_name="ap-northeast-1", bucket_name=os.environ["PICKTOSS_S3_BUCKET_NAME"])

sqs_config = SQSConfig(region_name="ap-northeast-1", queue_url=os.environ["PICKTOSS_AWS_SQS_QUEUE_URL"])

oauth_config = OauthConfig(
client_id=os.environ["PICKTOSS_OAUTH_CLIENT_ID"],
redirect_uri=os.environ["PICKTOSS_OAUTH_REDIRECT_URI"],
client_secret=os.environ["PICKTOSS_OAUTH_CLIENT_SECRET"],
callback_response_redirect_host=os.environ["PICKTOSS_OAUTH_CALLBACK_RESPONSE_REDIRECT_HOST"]
callback_response_redirect_host=os.environ["PICKTOSS_OAUTH_CALLBACK_RESPONSE_REDIRECT_HOST"],
)

jwt_config = JWTConfig(
Expand All @@ -125,8 +125,7 @@ def load_config() -> AppConfig:
)

discord_config = DiscordConfig(
bot_token=os.environ["PICKTOSS_DISCORD_BOT_TOKEN"],
channel_id=os.environ["PICKTOSS_DISCORD_CHANNEL_ID"]
bot_token=os.environ["PICKTOSS_DISCORD_BOT_TOKEN"], channel_id=os.environ["PICKTOSS_DISCORD_CHANNEL_ID"]
)

app_config = AppConfig(
Expand All @@ -139,7 +138,8 @@ def load_config() -> AppConfig:
jwt=jwt_config,
email=email_config,
discord=discord_config,
cors_allowed_origin=os.environ["PICKTOSS_CORS_ALLOWED_ORIGIN"]
cors_allowed_origin=os.environ["PICKTOSS_CORS_ALLOWED_ORIGIN"],
)

return app_config

10 changes: 3 additions & 7 deletions reminder/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,9 @@
member_repository=member_repository,
subscription_repository=subscription_repository,
subscription_service=subscription_service,
document_service=document_service
document_service=document_service,
)

feedback_service = FeedbackService(
feedback_repository=feedback_repository
)
feedback_service = FeedbackService(feedback_repository=feedback_repository)

payment_repository = PaymentService(
payment_repository=payment_repository
)
payment_repository = PaymentService(payment_repository=payment_repository)
37 changes: 11 additions & 26 deletions reminder/core/discord/discord_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,15 @@ def __init__(
self.base_url = base_url
self.url = base_url + f"/channels/{channel_id}/messages"
self.headers = {"Authorization": f"Bot {bot_token}"}

def report_llm_error(
self,
task: str,
error_type: LLMErrorType,
document_content: str,
error_message: str,
info: str = "",
llm_response: str = ""
llm_response: str = "",
):
"""Report LLM Error
Expand All @@ -48,42 +48,27 @@ def report_llm_error(

content = f"# Task: {task}\n## Error Type: {error_type.value}\n* KST: `{korea_now}`\n* UTC: `{utc_now}`\n{info}"
embeds = [
{
"title": "Document Content",
"description": document_content
},
{
"title": "Error Message",
"description": error_message
}
{"title": "Document Content", "description": document_content},
{"title": "Error Message", "description": error_message},
]

if error_type == LLMErrorType.INVALID_JSON_FORMAT:
embeds.insert(1, {
"title": "LLM Response",
"description": llm_response
},)
embeds.insert(
1,
{"title": "LLM Response", "description": llm_response},
)

body = {
"content": content,
"tts": False,
"embeds": embeds
}
body = {"content": content, "tts": False, "embeds": embeds}

response = requests.post(
url=self.url,
json=body,
headers=self.headers
)
response = requests.post(url=self.url, json=body, headers=self.headers)

try:
response.raise_for_status()
except HTTPError as e:
pass



discord_client = DiscordClient(
channel_id=cfg.discord.channel_id,
bot_token=cfg.discord.bot_token,
)
)
5 changes: 1 addition & 4 deletions reminder/core/llm/exception.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,5 @@ class InvalidLLMJsonResponseError(BaseCustomException):
def __init__(self, llm_response: str):
self.llm_response = llm_response
super().__init__(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Invalid LLM Response: {llm_response}"
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Invalid LLM Response: {llm_response}"
)


2 changes: 2 additions & 0 deletions reminder/domain/category/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@

router = APIRouter(tags=["Category"])


@router.get("/hello", status_code=status.HTTP_200_OK)
async def hello():
return {"hello": hello}


@router.post("/categories", status_code=status.HTTP_201_CREATED, response_model=CreateCategoryResponse)
async def create_category(
request: CreateCategoryRequest, session: DBSessionDep, member_id: str = Depends(get_current_member_id)
Expand Down
1 change: 0 additions & 1 deletion reminder/domain/category/exception.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,3 @@ def __init__(self, category_id: int):
class DuplicateCategoryNameError(BaseCustomException):
def __init__(self, name: str):
super().__init__(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Category with name {name} already exists")

2 changes: 1 addition & 1 deletion reminder/domain/category/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ async def find_or_none_by_id(self, session: AsyncSession, member_id: str, catego
query = select(Category).where(and_(Category.id == category_id, Category.member_id == member_id))
result = await session.execute(query)
return result.scalars().first()

async def find_or_none_by_name(self, session: AsyncSession, member_id: str, name: str) -> Category | None:
query = select(Category).where(and_(Category.name == name, Category.member_id == member_id))
result = await session.execute(query)
Expand Down
2 changes: 1 addition & 1 deletion reminder/domain/document/constant.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def get_current_subscription_max_document_num_by_subscription_plan(plan: Subscri
return PRO_PLAN_MONTHLY_MAX_DOCUMENT_NUM
else:
raise ValueError("Invalid subscription plan type")


def get_anytime_max_document_num_by_subscription_plan(plan: SubscriptionPlanType) -> int:
assert isinstance(plan, SubscriptionPlanType)
Expand Down
6 changes: 5 additions & 1 deletion reminder/domain/document/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,11 @@ async def upload_document(
return await document_service.upload_document(session=session, member_id=member_id, edocument=edocument)


@router.get("/categories/{category_id}/documents", status_code=status.HTTP_200_OK, response_model=GetAllDocumentsByCategoryResponse)
@router.get(
"/categories/{category_id}/documents",
status_code=status.HTTP_200_OK,
response_model=GetAllDocumentsByCategoryResponse,
)
async def get_all_documents_by_category(
category_id: int, session: DBSessionDep, member_id: str = Depends(get_current_member_id)
):
Expand Down
3 changes: 2 additions & 1 deletion reminder/domain/document/exception.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
FREE_PLAN_MONTHLY_MAX_DOCUMENT_NUM,
PRO_PLAN_MONTHLY_MAX_DOCUMENT_NUM,
FREE_PLAN_CURRENT_MAX_DOCUMENT_NUM,
PRO_PLAN_CURRENT_MAX_DOCUMENT_NUM
PRO_PLAN_CURRENT_MAX_DOCUMENT_NUM,
)


Expand All @@ -31,6 +31,7 @@ def __init__(self):
detail=f"Pro 플랜은 한달에 최대 {PRO_PLAN_MONTHLY_MAX_DOCUMENT_NUM}개의 문서를 업로드 할 수 있습니다.",
)


class FreePlanAnytimeDocumentUploadLimitExceedError(BaseCustomException):
def __init__(self):
super().__init__(
Expand Down
2 changes: 1 addition & 1 deletion reminder/domain/document/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ async def find_all_by_member_id(self, session: AsyncSession, member_id: str) ->
)
result = await session.execute(query)
return result.scalars().fetchall()

async def delete_by_member_id_and_id(self, session: AsyncSession, member_id: str, document_id: int) -> None:
document = await self.find_by_id(session, member_id, document_id)
if document is None:
Expand Down
29 changes: 14 additions & 15 deletions reminder/domain/document/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
FREE_PLAN_MONTHLY_MAX_DOCUMENT_NUM,
PRO_PLAN_MONTHLY_MAX_DOCUMENT_NUM,
FREE_PLAN_CURRENT_MAX_DOCUMENT_NUM,
PRO_PLAN_CURRENT_MAX_DOCUMENT_NUM
PRO_PLAN_CURRENT_MAX_DOCUMENT_NUM,
)
from reminder.domain.document.entity import EDocument
from reminder.domain.document.exception import (
Expand All @@ -23,7 +23,7 @@
FreePlanCurrentSubscriptionDocumentUploadLimitExceedError,
ProPlanCurrentSubscriptionDocumentUploadLimitExceedError,
FreePlanAnytimeDocumentUploadLimitExceedError,
ProPlanAnytimeDocumentUploadLimitExceedError
ProPlanAnytimeDocumentUploadLimitExceedError,
)
from reminder.domain.document.model import Document, DocumentUpload
from reminder.domain.document.repository import (
Expand Down Expand Up @@ -78,7 +78,9 @@ async def upload_document(
)

# 현재 시점에 업로드된 문서 개수: (제한 - Free: 3개, Pro: 15개)
current_num_uploaded_documents: int = await self.get_num_current_uploaded_documents_by_member_id(session, member_id)
current_num_uploaded_documents: int = await self.get_num_current_uploaded_documents_by_member_id(
session, member_id
)

# 현재 구독 사이클에 업로드한 문서 개수: (제한 - Free: 15개, Pro: 40개)
current_subscription_num_uploaded_documents: int = (
Expand All @@ -88,22 +90,22 @@ async def upload_document(

assert isinstance(plan_type, SubscriptionPlanType)
if plan_type == SubscriptionPlanType.FREE:
if current_subscription_num_uploaded_documents >= FREE_PLAN_MONTHLY_MAX_DOCUMENT_NUM: # 15개
if current_subscription_num_uploaded_documents >= FREE_PLAN_MONTHLY_MAX_DOCUMENT_NUM: # 15개
raise FreePlanCurrentSubscriptionDocumentUploadLimitExceedError()
if current_num_uploaded_documents >= FREE_PLAN_CURRENT_MAX_DOCUMENT_NUM: # 매 시점: 3개
if current_num_uploaded_documents >= FREE_PLAN_CURRENT_MAX_DOCUMENT_NUM: # 매 시점: 3개
raise FreePlanAnytimeDocumentUploadLimitExceedError()
elif plan_type == SubscriptionPlanType.PRO:
if current_subscription_num_uploaded_documents >= PRO_PLAN_MONTHLY_MAX_DOCUMENT_NUM: # 40개
if current_subscription_num_uploaded_documents >= PRO_PLAN_MONTHLY_MAX_DOCUMENT_NUM: # 40개
raise ProPlanCurrentSubscriptionDocumentUploadLimitExceedError()
if current_num_uploaded_documents >= PRO_PLAN_CURRENT_MAX_DOCUMENT_NUM: # 매 시점: 15개
if current_num_uploaded_documents >= PRO_PLAN_CURRENT_MAX_DOCUMENT_NUM: # 매 시점: 15개
raise ProPlanAnytimeDocumentUploadLimitExceedError()
else:
raise ValueError("Invalid Plan Type")

# Ensure document max size limit (15,000 characters)
if len(edocument.decode_contenet_str()) > DOCUMENT_MAX_LEN:
raise DocumentMaxLengthExceedError()

if len(edocument.decode_contenet_str()) < DOCUMENT_MIN_LEN:
raise DocumentMinLengthError()

Expand Down Expand Up @@ -174,7 +176,7 @@ async def get_document_by_id(self, session: AsyncSession, member_id: str, docume
],
content=content,
)

async def delete_document_by_id(self, session: AsyncSession, member_id: str, document_id: int) -> None:
await self.document_repository.delete_by_member_id_and_id(session, member_id, document_id)

Expand All @@ -196,11 +198,8 @@ async def get_num_uploaded_documents_for_current_subscription_by_member_id(
]

return len(current_subscription_document_uploads)

async def get_num_current_uploaded_documents_by_member_id(
self, session: AsyncSession, member_id: str
) -> int:
"""현재 업로드된 문서 개수
"""

async def get_num_current_uploaded_documents_by_member_id(self, session: AsyncSession, member_id: str) -> int:
"""현재 업로드된 문서 개수"""
documents = await self.document_repository.find_all_by_member_id(session, member_id)
return len(documents)
2 changes: 1 addition & 1 deletion reminder/domain/feedback/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
async def receive_feedback(
request: CreateFeedbackRequest, session: DBSessionDep, member_id: str = Depends(get_current_member_id)
):
await feedback_service.receive_feedback(session=session, content=request.content, member_id=member_id)
await feedback_service.receive_feedback(session=session, content=request.content, member_id=member_id)
13 changes: 5 additions & 8 deletions reminder/domain/feedback/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,12 @@
from reminder.domain.feedback.repository import FeedbackRepository
from reminder.domain.feedback.model import Feedback


class FeedbackService:
def __init__(self, feedback_repository: FeedbackRepository):
self.feedback_repository = feedback_repository



async def receive_feedback(self, session: AsyncSession, content: str, member_id: str):
feedback = Feedback(
content=content,
member_id=member_id
)

await self.feedback_repository.save(session, feedback)
feedback = Feedback(content=content, member_id=member_id)

await self.feedback_repository.save(session, feedback)
15 changes: 3 additions & 12 deletions reminder/domain/member/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,14 @@

class JWTError(BaseCustomException):
def __init__(self):
super().__init__(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="JWT Error"
)
super().__init__(status_code=status.HTTP_401_UNAUTHORIZED, detail="JWT Error")


class InvalidTokenScopeError(BaseCustomException):
def __init__(self):
super().__init__(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect token scope"
)
super().__init__(status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect token scope")


class MemberNotFoundError(BaseCustomException):
def __init__(self, member_id: str):
super().__init__(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Member with id {member_id} is not found"
)
super().__init__(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Member with id {member_id} is not found")
13 changes: 9 additions & 4 deletions reminder/domain/member/response/get_member_info_response.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,21 @@ class GetMemberInfoSubScriptionDto(BaseModel):
class GetMemberInfoDocumentDto(BaseModel):
# 현재
currentPossessDocumentNum: int = Field(..., examples=[7], description="현재 업로드 된 문서 개수")
currentSubscriptionCycleUploadedDocumentNum: int = Field(..., examples=[36], description="현재 구독 기간 동안 업로드한 문서 개수")
currentSubscriptionCycleUploadedDocumentNum: int = Field(
..., examples=[36], description="현재 구독 기간 동안 업로드한 문서 개수"
)

# Free
freePlanMaxPossessDocumentNum: int = Field(..., examples=[7], description="Free 플랜 최대 문서 보유 개수")
freePlanSubscriptionMaxUploadDocumentNum: int = Field(..., examples=[30], description="Free 플랜 구독 기간 최대 업로드 문서 개수")
freePlanSubscriptionMaxUploadDocumentNum: int = Field(
..., examples=[30], description="Free 플랜 구독 기간 최대 업로드 문서 개수"
)

# Pro
proPlanMaxPossessDocumentNum: int = Field(..., examples=[20], description="Pro 플랜 최대 문서 보유 개수")
proPlanSubscriptionMaxUploadDocumentNum: int = Field(..., examples=[50], description="Pro 플랜 구독 기간 최대 업로드 문서 개수")

proPlanSubscriptionMaxUploadDocumentNum: int = Field(
..., examples=[50], description="Pro 플랜 구독 기간 최대 업로드 문서 개수"
)


class GetMemberInfoQuizDto(BaseModel):
Expand Down
Loading

0 comments on commit aacf33e

Please sign in to comment.