From 5e1106f42799b93c3391958a897ebf5858843cda Mon Sep 17 00:00:00 2001 From: jagadeeswaran-zipstack Date: Tue, 30 Jul 2024 11:16:56 +0530 Subject: [PATCH 1/4] Fixed loader in coverage modal (#534) * fixed loader for coverage modal * code refactor --- .../OutputForDocModal.jsx | 23 +++++++++++-------- .../custom-tools/prompt-card/PromptCard.jsx | 2 +- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx index f9dc4a857..25eb22d31 100644 --- a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx +++ b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx @@ -82,7 +82,7 @@ function OutputForDocModal({ }, [open, singlePassExtractMode, isSinglePassExtractLoading]); useEffect(() => { - updatePromptOutput(); + updatePromptOutput(docOutputs); }, [docOutputs]); useEffect(() => { @@ -118,7 +118,7 @@ function OutputForDocModal({ const updatePromptOutput = (data) => { setPromptOutputs((prev) => { - const updatedPromptOutput = getUpdatedPromptOutput(data, prev); + const updatedPromptOutput = data || [...prev]; const keys = Object.keys(docOutputs); keys.forEach((key) => { @@ -130,10 +130,6 @@ function OutputForDocModal({ }); }; - const getUpdatedPromptOutput = (data, prev) => { - return data || [...prev]; - }; - const updatePromptOutputInstance = (updatedPromptOutput, docId, key) => { const index = findPromptOutputIndex(updatedPromptOutput, docId); const promptOutputInstance = createOrUpdatePromptOutputInstance( @@ -221,10 +217,17 @@ function OutputForDocModal({ const rowsData = []; const docs = moveSelectedDocToTop(); docs.forEach((item) => { - const output = data.find( - (outputValue) => outputValue?.document_manager === item?.document_id - ); - const key = `${output?.prompt_id}__${output?.document_manager}__${output?.profile_manager}`; + const output = data.find((outputValue) => { + const docId = + outputValue?.document_manager || + (outputValue?.key && getDocIdFromKey(outputValue?.key)) || + null; + return docId === item?.document_id; + }); + const key = `${promptId}__${item?.document_id}__${ + selectedProfile || profileManagerId + }`; + let status = outputStatus.fail; let message = displayPromptResult(output?.output, true); diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx index 35609066a..ef0ed947e 100644 --- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx +++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx @@ -318,7 +318,7 @@ function PromptCard({ if (runAllLLM) { let selectedProfiles = llmProfiles; - if (!coverAllDoc && selectedLlmProfiles?.length > 0) { + if (selectedLlmProfiles?.length) { selectedProfiles = llmProfiles.filter((profile) => selectedLlmProfiles.includes(profile?.profile_id) ); From 4c97569e03347d6bf75f0aa75d5ec59aa5b5d7d5 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M <117059509+chandrasekharan-zipstack@users.noreply.github.com> Date: Tue, 30 Jul 2024 11:27:07 +0530 Subject: [PATCH 2/4] fix: Reverted removal of structure tool related preferred envs (#533) Reverted removal of structure tool related preferred envs Co-authored-by: Gayathri <142381512+gaya3-zipstack@users.noreply.github.com> --- docker/scripts/merge_env.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/scripts/merge_env.py b/docker/scripts/merge_env.py index 73dcc4c6a..9a328b00c 100644 --- a/docker/scripts/merge_env.py +++ b/docker/scripts/merge_env.py @@ -10,7 +10,10 @@ # import sys -PREFERRED_BASE_ENV_KEYS = [] +PREFERRED_BASE_ENV_KEYS = [ + "STRUCTURE_TOOL_IMAGE_URL", + "STRUCTURE_TOOL_IMAGE_TAG", +] DEFAULT_AUTH_KEY = "unstract" DEFAULT_ADMIN_KEY = "admin" SET_DEFAULT_KEYS = { From ab4901d47895204ff9710aa946f91840a248d558 Mon Sep 17 00:00:00 2001 From: vishnuszipstack <117254672+vishnuszipstack@users.noreply.github.com> Date: Wed, 31 Jul 2024 09:57:39 +0530 Subject: [PATCH 3/4] mrq etl issue fixes (#535) * mrq etl issue fixes * sonar issu fix * pr comment fixes --- backend/pipeline/serializers/crud.py | 13 ++++++++ .../etl-task-deploy/EtlTaskDeploy.jsx | 30 ++++++++++++------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/backend/pipeline/serializers/crud.py b/backend/pipeline/serializers/crud.py index 56e5d1142..3480ac832 100644 --- a/backend/pipeline/serializers/crud.py +++ b/backend/pipeline/serializers/crud.py @@ -10,6 +10,7 @@ from pipeline.models import Pipeline from scheduler.helper import SchedulerHelper from utils.serializer_utils import SerializerUtils +from workflow_manager.endpoint.models import WorkflowEndpoint from backend.serializers import AuditSerializer from unstract.connectors.connectorkit import Connectorkit @@ -73,6 +74,18 @@ def _add_connector_data( connector_id=instance.connector_id, ) ) + if repr[PC.DESTINATION_NAME] == PC.NOT_CONFIGURED: + try: + check_manual_review = WorkflowEndpoint.objects.get( + workflow=instance.workflow, + endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION, + connection_type=WorkflowEndpoint.ConnectionType.MANUALREVIEW, + ) + if check_manual_review: + repr[PC.DESTINATION_NAME] = "Manual Review" + except Exception as ex: + logger.debug(f"Not a Manual review destination: {ex}") + return repr def to_representation(self, instance: Pipeline) -> OrderedDict[str, Any]: diff --git a/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx b/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx index a0383126f..bae1807df 100644 --- a/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx +++ b/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx @@ -95,21 +95,31 @@ const EtlTaskDeploy = ({ return null; }); }; - - const getWorkflows = () => { - const connectorType = type === "task" ? "FILESYSTEM" : "DATABASE"; + const fetchWorkflows = (type) => workflowApiService - .getWorkflowEndpointList("DESTINATION", connectorType) - .then((res) => { - const updatedData = res?.data.map((record) => ({ + .getWorkflowEndpointList("DESTINATION", type) + .then((res) => + res?.data.map((record) => ({ ...record, id: record.workflow, - })); - setWorkflowList(updatedData); - }) + })) + ) .catch(() => { - console.error("Unable to get workflow list"); + return []; }); + const getWorkflows = () => { + const connectorType = type === "task" ? "FILESYSTEM" : "DATABASE"; + setWorkflowList([]); + fetchWorkflows(connectorType).then((data) => { + if (connectorType === "DATABASE") { + fetchWorkflows("MANUALREVIEW").then((manualReviewData) => { + const combinedData = [...data, ...manualReviewData]; + setWorkflowList(combinedData); + }); + } else { + setWorkflowList(data); + } + }); }; useEffect(() => { From eb64aabb8a8712a3135bc26e0d2dbe97007c1a97 Mon Sep 17 00:00:00 2001 From: harini-venkataraman <115449948+harini-venkataraman@users.noreply.github.com> Date: Wed, 31 Jul 2024 10:30:07 +0530 Subject: [PATCH 4/4] [FIX] Removing preamble from embedding retrieval prompt (#539) * Removing preamble from embedding retrival prompt * retaining node score to 0 --------- Co-authored-by: Deepak K <89829542+Deepak-Kesavan@users.noreply.github.com> --- .../src/unstract/prompt_service/main.py | 34 ++----------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py index 6a06b90e1..f9a05506a 100644 --- a/prompt-service/src/unstract/prompt_service/main.py +++ b/prompt-service/src/unstract/prompt_service/main.py @@ -129,31 +129,6 @@ def construct_prompt( return prompt -def construct_prompt_for_engine( - preamble: str, - prompt: str, - postamble: str, - grammar_list: list[dict[str, Any]], -) -> str: - # Let's cleanup the context. Remove if 3 consecutive newlines are found - prompt = f"{preamble}\n\nQuestion or Instruction: {prompt}\n" - if grammar_list is not None and len(grammar_list) > 0: - prompt += "\n" - for grammar in grammar_list: - word = "" - synonyms = [] - if PSKeys.WORD in grammar: - word = grammar[PSKeys.WORD] - if PSKeys.SYNONYMS in grammar: - synonyms = grammar[PSKeys.SYNONYMS] - if len(synonyms) > 0 and word != "": - prompt += f'\nNote: You can consider that the word {word} is same as \ - {", ".join(synonyms)} in both the quesiton and the context.' # noqa - prompt += f"\n\n{postamble}" - prompt += "\n\n" - return prompt - - def authentication_middleware(func: Any) -> Any: def wrapper(*args: Any, **kwargs: Any) -> Any: token = AuthenticationMiddleware.get_token_from_auth_header(request) @@ -703,13 +678,8 @@ def run_retrieval( # type:ignore vector_index, retrieval_type: str, ) -> tuple[str, str]: - prompt = construct_prompt_for_engine( - preamble=tool_settings.get(PSKeys.PREAMBLE, ""), - prompt=output[PSKeys.PROMPTX], - postamble=tool_settings.get(PSKeys.POSTAMBLE, ""), - grammar_list=tool_settings.get(PSKeys.GRAMMAR, []), - ) - if retrieval_type is PSKeys.SUBQUESTION: + prompt = output[PSKeys.PROMPTX] + if retrieval_type == PSKeys.SUBQUESTION: subq_prompt = ( f"Generate a sub-question from the following verbose prompt that will" f" help extract relevant documents from a vector store:\n\n{prompt}"