refactor: split genai lambda layer into openai and langchain

lpm0073 · lpm0073 · commit 3d9292d25ef7 · 2024-01-22T15:30:37.000-06:00
diff --git a/api/terraform/lambda_info.tf b/api/terraform/lambda_info.tf
@@ -72,7 +72,7 @@ resource "aws_lambda_function" "info" {
   handler          = "openai_api.lambda_info.lambda_handler.handler"
   filename         = data.archive_file.lambda_info.output_path
   source_code_hash = data.archive_file.lambda_info.output_base64sha256
-  layers           = [aws_lambda_layer_version.genai.arn]
+  layers           = [aws_lambda_layer_version.openai.arn]
   tags             = var.tags
 
   environment {
diff --git a/api/terraform/lambda_langchain.tf b/api/terraform/lambda_langchain.tf
@@ -79,7 +79,7 @@ resource "aws_lambda_function" "lambda_langchain" {
   architectures    = var.compatible_architectures
   filename         = data.archive_file.lambda_langchain.output_path
   source_code_hash = data.archive_file.lambda_langchain.output_base64sha256
-  layers           = [aws_lambda_layer_version.genai.arn]
+  layers           = [aws_lambda_layer_version.openai.arn, aws_lambda_layer_version.langchain.arn]
   tags             = var.tags
 
   environment {
diff --git a/api/terraform/lambda_layer_genai.tf b/api/terraform/lambda_layer_genai.tf
diff --git a/api/terraform/lambda_layer_langchain.tf b/api/terraform/lambda_layer_langchain.tf
@@ -0,0 +1,62 @@
+#------------------------------------------------------------------------------
+# written by: Lawrence McDaniel
+#             https://lawrencemcdaniel.com/
+#
+# date:   sep-2023
+#
+# usage:  implement a Python Lambda layer with a virtual environment
+#         that includes the packages required by langchain API and LangChain.
+#------------------------------------------------------------------------------
+locals {
+  langchain_layer_slug              = "langchain"
+  langchain_layer_name              = "layer_${local.langchain_layer_slug}"
+  langchain_layer_parent_directory  = "${path.module}/python"
+  langchain_layer_source_directory  = "${local.langchain_layer_parent_directory}/${local.langchain_layer_name}"
+  langchain_layer_packaging_script  = "${local.langchain_layer_source_directory}/create_container.sh"
+  langchain_layer_package_folder    = local.langchain_layer_slug
+  langchain_layer_dist_build_path   = "${path.module}/build/"
+  langchain_layer_dist_package_name = "${local.langchain_layer_name}_dst.zip"
+}
+
+###############################################################################
+# Python package
+# https://alek-cora-glez.medium.com/deploying-aws-lambda-function-with-terraform-custom-dependencies-7874407cd4fc
+###############################################################################
+resource "null_resource" "package_langchain_layer" {
+  triggers = {
+    redeployment = sha1(jsonencode([
+      "${path.module}/lambda_layer.tf",
+      file("${local.langchain_layer_packaging_script}"),
+      file("${local.langchain_layer_source_directory}/Dockerfile"),
+      file("${local.langchain_layer_source_directory}/create_container.sh"),
+      file("${local.langchain_layer_source_directory}/requirements.txt"),
+      fileexists("${local.langchain_layer_source_directory}/${local.langchain_layer_dist_package_name}") ? filebase64("${local.langchain_layer_source_directory}/${local.langchain_layer_dist_package_name}") : null
+    ]))
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash"]
+    command     = local.langchain_layer_packaging_script
+
+    environment = {
+      SOURCE_CODE_PATH = local.langchain_layer_source_directory
+      RUNTIME          = var.lambda_python_runtime
+      CONTAINER_NAME   = local.langchain_layer_name
+      PACKAGE_NAME     = local.langchain_layer_dist_package_name
+    }
+  }
+}
+
+resource "aws_lambda_layer_version" "langchain" {
+  filename                 = "${local.langchain_layer_source_directory}/${local.langchain_layer_dist_package_name}"
+  source_code_hash         = fileexists("${local.langchain_layer_source_directory}/${local.langchain_layer_dist_package_name}") ? filebase64sha256("${local.langchain_layer_source_directory}/${local.langchain_layer_dist_package_name}") : "default"
+  layer_name               = local.langchain_layer_slug
+  compatible_architectures = var.compatible_architectures
+  compatible_runtimes      = [var.lambda_python_runtime]
+  lifecycle {
+    create_before_destroy = true
+  }
+  depends_on = [
+    null_resource.package_langchain_layer
+  ]
+}
diff --git a/api/terraform/lambda_layer_nlp.tf b/api/terraform/lambda_layer_nlp.tf
@@ -30,7 +30,7 @@ resource "null_resource" "package_nlp_layer" {
       file("${local.nlp_layer_source_directory}/Dockerfile"),
       file("${local.nlp_layer_source_directory}/create_container.sh"),
       file("${local.nlp_layer_source_directory}/requirements.txt"),
-      fileexists("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") ? filebase64("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") : null
+      fileexists("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") ? filebase64("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") : "default"
     ]))
   }
 
diff --git a/api/terraform/lambda_layer_openai.tf b/api/terraform/lambda_layer_openai.tf
@@ -0,0 +1,62 @@
+#------------------------------------------------------------------------------
+# written by: Lawrence McDaniel
+#             https://lawrencemcdaniel.com/
+#
+# date:   sep-2023
+#
+# usage:  implement a Python Lambda layer with a virtual environment
+#         that includes the packages required by OpenAI API and LangChain.
+#------------------------------------------------------------------------------
+locals {
+  openai_layer_slug              = "openai"
+  openai_layer_name              = "layer_${local.openai_layer_slug}"
+  openai_layer_parent_directory  = "${path.module}/python"
+  openai_layer_source_directory  = "${local.openai_layer_parent_directory}/${local.openai_layer_name}"
+  openai_layer_packaging_script  = "${local.openai_layer_source_directory}/create_container.sh"
+  openai_layer_package_folder    = local.openai_layer_slug
+  openai_layer_dist_build_path   = "${path.module}/build/"
+  openai_layer_dist_package_name = "${local.openai_layer_name}_dst.zip"
+}
+
+###############################################################################
+# Python package
+# https://alek-cora-glez.medium.com/deploying-aws-lambda-function-with-terraform-custom-dependencies-7874407cd4fc
+###############################################################################
+resource "null_resource" "package_openai_layer" {
+  triggers = {
+    redeployment = sha1(jsonencode([
+      "${path.module}/lambda_layer.tf",
+      file("${local.openai_layer_packaging_script}"),
+      file("${local.openai_layer_source_directory}/Dockerfile"),
+      file("${local.openai_layer_source_directory}/create_container.sh"),
+      file("${local.openai_layer_source_directory}/requirements.txt"),
+      fileexists("${local.openai_layer_source_directory}/${local.openai_layer_dist_package_name}") ? filebase64("${local.openai_layer_source_directory}/${local.openai_layer_dist_package_name}") : "default"
+    ]))
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash"]
+    command     = local.openai_layer_packaging_script
+
+    environment = {
+      SOURCE_CODE_PATH = local.openai_layer_source_directory
+      RUNTIME          = var.lambda_python_runtime
+      CONTAINER_NAME   = local.openai_layer_name
+      PACKAGE_NAME     = local.openai_layer_dist_package_name
+    }
+  }
+}
+
+resource "aws_lambda_layer_version" "openai" {
+  filename                 = "${local.openai_layer_source_directory}/${local.openai_layer_dist_package_name}"
+  source_code_hash         = fileexists("${local.openai_layer_source_directory}/${local.openai_layer_dist_package_name}") ? filebase64sha256("${local.openai_layer_source_directory}/${local.openai_layer_dist_package_name}") : null
+  layer_name               = local.openai_layer_slug
+  compatible_architectures = var.compatible_architectures
+  compatible_runtimes      = [var.lambda_python_runtime]
+  lifecycle {
+    create_before_destroy = true
+  }
+  depends_on = [
+    null_resource.package_openai_layer
+  ]
+}
diff --git a/api/terraform/lambda_openai_function.tf b/api/terraform/lambda_openai_function.tf
@@ -79,7 +79,7 @@ resource "aws_lambda_function" "lambda_openai_function" {
   architectures    = var.compatible_architectures
   filename         = data.archive_file.lambda_openai_function.output_path
   source_code_hash = data.archive_file.lambda_openai_function.output_base64sha256
-  layers           = [aws_lambda_layer_version.genai.arn, aws_lambda_layer_version.nlp.arn]
+  layers           = [aws_lambda_layer_version.openai.arn, aws_lambda_layer_version.nlp.arn]
   tags             = var.tags
 
   environment {
diff --git a/api/terraform/lambda_openai_v2.tf b/api/terraform/lambda_openai_v2.tf
@@ -79,7 +79,7 @@ resource "aws_lambda_function" "lambda_openai_v2" {
   architectures    = var.compatible_architectures
   filename         = data.archive_file.lambda_openai_v2.output_path
   source_code_hash = data.archive_file.lambda_openai_v2.output_base64sha256
-  layers           = [aws_lambda_layer_version.genai.arn]
+  layers           = [aws_lambda_layer_version.openai.arn]
   tags             = var.tags
 
   environment {
diff --git a/api/terraform/python/layer_langchain/.gitignore b/api/terraform/python/layer_langchain/.gitignore
diff --git a/api/terraform/python/layer_langchain/Dockerfile b/api/terraform/python/layer_langchain/Dockerfile
diff --git a/api/terraform/python/layer_langchain/README.md b/api/terraform/python/layer_langchain/README.md
diff --git a/api/terraform/python/layer_langchain/create_container.sh b/api/terraform/python/layer_langchain/create_container.sh
diff --git a/api/terraform/python/layer_langchain/requirements.txt b/api/terraform/python/layer_langchain/requirements.txt
@@ -0,0 +1,15 @@
+# -----------------------------------------------------------------------------
+# written by: Lawrence McDaniel
+#             https://lawrencemcdaniel.com
+#
+# usage:      Shared Python requirements for AWS Lambda functions.
+#             Create a virtual environment in the root of this repository
+#             named `venv`. Terraform modules will look for and include these
+#             requirements in the zip package for this layer.
+# -----------------------------------------------------------------------------
+
+# generative AI requirements
+# --------------------------
+langchain
+#langchain-experimental
+langchain-openai==0.0.3
diff --git a/api/terraform/python/layer_nlp/Dockerfile b/api/terraform/python/layer_nlp/Dockerfile
@@ -9,8 +9,5 @@ WORKDIR /var/task
 
 COPY requirements.txt .
 
-ENV PYTHONPATH "${PYTHONPATH}:python/lib/python3.11/site-packages"
-
 RUN yum install -y zip
 RUN pip install -r requirements.txt --target python/lib/python3.11/site-packages
-RUN python3 -m spacy download en_core_web_sm
diff --git a/api/terraform/python/layer_nlp/requirements.txt b/api/terraform/python/layer_nlp/requirements.txt
@@ -11,5 +11,4 @@
 # NLP requirements
 # --------------------------
 python-Levenshtein==0.23.0
-spacy==3.7.2
 pyyaml
diff --git a/api/terraform/python/layer_openai/.gitignore b/api/terraform/python/layer_openai/.gitignore
@@ -0,0 +1,3 @@
+*.zip
+venv
+archive
diff --git a/api/terraform/python/layer_openai/Dockerfile b/api/terraform/python/layer_openai/Dockerfile
@@ -0,0 +1,13 @@
+# Use an AWS Lambda Python runtime as the base image
+# https://hub.docker.com/r/amazon/aws-lambda-python
+# ------------------------------------------------------
+
+# Stage 1: Build dependencies for x86_64
+FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.11
+
+WORKDIR /var/task
+
+COPY requirements.txt .
+
+RUN yum install -y zip
+RUN pip install -r requirements.txt --target python/lib/python3.11/site-packages
diff --git a/api/terraform/python/layer_openai/README.md b/api/terraform/python/layer_openai/README.md
@@ -0,0 +1,3 @@
+# AWS Lambda Layer for OpenAI/Langchain Lambdas
+
+This layer contains the combined pip requirements for both lambda_langchain and lambda_openai_v2.
diff --git a/api/terraform/python/layer_openai/create_container.sh b/api/terraform/python/layer_openai/create_container.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#------------------------------------------------------------------------------
+# written by: Lawrence McDaniel
+#             https://lawrencemcdaniel.com/
+#
+# date:   nov-2023
+#
+# usage:  Lambda Python packaging tool.
+#         Called by Terraform "null_resource". Copies python
+#         module(s) plus any requirements to a dedicated folder so that
+#         it can be archived to a zip file for upload to
+#         AWS Lambda by Terraform.
+#------------------------------------------------------------------------------
+cd $SOURCE_CODE_PATH
+
+
+docker build -t $CONTAINER_NAME .
+docker rm $CONTAINER_NAME
+docker run --name $CONTAINER_NAME --entrypoint /bin/bash $CONTAINER_NAME -c "zip -r $PACKAGE_NAME ."
+
+# Delete the distribution package if it exists
+if [ -f $PACKAGE_NAME ]; then
+    rm $PACKAGE_NAME
+fi
+
+docker cp $CONTAINER_NAME:/var/task/$PACKAGE_NAME .
diff --git a/api/terraform/python/layer_openai/requirements.txt b/api/terraform/python/layer_openai/requirements.txt
@@ -10,9 +10,6 @@
 
 # generative AI requirements
 # --------------------------
-langchain
-langchain-experimental
-langchain-openai==0.0.3
 openai==1.7.2
 
 # general requirements
diff --git a/api/terraform/python/openai_api/lambda_openai_function/natural_language_processing.py b/api/terraform/python/openai_api/lambda_openai_function/natural_language_processing.py
@@ -5,27 +5,18 @@
 import string
 
 import Levenshtein
-import spacy
-
-
-nlp = spacy.load("en_core_web_sm")
 
 
 def clean_prompt(prompt: str) -> str:
     """
     Clean up the prompt by adding spaces before capital letters.
     Example: converts "WhoIsLawrenceMcDaniel" to "Who Is Lawrence McDaniel"
     """
+    pattern = r"(?<!Mc)([A-Z][a-z]+)|(?<!Mc)([A-Z]+)"
     retval = []
     for word in prompt.split():
         word = word.translate(str.maketrans("", "", string.punctuation))
-        doc = nlp(word)
-        words = re.sub("([A-Z][a-z]+)", r" \1", re.sub("([A-Z]+)", r" \1", word)).split()
-        words = (
-            ["".join(re.findall("[a-zA-Z]+", w)) for w in words]
-            if not any(ent.label_ in ["PERSON", "ORG"] for ent in doc.ents)
-            else [word.title()]
-        )
+        words = re.sub(pattern, r" \1\2", word).split()
         retval.extend(words)
     retval = " ".join(retval)
     return retval
diff --git a/requirements.txt b/requirements.txt
@@ -30,17 +30,24 @@ pydocstringformatter==0.7.3
 tox==4.12.0
 codespell==2.2.6
 
-# production
+# Lambda layer: openai
 # ------------
-python-dotenv==1.0.0
 openai==1.7.2
+
+# Lambda layer: langchain
+# ------------
 langchain
-langchain-experimental
+#langchain-experimental
 langchain-openai==0.0.3
+
+# Lambda layer: common
+# ------------
+python-dotenv==1.0.0
 pydantic==2.5.3
 pydantic-settings==2.1.0
-python-dotenv==1.0.0
 python-hcl2==4.3.2
+
+# Lambda layer: nlp
+# ------------
 python-Levenshtein==0.23.0
-spacy==3.7.2
 pyyaml

Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,7 @@ resource "null_resource" "package_nlp_layer" {`
`30`	`30`	`file("${local.nlp_layer_source_directory}/Dockerfile"),`
`31`	`31`	`file("${local.nlp_layer_source_directory}/create_container.sh"),`
`32`	`32`	`file("${local.nlp_layer_source_directory}/requirements.txt"),`
`33`		`- fileexists("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") ? filebase64("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") : null`
	`33`	`+ fileexists("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") ? filebase64("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") : "default"`
`34`	`34`	`]))`
`35`	`35`	`}`
`36`	`36`