From fc58359ea26af03b0b5a04eb881749790ed7598f Mon Sep 17 00:00:00 2001 From: LironEr Date: Fri, 17 Jan 2025 10:01:20 +0200 Subject: [PATCH 1/4] feat: Support passing extra args to pip install --- README.md | 1 + package.py | 28 ++++++++++++++++------------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ed1f32c8..704716e6 100644 --- a/README.md +++ b/README.md @@ -458,6 +458,7 @@ source_path = [ - `:zip [source] [destination]` is a special command which creates content of current working directory (first argument) and places it inside of path (second argument). - `pip_requirements` - Controls whether to execute `pip install`. Set to `false` to disable this feature, `true` to run `pip install` with `requirements.txt` found in `path`. Or set to another filename which you want to use instead. When `source_path` is passed as a string containing a path (and not a list of maps), and `requirements.txt` is present, `pip install` is automatically executed. - `pip_tmp_dir` - Set the base directory to make the temporary directory for pip installs. Can be useful for Docker in Docker builds. +- `pip_install_extra_args` - A list of additional pip arguments to add to the pip install command - `poetry_install` - Controls whether to execute `poetry export` and `pip install`. Set to `false` to disable this feature, `true` to run `poetry export` with `pyproject.toml` and `poetry.lock` found in `path`. When `source_path` is passed as a string containing a path (and not a list of maps), and `pyproject.toml` with a build system `poetry` is present, `poetry export` and `pip install` are automatically executed. - `poetry_export_extra_args` - A list of additional poetry arguments to add to the poetry export command - `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead. diff --git a/package.py b/package.py index 6e19846c..1497c4d7 100644 --- a/package.py +++ b/package.py @@ -687,7 +687,7 @@ def step(*x): def hash(path): source_paths.append(path) - def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): + def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None, pip_install_extra_args=[]): command = runtime requirements = path if os.path.isdir(path): @@ -703,11 +703,11 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): "available in system PATH".format(command) ) - step("pip", runtime, requirements, prefix, tmp_dir) + step("pip", runtime, requirements, prefix, tmp_dir, pip_install_extra_args) hash(requirements) def poetry_install_step( - path, poetry_export_extra_args=[], prefix=None, required=False, tmp_dir=None + path, poetry_export_extra_args=[], prefix=None, required=False, tmp_dir=None, pip_install_extra_args=[] ): pyproject_file = path if os.path.isdir(path): @@ -718,7 +718,7 @@ def poetry_install_step( "poetry configuration not found: {}".format(pyproject_file) ) else: - step("poetry", runtime, path, poetry_export_extra_args, prefix, tmp_dir) + step("poetry", runtime, path, poetry_export_extra_args, prefix, tmp_dir, pip_install_extra_args) hash(pyproject_file) pyproject_path = os.path.dirname(pyproject_file) poetry_lock_file = os.path.join(pyproject_path, "poetry.lock") @@ -819,6 +819,7 @@ def commands_step(path, commands): else: prefix = claim.get("prefix_in_zip") pip_requirements = claim.get("pip_requirements") + pip_install_extra_args = claim.get("pip_install_extra_args", []) poetry_install = claim.get("poetry_install") poetry_export_extra_args = claim.get("poetry_export_extra_args", []) npm_requirements = claim.get( @@ -833,6 +834,7 @@ def commands_step(path, commands): prefix, required=True, tmp_dir=claim.get("pip_tmp_dir"), + pip_install_extra_args=pip_install_extra_args, ) else: pip_requirements_step( @@ -840,6 +842,7 @@ def commands_step(path, commands): prefix, required=True, tmp_dir=claim.get("pip_tmp_dir"), + pip_install_extra_args=pip_install_extra_args, ) if poetry_install and runtime.startswith("python"): @@ -850,6 +853,7 @@ def commands_step(path, commands): poetry_export_extra_args=poetry_export_extra_args, required=True, tmp_dir=claim.get("poetry_tmp_dir"), + pip_install_extra_args=pip_install_extra_args, ) if npm_requirements and runtime.startswith("nodejs"): @@ -937,9 +941,9 @@ def execute(self, build_plan, zip_stream, query): else: zs.write_file(source_path, prefix=prefix, timestamp=ts) elif cmd == "pip": - runtime, pip_requirements, prefix, tmp_dir = action[1:] + runtime, pip_requirements, prefix, tmp_dir, pip_install_extra_args = action[1:] with install_pip_requirements( - query, pip_requirements, tmp_dir + query, pip_requirements, tmp_dir, pip_install_extra_args ) as rd: if rd: if pf: @@ -950,12 +954,12 @@ def execute(self, build_plan, zip_stream, query): # XXX: timestamp=0 - what actually do with it? zs.write_dirs(rd, prefix=prefix, timestamp=0) elif cmd == "poetry": - (runtime, path, poetry_export_extra_args, prefix, tmp_dir) = action[ + (runtime, path, poetry_export_extra_args, prefix, tmp_dir, pip_install_extra_args) = action[ 1: ] log.info("poetry_export_extra_args: %s", poetry_export_extra_args) with install_poetry_dependencies( - query, path, poetry_export_extra_args, tmp_dir + query, path, poetry_export_extra_args, tmp_dir, pip_install_extra_args ) as rd: if rd: if pf: @@ -1048,7 +1052,7 @@ def _zip_write_with_filter( @contextmanager -def install_pip_requirements(query, requirements_file, tmp_dir): +def install_pip_requirements(query, requirements_file, tmp_dir, pip_install_extra_args): # TODO: # 1. Emit files instead of temp_dir @@ -1125,7 +1129,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir): "--prefix=", "--target=.", "--requirement={}".format(requirements_filename), - ] + ] + pip_install_extra_args if docker: with_ssh_agent = docker.with_ssh_agent pip_cache_dir = docker.docker_pip_cache @@ -1175,7 +1179,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir): @contextmanager -def install_poetry_dependencies(query, path, poetry_export_extra_args, tmp_dir): +def install_poetry_dependencies(query, path, poetry_export_extra_args, tmp_dir, pip_install_extra_args): # TODO: # 1. Emit files instead of temp_dir @@ -1301,7 +1305,7 @@ def copy_file_to_target(file, temp_dir): "--prefix=", "--target=.", "--requirement=requirements.txt", - ], + ] + pip_install_extra_args, ] if docker: with_ssh_agent = docker.with_ssh_agent From f5f45109b46ffa3fdf5be0fe4f51febb3a5ba12d Mon Sep 17 00:00:00 2001 From: LironEr Date: Fri, 17 Jan 2025 10:14:05 +0200 Subject: [PATCH 2/4] run precommit --- modules/deploy/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/deploy/README.md b/modules/deploy/README.md index 6da1f6e8..f5f5d8ec 100644 --- a/modules/deploy/README.md +++ b/modules/deploy/README.md @@ -151,10 +151,10 @@ No modules. | [attach\_hooks\_policy](#input\_attach\_hooks\_policy) | Whether to attach Invoke policy to CodeDeploy role when before allow traffic or after allow traffic hooks are defined. | `bool` | `true` | no | | [attach\_triggers\_policy](#input\_attach\_triggers\_policy) | Whether to attach SNS policy to CodeDeploy role when triggers are defined | `bool` | `false` | no | | [auto\_rollback\_enabled](#input\_auto\_rollback\_enabled) | Indicates whether a defined automatic rollback configuration is currently enabled for this Deployment Group. | `bool` | `true` | no | -| [auto\_rollback\_events](#input\_auto\_rollback\_events) | List of event types that trigger a rollback. Supported types are DEPLOYMENT\_FAILURE and DEPLOYMENT\_STOP\_ON\_ALARM. | `list(string)` | 
[
  "DEPLOYMENT_STOP_ON_ALARM"
]
| no | +| [auto\_rollback\_events](#input\_auto\_rollback\_events) | List of event types that trigger a rollback. Supported types are DEPLOYMENT\_FAILURE and DEPLOYMENT\_STOP\_ON\_ALARM. | `list(string)` | 
[
  "DEPLOYMENT_STOP_ON_ALARM"
]
| no | | [aws\_cli\_command](#input\_aws\_cli\_command) | Command to run as AWS CLI. May include extra arguments like region and profile. | `string` | `"aws"` | no | | [before\_allow\_traffic\_hook\_arn](#input\_before\_allow\_traffic\_hook\_arn) | ARN of Lambda function to execute before allow traffic during deployment. This function should be named CodeDeployHook\_, to match the managed AWSCodeDeployForLambda policy, unless you're using a custom role | `string` | `""` | no | -| [codedeploy\_principals](#input\_codedeploy\_principals) | List of CodeDeploy service principals to allow. The list can include global or regional endpoints. | `list(string)` | 
[
  "codedeploy.amazonaws.com"
]
| no | +| [codedeploy\_principals](#input\_codedeploy\_principals) | List of CodeDeploy service principals to allow. The list can include global or regional endpoints. | `list(string)` | 
[
  "codedeploy.amazonaws.com"
]
| no | | [codedeploy\_role\_name](#input\_codedeploy\_role\_name) | IAM role name to create or use by CodeDeploy | `string` | `""` | no | | [create](#input\_create) | Controls whether resources should be created | `bool` | `true` | no | | [create\_app](#input\_create\_app) | Whether to create new AWS CodeDeploy app | `bool` | `false` | no | @@ -168,7 +168,7 @@ No modules. | [force\_deploy](#input\_force\_deploy) | Force deployment every time (even when nothing changes) | `bool` | `false` | no | | [function\_name](#input\_function\_name) | The name of the Lambda function to deploy | `string` | `""` | no | | [get\_deployment\_sleep\_timer](#input\_get\_deployment\_sleep\_timer) | Adds additional sleep time to get-deployment command to avoid the service throttling | `number` | `5` | no | -| [interpreter](#input\_interpreter) | List of interpreter arguments used to execute deploy script, first arg is path | `list(string)` | 
[
  "/bin/bash",
  "-c"
]
| no | +| [interpreter](#input\_interpreter) | List of interpreter arguments used to execute deploy script, first arg is path | `list(string)` | 
[
  "/bin/bash",
  "-c"
]
| no | | [run\_deployment](#input\_run\_deployment) | Run AWS CLI command to start the deployment | `bool` | `false` | no | | [save\_deploy\_script](#input\_save\_deploy\_script) | Save deploy script locally | `bool` | `false` | no | | [tags](#input\_tags) | A map of tags to assign to resources. | `map(string)` | `{}` | no | From 309eb12911ddb4c62cffa61bda5c7373c54831c1 Mon Sep 17 00:00:00 2001 From: LironEr Date: Fri, 17 Jan 2025 10:21:00 +0200 Subject: [PATCH 3/4] ruff fix --- package.py | 62 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/package.py b/package.py index 1497c4d7..1d1c3541 100644 --- a/package.py +++ b/package.py @@ -687,7 +687,9 @@ def step(*x): def hash(path): source_paths.append(path) - def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None, pip_install_extra_args=[]): + def pip_requirements_step( + path, prefix=None, required=False, tmp_dir=None, pip_install_extra_args=[] + ): command = runtime requirements = path if os.path.isdir(path): @@ -703,11 +705,23 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None, pip_i "available in system PATH".format(command) ) - step("pip", runtime, requirements, prefix, tmp_dir, pip_install_extra_args) + step( + "pip", + runtime, + requirements, + prefix, + tmp_dir, + pip_install_extra_args, + ) hash(requirements) def poetry_install_step( - path, poetry_export_extra_args=[], prefix=None, required=False, tmp_dir=None, pip_install_extra_args=[] + path, + poetry_export_extra_args=[], + prefix=None, + required=False, + tmp_dir=None, + pip_install_extra_args=[], ): pyproject_file = path if os.path.isdir(path): @@ -718,7 +732,15 @@ def poetry_install_step( "poetry configuration not found: {}".format(pyproject_file) ) else: - step("poetry", runtime, path, poetry_export_extra_args, prefix, tmp_dir, pip_install_extra_args) + step( + "poetry", + runtime, + path, + poetry_export_extra_args, + prefix, + tmp_dir, + pip_install_extra_args, + ) hash(pyproject_file) pyproject_path = os.path.dirname(pyproject_file) poetry_lock_file = os.path.join(pyproject_path, "poetry.lock") @@ -941,7 +963,13 @@ def execute(self, build_plan, zip_stream, query): else: zs.write_file(source_path, prefix=prefix, timestamp=ts) elif cmd == "pip": - runtime, pip_requirements, prefix, tmp_dir, pip_install_extra_args = action[1:] + ( + runtime, + pip_requirements, + prefix, + tmp_dir, + pip_install_extra_args, + ) = action[1:] with install_pip_requirements( query, pip_requirements, tmp_dir, pip_install_extra_args ) as rd: @@ -954,12 +982,21 @@ def execute(self, build_plan, zip_stream, query): # XXX: timestamp=0 - what actually do with it? zs.write_dirs(rd, prefix=prefix, timestamp=0) elif cmd == "poetry": - (runtime, path, poetry_export_extra_args, prefix, tmp_dir, pip_install_extra_args) = action[ - 1: - ] + ( + runtime, + path, + poetry_export_extra_args, + prefix, + tmp_dir, + pip_install_extra_args, + ) = action[1:] log.info("poetry_export_extra_args: %s", poetry_export_extra_args) with install_poetry_dependencies( - query, path, poetry_export_extra_args, tmp_dir, pip_install_extra_args + query, + path, + poetry_export_extra_args, + tmp_dir, + pip_install_extra_args, ) as rd: if rd: if pf: @@ -1179,7 +1216,9 @@ def install_pip_requirements(query, requirements_file, tmp_dir, pip_install_extr @contextmanager -def install_poetry_dependencies(query, path, poetry_export_extra_args, tmp_dir, pip_install_extra_args): +def install_poetry_dependencies( + query, path, poetry_export_extra_args, tmp_dir, pip_install_extra_args +): # TODO: # 1. Emit files instead of temp_dir @@ -1305,7 +1344,8 @@ def copy_file_to_target(file, temp_dir): "--prefix=", "--target=.", "--requirement=requirements.txt", - ] + pip_install_extra_args, + ] + + pip_install_extra_args, ] if docker: with_ssh_agent = docker.with_ssh_agent From e0e07b96d90ccd00853f8029b5283e96d0c6ab90 Mon Sep 17 00:00:00 2001 From: LironEr Date: Fri, 17 Jan 2025 10:23:30 +0200 Subject: [PATCH 4/4] Revert "run precommit" This reverts commit f5f45109b46ffa3fdf5be0fe4f51febb3a5ba12d. --- modules/deploy/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/deploy/README.md b/modules/deploy/README.md index f5f5d8ec..6da1f6e8 100644 --- a/modules/deploy/README.md +++ b/modules/deploy/README.md @@ -151,10 +151,10 @@ No modules. | [attach\_hooks\_policy](#input\_attach\_hooks\_policy) | Whether to attach Invoke policy to CodeDeploy role when before allow traffic or after allow traffic hooks are defined. | `bool` | `true` | no | | [attach\_triggers\_policy](#input\_attach\_triggers\_policy) | Whether to attach SNS policy to CodeDeploy role when triggers are defined | `bool` | `false` | no | | [auto\_rollback\_enabled](#input\_auto\_rollback\_enabled) | Indicates whether a defined automatic rollback configuration is currently enabled for this Deployment Group. | `bool` | `true` | no | -| [auto\_rollback\_events](#input\_auto\_rollback\_events) | List of event types that trigger a rollback. Supported types are DEPLOYMENT\_FAILURE and DEPLOYMENT\_STOP\_ON\_ALARM. | `list(string)` | 
[
  "DEPLOYMENT_STOP_ON_ALARM"
]
| no | +| [auto\_rollback\_events](#input\_auto\_rollback\_events) | List of event types that trigger a rollback. Supported types are DEPLOYMENT\_FAILURE and DEPLOYMENT\_STOP\_ON\_ALARM. | `list(string)` | 
[
  "DEPLOYMENT_STOP_ON_ALARM"
]
| no | | [aws\_cli\_command](#input\_aws\_cli\_command) | Command to run as AWS CLI. May include extra arguments like region and profile. | `string` | `"aws"` | no | | [before\_allow\_traffic\_hook\_arn](#input\_before\_allow\_traffic\_hook\_arn) | ARN of Lambda function to execute before allow traffic during deployment. This function should be named CodeDeployHook\_, to match the managed AWSCodeDeployForLambda policy, unless you're using a custom role | `string` | `""` | no | -| [codedeploy\_principals](#input\_codedeploy\_principals) | List of CodeDeploy service principals to allow. The list can include global or regional endpoints. | `list(string)` | 
[
  "codedeploy.amazonaws.com"
]
| no | +| [codedeploy\_principals](#input\_codedeploy\_principals) | List of CodeDeploy service principals to allow. The list can include global or regional endpoints. | `list(string)` | 
[
  "codedeploy.amazonaws.com"
]
| no | | [codedeploy\_role\_name](#input\_codedeploy\_role\_name) | IAM role name to create or use by CodeDeploy | `string` | `""` | no | | [create](#input\_create) | Controls whether resources should be created | `bool` | `true` | no | | [create\_app](#input\_create\_app) | Whether to create new AWS CodeDeploy app | `bool` | `false` | no | @@ -168,7 +168,7 @@ No modules. | [force\_deploy](#input\_force\_deploy) | Force deployment every time (even when nothing changes) | `bool` | `false` | no | | [function\_name](#input\_function\_name) | The name of the Lambda function to deploy | `string` | `""` | no | | [get\_deployment\_sleep\_timer](#input\_get\_deployment\_sleep\_timer) | Adds additional sleep time to get-deployment command to avoid the service throttling | `number` | `5` | no | -| [interpreter](#input\_interpreter) | List of interpreter arguments used to execute deploy script, first arg is path | `list(string)` | 
[
  "/bin/bash",
  "-c"
]
| no | +| [interpreter](#input\_interpreter) | List of interpreter arguments used to execute deploy script, first arg is path | `list(string)` | 
[
  "/bin/bash",
  "-c"
]
| no | | [run\_deployment](#input\_run\_deployment) | Run AWS CLI command to start the deployment | `bool` | `false` | no | | [save\_deploy\_script](#input\_save\_deploy\_script) | Save deploy script locally | `bool` | `false` | no | | [tags](#input\_tags) | A map of tags to assign to resources. | `map(string)` | `{}` | no |