Skip to content

Commit 9f49bbc

Browse files
Merge pull request #31 from epam/develop
Release 5.5.0, develop to main
2 parents b0e67fe + 5d934f4 commit 9f49bbc

File tree

130 files changed

+3520
-2311
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

130 files changed

+3520
-2311
lines changed

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,4 +251,5 @@ pip-selfcheck.json
251251

252252
.syndicate
253253
.tmp/
254-
**/.DS_Store
254+
**/.DS_Store
255+
.vscode/

CHANGELOG.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
66
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
77

8+
## [5.5.0] - 2024-08-07
9+
- return 429 status code if dynamodb provisioned capacity exceeded error
10+
- remove `accN` index from usage
11+
- scan each region in a separate process in order to reduce RAM usage
12+
- change `create_indexes` command. Now it ensures that indexes are up-to-date instead of recreating them
13+
- allow to build rulesets using rule comment field
14+
815
## [5.4.0] - 2024-07-09
916
- added `rule_source_id` and `excluded_rules` parameters to `POST /rulestets`.
1017
- added auto version resolving to all the `/rulesets` endpoints. Version parameters is optional

Makefile

Lines changed: 40 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,15 @@ AWS_REGION = $(shell aws configure get region)
1313

1414
EXECUTOR_IMAGE_NAME := rule-engine-executor # just dev image name
1515
EXECUTOR_IMAGE_TAG := latest
16-
SERVER_IMAGE_NAME := rule-engine
17-
SERVER_IMAGE_TAG := latest
16+
SERVER_IMAGE_NAME := public.ecr.aws/x4s4z8e1/syndicate/rule-engine
17+
SERVER_IMAGE_TAG ?= $(shell PYTHONPATH=./src python -B -c "from src.helpers.__version__ import __version__; print(__version__)")
1818

1919

2020
SYNDICATE_EXECUTABLE_PATH ?= $(shell which syndicate)
2121
SYNDICATE_CONFIG_PATH ?= .syndicate-config-main
2222
SYNDICATE_BUNDLE_NAME := custodian-service
2323

24+
HELM_REPO_NAME := syndicate
2425

2526
check-syndicate:
2627
@if [[ -z "$(SYNDICATE_EXECUTABLE_PATH)" ]]; then echo "No syndicate executable found"; exit 1; fi
@@ -80,17 +81,6 @@ fork-executor-image:
8081
# $(DOCKER_EXECUTABLE) build -t $(EXECUTOR_IMAGE_NAME):$(EXECUTOR_IMAGE_TAG) -f src/executor/Dockerfile --build-arg CUSTODIAN_SERVICE_PATH=custodian-as-a-service --build-arg CLOUD_CUSTODIAN_PATH=custodian-custom-core ..
8182

8283

83-
open-source-server-image:
84-
$(DOCKER_EXECUTABLE) build -t $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) -f src/onprem/Dockerfile-opensource .
85-
86-
fork-server-image:
87-
$(DOCKER_EXECUTABLE) build -t $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) -f src/onprem/Dockerfile .
88-
89-
90-
open-source-server-image-to-minikube:
91-
eval $(minikube -p minikube docker-env) && \
92-
$(DOCKER_EXECUTABLE) build -t $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) -f src/onprem/Dockerfile-opensource .
93-
9484
cli-dist:
9585
python -m pip install --upgrade build
9686
python -m build --sdist cli/
@@ -110,12 +100,6 @@ aws-ecr-push-executor:
110100
$(DOCKER_EXECUTABLE) push $$AWS_ACCOUNT_ID.dkr.ecr.$$AWS_REGION.amazonaws.com/$(EXECUTOR_IMAGE_NAME):$(EXECUTOR_IMAGE_TAG)
111101

112102

113-
aws-ecr-push-server:
114-
export AWS_REGION=$(AWS_REGION) AWS_ACCOUNT_ID=$(AWS_ACCOUNT_ID); \
115-
$(DOCKER_EXECUTABLE) tag $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) $$AWS_ACCOUNT_ID.dkr.ecr.$$AWS_REGION.amazonaws.com/$(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG); \
116-
$(DOCKER_EXECUTABLE) push $$AWS_ACCOUNT_ID.dkr.ecr.$$AWS_REGION.amazonaws.com/$(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)
117-
118-
119103
syndicate-update-lambdas: check-syndicate
120104
SDCT_CONF=$(SYNDICATE_CONFIG_PATH) $(SYNDICATE_EXECUTABLE_PATH) build --errors_allowed --bundle_name $(SYNDICATE_BUNDLE_NAME) -F
121105
SDCT_CONF=$(SYNDICATE_CONFIG_PATH) $(SYNDICATE_EXECUTABLE_PATH) update --update_only_types lambda --update_only_types lambda_layer --bundle_name $(SYNDICATE_BUNDLE_NAME) --replace_output
@@ -135,3 +119,40 @@ syndicate-update-api-gateway: check-syndicate
135119
syndicate-update-step-functions: check-syndicate
136120
# it does not remove the old api gateway
137121
SDCT_CONF=$(SYNDICATE_CONFIG_PATH) $(SYNDICATE_EXECUTABLE_PATH) deploy --deploy_only_types step_functions --replace_output --bundle_name $(SYNDICATE_BUNDLE_NAME)
122+
123+
124+
# images with fork which is default for now. Use src/onprem/Dockerfile-opensource for c7n from open source
125+
#make image-arm64
126+
#make image-amd64
127+
#make push-arm64
128+
#make push-amd64
129+
#make image-manifest
130+
#make push-manifest
131+
image-arm64:
132+
$(DOCKER_EXECUTABLE) build --platform linux/arm64 -t $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-arm64 -f src/onprem/Dockerfile .
133+
134+
image-amd64:
135+
$(DOCKER_EXECUTABLE) build --platform linux/amd64 -t $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-amd64 -f src/onprem/Dockerfile .
136+
137+
138+
image-manifest:
139+
-$(DOCKER_EXECUTABLE) manifest rm $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)
140+
$(DOCKER_EXECUTABLE) manifest create $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-arm64 $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-amd64
141+
$(DOCKER_EXECUTABLE) manifest annotate $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-arm64 --arch arm64
142+
$(DOCKER_EXECUTABLE) manifest annotate $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG) $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-amd64 --arch amd64
143+
144+
push-arm64:
145+
$(DOCKER_EXECUTABLE) push $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-arm64
146+
147+
148+
push-amd64:
149+
$(DOCKER_EXECUTABLE) push $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)-amd64
150+
151+
push-manifest:
152+
$(DOCKER_EXECUTABLE) manifest push $(SERVER_IMAGE_NAME):$(SERVER_IMAGE_TAG)
153+
154+
155+
push-helm-chart:
156+
helm package --dependency-update deployment/helm/rule-engine
157+
helm s3 push rule-engine-$(SERVER_IMAGE_TAG).tgz $(HELM_REPO_NAME)
158+
-rm rule-engine-$(SERVER_IMAGE_TAG).tgz

cli/CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
44
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
55
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
66

7+
## [5.5.0] - 2024-09-02
8+
- change job submit command to resolve credentials for tenant from all available places
9+
- add `--platform`, `--category`, `--service_section` and `--source` fields to `sre ruleset add`
10+
11+
## [5.4.1] - 2024-08-06
12+
- fix `inappropriate ioctl for device`
13+
- add environment variables: `SRE_CLI_RESPONSE_FORMAT`, `SRE_CLI_VERBOSE`, `SRE_CLI_NO_PROMPT`
14+
715
## [5.4.0] - 2024-07-09
816
- renamed `c7n` entrypoint to `sre`
917
- add 1 exit codes for all commands that failed

cli/api_module.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
22
"module_name": "srecli",
33
"cli_path": "/srecli/group",
4-
"mount_point": "/c7n"
4+
"mount_point": "/re"
55
}

cli/pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ dependencies = [
2121
sre = "srecli.group.sre:sre"
2222

2323
[tool.setuptools.dynamic]
24-
version = {attr = "srecli.version.__version__"}
24+
version = {attr = "srecli.__version__"}
2525

2626
[tool.setuptools.packages.find]
2727
where = ["."]

cli/srecli/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
2+
__version__ = '5.5.0'

cli/srecli/group/__init__.py

Lines changed: 26 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1+
import json
2+
import operator
3+
import shutil
4+
import sys
5+
import urllib.error
16
from abc import ABC, abstractmethod
27
from datetime import timezone
38
from functools import reduce, wraps
49
from http import HTTPStatus
5-
import operator
610
from itertools import islice
7-
import json
8-
import os
9-
import sys
1011
from pathlib import Path
1112
from typing import Any, Callable, TypedDict, cast
12-
import urllib.error
1313

1414
import click
1515
from dateutil.parser import isoparse
@@ -31,9 +31,13 @@
3131
NEXT_TOKEN_ATTR,
3232
NO_CONTENT_RESPONSE_MESSAGE,
3333
NO_ITEMS_TO_DISPLAY_RESPONSE_MESSAGE,
34-
JobType
34+
JobType,
35+
Env,
36+
MODULAR_ADMIN,
37+
STATUS_ATTR, SUCCESS_STATUS, ERROR_STATUS, CODE_ATTR, TABLE_TITLE_ATTR,
38+
REVERT_TO_JSON_MESSAGE, COLUMN_OVERFLOW
3539
)
36-
from srecli.service.logger import get_logger, get_user_logger, write_verbose_logs
40+
from srecli.service.logger import get_logger, enable_verbose_logs
3741

3842
CredentialsProvider = None
3943
try:
@@ -43,22 +47,7 @@
4347
pass
4448

4549

46-
# modular cli
47-
MODULAR_ADMIN = 'modules'
48-
SUCCESS_STATUS = 'SUCCESS'
49-
ERROR_STATUS = 'FAILED'
50-
STATUS_ATTR = 'status'
51-
CODE_ATTR = 'code'
52-
TABLE_TITLE_ATTR = 'table_title'
53-
# -----------
54-
5550
_LOG = get_logger(__name__)
56-
USER_LOG = get_user_logger(__name__)
57-
58-
REVERT_TO_JSON_MESSAGE = 'The command`s response is pretty huge and the ' \
59-
'result table structure can be broken.\nDo you want ' \
60-
'to show the response in the JSON format?'
61-
COLUMN_OVERFLOW = 'Column has overflown, within the table representation.'
6251

6352

6453
class TableException(Exception):
@@ -170,10 +159,12 @@ def wrapper(*args, **kwargs):
170159
if Path(__file__).parents[3].name == MODULAR_ADMIN: # TODO check some other way
171160
modular_mode = True
172161

173-
json_view = kwargs.pop('json')
174-
verbose = kwargs.pop('verbose')
162+
json_view = Env.RESPONSE_FORMAT.get() == 'json' or kwargs.get('json')
163+
verbose = Env.VERBOSE.get() or kwargs.get('verbose') # todo verbose can be enabled earlier if from env
164+
kwargs.pop('json', None)
165+
kwargs.pop('verbose', None)
175166
if verbose:
176-
write_verbose_logs()
167+
enable_verbose_logs()
177168
ctx = cast(click.Context, click.get_current_context())
178169
self.update_context(ctx)
179170
try:
@@ -204,7 +195,10 @@ def wrapper(*args, **kwargs):
204195
items_per_column=ctx.obj['config'].items_per_column,
205196
attributes_order=self._attributes_order
206197
)
207-
table = printer.print(prepared)
198+
table = printer.print(
199+
prepared,
200+
raise_on_overflow=not Env.NO_PROMPT.get()
201+
)
208202
except ColumnOverflow as ce:
209203

210204
_LOG.info(f'Awaiting user to respond to - {ce!r}.')
@@ -253,6 +247,8 @@ def format(self, resp: CustodianResponse) -> dict:
253247
if resp.code == HTTPStatus.NO_CONTENT:
254248
return {MESSAGE_ATTR: NO_CONTENT_RESPONSE_MESSAGE}
255249
elif isinstance(resp.exc, json.JSONDecodeError):
250+
if not resp.data and resp.code:
251+
return {MESSAGE_ATTR: resp.code.phrase}
256252
return {MESSAGE_ATTR: f'Invalid JSON received: {resp.exc.msg}'}
257253
elif isinstance(resp.exc, urllib.error.URLError):
258254
return {MESSAGE_ATTR: f'Cannot send a request: {resp.exc.reason}'}
@@ -308,7 +304,7 @@ def _format_er(e):
308304

309305
def format(self, resp: CustodianResponse) -> dict:
310306
base = {
311-
CODE_ATTR: resp.code,
307+
CODE_ATTR: resp.code or HTTPStatus.SERVICE_UNAVAILABLE.value,
312308
STATUS_ATTR: SUCCESS_STATUS if resp.ok else ERROR_STATUS,
313309
TABLE_TITLE_ATTR: self.modular_table_title
314310
}
@@ -399,7 +395,7 @@ def key(tpl):
399395
else:
400396
formatted = self._items_table(data)
401397

402-
overflow = formatted.index('\n') > os.get_terminal_size().columns
398+
overflow = formatted.index('\n') > shutil.get_terminal_size().columns
403399
if overflow and raise_on_overflow:
404400
raise ColumnOverflow(table=formatted)
405401
return formatted
@@ -465,14 +461,14 @@ def response(*args, **kwargs):
465461

466462
# callbacks
467463
def convert_in_upper_case_if_present(ctx, param, value):
468-
if isinstance(value, list | tuple):
464+
if isinstance(value, (list, tuple)):
469465
return [each.upper() for each in value]
470466
elif value:
471467
return value.upper()
472468

473469

474470
def convert_in_lower_case_if_present(ctx, param, value):
475-
if isinstance(value, list):
471+
if isinstance(value, (list, tuple)):
476472
return [each.lower() for each in value]
477473
elif value:
478474
return value.lower()
@@ -499,17 +495,6 @@ def build_account_option(**kwargs) -> Callable:
499495
return click.option('--account_number', '-acc', **params)
500496

501497

502-
def build_tenant_display_name_option(**kwargs) -> Callable:
503-
params = dict(
504-
type=str,
505-
required=True,
506-
help='The name of the target tenant group',
507-
callback=convert_in_lower_case_if_present
508-
)
509-
params.update(kwargs)
510-
return click.option('--tenant_display_name', '-tdn', **params)
511-
512-
513498
def build_iso_date_option(*args, **kwargs) -> Callable:
514499
help_iso = 'ISO 8601 format. Example: 2021-09-22T00:00:00.000000'
515500
params = dict(type=isoparse, required=False)
@@ -563,7 +548,6 @@ def build_limit_option(**kwargs) -> Callable:
563548

564549

565550
tenant_option = build_tenant_option()
566-
tenant_display_name_option = build_tenant_display_name_option()
567551
account_option = build_account_option()
568552

569553
optional_job_type_option = build_job_type_option()

cli/srecli/group/integrations.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import click
22

33
from srecli.group.integrations_dojo import dojo
4-
from srecli.group.integrations_sre import sre
4+
from srecli.group.integrations_re import re
55
from srecli.group.integrations_chronicle import chronicle
66

77

@@ -11,5 +11,5 @@ def integrations():
1111

1212

1313
integrations.add_command(dojo)
14-
integrations.add_command(sre)
14+
integrations.add_command(re)
1515
integrations.add_command(chronicle)

cli/srecli/group/integrations_sre.py renamed to cli/srecli/group/integrations_re.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,15 @@
99
from srecli.service.constants import AWS, AZURE, GOOGLE, KUBERNETES
1010

1111

12-
@click.group(name='sre')
13-
def sre():
12+
@click.group(name='re')
13+
def re():
1414
"""
1515
Manages Rule engine integration (self integration for Maestro)
1616
:return:
1717
"""
1818

1919

20-
@sre.command(cls=ViewCommand, name='add')
20+
@re.command(cls=ViewCommand, name='add')
2121
@build_tenant_option(multiple=True)
2222
@click.option('--all_tenants', is_flag=True,
2323
help='Whether to activate integration for all tenants')
@@ -84,7 +84,7 @@ def add(ctx: ContextObj, tenant_name: tuple[str, ...], all_tenants: bool,
8484
)
8585

8686

87-
@sre.command(cls=ViewCommand, name='describe')
87+
@re.command(cls=ViewCommand, name='describe')
8888
@cli_response()
8989
def describe(ctx: ContextObj, customer_id):
9090
"""
@@ -93,7 +93,7 @@ def describe(ctx: ContextObj, customer_id):
9393
return ctx['api_client'].sre_describe(customer_id=customer_id)
9494

9595

96-
@sre.command(cls=ViewCommand, name='delete')
96+
@re.command(cls=ViewCommand, name='delete')
9797
@cli_response()
9898
def delete(ctx: ContextObj, customer_id):
9999
"""
@@ -102,7 +102,7 @@ def delete(ctx: ContextObj, customer_id):
102102
return ctx['api_client'].sre_delete(customer_id=customer_id)
103103

104104

105-
@sre.command(cls=ViewCommand, name='update')
105+
@re.command(cls=ViewCommand, name='update')
106106
@click.option('--add_tenant', '-at', type=str, multiple=True,
107107
help='Tenants to activate')
108108
@click.option('--exclude_tenant', '-et', type=str, multiple=True,

0 commit comments

Comments
 (0)