diff --git a/README.md b/README.md index dcc2be8a..f9133f6d 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Uncoder IO can be run on-prem without a need for an internet connection, thus su - FortiSIEM Rule - `fortisiem-rule` - LogRhythm Axon Rule - `axon-ads-rule` - LogRhythm Axon Query - `axon-ads-query` +- LogRhythm SIEM Query - `siem-json-query` IOC-based queries can be generated in the following formats: diff --git a/uncoder-core/app/translator/mappings/platforms/logrhythm_siem/default.yml b/uncoder-core/app/translator/mappings/platforms/logrhythm_siem/default.yml new file mode 100644 index 00000000..dd72d6c7 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/logrhythm_siem/default.yml @@ -0,0 +1,309 @@ +platform: LogRhythm SIEM +source: default + + +field_mapping: + EventID: vendor_information.id + Channel: general_information.log_source.type_name + ComputerName: origin.host.name + FileName: object.file.name + ProcessId: object.process.id + Image: object.process.name + AccountEmail: unattributed.account.email_address + ContextInfo: general_information.raw_message + CurrentDirectory: object.process.path + ParentProcessId: object.process.parent_process.id + ParentImage: object.process.parent_process.path + ParentCommandLine: object.process.parent_process.command_line + TargetFilename: object.file.name + SourceIp: origin.host.ip_address.value + SourceHostname: origin.host.name + SourcePort: origin.host.network_port.value + DestinationIp: target.host.ip_address.value + DestinationHostname: + - target.host.name + - target.host.domain + DestinationPort: target.host.network_port.value + DestinationPortName: action.network.protocol.name + ImageLoaded: object.file.path + SignatureStatus: object.process.signature.status + SourceProcessId: object.process.id + SourceImage: object.process.name + Device: object.process.path + Destination: object.process.name + QueryName: action.dns.query + QueryStatus: action.dns.result + CommandName: object.process.command_line + CommandPath: object.process.path + HostApplication: object.script.command_line + HostName: origin.host.name + ScriptName: object.script.name + ScriptBlockText: object.script.command_line + ScriptBlockId: object.script.id + Application: object.process.name + ClientAddress: origin.host.ip_address.value + ClientName: origin.host.domain.name + DestAddress: target.host.ip_address.value + DestPort: target.host.network_port.value + IpAddress: origin.host.ip_address.value + IpPort: origin.host.network_port.value + NewProcessId: object.process.id + NewProcessName: object.process.name + ParentProcessName: object.process.parent_process.name + ProcessName: object.process.name + SourceAddress: origin.host.ip_address.value + WorkstationName: origin.host.name + destination.port: target.host.network_port.value + dst: target.host.ip_address.value + dst_ip: target.host.ip_address.value + dst_port: target.host.network_port.value + network_application: + - action.network.protocol.name + - object.url.protocol + network_protocol: action.network.protocol.name + proto: action.network.protocol.name + src: origin.host.ip_address.value + src_ip: origin.host.ip_address.value + src_port: origin.host.network_port.value + action: action.command + mqtt_action: action.command + smb_action: action.command + tunnel_action: action.command + arg: object.process.command_args + ftp_arg: object.process.command_args + mysql_arg: object.process.command_args + pop3_arg: object.process.command_args + client: origin.host.ip_address.value + command: action.command + ftp_command: action.command + irc_command: action.command + pop3_command: action.command + duration: action.duration + from: origin.account.email_address + kerberos_from: origin.account.email_address + smtp_from: origin.account.email_address + method: action.network.http_method + http_method: action.network.http_method + sip_method: action.network.http_method + name: object.file.name + smb_files_name: object.file.name + software_name: object.file.name + weird_name: object.file.name + path: object.file.path + smb_mapping_path: object.file.path + smb_files_path: object.file.path + smtp_files_path: object.file.path + password: object.file.name + reply_to: target.account.email_address + response_body_len: action.network.byte_information.received + request_body_len: action.network.byte_information.sent + rtt: action.duration + status_code: action.result.code + known_certs_subject: object.certificate.subject + sip_subject: object.email_message.subject + smtp_subject: object.email_message.subject + ssl_subject: object.certificate.subject + username: origin.account.name + uri: object.url.path + user: origin.account.name + user_agent: action.user_agent + http_user_agent: action.user_agent + gquic_user_agent: action.user_agent + sip_user_agent: action.user_agent + smtp_user_agent: action.user_agent + version: object.file.version + gquic_version: object.file.version + http_version: object.file.version + ntp_version: object.file.version + socks_version: object.file.version + snmp_version: object.file.version + ssh_version: object.file.version + tls_version: object.file.version + answer: action.dns.result + question_length: action.network.byte_information.total + record_type: action.dns.record_type + parent_domain: target.host.domain + cs-bytes: action.network.byte_information.received + r-dns: target.host.domain + sc-bytes: action.network.byte_information.received + sc-status: action.result.code + c-uri: object.url.complete + c-uri-extension: object.url.type + c-uri-query: object.url.query + c-uri-stem: object.url.path + c-useragent: action.user_agent + cs-host: + - target.host.name + - target.host.domain + cs-method: action.network.http_method + cs-version: object.file.version + uid: action.session.id + endpoint: origin.host.name + domain: target.host.domain + host_name: target.host.name + client_fqdn: origin.host.name + requested_addr: target.host.ip_address.value + server_addr: target.host.ip_address.value + qtype: action.dns.record_type + qtype_name: action.dns.record_type + query: action.dns.query + rcode_name: action.dns.result + md5: unattributed.hash.md5 + sha1: unattributed.hash.sha1 + sha256: unattributed.hash.sha256 + sha512: unattributed.hash.sha512 + filename: object.file.name + host: + - unattributed.host.name + - unattributed.host.ip_address.value + domainname: unattributed.host.name + hostname: unattributed.host.name + server_nb_computer_name: unattributed.host.name + server_tree_name: unattributed.host.name + server_dns_computer_name: unattributed.host.name + machine: unattributed.host.name + os: origin.host.os.platform + mac: unattributed.host.mac_address + result: + - action.result.message + - action.result.code + - action.result.reason + mailfrom: origin.account.email_address + rcptto: target.account.email_address + second_received: target.account.email_address + server_name: unattributed.host.name + c-ip: origin.host.ip_address.value + cs-uri: object.url.path + cs-uri-query: object.url.query + cs-uri-stem: object.url.path + clientip: origin.host.ip_address.value + clientIP: origin.host.ip_address.value + dest_domain: + - target.host.name + - target.host.domain + dest_ip: target.host.ip_address.value + dest_port: target.host.network_port.value + agent.version: object.file.version + destination.hostname: + - target.host.name + - target.host.domain + DestinationAddress: + - target.host.name + - target.host.domain + - target.host.ip_address.value + DestinationIP: target.host.ip_address.value + dst-ip: target.host.ip_address.value + dstip: target.host.ip_address.value + dstport: target.host.ip_address.value + Host: target.host.name + HostVersion: object.file.version + http_host: + - target.host.name + - target.host.domain + - target.host.ip_address.value + http_uri: object.url.path + http_url: object.url.complete + http.request.url-query-params: object.url.query + HttpMethod: action.network.http_method + in_url: object.url.path + post_url_parameter: object.url.path + Request_Url: object.url.complete + request_url: object.url.complete + request_URL: object.url.complete + RequestUrl: object.url.complete + resource.url: object.url.path + resource.URL: object.url.path + sc_status: action.result.code + sender_domain: + - target.host.name + - target.host.domain + service.response_code: action.result.code + source: + - origin.host.name + - origin.host.domain.name + - origin.host.ip_address.value + SourceAddr: origin.host.ip_address.value + SourceIP: origin.host.ip_address.value + SourceNetworkAddress: origin.host.ip_address.value + srcip: origin.host.ip_address.value + Status: action.result.code + status: action.result.code + url: object.url.path + URL: object.url.path + url_query: object.url.query + url.query: object.url.query + uri_path: object.url.path + user_agent.name: action.user_agent + user-agent: action.user_agent + User-Agent: action.user_agent + useragent: action.user_agent + UserAgent: action.user_agent + User_Agent: action.user_agent + web_dest: + - target.host.name + - target.host.domain + - target.host.ip_address.value + - object.url.domain + web.dest: + - target.host.name + - target.host.domain + - target.host.ip_address.value + - object.url.domain + Web.dest: + - target.host.name + - target.host.domain + - target.host.ip_address.value + - object.url.domain + web.host: + - target.host.name + - target.host.domain + - target.host.ip_address.value + - object.url.domain + Web.host: + - target.host.name + - target.host.domain + - target.host.ip_address.value + - object.url.domain + web_method: action.network.http_method + Web_method: action.network.http_method + web.method: action.network.http_method + Web.method: action.network.http_method + web_src: origin.host.ip_address.value + web_status: action.result.code + Web_status: action.result.code + web.status: action.result.code + Web.status: action.result.code + web_uri: object.url.path + web_url: object.url.complete + destination.ip: target.host.ip_address.value + source.ip: origin.host.ip_address.value + source.port: origin.host.ip_address.value + Computer: + - target.host.name + - target.host.domain + - target.host.ip_address.value + OriginalFileName: object.file.name + User: origin.account.name + EventType: action.command + TargetObject: + - object.registry_object.key + - object.registry_object.path + - object.resource.name + CommandLine: object.process.command_line + type: + - action.command + - action.type + - action.session.type + a0: + - object.process.command_line + - object.process.command_args + - object.process.name + cs-user-agent: action.user_agent + blocked: + - action.message + - action.result.reason + cs-ip: origin.host.ip_address.value + SubjectLogonId: action.session.id + SubjectUserName: origin.account.name + SubjectUserSid: origin.account.id + SubjectDomainName: origin.account.domain diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/__init__.py b/uncoder-core/app/translator/platforms/logrhythm_siem/__init__.py new file mode 100644 index 00000000..8a1fbde6 --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/__init__.py @@ -0,0 +1,2 @@ +# from app.translator.platforms.logrhythm_siem.renders.logrhythm_siem_query import LogRhythmSiemQueryRender # noqa: F401 +from app.translator.platforms.logrhythm_siem.renders.logrhythm_siem_rule import LogRhythmSiemRuleRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/const.py b/uncoder-core/app/translator/platforms/logrhythm_siem/const.py new file mode 100644 index 00000000..2e8902ef --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/const.py @@ -0,0 +1,94 @@ +from app.translator.core.custom_types.meta_info import SeverityType +from app.translator.core.models.platform_details import PlatformDetails + +UNMAPPED_FIELD_DEFAULT_NAME = "general_information.raw_message" + +DEFAULT_LOGRHYTHM_Siem_RULE = { + "title": "Default LogRhythm Siem rule", + "version": 1, + "description": "Default LogRhythm Siem rule description.", + "maxMsgsToQuery": 30000, + "logCacheSize": 10000, + "aggregateLogCacheSize": 10000, + "queryTimeout": 60, + "isOriginatedFromWeb": False, + "webLayoutId": 0, + "queryRawLog": True, + "queryFilter": { + "msgFilterType": 2, + "isSavedFilter": False, + "filterGroup": { + "filterItemType": 1, + "fieldOperator": 1, + "filterMode": 1, + "filterGroupOperator": 0, + + "filterItems":"query", + "name": "Filter Group", + # "raw": "query" # FOR DEBUG REASONS + } + }, + "queryEventManager": False, + "useDefaultLogRepositories": True, + "dateCreated": "2024-06-05T22:47:06.3683942Z", + "dateSaved": "2024-06-05T22:47:06.3683942Z", + "dateUsed": "2024-06-05T22:47:06Z", + "includeDiagnosticEvents": True, + "searchMode": 2, + "webResultMode": 0, + "nextPageToken": "", + "pagedTimeout": 300, + "restrictedUserId": 0, + "createdVia": 0, + "searchType": 1, + "queryOrigin": 0, + "searchServerIPAddress": None, + "dateCriteria": { + "useInsertedDate": False, + "lastIntervalValue": 24, + "lastIntervalUnit": 7 + }, + "repositoryPattern": "", + "ownerId": 227, + "searchId": 0, + "queryLogSourceLists": [], + "queryLogSources": [], + "logRepositoryIds": [], + "refreshRate": 0, + "isRealTime": False, + "objectSecurity": { + "objectId": 0, + "objectType": 20, + "readPermissions": 0, + "writePermissions": 0, + "entityId": 1, + "ownerId": 227, + "canEdit": True, + "canDelete": False, + "canDeleteObject": False, + "entityName": "", + "ownerName": "", + "isSystemObject": True + }, + "enableIntelligentIndexing": False +} + +PLATFORM_DETAILS = {"group_id": "siem-ads", "group_name": "LogRhythm Siem"} + +LOGRHYTHM_Siem_QUERY_DETAILS = { + "platform_id": "siem-ads-query", + "name": "LogRhythm Siem Query", + "platform_name": "Query", + **PLATFORM_DETAILS, +} + +LOGRHYTHM_Siem_RULE_DETAILS = { + "platform_id": "siem-ads-rule", + "name": "LogRhythm Siem Search API", + "platform_name": "Search API", + "first_choice": 0, + **PLATFORM_DETAILS, +} + +# logrhythm_siem_query_details = PlatformDetails(**LOGRHYTHM_Siem_QUERY_DETAILS) +logrhythm_siem_rule_details = PlatformDetails(**LOGRHYTHM_Siem_RULE_DETAILS) diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/escape_manager.py b/uncoder-core/app/translator/platforms/logrhythm_siem/escape_manager.py new file mode 100644 index 00000000..ec576ea6 --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/escape_manager.py @@ -0,0 +1,37 @@ +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.escape_manager import EscapeManager +from app.translator.core.models.escape_details import EscapeDetails + + +class LogRhythmQueryEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [EscapeDetails(pattern=r"'", escape_symbols=r"''")], + ValueType.regex_value: [ + EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\"), + EscapeDetails(pattern=r"\*", escape_symbols=r"\\*"), + EscapeDetails(pattern=r"\.", escape_symbols=r"\\."), + EscapeDetails(pattern=r"\^", escape_symbols=r"\\^"), + EscapeDetails(pattern=r"\$", escape_symbols=r"\\$"), + EscapeDetails(pattern=r"\|", escape_symbols=r"\\|"), + EscapeDetails(pattern=r"\?", escape_symbols=r"\\?"), + EscapeDetails(pattern=r"\+", escape_symbols=r"\\+"), + EscapeDetails(pattern=r"\(", escape_symbols=r"\\("), + EscapeDetails(pattern=r"\)", escape_symbols=r"\\)"), + EscapeDetails(pattern=r"\[", escape_symbols=r"\\["), + EscapeDetails(pattern=r"\]", escape_symbols=r"\\]"), + EscapeDetails(pattern=r"\{", escape_symbols=r"\\{"), + EscapeDetails(pattern=r"\}", escape_symbols=r"\\}"), + ], + } + + +class LogRhythmRuleEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [EscapeDetails(pattern=r"'", escape_symbols=r"''")] + } + + +logrhythm_query_escape_manager = LogRhythmQueryEscapeManager() +logrhythm_rule_escape_manager = LogRhythmRuleEscapeManager() diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/mapping.py b/uncoder-core/app/translator/platforms/logrhythm_siem/mapping.py new file mode 100644 index 00000000..8e032a08 --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/mapping.py @@ -0,0 +1,47 @@ +from typing import Optional + +from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping + + +class LogRhythmSiemLogSourceSignature(LogSourceSignature): + def __init__(self, default_source: Optional[dict] = None): + self._default_source = default_source or {} + + def is_suitable(self) -> bool: + return True + + def __str__(self) -> str: + return "general_information.log_source.type_name" + + +class LogRhythmSiemMappings(BasePlatformMappings): + def prepare_mapping(self) -> dict[str, SourceMapping]: + source_mappings = {} + for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): + log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict) + fields_mapping = self.prepare_fields_mapping(field_mapping=mapping_dict.get("field_mapping", {})) + source_mappings[DEFAULT_MAPPING_NAME] = SourceMapping( + source_id=DEFAULT_MAPPING_NAME, log_source_signature=log_source_signature, fields_mapping=fields_mapping + ) + return source_mappings + + def prepare_log_source_signature(self, mapping: dict) -> LogRhythmSiemLogSourceSignature: + default_log_source = mapping.get("default_log_source") + return LogRhythmSiemLogSourceSignature(default_source=default_log_source) + + def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMapping]: + suitable_source_mappings = [] + for source_mapping in self._source_mappings.values(): + if source_mapping.source_id == DEFAULT_MAPPING_NAME: + continue + + if source_mapping.fields_mapping.is_suitable(field_names): + suitable_source_mappings.append(source_mapping) + + if not suitable_source_mappings: + suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]] + + return suitable_source_mappings + + +logrhythm_siem_mappings = LogRhythmSiemMappings(platform_dir="logrhythm_siem") diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/__init__.py b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py new file mode 100644 index 00000000..7d710f94 --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py @@ -0,0 +1,274 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.a +----------------------------------------------------------------- +""" + +from typing import Union + +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.custom_types.tokens import LogicalOperatorType +from app.translator.core.custom_types.values import ValueType +from app.translator.core.exceptions.core import StrictPlatformException +from app.translator.core.exceptions.render import BaseRenderException +from app.translator.core.mapping import LogSourceSignature, SourceMapping +from app.translator.core.models.field import FieldValue, Keyword +from app.translator.core.models.identifier import Identifier +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.managers import render_manager +# from app.translator.platforms.logrhythm_siem.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_siem_query_details +from app.translator.platforms.logrhythm_siem.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_siem_rule_details +from app.translator.platforms.logrhythm_siem.escape_manager import logrhythm_query_escape_manager +from app.translator.platforms.logrhythm_siem.mapping import LogRhythmSiemMappings, logrhythm_siem_mappings + + +class LogRhythmRegexRenderException(BaseRenderException): + ... + + +class LogRhythmSiemFieldValueRender(BaseFieldValueRender): + details: PlatformDetails = logrhythm_siem_rule_details + # details: PlatformDetails = logrhythm_siem_query_details + escape_manager = logrhythm_query_escape_manager + + def __is_complex_regex(self, regex: str) -> bool: + regex_items = ("[", "]", "(", ")", "{", "}", "+", "?", "^", "$", "\\d", "\\w", "\\s", "-") + return any(v in regex for v in regex_items) + + def __is_contain_regex_items(self, value: str) -> bool: + regex_items = ("[", "]", "(", ")", "{", "}", "*", "+", "?", "^", "$", "|", ".", "\\d", "\\w", "\\s", "\\", "-") + return any(v in value for v in regex_items) + + def __regex_to_str_list(self, value: Union[int, str]) -> list[list[str]]: # noqa: PLR0912 + value_groups = [] + + stack = [] # [(element: str, escaped: bool)] + + for char in value: + if char == "\\": + if stack and stack[-1][0] == "\\" and stack[-1][1] is False: + stack.pop() + stack.append((char, True)) + else: + stack.append(("\\", False)) + elif char == "|": + if stack and stack[-1][0] == "\\" and stack[-1][1] is False: + stack.pop() + stack.append((char, True)) + elif stack: + value_groups.append("".join(element[0] for element in stack)) + stack = [] + else: + stack.append((char, False)) + if stack: + value_groups.append("".join(element[0] for element in stack if element[0] != "\\" or element[-1] is True)) + + joined_components = [] + for value_group in value_groups: + inner_joined_components = [] + not_joined_components = [] + for i in range(len(value_group)): + if value_group[i] == "*" and i > 0 and value_group[i - 1] != "\\": + inner_joined_components.append("".join(not_joined_components)) + not_joined_components = [] + else: + not_joined_components.append(value_group[i]) + if not_joined_components: + inner_joined_components.append("".join(not_joined_components)) + joined_components.append(inner_joined_components) + + return joined_components + + def __unmapped_regex_field_to_contains_string(self, field: str, value: str) -> str: + if self.__is_complex_regex(value): + raise LogRhythmRegexRenderException + values = self.__regex_to_str_list(value) + return ( + "(" + + self.or_token.join( + " AND ".join(f'{field} CONTAINS "{self.apply_value(value)}"' for value in value_list) + for value_list in values + ) + + ")" + ) + + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, str): + return f'{field} = "{self.apply_value(value)}"' + if isinstance(value, list): + prepared_values = ", ".join(f"{self.apply_value(v)}" for v in value) + operator = "IN" if all(isinstance(v, str) for v in value) else "in" + return f"{field} {operator} [{prepared_values}]" + return f'{field} = "{self.apply_value(value)}"' + + def less_modifier(self, field: str, value: Union[int, str]) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, int): + return f"{field} < {value}" + return f"{field} < '{self.apply_value(value)}'" + + def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, int): + return f"{field} <= {value}" + return f"{field} <= {self.apply_value(value)}" + + def greater_modifier(self, field: str, value: Union[int, str]) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, int): + return f"{field} > {value}" + return f"{field} > {self.apply_value(value)}" + + def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, int): + return f"{field} >= {value}" + return f"{field} >= {self.apply_value(value)}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, list): + return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" + if isinstance(value, int): + return f"{field} != {value}" + return f"{field} != {self.apply_value(value)}" + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" + return f'{field} CONTAINS "{self.apply_value(value)}"' + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" + if isinstance(value, str) and field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + applied_value = self.apply_value(value, value_type=ValueType.regex_value) + return f'{field} matches ".*{applied_value}$"' + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" + if isinstance(value, str) and field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + applied_value = self.apply_value(value, value_type=ValueType.regex_value) + return f'{field} matches "^{applied_value}.*"' + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME and self.__is_contain_regex_items(value): + if isinstance(value, str): + return self.__unmapped_regex_field_to_contains_string(field, value) + if isinstance(value, list): + return self.or_token.join( + self.__unmapped_regex_field_to_contains_string(field=field, value=v) for v in value + ) + if isinstance(value, list): + return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" + if isinstance(value, str) and field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + return f'{field} matches "{value}"' + + def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 + if isinstance(value, list): + rendered_keywords = [f'{UNMAPPED_FIELD_DEFAULT_NAME} CONTAINS "{v}"' for v in value] + return f"({self.or_token.join(rendered_keywords)})" + return f'{UNMAPPED_FIELD_DEFAULT_NAME} CONTAINS "{value}"' + + +@render_manager.register +class LogRhythmSiemQueryRender(PlatformQueryRender): + details: PlatformDetails = logrhythm_siem_rule_details + # details: PlatformDetails = logrhythm_siem_query_details + + or_token = "OR" + and_token = "AND" + not_token = "NOT" + + field_value_render = LogRhythmSiemFieldValueRender(or_token=or_token) + + mappings: LogRhythmSiemMappings = logrhythm_siem_mappings + comment_symbol = "//" + is_single_line_comment = True + is_strict_mapping = True + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"AND {query}" if query else "" + + def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 + return str(log_source_signature) + + def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + if isinstance(token, FieldValue) and token.field: + try: + mapped_fields = self.map_field(token.field, source_mapping) + except StrictPlatformException: + try: + return self.field_value_render.apply_field_value( + field=UNMAPPED_FIELD_DEFAULT_NAME, operator=token.operator, value=token.value + ) + except LogRhythmRegexRenderException as exc: + raise LogRhythmRegexRenderException( + f"Uncoder does not support complex regexp for unmapped field:" + f" {token.field.source_name} for LogRhythm Siem" + ) from exc + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] + ) + return self.group_token % joined if len(mapped_fields) > 1 else joined + + return super().apply_token(token, source_mapping) + + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + queries_map = {} + source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) + + for source_mapping in source_mappings: + prefix = self.generate_prefix(source_mapping.log_source_signature) + if "product" in query_container.meta_info.parsed_logsources: + prefix = f"{prefix} CONTAINS {query_container.meta_info.parsed_logsources['product'][0]}" + else: + prefix = f"{prefix} CONTAINS anything" + + result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) + rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) + not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported + finalized_query = self.finalize_query( + prefix=prefix, + query=result, + functions=rendered_functions.rendered, + not_supported_functions=not_supported_functions, + meta_info=query_container.meta_info, + source_mapping=source_mapping, + ) + if return_only_first_query_ctx_var.get() is True: + return finalized_query + queries_map[source_mapping.source_id] = finalized_query + + return self.finalize(queries_map) diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py new file mode 100644 index 00000000..b68c08c8 --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py @@ -0,0 +1,676 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +import ast +import copy +from datetime import datetime, timedelta +import json +import random +import re +import traceback +from typing import Optional + +from app.translator.core.custom_types.meta_info import SeverityType +from app.translator.core.mapping import SourceMapping +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer +from app.translator.managers import render_manager +from app.translator.platforms.logrhythm_siem.const import DEFAULT_LOGRHYTHM_Siem_RULE, logrhythm_siem_rule_details +from app.translator.platforms.logrhythm_siem.escape_manager import logrhythm_rule_escape_manager +from app.translator.platforms.logrhythm_siem.renders.logrhythm_siem_query import ( + # LogRhythmSiemFieldValue + # LogRhythmSiemFieldValue, + LogRhythmSiemFieldValueRender, + LogRhythmSiemQueryRender, +) +from app.translator.tools.utils import get_rule_description_str + +_AUTOGENERATED_TEMPLATE = "Autogenerated LogRhythm Siem Rule" +_SEVERITIES_MAP = { + SeverityType.critical: SeverityType.critical, + SeverityType.high: SeverityType.high, + SeverityType.medium: SeverityType.medium, + SeverityType.low: SeverityType.low, + SeverityType.informational: SeverityType.low, +} + + +class LogRhythmSiemRuleFieldValue(LogRhythmSiemFieldValueRender): +# class LogRhythmSiemRuleFieldValue(LogRhythmSiemFieldValue): + details: PlatformDetails = logrhythm_siem_rule_details + escape_manager = logrhythm_rule_escape_manager + + +@render_manager.register +class LogRhythmSiemRuleRender(LogRhythmSiemQueryRender): +# class LogRhythmSiemRuleRender(): + details: PlatformDetails = logrhythm_siem_rule_details + or_token = "or" + field_value_map = LogRhythmSiemRuleFieldValue(or_token=or_token) + + + # Function to generate ISO 8601 formatted date within the last 24 hours + def generate_iso_date(self): + now = datetime.utcnow() + past_date = now - timedelta(hours=random.randint(0, 23), minutes=random.randint(0, 59), seconds=random.randint(0, 59)) + return past_date.isoformat() + 'Z' + + + def generate_timestamps(self): + iso_date = self.generate_iso_date() + date_created = iso_date + date_saved = iso_date + date_used = iso_date + date_used = (date_used.split('.')[0]) + "Z" + return_obj = [] + + return_obj.append(date_created) + return_obj.append(date_saved) + return_obj.append(date_used) + return return_obj + + + def finalize_query( + self, + prefix: str, + query: str, + functions: str, + meta_info: Optional[MetaInfoContainer] = None, + source_mapping: Optional[SourceMapping] = None, + not_supported_functions: Optional[list] = None, + *args, # noqa: ARG002 + **kwargs, # noqa: ARG002 + ) -> str: + query = super().finalize_query(prefix=prefix, query=query, functions=functions) + rule = copy.deepcopy(DEFAULT_LOGRHYTHM_Siem_RULE) + ''' Parse out query - originally for "filter". + example how it should look like: + { + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": 29, # Must be altered + "values": [ + { + "filterType": 29, # Must be altered + "valueType": 4, + "value": { + "value": "moo", + "matchType": 0 + }, + "displayValue": "moo" + } + ], + "name": "User (Origin)" + } + + ANDs & CONTAINS needs to be broken out + ''' + # rule["queryFilter"]["filterGroup"]["raw"] = query # DEBUG + query = self.gen_filter_item(query) + # rule["observationPipeline"]["pattern"]["operations"][0]["logObserved"]["filter"] = query + rule["queryFilter"]["filterGroup"]["filterItems"] = query + + rule["title"] = meta_info.title or _AUTOGENERATED_TEMPLATE + rule["description"] = get_rule_description_str( + description=meta_info.description or rule["description"] or _AUTOGENERATED_TEMPLATE, + author=meta_info.author, + license_=meta_info.license, + ) + + # Set the time, default is last 24 hours + gen_time = self.generate_timestamps() + rule["dateCreated"] = gen_time[0] + rule["dateSaved"] = gen_time[1] + rule["dateUsed"] = gen_time[2] + + json_rule = json.dumps(rule, indent=4, sort_keys=False) + if not_supported_functions: + rendered_not_supported = self.render_not_supported_functions(not_supported_functions) + return json_rule + rendered_not_supported + return json_rule + + + def pull_filter_item(self, f_type): + if f_type == 'User (Origin)': + f_type = 'Login' + + filter_type = { + 'IDMGroupForAccount': [53, 2, 'User (Impacted) by Active Directory Group'], + 'Address': [44, 11, 'Address (Sender or Recipient)'], + 'Amount': [64, 10, 'Amount'], + 'Application': [97, 11, 'PolyList Item'], + 'MsgClass': [10, 2, 'Classification'], + 'Command': [112, 4, 'Command'], + 'CommonEvent': [11, 2, 'Common Event'], + 'Direction': [2, 2, 'Direction'], + 'Duration': [62, 10, 'Duration'], + 'Group': [38, 11, 'Group'], + 'BytesIn': 58, + 'BytesOut': 59, + 'BytesInOut': 95, + 'DHost': 100, + 'Host': [98, 11, 'PolyList Item'], + 'SHost': 99, + 'ItemsIn': 60, + 'ItemsOut': 61, + 'ItemsInOut': [96, 10, 'Host (Impacted) Packets Total'], + 'DHostName': 25, + 'HostName': [23, 4, 'HostName (Origin or Impacted)'], + 'SHostName': 24, + 'KnownService': [16, 2, 'Known Application'], + 'DInterface': 108, + 'Interface': [133, 4, 'Interface (Origin or Impacted)'], + 'SInterface': 107, + 'DIP': 19, + 'IP': [17, 5, 'IP Address (Origin or Impacted)'], + 'SIP': 18, + 'DIPRange': 22, + 'IPRange': 20, + 'SIPRange': 21, + 'KnownDHost': 15, + 'KnownHost': [13, 2, 'Known Host (Origin or Impacted)'], + 'KnownSHost': 14, + 'Location': [87, 2, 'Location (Origin or Impacted)'], + 'SLocation': [85, 2, 'Location (Origin)'], + 'DLocation': [86, 2, 'Location (Impacted)'], + 'MsgSource': [7, 2, 'Log Source'], + 'Entity': [6, 2, 'Log Source Entity'], + 'RootEntity': 136, + 'MsgSourceType': [9, 2, 'Log Source Type'], + 'DMAC': 104, + 'MAC': [132, 4, 'MAC Address (Origin or Impacted)'], + 'SMAC': 103, + 'Message': [35, 4, 'Log Message'], + 'MPERule': 12, + 'DNATIP': 106, + 'NATIP': [126, 5, 'NAT IP Address (Origin or Impacted)'], + 'SNATIP': 105, + 'DNATIPRange': 125, + 'NATIPRange': 127, + 'SNATIPRange': 124, + 'DNATPort': 115, + 'NATPort': [130, 2, 'NAT TCP/UDP Port (Origin or Impacted)'], + 'SNATPort': 114, + 'DNATPortRange': 129, + 'NATPortRange': 131, + 'SNATPortRange': 128, + 'DNetwork': 50, + 'Network': [51, 2, 'Network (Origin or Impacted)'], + 'SNetwork': 49, + 'Object': [34, 11, 'Object'], + 'ObjectName': [113, 4, 'Object Name'], + 'Login': 29, + 'IDMGroupForLogin': [52, 2, 'User (Origin) by Active Directory Group'], + 'Priority': [3, 10, 'Risk Based Priority (RBP)'], + 'Process': [41, 4, 'Process Name'], + 'PID': [109, 2, 'Process ID'], + 'Protocol': [28, 1, 'Protocol'], + 'Quantity': [63, 10, 'Quantity'], + 'Rate': [65, 10, 'Rate'], + 'Recipient': [32, 11, 'Recipient'], + 'Sender': [31, 11, 'Sender'], + 'Session': [40, 4, 'Session'], + 'Severity': [110, 4, 'Severity'], + 'Size': [66, 10, 'Size'], + 'Subject': [33, 4, 'Subject'], + 'DPort': 27, + 'Port': [45,2, 'TCP/UDP Port (Origin or Impacted)'], + 'SPort': 26, + 'DPortRange': 47, + 'PortRange': [48, 9, 'TCP/UDP Port Range (Origin or Impacted)'], + 'SPortRange': 46, + 'URL': [42, 4, 'URL'], + 'Account': 30, + 'User': [43, 4, 'User (Origin or Impacted)'], + 'IDMGroupForUser': [54, 2, 'User by Active Directory Group'], + 'VendorMsgID': [37, 4, 'Vendor Message ID'], + 'Version': [111, 4, 'Version'], + 'SZone': 93, + 'DZone': 94, + 'FilterGroup': 1000, + 'PolyListItem': 1001, + 'Domain': [39, 11, 'Domain Impacted'], + 'DomainOrigin': [137, 11, 'Domain Origin'], + 'Hash': [138, 4, 'Hash'], + 'Policy': [139, 4, 'Policy'], + 'VendorInfo': [140, 4, 'Vendor Info'], + 'Result': [141, 4, 'Result'], + 'ObjectType': [142, 4, 'Object Type'], + 'CVE': [143,11, 'CVE'], + 'UserAgent': [144, 4, 'User Agent'], + 'ParentProcessId': [145, 4, 'Parent Process ID'], + 'ParentProcessName': [146, 4, 'Parent Process Name'], + 'ParentProcessPath': [147, 4, 'Parent Process Path'], + 'SerialNumber': [148, 4, 'Serial Number'], + 'Reason': [149, 4, 'Reason'], + 'Status': [150, 4, 'Status'], + 'ThreatId': [151, 4, 'Threat ID'], + 'ThreatName': [152, 4, 'Threat Name'], + 'SessionType': [153, 4, 'Session Type'], + 'Action': [154,13, 'Action'], + 'ResponseCode': [155, 4, 'Response Code'], + 'UserOriginIdentityID': 167, + 'Identity': 160, + 'UserImpactedIdentityID': [168,2], + 'SenderIdentityID': [169, 2, 'Sender Identity'], + 'RecipientIdentityID': 170 + } + + value_type = { + 'Byte': 0, + 'Int16': 1, + 'Int32': 2, + 'Int64': 3, + 'String': 4, + 'IPAddress': 5, + 'IPAddressrange': 6, + 'TimeOfDay': 7, + 'DateRange': 8, + 'PortRange': 9, + 'Quantity': 10, + 'ListReference': 11, + 'ListSet': 12, + 'Null': 13, + 'INVALID': 99 + } + if f_type in filter_type: + r_f = filter_type[f_type] + else: + print(f'filterType name reference was not found: {f_type}') + r_f = 0000 + # v_t = value_type[v_type] + return r_f + + + def process_sub_conditions(self, sub_conditions, items): + parsed_conditions = [] + for sub_condition in sub_conditions: + try: + # Parse the sub_condition + # This generally is preventing many things from parsing properly + if sub_condition.startswith('target.host.network_port.value') or \ + sub_condition.startswith('general_information.log_source.type_name CONTAINS target.host.network_port.value'): + sub_condition = sub_condition.replace('general_information.log_source.type_name CONTAINS ','') + matches = re.findall(r'(target\.host\.network_port\.value) in \[([^\]]+)\]', sub_condition) + if matches: + for match in matches: + # lrTODO : this needs to go into process_match + # items = self.process_match(match) + field, value = match + f_t = self.field_translation(field) + f_number = self.pull_filter_item(f_t) + port_list = [int(x.strip()) for x in value.split(',')] + collected_values = [] + for port in port_list: + collected_values.append({ + "filterType": f_number[0], + "valueType": f_number[1], + "value": port, + "displayValue": str(port) + }) + items.append({ + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": f_number[0], + "values": collected_values, + "name": f_number[2] + # "name": f_t + }) + # for port in port_list: + # items.append({ + # "filterItemType": 0, + # "fieldOperator": 0, + # "filterMode": 1, + # "filterType": f_number[0], + # "values":[{ + # "filterType": f_number[0], + # "valueType": f_number[1], + # "value": { + # "value":port, + # "matchType": 0 + # }, + # "displayValue": str(port) + # }], + # "name": f_number[2] + # # "name": f_t + # }) + elif 'CONTAINS' in sub_condition: + try: + matches = re.findall(r'(\w+\.\w+\.\w+) CONTAINS "([^"]+)"', sub_condition)[0] + except: + matches = re.findall(r'(\w+\.\w+\.\w+) CONTAINS "([^"]+)"', sub_condition) + if not matches: + matches = re.findall(r'(\w+\.\w+) CONTAINS "([^"]+)"', sub_condition) + if 'matches ' in sub_condition: + match = matches[0] + # Some instances have a matches after a contains, + s_match = sub_condition.split('matches ')[1] + s_c_filtered = s_match.replace('"','').replace('.*','') + result = (match, s_c_filtered) + # result = (match, match + s_c_filtered) + f_t = self.field_translation(result[0]) + f_number = self.pull_filter_item(f_t) + v_t = result[1] + if result[0] in result[1]: + v_t = v_t.replace(result[0],'') + items.append({ + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": f_number[0], + "values":[{ + "filterType": f_number[0], + "valueType": f_number[1], + "value": { + "value":result[1].replace('\\', '/'), + "matchType": 0 + }, + "displayValue": str(result[1].replace('\\', '/')) + }], + "name": f_number[2] + # "name": f_t + }) + if len(matches) == 2: + f_t = self.field_translation(matches[0]) + f_number = self.pull_filter_item(f_t) + items.append({ + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": f_number[0], + "values":[{ + "filterType": f_number[0], + "valueType": f_number[1], + "value": { + "value":matches[1].replace('\\', '/'), + "matchType": 0 + }, + "displayValue": str(matches[1].replace('\\', '/')) + }], + "name": f_number[2] + }) + else: + for match in matches: + items = self.process_match(match) + + # Parse the sub_condition + elif 'AND' in sub_condition: + matches = re.findall(r'(\w+\.\w+\.\w+) AND "([^"]+)"', sub_condition) + if not matches: + matches = re.findall(r'(\w+\.\w+\.\w+) matches ([^\s]+)', sub_condition) + # field, value = re.findall(r'(\w+\.\w+\.\w+) CONTAINS "([^"]+)"', sub_condition)[0] + if matches: + if len(matches) == 2: + f_t = self.field_translation(matches[0][0]) + f_v = matches[0][1] + f_v = f_v.replace('\"','') + f_number = self.pull_filter_item(f_t) + items.append({ + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": f_number[0], + "values":[{ + "filterType": f_number[0], + "valueType": f_number[1], + "value": { + "value": f_v.replace('\\', '/'), + "matchType": 0 + }, + "displayValue": str(f_v.replace('\\', '/')) + }], + "name": f_number[2] + }) + else: + for match in matches: + items = self.process_match(match) + elif 'matches' in sub_condition: + matches = re.findall(r'(\w+\.\w+\.\w+) matches "([^"]+)"', sub_condition) + if matches: + for match in matches: + items = self.process_match(match) + elif 'origin.account.name' in sub_condition or 'User' in sub_condition: + if 'User' in sub_condition: + matches = re.findall(r'User: "([^"]+)"', sub_condition) + else: + matches = re.findall(r'origin\.account\.name = "([^"]+)"', sub_condition) + if matches: + for match in matches: + items = self.process_match(match) + elif 'object.file.name' in sub_condition or 'TargetFilename' in sub_condition: + if 'TargetFilename' in sub_condition: + matches = re.findall(r'TargetFilename: "([^"]+)"', sub_condition) + else: + matches = re.findall(r'object\.file\.name = "([^"]+)"', sub_condition) + if matches: + for match in matches: + items = self.process_match(match) + except Exception as e: + print(f'Error processing sub_condition: {sub_condition}') + print(f'Error: {e}') + traceback.print_exc() + # print(f'items END: {items}\nParsed_condition: {parsed_conditions}') + return items + + + + def gen_filter_item(self, query): + ''' Collection of general observations found that breaks translation ''' + # Removing the product[0] AND to be more global friendly with parsing + query = query.replace("\\\'product\\\'", "'product'") + query = query.replace('query_container.meta_info.parsed_logsources[\'product\'][0] AND ', '') + + # Random + query = query.replace('anything AND ', '') # LR Siem cannot really handle this query + + # Remove outer parentheses + # Split the query into individual conditions + query = re.sub(r'^\(\(|\)\)$', '', query) + conditions = re.split(r'\)\s*or \s*\(', query) + parsed_conditions = [] + + for condition in conditions: + # Remove inner parentheses + condition = condition.strip('()') + condition = condition.replace('((', '') + condition = condition.replace('(', '') + sub_conditions = re.split(r'\s*or \s*', condition) + # sub_conditions = re.split(r'\s*AND\s*', condition) + items = [] + + current_found = parsed_conditions.append(self.process_sub_conditions(sub_conditions, items)) + parsed_conditions.append(current_found) + return parsed_conditions[0] + + + def field_translation(self, field): + if field == 'origin.account.name': + field = 'User (Origin)' + elif field == 'general_information.raw_message': + field = 'Message' + elif field in { 'object.process.command_line', + 'object.script.command_line' + }: + field = 'Command' + elif field in { 'object.registry_object.path', + 'object.registry_object.key', + 'object.resource.name' + }: + field = 'Object' + elif field in { 'target.host.ip_address.value', + 'target.host.ip_address.value' + }: + field = 'Address' + elif field in { 'target.host.name', + 'target.host.domain' + }: + field = 'DHost' + elif field in { 'action.network.byte_information.received', + 'action.network.byte_information.received' + }: + field = 'BytesIn' + elif field == 'unattributed.host.mac_address': + field = 'MAC' + elif field == 'action.network.http_method': + field = 'SIP' + elif field in { 'origin.url.path', + 'action.dns.query' + }: + field = 'URL' + elif field == 'origin.host.domain': + field = 'SHostName' + elif field == 'target.host.domain': + field = 'Host' + elif field == 'action.network.byte_information.sent': + field = 'BytesOut' + elif field == 'action.network.byte_information.total': + field = 'BytesInOut' + elif field == 'object.process.name': + field = 'Process' + elif field == 'action.duration': + field = 'Duration' + elif field == 'process.parent_process.path': + field = 'ParentProcessPath' + elif field == 'object.process.parent_process.name': + field = 'ParentProcessName' + elif field == 'object.file.name' or field == 'TargetFilename': + field = 'Object' + elif field == 'target.host.network_port.value': + field = 'Port' + return field + + + def process_match(self, match): + field, value = match + keywords = ["AND", "CONTAINS", "AND NOT", "IN", "NOT"] + items = [] + + # Regex to extract field and value + # pattern = re.compile(r'(\S+)\s+(CONTAINS|NOT |IN|NOT IN|AND|OR)\s+(.+)') + pattern = re.compile(r'((?:NOT \s+)?\S+)\s+(CONTAINS|in|IN|NOT IN|AND|OR|AND NOT|NOT)\s+(.+)') + + # Initial tuple block + i_field = field + i_block = value.split(' ')[0] + i_block = i_block.replace('"','') + i_v = (i_field, i_block) + # i_block = re.split(f'\\s+{keyword}\\s+', value) + # print(f'item append len : {len(i_v)}') + items.extend(self.item_append(i_v)) + # Split the value based on the keywords + for keyword in keywords: + if keyword in value: + parts = re.split(f'\\s+{keyword}\\s+', value) + for part in parts: + part = part.strip() + if part: + # MUST match array list AND, CONTAINS, etc. otherwise skip + match_obj = pattern.match(part) + if match_obj == None and part.startswith('NOT '): + k = part.replace('NOT ','') + match_obj = pattern.match(k) + if match_obj: + # Construct a new match tuple and process it + new_field = match_obj.group(1) + # lr_TODO : clean value + new_value = match_obj.group(3) + current_match = (new_field, self.clean_value(new_value)) + items.extend(self.item_append(current_match)) + break + else: + # If no keywords are found, process the original match + no_keywords = (field, value) + items.extend(self.item_append(no_keywords)) + + return items + + + def clean_value(self, value): + return_value = value.replace(')','') + return return_value + + + def item_append(self, match): + items = [] + field, value = match + f_t = self.field_translation(field=field) + if f_t == 'Port': + if isinstance(value, list) or value.startswith('[') and value.endswith(']'): + value = self.check_array(value) + port_list = [int(x.strip()) for x in value.split(',')] + for port in port_list: + items.append(self.new_item(f_t=f_t, value=port)) + else: + items.append(self.new_item(f_t=f_t, value=value)) + else: + items.append(self.new_item(f_t=f_t, value=value)) + return items + + + + def new_item(self, f_t, value): + # Clean if value is string, else just feed it through (could be port number) + if isinstance(value, str): + value = value.replace('\\', '/') + f_number = self.pull_filter_item(f_t) + # i = { + # "filterType": self.pull_filter_item(f_t), + # "valueType": 4, + # "value": { + # "value": value, + # "matchType": 0 + # }, + # "displayValue": f_t + # } + + i = { + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": f_number[0], + "values":[ + { + "filterType": f_number[0], + "valueType": f_number[1], + "value": { + "value": value.replace('\\', '/'), + "matchType": 0 + }, + "displayValue": str(value.replace('\\', '/')) + } + ], + "name": f_number[2] + } + return i + + + def check_array(self, s): + if isinstance(s, str): + if re.match(r'^\[.*\]$', s.strip()): + try: + # Safely eval the str to a Py List + array = ast.literal_eval(s) + if isinstance(array, list): + return array + except (ValueError, SyntaxError): + pass + return None + else: + return s \ No newline at end of file