Merge pull request #24 from CybercentreCanada/assemblyline/issues/101

Assemblyline/issues/101

.vscode/settings.json

@@ -1,6 +1,6 @@
 {
     "editor.codeActionsOnSave": {
-        "source.organizeImports": true,
+        "source.organizeImports": "explicit"
     },
     "editor.formatOnSave": true,
     "editor.rulers": [
@@ -18,21 +18,15 @@
         "--profile=black",
         // "--src=${workspaceFolder}"
     ],
-    "python.formatting.autopep8Args": [
-        "--max-line-length",
-        "120",
-        "--experimental"
-    ],
-    "python.formatting.provider": "black",
-    "python.formatting.blackArgs": [
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
+    "black-formatter.args": [
         "--line-length=120"
     ],
-    "python.linting.enabled": true,
-    "python.linting.flake8Enabled": true,
-    "python.linting.flake8Args": [
+    "flake8.args": [
         "--max-line-length=120",
         //Added the ignore of E203 for now : https://github.com/PyCQA/pycodestyle/issues/373
         "--ignore=E203,W503"
     ],
-    "python.linting.pylintEnabled": false,
 }

CONTRIBUTING.md

@@ -5,7 +5,7 @@ This guide covers the basics of how to contribute to the Assemblyline project.
 Python code should follow the PEP8 guidelines defined here: [PEP8 Guidelines](https://www.python.org/dev/peps/pep-0008/).
 
 ## Tell us want you want to build/fix
-Before you start coding anything you should connect with the [Assemblyline community](https://groups.google.com/d/forum/cse-cst-assemblyline) to make sure no one else is working on the same thing and that whatever you are going to build still fits with the vision off the system.
+Before you start coding anything you should connect with the Assemblyline community via the [Assemblyline Discord server](https://discord.gg/GUAy9wErNu) and/or the [central Assemblyline GitHub project](https://github.com/CybercentreCanada/assemblyline/issues) to make sure no one else is working on the same thing and that whatever you are going to build still fits with the vision of the system.
 
 ## Git workflow
 
@@ -22,4 +22,4 @@ If you've worked on a new service that you want to be included in the default se
 
 Even if you try to merge in your pull request, you will be denied. Only a few people in our team are allowed to merge code into our repositories.
 
-We check for new pull requests every day and will merge them in once they have been approved by someone in our team.
+We check for new pull requests every day and will merge them in once they have been approved by someone in our team.

metapeek.py

@@ -203,6 +203,14 @@ def fna_check_unicode_bidir_ctrls(filename, f_ext):
         if not any(c in filename for c in BIDIR_CTRLS):
             return False
 
+        # get_display does not handle Explicit Directional Isolates, so we'll replace them.
+        filename = (
+            filename.replace("\u2066", "\u202A")  # Replace LRI with LRE
+            .replace("\u2067", "\u202B")  # Replace RLI with RLE
+            .replace("\u2068", "")  # Replace FSI with nothing?
+            .replace("\u2069", "\u202C")  # Replace PDI with PDF
+        )
+
         _, f_ext_display = os.path.splitext(get_display(filename))
         f_ext_display = remove_bidir_unicode_controls(f_ext_display)
 

service_manifest.yml

@@ -15,6 +15,7 @@ disable_cache: false
 enabled: true
 is_external: false
 licence_count: 0
+privileged: true
 
 config: {}
 
@@ -55,4 +56,4 @@ heuristics:
 docker_config:
   image: ${REGISTRY}cccs/assemblyline-service-metapeek:$SERVICE_TAG
   cpu_cores: 0.4
-  ram_mb: 256
+  ram_mb: 2048

tests/results/2166d65cfd532d5cbe163e9ea3fe7eebd34d81efe42fb64393eff04c037bcdcb/result.json

@@ -11,6 +11,7 @@
         "classification": "TLP:C",
         "depth": 0,
         "heuristic": null,
+        "promote_to": null,
         "tags": {
           "file": {
             "name": {
@@ -38,6 +39,7 @@
           "score_map": {},
           "signatures": {}
         },
+        "promote_to": null,
         "tags": {
           "file": {
             "name": {

tests/results/4178f38c423778bb19dd26983c6814706ea21ef45588709d4945a79e229bf5fe/result.json

@@ -11,6 +11,7 @@
         "classification": "TLP:C",
         "depth": 0,
         "heuristic": null,
+        "promote_to": null,
         "tags": {
           "file": {
             "name": {
@@ -38,6 +39,7 @@
           "score_map": {},
           "signatures": {}
         },
+        "promote_to": null,
         "tags": {
           "file": {
             "behavior": [

tests/results/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8_rtlo1/result.json

@@ -11,6 +11,7 @@
         "classification": "TLP:C",
         "depth": 0,
         "heuristic": null,
+        "promote_to": null,
         "tags": {
           "file": {
             "name": {
@@ -38,6 +39,7 @@
           "score_map": {},
           "signatures": {}
         },
+        "promote_to": null,
         "tags": {
           "file": {
             "behavior": [

tests/results/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8_rtlo7/result.json

@@ -11,6 +11,7 @@
         "classification": "TLP:C",
         "depth": 0,
         "heuristic": null,
+        "promote_to": null,
         "tags": {
           "file": {
             "name": {
@@ -38,6 +39,7 @@
           "score_map": {},
           "signatures": {}
         },
+        "promote_to": null,
         "tags": {
           "file": {
             "behavior": [

tests/test_bidi.py

@@ -0,0 +1,16 @@
+from unittest.mock import create_autospec
+
+from assemblyline_v4_service.common.request import ServiceRequest
+
+import metapeek
+
+
+def test_Explicit_Directional_Isolates():
+    file_name = "‎⁨‏واتساب⁩.pdf"
+
+    mock = create_autospec(ServiceRequest)
+    mock.file_name = file_name
+    mock.file_type = "document/pdf"
+
+    m = metapeek.MetaPeek()
+    m.check_file_name_anomalies(mock)