Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Twilio credentials markup #173

Merged
merged 2 commits into from
Nov 11, 2024
Merged

Twilio credentials markup #173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
81d2575
Twilio credentials markup
babenek Nov 6, 2024
d4f6aa4
Removed extra false twilio
babenek Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions .ci/benchmark.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
META MD5 ef775241a6d575ff10f7220dcfadf5d7
DATA MD5 51b6d4e4debbd374fc184f2b691e0bb8
DATA: 16344639 interested lines. MARKUP: 62827 items
META MD5 72b4b7db8a2ffef0f19e802c09032e14
DATA MD5 abd9c025d5c323af814fbeb33f469c90
DATA: 16342283 interested lines. MARKUP: 62020 items
FileType FileNumber ValidLines Positives Negatives Templates
--------------- ------------ ------------ ----------- ----------- -----------
194 28318 71 418 90
Expand Down Expand Up @@ -55,7 +55,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.erb 13 323 27
.erl 4 96 7
.ex 25 4968 5 98 5
.example 17 1838 76 38 52
.example 17 1838 77 38 52
.exs 24 4842 8 187 4
.ext 5 211 1 4 2
.fsproj 1 75 1 2
Expand Down Expand Up @@ -159,7 +159,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.pyx 2 1094 23
.r 4 62 6 3 1
.rake 2 51 2
.rb 860 131838 254 3340 615
.rb 838 129482 281 2529 613
.re 1 31 1
.red 1 159 1
.release 1 13 4
Expand Down Expand Up @@ -222,7 +222,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.yml 419 36169 559 889 376
.zsh 6 872 12
.zsh-theme 1 97 1
TOTAL: 10254 16344639 12227 50501 5104
TOTAL: 10232 16342283 12255 49690 5102
credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0
Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1
------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ----
Expand All @@ -242,7 +242,7 @@ CMD Password 21 128 6
CMD Secret 1 1 0 0 0 1 1 0.000000 1.000000 0.500000 0.000000
CMD Token 6 0 0 0 0 0 6 1.000000 0.000000 0.000000
Certificate 24 471 0 0 0 471 24 0.000000 1.000000 0.951515 0.000000
Credential 93 419 76 0 0 495 93 0.000000 1.000000 0.841837 0.000000
Credential 91 421 76 0 0 497 91 0.000000 1.000000 0.845238 0.000000
Docker Swarm Token 2 0 0 0 0 0 2 1.000000 0.000000 0.000000
Dropbox App secret 64 139 1 0 0 140 64 0.000000 1.000000 0.686275 0.000000
Facebook Access Token 0 1 0 0 0 1 0 0.000000 1.000000
Expand All @@ -259,7 +259,7 @@ Jira / Confluence PAT token 0 4 0
Jira 2FA 15 6 1 0 0 7 15 0.000000 1.000000 0.318182 0.000000
Key 3909 15717 485 0 0 16202 3909 0.000000 1.000000 0.805629 0.000000
Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000
Other 8 8292 1 0 0 8293 8 0.000000 1.000000 0.999036 0.000000
Other 8 7445 1 0 0 7446 8 0.000000 1.000000 0.998927 0.000000
PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000
Password 1869 7535 2680 0 0 10215 1869 0.000000 1.000000 0.845333 0.000000
Salt 47 76 1 0 0 77 47 0.000000 1.000000 0.620968 0.000000
Expand All @@ -268,7 +268,7 @@ Seed 1 6 0
Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000
Tencent WeChat API App ID 6 0 0 0 0 0 6 1.000000 0.000000 0.000000
Token 643 4170 454 0 0 4624 643 0.000000 1.000000 0.877919 0.000000
Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000
Twilio Credentials 30 39 0 0 0 39 30 0.000000 1.000000 0.565217 0.000000
URL Credentials 210 156 216 0 0 372 210 0.000000 1.000000 0.639175 0.000000
UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000
12227 50501 5104 0 0 0 50501 12227 0.000000 1.000000 0.805079 0.000000
12255 49690 5102 0 0 0 49690 12255 0.000000 1.000000 0.802163 0.000000
12 changes: 8 additions & 4 deletions download_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ def obfuscate_jwt(value: str) -> str:
b'"n":', b'"nbf":', b'"nonce":', b'"oth":', b'"p":', b'"p2c":', b'"p2s":', b'"password":', b'"ppt":',
b'"q":', b'"qi":', b'"role":', b'"secret":', b'"sub":', b'"svt":', b'"tag":', b'"token":', b'"typ":',
b'"url":', b'"use":', b'"x":', b'"x5c":', b'"x5t":', b'"x5t#S256":', b'"x5u":', b'"y":', b'"zip":'
]:
]:
# safe words to keep JSON structure (false, true, null)
# and important JWT ("alg", "type", ...)
if decoded[n:n + len(wrd)] == wrd:
Expand Down Expand Up @@ -287,7 +287,7 @@ def obfuscate_jwt(value: str) -> str:


def get_obfuscated_value(value, meta_row: MetaRow):
if "Info" == meta_row.PredefinedPattern or meta_row.Category in ["IPv4", "IPv6"]:
if "Info" == meta_row.PredefinedPattern:
# not a credential - does not required obfuscation
obfuscated_value = value
elif value.startswith("Apikey "):
Expand Down Expand Up @@ -333,8 +333,11 @@ def get_obfuscated_value(value, meta_row: MetaRow):
obfuscated_value = value[:9] + generate_value(value[9:])
elif value.startswith("hooks.slack.com/services/"):
obfuscated_value = "hooks.slack.com/services/" + generate_value(value[25:])
elif value.startswith("wx") and 18 == len(value):
obfuscated_value = "wx" + generate_value(value[2:])
elif (value.startswith("wx") and 18 == len(value)
or (any(value.startswith(x) for x in
["AC", "AD", "AL", "CA", "CF", "CL", "CN", "CR", "FW", "IP", "KS", "MM", "NO", "PK", "PN", "QU", "RE",
"SC", "SD", "SK", "SM", "TR", "UT", "XE", "XR"]) and 34 == len(value))):
obfuscated_value = value[:2] + generate_value(value[2:])
elif ".apps.googleusercontent.com" in value:
pos = value.index(".apps.googleusercontent.com")
obfuscated_value = generate_value(value[:pos]) + ".apps.googleusercontent.com" + generate_value(
Expand Down Expand Up @@ -376,6 +379,7 @@ def check_asc_or_desc(line_data_value: str) -> bool:
continue
return False


def generate_value(value):
"""Wrapper to skip obfuscation with false positive or negatives"""
pattern_keyword = re.compile(r"(api|pass|pw[d\b])", flags=re.IGNORECASE)
Expand Down
1 change: 1 addition & 0 deletions meta/31423103.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,3 +112,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
1023934,a3046da0,GitHub,31423103,data/31423103/test/a3046da0.example,46,46,F,F,,,F,F,,,,,0,0,F,F,F,Key
1338567,a3046da0,GitHub,31423103,data/31423103/test/a3046da0.example,47,47,F,F,12,44,F,F,,,,,0.0,0,F,F,F,Key
1339450,a3046da0,GitHub,31423103,data/31423103/test/a3046da0.example,20,20,T,F,12,48,F,F,,,,,0.0,0,F,F,F,UUID
1479653,a3046da0,GitHub,31423103,data/31423103/test/a3046da0.example,43,43,T,F,11,45,F,F,,,,,0.0,0,F,F,F,Twilio Credentials
Loading
Loading