Skip to content

Commit 5b190a0

Browse files
inbaltakoinbaltako
authored
Merge pull request #46 from securenative/dev
Remove pii data
2 parents 00e68a0 + 007b4e7 commit 5b190a0

File tree

10 files changed

+175
-7
lines changed

10 files changed

+175
-7
lines changed

‎.github/workflows/publish.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,12 @@ jobs:
2424
uses: actions/setup-python@v2
2525
with:
2626
python-version: '3.6'
27+
2728
- name: Install dependencies
2829
run: |
2930
python -m pip install --upgrade pip
3031
pip install setuptools wheel twine
32+
3133
- name: Build and publish
3234
env:
3335
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}

‎README.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,4 +193,29 @@ from securenative.config.securenative_options import SecureNativeOptions
193193

194194
options = SecureNativeOptions(api_key="YOUR_API_KEY", max_events=10, log_level="ERROR", proxy_headers=['CF-Connecting-IP'])
195195
securenative = SecureNative.init_with_options(options)
196+
```
197+
198+
199+
## Remove PII Data From Headers
200+
201+
By default, SecureNative SDK remove any known pii headers from the received request.
202+
We also support using custom pii headers and regex matching via configuration, for example:
203+
204+
### Option 1: Using config file
205+
```ini
206+
SECURENATIVE_API_KEY: "YOUR_API_KEY"
207+
SECURENATIVE_PII_HEADERS: ["apiKey"]
208+
```
209+
210+
Initialize sdk as shown above.
211+
212+
### Options 2: Using ConfigurationBuilder
213+
214+
```python
215+
from securenative.securenative import SecureNative
216+
from securenative.config.securenative_options import SecureNativeOptions
217+
218+
219+
options = SecureNativeOptions(api_key="YOUR_API_KEY", max_events=10, log_level="ERROR", pii_regex_pattern='((?i)(http_auth_)(\w+)?)')
220+
securenative = SecureNative.init_with_options(options)
196221
```

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.3.3
1+
0.3.6

‎securenative/config/configuration_manager.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,9 @@ def load_config(cls, resource_path):
6969
"SECURENATIVE_FAILOVER_STRATEGY",
7070
options.fail_over_strategy),
7171
proxy_headers=cls._get_env_or_default(properties, "SECURENATIVE_PROXY_HEADERS",
72-
options.proxy_headers))
72+
options.proxy_headers),
73+
pii_headers=cls._get_env_or_default(properties, "SECURENATIVE_PII_HEADERS",
74+
options.pii_headers),
75+
pii_regex_pattern=cls._get_env_or_default(properties,
76+
"SECURENATIVE_PII_REGEX_PATTERN",
77+
options.pii_regex_pattern))

‎securenative/config/securenative_options.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ class SecureNativeOptions(object):
55

66
def __init__(self, api_key=None, api_url="https://api.securenative.com/collector/api/v1", interval=1000,
77
max_events=1000, timeout=1500, auto_send=True, disable=False, log_level="CRITICAL",
8-
fail_over_strategy=FailOverStrategy.FAIL_OPEN.value, proxy_headers=None):
8+
fail_over_strategy=FailOverStrategy.FAIL_OPEN.value, proxy_headers=None,
9+
pii_headers=None, pii_regex_pattern=None):
910

1011
if proxy_headers is None:
1112
proxy_headers = []
@@ -24,3 +25,5 @@ def __init__(self, api_key=None, api_url="https://api.securenative.com/collector
2425
self.disable = disable
2526
self.log_level = log_level
2627
self.proxy_headers = proxy_headers
28+
self.pii_headers = pii_headers
29+
self.pii_regex_pattern = pii_regex_pattern

‎securenative/context/securenative_context.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ def from_http_request(request, options):
2121
client_token = None
2222

2323
try:
24-
headers = dict(request.headers)
24+
headers = RequestUtils.get_headers_from_request(request.headers, options)
2525
except Exception:
2626
headers = None
2727

‎securenative/utils/request_utils.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1+
import re
2+
13
from securenative.utils.ip_utils import IpUtils
24

35

46
class RequestUtils(object):
57
SECURENATIVE_COOKIE = "_sn"
68
SECURENATIVE_HEADER = "x-securenative"
79
IP_HEADERS = ["HTTP_X_FORWARDED_FOR", "X_FORWARDED_FOR", "REMOTE_ADDR", "x-forwarded-for", "x-client-ip", "x-real-ip", "x-forwarded", "x-cluster-client-ip", "forwarded-for", "forwarded", "via"]
10+
PII_HEADERS = ['authorization', 'access_token', 'apikey', 'password', 'passwd', 'secret', 'api_key']
811

912
@staticmethod
1013
def get_secure_header_from_request(headers):
@@ -15,7 +18,7 @@ def get_secure_header_from_request(headers):
1518

1619
@staticmethod
1720
def get_client_ip_from_request(request, options):
18-
if options and len(options.proxy_headers) > 0:
21+
if options and options.proxy_headers and len(options.proxy_headers) > 0:
1922
for header in options.proxy_headers:
2023
try:
2124
if request.environ.get(header) is not None:
@@ -76,3 +79,21 @@ def get_valid_ip(ips):
7679

7780
if IpUtils.is_loop_back(ip):
7881
return ip
82+
83+
@staticmethod
84+
def get_headers_from_request(headers, options=None):
85+
h = {}
86+
if options and options.pii_headers and len(options.pii_headers) > 0:
87+
for header in headers:
88+
if header not in options.pii_headers and header.upper() not in options.pii_headers:
89+
h[header] = headers[header]
90+
elif options and options.pii_regex_pattern:
91+
for header in headers:
92+
if not re.search(options.pii_regex_pattern, header):
93+
h[header] = headers[header]
94+
else:
95+
for header in headers:
96+
if header not in RequestUtils.PII_HEADERS and header.upper() not in RequestUtils.PII_HEADERS:
97+
h[header] = headers[header]
98+
99+
return h

‎securenative/utils/version_utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ class VersionUtils(object):
22

33
@staticmethod
44
def get_version():
5-
return "0.3.3"
5+
return "0.3.6"

‎tests/configuration_manager_test.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,9 @@ def test_parse_config_file_correctly(self):
6363
"SECURENATIVE_DISABLE": "False",
6464
"SECURENATIVE_LOG_LEVEL": "Critical",
6565
"SECURENATIVE_FAILOVER_STRATEGY": "fail-closed",
66-
"SECURENATIVE_PROXY_HEADERS": "CF-Connecting-IP,Some-Random-Ip"
66+
"SECURENATIVE_PROXY_HEADERS": "CF-Connecting-IP,Some-Random-Ip",
67+
"SECURENATIVE_PII_HEADERS": "authentication,api_key",
68+
"SECURENATIVE_PII_REGEX_PATTERN": "/auth/i"
6769
}
6870

6971
self.create_ini_file(config)
@@ -79,7 +81,9 @@ def test_parse_config_file_correctly(self):
7981
self.assertEqual(options.log_level, "Critical")
8082
self.assertEqual(options.max_events, "100")
8183
self.assertEqual(options.timeout, "1500")
84+
self.assertEqual(options.pii_regex_pattern, "/auth/i")
8285
self.assertEqual(options.proxy_headers, ["CF-Connecting-IP", "Some-Random-Ip"])
86+
self.assertEqual(options.pii_headers, ["authentication", "api_key"])
8387

8488
@unittest.skipIf(platform.system() == "Windows" or platform.system() == "windows", "test not supported on windows")
8589
def test_ignore_unknown_config_in_properties_file(self):

‎tests/request_utils_test.py‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,3 +308,111 @@ def test_extraction_priority_without_x_forwarded_for(self):
308308
client_ip = RequestUtils.get_client_ip_from_request(request, options)
309309

310310
self.assertEqual("203.0.113.1", client_ip)
311+
312+
def test_strip_down_pii_data_from_headers(self):
313+
headers = {
314+
'Host': 'net.example.com',
315+
'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 (.NET CLR 3.5.30729)',
316+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
317+
'Accept-Language': 'en-us,en;q=0.5',
318+
'Accept-Encoding': 'gzip,deflate',
319+
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
320+
'Keep-Alive': '300',
321+
'Connection': 'keep-alive',
322+
'Cookie': 'PHPSESSID=r2t5uvjq435r4q7ib3vtdjq120',
323+
'Pragma': 'no-cache',
324+
'Cache-Control': 'no-cache',
325+
'authorization': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
326+
'access_token': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
327+
'apikey': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
328+
'password': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
329+
'passwd': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
330+
'secret': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
331+
'api_key': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z'
332+
}
333+
334+
with requests_mock.Mocker(real_http=True) as request:
335+
request.headers = headers
336+
337+
h = RequestUtils.get_headers_from_request(request.headers)
338+
339+
self.assertEqual(h.get('authorization'), None)
340+
self.assertEqual(h.get('access_token'), None)
341+
self.assertEqual(h.get('apikey'), None)
342+
self.assertEqual(h.get('password'), None)
343+
self.assertEqual(h.get('passwd'), None)
344+
self.assertEqual(h.get('secret'), None)
345+
self.assertEqual(h.get('api_key'), None)
346+
347+
def test_strip_down_pii_data_from_custom_headers(self):
348+
headers = {
349+
'Host': 'net.example.com',
350+
'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 (.NET CLR 3.5.30729)',
351+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
352+
'Accept-Language': 'en-us,en;q=0.5',
353+
'Accept-Encoding': 'gzip,deflate',
354+
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
355+
'Keep-Alive': '300',
356+
'Connection': 'keep-alive',
357+
'Cookie': 'PHPSESSID=r2t5uvjq435r4q7ib3vtdjq120',
358+
'Pragma': 'no-cache',
359+
'Cache-Control': 'no-cache',
360+
'authorization': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
361+
'access_token': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
362+
'apikey': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
363+
'password': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
364+
'passwd': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
365+
'secret': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
366+
'api_key': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z'
367+
}
368+
369+
with requests_mock.Mocker(real_http=True) as request:
370+
request.headers = headers
371+
372+
options = SecureNativeOptions(pii_headers=['authorization', 'access_token', 'apikey', 'password',
373+
'passwd', 'secret', 'api_key'])
374+
h = RequestUtils.get_headers_from_request(request.headers, options)
375+
376+
self.assertEqual(h.get('authorization'), None)
377+
self.assertEqual(h.get('access_token'), None)
378+
self.assertEqual(h.get('apikey'), None)
379+
self.assertEqual(h.get('password'), None)
380+
self.assertEqual(h.get('passwd'), None)
381+
self.assertEqual(h.get('secret'), None)
382+
self.assertEqual(h.get('api_key'), None)
383+
384+
def test_strip_down_pii_data_from_regex_pattern(self):
385+
headers = {
386+
'Host': 'net.example.com',
387+
'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 (.NET CLR 3.5.30729)',
388+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
389+
'Accept-Language': 'en-us,en;q=0.5',
390+
'Accept-Encoding': 'gzip,deflate',
391+
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
392+
'Keep-Alive': '300',
393+
'Connection': 'keep-alive',
394+
'Cookie': 'PHPSESSID=r2t5uvjq435r4q7ib3vtdjq120',
395+
'Pragma': 'no-cache',
396+
'Cache-Control': 'no-cache',
397+
'http_auth_authorization': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
398+
'http_auth_access_token': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
399+
'http_auth_apikey': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
400+
'http_auth_password': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
401+
'http_auth_passwd': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
402+
'http_auth_secret': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z',
403+
'http_auth_api_key': 'ylSkZIjbdWybfs4fUQe9BqP0LH5Z'
404+
}
405+
406+
with requests_mock.Mocker(real_http=True) as request:
407+
request.headers = headers
408+
409+
options = SecureNativeOptions(pii_regex_pattern='((?i)(http_auth_)(\w+)?)')
410+
h = RequestUtils.get_headers_from_request(request.headers, options)
411+
412+
self.assertEqual(h.get('http_auth_authorization'), None)
413+
self.assertEqual(h.get('http_auth_access_token'), None)
414+
self.assertEqual(h.get('http_auth_apikey'), None)
415+
self.assertEqual(h.get('http_auth_password'), None)
416+
self.assertEqual(h.get('http_auth_passwd'), None)
417+
self.assertEqual(h.get('http_auth_secret'), None)
418+
self.assertEqual(h.get('http_auth_api_key'), None)

0 commit comments

Comments
 (0)