Fix GIL lock problems due to threading and add identifier finding

koivunen · koivunen · commit ee9cd87a4ac0 · 2021-08-23T18:02:51.000+03:00
diff --git a/config.py b/config.py
@@ -9,4 +9,5 @@
 
 homenetwork = ipaddress.ip_network('192.168.0.0/24')
 homenetwork_router = ipaddress.ip_address('192.168.0.1')
-aggregate_google=True # That is a lot of domains
+aggregate_google=True # That is a lot of domains
+ignored_domains=["osoite.local"]
diff --git a/connections.py b/connections.py
@@ -37,7 +37,6 @@ def getHomeIPFromConnection(conn):
 
 
 def getConnections():
-	global conns_dirty
 	"""
 	Returns connections list:
 	Dict of 'conntrackid': {'bytes': '2519',
@@ -54,13 +53,17 @@ def getConnections():
                'src': IPv4Address('192.168.0.142'),
                'state': 'ESTABLISHED'}					# if tcp connection, shows conntrack state
 	"""
+	
+	global conns_dirty,conns_mainthread
+	
+	# TODO: cleanup!!!
+	#assert threading.current_thread() is threading.main_thread()
+	
 	if conns_dirty:
 		conns_dirty=False
 		with conns_lock:
-			for k,v in conns.items():
-				if k not in conns_mainthread:
-					conns_mainthread[k]=v
-
+			conns_mainthread=dict(conns) # GIL cannot yield in a dict copy
+			
 	return conns_mainthread
 
 
diff --git a/connvis b/connvis
@@ -1 +1,6 @@
-sudo python3 main.py "$@"
+reset -I
+(
+	sudo python3 main.py "$@"
+)
+reset -I
+
diff --git a/ip2dns.py b/ip2dns.py
@@ -50,20 +50,30 @@ def load(*args):
 psl = PublicSuffixList()
 
 def shorten(domain):
-	return psl.privatesuffix(domain)
+	if not domain:
+		return domain
 
-import dnsmasq_parser
+	ret = psl.privatesuffix(domain)
+	if ret:
+		return ret
+	if not psl.is_public(domain):
+		print("FIXME: PSL could not digest:",domain)
+	return domain # it IS a private suffix or not a domain
+
+import dnsmasq_parser,config
 def onJournalMessage(entry):
 	l=entry["MESSAGE"]
 
 	r=dnsmasq_parser.parse_reply(l)
 	if r:
 		ip=r["ip"]
 		domain=r["domain"]
+		if domain in config.ignored_domains:
+			return
 		if not "NODATA" in ip and not "NXDOMAIN" in ip:
 			feed(ip,domain)
 		return r
-
+	
 def seedFromDnsmasq():
 
 	import select
diff --git a/monitor_domains.py b/monitor_domains.py
@@ -49,6 +49,7 @@ def lastDomainResolveTime(domain,short=False):
 import dnsmasq_parser
 import ip2dns
 import ipaddress
+import config
 def onJournalMessage(entry):
 	l=entry["MESSAGE"]
 	ts=entry['__REALTIME_TIMESTAMP'] #we are parsing real time, no use
@@ -57,6 +58,9 @@ def onJournalMessage(entry):
 	if q:
 		source = ipaddress.ip_address(q["source"])
 		domain=q["query"]
+		if domain in config.ignored_domains:
+			return
+
 		sdomain=ip2dns.shorten(domain)
 		now=time.time()
 
diff --git a/static/sankey/index.js b/static/sankey/index.js
@@ -59,8 +59,8 @@ async function updateLoop() {
 			await sleep(333);
 		};
 
+		var data = loadedData;
 		try {
-			var data = loadedData;
 			if (!data) {
 				let response = await fetch('/sankey.json');
 				lastDataJson = await response.text();
diff --git a/templates/hello.html b/templates/hello.html
@@ -19,9 +19,13 @@
 
 </head>
 <body>
-
+	
 	<h1>CONNVIS Real Time Network Monitoring and Visualizer</h1>
 	<br/>
+	<span><b>Your identifier / tunnisteesi: </b><h2 class="yourname">{{ yourname }}</h2></span>
+	<br/>
+	<hr/>
+	<br/>
 	<a class="vislink" href="sankey/index.html">Local device to destinations</a> (<a href="sankey.json">json</a>)
 	<br/>
 	<a class="vislink" href="geo/index.html">Targets on a world map</a> (<a href="geo.json">json</a>)
@@ -41,8 +45,9 @@ <h1>CONNVIS Real Time Network Monitoring and Visualizer</h1>
 	<br/>
 	<br/>
 	<br/>
-	<br/>
 	<hr>
+	<br/>
+	<br/>
 	 <small>This site or product includes IP2Location LITE data available from <a href="https://lite.ip2location.com">https://lite.ip2location.com</a>.</small>
 </body>
 </html>
diff --git a/utils.py b/utils.py
@@ -22,7 +22,40 @@ def shortenHomeIPMemorableUniq(homeip,original_important=True,net=config.homenet
 		homeipname=f"{homeip}\n({homeipname})"
 	return homeipname
 
+#TODO
+"""  File "/home/display/connvis/utils.py", line 26, in aggregateBigCompanies
+    if config.aggregate_google and "google" in identifier.lower(): # gvt2.com gstatic.om doubleclick
+AttributeError: 'NoneType' object has no attribute 'lower'
+130.232.129.162 - - [20/Aug/2021 13:49:50] "GET /dnsbar.json HTTP/1.1" 500 -
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.9/dist-packages/flask/app.py", line 2088, in __call__
+    return self.wsgi_app(environ, start_response)
+  File "/usr/local/lib/python3.9/dist-packages/flask/app.py", line 2073, in wsgi_app
+    response = self.handle_exception(e)
+  File "/usr/local/lib/python3.9/dist-packages/flask/app.py", line 2070, in wsgi_app
+    response = self.full_dispatch_request()
+  File "/usr/local/lib/python3.9/dist-packages/flask/app.py", line 1515, in full_dispatch_request
+    rv = self.handle_user_exception(e)
+  File "/usr/local/lib/python3.9/dist-packages/flask/app.py", line 1513, in full_dispatch_request
+    rv = self.dispatch_request()
+  File "/usr/local/lib/python3.9/dist-packages/flask/app.py", line 1499, in dispatch_request
+    return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
+  File "/home/display/connvis/vishttp.py", line 93, in dnsbar
+    return jsonify(dnsBarProvider())
+  File "/home/display/connvis/dnsvis.py", line 43, in provider
+    shortdomain=utils.aggregateBigCompanies(shortdomain)
+  File "/home/display/connvis/utils.py", line 26, in aggregateBigCompanies
+    if config.aggregate_google and "google" in identifier.lower(): # gvt2.com gstatic.om doubleclick
+AttributeError: 'NoneType' object has no attribute 'lower'
+130.232.129.162 - - [20/Aug/2021 13:49:53] "GET /dnsbar.json HTTP/1.1" 500 -
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.9/dist-packages/flask/app.py", line 2088, in __call__
+  """
+  
 def aggregateBigCompanies(identifier,where=False):
+	if not identifier: #TODO: ???
+		return identifier
+
 	if config.aggregate_google and "google" in identifier.lower(): # gvt2.com gstatic.om doubleclick
 			return "google.com" 
 	return identifier
diff --git a/vishttp.py b/vishttp.py
@@ -113,10 +113,15 @@ def setSankeyProvider(p):
 def sankeyjson():
 	return jsonify(SankeyProvider())
 
-
+import ipaddress,utils
 @app.route("/")
 def main():
-	return render_template("hello.html")
+	yourip=ipaddress.ip_address(request.remote_addr)
+	yourname=request.remote_addr
+	if yourip in config.homenetwork:
+		yourname=utils.shortenHomeIPMemorableUniq(yourip,True)
+		
+	return render_template("hello.html",yourip=yourip,yourname=yourname)
 
 
 def run(use_reloader=False):
