[CLEANUP]

flask_api.py

@@ -9,12 +9,12 @@
 from toolkit.analysis.keywords import generate_results
 from flask import Flask, render_template, request
 import logging
-logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.DEBUG, datefmt='%m/%d/%Y %I:%M:%S %p')
+logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',
+                    level=logging.DEBUG, datefmt='%m/%d/%Y %I:%M:%S %p')
 
 app = Flask(__name__, template_folder='toolkit/templates')
 
 
-
 def initialize_db(conn):
     """Initialize the DB
 
@@ -53,9 +53,6 @@ def update_or_insert_graph_in_db(conn, urls, maximum, update=False):
     return render_template("bokeh.html", script=script, div=div, domain=domain, template="Flask", time=datetime.now().strftime("%m/%d/%Y, %H:%M:%S"))
 
 
-
-
-
 def generate_interactive_graph(conn, urls, relaunch, maxi_urls):
     if urls is None:
         return "Empty Url paramaters"
@@ -99,6 +96,7 @@ def interactive_graph():
         return generate_interactive_graph(conn, urls, relaunch, maxi_urls)
     conn.close()
 
+
 @app.route('/api/headers')
 def find_headers():
     value = request.args.get('url')
@@ -107,15 +105,16 @@ def find_headers():
     else:
         return "Please input a valid value like this: /api/headers?url='https://primates.dev'"
 
+
 @app.route('/api/serp')
 def find_rank_query():
     query = request.args.get('query')
     domain = request.args.get('domain')
     tld = request.args.get('tld')
     lang = request.args.get('lang')
-    print (lang)
+    print(lang)
     if query and domain:
-        return rank(domain,query, lang=lang, tld=tld)
+        return rank(domain, query, lang=lang, tld=tld)
     else:
         return 'Please input a valid value like this: /api/serp?domain=primates.dev&query=parse api xml response&tld=com&lang=en'
 
@@ -125,14 +124,12 @@ def find_keywords_query():
     conn = conf.create_connection("visited.db")
     query = request.args.get('query')
     if query:
-        return analysis.get_query_results(conn,query)
-        
+        return analysis.get_query_results(conn, query)
+
     else:
         return 'Please input a valid value like this: /api/analysis/keywords?query=parse api xml response'
 
 
-
-
 if __name__ == '__main__':
 
     conn = conf.create_connection("visited.db")
@@ -144,4 +141,4 @@ def find_keywords_query():
         logging.warning("Error! cannot create the database connection.")
 
     logging.info("DB running")
-    app.run(debug=True, port=5000)  # run app in debug mode on port 5000
+    app.run(port=5000)  # run app in debug mode on port 5000

toolkit/db/analysis.py

@@ -2,6 +2,7 @@
 from toolkit.analysis import keywords
 import json
 
+
 def insert_query_db(conn, values):
     """Insert a query into the DB
 
@@ -15,13 +16,15 @@ def insert_query_db(conn, values):
     cur.execute(sql, values)
     conn.commit()
 
+
 def select_query(conn, query):
     cur = conn.cursor()
     cur.execute("SELECT * FROM keywords WHERE query=?", (query,))
 
     row = cur.fetchall()
     return row
 
+
 def update_running_db(conn, query, status):
     """Update running table where url = input
 
@@ -37,6 +40,7 @@ def update_running_db(conn, query, status):
     cur.execute(sql, (status, query))
     conn.commit()
 
+
 def update_result_db(conn, task):
     sql = ''' UPDATE keywords
               SET
@@ -48,6 +52,7 @@ def update_result_db(conn, task):
     cur.execute(sql, task)
     conn.commit()
 
+
 def finished_all_jobs(conn):
     sql = ''' UPDATE keywords
               SET
@@ -68,13 +73,9 @@ def get_query_results(conn, query, redo=False):
             return json.loads(check_exist[0][2])
         print(check_exist)
     else:
-        insert_query_db(conn, (query,"","RUNNING",datetime.datetime.now()))
+        insert_query_db(conn, (query, "", "RUNNING", datetime.datetime.now()))
         results = keywords.generate_results(query, 20)
-        update_result_db(conn, ("FINISHED", datetime.datetime.now(), json.dumps(results), query))
+        update_result_db(
+            conn, ("FINISHED", datetime.datetime.now(), json.dumps(results), query))
         return results
     return "Blabla"
-
-
-
-
-

toolkit/db/conf.py

@@ -26,7 +26,6 @@
                             """
 
 
-
 def create_table(conn, create_table_sql):
     """Create a table in the database
 
@@ -40,8 +39,9 @@ def create_table(conn, create_table_sql):
     except sqlite3.Error as e:
         logging.warning(e)
 
+
 def create_connection(db_file):
-    
+
     conn = None
     try:
         conn = sqlite3.connect(db_file)
@@ -51,6 +51,7 @@ def create_connection(db_file):
 
     return conn
 
+
 def update_running_db_stopped(conn):
     task = ["STOPPED"]
     sql = ''' UPDATE running
@@ -65,4 +66,3 @@ def update_running_db_stopped(conn):
     cur = conn.cursor()
     cur.execute(sql, ("FINISHED",))
     conn.commit()
-

toolkit/db/seo.py

@@ -21,6 +21,7 @@ def update_running_status(conn, urls, status="STOPPED", already_exists=True):
     else:
         update_running_db(conn, ("STOPPED", urls))
 
+
 def insert_url_db(conn, result):
     """Insert an url into the DB
 
@@ -83,8 +84,6 @@ def update_running_db(conn, task):
     conn.commit()
 
 
-
-
 def select_visited(conn, urls):
     cur = conn.cursor()
     cur.execute("SELECT * FROM visited WHERE urls=?", (urls,))

toolkit/lib/http_tools.py

@@ -1,6 +1,7 @@
 import requests
 from bs4 import BeautifulSoup
 
+
 def request_status_code(url):
     try:
         response = requests.get(url)

toolkit/seo/headers.py

@@ -1,5 +1,6 @@
 from toolkit.lib.http_tools import request_parse, request_status_code
 
+
 def find_all_headings(soup):
     headings = {"h1": {"count": 0, "values": []}, "h2": {"count": 0, "values": []},
                 "h3": {"count": 0, "values": []}, "h4": {"count": 0, "values": []},
@@ -9,6 +10,7 @@ def find_all_headings(soup):
         headings[heading.name]["count"] += 1
     return headings
 
+
 def find_all_headers_url(url):
     soup = request_parse(url)
     if soup:
@@ -17,7 +19,6 @@ def find_all_headers_url(url):
         return {"Error": "No headers found or error in the url"}
 
 
-
 def print_all_headers(headers_list):
     for key in headers_list:
         for i in headers_list[key]["header"]:

toolkit/seo/rank.py

@@ -1,31 +1,33 @@
 from googlesearch import search
 import logging
 
-def rank(domain, query, lang = 'en', tld = 'com'):
+
+def rank(domain, query, lang='en', tld='com'):
     my_results_list = []
     lang = lang
     tld = tld
     if lang is None:
         lang = 'en'
     if tld is None:
-        tld = 'com' 
+        tld = 'com'
 
     for i in search(query,        # The query you want to run
-                    tld = 'com',  # The top level domain
-                    lang = 'en',  # The language
-                    num = 10,     # Number of results per page
-                    start = 0,    # First result to retrieve
-                    stop = 100,  # Last result to retrieve
-                    pause = 2.0,  # Lapse between HTTP requests
-                ):
+                    tld='com',  # The top level domain
+                    lang='en',  # The language
+                    num=10,     # Number of results per page
+                    start=0,    # First result to retrieve
+                    stop=100,  # Last result to retrieve
+                    pause=2.0,  # Lapse between HTTP requests
+                    ):
         my_results_list.append(i)
         logging.debug(str(my_results_list.index(i)) + ": " + str(i))
         if domain in i:
             return {"pos": my_results_list.index(i) + 1, "url": i}
     return {"pos": -1, "url": None}
 
+
 if __name__ == "__main__":
-    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.DEBUG, datefmt='%m/%d/%Y %I:%M:%S %p')
+    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',
+                        level=logging.DEBUG, datefmt='%m/%d/%Y %I:%M:%S %p')
 
     print(rank("primates.dev", "parse xml response python"))
-