Merge pull request #250 from hillsandales/add-supplier-automationdire…

…ct-and-jameco

Add suppliers Automation Direct and Jameco

.gitignore

@@ -9,3 +9,4 @@ build/
 kintree/tests/*
 .coverage
 htmlcov/
+.vscode/launch.json

kintree/common/tools.py

@@ -60,17 +60,40 @@ def create_library(library_path: str, symbol: str, template_lib: str):
         copyfile(template_lib, new_kicad_sym_file)
 
 
-def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=False, requests_lib=False, silent=False):
+def get_image_with_retries(url, headers, retries=3, wait=5, silent=False):
+    """ Method to download image with cloudscraper library and retry attempts"""
+    import cloudscraper
+    import time
+    scraper = cloudscraper.create_scraper()
+    for attempt in range(retries):
+        try:
+            response = scraper.get(url, headers=headers)
+            if response.status_code == 200:
+                return response
+            else:
+                cprint(f'[INFO]\tWarning: Image download Attempt {attempt + 1} failed with status code {response.status_code}. Retrying in {wait} seconds...', silent=silent)
+        except Exception as e:
+            cprint(f'[INFO]\tWarning: Image download Attempt {attempt + 1} encountered an error: {e}. Retrying in {wait} seconds...', silent=silent)
+        time.sleep(wait)
+    cprint('[INFO]\tWarning: All Image download attempts failed. Could not retrieve the image.', silent=silent)
+    return None
+
+
+def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=False, requests_lib=False, try_cloudscraper=False, silent=False):
     ''' Standard method to download URL content, with option to save to local file (eg. images) '''
 
     import socket
     import urllib.request
     import requests
 
+    # A more detailed headers was needed for request to Jameco
     headers = {
-        'User-Agent': 'Mozilla/5.0',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
-        'Accept-Language': 'en-US,en;q=0.5',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36',
+        'Accept': 'applicaiton/json,image/webp,image/apng,image/*,*/*;q=0.8',
+        'Accept-Encoding': 'Accept-Encoding: gzip, deflate, br',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Connection': 'keep-alive',
+        'Cache-Control': 'no-cache',
     }
 
     # Set default timeout for download socket
@@ -95,17 +118,33 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=
                     return None
                 with open(fileoutput, 'wb') as file:
                     file.write(response.content)
+            elif try_cloudscraper:
+                response = get_image_with_retries(url, headers=headers)
+                if filetype.lower() not in response.headers['Content-Type'].lower():
+                    cprint(f'[INFO]\tWarning: {filetype} download returned the wrong file type', silent=silent)
+                    return None
+                with open(fileoutput, 'wb') as file:
+                    file.write(response.content)
             else:
                 (file, headers) = urllib.request.urlretrieve(url, filename=fileoutput)
                 if filetype.lower() not in headers['Content-Type'].lower():
                     cprint(f'[INFO]\tWarning: {filetype} download returned the wrong file type', silent=silent)
                     return None
             return file
         else:
-            url_data = urllib.request.urlopen(url)
-            data = url_data.read()
-            data_json = json.loads(data.decode('utf-8'))
-            return data_json
+            # some suppliers work with requests.get(), others need urllib.request.urlopen()
+            try:
+                response = requests.get(url)
+                data_json = response.json()
+                return data_json
+            except requests.exceptions.JSONDecodeError:
+                try:
+                    url_data = urllib.request.urlopen(url)
+                    data = url_data.read()
+                    data_json = json.loads(data.decode('utf-8'))
+                    return data_json
+                finally:
+                    pass
     except (socket.timeout, requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout):
         cprint(f'[INFO]\tWarning: {filetype} download socket timed out ({timeout}s)', silent=silent)
     except (urllib.error.HTTPError, requests.exceptions.ConnectionError):
@@ -136,9 +175,14 @@ def download_with_retry(url: str, full_path: str, silent=False, **kwargs) -> str
     if not file:
         # Try with requests library
         file = download(url, fileoutput=full_path, enable_headers=True, requests_lib=True, silent=silent, **kwargs)
+
+    if not file:
+        # Try with cloudscraper
+        file = download(url, fileoutput=full_path, enable_headers=True, requests_lib=False, try_cloudscraper=True, silent=silent, **kwargs)
 
     # Still nothing
     if not file:
         return False
 
+    cprint('[INFO]\tSuccess: Part image downloaded', silent=silent)
     return True

kintree/config/automationdirect/automationdirect_api.yaml

@@ -0,0 +1,5 @@
+AUTOMATIONDIRECT_API_ROOT_URL: "https://www.automationdirect.com"
+AUTOMATIONDIRECT_API_URL: "https://www.automationdirect.com/ajax?&fctype=adc.falcon.search.SearchFormCtrl&cmd=AjaxSearch"
+AUTOMATIONDIRECT_API_SEARCH_QUERY: "&searchquery="              # can be anything but probably best to set to search term
+AUTOMATIONDIRECT_API_SEARCH_STRING: "&solrQueryString=q%3D"     # append search term to this and combine with other params
+AUTOMATIONDIRECT_API_IMAGE_PATH: "https://cdn.automationdirect.com/images/products/medium/m_" # image path for medium size image on product page

kintree/config/automationdirect/automationdirect_config.yaml

@@ -0,0 +1,11 @@
+SUPPLIER_DATABASE_NAME: Automation Direct
+SEARCH_NAME: null
+SEARCH_DESCRIPTION: null
+SEARCH_REVISION: null
+SEARCH_KEYWORDS: null
+SEARCH_SKU: null
+SEARCH_MANUFACTURER: null
+SEARCH_MPN: null
+SEARCH_SUPPLIER_URL: null
+SEARCH_DATASHEET: null
+EXTRA_FIELDS: null

kintree/config/config_interface.py

@@ -78,7 +78,7 @@ def load_config(path):
 
             dump_file(user_settings, os.path.join(path_to_user_files, filename))
 
-    for dir in ['user', 'inventree', 'kicad', 'digikey', 'mouser', 'element14', 'lcsc', 'tme']:
+    for dir in ['user', 'inventree', 'kicad', 'digikey', 'mouser', 'element14', 'lcsc', 'tme', 'jameco', 'automationdirect']:
         try:
             # Load configuration
             config_files = os.path.join(path_to_root, dir, '')

kintree/config/inventree/suppliers.yaml

@@ -10,6 +10,12 @@ Element14:
 Farnell:
   enable: true
   name: Farnell
+Jameco:
+  enable: true
+  name: Jameco
+AutomationDirect:
+  enable: true
+  name: Automation Direct
 Newark:
   enable: true
   name: Newark

kintree/config/jameco/jameco_api.yaml

@@ -0,0 +1 @@
+JAMECO_API_URL: https://ahzbkf.a.searchspring.io/api/search/search.json?ajaxCatalog=v3&resultsFormat=native&siteId=ahzbkf&q=

kintree/config/jameco/jameco_config.yaml

@@ -0,0 +1,11 @@
+SUPPLIER_INVENTREE_NAME: Jameco Electronics
+SEARCH_NAME: null
+SEARCH_DESCRIPTION: null
+SEARCH_REVISION: null
+SEARCH_KEYWORDS: null
+SEARCH_SKU: null
+SEARCH_MANUFACTURER: null
+SEARCH_MPN: null
+SEARCH_SUPPLIER_URL: null
+SEARCH_DATASHEET: null
+EXTRA_FIELDS: null

kintree/config/settings.py

@@ -149,9 +149,11 @@ def load_suppliers():
     for supplier, data in CONFIG_SUPPLIERS.items():
         if data['enable']:
             if data['name']:
-                SUPPORTED_SUPPLIERS_API.append(data['name'])
+                supplier_name = data['name'].replace(' ', '')
+                SUPPORTED_SUPPLIERS_API.append(supplier_name)
             else:
-                SUPPORTED_SUPPLIERS_API.append(supplier)
+                supplier_key = supplier.replace(' ', '')
+                SUPPORTED_SUPPLIERS_API.append(supplier_key)
 
 
 load_suppliers()
@@ -178,6 +180,14 @@ def load_suppliers():
 CONFIG_LCSC = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'lcsc_config.yaml'))
 CONFIG_LCSC_API = os.path.join(CONFIG_USER_FILES, 'lcsc_api.yaml')
 
+# JAMECO user configuration
+CONFIG_JAMECO = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'jameco_config.yaml'))
+CONFIG_JAMECO_API = os.path.join(CONFIG_USER_FILES, 'jameco_api.yaml')
+
+# AUTOMATIONDIRECT user configuration
+CONFIG_AUTOMATIONDIRECT = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'automationdirect_config.yaml'))
+CONFIG_AUTOMATIONDIRECT_API = os.path.join(CONFIG_USER_FILES, 'automationdirect_api.yaml')
+
 # TME user configuration
 CONFIG_TME = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'tme_config.yaml'))
 CONFIG_TME_API = os.path.join(CONFIG_USER_FILES, 'tme_api.yaml')

kintree/database/inventree_api.py

@@ -512,8 +512,9 @@ def create_part(category_id: int, name: str, description: str, revision: str, ip
             'component': True,
             'purchaseable': True,
         })
-    except Exception:
+    except Exception as e:
         cprint('[TREE]\tError: Part creation failed. Check if Ki-nTree settings match InvenTree part settings.', silent=settings.SILENT)
+        cprint(repr(e), silent=settings.SILENT)
         return 0
 
     if part:

kintree/database/inventree_interface.py

@@ -5,7 +5,7 @@
 from ..common.tools import cprint
 from ..config import config_interface
 from ..database import inventree_api
-from ..search import search_api, digikey_api, mouser_api, element14_api, lcsc_api, tme_api
+from ..search import search_api, automationdirect_api, digikey_api, mouser_api, element14_api, lcsc_api, jameco_api, tme_api
 
 category_separator = '/'
 
@@ -401,8 +401,13 @@ def get_value_from_user_key(user_key: str, default_key: str, default_value=None)
             user_search_key = settings.CONFIG_ELEMENT14.get(user_key, None)
         elif supplier == 'LCSC':
             user_search_key = settings.CONFIG_LCSC.get(user_key, None)
+        elif supplier == 'Jameco':
+            user_search_key = settings.CONFIG_JAMECO.get(user_key, None)
         elif supplier == 'TME':
             user_search_key = settings.CONFIG_TME.get(user_key, None)
+        elif supplier == 'AutomationDirect':
+            user_search_key = settings.CONFIG_AUTOMATIONDIRECT.get(user_key, None)
+
         else:
             return default_value
 
@@ -425,8 +430,12 @@ def get_value_from_user_key(user_key: str, default_key: str, default_value=None)
         default_search_keys = element14_api.get_default_search_keys()
     elif supplier == 'LCSC':
         default_search_keys = lcsc_api.get_default_search_keys()
+    elif supplier == 'Jameco':
+        default_search_keys = jameco_api.get_default_search_keys()
     elif supplier == 'TME':
         default_search_keys = tme_api.get_default_search_keys()
+    elif supplier == 'AutomationDirect':
+        default_search_keys = automationdirect_api.get_default_search_keys()
     else:
         # Empty array of default search keys
         default_search_keys = [''] * len(digikey_api.get_default_search_keys())
@@ -459,8 +468,12 @@ def supplier_search(supplier: str, part_number: str, test_mode=False) -> dict:
 
     store = ''
     if supplier in ['Farnell', 'Newark', 'Element14']:
-        element14_config = config_interface.load_file(settings.CONFIG_ELEMENT14_API)
-        store = element14_config.get(f'{supplier.upper()}_STORE', '').replace(' ', '')
+        try:
+            element14_config = config_interface.load_file(settings.CONFIG_ELEMENT14_API)
+            store = element14_config.get(f'{supplier.upper()}_STORE', '').replace(' ', '')
+        except AttributeError:
+            cprint(f'\n[INFO]\tWarning: {supplier.upper()}_STORE value not found', silent=False)
+
     search_filename = f"{settings.search_results['directory']}{supplier}{store}_{part_number}{settings.search_results['extension']}"
     # Get cached data, if cache is enabled (else returns None)
     part_cache = search_api.load_from_file(search_filename, test_mode)
@@ -478,8 +491,12 @@ def supplier_search(supplier: str, part_number: str, test_mode=False) -> dict:
             part_info = element14_api.fetch_part_info(part_number, supplier)
         elif supplier == 'LCSC':
             part_info = lcsc_api.fetch_part_info(part_number)
+        elif supplier == 'Jameco':
+            part_info = jameco_api.fetch_part_info(part_number)
         elif supplier == 'TME':
             part_info = tme_api.fetch_part_info(part_number)
+        elif supplier == 'AutomationDirect':
+            part_info = automationdirect_api.fetch_part_info(part_number)
 
     # Check supplier data exist
     if not part_info:

kintree/gui/views/settings.py

@@ -81,6 +81,13 @@
             ft.TextField(),
             None,
         ]
+    elif supplier == 'Jameco':
+        jameco_api_settings = config_interface.load_file(global_settings.CONFIG_JAMECO_API)
+        supplier_settings[supplier]['API URL'] = [
+            jameco_api_settings['JAMECO_API_URL'],
+            ft.TextField(),
+            None,
+        ]
     elif supplier == 'TME':
         tme_api_settings = config_interface.load_file(global_settings.CONFIG_TME_API)
         supplier_settings[supplier]['API Token'] = [
@@ -103,6 +110,34 @@
             ft.TextField(),
             None,
         ]
+    elif supplier == 'AutomationDirect':
+        automationdirect_api_settings = config_interface.load_file(global_settings.CONFIG_AUTOMATIONDIRECT_API)
+        supplier_settings[supplier]['API Top-Level Root Domain'] = [
+            automationdirect_api_settings['AUTOMATIONDIRECT_API_ROOT_URL'],
+            ft.TextField(),
+            None,
+        ]
+        supplier_settings[supplier]['API URL Path'] = [
+            automationdirect_api_settings['AUTOMATIONDIRECT_API_URL'],
+            ft.TextField(),
+            None,
+        ]
+        supplier_settings[supplier]['API Search Query'] = [
+            automationdirect_api_settings['AUTOMATIONDIRECT_API_SEARCH_QUERY'],
+            ft.TextField(),
+            None,
+        ]
+        supplier_settings[supplier]['API Search String'] = [
+            automationdirect_api_settings['AUTOMATIONDIRECT_API_SEARCH_STRING'],
+            ft.TextField(),
+            None,
+        ]
+        supplier_settings[supplier]['API Image Path URL'] = [
+            automationdirect_api_settings['AUTOMATIONDIRECT_API_IMAGE_PATH'],
+            ft.TextField(),
+            None,
+        ]
+
 
 SETTINGS = {
     'User Settings': {
@@ -672,6 +707,15 @@ def save_s(self, e: ft.ControlEvent, supplier: str, show_dialog=True):
             }
             lcsc_settings = {**settings_from_file, **updated_settings}
             config_interface.dump_file(lcsc_settings, global_settings.CONFIG_LCSC_API)
+        elif supplier == 'Jameco':
+            # Load settings from file
+            settings_from_file = config_interface.load_file(global_settings.CONFIG_JAMECO_API)
+            # Update settings values
+            updated_settings = {
+                'JAMECO_API_URL': SETTINGS[self.title][supplier]['API URL'][1].value,
+            }
+            jameco_settings = {**settings_from_file, **updated_settings}
+            config_interface.dump_file(jameco_settings, global_settings.CONFIG_JAMECO_API)
         elif supplier == 'TME':
             # Load settings from file
             settings_from_file = config_interface.load_file(global_settings.CONFIG_TME_API)
@@ -684,7 +728,20 @@ def save_s(self, e: ft.ControlEvent, supplier: str, show_dialog=True):
             }
             tme_settings = {**settings_from_file, **updated_settings}
             config_interface.dump_file(tme_settings, global_settings.CONFIG_TME_API)
-
+        elif supplier == 'AutomationDirect':
+            # Load settings from file
+            settings_from_file = config_interface.load_file(global_settings.CONFIG_AUTOMATIONDIRECT_API)
+            # Update settings values
+            updated_settings = {
+                'AUTOMATIONDIRECT_API_ROOT_URL': SETTINGS[self.title][supplier]['API Top-Level Root Domain'][1].value,
+                'AUTOMATIONDIRECT_API_URL': SETTINGS[self.title][supplier]['API URL Path'][1].value,
+                'AUTOMATIONDIRECT_API_SEARCH_QUERY': SETTINGS[self.title][supplier]['API Search Query'][1].value,
+                'AUTOMATIONDIRECT_API_SEARCH_STRING': SETTINGS[self.title][supplier]['API Search String'][1].value,
+                'AUTOMATIONDIRECT_API_IMAGE_PATH': SETTINGS[self.title][supplier]['API Image Path URL'][1].value,
+            }
+            automationdirect_settings = {**settings_from_file, **updated_settings}
+            config_interface.dump_file(automationdirect_settings, global_settings.CONFIG_AUTOMATIONDIRECT_API)
+
         if show_dialog:
             self.show_dialog(
                 d_type=DialogType.VALID,
@@ -711,6 +768,12 @@ def test_s(self, e: ft.ControlEvent, supplier: str):
         elif supplier == 'TME':
             from ...search import tme_api
             result = tme_api.test_api()
+        elif supplier == 'Jameco':
+            from ...search import jameco_api
+            result = jameco_api.test_api()
+        elif supplier == 'AutomationDirect':
+            from ...search import automationdirect_api
+            result = automationdirect_api.test_api()
 
         if result:
             self.show_dialog(