Merge pull request #170 from sparkmicro/1.0.x

1.0.3

.github/workflows/test_deploy.yaml

@@ -51,6 +51,8 @@ jobs:
       TOKEN_DIGIKEY: ${{ secrets.TOKEN_DIGIKEY }}
       DIGIKEY_CLIENT_ID: ${{ secrets.DIGIKEY_CLIENT_ID }}
       DIGIKEY_CLIENT_SECRET: ${{ secrets.DIGIKEY_CLIENT_SECRET }}
+      TME_API_TOKEN: ${{ secrets.TME_API_TOKEN }}
+      TME_API_SECRET: ${{ secrets.TME_API_SECRET }}
 
     continue-on-error: true
     strategy:

kintree/common/tools.py

@@ -67,17 +67,22 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=
     import urllib.request
     import requests
 
+    headers = {
+        'User-Agent': 'Mozilla/5.0',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+    }
+
     # Set default timeout for download socket
     socket.setdefaulttimeout(timeout)
-    if enable_headers:
+    if enable_headers and not requests_lib:
         opener = urllib.request.build_opener()
-        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
+        opener.addheaders = list(headers.items())
         urllib.request.install_opener(opener)
     try:
         if filetype == 'Image' or filetype == 'PDF':
             # Enable use of requests library for downloading files (some URLs do NOT work with urllib)
             if requests_lib:
-                headers = {'User-agent': 'Mozilla/5.0'}
                 response = requests.get(url, headers=headers, timeout=timeout, allow_redirects=True)
                 if filetype.lower() not in response.headers['Content-Type'].lower():
                     cprint(f'[INFO]\tWarning: {filetype} download returned the wrong file type', silent=silent)
@@ -99,7 +104,7 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=
         cprint(f'[INFO]\tWarning: {filetype} download socket timed out ({timeout}s)', silent=silent)
     except (urllib.error.HTTPError, requests.exceptions.ConnectionError):
         cprint(f'[INFO]\tWarning: {filetype} download failed (HTTP Error)', silent=silent)
-    except (urllib.error.URLError, ValueError):
+    except (urllib.error.URLError, ValueError, AttributeError):
         cprint(f'[INFO]\tWarning: {filetype} download failed (URL Error)', silent=silent)
     except requests.exceptions.SSLError:
         cprint(f'[INFO]\tWarning: {filetype} download failed (SSL Error)', silent=silent)
@@ -114,7 +119,7 @@ def download_with_retry(url: str, full_path: str, silent=False, **kwargs) -> str
     if not url:
         cprint('[INFO]\tError: Missing image URL', silent=silent)
         return False
-
+    
     # Try without headers
     file = download(url, fileoutput=full_path, silent=silent, **kwargs)
 

kintree/config/tme/tme_api.yaml

@@ -1,4 +1,4 @@
 TME_API_TOKEN: NULL
 TME_API_SECRET: NULL
-TME_API_COUNTRY: NULL
-TME_API_LANGUAGE: NULL
+TME_API_COUNTRY: US
+TME_API_LANGUAGE: EN

kintree/database/inventree_interface.py

@@ -262,7 +262,11 @@ def translate_form_to_inventree(part_info: dict, category_tree: list, is_custom=
     inventree_part['supplier_link'] = part_info['supplier_link'].replace(' ', '%20')
     inventree_part['datasheet'] = part_info['datasheet'].replace(' ', '%20')
     # Image URL is not shown to user so force default key/value
-    inventree_part['image'] = part_info['image'].replace(' ', '%20')
+    try:
+        inventree_part['image'] = part_info['image'].replace(' ', '%20')
+    except AttributeError:
+        # Part image URL is null (no product picture)
+        pass
     inventree_part['pricing'] = part_info.get('pricing', {})
 
     # Load parameters map
@@ -575,12 +579,12 @@ def inventree_create(part_info: dict, kicad=False, symbol=None, footprint=None,
                     cprint('[TREE]\tWarning: Failed to upload part image', silent=settings.SILENT)
             if inventree_part['datasheet'] and settings.DATASHEET_UPLOAD:
                 # Upload datasheet
-                datasheet_link = inventree_api.upload_part_datasheet(
-                    inventree_part['datasheet'], part_pk)
+                datasheet_link = inventree_api.upload_part_datasheet(inventree_part['datasheet'], part_pk)
                 if not datasheet_link:
                     cprint('[TREE]\tWarning: Failed to upload part datasheet', silent=settings.SILENT)
-                else:
-                    inventree_part['datasheet'] = datasheet_link
+                # TODO: this is messing up with the datasheet download to local folder
+                # else:
+                #     inventree_part['datasheet'] = datasheet_link
 
         if kicad:
             try:
@@ -753,7 +757,7 @@ def inventree_create_alternate(part_info: dict, part_id='', part_ipn='', show_pr
     manufacturer_mpn = part_info.get('manufacturer_part_number', '')
     datasheet = part_info.get('datasheet', '')
 
-    # if datasheet upload is enabled and no attechment present yet upload
+    # if datasheet upload is enabled and no attachment present yet then upload
     if settings.DATASHEET_UPLOAD and not part.getAttachments():
         if datasheet:
             inventree_api.upload_part_datasheet(part_id=part_pk,

kintree/gui/views/main.py

@@ -11,7 +11,7 @@
 from .common import DropdownWithSearch, SwitchWithRefs
 from .common import handle_transition
 # Tools
-from ...common.tools import cprint, download
+from ...common.tools import cprint, download_with_retry
 # Settings
 from ...common import progress
 from ...config import settings, config_interface
@@ -1321,7 +1321,7 @@ def create_part(self, e=None):
                     f'{part_info.get("IPN", "datasheet")}.pdf',
                 )
                 cprint('\n[MAIN]\tDownloading Datasheet')
-                if download(datasheet_url, filetype='PDF', fileoutput=filename, timeout=10):
+                if download_with_retry(datasheet_url, filename, filetype='PDF', timeout=10):
                     cprint(f'[INFO]\tSuccess: Datasheet saved to {filename}')
         # Open browser
         if settings.ENABLE_INVENTREE:

kintree/search/tme_api.py

@@ -38,26 +38,34 @@ def check_environment() -> bool:
 def setup_environment(force=False) -> bool:
     if not check_environment() or force:
         tme_api_settings = config_interface.load_file(settings.CONFIG_TME_API)
-        os.environ['TME_API_TOKEN'] = tme_api_settings['TME_API_TOKEN']
-        os.environ['TME_API_SECRET'] = tme_api_settings['TME_API_SECRET']
+        os.environ['TME_API_TOKEN'] = tme_api_settings.get('TME_API_TOKEN', None)
+        os.environ['TME_API_SECRET'] = tme_api_settings.get('TME_API_SECRET', None)
 
     return check_environment()
 
 
 # Based on TME API snippets mentioned in API documentation: https://developers.tme.eu/documentation/download
 # https://github.com/tme-dev/TME-API/blob/master/Python/call.py
 def tme_api_request(endpoint, tme_api_settings, part_number, api_host='https://api.tme.eu', format='json'):
+    TME_API_TOKEN = tme_api_settings.get('TME_API_TOKEN', None)
+    TME_API_SECRET = tme_api_settings.get('TME_API_SECRET', None)
+
     params = collections.OrderedDict()
-    params['Country'] = tme_api_settings['TME_API_COUNTRY']
-    params['Language'] = tme_api_settings['TME_API_LANGUAGE']
+    params['Country'] = tme_api_settings.get('TME_API_COUNTRY', 'US')
+    params['Language'] = tme_api_settings.get('TME_API_LANGUAGE', 'EN')
     params['SymbolList[0]'] = part_number
-    params['Token'] = tme_api_settings['TME_API_TOKEN']
+    if not TME_API_TOKEN and not TME_API_SECRET:
+        TME_API_TOKEN = os.environ.get('TME_API_TOKEN', None)
+        TME_API_SECRET = os.environ.get('TME_API_SECRET', None)
+    if not TME_API_TOKEN and not TME_API_SECRET:
+        return None
+    params['Token'] = TME_API_TOKEN
 
     url = api_host + endpoint + '.' + format
     encoded_params = urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
     signature_base = 'POST' + '&' + urllib.parse.quote(url, '') + '&' + urllib.parse.quote(encoded_params, '')
     hmac_value = hmac.new(
-        tme_api_settings['TME_API_SECRET'].encode(),
+        TME_API_SECRET.encode(),
         signature_base.encode(),
         hashlib.sha1
     ).digest()

run_tests.py

@@ -2,7 +2,7 @@
 import sys
 
 import kintree.config.settings as settings
-from kintree.common.tools import cprint, create_library, download, download_with_retry
+from kintree.common.tools import cprint, create_library, download_with_retry
 from kintree.config import config_interface
 from kintree.database import inventree_api, inventree_interface
 from kintree.kicad import kicad_interface
@@ -386,20 +386,26 @@ def check_result(status: str, new_part: bool) -> bool:
                     test_pdf_urllib = 'https://www.seielect.com/Catalog/SEI-CF_CFM.pdf'
                     # Test different download methods for images
                     if not download_with_retry(test_image_urllib, './image1.jpg', silent=True, filetype='Image'):
+                        print(' [1] ')
                         method_success = False
-                    if not download_with_retry(test_image_requestslib, './image2.jpg', silent=True, filetype='Image',):
+                    if not download_with_retry(test_image_requestslib, './image2.jpg', silent=True, filetype='Image'):
+                        print(' [2] ')
                         method_success = False
                     # Test PDF
-                    if not download(test_pdf_urllib, filetype='PDF', fileoutput='./datasheet.pdf', silent=True):
+                    if not download_with_retry(test_pdf_urllib, './datasheet.pdf', silent=True, filetype='PDF'):
+                        print(' [3] ')
                         method_success = False
                     # Wrong folder
-                    if download(test_pdf_urllib, filetype='PDF', fileoutput='./myfolder/datasheet.pdf', silent=True):
+                    if download_with_retry(test_pdf_urllib, './myfolder/datasheet.pdf', silent=True, filetype='PDF'):
+                        print(' [4] ')
                         method_success = False
                     # Test erroneous URL
                     if download_with_retry('http', '', silent=True):
+                        print(' [5] ')
                         method_success = False
                     # Test empty URL
                     if download_with_retry('', '', silent=True):
+                        print(' [6] ')
                         method_success = False
 
                 elif method_idx == 9: