Merge pull request #246 from sparkmicro/pr/T0jan/233

Better handling of datasheet links (local and server)

kintree/__init__.py

@@ -4,8 +4,8 @@
 # VERSION INFORMATION
 version_info = {
     'MAJOR_REVISION': 1,
-    'MINOR_REVISION': 0,
-    'RELEASE_STATUS': '0dev',
+    'MINOR_REVISION': 1,
+    'RELEASE_STATUS': '99dev',
 }
 
 __version__ = '.'.join([str(v) for v in version_info.values()])

kintree/common/tools.py

@@ -80,6 +80,12 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=
         opener.addheaders = list(headers.items())
         urllib.request.install_opener(opener)
     try:
+        if filetype == 'PDF':
+            # some distributors/manufacturers implement
+            # redirects which don't allow direct downloads
+            if 'gotoUrl' in url and 'www.ti.com' in url:
+                mpn = url.split('%2F')[-1]
+                url = f'https://www.ti.com/lit/ds/symlink/{mpn}.pdf'
         if filetype == 'Image' or filetype == 'PDF':
             # Enable use of requests library for downloading files (some URLs do NOT work with urllib)
             if requests_lib:

kintree/database/inventree_api.py

@@ -462,30 +462,32 @@ def upload_part_image(image_url: str, part_id: int) -> bool:
         return False
 
 
-def upload_part_datasheet(datasheet_url: str, part_id: int) -> str:
+def upload_part_datasheet(datasheet_url: str, part_ipn: int, part_pk: int) -> str:
     ''' Upload InvenTree part attachment'''
     global inventree_api
 
-    # Get attachment full path
-    datasheet_name = f'{os.path.basename(datasheet_url)}'
-    # inventree needs .pdf at the end of filename to recognize a PDF
-    if not datasheet_name.lower().endswith('.pdf'):
-        datasheet_name += '.pdf'
-    datasheet_location = settings.search_datasheets + datasheet_name
-
-    # Download image (multiple attempts)
-    if not download_with_retry(datasheet_url,
-                               datasheet_location,
-                               filetype='PDF',
-                               timeout=10):
-        return ''
+    datasheet_name = f'{part_ipn}.pdf'
+    # Get datasheet path based on user settings for local storage
+    if settings.DATASHEET_SAVE_ENABLED:
+        datasheet_location = os.path.join(settings.DATASHEET_SAVE_PATH, datasheet_name)
+    else:
+        datasheet_location = os.path.join(settings.search_datasheets, datasheet_name)
+
+    if not os.path.isfile(datasheet_location):
+        # Download datasheet (multiple attempts)
+        if not download_with_retry(
+            datasheet_url,
+            datasheet_location,
+            filetype='PDF',
+            timeout=10
+        ):
+            return ''
 
     # Upload Datasheet to InvenTree
-    part = Part(inventree_api, part_id)
+    part = Part(inventree_api, part_pk)
     if part:
         try:
             attachment = part.uploadAttachment(attachment=datasheet_location)
-            os.remove(datasheet_location)
             return f'{inventree_api.base_url.strip("/")}{attachment["attachment"]}'
         except Exception:
             return ''

kintree/database/inventree_interface.py

@@ -646,14 +646,17 @@ def inventree_create(part_info: dict, stock=None, kicad=False, symbol=None, foot
                 image_result = inventree_api.upload_part_image(inventree_part['image'], part_pk)
                 if not image_result:
                     cprint('[TREE]\tWarning: Failed to upload part image', silent=settings.SILENT)
-            if inventree_part['datasheet'] and settings.DATASHEET_UPLOAD:
-                # Upload datasheet
-                datasheet_link = inventree_api.upload_part_datasheet(inventree_part['datasheet'], part_pk)
-                if not datasheet_link:
-                    cprint('[TREE]\tWarning: Failed to upload part datasheet', silent=settings.SILENT)
-                # TODO: this is messing up with the datasheet download to local folder
-                # else:
-                #     inventree_part['datasheet'] = datasheet_link
+        if inventree_part['datasheet'] and settings.DATASHEET_UPLOAD:
+            # Upload datasheet
+            datasheet_link = inventree_api.upload_part_datasheet(
+                datasheet_url=inventree_part['datasheet'],
+                part_ipn=inventree_part['IPN'],
+                part_pk=part_pk,
+            )
+            if not datasheet_link:
+                cprint('[TREE]\tWarning: Failed to upload part datasheet', silent=settings.SILENT)
+            else:
+                cprint('[TREE]\tSuccess: Uploaded part datasheet', silent=settings.SILENT)
 
         if kicad:
             try:
@@ -843,8 +846,14 @@ def inventree_create_alternate(part_info: dict, part_id='', part_ipn='', show_pr
     if settings.DATASHEET_UPLOAD and not attachment:
         if datasheet:
             part_info['datasheet'] = inventree_api.upload_part_datasheet(
-                part_id=part_pk,
-                datasheet_url=datasheet)
+                datasheet_url=datasheet,
+                part_ipn=part_ipn,
+                part_pk=part_id,
+            )
+            if not part_info['datasheet']:
+                cprint('[TREE]\tWarning: Failed to upload part datasheet', silent=settings.SILENT)
+            else:
+                cprint('[TREE]\tSuccess: Uploaded part datasheet', silent=settings.SILENT)
     # if an attachment is present, set it as the datasheet field
     if attachment:
         part_info['datasheet'] = f'{inventree_api.inventree_api.base_url.strip("/")}{attachment[0]["attachment"]}'

kintree/gui/views/main.py

@@ -1546,17 +1546,23 @@ def create_part(self, e=None):
             return self.process_cancel()
 
         # Final operations
-        # Download datasheet
+        # Download a local version of the part datasheet
         if settings.DATASHEET_SAVE_ENABLED:
-            datasheet_url = part_info.get('datasheet', None)
-            if datasheet_url:
-                filename = os.path.join(
-                    settings.DATASHEET_SAVE_PATH,
-                    f'{part_info.get("IPN", "datasheet")}.pdf',
-                )
-                cprint('\n[MAIN]\tDownloading Datasheet')
-                if download_with_retry(datasheet_url, filename, filetype='PDF', timeout=10):
-                    cprint(f'[INFO]\tSuccess: Datasheet saved to {filename}')
+            filename = os.path.join(
+                settings.DATASHEET_SAVE_PATH,
+                f'{part_info.get("IPN", "datasheet")}.pdf',
+            )
+            if settings.DATASHEET_UPLOAD and os.path.isfile(filename):
+                # Datasheet was already downloaded
+                cprint('\n[MAIN]\tDatasheet')
+                cprint(f'[INFO]\tSuccess: Datasheet file exists ({filename})')
+            else:
+                # Datasheet needs to be downloaded
+                datasheet_url = part_info.get('datasheet', None)
+                if datasheet_url:
+                    cprint('\n[MAIN]\tDownloading Datasheet')
+                    if download_with_retry(datasheet_url, filename, filetype='PDF', timeout=10):
+                        cprint(f'[INFO]\tSuccess: Datasheet saved to {filename}')
         # Open browser
         if settings.ENABLE_INVENTREE:
             if part_info.get('inventree_url', None):

run_tests.py

@@ -271,7 +271,7 @@ def check_result(status: str, new_part: bool) -> bool:
                             cprint(f'[DBUG]\tpart_pk = {part_pk}')
 
                     # Disable datasheet download/upload after first part (to speed up testing)
-                    settings.DATASHEET_UPLOAD = False
+                    # settings.DATASHEET_UPLOAD = False
 
         if ENABLE_TEST_METHODS:
             methods = [

tests/files/inventree_dev.yaml

@@ -1,8 +1,8 @@
-DATASHEET_UPLOAD: false
+DATASHEET_UPLOAD: true
 ENABLE: true
 ENABLE_PROXY: false
 PASSWORD: !!binary |
   WVdSdGFXND0=
 SERVER_ADDRESS: http://127.0.0.1:8000/
 USERNAME: admin
-PROXIES: null
+PROXIES: null