feat: support accessing private S3 basins (close #174)

ZELLMECHANIK-DRESDEN · Nov 7, 2024 · baa3f88 · baa3f88
1 parent f021a48
commit baa3f88
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 2 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,10 +1,12 @@
 2.19.0
+ - feat: support accessing private S3 basins (#174)
  - enh: enable lut selection in analysis view plots (#71)
  - fix: only allow user to choose "R.exe" in R binary manual search (#173)
  - enh: correctly show system R binary in settings
  - ref: migrate from pyqt5 to pyqt6 (#165)
  - ref: migrate from pkg_resources to importlib.resources (#155)
  - setup: support Python 3.12
+ - setup: new dependency "boto3" via "dclab[s3]""
 2.18.2
  - enh: enable lut selection to plot isoelasticity lines (#71)
  - enh: replace findText with findData of comboBox_visc_model

diff --git a/pyproject.toml b/pyproject.toml
@@ -30,7 +30,7 @@ classifiers = [
 license = {text = "GPL v3"}
 dependencies = [
     "fcswrite>=0.5.1",
-    "dclab[dcor,http]>=0.62.5",
+    "dclab[dcor,http,s3]>=0.62.5",
     "h5py>=2.8.0",
     "numpy>=1.21",  # CVE-2021-33430
     "pyqt6",

diff --git a/shapeout2/gui/main.py b/shapeout2/gui/main.py
@@ -77,6 +77,7 @@ def __init__(self, *arguments):
         QtCore.QCoreApplication.setOrganizationDomain("zellmechanik.com")
         QtCore.QCoreApplication.setApplicationName("shapeout2")
         QtCore.QSettings.setDefaultFormat(QtCore.QSettings.Format.IniFormat)
+
         #: Shape-Out settings
         self.settings = QtCore.QSettings()
         # Register custom DCOR CA bundle directory with dclab
@@ -90,6 +91,18 @@ def __init__(self, *arguments):
         # open a session with private data.
         api_key = self.settings.value("dcor/api key", "")
         dclab.rtdc_dataset.fmt_dcor.api.APIHandler.add_api_key(api_key)
+        # Register S3 access settings in dclab
+        s3_endpoint_url = self.settings.value("s3/endpoint url", "")
+        if s3_endpoint_url:
+            dclab.rtdc_dataset.fmt_s3.S3_ENDPOINT_URL = s3_endpoint_url
+        s3_access_key_id = self.settings.value("s3/access key id", "")
+        if s3_access_key_id:
+            dclab.rtdc_dataset.fmt_s3.S3_ACCESS_KEY_ID = s3_access_key_id
+        s3_secret_access_key = self.settings.value("s3/secret access key", "")
+        if s3_secret_access_key:
+            dclab.rtdc_dataset.fmt_s3.S3_SECRET_ACCESS_KEY = \
+                s3_secret_access_key
+
         #: Analysis pipeline
         self.pipeline = pipeline.Pipeline()
         #: Extensions

diff --git a/shapeout2/gui/preferences.py b/shapeout2/gui/preferences.py
@@ -60,6 +60,9 @@ def __init__(self, parent, *args, **kwargs):
             ["dcor/servers", self.dcor_servers, ["dcor.mpl.mpg.de"]],
             ["dcor/use ssl", self.dcor_use_ssl, "1"],
             ["lme4/r path", self.lme4_rpath, rdefault],
+            ["s3/endpoint url", self.lineEdit_s3_endpoint_url, ""],
+            ["s3/access key id", self.lineEdit_s3_access_key_id, ""],
+            ["s3/secret access key", self.lineEdit_s3_secret_access_key, ""],
         ]
 
         # extensions

diff --git a/shapeout2/gui/preferences.ui b/shapeout2/gui/preferences.ui
@@ -89,7 +89,7 @@
             <string/>
            </property>
            <property name="echoMode">
-            <enum>QLineEdit::Normal</enum>
+            <enum>QLineEdit::Password</enum>
            </property>
            <property name="placeholderText">
             <string>optional (to access private data)</string>
@@ -147,6 +147,100 @@
        </item>
       </layout>
      </widget>
+     <widget class="QWidget" name="tab">
+      <attribute name="title">
+       <string>S3 basins</string>
+      </attribute>
+      <layout class="QVBoxLayout" name="verticalLayout_9">
+       <item>
+        <widget class="QLabel" name="label_10">
+         <property name="maximumSize">
+          <size>
+           <width>500</width>
+           <height>16777215</height>
+          </size>
+         </property>
+         <property name="text">
+          <string>If you are working with DC datasets that have basins in an S3 object store, you can specify the corresponding access credentials here. Please restart Shape-Out for the settings to take effect.</string>
+         </property>
+         <property name="wordWrap">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <layout class="QFormLayout" name="formLayout_3">
+         <item row="0" column="0">
+          <widget class="QLabel" name="label_7">
+           <property name="text">
+            <string>Endpoint URL</string>
+           </property>
+          </widget>
+         </item>
+         <item row="0" column="1">
+          <widget class="QLineEdit" name="lineEdit_s3_endpoint_url">
+           <property name="toolTip">
+            <string>This is the endpoint URL that boto3 will attempt to connect to</string>
+           </property>
+           <property name="placeholderText">
+            <string>e.g. https://objectstore.hpccloud.mpcdf.mpg.de</string>
+           </property>
+          </widget>
+         </item>
+         <item row="1" column="0">
+          <widget class="QLabel" name="label_8">
+           <property name="text">
+            <string>Access Key ID</string>
+           </property>
+          </widget>
+         </item>
+         <item row="1" column="1">
+          <widget class="QLineEdit" name="lineEdit_s3_access_key_id">
+           <property name="toolTip">
+            <string>This is the access key for accessing .rtdc resources on S3</string>
+           </property>
+           <property name="placeholderText">
+            <string>e.g. 1ad3afa49c7241bac142ed4108343ea</string>
+           </property>
+          </widget>
+         </item>
+         <item row="2" column="0">
+          <widget class="QLabel" name="label_9">
+           <property name="text">
+            <string>Secret Acces Key</string>
+           </property>
+          </widget>
+         </item>
+         <item row="2" column="1">
+          <widget class="QLineEdit" name="lineEdit_s3_secret_access_key">
+           <property name="toolTip">
+            <string>This is the secret access key matching the Access Key ID</string>
+           </property>
+           <property name="echoMode">
+            <enum>QLineEdit::Password</enum>
+           </property>
+           <property name="placeholderText">
+            <string>e.g. a3a4862a92643be730c946d267015e20</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item>
+        <spacer name="verticalSpacer_5">
+         <property name="orientation">
+          <enum>Qt::Vertical</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>20</width>
+           <height>281</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+      </layout>
+     </widget>
      <widget class="QWidget" name="tab_r">
       <attribute name="title">
        <string>R/lme4</string>