From 0ae67b4efd459cb1a8e209ad055e9927553c0732 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Tue, 31 Oct 2023 08:26:58 -0400 Subject: [PATCH 1/3] Working With S3 Service The plugin now seems to work with the S3 service. --- .../java/org/vcell/vcellfiji/N5ImageHandler.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java b/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java index d28d167..1c62954 100644 --- a/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java +++ b/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java @@ -1,11 +1,13 @@ package org.vcell.vcellfiji; +import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.AnonymousAWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.AmazonS3URI; import ij.ImagePlus; @@ -180,12 +182,23 @@ public void createS3Client(String url, HashMap credentials, Hash this.s3ObjectKey = s3URI.getKey(); this.bucketName = s3URI.getBucket(); defaultRegion = s3URI.getRegion(); + if(s3URI.isPathStyle()){ + s3ClientBuilder.withPathStyleAccessEnabled(true); + } } // otherwise assume it is one of our URLs + // http://vcell:8000/bucket/object/object2 catch (IllegalArgumentException e){ String[] pathSubStrings = uri.getPath().split("/", 3); this.s3ObjectKey = pathSubStrings[2]; this.bucketName = pathSubStrings[1]; + s3ClientBuilder.withPathStyleAccessEnabled(true); + s3ClientBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(uri.getScheme() + "://" + uri.getAuthority(), defaultRegion)); + if(credentials != null){ + s3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(credentials.get("AccessKey"), credentials.get("SecretKey")))); + } + this.s3Client = s3ClientBuilder.build(); + return; } if(credentials != null){ From cbe81460c14f38fa994b973f609e9c8fbbb9a51f Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Tue, 31 Oct 2023 10:28:09 -0400 Subject: [PATCH 2/3] Fix Tests and Refactor Fixed the test cases that where failing with the new method for retrieving S3 objects. Also refactored the ImageJ plugin somewhat. --- .../org/vcell/vcellfiji/N5ImageHandler.java | 41 ++++++++----------- .../vcell/vcellfiji/N5ImageHandlerTest.java | 14 +++---- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java b/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java index 1c62954..a8df0a5 100644 --- a/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java +++ b/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java @@ -174,17 +174,32 @@ public void loadN5Dataset(String selectedDataset) throws IOException { public void createS3Client(String url, HashMap credentials, HashMap endpoint){ AmazonS3ClientBuilder s3ClientBuilder = AmazonS3ClientBuilder.standard(); URI uri = URI.create(url); - String defaultRegion = "site2-low"; + + if(credentials != null){ + s3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(credentials.get("AccessKey"), credentials.get("SecretKey")))); + } // believe that it's a s3 URL try{ AmazonS3URI s3URI = new AmazonS3URI(uri); this.s3ObjectKey = s3URI.getKey(); this.bucketName = s3URI.getBucket(); - defaultRegion = s3URI.getRegion(); if(s3URI.isPathStyle()){ s3ClientBuilder.withPathStyleAccessEnabled(true); } + if(endpoint != null){ + s3ClientBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint.get("Endpoint"), endpoint.get("Region"))); + this.s3Client = s3ClientBuilder.build(); + return; + } + // if nothing is given, default user and return so that code after if statement does not execute + if(endpoint == null && credentials == null){ + this.s3Client = AmazonS3ClientBuilder.standard().withRegion(s3URI.getRegion()).withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())).build(); + return; + } + //creds not null, but region is + this.s3Client = s3ClientBuilder.withRegion(s3URI.getRegion()).build(); + return; } // otherwise assume it is one of our URLs // http://vcell:8000/bucket/object/object2 @@ -193,30 +208,10 @@ public void createS3Client(String url, HashMap credentials, Hash this.s3ObjectKey = pathSubStrings[2]; this.bucketName = pathSubStrings[1]; s3ClientBuilder.withPathStyleAccessEnabled(true); - s3ClientBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(uri.getScheme() + "://" + uri.getAuthority(), defaultRegion)); - if(credentials != null){ - s3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(credentials.get("AccessKey"), credentials.get("SecretKey")))); - } - this.s3Client = s3ClientBuilder.build(); - return; - } - - if(credentials != null){ - s3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(credentials.get("AccessKey"), credentials.get("SecretKey")))); - } - if(endpoint != null){ - s3ClientBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint.get("Endpoint"), endpoint.get("Region"))); + s3ClientBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(uri.getScheme() + "://" + uri.getAuthority(), "site2-low")); this.s3Client = s3ClientBuilder.build(); return; } - // if nothing is given, default user and return so that code after if statement does not execute - if(endpoint == null && credentials == null){ - this.s3Client = AmazonS3ClientBuilder.standard().withRegion(defaultRegion).withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())).build(); - return; - } - - //creds not null, but region is - this.s3Client = s3ClientBuilder.withRegion(defaultRegion).build(); } diff --git a/src/test/java/org/vcell/vcellfiji/N5ImageHandlerTest.java b/src/test/java/org/vcell/vcellfiji/N5ImageHandlerTest.java index 92f9a88..b7b08ab 100644 --- a/src/test/java/org/vcell/vcellfiji/N5ImageHandlerTest.java +++ b/src/test/java/org/vcell/vcellfiji/N5ImageHandlerTest.java @@ -65,22 +65,22 @@ public void testS3Client() throws IOException { credentials.put("AccessKey", "jj"); credentials.put("SecretKey", "jj"); - final String s3ProxyURL = "/" + this.n5FileName; + final String s3ProxyURI = "http://localhost:4000/" + this.n5FileName; N5ImageHandler n5ImageHandler = new N5ImageHandler(); // Environment variables are set in github actions VM -// n5ImageHandler.createS3Client(s3ProxyURL, null, null); -// this.remoteN5ImgPlusTests(n5ImageHandler); + n5ImageHandler.createS3Client(s3ProxyURI, null, null); + this.remoteN5ImgPlusTests(n5ImageHandler); - n5ImageHandler.createS3Client(s3ProxyURL, null, s3Endpoint); + n5ImageHandler.createS3Client(s3ProxyURI, null, s3Endpoint); this.remoteN5ImgPlusTests(n5ImageHandler); -// n5ImageHandler.createS3Client(s3ProxyURL, credentials, null); -// this.remoteN5ImgPlusTests(n5ImageHandler); + n5ImageHandler.createS3Client(s3ProxyURI, credentials, null); + this.remoteN5ImgPlusTests(n5ImageHandler); - n5ImageHandler.createS3Client(s3ProxyURL, credentials, s3Endpoint); + n5ImageHandler.createS3Client(s3ProxyURI, credentials, s3Endpoint); this.remoteN5ImgPlusTests(n5ImageHandler); } From 8b3c1a00b4cfc01105bc4200e8cb7f9110eef441 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Thu, 2 Nov 2023 08:48:29 -0400 Subject: [PATCH 3/3] Multi Thread GUI Make long tasks be done within a worker thread to keep GUI responsive. --- .../org/vcell/vcellfiji/N5ImageHandler.java | 128 +++++++++++++----- .../java/org/vcell/vcellfiji/UI/VCellGUI.java | 4 +- 2 files changed, 96 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java b/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java index a8df0a5..0d2be9f 100644 --- a/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java +++ b/src/main/java/org/vcell/vcellfiji/N5ImageHandler.java @@ -1,13 +1,11 @@ package org.vcell.vcellfiji; -import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.AnonymousAWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.AmazonS3URI; import ij.ImagePlus; @@ -33,53 +31,113 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.List; /* Able to open N5 files locally, display the datasets that can be chosen from it, and open the datasets within ImageJ. Flow of operations is select an N5 file, get dataset list, select a dataset and then open it. + http://vcellapi-beta.cam.uchc.edu:8088/test/test.n5 + */ @Plugin(type = Command.class, menuPath = "Plugins>VCell>N5 Dataset Viewer") -public class N5ImageHandler implements Command{ +public class N5ImageHandler implements Command, ActionListener { private VCellGUI vGui; private File selectedLocalFile; private AmazonS3 s3Client; private String bucketName; - private String s3ObjectKey; + private SwingWorker, ArrayList> n5DatasetListUpdater; + @Override + public void actionPerformed(ActionEvent e) { + enableCriticalButtons(false); + + if(e.getSource() == vGui.localFileDialog){ + n5DatasetListUpdater= new SwingWorker, ArrayList>() { + @Override + protected ArrayList doInBackground() throws Exception { + selectedLocalFile = vGui.localFileDialog.getSelectedFile(); + ArrayList n5DataSetList = getN5DatasetList(); + publish(n5DataSetList); + return null; + } + + @Override + protected void done() { + enableCriticalButtons(true); + } + + @Override + protected void process(List> chunks) { + displayN5Results(chunks.get(0)); + } + }; + n5DatasetListUpdater.execute(); + } + + else if (e.getSource() == vGui.okayButton) { + SwingWorker swingWorker = new SwingWorker() { + @Override + protected Object doInBackground() throws Exception { + try{ + loadN5Dataset(vGui.datasetList.getSelectedValue()); + return null; + } + catch (IOException ex){ + return null; + } + } + + @Override + protected void done() { + enableCriticalButtons(true); + } + }; + swingWorker.execute(); + } + // https://stackoverflow.com/questions/16937997/java-swingworker-thread-to-update-main-gui + // Why swing updating does not work + + else if (e.getSource() == vGui.remoteFileSelection.submitS3Info){ + n5DatasetListUpdater= new SwingWorker, ArrayList>() { + @Override + protected ArrayList doInBackground() throws Exception { + createS3Client(vGui.remoteFileSelection.getS3URL(), vGui.remoteFileSelection.returnCredentials(), vGui.remoteFileSelection.returnEndpoint()); + ArrayList list = getS3N5DatasetList(); + publish(list); + return null; + } + + @Override + protected void done() { + enableCriticalButtons(true); + } + + @Override + protected void process(List> chunks) { + displayN5Results(chunks.get(0)); + } + }; + n5DatasetListUpdater.execute(); + } + } + + private void enableCriticalButtons(boolean enable) { + vGui.remoteFileSelection.submitS3Info.setEnabled(enable); + vGui.okayButton.setEnabled(enable); + vGui.LocalFiles.setEnabled(enable); + vGui.remoteFiles.setEnabled(enable); + } @Override public void run() { this.vGui = new VCellGUI(); - this.vGui.localFileDialog.addActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - selectedLocalFile = vGui.localFileDialog.getSelectedFile(); - displayN5Results(getN5DatasetList()); - } - }); + this.vGui.localFileDialog.addActionListener(this); - this.vGui.okayButton.addActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - try{ - loadN5Dataset(vGui.datasetList.getSelectedValue()); - } - catch (IOException ex){ - return; - } - } - }); - - this.vGui.remoteFileSelection.submitS3Info.addActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - createS3Client(vGui.remoteFileSelection.getS3URL(), vGui.remoteFileSelection.returnCredentials(), vGui.remoteFileSelection.returnEndpoint()); - ArrayList list = getS3N5DatasetList(); - displayN5Results(list); - } - }); + this.vGui.okayButton.addActionListener(this); + + this.vGui.remoteFileSelection.submitS3Info.addActionListener(this); } public ArrayList getN5DatasetList(){ @@ -99,9 +157,7 @@ public ArrayList getN5DatasetList(){ } public void displayN5Results(ArrayList datasetList){ - SwingUtilities.invokeLater(() ->{ - this.vGui.updateDatasetList(datasetList); - }); + this.vGui.updateDatasetList(datasetList); } @@ -236,4 +292,8 @@ public File getSelectedLocalFile() { return selectedLocalFile; } + public static void main(String[] args) { + N5ImageHandler n5ImageHandler = new N5ImageHandler(); + n5ImageHandler.run(); + } } diff --git a/src/main/java/org/vcell/vcellfiji/UI/VCellGUI.java b/src/main/java/org/vcell/vcellfiji/UI/VCellGUI.java index eb5b40b..ac40072 100644 --- a/src/main/java/org/vcell/vcellfiji/UI/VCellGUI.java +++ b/src/main/java/org/vcell/vcellfiji/UI/VCellGUI.java @@ -6,14 +6,14 @@ import java.util.ArrayList; public class VCellGUI extends JFrame { - private JButton LocalFiles; + public JButton LocalFiles; public JPanel mainPanel; private final JFrame jFrame; public JFileChooser localFileDialog; private JToolBar menuBar; public JList datasetList; private JScrollPane resultsScrollPane; - private JButton remoteFiles; + public JButton remoteFiles; public JButton okayButton; public JCheckBox openVirtualCheckBox;