Caltech-IPAC · robyww · Dec 18, 2024
diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/LockingVisNetwork.java b/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/LockingVisNetwork.java
@@ -20,8 +20,10 @@
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -34,6 +36,7 @@
 public class LockingVisNetwork {
 
     private static final Map<BaseNetParams, Object> _activeRequest = Collections.synchronizedMap(new HashMap<>());
+    private static final Map<BaseNetParams, DownloadProgress> activeListeners = Collections.synchronizedMap(new HashMap<>());
 
     public static FileInfo retrieveURL(AnyUrlParams params) throws FailedRequestException {
         return lockingRetrieve(params, null);
@@ -64,9 +67,10 @@ private static FileInfo lockingRetrieve(BaseNetParams params, ServiceCaller svcC
         confirmParamsType(params);
         try {
             Object lockKey= _activeRequest.computeIfAbsent(params, k -> new Object());
+            DownloadProgress dl= makeDownloadProgressOrFind(params);
             synchronized (lockKey) {
                 return (params instanceof AnyUrlParams urlP) ?
-                    retrieveURL( urlP, makeDownloadProgress(params) ) :
+                    retrieveURL(urlP, dl) :
                         (params instanceof ImageServiceParams isParam) ?
                                 retrieveService(isParam, svcCaller) :
                                 retrieveServiceCaller((ServiceCallerParams) params);
@@ -78,9 +82,22 @@ private static FileInfo lockingRetrieve(BaseNetParams params, ServiceCaller svcC
         }
     }
 
-    private static DownloadProgress makeDownloadProgress(BaseNetParams params) { // todo: generalize beyond just plotId
+
+
+    private static DownloadProgress makeDownloadProgressOrFind(BaseNetParams params) { // todo: generalize beyond just plotId
         if (params.getStatusKey()== null) return null;
-        return new DownloadProgress(params.getStatusKey(), params.getPlotId());
+        if (!(params instanceof AnyUrlParams)) return null;
+        DownloadProgress dl;
+
+        if (activeListeners.containsKey(params)) {
+            dl = activeListeners.get(params);
+            dl.addPlotId(params.getPlotId());
+        }
+        else {
+            dl= new DownloadProgress(params.getStatusKey(), params.getPlotId());
+            activeListeners.put(params, dl);
+        }
+        return dl;
     }
 
     private static void confirmParamsType(NetParams params) throws FailedRequestException {
@@ -122,6 +139,7 @@ private static FileInfo retrieveURL(AnyUrlParams params, DownloadListener dl) th
             var ops= URLDownload.Options.listenerOp(params.getMaxSizeToDownload(), dl);
             fileInfo= URLDownload.getDataToFile(params.getURL(), fileName, params.getCookies(), params.getHeaders(), ops);
             if (fileInfo.getResponseCode()==200) CacheHelper.putFileInfo(params,fileInfo);
+            activeListeners.remove(params);
             return fileInfo;
         } catch (Exception e) {
             Logger.warn(e.toString());
@@ -139,13 +157,15 @@ public interface CanCallService {};
     private static class DownloadProgress implements DownloadListener {
 
         private final String _key;
-        private final String _plotId;
+        private final List<String> plotIdList= new ArrayList<>();
 
         DownloadProgress(String key, String plotId) {
             _key = key;
-            _plotId = plotId;
+            plotIdList.add(plotId);
         }
 
+        void addPlotId(String plotId) {plotIdList.add(plotId);}
+
         public void dataDownloading(DownloadEvent ev) {
             if (_key == null) return;
             String offStr = "";
@@ -155,7 +175,9 @@ public void dataDownloading(DownloadEvent ev) {
                 offStr = " of " + FileUtil.getSizeAsString(ev.getMax(),true);
             }
             String messStr = "Retrieved " + FileUtil.getSizeAsString(current,true) + offStr;
-            PlotServUtils.updateProgress(_key, _plotId, ProgressStat.PType.DOWNLOADING, messStr);
+            for(var plotId : plotIdList) {
+                PlotServUtils.updateProgress(_key,plotId, ProgressStat.PType.DOWNLOADING, messStr);
+            }
         }
     }
 

diff --git a/src/firefly/java/edu/caltech/ipac/table/io/SpectrumMetaInspector.java b/src/firefly/java/edu/caltech/ipac/table/io/SpectrumMetaInspector.java
@@ -76,7 +76,7 @@ public class SpectrumMetaInspector {
     private static final String SPEC_TI_AXIS_ACCURACY= SPEC_TI_AXIS+ACCURACY;
     private static final String SPEC_ORDER = "spec:Data.SpectralAxis.Order";
     private static final String SPEC_REL_ORDER= "spec:Data.SpectralAxis.RelOrder";
-    private static final String spec10Version= "Spectrum v1.0";
+    private static final String spec10Version= "spectrum v1.0";
     private static final String VOCLASS= "VOCLASS";
 
     private static final String SPEC_SPECTRUM= "spec:Spectrum";
@@ -116,12 +116,9 @@ public static void searchForSpectrum(DataGroup dg, BasicHDU<?> hdu, boolean spec
         String utype;
         if (hdu!=null) {
             Header h= hdu.getHeader();
-            String voclass= h.getStringValue(VOCLASS);
-            if (voclass==null) voclass= "";
+            String voclass= h.getStringValue(VOCLASS,"").toLowerCase();
             utype= FitsReadUtil.getUtype(h);
-            if (utype==null && !spectrumHint &&
-                    !voclass.equals(spec10Version) &&
-                    !voclass.toLowerCase().startsWith("spectrum") ) {
+            if (utype==null && !spectrumHint && (voclass.equals(spec10Version) || voclass.startsWith("spectrum") )) {
                 utype= SPEC_SPECTRUM;
             }
             if (utype!=null) {

diff --git a/src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java b/src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java
@@ -221,7 +221,7 @@ public static HttpResultInfo getDataFromURL(URL url,
             Map<String,List<String>> reqProp= conn.getRequestProperties();
             pushPostData(conn, postData);
 
-            logHeader(postData, conn, reqProp);
+            logHeader(url.toString(), postData, conn, reqProp);
             ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
             netCopy(makeAnyInStream(conn, false), out, conn, 0, null);
             byte[] results = out.toByteArray();
@@ -247,7 +247,7 @@ public static HttpResultInfo getHeaderFromURL(URL url,
                 conn.setReadTimeout(timeoutInSec * 1000);
             }
             ((HttpURLConnection)conn).setRequestMethod("HEAD");
-            logHeader(null, conn, conn.getRequestProperties());
+            logHeader(url.toString(), null, conn, conn.getRequestProperties());
             Set<Map.Entry<String,List<String>>> hSet = getResponseCode(conn)==-1 ? null : conn.getHeaderFields().entrySet();
             HttpResultInfo result= new HttpResultInfo(null,getResponseCode(conn),conn.getContentType(),getSugestedFileName(conn));
 
@@ -367,6 +367,7 @@ public static FileInfo getDataToFile(URLConnection conn,
                                          int redirectCnt) throws FailedRequestException {
 
         try {
+            String originalUrl= conn.getURL().toString();
             FileInfo outFileData;
             Map<String, List<String>> reqProp = conn.getRequestProperties();
             Map<String, List<String>> sendHeaders = null;
@@ -396,7 +397,7 @@ public static FileInfo getDataToFile(URLConnection conn,
             //------
             //---From here on the server should be responding
             //------
-            logHeader(postData, conn, sendHeaders);
+            logHeader(originalUrl, postData, conn, sendHeaders);
             validFileSize(conn, ops.maxFileSize);
             netCopy(makeAnyInStream(conn, ops.uncompress), makeOutStream(outfile), conn, ops.maxFileSize, ops.dl);
             long elapse = System.currentTimeMillis() - start;
@@ -562,9 +563,11 @@ private static void logError(URL url, Map<String,String> postData, Exception e)
         _log.warn(strList.toArray(new String[0]));
     }
 
-    public static void logHeader(URLConnection conn) { logHeader(null, conn, null); }
+    public static void logHeader(URLConnection conn) { logHeader(null,null, conn, null); }
 
-    private static void logHeader(Map<String,String> postData, URLConnection conn, Map<String,List<String>> sendHeaders) {
+    public static void logHeader(String originalUrl, URLConnection conn) { logHeader(originalUrl,null, conn, null); }
+
+    private static void logHeader(String originalUrl,  Map<String,String> postData, URLConnection conn, Map<String,List<String>> sendHeaders) {
         StringBuffer workBuff;
         try {
             String verb= "";
@@ -575,6 +578,9 @@ private static void logHeader(Map<String,String> postData, URLConnection conn, M
             if (conn.getURL() != null) {
                 outStr.add("----------Sending " + verb);
                 outStr.add( conn.getURL().toString());
+                if (originalUrl!=null && !conn.getURL().toString().equals(originalUrl)) {
+                    outStr.add( StringUtils.pad(20, "Original URL")+": "+originalUrl);
+                }
                 if (sendHeaders!=null) {
                     for(Map.Entry<String,List<String>> se: sendHeaders.entrySet()) {
                         workBuff = new StringBuffer(100);

diff --git a/src/firefly/js/core/ClickToAction.js b/src/firefly/js/core/ClickToAction.js
@@ -55,13 +55,13 @@ export function makeSearchActionObj({ cmd, groupId, label='', tip, searchType, m
     return {cmd, label, tip, searchType, min, max, supported, execute, verb, searchDesc, groupId};
 }
 
-export function getSearchTypeDesc(sa, wp, size, areaPtsLength) {
+export function getSearchTypeDesc({sa, wp, size, areaPtsLength,tbl_id}) {
 
     const {searchDesc, searchType, verb, label, min, max}= sa;
     if (searchDesc) {
         if (isString(searchDesc)) return searchDesc;
         if (isFunction(searchDesc)) {
-            const title= searchDesc(wp,size,areaPtsLength);
+            const title= searchDesc({wp,size,areaPtsLength,tbl_id});
             if (isString(title)) return title;
         }
     }

diff --git a/src/firefly/js/data/MetaConst.js b/src/firefly/js/data/MetaConst.js
@@ -34,6 +34,11 @@ export const MetaConst = {
      */
     FITS_EXTRACTION_TYPE: 'FitsExtractionType',
 
+    /**
+     * Search Target - the search target associated with this table
+     */
+    SEARCH_TARGET: 'SearchTarget',
+
     /**
      * An world point in a fits FILE that is associated with this table
      */
@@ -182,6 +187,11 @@ export const MetaConst = {
     /** the column name with public release date info;  null is considered not public */
     RELEASE_DATE_COL : 'RELEASE_DATE_COL',
 
+    /**
+     * any ID can be added to a table search to identify the source for the purpose of getting source specific obscore preferences
+     */
+    OBSCORE_SOURCE_ID: 'OBSCORE_SOURCE_ID',
+
     /**
      * if defined this table container a moving object, setting this will override any catalog evaluation
      * value can me true or false, if true then evaluate this table as a moving obj, if false then ignore
@@ -229,6 +239,12 @@ export const MetaConst = {
      */
     UPLOAD_TABLE: 'UploadTable',
 
+    /**
+     * if defined and an object, Then this will be table specific DataProductsFactory options
+     * @see DataProductsFactoryOptions
+     */
+    DATA_PRODUCTS_FACTORY_OPTIONS: 'DataProductFactoryOptions',
+
     /** @deprecated use CENTER_COLUMN */
     CATALOG_COORD_COLS : 'CatalogCoordColumns',
 

diff --git a/src/firefly/js/drawingLayers/HiPSMOC.js b/src/firefly/js/drawingLayers/HiPSMOC.js
@@ -61,15 +61,20 @@ function loadMocFitsWatcher(action, cancelSelf, params, dispatch, getState) {
         if (!dl) return;
         const preloadedTbl= tablePreloaded && getTblById(tbl_id);
 
+        const filterObj= dl.maxFetchDepth ? {filters : `"${mocFitsInfo.uniqColName}" < ${4*(4**(dl.maxFetchDepth+1))}`} : {};
         if (!dl.mocTable) { // moc table is not yet loaded
             let tReq;
             if (preloadedTbl){ //load by getting the full version of a already loaded table
                 tReq= cloneRequest(preloadedTbl.request,
-                    { startIdx : 0, pageSize : MAX_ROW, inclCols: mocFitsInfo.uniqColName });
+                    { startIdx : 0, pageSize : MAX_ROW, inclCols: mocFitsInfo.uniqColName, ...filterObj});
             }
             else if (fitsPath) {       // load by getting file on server
+
                 tReq = makeTblRequest('userCatalogFromFile', 'Table Upload',
-                    {filePath: fitsPath, sourceFrom: 'isLocal'},
+                    {
+                        filePath: fitsPath, sourceFrom: 'isLocal',
+                        ...filterObj,
+                    },
                     {tbl_id: mocFitsInfo.tbl_id, pageSize: MAX_ROW, inclCols: mocFitsInfo.uniqColName});
             }
             if (!tReq) return;
@@ -162,6 +167,7 @@ function creator(initPayload) {
     dl.mocTable= undefined;
     dl.rootTitle= dl.title;
     dl.mocGroupDefColorId= mocGroupDefColorId;
+    dl.maxFetchDepth= initPayload.maxFetchDepth;
 
     dispatchAddActionWatcher({
         callback:loadMocFitsWatcher,

diff --git a/src/firefly/js/metaConvert/AnalysisUtils.js b/src/firefly/js/metaConvert/AnalysisUtils.js
@@ -115,18 +115,19 @@ export function makeAnalysisGetGridDataProduct(makeReq) {
  * @param obj.menuKey
  * @param obj.dataTypeHint
  * @param {ServiceDescriptorDef} [obj.serDef]
+ * @param {DatalinkData} [obj.dlData]
  * @param [obj.originalTitle]
  * @param {DataProductsFactoryOptions} [obj.options]
  * @return {function}
  */
 export function makeAnalysisActivateFunc({table, row, request, activateParams, menuKey,
-                                             dataTypeHint, serDef, originalTitle, options}) {
+                                             dataTypeHint, serDef, originalTitle, options, dlData}) {
     const analysisActivateFunc = async (menu, userInputParams) => {
         const {dpId}= activateParams;
         dispatchUpdateDataProducts(dpId, dpdtWorkingMessage(LOADING_MSG,menuKey));
         // do the uploading and analyzing
         const dPDisplayType= await doUploadAndAnalysis({ table, row, request, activateParams, dataTypeHint, options, menu,
-            serDef, userInputParams, analysisActivateFunc, originalTitle, menuKey});
+            serDef, dlData, userInputParams, analysisActivateFunc, originalTitle, menuKey});
         // activate the result of the analysis
        dispatchResult(dPDisplayType, menu,menuKey,dpId, serDef, analysisActivateFunc);
     };

diff --git a/src/firefly/js/metaConvert/DataProductsCntlr.js b/src/firefly/js/metaConvert/DataProductsCntlr.js
@@ -47,6 +47,7 @@ function activateMenuItemActionCreator(rawAction) {
         const menuItem= menu.find( (m) => m.menuKey===menuKey);
         if (!menuItem) return;
         if (menuItem.displayType===DPtypes.DOWNLOAD) doDownload(menuItem.url);
+        else if (menuItem.displayType===DPtypes.EXTRACT) doExtract(menuItem);
         else dispatcher(rawAction);
     };
 }
@@ -66,6 +67,10 @@ function activateFileMenuItemActionCreator(rawAction) {
     };
 }
 
+function doExtract(menuItem) {
+    menuItem?.activate();
+}
+
 export function doDownload(url) {
     const serviceURL = getRootURL() + 'servlet/Download';
     const tmpUrl= url.toLowerCase();

diff --git a/src/firefly/js/metaConvert/DataProductsFactory.js b/src/firefly/js/metaConvert/DataProductsFactory.js
@@ -4,7 +4,7 @@
 
 import {isArray, once} from 'lodash';
 import {MetaConst} from '../data/MetaConst.js';
-import {getMetaEntry} from '../tables/TableUtil';
+import {getMetaEntry, getObjectMetaEntry} from '../tables/TableUtil';
 import {hasObsCoreLikeDataProducts, isDatalinkTable} from '../voAnalyzer/TableAnalysis.js';
 import {hasServiceDescriptors} from '../voAnalyzer/VoDataLinkServDef.js';
 import {Band} from '../visualize/Band';
@@ -284,14 +284,13 @@ function initConverterTemplates() {
  * @typedef {Object} DataProductsFactoryOptions
  *
  * @prop {boolean} [allowImageRelatedGrid]
- * @prop {boolean} [allowServiceDefGrid] - // todo: this is redundant, should remove
  * @prop {boolean} [singleViewImageOnly] - if true, the view will only show the primary image product
  * @prop {boolean} [singleViewTableOnly] - if true, the view will only show the primary table product
  * @prop {String} [dataLinkInitialLayout] - determine how a datalink obscore table trys to show the data layout, must be 'single', 'gridRelated', 'gridFull';
  * @prop {boolean} [activateServiceDef] - if possible then call the service def without giving option for user input
  * @prop {boolean} [threeColor] - if true, then for images in related grid, show the threeColor option
  * @prop {number} [maxPlots] - maximum number of plots in grid mode, this will override the factory default
- * @prop {boolean} [canGrid] - some specific factories might have parameters that override this parameter (e.g. allowServiceDefGrid)
+ * @prop {boolean} [canGrid] - some specific factories might have parameters that override this parameter
  * @prop [initialLayout]
  * @prop {string} [tableIdBase] - any tbl_id will use this string for its base
  * @prop {string} [chartIdBase] - any chartId will use this string for its base
@@ -302,6 +301,7 @@ function initConverterTemplates() {
  *                The values in this object will override one or more parameters to a service descriptor.
  *                The following are used with this prop by service descriptors to build the url to include input from the UI.
  *                see- ServDescProducts.js getComponentInputs()
+ * @prop {object} datalinkTblRequestOptions = add addition options to table request
  * @prop {Array.<string>} [paramNameKeys] - name of the parameters to put in the url from the getComponentState() return object
  * @prop {Array.<string>} [ucdKeys] - same as above but can be specified by UCD
  * @prop {Array.<string>} [utypeKeys] - same as above but can be specified by utype
@@ -313,19 +313,19 @@ function initConverterTemplates() {
 export const getDefaultFactoryOptions= once(() => ({
     dataProductsComponentKey: DEFAULT_DATA_PRODUCTS_COMPONENT_KEY,
     allowImageRelatedGrid: false,
-    allowServiceDefGrid: false, // todo: this is redundant, should remove
     singleViewImageOnly:false,
     singleViewTableOnly:false,
     dataLinkInitialLayout: 'single', //determine how a datalink obscore table trys to show the data layout, must be 'single', 'gridRelated', 'gridFull';
     activateServiceDef: false,
     threeColor: undefined,
     maxPlots: undefined,
-    canGrid: undefined, // some specific factories might have parameters that override this parameter (e.g. allowServiceDefGrid)
+    canGrid: undefined, // some specific factories might have parameters that override this parameter
     initialLayout: undefined, //todo - an datalink use this?
     tableIdBase: undefined,
     chartIdBase: undefined,
     tableIdList: [], // list of ids
     chartIdList: [],// list of ids
+    datalinkTblRequestOptions: {},
     paramNameKeys: [],
     ucdKeys: [],
     utypeKeys: [],
@@ -386,9 +386,9 @@ export function makeDataProductsConverter(table, factoryKey= undefined) {
         threeColor: options.threeColor ?? t.threeColor ?? false,
         dataProductsComponentKey: options.dataProductsComponentKey
     };
-    const retObj= t.create(table,pT, options);
+    const metaOptions= getObjectMetaEntry(table, MetaConst.DATA_PRODUCTS_FACTORY_OPTIONS, {});
+    const retObj= t.create(table,pT, {...options, ...metaOptions});
     return {options, ...retObj};
-
 }