Allow core retry call with different options

lib/core.js

@@ -1437,7 +1437,8 @@
         responseStatusCode: 'responseStatusCode',
         responseError: 'responseError',
         timeout: 'timeout',
-        fallback: 'fallback'
+        fallback: 'fallback',
+        retry: 'retry'        // Make same iframely call with different options.
     };
 
     function findRedirectError(result) {
@@ -1458,6 +1459,21 @@
         }
     }
 
+    function findRetryError(result) {
+        if (result) {
+            for(var i = 0; i < result.length; i++) {
+                var r = result[i];
+                var retry = r.error && r.error[SYS_ERRORS.retry];
+                if (typeof retry === "object") {
+                    log('   -- plugin retry (by "' + r.method.pluginId + '")');
+                    return retry;
+                } else if (retry && typeof retry !== "object") {
+                    log('   -- skip plugin retry, not object (by "' + r.method.pluginId + '")', retry);
+                }
+            }
+        }
+    }
+
     function findResponseError(result, uri) {
         if (result) {
             for(var i = 0; i < result.length; i++) {
@@ -1739,6 +1755,9 @@
             options = {};
         }
 
+        // Store original to run on retry.
+        var originalOptions = {...options};
+
         var initialCallback = cb;
         var fallbackInfo;
 
@@ -1786,6 +1805,11 @@
             return;
         }
 
+        if (options.retriesCount && options.retriesCount > (options.maxRedirects || CONFIG.MAX_REDIRECTS)) {
+            callbackWithErrorCode('retry loop', options, cb);
+            return;
+        }
+
         // Remove default :443 and :80 from uri.
         uri = uri
             .replace(/^(http:\/\/[^\/]+):80(?!\d)/, '$1')
@@ -1952,6 +1976,25 @@
                     return;
                 }
 
+                // Find retry command.
+                var retry_data = findRetryError(result);
+                if (retry_data 
+                    && options.retriesCount
+                    // Prerender not changed
+                    && !!options.prerender === !!retry_data.prerender
+                    // User agent not changed
+                    && options.user_agent == retry_data.user_agent) {
+                    log('   -- ignore recursive prerender retry');
+                } else if (retry_data) {
+                    abortCurrentRequest();
+                    var retryOptions = Object.assign({}, originalOptions, retry_data);
+                    retryOptions.retriesCount = (retryOptions.retriesCount || 0) + 1;
+                    retryOptions.refresh = true;
+                    run(uri, retryOptions, cb);
+                    aborted = true;
+                    return;
+                }
+
                 // Gather results.
                 // Run before `responseError` to collect data and send in error.
                 var hasNewData = useResult(usedMethods, context, pluginsContexts, allResults, result, options, asyncMethodCb);

lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js

@@ -7,6 +7,7 @@
 
 export function CollectingHandlerForMutliTarget(cbsArray){
     this._cbsArray = cbsArray || [];
+    this._virtualCount = 0;
     this.events = [];
 }
 
@@ -35,11 +36,16 @@ EVENTS.forEach(function(name) {
     };
 });
 
+CollectingHandlerForMutliTarget.prototype.hasNoHandlers = function() {
+    return this._cbsArray.length === 0 && this._virtualCount === 0;
+};
+
 CollectingHandlerForMutliTarget.prototype.addHandler = function(cbs) {
+    const wasEmpty = this.hasNoHandlers();
     this._cbsArray.push(cbs);
     this._emitEventsFor(cbs);
 
-    if (this._cbsArray.length === 1) {
+    if (wasEmpty) {
         // Got first handler, resume stream.
         this.onFirstHandler();
     }
@@ -52,14 +58,34 @@ CollectingHandlerForMutliTarget.prototype.removeHandler = function(cbs) {
         var idx = that._cbsArray.indexOf(cbs);
         if (idx > -1) {
             that._cbsArray.splice(idx, 1);
-            if (that._cbsArray.length === 0) {
+            if (that.hasNoHandlers()) {
                 // No handlers, pause stream.
                 that.onNoHandlers();
             }
         }
     });
 };
 
+CollectingHandlerForMutliTarget.prototype.addVirtualHandler = function() {
+    const wasEmpty = this.hasNoHandlers();
+    this._virtualCount++;
+    if (wasEmpty) {
+        this.onFirstHandler();
+    }
+};
+
+CollectingHandlerForMutliTarget.prototype.removeVirtualHandler = function() {
+    var that = this;
+    process.nextTick(function() {
+        if (that._virtualCount > 0) {
+            that._virtualCount--;
+            if (that.hasNoHandlers()) {
+                that.onNoHandlers();
+            }
+        }
+    });
+};
+
 CollectingHandlerForMutliTarget.prototype.emitCb = function(event) {
     this.events.push(event);
     this.callCb(event);
@@ -128,4 +154,4 @@ CollectingHandlerForMutliTarget.prototype._emitEventsFor = function(cbs) {
     }
 };
 
-export const notPlugin = true;
+export const notPlugin = true;

lib/plugins/system/htmlparser/HtmlHandler.js

@@ -0,0 +1,39 @@
+export class HtmlHandler {
+
+    constructor() {
+        this.text = '';
+        this._onEndCallbacks = [];
+        this._ended = false;
+    }
+
+    onData(chunk) {
+        // TODO: decode?
+        this.text += chunk;
+    }
+
+    onEnd(callback) {
+        if (typeof callback === 'function') {
+            if (this._ended) {
+                callback(this.text);
+            } else {
+                this._onEndCallbacks.push(callback);
+            }
+        }
+    }
+
+    end() {
+        this._ended = true;
+        for (const cb of this._onEndCallbacks) {
+            cb(this.text);
+        }
+        this._onEndCallbacks = [];
+    }
+
+    attach(resp) {
+        resp.on('data', chunk => this.onData(chunk));
+        resp.on('end', () => this.end());
+        return this;
+    }
+}
+
+export const notPlugin = true;

lib/plugins/system/htmlparser/htmlparser.js

@@ -7,11 +7,13 @@ import * as metaUtils from '../meta/utils.js';
 import { extendCookiesJar } from '../../../fetch.js';
 var getUrlFunctional = utils.getUrlFunctional;
 import { CollectingHandlerForMutliTarget } from './CollectingHandlerForMutliTarget.js';
+import { HtmlHandler } from './HtmlHandler.js';
 
 export default {
 
     provides: [
         'self',
+        'htmlresponse',
         '__nonHtmlContentData',
         '__nonHtmlContentResponse',
         '__statusCode'
@@ -201,11 +203,22 @@ export default {
                 handler.abortController = abortController;
                 handler.h2 = resp.h2;
 
-                // Do before resume?
-                cb(null, {
+                var result = {
                     htmlparser: handler
-                });
+                };
+
+                if (options.enableHtmlResponse) {
+                    // Collect full response text.
+                    var htmlHandler = new HtmlHandler();
+                    // Bind on data, on end.
+                    htmlHandler.attach(resp);
 
+                    result.htmlresponse = htmlHandler;
+                }
+
+                // Do before resume?
+                cb(null, result);
+
                 // Proxy data.
                 resp.on('data', parser.write.bind(parser));
                 resp.on('end', parser.end.bind(parser));

lib/plugins/system/meta/utils.js

@@ -3,7 +3,7 @@ export function getMetaCacheKey(url, whitelistRecord, options) {
     var meta_key = 'meta';
 
     if (options.metaKeyPrefix) {
-        meta_key += ':' + options.metaKeyPrefix;    
+        meta_key += ':' + options.metaKeyPrefix;
     }
 
     meta_key += ':' + url;

lib/utils.js

@@ -102,11 +102,15 @@ export function prepareRequestOptions(request_options, options) {
                     if (typeof prerender_option === 'string') {
                         request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${prerender_option}`;
                     }
+
+                    if (typeof options?.prerender?.overrideOptions === 'function') {
+                        options.prerender.overrideOptions(request_options);
+                    }
                 }
             }
 
             // Some calls (like oembed) use basic options without `getDomainOptions`.
-            var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender');
+            var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender') || options?.prerender;
             if (enable_domain_prerender) {
                 setPrerender(enable_domain_prerender);
             }

modules/api/views.js

@@ -140,6 +140,7 @@ export default function(app) {
                     dataMode: getBooleanParam(req, 'dataMode'),
                     forceParams: req.query.meta === "true" ? CONFIG.DEBUG_CONTEXTS : null,
                     whitelist: getBooleanParam(req, 'whitelist'),
+                    // TOOD: obsolete?
                     readability: getBooleanParam(req, 'readability'),
                     getWhitelistRecord: whitelist.findWhitelistRecordFor,
                     maxWidth: getIntParam(req, 'maxwidth') || getIntParam(req, 'max-width'),
@@ -158,9 +159,16 @@ export default function(app) {
             }
 
             if (result.safe_html) {
+                var readerJsParams = new url.URLSearchParams({
+                    uri: uri
+                });
+                if (req.query.dataMode) {
+                    readerJsParams.set('dataMode', req.query.dataMode);
+                }
+
                 cache.set('html:' + version + ':' + uri, result.safe_html);
                 result.links.unshift({
-                    href: CONFIG.baseAppUrl + "/reader.js?uri=" + encodeURIComponent(uri),
+                    href: CONFIG.baseAppUrl + '/reader.js?' + readerJsParams.toString(),
                     type: CONFIG.T.javascript,
                     rel: [CONFIG.R.reader, CONFIG.R.inline]
                 });
@@ -280,7 +288,9 @@ export default function(app) {
                     iframelyCore.run(uri, {
                         v: '1.3',
                         getWhitelistRecord: whitelist.findWhitelistRecordFor,
-                        readability: true
+                        // TOOD: obsolete?
+                        readability: true,
+                        dataMode: getBooleanParam(req, 'dataMode'),
                     }, function(error, data) {
 
                         if (!data || !data.safe_html) {