Javascript rendering - detecting window.location changes #28

replaysMike · 2022-03-30T20:01:45Z

What would be your recommended way of dealing with window.location changes on the page? I'm crawling sites that have a method that looks something like the following probably to break crawlers:

function iframeOnLoad(){
  var reqUrl='https://domain.com/page_i_want';
  setTimeout(function() { window.location = reqUrl }, 3000);
}

<iframe onload="iframeOnload()" />

Assuming PhantonJs is rendering this, is it possible to detect url changes when window.location is set via JS? I could maybe write some custom addons but I'm not sure if this is already handled somehow.

replaysMike · 2022-03-30T20:35:19Z

I think I've got a starting point. If I modify the phantomJs RunScript in RunPhantomJsWithStaticWaitTime():

function printArgs() {
    var i, ilen;
    for (i = 0, ilen = arguments.length; i < ilen; ++i) {
        console.log('    arguments[' + i + '] = ' + JSON.stringify(arguments[i]));
    }
    console.log('');
}
page.onNavigationRequested = function() {
    console.log('onNavigationRequested');
    printArgs.apply(this, arguments);
};

the URL shows up in the rendered javascript output as long as I set the JavascriptRenderingWaitTimeInMilliseconds long enough. I should be able to wire in some detection somewhere in here by changing the console.log entries to output anchor href links.

replaysMike · 2022-03-30T21:39:44Z

Here is what I settled on, not sure if you want to include it in AbotX2 default implementation:

var system = require('system');
var inputData = system/*......code snippet excluded......*/
page.settings.userAgent = input.UserAgent;

// js redirect detection
var detectedUrls = [];
page.onNavigationRequested = function() {
    if (arguments.length > 0) {
        var url = arguments[0];
        if (url !== 'about:blank' && url !== input.Url) {
            detectedUrls.push(url);
        }
    }
};

page.setContent(input.RawHtml, input.Url);
                        
window.setTimeout(function () {
    var content = page.content;
    if (detectedUrls && detectedUrls.length > 0) {
        var detectedUrlLinksString = '';
        for(var i = 0; i < detectedUrls.length; i++) {
            detectedUrlLinksString = detectedUrlLinksString + '<a href=' + detectedUrls[i] + '>Redirect</a>';
        }
        if (new RegExp('</body>', 'i').test(content))
            content = content.replace(/<\/body>/gi, detectedUrlLinksString + '</body>');
        else if (new RegExp('</html>', 'i').test(content))
            content = content.replace(/<\/html>/gi, '<body>' + detectedUrlLinksString + '</body></html>');
        else
            content = content + detectedUrlLinksString;
    } else {
        // no detected urls
    }

    console.log(content);
    phantom.exit();
}, input.WaitToRenderTimeInMilliseconds);

sjdirect · 2022-03-30T23:59:51Z

Hi appreciate circling back to share your solution. It does seem like a reasonable approach. I don't think it's something we should add back to the main solution but lets leave this up for others to find in case they have similar needs.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Javascript rendering - detecting window.location changes #28

Javascript rendering - detecting window.location changes #28

replaysMike commented Mar 30, 2022

replaysMike commented Mar 30, 2022 •

edited

Loading

replaysMike commented Mar 30, 2022 •

edited

Loading

sjdirect commented Mar 30, 2022

Javascript rendering - detecting window.location changes #28

Javascript rendering - detecting window.location changes #28

Comments

replaysMike commented Mar 30, 2022

replaysMike commented Mar 30, 2022 • edited Loading

replaysMike commented Mar 30, 2022 • edited Loading

sjdirect commented Mar 30, 2022

replaysMike commented Mar 30, 2022 •

edited

Loading

replaysMike commented Mar 30, 2022 •

edited

Loading