diff --git a/.gitignore b/.gitignore index 7d65964c0..b604acc93 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,11 @@ -.vagrant - -build/* -src/icons/converted/* - -node_modules -npm-debug.log - -tags +/build/* +/src/icons/converted/* +/node_modules +/npm-debug.log !.gitkeep +# Junk files Thumbs.db Desktop.ini *.DS_Store diff --git a/.travis.yml b/.travis.yml index d9239161a..ca6c98893 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,9 @@ -language: node_js -node_js: - - "node" sudo: false -dist: trusty -before_install: - - export WAPPALYZER_ROOT=$TRAVIS_BUILD_DIR - - export WAPPALYZER_NODE_PATH=$TRAVIS_BUILD_DIR - - export PATH=$PATH:$TRAVIS_BUILD_DIR/bin - - ln -s docker/node/package.json package.json -before_script: npm i -g manifoldjs +services: docker after_success: - sha256sum build/* > build/SHA256SUMS - cat build/SHA256SUMS +script: ./run build deploy: provider: releases api_key: @@ -22,9 +14,5 @@ deploy: on: repo: AliasIO/Wappalyzer tags: true -after_script: ls -l --block-size=kB build -cache: - directories: - - node_modules env: - CXX=g++-4.8 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index bffe6a2e1..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,3 +0,0 @@ -# Contributing - -You can find our contribution guide [in our wiki](https://github.com/AliasIO/Wappalyzer/wiki/Contributing). diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..e10b84532 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM alpine + +MAINTAINER Elbert Alias + +ENV WAPPALYZER_DIR=/opt/wappalyzer + +RUN apk add --no-cache \ + bash \ + curl \ + fontconfig \ + nodejs \ + optipng \ + zip + +RUN mkdir -p /usr/share && \ + cd /usr/share \ + && curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 | tar xj \ + && ln -s /usr/share/phantomjs/phantomjs /usr/bin/phantomjs + +RUN apk del \ + curl + +RUN npm i -g \ + jsonlint-cli \ + manifoldjs \ + svg2png-many + +RUN mkdir -p $WAPPALYZER_DIR + +WORKDIR $WAPPALYZER_DIR + +CMD [ "./bin/run" ] diff --git a/bin/wappalyzer-build b/bin/build similarity index 51% rename from bin/wappalyzer-build rename to bin/build index a07a85b59..af6a94195 100755 --- a/bin/wappalyzer-build +++ b/bin/build @@ -1,82 +1,57 @@ #!/bin/bash -if [ -z $WAPPALYZER_ROOT ] -then - echo "\$WAPPALYZER_ROOT not set" +cd "$(dirname $0)/.." - exit 1 -fi - -if [ -z $WAPPALYZER_NODE_PATH ] -then - echo "\$WAPPALYZER_NODE_PATH not set" - - exit 1 -fi - -set -eu - -wappalyzer validate +./bin/validate echo "Prettifying apps.json..." -node $WAPPALYZER_NODE_PATH/node_modules/jsonlint/lib/cli.js -ist $'\t' $WAPPALYZER_ROOT/src/apps.json - -echo "Compressing PNG and SVG icons..." - -node $WAPPALYZER_NODE_PATH/node_modules/imagemin-cli/cli.js $WAPPALYZER_ROOT/src/icons/* --out-dir $WAPPALYZER_ROOT/src/icons 2>&1 > /dev/null +jsonlint-cli -ist $'\t' src/apps.json echo "Converting SVG icons to PNG..." -OIFS="$IFS" -IFS=$'\n' - -for svg in $(find $WAPPALYZER_ROOT/src/icons -type f -name "*.svg") -do - echo " Converting $(basename "$svg")..." - - dest="$WAPPALYZER_ROOT/src/icons/converted/$(basename "$svg").png" - - if [[ -f "$dest" ]]; then - rm "$dest" - fi +#svg2png-many -i src/icons/ -o src/icons/converted/ --width=32 --height=32 - node $WAPPALYZER_NODE_PATH/node_modules/svg2png/bin/svg2png-cli "$svg" --width=32 --height=32 --output "$dest" || true -done +echo "Compressing PNG icons..." -IFS="$OIFS" +optipng -quiet "src/icons/*.png" +optipng -quiet "src/icons/converted/*.png" -echo "Compressing converted PNG icons..." - -node $WAPPALYZER_NODE_PATH/node_modules/imagemin-cli/cli.js $WAPPALYZER_ROOT/src/icons/converted/* $WAPPALYZER_ROOT/src/icons/converted - -wappalyzer links # WebExtension -echo "Building WebExtension driver..." +echo "Building WebExtension..." -webextension_dir=$WAPPALYZER_ROOT/src/drivers/webextension +webextension_dir=src/drivers/webextension pushd $webextension_dir > /dev/null -zip -qr $WAPPALYZER_ROOT/build/wappalyzer_webextension.zip . +zip -qr ../../../build/wappalyzer_webextension.zip . popd > /dev/null # Edge +echo "Building Edge application..." -pushd $WAPPALYZER_ROOT/build > /dev/null +pushd build > /dev/null + +webextension_dir="../$webextension_dir" +manifest_dir="Wappalyzer/edgeextension/manifest" mv $webextension_dir/manifest.json $webextension_dir/manifest.webextension.json mv $webextension_dir/manifest.edge.json $webextension_dir/manifest.json manifoldjs -l debug -p edgeextension -f edgeextension -m $webextension_dir/manifest.json +# Replace symlinks with actual files +rm $manifest_dir/Extension/images/icons + +cp ../src/apps.json $manifest_dir/Extension +cp ../src/wappalyzer.js $manifest_dir/Extension/js +cp -r ../src/icons $manifest_dir/Extension/images + mv $webextension_dir/manifest.json $webextension_dir/manifest.edge.json mv $webextension_dir/manifest.webextension.json $webextension_dir/manifest.json -manifest_dir="Wappalyzer/edgeextension/manifest" - sed -i 's/INSERT-YOUR-PACKAGE-IDENTITY-NAME-HERE/Wappalyzer/' $manifest_dir/appxmanifest.xml sed -i 's/INSERT-YOUR-PACKAGE-IDENTITY-PUBLISHER-HERE/Wappalyzer/' $manifest_dir/appxmanifest.xml sed -i 's/INSERT-YOUR-PACKAGE-PROPERTIES-PUBLISHERDISPLAYNAME-HERE/Wappalyzer/' $manifest_dir/appxmanifest.xml @@ -94,4 +69,4 @@ rm -rf Wappalyzer popd > /dev/null -echo "Done. Builds have been created in $WAPPALYZER_ROOT/build." +echo "Done." diff --git a/bin/wappalyzer-help b/bin/help similarity index 84% rename from bin/wappalyzer-help rename to bin/help index d832f88c1..736fc925f 100755 --- a/bin/wappalyzer-help +++ b/bin/help @@ -1,7 +1,7 @@ #!/bin/bash cat << 'EOF' -usage: wappalyzer [] +usage: ./run [] The following commands are available: build Package drivers diff --git a/bin/links b/bin/links new file mode 100755 index 000000000..c7905c0c4 --- /dev/null +++ b/bin/links @@ -0,0 +1,14 @@ +#!/bin/bash + +cd "$(dirname $0)/../src" + +echo "Creating links..." + +ln -nsf ../../apps.json drivers/npm +ln -nsf ../../wappalyzer.js drivers/npm + +ln -nsf ../../apps.json drivers/webextension +ln -nsf ../../../wappalyzer.js drivers/webextension/js +ln -nsf ../../../icons/ drivers/webextension/images/icons + +echo "Done." diff --git a/bin/run b/bin/run new file mode 100755 index 000000000..f7a29c820 --- /dev/null +++ b/bin/run @@ -0,0 +1,11 @@ +#!/bin/bash + +cd "$(dirname $0)" + +if [[ -f "$1" ]]; then + ./$1 $@ +else + ./help + + exit 1 +fi diff --git a/bin/validate b/bin/validate new file mode 100755 index 000000000..1d6587600 --- /dev/null +++ b/bin/validate @@ -0,0 +1,17 @@ +#!/bin/bash + +cd "$(dirname $0)/.." + +set -eu + +echo "Validating apps.json..." + +jsonlint-cli -s schema.json src/apps.json + +echo "Validating regular expressions..." + +./bin/validate-regex + +echo "Validating icons..." + +./bin/validate-icons diff --git a/bin/validate-icons b/bin/validate-icons new file mode 100755 index 000000000..e1e629bbe --- /dev/null +++ b/bin/validate-icons @@ -0,0 +1,43 @@ +#!/usr/bin/env node + +var + app, + fs = require('fs'), + fileType = require('../node_modules/file-type'), + readChunk = require('../node_modules/read-chunk') + isSvg = require('../node_modules/is-svg') + json = require('../src/apps.json'); + +for (app in json.apps) { + (function(app) { + var + basePath = 'src/icons/'; + iconPath = json.apps[app].icon || 'default.svg'; + path = basePath + iconPath, + ext = iconPath.substr(iconPath.length - 4); + + if ( ext !== '.png' && ext !== '.svg' ) { + throw err = new Error('Icon file extension specified for app "' + app + '" is not ".png" or ".svg": src/icons/' + iconPath); + } + + fs.exists(path, function(exists) { + if ( exists ) { + if ( ext === '.png' ) { + var buffer = fileType(readChunk.sync(path, 0, 262)); + + if ( buffer === null ) { + throw new Error('Unknown mimetype or bad file for "' + app + '": src/icons/' + iconPath); + } else if ( buffer.mime !== 'image/png' ) { + throw new Error('Incorrect mimetype "' + buffer.mime + '" when expected PNG for app "' + app + '": src/icons/' + iconPath); + } + } else if ( type === 'svg' ) { + if ( !isSvg(fs.readFileSync(path)) ) { + throw new Error('Incorrect mimetype when expected SVG for app "' + app + '": src/icons/' + iconPath); + } + } + } else { + throw Error('Missing file for app "' + app + '": src/icons/' + iconPath); + } + }); + }(app)); +} diff --git a/bin/wappalyzer-validate-regex b/bin/validate-regex similarity index 94% rename from bin/wappalyzer-validate-regex rename to bin/validate-regex index f7976c116..423aa2c69 100755 --- a/bin/wappalyzer-validate-regex +++ b/bin/validate-regex @@ -2,8 +2,7 @@ var app, - modulesPath = process.env.WAPPALYZER_NODE_PATH !== undefined ? process.env.WAPPALYZER_NODE_PATH + '/node_modules/' : '', - json = require(process.env.WAPPALYZER_ROOT + '/src/apps.json'); + json = require('../src/apps.json'); for ( app in json.apps ) { ['headers', 'html', 'env', 'meta', 'script'].forEach(function(type) { diff --git a/bin/wappalyzer b/bin/wappalyzer deleted file mode 100755 index 5c1cb8261..000000000 --- a/bin/wappalyzer +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ -f "$0-$1" ] -then - $0-$1 ${*:2} -else - $0-help - - exit 1 -fi - -exit diff --git a/bin/wappalyzer-links b/bin/wappalyzer-links deleted file mode 100755 index ae75f4432..000000000 --- a/bin/wappalyzer-links +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -path=$1 - -if [ -z $path ] -then - if [ -z $WAPPALYZER_ROOT ] - then - echo "-$(basename $0): No path specified" - - exit 1 - fi - - path=$WAPPALYZER_ROOT -fi - -set -eu - -if [ ! -d $path/src ] -then - echo "-$(basename $0): Incorrect path" - - exit 1 -fi - -path="$path/src" - -echo "Creating hard links..." - -ln -f $path/wappalyzer.js $path/drivers/phantomjs -ln -f $path/apps.json $path/drivers/phantomjs - -ln -f $path/wappalyzer.js $path/drivers/webextension/js -ln -f $path/apps.json $path/drivers/webextension -ln -f $path/icons/*.png $path/drivers/webextension/images/icons -ln -f $path/icons/*.svg $path/drivers/webextension/images/icons -ln -f $path/utils/*.js $path/drivers/webextension/js - -if [ "$(compgen -G "$path/icons/converted/*.png" | head -n1)" ]; then - ln -f $path/icons/converted/*.png $path/drivers/webextension/images/icons/converted -fi - -echo "OK" - -exit 0 diff --git a/bin/wappalyzer-validate b/bin/wappalyzer-validate deleted file mode 100755 index 0106c4d18..000000000 --- a/bin/wappalyzer-validate +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -path="$1" - -if [ -z "$path" ]; then - if [ -z "$WAPPALYZER_ROOT" ]; then - echo "-$(basename $0): No path specified" - - exit 1 - fi - - path="$WAPPALYZER_ROOT" -fi - -set -eu - -if [ ! -d "$path/src" ]; then - echo "-$(basename $0): Incorrect path" - - exit 1 -fi - -path="$path/src" - -echo "Validating apps.json..." - -node "$WAPPALYZER_NODE_PATH/node_modules/jsonlint/lib/cli.js" --quiet -V "$WAPPALYZER_ROOT/schema.json" "$path/apps.json" - -echo "Validating regular expressions..." - -wappalyzer validate-regex - -echo "Validating icons..." - -wappalyzer validate-icons diff --git a/bin/wappalyzer-validate-icons b/bin/wappalyzer-validate-icons deleted file mode 100755 index 185511826..000000000 --- a/bin/wappalyzer-validate-icons +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env node - -var - app, - modulesPath = process.env.WAPPALYZER_NODE_PATH !== undefined ? process.env.WAPPALYZER_NODE_PATH + '/node_modules/' : '', - fs = require('fs'), - fileType = require(modulesPath + 'file-type'), - readChunk = require(modulesPath + 'read-chunk') - isSvg = require(modulesPath + 'is-svg') - json = require(process.env.WAPPALYZER_ROOT + '/src/apps.json'); - -for (app in json.apps) { - (function(app) { - var basePath = process.env.WAPPALYZER_ROOT + '/src/icons/'; - var iconPath = json.apps[app].icon || 'default.svg'; - var path = basePath + iconPath; - - var type; - if (path.substr(path.length - 4) === '.png') { - type = "PNG"; - } - else if (path.substr(path.length - 4) === '.svg') { - type = "SVG"; - } - else { - var err = new Error('Icon file extension specified for app "' + app + '" is not ".png" or ".svg": src/icons/' + iconPath); - throw err; - } - - fs.exists(path, function(exists) { - if (exists) { - if (type === "PNG") { - var buffer = fileType(readChunk.sync(path, 0, 262)); - if (buffer === null) { - var err = new Error('Unknown mimetype or bad file for "' + app + '": src/icons/' + iconPath); - throw err; - } - else if (buffer.mime !== 'image/png') { - var err = new Error('Incorrect mimetype "' + buffer.mime + '" when expected PNG for app "' + app + '": src/icons/' + iconPath); - throw err; - } - } - else if (type === "SVG") { - if (!isSvg(fs.readFileSync(path))) { - var err = new Error('Incorrect mimetype when expected SVG for app "' + app + '": src/icons/' + iconPath); - throw err; - } - } - } else { - var err = new Error('Missing file for app "' + app + '": src/icons/' + iconPath); - throw err; - } - }); - }(app)); -}; diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index c35776024..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,72 +0,0 @@ -FROM phusion/baseimage - -MAINTAINER Elbert Alias - -ENV DEBIAN_FRONTEND noninteractive - -ENV WAPPALYZER_ROOT /home/wappalyzer/synced -ENV WAPPALYZER_NODE_PATH /home/wappalyzer/node - -# Install packages -RUN sed -i 's/^deb-src\ /\#deb-src\ /g' /etc/apt/sources.list -RUN sed -i 's/archive\.ubuntu\.com/au.archive.ubuntu.com/g' /etc/apt/sources.list -RUN sed -i 's/security\.ubuntu\.com/au.archive.ubuntu.com/g' /etc/apt/sources.list -RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \ - apt-get install -y \ - nodejs \ - bzip2 \ - zip \ - libfreetype6 \ - libfontconfig \ - rsync && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - - -# Add user -RUN useradd -ms /bin/bash wappalyzer && usermod -a -G docker_env wappalyzer && echo "wappalyzer:wappalyzer" | chpasswd -RUN echo 'wappalyzer ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers - -WORKDIR /home/wappalyzer - -RUN su wappalyzer -c "mkdir bin synced" -RUN su wappalyzer -c "echo \"export PATH=\$PATH:/home/wappalyzer/bin:\\$WAPPALYZER_ROOT/bin\" | cat - .profile > /tmp/profile && mv /tmp/profile .profile" - -RUN echo "cd \$WAPPALYZER_ROOT" >> .bashrc -RUN echo "wappalyzer" >> .bashrc - - -# PhantomJS -# RUN su wappalyzer -c "\ -# mkdir phantomjs && \ -# curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 | tar xvjC phantomjs && \ -# ln -s /home/wappalyzer/phantomjs/phantomjs-*-linux-x86_64/bin/phantomjs /usr/bin/phantomjs" - - -# Node JS -RUN su wappalyzer -c "mkdir $WAPPALYZER_NODE_PATH" - -ADD node/package.json $WAPPALYZER_NODE_PATH/package.json - -RUN su wappalyzer -c "cd $WAPPALYZER_NODE_PATH && npm i" - -RUN npm i -g manifoldjs - - -# SSH -RUN rm -f /etc/service/sshd/down -RUN su wappalyzer -c "mkdir .ssh && chmod 700 .ssh" - -ADD ssh/insecure.key.pub /tmp/insecure.key.pub - -RUN su wappalyzer -c "cat /tmp/insecure.key.pub >> .ssh/authorized_keys && chmod 600 .ssh/authorized_keys" && rm -f /tmp/insecure.key.pub - - -# Fix the `stdin: is not a tty` error in Vagrant -RUN sed -i 's/^mesg n$/tty -s \&\& mesg n/g' /root/.profile - -EXPOSE 22 - -RUN echo "/usr/sbin/sshd -D" > /etc/my_init.d/sshd.sh - -CMD ["/sbin/my_init"] diff --git a/docker/node/package.json b/docker/node/package.json deleted file mode 100644 index cbfe4c39e..000000000 --- a/docker/node/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "wappalyzer-test", - "private": true, - "license": "GPLv3", - "bugs": { - "url": "https://github.com/AliasIO/Wappalyzer/issues" - }, - "homepage": "https://github.com/AliasIO/Wappalyzer", - "description": "Build test for Wappalyzer", - "repository": "AliasIO/Wappalyzer", - "scripts": { - "test": "./bin/wappalyzer build" - }, - "devDependencies": { - "file-type": "3.8.*", - "is-svg": "2.0.*", - "read-chunk": "2.0.*", - "jsonlint": "*", - "jpm": "*", - "imagemin-cli": "2.*.*", - "phantomjs-prebuilt": "*", - "svg2png": "3.*.*" - }, - "engines": { - "node": ">= 4" - } -} diff --git a/docker/ssh/insecure.key b/docker/ssh/insecure.key deleted file mode 100644 index bf599e805..000000000 --- a/docker/ssh/insecure.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAqVo54bv3J4K62IDCUIVY7gF2iBOR8fub5c9t6j4MbsRQDmU0 -yxhN1+7fMCERcoMSelAb/SYSzKE8azLugP7hsq4t+/xJijCB22uriFhKMpu4ySoA -6PVNbfbIptaVo6holVIMRKPXTg1hU6lr2DWMKB5AlGmz7ZHmQgX9iBoV8PmMYOF5 -UpiwfokcNnIwDiCCpd19G/lT6FlCVZHZdeZh41FWke8oQgWevM8fdGJkZ+a7Lgdm -6A3WlvI0AIz9KWfXlXAOvthi4QlB4SvSCwRQbKdHbW5UBRShucjlce3hZO9Q6ROC -Ta1zgvscleKd63yHSQ18Z2+j1t5T3HJzhOn1ywIDAQABAoIBAFdbcWwd15oL93Hi -yVlbz8nF+2yqG6ilT3+2Bk7EfwlMPreOQTNacB0o3PwosqVNM2Tgme91ilEAAviR -JzJnR+kxrArF0vFzoCidW7hTQsLkTOeHwTxp/OzyVgKpwjrPjlMoJafqlA9xG3la -dYTr6kZvhISKGgQ8NHhuOp/QRO0uFaQD8l0kVCEufgqbs6gBY7aAkNf2iogSms9w -139zwsXeSg2Vm7Img1PVrRsRM5qbTNBMIim+9q4fSrqP+rYF9lU1p68IzmG/wY2o -0ElwRoatVVV9Rvk00a1qBGNaT39oVEtryIiAlXPno+KW5dyVx53o9vjWHDz44yBN -G8/EgEECgYEA3UJtj5ikEdhquzy1WfbZwZVmTCJLMefiVui00RLV4Q/r/4Uojdsb -L/QCAplpbHUrj7O8Lp9CuXSnUNa4CkrTidj2wkDWDFl1EBLJNz1zhJjs6Aaw2u1t -SV6ULgx9Y25oiStygkJhon+wlNF9HRSCFeNkBNNjpkD2gEifxnHIDLsCgYEAw/Fi -OHFvVPx0/DS3FD/MEJAAZaGdg8Zb2TCmCIVjoj0tZOTDvwKIEYCQ41vADDLlCSGg -XOgNXvosSZtezmJxnkzO3NYXE4XZ7bPVMamuqbKXXzaee3tPmiFaeZfiB4tj7uxh -tWQdUicSMQ8WPqyTB5wmcF7k3L4TzrjSLOi5MjECgYEAnsLYfI3pREjGpN01v3zp -2QbLZVvCGugpN3U4sUBTqW9URn+Ujt0mD+2FN7o+tC0K+czc/ZMXPJ1gpRe87N2u -Fh5RSSder3PNFKB2MyXr6rwUWZTbd+ufXGL57mTV3+/MIIjFXIQnLIN1AAOlQ6WA -hhYooXHQ8e1xMrFpu+nOwpsCgYAanYEf9ZiPzLL/RxHPOFM5tLW5GFjS+3Cicc87 -vRKpiPHPkJteKLPe3L3RhDbz8T3XHbAwrnySeBzn6qjYE9snC3vV12bPChFuDk1Z -L5lAB7g0ALFKSJepcfpeJdlX+QEcBRPIlc69x/zamRgGb16RXA6EmP8xVMhP/h7e -ZsYxIQKBgBqOaZNHwaG6PqP2HCjc1keH4wQVx9eTV5y4/B5FJbMeK05vV2rBHoxA -P/dXE85olj8D/9E88xJ1V716kBeONcKFdbd3VfY5bo2VBz+piOaG3EBQOLIbd4/C -KVtgOWqI7WTLPIgpXlG5MA801RwhVyk1phmwhvW/Kb3lWsK1vkOe ------END RSA PRIVATE KEY----- diff --git a/docker/ssh/insecure.key.pub b/docker/ssh/insecure.key.pub deleted file mode 100644 index 48f46c871..000000000 --- a/docker/ssh/insecure.key.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCpWjnhu/cngrrYgMJQhVjuAXaIE5Hx+5vlz23qPgxuxFAOZTTLGE3X7t8wIRFygxJ6UBv9JhLMoTxrMu6A/uGyri37/EmKMIHba6uIWEoym7jJKgDo9U1t9sim1pWjqGiVUgxEo9dODWFTqWvYNYwoHkCUabPtkeZCBf2IGhXw+Yxg4XlSmLB+iRw2cjAOIIKl3X0b+VPoWUJVkdl15mHjUVaR7yhCBZ68zx90YmRn5rsuB2boDdaW8jQAjP0pZ9eVcA6+2GLhCUHhK9ILBFBsp0dtblQFFKG5yOVx7eFk71DpE4JNrXOC+xyV4p3rfIdJDXxnb6PW3lPccnOE6fXL elbert@home diff --git a/package.json b/package.json new file mode 100644 index 000000000..2148f0f67 --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "dependencies": { + "file-type": "3.8.*", + "is-svg": "2.0.*", + "read-chunk": "2.0.*" + } +} + diff --git a/run b/run new file mode 100755 index 000000000..237565872 --- /dev/null +++ b/run @@ -0,0 +1,19 @@ +#!/bin/bash + +cd "$(dirname $0)" + +if [[ -z "$(which docker)" ]]; then + echo "Please install Docker from https://www.docker.com" + + exit 1 +fi + +docker="docker run --rm -v "$(pwd):/opt/wappalyzer" -it wappalyzer/dev" + +if [[ ! -d "node_modules" ]]; then + $docker ./bin/run links + + $docker npm i +fi + +$docker ./bin/run $@ diff --git a/src/drivers/phantomjs/.gitignore b/src/drivers/npm/.gitignore similarity index 100% rename from src/drivers/phantomjs/.gitignore rename to src/drivers/npm/.gitignore diff --git a/src/drivers/phantomjs/Dockerfile b/src/drivers/npm/Dockerfile similarity index 100% rename from src/drivers/phantomjs/Dockerfile rename to src/drivers/npm/Dockerfile diff --git a/src/drivers/phantomjs/README.md b/src/drivers/npm/README.md similarity index 100% rename from src/drivers/phantomjs/README.md rename to src/drivers/npm/README.md diff --git a/src/drivers/phantomjs/driver.js b/src/drivers/npm/driver.js similarity index 100% rename from src/drivers/phantomjs/driver.js rename to src/drivers/npm/driver.js diff --git a/src/drivers/phantomjs/index.js b/src/drivers/npm/index.js similarity index 100% rename from src/drivers/phantomjs/index.js rename to src/drivers/npm/index.js diff --git a/src/drivers/phantomjs/package.json b/src/drivers/npm/package.json similarity index 100% rename from src/drivers/phantomjs/package.json rename to src/drivers/npm/package.json diff --git a/src/drivers/webextension/.gitignore b/src/drivers/webextension/.gitignore index 602df1590..3e368c431 100644 --- a/src/drivers/webextension/.gitignore +++ b/src/drivers/webextension/.gitignore @@ -1,7 +1,6 @@ -apps.json -images/icons/converted/*.png -images/icons/*.png -images/icons/*.svg -js/wappalyzer.js -js/iframe.js -js/network.js +/apps.json +/images/icons/converted/* +/images/icons/* +/js/wappalyzer.js +/js/iframe.js +/js/network.js diff --git a/src/drivers/webextension/images/icons b/src/drivers/webextension/images/icons new file mode 120000 index 000000000..f7467b9a7 --- /dev/null +++ b/src/drivers/webextension/images/icons @@ -0,0 +1 @@ +../../../icons/ \ No newline at end of file diff --git a/src/drivers/webextension/images/icons/.gitkeep b/src/drivers/webextension/images/icons/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/drivers/webextension/images/icons/converted/.gitkeep b/src/drivers/webextension/images/icons/converted/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/icons/3dCart.png b/src/icons/3dCart.png index a8fc57467..64ca4abc8 100644 Binary files a/src/icons/3dCart.png and b/src/icons/3dCart.png differ diff --git a/src/icons/ActOn.png b/src/icons/ActOn.png index d288e5116..de6b48edd 100644 Binary files a/src/icons/ActOn.png and b/src/icons/ActOn.png differ diff --git a/src/icons/Amazon-Cloudfront.svg b/src/icons/Amazon-Cloudfront.svg index a28de1faa..7454c8e58 100755 --- a/src/icons/Amazon-Cloudfront.svg +++ b/src/icons/Amazon-Cloudfront.svg @@ -1,23 +1 @@ - - NetworkingContentDelivery - - - - - - - - - - - - - - - - - - - - - +NetworkingContentDelivery \ No newline at end of file diff --git a/src/icons/AngularJS.svg b/src/icons/AngularJS.svg index 649c9f92f..003deea80 100644 --- a/src/icons/AngularJS.svg +++ b/src/icons/AngularJS.svg @@ -1,74 +1 @@ - - - -image/svg+xml \ No newline at end of file + \ No newline at end of file diff --git a/src/icons/Apache Traffic Server.png b/src/icons/Apache Traffic Server.png index f5cca2ecb..0faf78e0f 100644 Binary files a/src/icons/Apache Traffic Server.png and b/src/icons/Apache Traffic Server.png differ diff --git a/src/icons/Apache.svg b/src/icons/Apache.svg index 2494ae756..775665d1c 100644 --- a/src/icons/Apache.svg +++ b/src/icons/Apache.svg @@ -1,3 +1 @@ - - -image/svg+xml + \ No newline at end of file diff --git a/src/icons/Aurelia.svg b/src/icons/Aurelia.svg index 3bf9ad8fd..3d157b859 100644 --- a/src/icons/Aurelia.svg +++ b/src/icons/Aurelia.svg @@ -1,64 +1 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/src/icons/Bootstrap Table.svg b/src/icons/Bootstrap Table.svg index 2f05ef1f4..a5d95e778 100644 --- a/src/icons/Bootstrap Table.svg +++ b/src/icons/Bootstrap Table.svg @@ -1,4 +1 @@ - - - BT - +BT \ No newline at end of file diff --git a/src/icons/Clipboard.js.svg b/src/icons/Clipboard.js.svg index 007a950a7..aa2c564d3 100644 --- a/src/icons/Clipboard.js.svg +++ b/src/icons/Clipboard.js.svg @@ -1,3 +1 @@ - - - + \ No newline at end of file diff --git a/src/icons/Comandia.svg b/src/icons/Comandia.svg index 9c91aef7e..24a7be8df 100644 --- a/src/icons/Comandia.svg +++ b/src/icons/Comandia.svg @@ -1,9 +1 @@ - - - - - - - - - \ No newline at end of file + \ No newline at end of file diff --git a/src/icons/Express.png b/src/icons/Express.png index e89a824d9..8f3461e69 100644 Binary files a/src/icons/Express.png and b/src/icons/Express.png differ diff --git a/src/icons/Fastly.svg b/src/icons/Fastly.svg index 05ae7d05d..bb7a0ded2 100644 --- a/src/icons/Fastly.svg +++ b/src/icons/Fastly.svg @@ -1,31 +1 @@ - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/src/icons/Firebase.png b/src/icons/Firebase.png index 41e5de9ea..4cfb7b3af 100644 Binary files a/src/icons/Firebase.png and b/src/icons/Firebase.png differ diff --git a/src/icons/FlexSlider.png b/src/icons/FlexSlider.png old mode 100755 new mode 100644 index 9c264278f..b1cb7fcb7 Binary files a/src/icons/FlexSlider.png and b/src/icons/FlexSlider.png differ diff --git a/src/icons/ForkCMS.png b/src/icons/ForkCMS.png old mode 100755 new mode 100644 index 5057b873c..3cde28cd1 Binary files a/src/icons/ForkCMS.png and b/src/icons/ForkCMS.png differ diff --git a/src/icons/Froala.svg b/src/icons/Froala.svg index 725e7ff3c..dfd7d0421 100644 --- a/src/icons/Froala.svg +++ b/src/icons/Froala.svg @@ -1 +1 @@ -Froala \ No newline at end of file +Froala \ No newline at end of file diff --git a/src/icons/Google Search Appliance.png b/src/icons/Google Search Appliance.png index 2e1f97ece..71bb47077 100644 Binary files a/src/icons/Google Search Appliance.png and b/src/icons/Google Search Appliance.png differ diff --git a/src/icons/Homeland.png b/src/icons/Homeland.png index a269934c4..e8ca492f0 100644 Binary files a/src/icons/Homeland.png and b/src/icons/Homeland.png differ diff --git a/src/icons/Kamva.svg b/src/icons/Kamva.svg index 8bbf89c96..0304de60a 100644 --- a/src/icons/Kamva.svg +++ b/src/icons/Kamva.svg @@ -1,212 +1 @@ - - - -image/svg+xml + \ No newline at end of file diff --git a/src/icons/Klarna.svg b/src/icons/Klarna.svg index e611627af..0bb061c85 100644 --- a/src/icons/Klarna.svg +++ b/src/icons/Klarna.svg @@ -1,12 +1 @@ - - - - klarna - Created with Sketch. - - - - - - - \ No newline at end of file +klarna \ No newline at end of file diff --git a/src/icons/Medium.svg b/src/icons/Medium.svg index f85a5519f..afaa40a70 100755 --- a/src/icons/Medium.svg +++ b/src/icons/Medium.svg @@ -1,9 +1 @@ - - - - - - - - - + \ No newline at end of file diff --git a/src/icons/Netlify.svg b/src/icons/Netlify.svg index 370591793..fa36dd539 100644 --- a/src/icons/Netlify.svg +++ b/src/icons/Netlify.svg @@ -1,9 +1 @@ - - - - - - - - - + \ No newline at end of file diff --git a/src/icons/Neto.svg b/src/icons/Neto.svg index a4fabcbe3..4ed0473bf 100644 --- a/src/icons/Neto.svg +++ b/src/icons/Neto.svg @@ -1,36 +1 @@ - - - - - - - - - - - + \ No newline at end of file diff --git a/src/icons/PHP.svg b/src/icons/PHP.svg index e5799b153..894c7f342 100644 --- a/src/icons/PHP.svg +++ b/src/icons/PHP.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/src/icons/Play.svg b/src/icons/Play.svg index 10925d767..098d6b665 100644 --- a/src/icons/Play.svg +++ b/src/icons/Play.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/src/icons/Plotly.png b/src/icons/Plotly.png index 20f7b24d8..b970858cb 100644 Binary files a/src/icons/Plotly.png and b/src/icons/Plotly.png differ diff --git a/src/icons/Raspbian.svg b/src/icons/Raspbian.svg index 2131cf097..ceed5e799 100644 --- a/src/icons/Raspbian.svg +++ b/src/icons/Raspbian.svg @@ -1,17 +1 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file + \ No newline at end of file diff --git a/src/icons/Revel.png b/src/icons/Revel.png index f8e9b6ef5..89494132d 100644 Binary files a/src/icons/Revel.png and b/src/icons/Revel.png differ diff --git a/src/icons/RockRMS.svg b/src/icons/RockRMS.svg index 1a782b9bb..6b352f6ca 100644 --- a/src/icons/RockRMS.svg +++ b/src/icons/RockRMS.svg @@ -1,12 +1 @@ - - - - rock-logo - Created with Sketch. - - - - - \ No newline at end of file +rock-logo \ No newline at end of file diff --git a/src/icons/Salesforce.svg b/src/icons/Salesforce.svg index c5635099f..8209a8af9 100644 --- a/src/icons/Salesforce.svg +++ b/src/icons/Salesforce.svg @@ -1,159 +1 @@ - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/src/icons/Scholica.svg b/src/icons/Scholica.svg index b35216915..d136a5b2c 100644 --- a/src/icons/Scholica.svg +++ b/src/icons/Scholica.svg @@ -1,24 +1 @@ - - - - logo-green-icon - Created with Sketch. - - - - - - - - - \ No newline at end of file +logo-green-icon \ No newline at end of file diff --git a/src/icons/Shapecss.svg b/src/icons/Shapecss.svg index 8d577bec6..71026cd91 100644 --- a/src/icons/Shapecss.svg +++ b/src/icons/Shapecss.svg @@ -1,27 +1 @@ - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/src/icons/ShellInABox.png b/src/icons/ShellInABox.png index 036151609..3bb29f8c6 100644 Binary files a/src/icons/ShellInABox.png and b/src/icons/ShellInABox.png differ diff --git a/src/icons/SpinCMS.png b/src/icons/SpinCMS.png index f6e947831..893beccae 100644 Binary files a/src/icons/SpinCMS.png and b/src/icons/SpinCMS.png differ diff --git a/src/icons/SumoMe.png b/src/icons/SumoMe.png index b70b8f935..10f30c3bd 100644 Binary files a/src/icons/SumoMe.png and b/src/icons/SumoMe.png differ diff --git a/src/icons/SweetAlert2.png b/src/icons/SweetAlert2.png index 1d7edfcf1..bba68625d 100644 Binary files a/src/icons/SweetAlert2.png and b/src/icons/SweetAlert2.png differ diff --git a/src/icons/Vinala.png b/src/icons/Vinala.png index 8aa46505e..b453eb188 100644 Binary files a/src/icons/Vinala.png and b/src/icons/Vinala.png differ diff --git a/src/icons/YouTrack.png b/src/icons/YouTrack.png index 4a00a51d4..c9aee2f8e 100644 Binary files a/src/icons/YouTrack.png and b/src/icons/YouTrack.png differ diff --git a/src/icons/adminer.png b/src/icons/adminer.png index 231efe4d6..e0911caa5 100644 Binary files a/src/icons/adminer.png and b/src/icons/adminer.png differ diff --git a/src/icons/apostrophecms.svg b/src/icons/apostrophecms.svg index d71a5b5db..ca5fc0d41 100644 --- a/src/icons/apostrophecms.svg +++ b/src/icons/apostrophecms.svg @@ -1,12 +1 @@ - - - - 256 - Created with Sketch. - - - - - - - \ No newline at end of file +256 \ No newline at end of file diff --git a/src/icons/aws-ec2.svg b/src/icons/aws-ec2.svg index f6da9f51d..bb4f86907 100755 --- a/src/icons/aws-ec2.svg +++ b/src/icons/aws-ec2.svg @@ -1,13 +1 @@ - - - - - - - - - - - - - \ No newline at end of file + \ No newline at end of file diff --git a/src/icons/aws-s3.svg b/src/icons/aws-s3.svg index 70a784357..3c8c386a2 100755 --- a/src/icons/aws-s3.svg +++ b/src/icons/aws-s3.svg @@ -1,17 +1 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file + \ No newline at end of file diff --git a/src/icons/cloudcart.svg b/src/icons/cloudcart.svg index 5d5c41cf9..e7029a935 100644 --- a/src/icons/cloudcart.svg +++ b/src/icons/cloudcart.svg @@ -1,24 +1 @@ - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/src/icons/default.svg b/src/icons/default.svg index af0f6be0c..fbbd408a7 100644 --- a/src/icons/default.svg +++ b/src/icons/default.svg @@ -1,19 +1 @@ - - - - Favicon - Created with Sketch. - - - - - - - - - - - - - - \ No newline at end of file +Favicon \ No newline at end of file diff --git a/src/icons/eCommercePy.png b/src/icons/eCommercePy.png index 16fb0d0b9..ae9d304fb 100644 Binary files a/src/icons/eCommercePy.png and b/src/icons/eCommercePy.png differ diff --git a/src/icons/ebis.png b/src/icons/ebis.png index 1c572e816..463d82d95 100644 Binary files a/src/icons/ebis.png and b/src/icons/ebis.png differ diff --git a/src/icons/ec-cube.png b/src/icons/ec-cube.png index beb4e7123..1097be68e 100644 Binary files a/src/icons/ec-cube.png and b/src/icons/ec-cube.png differ diff --git a/src/icons/etherpad.png b/src/icons/etherpad.png index 8cf4ddebf..58211c95f 100644 Binary files a/src/icons/etherpad.png and b/src/icons/etherpad.png differ diff --git a/src/icons/gerrit.svg b/src/icons/gerrit.svg index 160dd52e3..1b8d4e217 100644 --- a/src/icons/gerrit.svg +++ b/src/icons/gerrit.svg @@ -1,8 +1 @@ - - - - - - - - + \ No newline at end of file diff --git a/src/icons/gogs.png b/src/icons/gogs.png index e64bfcc2e..dcc329ca7 100644 Binary files a/src/icons/gogs.png and b/src/icons/gogs.png differ diff --git a/src/icons/gravityforms.svg b/src/icons/gravityforms.svg index b2c4ee7c6..43130e50a 100644 --- a/src/icons/gravityforms.svg +++ b/src/icons/gravityforms.svg @@ -1 +1 @@ -gravityforms +gravityforms \ No newline at end of file diff --git a/src/icons/iPresta.png b/src/icons/iPresta.png index f493a4650..7ab9809ed 100644 Binary files a/src/icons/iPresta.png and b/src/icons/iPresta.png differ diff --git a/src/icons/infusionsoft.svg b/src/icons/infusionsoft.svg index 50f50d016..c2b7a1253 100644 --- a/src/icons/infusionsoft.svg +++ b/src/icons/infusionsoft.svg @@ -1 +1 @@ -infusionsoft \ No newline at end of file +infusionsoft \ No newline at end of file diff --git a/src/icons/kibana.svg b/src/icons/kibana.svg index 419dcbd87..595a80598 100644 --- a/src/icons/kibana.svg +++ b/src/icons/kibana.svg @@ -1,19 +1 @@ - - - - - - - - - - - - + \ No newline at end of file diff --git a/src/icons/mailchimp.svg b/src/icons/mailchimp.svg index 2456a7208..5831b880a 100644 --- a/src/icons/mailchimp.svg +++ b/src/icons/mailchimp.svg @@ -1 +1 @@ -mailchimp +mailchimp \ No newline at end of file diff --git a/src/icons/mattermost.png b/src/icons/mattermost.png index 7375f3e61..cca83f7a0 100644 Binary files a/src/icons/mattermost.png and b/src/icons/mattermost.png differ diff --git a/src/icons/meliscmsv2.png b/src/icons/meliscmsv2.png index 39ec358a6..c545b2882 100644 Binary files a/src/icons/meliscmsv2.png and b/src/icons/meliscmsv2.png differ diff --git a/src/icons/moon.png b/src/icons/moon.png index 2fdae7cc3..eb8ff3afd 100644 Binary files a/src/icons/moon.png and b/src/icons/moon.png differ diff --git a/src/icons/octopress.png b/src/icons/octopress.png index 0f2506730..d65b4ba96 100644 Binary files a/src/icons/octopress.png and b/src/icons/octopress.png differ diff --git a/src/icons/pygments.png b/src/icons/pygments.png index 222c8951c..3f3be26e6 100644 Binary files a/src/icons/pygments.png and b/src/icons/pygments.png differ diff --git a/src/icons/storyblok.png b/src/icons/storyblok.png index daf5882d1..404358d94 100644 Binary files a/src/icons/storyblok.png and b/src/icons/storyblok.png differ diff --git a/src/icons/translucide.svg b/src/icons/translucide.svg index 1cadcf7ac..ed882e815 100644 --- a/src/icons/translucide.svg +++ b/src/icons/translucide.svg @@ -1,36 +1 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/src/icons/typecho.svg b/src/icons/typecho.svg index b447bb51b..e43dcb1f9 100644 --- a/src/icons/typecho.svg +++ b/src/icons/typecho.svg @@ -1,10 +1 @@ - - - typecho-logo - Created with Sketch (http://www.bohemiancoding.com/sketch) - - - - - - \ No newline at end of file +typecho-logo \ No newline at end of file diff --git a/src/icons/wp_super_cache.png b/src/icons/wp_super_cache.png index 87e8d17ed..819cab29f 100644 Binary files a/src/icons/wp_super_cache.png and b/src/icons/wp_super_cache.png differ diff --git a/src/utils/iframe.js b/src/utils/iframe.js deleted file mode 100644 index d37f91ef1..000000000 --- a/src/utils/iframe.js +++ /dev/null @@ -1,1166 +0,0 @@ -'use strict'; - -(function(win) { - -var exports = {}; - -(function(exports) { - - var utils = { - normalizeUrl: function(url) { - - return this.hashUrl(url) || null; - - }, - - getReferrer: function() { - return this.normalizeUrl(document.referrer); - }, - - getPageUrl: function() { - return this.normalizeUrl(window.location.href); - }, - hashUrl: function(url) { - var a, - result; - - if ( !url || url.indexOf('http') !== 0 ) { - return null; - } - - a = document.createElement('a'); - a.href = url; - - result = a.protocol + '//' + a.hostname + '/'; - - if ( a.pathname && a.pathname !== '/' ) { - result += this.hashCode(a.pathname); - } - - if ( a.search ) { - result += '?' + this.hashCode(a.search); - } - - if ( a.hash ) { - result += '#' + this.hashCode(a.hash); - } - - return result; - }, - - hashCode: function(str) { - var hash = 0, - kar, - i; - - if ( str.length === 0 ) { - return hash; - } - - for ( i = 0; i < str.length; i++ ) { - kar = str.charCodeAt(i); - hash = ((hash << 5) - hash) + kar; - hash = hash & hash; - } - - return hash + Math.pow(2, 32); - }, - - realArray: function(a) { - return Array.prototype.slice.apply(a); - }, - - onDocLoaded: function(doc, callback) { - if ( doc.readyState === 'loading' ) { - doc.addEventListener('DOMContentLoaded', callback); - } else { - callback(); - } - }, - - SCRIPT_IN_WINDOW_TOP: window === window.top, - - isFriendlyWindow: function(win) { - - var href; - try { - href = win.location.href; - } catch(e) { - return false; - } - return true; - }, - - elementWindow: function(el) { - return el.ownerDocument.defaultView; - }, - - viewport: function(win) { - return {width: win.innerWidth, height: win.innerHeight}; - }, - - parseQS: function(qs) { - if ( qs.indexOf('http') === 0 ) { - qs = qs.split('?')[1]; - } - var i, kvs, key, val; - var dict = {}; - qs = qs.split('&'); - for ( i = 0; i < qs.length; i++ ) { - kvs = qs[i].split('='); - key = kvs[0]; - val = kvs.slice(1).join('='); - try { - dict[key] = window.decodeURIComponent(val); - } catch (e) { - - continue; - } - } - return dict; - }, - sendToBackground: function(message, event, responseMessage, onResponse) { - if ( typeof browser !== 'undefined' ) { - var response = browser.runtime.sendMessage(message); - response.then(onResponse); - } else if ( typeof chrome !== 'undefined' ) { - chrome.runtime.sendMessage(message, onResponse); - } else if ( window.self.port ) { - window.self.port.on(responseMessage, onResponse); - window.self.port.emit(event, message); - } - }, - - ifTrackingEnabled: function(callback, elseCallback) { - - this.sendToBackground( - 'is_tracking_enabled', - '', - 'tracking_enabled_response', - function(message) { - if ( message.tracking_enabled ) { - - callback(); - } else { - - elseCallback(); - } - } - ); - - } - }; - - utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent); - utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME; - - function LogGenerator() { - this.msgNum = 0; - this.pageMeta = { - 'url': utils.getPageUrl(), - 'isHP': window.location.pathname === '/', - 'referrer': utils.getReferrer(), - 'rand': Math.floor(Math.random() * 10e12), - 'startTime': new Date().getTime() - }; - } - - LogGenerator.prototype = { - log: function(event, opt_assets, opt_pageTags) { - var opt_video_assets; - if ( event === 'video' || event === 'invalid-video' ) { - opt_video_assets = opt_assets || []; - opt_assets = []; - } else { - opt_video_assets = []; - opt_assets = opt_assets || []; - } - var result = { - doc: this.pageMeta, - event: event, - video_assets: opt_video_assets, - assets: opt_assets, - version: '3', - mrev: '9c4d5b3-c', - msgNum: this.msgNum, - timestamp: new Date().getTime(), - pageVis: document.visibilityState, - pageFoc: document.hasFocus(), - pageTags: opt_pageTags || [] - }; - this.msgNum++; - return result; - } - }; - - utils.LogGenerator = LogGenerator; - - exports.utils = utils; -})(exports); - -(function(exports) { - - var SizeMatcher = { - VALID_AD_SIZES: [ - [300, 50], - [320, 50], - [160, 600], - [300, 250], - [300, 600], - [300, 1050], - [336, 280], - [336, 850], - [468, 60], - [728, 90], - [728, 250], - [728, 270], - [970, 66], - [970, 90], - [970, 125], - [970, 250], - [970, 400], - [970, 415], - [1280, 100] - ], - - PX_SIZE_TOL: 10, - - getMatchedAdSize: function(width, height) { - - if ( !this.set ) { - this.set = this._makeSizeSet(); - } - - return this.set[Math.round(width) + 'x' + Math.round(height)]; - }, - - elementIsAdShaped: function(el) { - return !!this.getMatchedAdSizeForElement(el); - }, - - getMatchedAdSizeForElement: function(el) { - var rect = el.getBoundingClientRect(); - return this.getMatchedAdSize(rect.width, rect.height); - }, - - _makeSizeSet: function() { - var set = {}; - var i; - var xfuz; - var yfuz; - var size; - var width; - var height; - - for ( i = 0; i < this.VALID_AD_SIZES.length; i++ ) { - for ( xfuz = -this.PX_SIZE_TOL; xfuz <= this.PX_SIZE_TOL; xfuz++ ) { - for ( yfuz = -this.PX_SIZE_TOL; yfuz <= this.PX_SIZE_TOL; yfuz++ ) { - size = this.VALID_AD_SIZES[i]; - width = size[0] + xfuz; - height = size[1] + yfuz; - set[width + 'x' + height] = size; - } - } - } - return set; - } - }; - - var Throttler = { - MAX_SEARCHES_PER_WINDOW: 10, - MAX_SEARCHES_PER_ELEMENT: 2, - - countSearch: function(el) { - if ( typeof el.searches !== 'number' ) { - el.searches = 0; - } - - el.searches += 1; - }, - - throttle: function(el, max) { - if ( typeof el.searches === 'number' && el.searches >= max ) { - return true; - } - return false; - }, - - throttleElement: function(el) { - return this.throttle(el, this.MAX_SEARCHES_PER_ELEMENT); - }, - - throttleWin: function(win) { - return this.throttle(win, this.MAX_SEARCHES_PER_WINDOW); - }, - - getCount: function(el) { - return el.searches || 0; - } - }; - - function TopSearcher(win) { - this.win = win; - this.doc = win.document; - } - - TopSearcher.prototype.search = function() { - var candidates = exports.utils.realArray(this.doc.querySelectorAll('img, object, embed')), - html5Ad, - ads = []; - - ads = ads.concat(candidates.filter(function(el) { - if ( !el.mpAdFound && !Throttler.throttleElement(el) ) { - Throttler.countSearch(el); - if ( (el.tagName !== 'IMG' || isStandardImage(el)) && SizeMatcher.elementIsAdShaped(el) ) { - el.mpAdFound = true; - return true; - } - } - return false; - })); - - html5Ad = this._mainGetHTMLAd(); - if ( html5Ad ) { - html5Ad.html5 = true; - html5Ad.mpAdFound = true; - ads.push(html5Ad); - } - - return ads; - }; - - TopSearcher.prototype._mainGetHTMLAd = function() { - var styles = this.doc.querySelectorAll('div > style, div > link[rel="stylesheet"]'), - i, div; - for ( i = 0; i < styles.length; i++ ) { - div = styles[i].parentNode; - if ( !div.mpAdFound && SizeMatcher.elementIsAdShaped(div) && this._jumpedOut(div) ) { - return div; - } - } - }; - - TopSearcher.prototype._jumpedOut = function(el) { - var siblings, ifrs; - siblings = exports.utils.realArray(el.parentNode.children); - ifrs = siblings.filter(function(el) { - return el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0; - }); - return ifrs.length > 0; - }; - - function IframeSearcher(win) { - this.MIN_AD_AREA = 14000; - this.MIN_WINDOW_PX = 10; - - this.win = win; - this.doc = win.document; - this.body = win.document.body; - this.winClickTag = win.clickTag; - this.adSizeMeta = this._getAdSizeMeta(); - this.numElementsInBody = (this.body && this.body.querySelectorAll('*').length) || 0; - - this.shouldSearchWindow = false; - if ( !this.win.mpAdFound && this.body && !Throttler.throttleWin(this.win) ) { - this.winWidth = this.win.innerWidth; - this.winHeight = this.win.innerHeight; - if ( this._meetsMinAdSize(this.winWidth, this.winHeight) && !this._containsLargeIframes() ) { - this.shouldSearchWindow = true; - } - } - - } - - IframeSearcher.prototype.search = function() { - var ad; - - if ( this.shouldSearchWindow ) { - ad = this._search(); - if ( ad ) { - ad.mpAdFound = true; - win.mpAdFound = true; - return ad; - } - Throttler.countSearch(this.win); - } - - return null; - }; - - IframeSearcher.prototype._search = function() { - var _this = this, - stdCandidates, - html5Candidates, - stdEl, - html5El; - - stdCandidates = this.body.querySelectorAll('img, object, embed'); - - stdEl = getFirst(stdCandidates, function(el) { - if ( !el.mpAdFound && - !Throttler.throttleElement(el) && - (el.tagName !== 'IMG' || isStandardImage(el)) && - _this._elementIsAtLeastAsBigAsWindow(el)) - { - return true; - } - Throttler.countSearch(el); - return false; - }); - - if ( stdEl ) { - return stdEl; - } - - if ( this._isHTML5Iframe() ) { - html5Candidates = this.doc.querySelectorAll('body, canvas, button, video, svg, div'); - html5El = getFirst(html5Candidates, function(el) { - - if ( _this._elementIsAtLeastAsBigAsWindow(el) ) { - return true; - } - Throttler.countSearch(el); - return false; - }); - } - - if ( html5El ) { - html5El.html5 = true; - html5El.winClickTag = this.winClickTag; - html5El.adSizeMeta = this.adSizeMeta; - return html5El; - } - - return null; - }; - - IframeSearcher.prototype._isHTML5Iframe = function() { - if ( this.winClickTag || this.adSizeMeta ) { - return true; - } - - if ( this.doc.querySelectorAll('canvas', 'button', 'video', 'svg').length > 0 ) { - return true; - } - - if ( this.numElementsInBody >= 5 && Throttler.getCount(this.win) > 0 && this.doc.querySelectorAll('div').length > 0 ) { - return true; - } - - return false; - }; - - IframeSearcher.prototype._elementIsAtLeastAsBigAsWindow = function(el) { - var rect = el.getBoundingClientRect(), - tol = 0.95; - - return rect.width >= (tol * this.winWidth) && rect.height >= (tol * this.winHeight); - }; - - IframeSearcher.prototype._meetsMinAdSize = function(width, height) { - return (width * height) >= this.MIN_AD_AREA; - }; - - IframeSearcher.prototype._containsLargeIframes = function() { - var iframes = this.doc.querySelectorAll('iframe'); - var rect; - var i; - for ( i = 0; i < iframes.length; i++ ) { - rect = iframes[i].getBoundingClientRect(); - if ( rect.width > this.MIN_WINDOW_PX || rect.height > this.MIN_WINDOW_PX ) { - return true; - } - } - return false; - }; - - IframeSearcher.prototype._getAdSizeMeta = function() { - var adSizeMeta = this.doc.querySelectorAll('meta[name="ad.size"]'); - if ( adSizeMeta.length > 0 ) { - return adSizeMeta[0].content; - } else { - return null; - } - }; - - function getFirst(arr, testFn) { - var i, el; - for ( i = 0; i < arr.length; i++ ) { - el = arr[i]; - if ( testFn(el) ) { - return el; - } - } - return null; - } - - function isStandardImage(img) { - - return img.src && (img.parentNode.tagName === 'A' || img.getAttribute('onclick')); - } - - function getFriendlyIframes(win) { - var iframes = win.document.querySelectorAll('iframe'); - iframes = exports.utils.realArray(iframes); - var friendlyIframes = iframes.filter(function(ifr) { - return exports.utils.isFriendlyWindow(ifr.contentWindow); - }); - return friendlyIframes; - } - - function findAds(win) { - var i, - iframes, - searcher, - ad, - ads = []; - - if ( win === win.top ) { - searcher = new TopSearcher(win); - ads = ads.concat(searcher.search()); - } else { - searcher = new IframeSearcher(win); - ad = searcher.search(); - if ( ad ) { - ads.push(ad); - } - } - - iframes = getFriendlyIframes(win); - for ( i = 0; i < iframes.length; i++ ) { - ads = ads.concat(findAds(iframes[i].contentWindow)); - } - - return ads; - } - - exports.adfinder = { - getMatchedAdSize: SizeMatcher.getMatchedAdSize.bind(SizeMatcher), - findAds: findAds - }; -})(exports); - -(function(exports) { - - var parser = { - TAGS_WITH_SRC_ATTR: { - 'IMG': true, - 'SCRIPT': true, - 'IFRAME': true, - 'EMBED': true - }, - - MAX_ATTR_LEN: 100, - - getUrl: function(el, params) { - var url; - - if ( this.TAGS_WITH_SRC_ATTR.hasOwnProperty(el.tagName) ) { - url = el.src; - - } else if ( el.tagName === 'OBJECT' ) { - url = el.data || (params && params.movie) || null; - - } else if ( el.tagName === 'A' ) { - url = el.href; - } - - if ( url && url.indexOf('http') === 0 ) { - return url; - } else { - return null; - } - }, - - getParams: function(el) { - if ( el.tagName !== 'OBJECT' ) { - return null; - } - - var i, child; - var params = {}; - var children = el.children; - for ( i = 0; i < children.length; i++ ) { - child = children[i]; - if ( child.tagName === 'PARAM' && child.name ) { - - params[child.name.toLowerCase()] = child.value; - } - } - return params; - }, - - getPosition: function(el) { - var rect = el.getBoundingClientRect(); - var win = exports.utils.elementWindow(el); - - return { - width: Math.round(rect.width), - height: Math.round(rect.height), - left: Math.round(rect.left + win.pageXOffset), - top: Math.round(rect.top + win.pageYOffset) - }; - }, - - getFlashvars: function(el, params, url) { - var flashvars; - var urlQS = url && url.split('?')[1]; - - if ( el.tagName === 'EMBED' ) { - flashvars = el.getAttribute('flashvars') || urlQS; - - } else if ( el.tagName === 'OBJECT' ) { - flashvars = params.flashvars || el.getAttribute('flashvars') || urlQS; - } - - return (flashvars && exports.utils.parseQS(flashvars)) || null; - }, - - findClickThru: function(el, flashvars) { - var key; - if ( el.tagName === 'IMG' && el.parentElement.tagName === 'A' ) { - return el.parentElement.href; - } else if ( flashvars ) { - for ( key in flashvars ) { - if ( flashvars.hasOwnProperty(key) ) { - - if ( key.toLowerCase().indexOf('clicktag') === 0 ) { - return flashvars[key]; - } - } - } - } - return null; - }, - - getAttr: function(el, name) { - var val = el.getAttribute(name); - - if ( val && val.slice && val.toString ) { - - return val.slice(0, this.MAX_ATTR_LEN).toString(); - } else { - return null; - } - }, - - putPropIfExists: function(obj, name, val) { - if ( val ) { - obj[name] = val; - } - }, - - putAttrIfExists: function(obj, el, name) { - var val = this.getAttr(el, name); - this.putPropIfExists(obj, name, val); - }, - - elementToJSON: function(el, opt_findClickThru) { - var pos = this.getPosition(el); - var params = this.getParams(el); - var url = this.getUrl(el, params); - var flashvars = this.getFlashvars(el, params, url); - var clickThru = opt_findClickThru && this.findClickThru(el, flashvars); - var json = { - tagName: el.tagName, - width: pos.width, - height: pos.height, - left: pos.left, - top: pos.top, - children: [] - }; - - if ( params ) { - - delete params.flashvars; - } - - this.putAttrIfExists(json, el, 'id'); - this.putAttrIfExists(json, el, 'class'); - this.putAttrIfExists(json, el, 'name'); - - this.putPropIfExists(json, 'flashvars', flashvars); - this.putPropIfExists(json, 'url', url); - this.putPropIfExists(json, 'params', params); - this.putPropIfExists(json, 'clickThru', clickThru); - - return json; - } - }; - - exports.parser = { elementToJSON: parser.elementToJSON.bind(parser) }; -})(exports); - -(function(exports) { - - var ContextManager = function(adData) { - this.adData = adData; - }; - - ContextManager.prototype = { - CONTAINER_SIZE_TOL: 0.4, - ASPECT_RATIO_FOR_LEADERBOARDS: 2, - - isValidContainer: function(el, opt_curWin) { - - var cWidth = el.clientWidth; - var cHeight = el.clientHeight; - - var adWidth = this.adData.width; - var adHeight = this.adData.height; - - var winWidth = opt_curWin && opt_curWin.innerWidth; - var winHeight = opt_curWin && opt_curWin.innerHeight; - var similarWin = opt_curWin && this.withinTol(adWidth, winWidth) && this.withinTol(adHeight, winHeight); - - var similarSizeX = this.withinTol(adWidth, cWidth); - var similarSizeY = this.withinTol(adHeight, cHeight); - var adAspect = adWidth / adHeight; - - return similarWin || el.tagName === 'A' || (adAspect >= this.ASPECT_RATIO_FOR_LEADERBOARDS && similarSizeY) || (similarSizeX && similarSizeY); - }, - - withinTol: function(adlen, conlen) { - var pct = (conlen - adlen) / adlen; - - return pct <= this.CONTAINER_SIZE_TOL; - }, - - serializeElements: function(el) { - if ( !el ) { - return; - } - var i; - var ifrWin; - var adId = this.adData.adId; - var elIsAd = false; - - if ( adId && el[adId] && el[adId].isAd === true ) { - elIsAd = true; - } - - var json = exports.parser.elementToJSON(el, elIsAd); - var childJSON; - - if ( elIsAd ) { - json.adId = adId; - this.adData.element = {}; - - var keys = Object.keys(json); - for ( i = 0; i < keys.length; i++ ) { - var key = keys[i]; - if ( key !== 'children' && key !== 'contents' ) { - this.adData.element[key] = json[key]; - } - } - } - - var children = exports.utils.realArray(el.children).filter(function(el) { - var param = el.tagName === 'PARAM'; - var inlineScript = el.tagName === 'SCRIPT' && !(el.src && el.src.indexOf('http') >= 0); - var noScript = el.tagName === 'NOSCRIPT'; - return !(param || inlineScript || noScript); - }); - - for ( i = 0; i < children.length; i++ ) { - childJSON = this.serializeElements(children[i]); - if ( childJSON ) { - json.children.push(childJSON); - } - } - - if ( el.tagName === 'IFRAME' ) { - ifrWin = el.contentWindow; - - if ( adId && el[adId] && el[adId].needsWindow ) { - - json.contents = this.adData.serializedIframeContents; - el[adId].needsWindow = false; - delete this.adData.serializedIframeContents; - - } else if ( exports.utils.isFriendlyWindow(ifrWin) ) { - - childJSON = this.serializeElements(ifrWin.document.documentElement); - if ( childJSON ) { - json.contents = childJSON; - } - } - } - - if ( json.children.length > 0 || json.adId || json.tagName === 'IFRAME' || json.url ) { - return json; - } else { - return null; - } - }, - - captureHTML: function(containerEl) { - this.adData.context = this.serializeElements(containerEl); - }, - - nodeCount: function(el) { - return el.getElementsByTagName('*').length + 1; - }, - - highestContainer: function(curWin, referenceElement) { - var curContainer = referenceElement; - var docEl = curWin.document.documentElement; - var parentContainer; - - if ( curWin !== curWin.top && this.isValidContainer(docEl, curWin) ) { - return docEl; - } - - while ( true ) { - parentContainer = curContainer.parentElement; - if ( parentContainer && this.isValidContainer(parentContainer) ) { - curContainer = parentContainer; - } else { - return curContainer; - } - } - } - }; - - var tagfinder = { - - setPositions: function(adData, opt_el, opt_winPos) { - var el = opt_el || adData.context; - var winPos = opt_winPos || {left: 0, top: 0}; - var ifrPos; - - el.left += winPos.left; - el.top += winPos.top; - - if ( el.children ) { - el.children.forEach(function(child) { - this.setPositions(adData, child, winPos); - }, this); - } - - if ( el.contents ) { - ifrPos = {left: el.left, top: el.top}; - this.setPositions(adData, el.contents, ifrPos); - } - - if ( el.adId === adData.adId ) { - adData.element.left = el.left; - adData.element.top = el.top; - } - }, - - appendTags: function(adData, referenceElement) { - var mgr = new ContextManager(adData); - var curWin = exports.utils.elementWindow(referenceElement); - var highestContainer; - - while ( true ) { - highestContainer = mgr.highestContainer(curWin, referenceElement); - mgr.captureHTML(highestContainer); - if ( curWin === curWin.top ) { - break; - } else { - - curWin.mpAdFound = true; - - mgr.adData.serializedIframeContents = mgr.adData.context; - - if ( exports.utils.isFriendlyWindow(curWin.parent) ) { - referenceElement = curWin.frameElement; - referenceElement[mgr.adData.adId] = {needsWindow: true}; - curWin = curWin.parent; - } else { - break; - } - } - } - return { - referenceElement:referenceElement, - highestContainer: highestContainer - }; - } - }; - - exports.tagfinder = tagfinder; -})(exports); - -(function(exports) { - var _onAdFound; - var _logGen = new exports.utils.LogGenerator(); - var _pageTags; - var INIT_MS_BW_SEARCHES = 2000; - var PAGE_TAG_RE = new RegExp('gpt|oascentral'); - var POST_MSG_ID = '1490888598-28717-31700-14775-21098'; - var AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)'); - - function getPageTags(doc) { - var scripts = doc.getElementsByTagName('script'); - var pageTags = []; - scripts = exports.utils.realArray(scripts); - scripts.forEach(function(script) { - if ( PAGE_TAG_RE.exec(script.src) ) { - pageTags.push({'tagName': 'SCRIPT', 'url': script.src}); - } - }); - return pageTags; - } - - function messageAllParentFrames(adData) { - - adData.postMessageId = POST_MSG_ID; - - adData = JSON.stringify(adData); - - var win = window; - while ( win !== win.top ) { - win = win.parent; - win.postMessage(adData, '*'); - } - } - - function appendTagsAndSendToParent(adData, referenceElement) { - var results = exports.tagfinder.appendTags(adData, referenceElement); - if ( exports.utils.SCRIPT_IN_HOSTILE_IFRAME ) { - messageAllParentFrames(adData); - - } else if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { - - exports.tagfinder.setPositions(adData); - - adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height); - if ( !adData.matchedSize ) { - - if ( AD_SERVER_RE.exec(results.referenceElement.id) ) { - adData.matchedSize = [adData.width, adData.height]; - adData.oddSize = true; - } else { - - return; - } - } - delete adData.width; - delete adData.height; - adData.curPageUrl = exports.utils.getPageUrl(); - _pageTags = _pageTags || getPageTags(document); - var log = _logGen.log('ad', [adData], _pageTags); - - if ( _onAdFound ) { - - _onAdFound(log, results.referenceElement); - - } - } - } - - function extractAdsWrapper() { - if ( exports.utils.SCRIPT_IN_WINDOW_TOP || document.readyState === 'complete' ) { - extractAds(); - } - setTimeout( - function() { extractAdsWrapper(); }, INIT_MS_BW_SEARCHES - ); - } - - function extractAds() { - var ads = exports.adfinder.findAds(window); - ads.forEach(function(ad) { - - var startTime = new Date().getTime(); - var adId = startTime + '-' + Math.floor(Math.random() * 10e12); - - var adData = { - width: Math.round(ad.offsetWidth), - height: Math.round(ad.offsetHeight), - startTime: startTime, - adId: adId, - html5: ad.html5 || false - }; - - if ( ad.html5 ) { - adData.adSizeMeta = ad.adSizeMeta || null; - adData.winClickTag = ad.winClickTag || null; - } - - ad[adId] = { isAd: true }; - - appendTagsAndSendToParent(adData, ad); - }); - } - - function isChildWin(myWin, otherWin) { - var parentWin = otherWin.parent; - while ( parentWin !== otherWin ) { - if ( parentWin === myWin ) { - return true; - } - otherWin = parentWin; - parentWin = parentWin.parent; - } - return false; - } - - function iframeFromWindow(win, winToMatch) { - var i, ifr, ifrWin, - iframes = win.document.querySelectorAll('iframe'); - - for ( i = 0; i < iframes.length; i++ ) { - ifr = iframes[i]; - if ( ifr.contentWindow === winToMatch ) { - return ifr; - } - } - - for ( i = 0; i < iframes.length; i++ ) { - ifrWin = iframes[i].contentWindow; - if ( exports.utils.isFriendlyWindow(ifrWin) ) { - ifr = iframeFromWindow(ifrWin, winToMatch); - if ( ifr ) { - return ifr; - } - } - } - } - - function onPostMessage(event) { - var adData, - ifrWin = event.source, - - myWin = window.document.defaultView, - ifrTag; - - try { - - adData = JSON.parse(event.data); - } catch(e) { - - return; - } - - if ( adData.postMessageId === POST_MSG_ID ) { - - delete adData.postMessageId; - - event.stopImmediatePropagation(); - - if ( isChildWin(myWin, ifrWin) ) { - if ( exports.utils.isFriendlyWindow(ifrWin) ) { - ifrTag = ifrWin.frameElement; - } else { - ifrTag = iframeFromWindow(myWin, ifrWin); - } - - if ( ifrTag ) { - ifrTag[adData.adId] = {needsWindow: true}; - appendTagsAndSendToParent(adData, ifrTag); - } - } - } - } - - function onVideoMessage(msg, sender, callback) { - var log; - if ( msg.event === 'new-video-ad' ) { - msg.assets.forEach(function(asset) { - - }); - log = _logGen.log('video', msg.assets); - } else { - log = _logGen.log('invalid-video', msg.assets); - } - - msg.assets.forEach(function(a) {delete a.isVideo;}); - log.displayAdFound = msg.displayAdFound; - log.requests = msg.requests; - log.data = msg.event_data; - - log.doc.finalPageUrl = log.doc.url; - log.doc.url = exports.utils.normalizeUrl(msg.origUrl); - - _onAdFound(log); - } - - function addBackgroundListener(event, callback) { - if ( typeof browser !== 'undefined' ) { - browser.runtime.onMessage.addListener(function(msg) { - if ( msg.event === event ) { - callback(msg); - } - }); - } else if ( typeof chrome !== 'undefined' ) { - chrome.runtime.onMessage.addListener(function(msg) { - if ( msg.event === event ) { - callback(msg); - } - }); - } else if ( window.self.port ) { - window.self.port.on(event, callback); - } - } - - exports.coordinator = { - addPostMessageListener: function() { - if ( !exports.utils.SCRIPT_IN_FRIENDLY_IFRAME ) { - window.addEventListener('message', onPostMessage, false); - } - }, - - init: function(onAdFound) { - - if ( exports.utils.SCRIPT_IN_FRIENDLY_IFRAME ) { - return false; - } - - _onAdFound = onAdFound; - if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { - var log = _logGen.log('page'); - onAdFound(log); - - window.addEventListener('beforeunload', function(event) { - var log = _logGen.log('unload'); - log.timing = window.performance.timing; - onAdFound(log); - }); - - addBackgroundListener('new-video-ad', onVideoMessage); - addBackgroundListener('new-invalid-video-ad', onVideoMessage); - - } - - exports.utils.onDocLoaded(document, extractAdsWrapper); - } - }; - -})(exports); - -if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { - window.adparser = { - init: exports.coordinator.init, - addPostMessageListener: exports.coordinator.addPostMessageListener, - ifTrackingEnabled: exports.utils.ifTrackingEnabled, - sendToBackground: exports.utils.sendToBackground - }; -} else { - exports.coordinator.addPostMessageListener(); - exports.utils.ifTrackingEnabled( - function() { - exports.coordinator.init(function() {}); - }, - function() {} - ); -} -})(window); -(function(adparser) { - function onAdFound(log) { - adparser.sendToBackground({ id: 'ad_log', subject: log }, 'ad_log', '', function(){}); - } - - if ( window === window.top ) { - adparser.addPostMessageListener(); - adparser.ifTrackingEnabled( - function() { - adparser.init(onAdFound); - }, - function() {} - ) - } -})(window.adparser); diff --git a/src/utils/network.js b/src/utils/network.js deleted file mode 100644 index b2f2ea018..000000000 --- a/src/utils/network.js +++ /dev/null @@ -1,784 +0,0 @@ -'use strict'; -(function() { - - function isChrome() { - return (typeof chrome !== 'undefined' && - window.navigator.userAgent.match(/Chrom(?:e|ium)\/([0-9\.]+)/)); - } - - var browserProxy; - if ( isChrome() ) { - browserProxy = chrome; - } else { - browserProxy = browser; - } - - var MIN_FF_MAJOR_VERSION = 51; - - var requiredBrowserApis = [ - browserProxy.webNavigation, - browserProxy.tabs, - browserProxy.webRequest, - browserProxy.runtime - ]; - var areListenersRegistered = false; - var secBefore = 2000; - var secAfter = 5000; - var secBetweenDupAssets = 10e3; - var minVidSize = 500e3; - var maxVidSize = 25e6; - var maxContentRange = 25e6; - var videoExtensions = [ - 'af', '3gp', 'asf', 'avchd', 'avi', 'cam', 'dsh', 'flv', 'm1v', 'm2v', - 'fla', 'flr', 'sol', 'm4v', 'mkv', 'wrap', 'mng', 'mov', 'mpeg', 'mpg', - 'mpe', 'mp4', 'mxf', 'nsv', 'ogg', 'rm', 'svi', 'smi', 'wmv', 'webm' - ]; - var extensionsReg = new RegExp('\\.' + videoExtensions.join('$|\\.') + '$'); - var videoContentTypesPrefixes = ['binary/octet-stream', 'video/', 'flv-application/', 'media']; - - var bannedContentTypes = ['video/mp2t','video/f4m','video/f4f']; - var bannedFiletypes = ['ts']; - var bannedFiletypesReg = new RegExp('\\.' + bannedFiletypes.join('$|\\.') + '$'); - var whitelistReqTypes = ['object', 'xmlhttprequest', 'other']; - - var topVideoAssetDomains = [ - '2mdn.net', - 'adap.tv', - 'adnxs.com', - 'adsrvr.org', - 'btrll.com', - 'celtra.com', - 'flashtalking.com', - 'flite.com', - 'innovid.com', - 'jivox.com', - 'mixpo.com', - 'nytimes.com', - 'playwire.com', - 'selectmedia.asia', - 'serving-sys.com', - 'solvemedia.com', - 'spotible.com', - 'teads.tv', - 'tribalfusion.com', - 'tubemogul.com', - 'videologygroup.com', - 'washingtonpost.com' - ]; - - if ( !String.prototype.endsWith ) { - String.prototype.endsWith = function(searchString, position) { - var subjectString = this.toString(); - if ( typeof position !== 'number' || !isFinite(position) || - Math.floor(position) !== position || position > subjectString.length) { - position = subjectString.length; - } - position -= searchString.length; - var lastIndex = subjectString.indexOf(searchString, position); - return lastIndex !== -1 && lastIndex === position; - }; - } - - function getFrame(getFrameDetails, callback) { - if ( typeof chrome !== 'undefined' ) { - chrome.webNavigation.getFrame(getFrameDetails, callback); - } else if ( typeof browser !== 'undefined' ) { - var gettingFrame = browser.webNavigation.getFrame(getFrameDetails); - gettingFrame.then(callback); - } - } - - function ifBrowserValid(callback, elseCallback) { - if ( isChrome() ) { - - callback(); - } else if ( typeof browser !== 'undefined' ) { - try { - var gettingInfo = browser.runtime.getBrowserInfo(); - gettingInfo.then(function(browserInfo) { - var browserVersion = parseInt(browserInfo.version.split('.')[0]); - - if ( browserInfo.name === 'Firefox' && - browserVersion >= MIN_FF_MAJOR_VERSION) { - callback(); - } else { - elseCallback(); - } - }); - } catch (err) { - - elseCallback(); - } - } else { - elseCallback(); - } - } - - function isTrackingEnabled() { - - return parseInt(localStorage.tracking, 10); - - } - - function isPixelRequest(request) { - return (request.type === 'image' || request.responseStatus === 204) && - request.size <= 1000; - } - - function isVpaidOrVastRequest(request) { - var lowerCaseUrl = request.url.toLowerCase(); - return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1; - } - - function hasValidRequestType(request) { - return whitelistReqTypes.indexOf(request.type) >= 0; - } - - function stripQueryParams(url) { - return url.split('?', 1)[0]; - } - - function parseHostnameFromUrl(url) { - var parser = document.createElement('a'); - parser.href = url; - return parser.hostname; - } - - function hasDomain(url, domain) { - return parseHostnameFromUrl(url).endsWith(domain); - } - - function findHeader(headers, key) { - var header; - for ( var i = 0; i < headers.length; i += 1 ) { - header = headers[i]; - if ( header.name.toLowerCase() === key ) { - return header; - } - } - return null; - } - - function validVideoType(vtype) { - var goodType = videoContentTypesPrefixes.some(function(prefix) { - return vtype.indexOf(prefix) === 0; - }); - return goodType; - } - - function assetMsgKey(assetReq) { - var url = stripQueryParams(assetReq.url); - var key = assetReq.frameId + '-' + url; - return key; - } - - var PageNetworkTrafficCollector = function(tabId) { - this.tabId = tabId; - this.displayAdFound = false; - this.requests = {}; - this.msgsBeingSent = {}; - this.assetsSeen = {}; - this.allRedirects = {}; - }; - - var globalPageContainer = { - collectors: {}, - dyingCollectors: {}, - - cleanupCollector: function(tabId) { - if ( tabId in this.collectors ) { - delete globalPageContainer.collectors[tabId]; - } - }, - - onNewNavigation: function(details) { - var tabId = details.tabId; - this.cleanupCollector(tabId); - - if ( isTrackingEnabled() ) { - if ( !areListenersRegistered ) { - - registerListeners(); - } - this.collectors[tabId] = new PageNetworkTrafficCollector(tabId); - } else { - if ( areListenersRegistered ) { - - unregisterListeners(); - } - } - }, - - onNavigationCommitted: function(details) { - - }, - - onNavigationCompleted: function(details) { - - }, - - onTabClose: function(tabId, closeInfo) { - - this.cleanupCollector(tabId); - delete this.collectors[tabId]; - }, - - onDisplayAdFound: function(tabId) { - this.collectors[tabId].displayAdFound = true; - }, - - getRandId: function() { - return String(Math.floor(Math.random() * 1e9)); - }, - - getCollector: function(tabId) { - if ( this.collectors.hasOwnProperty(tabId) ) { - return this.collectors[tabId]; - } - return null; - }, - - forwardCall: function(details, collectorMemberFunction) { - var collector = this.getCollector(details.tabId); - if ( collector !== null ) { - collectorMemberFunction.apply(collector, [details]); - } - } - }; - - PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() { - var logMessage = Array.from(arguments).join(' '); - var message = {message: logMessage, event: 'console-log-message'}; - browserProxy.tabs.sendMessage(this.tabId, message); - }; - - PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, isValidAd) { - var msg = {}; - msg.assets = []; - msg.event_data = {}; - if ( isValidAd ) { - msg.event = 'new-video-ad'; - msg.requests = reqs; - msg.requests.sort(function(reqA, reqB) {return reqA.requestTimestamp - reqB.requestTimestamp;}); - if ( assetReq ) { - msg.assets = [assetReq]; - } - } else { - msg.requests = reqs.map(function(request) { - return parseHostnameFromUrl(request.url); - }); - msg.assets = [{ - - url: parseHostnameFromUrl(assetReq.url), - - contentType: assetReq.contentType, - size: assetReq.size - }]; - msg.event = 'new-invalid-video-ad'; - } - msg.origUrl = curPageUrl; - msg.displayAdFound = this.displayAdFound; - - browserProxy.tabs.sendMessage(this.tabId, msg); - }; - - PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) { - return url + ':' + frameId; - }; - - PageNetworkTrafficCollector.prototype.seenBefore = function(request) { - var oldTime = this.assetsSeen[assetMsgKey(request)]; - if ( oldTime && (request.requestTimestamp-oldTime < secBetweenDupAssets)){ - - return true; - } - return false; - }; - - PageNetworkTrafficCollector.prototype.recordSeenAsset = function(request) { - this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp; - }; - - PageNetworkTrafficCollector.prototype.onBeforeRequest = function(details) { - var req = { - url: details.url, - type: details.type, - httpMethod: details.method, - frameId: details.frameId, - parentFrameId: details.parentFrameId, - requestTimestamp: details.timeStamp, - }; - this.requests[details.requestId] = req; - }; - - PageNetworkTrafficCollector.prototype.onSendHeaders = function(details) { - var request, header; - request = this.requests[details.requestId]; - header = request && findHeader(details.requestHeaders, 'x-requested-with'); - if ( header && header.value.toLowerCase().indexOf('flash') > -1 ) { - request.from_flash = true; - } - }; - - PageNetworkTrafficCollector.prototype.onHeadersReceived = function(details) { - var getFrameDetails = { - tabId: details.tabId, - processId: null, - frameId: details.frameId - }; - var pageNetworkTrafficController = this; - getFrame(getFrameDetails, function(frameDetails) { - if ( frameDetails && frameDetails.url ) { - pageNetworkTrafficController._onHeadersReceived(details, frameDetails); - } - }); - }; - - PageNetworkTrafficCollector.prototype._onHeadersReceived = function(details, frameDetails) { - var contentSize, contentRange; - - var request = this.requests[details.requestId]; - if ( request ) { - var redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)]; - var header = request && findHeader(details.responseHeaders, 'content-type'); - var contentType = header && header.value.toLowerCase(); - - if ( contentType){ - request.contentType = contentType; - } - header = request && findHeader(details.responseHeaders, 'content-length'); - contentSize = header && header.value; - if ( contentSize ) { - request.size = request.size || 0; - request.size += parseInt(contentSize); - } - header = request && findHeader(details.responseHeaders, 'content-range'); - contentRange = header && header.value; - if ( contentRange ) { - request.contentRange = parseInt(contentRange.split('/')[1]); - } - - var frameUrl = null; - if ( frameDetails && frameDetails.url ) { - frameUrl = frameDetails.url; - } - if ( !this.bannedRequest(request) && - (this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) { - request.isVideo = true; - } - } - }; - - PageNetworkTrafficCollector.prototype.onBeforeRedirect = function(details) { - var request = this.requests[details.requestId]; - if ( request ) { - if ( request.redirects ) { - request.redirects.push(details.redirectUrl); - } else { - request.redirects = [details.redirectUrl]; - } - this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request; - } - }; - - PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function(url) { - var re = /video_masthead/; - return this.hasYoutubeDomain(url) && re.test(url); - }; - PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function(srcUrl, destUrl) { - if ( !this.hasYoutubeDomain(srcUrl) ) { - return false; - } - - var re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/; - return re.test(destUrl); - }; - PageNetworkTrafficCollector.prototype.processResponse = function(requestDetails, frameDetails) { - var request; - if ( requestDetails ) { - request = this.requests[requestDetails.requestId]; - if ( request ) { - request.responseStatus = requestDetails.statusCode; - request.responseTimestamp = requestDetails.timeStamp; - - var frameUrl = null; - if ( frameDetails && frameDetails.url ) { - frameUrl = frameDetails.url; - } - - var requestUrl = null; - if ( request.url ) { - requestUrl = request.url; - } - - if ( this.isYoutubeAdReq(frameUrl, requestUrl) ) { - var videoId = this.parseYoutubeVideoIdFromUrl(requestUrl); - if ( videoId ) { - request.isYoutubeAd = true; - request.isVideo = true; - request.url = 'https://www.youtube.com/watch?v=' + this.parseYoutubeVideoIdFromUrl(requestUrl); - } - } else if ( !this.bannedRequest(request) && - (this.isVideo || this.isVideoReq(frameUrl, request))) { - request.isVideo = true; - } - - if ( request.isVideo ) { - - var msgKey = assetMsgKey(request); - this.msgsBeingSent[msgKey] = request; - if ( !this.seenBefore(request) ) { - this.sendMsgWhenQuiet(msgKey); - } - this.recordSeenAsset(request); - } - } - } - }; - - PageNetworkTrafficCollector.prototype.onResponseStarted = function(responseDetails) { - if ( responseDetails.frameId < 0 ) { - responseDetails.frameId = 99999; - - } - var getFrameDetails = { - tabId: responseDetails.tabId, - processId: null, - frameId: responseDetails.frameId - }; - var pageNetworkTrafficController = this; - getFrame(getFrameDetails, function(frameDetails) { - if ( frameDetails && frameDetails.url ) { - pageNetworkTrafficController.processResponse(responseDetails, frameDetails); - } - }); - }; - - PageNetworkTrafficCollector.prototype.hasBannedFiletype = function(request) { - var url = stripQueryParams(request.url); - if ( bannedFiletypesReg.exec(url) ) { - return true; - } else { - return false; - } - }; - - PageNetworkTrafficCollector.prototype.checkContentHeaders = function(request) { - if ( request.contentType && validVideoType(request.contentType) ) { - return true; - } - return false; - }; - - PageNetworkTrafficCollector.prototype.checkUrlExtension = function(request) { - var url = stripQueryParams(request.url); - if ( extensionsReg.exec(url) ) { - return true; - } else { - return false; - } - }; - - PageNetworkTrafficCollector.prototype.isVideoReq = function(srcUrl, request) { - if ( this.isYoutubeVideoRequest(srcUrl, request.url) ) { - return false; - } - return this.checkUrlExtension(request) || this.checkContentHeaders(request); - }; - PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function(url) { - var hostname = parseHostnameFromUrl(url) ; - if ( hostname === 'www.youtube.com' ) { - return true; - } - return false; - }; - PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function(url) { - var re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/; - var match = re.exec(url); - if ( match && match.length > 1 ) { - return match[1]; - } - - re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/; - match = re.exec(url); - if ( match && match.length > 1 ) { - return match[1]; - } - - re = /^https?:\/\/www\.youtube\.com\/watch\?v=(.*$)/; - match = re.exec(url); - if ( match && match.length > 1 ) { - return match[1]; - } - return null; - }; - - PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function(url) { - var re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/; - return re.test(url); - }; - PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function(srcUrl, destUrl) { - - if ( !this.hasYoutubeDomain(srcUrl) || - !this.isYoutubeGetVideoInfoReq(destUrl)) { - return false; - } - if ( this.parseYoutubeVideoIdFromUrl(srcUrl) === - this.parseYoutubeVideoIdFromUrl(destUrl) && - !this.isYoutubeMastheadRequest(destUrl)) { - return false; - } - return true; - }; - - PageNetworkTrafficCollector.prototype.bannedRequest = function(request) { - return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request); - }; - - PageNetworkTrafficCollector.prototype.bannedVideoType = function(request) { - var badType = false; - if ( request.contentType ) { - badType = bannedContentTypes.some(function(prefix) { - return request.contentType.indexOf(prefix) >= 0; - }); - } - return badType; - }; - - PageNetworkTrafficCollector.prototype.bannedVideoSize = function(request) { - if ( request.size !== null ) { - if ( request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange ) { - return true; - } - } - return false; - }; - - PageNetworkTrafficCollector.prototype.grabTagReqs = function(tabRequests, assetRequest) { - var minTimestamp, maxTimestamp; - minTimestamp = assetRequest.requestTimestamp - secBefore; - maxTimestamp = assetRequest.requestTimestamp + secAfter; - - var filteredRequests = tabRequests.filter(function(request) { - return (request.requestTimestamp > minTimestamp && - request.requestTimestamp < maxTimestamp && - request.frameId === assetRequest.frameId && - request.url !== assetRequest.url && - (hasValidRequestType(request) || - isPixelRequest(request))); - }); - - return filteredRequests; - }; - - PageNetworkTrafficCollector.prototype.isValidVideoAd = function(assetRequest, tagRequests) { - var hasVpaidOrVastRequest = tagRequests.some(function(tagRequest) { - return isVpaidOrVastRequest(tagRequest); - }); - - if ( assetRequest.isYoutubeAd ) { - return true; - } - if ( hasVpaidOrVastRequest ) { - return true; - } - var hasTopVideoAssetDomain = topVideoAssetDomains.some(function(assetDomain) { - return hasDomain(assetRequest.url, assetDomain); - }); - - return hasTopVideoAssetDomain; - }; - - PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function(msgKey) { - var _this = this, - origPageUrl, msgAssetReq; - msgAssetReq = this.msgsBeingSent[msgKey]; - browserProxy.tabs.get(this.tabId, function(tab) {origPageUrl = tab.url;}); - - setTimeout(function() { - var rawRequests = []; - if ( globalPageContainer.collectors[_this.tabId] === _this ) { - for ( var reqId in _this.requests ) { - rawRequests.push(_this.requests[reqId]); - } - var tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq); - - if ( _this.isValidVideoAd(msgAssetReq, tagReqs) ) { - _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, true); - } else { - - _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, false); - } - - } else { - - } - delete _this.msgsBeingSent[msgKey]; - }, secAfter+secBefore); - }; - - PageNetworkTrafficCollector.prototype.existingMessage = function(candidateRequest) { - var frameMsg = this.msgsBeingSent[candidateRequest.frameId]; - if ( frameMsg ) { - return frameMsg; - } else { - return null; - } - }; - - function onBeforeRequestListener(details) { - globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest); - } - - function onSendHeadersListener(details) { - globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders); - } - - function onHeadersReceivedListener(details) { - globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived); - } - - function onBeforeRedirectListener(details) { - globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect); - } - - function onResponseStartedListener(details) { - globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted); - } - - function onCommittedListener(details) { - if ( details.frameId === 0 ) { - globalPageContainer.onNavigationCommitted(details); - } - } - - function onCompletedListener(details) { - if ( details.frameId === 0 ) { - globalPageContainer.onNavigationCompleted(details); - } - } - - function onRemovedListener(tabId, closeInfo) { - globalPageContainer.onTabClose(tabId, closeInfo); - } - - function onMessageListener(message, sender, sendResponse) { - if ( message.event === 'new-ad' && message.data.event === 'ad' ) { - var tabId = sender.tab.id; - if ( tabId ) { - globalPageContainer.onDisplayAdFound(tabId); - } - } - } - - function registerListeners() { - - browserProxy.webRequest.onBeforeRequest.addListener( - onBeforeRequestListener, - {urls: ['http://*/*', 'https://*/*']}, - [] - ); - - browserProxy.webRequest.onSendHeaders.addListener( - onSendHeadersListener, - {urls: ['http://*/*', 'https://*/*']}, - ['requestHeaders'] - ); - - browserProxy.webRequest.onHeadersReceived.addListener( - onHeadersReceivedListener, - {urls: ['http://*/*', 'https://*/*']}, - ['responseHeaders'] - ); - - browserProxy.webRequest.onBeforeRedirect.addListener( - onBeforeRedirectListener, - {urls: ['http://*/*', 'https://*/*']}, - [] - ); - - browserProxy.webRequest.onResponseStarted.addListener( - onResponseStartedListener, - {urls: ['http://*/*', 'https://*/*']}, - ['responseHeaders'] - ); - - browserProxy.webNavigation.onCommitted.addListener(onCommittedListener); - browserProxy.webNavigation.onCompleted.addListener(onCompletedListener); - browserProxy.tabs.onRemoved.addListener(onRemovedListener); - browserProxy.runtime.onMessage.addListener(onMessageListener); - - areListenersRegistered = true; - } - - function unregisterListeners() { - - browserProxy.webRequest.onBeforeRequest.removeListener( - onBeforeRequestListener - ); - - browserProxy.webRequest.onSendHeaders.removeListener( - onSendHeadersListener - ); - - browserProxy.webRequest.onHeadersReceived.removeListener( - onHeadersReceivedListener - ); - - browserProxy.webRequest.onBeforeRedirect.removeListener( - onBeforeRedirectListener - ); - - browserProxy.webRequest.onResponseStarted.removeListener( - onResponseStartedListener - ); - - browserProxy.webNavigation.onCommitted.removeListener(onCommittedListener); - browserProxy.webNavigation.onCompleted.removeListener(onCompletedListener); - browserProxy.tabs.onRemoved.removeListener(onRemovedListener); - browserProxy.runtime.onMessage.removeListener(onMessageListener); - - areListenersRegistered = false; - } - - function areRequiredBrowserApisAvailable() { - return requiredBrowserApis.every(function(api) { - return typeof api !== 'undefined'; - }); - } - - if ( areRequiredBrowserApisAvailable() ) { - ifBrowserValid( - function() { - browserProxy.webNavigation.onBeforeNavigate.addListener( - function(details) { - if ( details.frameId === 0 ) { - globalPageContainer.onNewNavigation(details); - } - }, - { - url: [{urlMatches: 'http://*/*'}, {urlMatches: 'https://*/*'}] - } - ); - }, function() { - - } - ); - } - - browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) { - if ( request === 'is_browser_valid' ) { - ifBrowserValid( - sendResponse({'browser_valid': true}), - sendResponse({'browser_valid': false}) - ); - } - }); - - browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) { - if ( request === 'is_tracking_enabled' ) { - sendResponse({'tracking_enabled': isTrackingEnabled()}); - } - }); - -})();