scraper.js

// Generated by CoffeeScript 1.8.0
(function() {
  var Promise, Scraper, fs, request, scraper;

  request = require("request-promise");

  Promise = require("bluebird");

  fs = require("fs");

  Scraper = (function() {
    function Scraper() {}

    Scraper.prototype._endpoint = "https://api.justwatch.com";

    Scraper.prototype._popular = function(locale) {
      var item_count, items_list, page_size, _i, _ref, _results;
      page_size = 200;
      item_count = 1000;
      items_list = [];
      return Promise.each((function() {
        _results = [];
        for (var _i = 0, _ref = item_count / page_size; 0 <= _ref ? _i < _ref : _i > _ref; 0 <= _ref ? _i++ : _i--){ _results.push(_i); }
        return _results;
      }).apply(this), (function(_this) {
        return function(i) {
          i += 1;
          console.log("* Downloading titles " + (page_size * i) + "/" + item_count);
          return request({
            uri: "" + _this._endpoint + "/titles/" + locale + "/popular",
            method: 'POST',
            gzip: true,
            json: true,
            body: {
              content_types: ["show", "movie"],
              page: i,
              page_size: page_size
            }
          }).then(function(data) {
            return items_list = items_list.concat(data.items);
          });
        };
      })(this)).then(function() {
        return items_list;
      });
    };

    Scraper.prototype._show = function(id, locale) {
      var url;
      console.log("* Downloading show data for ID: #" + id);
      url = "" + this._endpoint + "/titles/show/" + id + "/locale/" + locale;
      return request.get({
        uri: url,
        json: true
      });
    };

    Scraper.prototype.scrape = function(locale) {
      if (locale == null) {
        locale = "en_US";
      }
      return this._popular(locale).then((function(_this) {
        return function(titles) {
          return Promise.all(titles.map(function(title) {
            var data;
            data = {
              title: title.title,
              year: title.original_release_year,
              type: title.object_type
            };
            if (data.type === "movie") {
              return data;
            }
            return _this._show(title.id, locale).then(function(show) {
              data.season_count = show.seasons.length;
              return data;
            });
          }));
        };
      })(this)).then(function(titles) {
        return {
          titles: titles,
          title_count: titles.length,
          locale: locale
        };
      });
    };

    return Scraper;

  })();

  module.exports = Scraper;

  if (require.main === module) {
    scraper = new Scraper();
    scraper.scrape(process.argv[2]).then(function(response) {
      var data, filePath;
      filePath = "" + __dirname + "/output.txt";
      data = JSON.stringify(response);
      return fs.writeFile(filePath, data, function() {
        return console.log("* File saved to " + filePath);
      });
    });
  }

}).call(this);