diff --git a/Rakefile b/Rakefile index 1ce5ff2..260005b 100644 --- a/Rakefile +++ b/Rakefile @@ -5,6 +5,7 @@ require 'rubygems' #require "rake/testtask" #require 'rcov/rcovtask' $: << File.dirname(__FILE__) +require 'lib/awsbase/require_relative.rb' require 'lib/right_aws.rb' testglobs = ["test/ts_right_aws.rb"] @@ -86,4 +87,11 @@ task :testacf do require 'test/acf/test_acf.rb' end +desc "Test Alexa interface" +task :testalexa do + require 'test/test_credentials' + TestCredentials.get_credentials + require 'test/alexa/test_alexa.rb' +end + # vim: syntax=Ruby diff --git a/aws.gemspec b/aws.gemspec index 7de4545..7d4a4ec 100644 --- a/aws.gemspec +++ b/aws.gemspec @@ -1,6 +1,6 @@ # Generated by jeweler -# DO NOT EDIT THIS FILE DIRECTLY -# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec' +# DO NOT EDIT THIS FILE +# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec` # -*- encoding: utf-8 -*- Gem::Specification.new do |s| @@ -9,43 +9,58 @@ Gem::Specification.new do |s| s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Travis Reeder", "Chad Arimura", "RightScale"] - s.date = %q{2011-05-08} - s.description = %q{AWS Ruby Library for interfacing with Amazon Web Services including EC2, S3, SQS, SimpleDB and most of their other services as well. By http://www.appoxy.com} + s.date = %q{2010-11-11} + s.description = %q{AWS Ruby Library for interfacing with Amazon Web Services.} s.email = %q{travis@appoxy.com} s.extra_rdoc_files = [ "README.markdown" ] s.files = [ - "lib/acf/acf_interface.rb", - "lib/aws.rb", - "lib/awsbase/aws_response_array.rb", - "lib/awsbase/awsbase.rb", - "lib/awsbase/benchmark_fix.rb", - "lib/awsbase/errors.rb", - "lib/awsbase/parsers.rb", - "lib/awsbase/require_relative.rb", - "lib/awsbase/utils.rb", - "lib/ec2/ec2.rb", - "lib/ec2/mon_interface.rb", - "lib/elb/elb_interface.rb", - "lib/iam/iam.rb", - "lib/rds/rds.rb", - "lib/right_aws.rb", - "lib/s3/bucket.rb", - "lib/s3/grantee.rb", - "lib/s3/key.rb", - "lib/s3/s3.rb", - "lib/s3/s3_interface.rb", - "lib/sdb/active_sdb.rb", - "lib/sdb/sdb_interface.rb", - "lib/ses/ses.rb", - "lib/sqs/sqs.rb", - "lib/sqs/sqs_interface.rb" + "lib/acf/right_acf_interface.rb", + "lib/alexa/alexa.rb", + "lib/aws.rb", + "lib/awsbase/aws_response_array.rb", + "lib/awsbase/benchmark_fix.rb", + "lib/awsbase/right_awsbase.rb", + "lib/awsbase/support.rb", + "lib/ec2/right_ec2.rb", + "lib/ec2/right_mon_interface.rb", + "lib/elb/elb_interface.rb", + "lib/rds/rds.rb", + "lib/right_aws.rb", + "lib/s3/right_s3.rb", + "lib/s3/right_s3_interface.rb", + "lib/sdb/active_sdb.rb", + "lib/sdb/right_sdb_interface.rb", + "lib/sqs/right_sqs.rb", + "lib/sqs/right_sqs_interface.rb" ] s.homepage = %q{http://github.com/appoxy/aws/} s.require_paths = ["lib"] - s.rubygems_version = %q{1.7.2} - s.summary = %q{AWS Ruby Library for interfacing with Amazon Web Services. By http://www.appoxy.com} + s.rubygems_version = %q{1.3.7} + s.summary = %q{AWS Ruby Library for interfacing with Amazon Web Services.} + s.test_files = [ + "test/acf/test_acf.rb", + "test/acf/test_helper.rb", + "test/alexa/test_alexa.rb", + "test/alexa/test_helper.rb", + "test/ec2/test_ec2.rb", + "test/ec2/test_helper.rb", + "test/ec2/test_mon.rb", + "test/elb/test_elb.rb", + "test/http_connection.rb", + "test/rds/test_rds.rb", + "test/s3/test_helper.rb", + "test/s3/test_s3.rb", + "test/s3/test_s3_stubbed.rb", + "test/sdb/test_active_sdb.rb", + "test/sdb/test_helper.rb", + "test/sdb/test_sdb.rb", + "test/sqs/test_helper.rb", + "test/sqs/test_sqs.rb", + "test/test_credentials.rb", + "test/ts_right_aws.rb" + ] if s.respond_to? :specification_version then s.specification_version = 3 @@ -65,4 +80,3 @@ Gem::Specification.new do |s| s.add_dependency(%q, [">= 0"]) end end - diff --git a/hackerdude-aws.gemspec b/hackerdude-aws.gemspec new file mode 100644 index 0000000..19e9c99 --- /dev/null +++ b/hackerdude-aws.gemspec @@ -0,0 +1,84 @@ +# Generated by jeweler +# DO NOT EDIT THIS FILE DIRECTLY +# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec' +# -*- encoding: utf-8 -*- + +Gem::Specification.new do |s| + s.name = %q{hackerdude-aws} + s.version = "2.3.26" + + s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= + s.authors = ["Travis Reeder", "Chad Arimura", "RightScale", "David Martinez"] + s.date = %q{2010-12-13} + s.description = %q{AWS Ruby Library for interfacing with Amazon Web Services. Alexa-experimental branch.} + s.email = %q{travis@appoxy.com} + s.extra_rdoc_files = [ + "README.markdown" + ] + s.files = [ + "lib/acf/right_acf_interface.rb", + "lib/aws.rb", + "lib/awsbase/aws_response_array.rb", + "lib/awsbase/benchmark_fix.rb", + "lib/awsbase/right_awsbase.rb", + "lib/awsbase/support.rb", + "lib/ec2/right_ec2.rb", + "lib/ec2/right_mon_interface.rb", + "lib/elb/elb_interface.rb", + "lib/rds/rds.rb", + "lib/right_aws.rb", + "lib/s3/right_s3.rb", + "lib/s3/right_s3_interface.rb", + "lib/sdb/active_sdb.rb", + "lib/sdb/right_sdb_interface.rb", + "lib/sqs/right_sqs.rb", + "lib/sqs/right_sqs_interface.rb" + ] + s.homepage = %q{http://github.com/hackerdude/aws/} + s.require_paths = ["lib"] + s.rubygems_version = %q{1.3.7} + s.summary = %q{AWS Ruby Library for interfacing with Amazon Web Services. Hackerdude branch including Alexa (Experimental).} + s.test_files = [ + "test/acf/test_acf.rb", + "test/acf/test_helper.rb", + "test/ec2/test_ec2.rb", + "test/ec2/test_helper.rb", + "test/ec2/test_mon.rb", + "test/elb/test_elb.rb", + "test/http_connection.rb", + "test/rds/test_rds.rb", + "test/s3/test_helper.rb", + "test/s3/test_s3.rb", + "test/s3/test_s3_stubbed.rb", + "test/sdb/test_active_sdb.rb", + "test/sdb/test_helper.rb", + "test/sdb/test_sdb.rb", + "test/sqs/test_helper.rb", + "test/sqs/test_sqs.rb", + "test/test_credentials.rb", + "test/ts_right_aws.rb" + ] + + if s.respond_to? :specification_version then + current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION + s.specification_version = 3 + + if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then + s.add_runtime_dependency(%q, [">= 0"]) + s.add_runtime_dependency(%q, [">= 0"]) + s.add_runtime_dependency(%q, [">= 0"]) + s.add_runtime_dependency(%q, [">= 0"]) + else + s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) + end + else + s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) + end +end + diff --git a/lib/alexa/alexa.rb b/lib/alexa/alexa.rb new file mode 100644 index 0000000..a0eaaa8 --- /dev/null +++ b/lib/alexa/alexa.rb @@ -0,0 +1,244 @@ +module Aws +class Alexa + include AwsBaseInterface + DEFAULT_HOST = "awis.amazonaws.com" + DEFAULT_PATH = "/" + API_VERSION = "2005-07-11" + DEFAULT_PROTOCOL = 'http' + DEFAULT_PORT = 80 + + VALID_URLINFO_RESPONSE_GROUPS = [ + :related_links, + :categories, + :rank, + :rank_by_country, + :rank_by_city, + :usage_stats, + :contact_info, + :adult_content, + :speed, + :language, + :keywords, + :owned_domains, + :links_in_count, + :site_data + ] + # TODO - the docs mentions :popups on the meta groups, but it's not on the response group. Oversight? Check to see if that kind of info is being passed + + VALID_CATEGORY_BROWSE_RESPONSE_GROUPS = [ :categories, + :related_categories, + :language_categories, + :letter_bars + ] + + META_GROUPS = { + :related=>[:related_links, :categories], + :traffic_data=>[:rank, :usage_stats], + :content_data=>[:site_data, :adult_content, :popups, :speed, :language] + } + @@bench = AwsBenchmarkingBlock.new + def self.bench_xml + @@bench.xml + end + def self.bench_ec2 + @@bench.service + end + + @@api = ENV['ALEXA_API_VERSION'] || API_VERSION + def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) + init({ :name => 'ALEXA', + :default_host => ENV['ALEXA_URL'] ? URI.parse(ENV['ALEXA_URL']).host : DEFAULT_HOST, + :default_port => ENV['ALEXA_URL'] ? URI.parse(ENV['ALEXA_URL']).port : DEFAULT_PORT, + :default_service => ENV['ALEXA_URL'] ? URI.parse(ENV['ALEXA_URL']).path : DEFAULT_PATH, + :default_protocol => ENV['ALEXA_URL'] ? URI.parse(ENV['ALEXA_URL']).scheme : DEFAULT_PROTOCOL, + :api_version => API_VERSION }, + aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'] , + aws_secret_access_key|| ENV['AWS_SECRET_ACCESS_KEY'], + params) + # EC2 doesn't really define any transient errors to retry, and in fact, + # when they return a 503 it is usually for 'request limit exceeded' which + # we most certainly should not retry. So let's pare down the list of + # retryable errors to InternalError only (see AwsBase for the default + # list) + amazon_problems = ['InternalError'] + end + + def generate_request(action, params={}) #:nodoc: + service_hash = {"Action" => action, + "AWSAccessKeyId" => @aws_access_key_id, + "Version" => @@api } + service_hash.update(params) + service_params = signed_service_params(@aws_secret_access_key, service_hash, :get, @params[:server], @params[:service]) + + # use POST method if the length of the query string is too large + if service_params.size > 2000 + if signature_version == '2' + # resign the request because HTTP verb is included into signature + service_params = signed_service_params(@aws_secret_access_key, service_hash, :post, @params[:server], @params[:service]) + end + request = Net::HTTP::Post.new(@params[:service]) + request.body = service_params + request['Content-Type'] = 'application/x-www-form-urlencoded' + else + request = Net::HTTP::Get.new("#{@params[:service]}?#{service_params}") + end + # prepare output hash + { :request => request, + :server => @params[:server], + :port => @params[:port], + :protocol => @params[:protocol] } + end + + def request_info(request, parser) #:nodoc: + thread = @params[:multi_thread] ? Thread.current : Thread.main + thread[:ec2_connection] ||= Rightscale::HttpConnection.new(:exception => AwsError, :logger => @logger) + request_info_impl(thread[:ec2_connection], @@bench, request, parser) + end + + # Quickly return the alexa URL info, (rank only) + # + # Example: + # rank("http://www.yahoo.com")[:rank][:text] # 4 + def rank(url, params={}, cache_for = nil) + result = url_info(url, {:response_groups=>:rank}) + return result[:url_info_response][:response][:url_info_result][:alexa][:traffic_data] + end + + # Retrieves the Alexa URL info for a URL. + # By default it returns every response group. + # Override response_groups to set your own. + # + # Example: + # url_info("http://www.google.com")[:url_info_response].. + def url_info(url, opts={}, cache_for=nil, parser = QAlexaUrlInfoParser) + options = {:response_groups=>VALID_URLINFO_RESPONSE_GROUPS}.merge(opts) + request_hash = {} + request_hash['Url'] = url + request_hash['ResponseGroup'] = response_groups_to_param(options[:response_groups]) + params.each do |key, val| + request_hash.merge! hash_params(key, (val.is_a?(Array) ? val : [val])) + end + link = generate_request('UrlInfo', request_hash) + request_cache_or_info(cache_for, link, parser, @@bench, cache_for) + rescue Exception + on_exception + end + + # Returns a list of subcategories inside a category + # + def category_browse(path, opts={}, cache_for=nil, parser = QAlexaUrlInfoParser) + options = {:response_groups=>VALID_CATEGORY_BROWSE_RESPONSE_GROUPS}.merge(opts) + request_hash = {} + request_hash['Path'] = path + request_hash['ResponseGroup'] = response_groups_to_param(options[:response_groups]) + params.each do |key, val| + request_hash.merge! hash_params(key, (val.is_a?(Array) ? val : [val])) + end + link = generate_request('CategoryBrowse', request_hash) + request_cache_or_info(cache_for, link, parser, @@bench, cache_for) + rescue Exception + on_exception + end + + # Returns a list of category listings + def category_listings(path, cache_for=nil, parser = QAlexaUrlInfoParser) + request_hash = {} + request_hash['Path'] = path + request_hash['ResponseGroup'] = "Listings" + params.each do |key, val| + request_hash.merge! hash_params(key, (val.is_a?(Array) ? val : [val])) + end + link = generate_request('CategoryListings', request_hash) + request_cache_or_info(cache_for, link, parser, @@bench, cache_for) + rescue Exception + on_exception + end + + def sites_linking_in(path, cache_for, parser = QAlexaUrlInfoParser) + request_hash = {} + request_hash['Path'] = path + request_hash['ResponseGroup'] = "SitesLinkingIn" + params.each do |key, val| + request_hash.merge! hash_params(key, (val.is_a?(Array) ? val : [val])) + end + link = generate_request('CategoryListings', request_hash) + request_cache_or_info(cache_for, link, parser, @@bench, cache_for) + rescue Exception + on_exception + end + + def traffic_history + throw ArgumentError.new("Not Implemented. Sorry!") + end + + private + + def response_groups_to_param(groups) + actual_groups = groups.is_a?(Array) ? groups : [groups] + actual_groups.collect{|g| camelize(g.to_s) }.join(",") + end + + # This is ye olde ActiveSupport camelize, just without the dependency + def camelize(string) + string.split('_').map {|w| w.capitalize}.join + end + +end + + class QAlexaElementHash < Hash + attr_accessor :finalized + def last + self + end + def to_s + return has_key?(:text) ? self[:text] : super.to_s + end + end + + class QAlexaUrlInfoParser < AwsParser + + def initialize(args) + super(args) + @result = {} + end + + def tagstart(name, attr) + item = current_item(name) + _attr = attr.dup + _attr.delete("xmlns:aws") + item.merge!(_attr) + end + + def tagend(name) + element = current_item(name) + element[:text] = @text.strip + element.finalized = true + end + + def current_item(name) + outer = @result + inner = nil + path_array = @xmlpath.split('/') + path_array << name + path_array.collect{|s| s.sub('aws:','')}.each{|_xpath_element| + name_sym = symbol_for(_xpath_element) + inner = outer[name_sym] || QAlexaElementHash.new + inner = inner.last # Sometimes we may get an array. + if inner.finalized + old_inner = inner + inner = QAlexaElementHash.new + outer[name_sym] = [old_inner, inner] + end + outer[name_sym] = inner if ! outer.has_key?(name_sym) + outer = inner + } + return inner.nil? ? outer : inner + end + + def symbol_for(name) + sym_name = Aws::Utils.underscore(name.sub('aws:','')).to_sym + end + + end + +end diff --git a/lib/aws.rb b/lib/aws.rb index 79879b8..8833234 100644 --- a/lib/aws.rb +++ b/lib/aws.rb @@ -27,7 +27,3 @@ require 'acf/acf_interface' require 'elb/elb_interface' require 'rds/rds' -require 'iam/iam' -require 'ses/ses' - - diff --git a/lib/right_aws.rb b/lib/right_aws.rb index 75867d7..4c6913a 100644 --- a/lib/right_aws.rb +++ b/lib/right_aws.rb @@ -47,6 +47,7 @@ require 'sdb/sdb_interface' require 'acf/acf_interface' require 'elb/elb_interface' +require 'alexa/alexa' # backwards compatible. diff --git a/test/alexa/test_alexa.rb b/test/alexa/test_alexa.rb new file mode 100644 index 0000000..17892fd --- /dev/null +++ b/test/alexa/test_alexa.rb @@ -0,0 +1,53 @@ +require File.dirname(__FILE__) + '/test_helper.rb' +require 'pp' +require File.dirname(__FILE__) + '/../test_credentials.rb' + +require 'ruby-debug' +Debugger.start + +# Tests the Alexa AWS implementation +# Note this is very preliminary code. The code just spits back hashes +# and it has no concept of types just yet. The parser can +# be overriden on the calls if you need stronger types and must +# have something right away. +class TestAlexa < Test::Unit::TestCase + + def setup + TestCredentials.get_credentials + @alexa = Aws::Alexa.new(TestCredentials.aws_access_key_id, + TestCredentials.aws_secret_access_key) + end + + # Quick rank request + def test_rank + return + TestCredentials.get_credentials + rank = @alexa.rank("http://www.youtube.com") + assert ! rank.empty? + assert ! rank[:rank][:text].empty? + assert rank[:rank][:text].to_i > 0 + assert ! rank[:data_url][:text].empty? + end + + def test_urlinfo + TestCredentials.get_credentials + result = @alexa.url_info("http://www.yahoo.com") + assert result[:url_info_response][:response][:url_info_result][:alexa][:contact_info][:company_stock_ticker][:text] == "YHOO" + end + + def test_category_browse + TestCredentials.get_credentials + category_browse = @alexa.category_browse("Top/Computers/Software/Operating_Systems") + assert ! category_browse.empty? + assert ! category_browse[:category_browse_response][:response][:category_browse_result][:alexa][:category_browse][:categories][:category].first[:path][:text].empty? + + end + + def test_category_listings + TestCredentials.get_credentials + category_browse = @alexa.category_listings("Top/Computers/Software/Operating_Systems") + assert ! category_browse.empty? + assert category_browse[:category_listings_response][:response][:category_listings_result][:alexa][:category_listings][:listings][:listing].length > 0 + end + +end diff --git a/test/alexa/test_helper.rb b/test/alexa/test_helper.rb new file mode 100644 index 0000000..afb6844 --- /dev/null +++ b/test/alexa/test_helper.rb @@ -0,0 +1,2 @@ +require 'test/unit' +require File.dirname(__FILE__) + '/../../lib/aws'