From 301d770bdd3e4a7b3311fac43a0c249fa853e179 Mon Sep 17 00:00:00 2001 From: Mike Bajur Date: Wed, 4 Jun 2014 16:55:45 +0200 Subject: [PATCH 1/6] Add facebook plugin --- lib/viddl-rb.rb | 6 +++--- plugins/facebook.rb | 25 +++++++++++++++++++++++++ spec/integration/url_extraction_spec.rb | 24 +++++++++++++++--------- 3 files changed, 43 insertions(+), 12 deletions(-) create mode 100755 plugins/facebook.rb diff --git a/lib/viddl-rb.rb b/lib/viddl-rb.rb index 78150d0..a32746f 100755 --- a/lib/viddl-rb.rb +++ b/lib/viddl-rb.rb @@ -26,8 +26,8 @@ def self.io=(io_object) #set the default PluginBase io object to a StringIO instance. #this will suppress any standard output from the plugins. self.io = StringIO.new - - #returns an array of hashes containing the download url(s) and filenames(s) + + #returns an array of hashes containing the download url(s) and filenames(s) #for the specified video url. #if the url does not match any plugin, return nil and if a plugin #throws an error, throw PluginError. @@ -36,7 +36,7 @@ def self.io=(io_object) def self.get_urls_names(url) plugin = PluginBase.registered_plugins.find { |p| p.matches_provider?(url) } - if plugin + if plugin begin #we'll end up with an array of hashes with they keys :url and :name urls_filenames = plugin.get_urls_and_filenames(url) diff --git a/plugins/facebook.rb b/plugins/facebook.rb new file mode 100755 index 0000000..5326b65 --- /dev/null +++ b/plugins/facebook.rb @@ -0,0 +1,25 @@ +require 'uri' +require 'cgi' + +class Facebook < PluginBase + + #this will be called by the main app to check whether this plugin is responsible for the url passed + def self.matches_provider?(url) + url.include?("facebook.com") + end + + def self.get_urls_and_filenames(url, options = {}) + video_page = open(url).read + + title = video_page.scan(/(.+)<\/title>/).flatten.first + + download_url = video_page[/https.*.mp4/].gsub(/\\u([\da-fA-F]{4})/) {|m| [$1].pack("H*").unpack("n*").pack("U*")} + download_url = CGI::unescape(download_url) + download_url = URI::extract(download_url.gsub('\\', '')).first + + file_name = PluginBase.make_filename_safe(title) + '.mp4' + + [{:url => download_url, :name => file_name}] + end +end + diff --git a/spec/integration/url_extraction_spec.rb b/spec/integration/url_extraction_spec.rb index d8026c6..89d5131 100755 --- a/spec/integration/url_extraction_spec.rb +++ b/spec/integration/url_extraction_spec.rb @@ -31,9 +31,9 @@ def test_youtube_vevo assert_equal $?, 0 can_download_test(result) end - + def test_arte_plus_seven - response = RestClient.get('http://www.arte.tv/guide/de/plus7.json?regions=default%2CEUR_DE_FR%2CDE_FR%2CSAT%2CALL').to_str + response = RestClient.get('http://www.arte.tv/guide/de/plus7.json?regions=default%2CEUR_DE_FR%2CDE_FR%2CSAT%2CALL').to_str test_url = "http://www.arte.tv" + MultiJson.load(response)['videos'][0]['url'] puts "Running test using URL: #{test_url}" result = `ruby bin/viddl-rb #{test_url} --url-only` @@ -48,7 +48,7 @@ def test_youtube_different_formats can_download_test(result) assert result.include?("itag=18") - result2 = `ruby bin/viddl-rb http://www.youtube.com/watch?v=Zj3tYO9co44 --url-only --quality *:720:*` + result2 = `ruby bin/viddl-rb http://www.youtube.com/watch?v=Zj3tYO9co44 --url-only --quality *:720:*` assert_equal $?, 0 can_download_test(result2) assert result2.include?("itag=22") @@ -70,7 +70,7 @@ def test_vimeo_sd_video result = `ruby bin/viddl-rb http://vimeo.com/38372260 --url-only` assert_equal $?, 0 can_download_test(result) {|url_output| curl_code_grabber(url_output) } - end + end def test_soundcloud result = `ruby bin/viddl-rb http://soundcloud.com/rjchevalier/remembering-mavi-koy-wip --url-only` @@ -96,6 +96,12 @@ def test_bandcamp can_download_test(result) { |url_output| http_code_grabber(CGI::unescape(url_output), {:method => :get}) } end + def test_facebook + result = `ruby bin/viddl-rb https://www.facebook.com/photo.php?v=101503003357454&set=vb.310080259100701&type=2&theater --url-only` + assert_equal $?, 0 + can_download_test(result) { |url_output| http_code_grabber(CGI::unescape(url_output), {:method => :get}) } + end + # NOTE: The Metacafe tests are skipped because plugin is currently broken. def test_metacafe @@ -120,13 +126,13 @@ def test_dailymotion_hq assert_equal $?, 0 can_download_test(result) { |url_output| http_code_grabber(CGI::unescape(url_output)) } end - + private - + def can_download_test(result, &grabber) url_output = result.split("\n").last assert_includes(CGI.unescape(url_output), 'http://') - code_grabber = grabber || proc { |url_output| http_code_grabber(url_output) } + code_grabber = grabber || proc { |url_output| http_code_grabber(url_output) } tries = 0 http_response_code = 0 @@ -139,10 +145,10 @@ def can_download_test(result, &grabber) else puts "Retrying HTTP Call because of: #{e.message}" sleep 5 - retry + retry end end - + assert_equal(200, http_response_code) end end From cc1567f30745ce129df87c2afd9537e3a2812dc5 Mon Sep 17 00:00:00 2001 From: Mike Bajur Date: Thu, 5 Jun 2014 14:35:27 +0200 Subject: [PATCH 2/6] Make url_extraction_spec respect SSL video urls --- plugins/youtube/ciphers.yml | 1 + spec/integration/url_extraction_spec.rb | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/youtube/ciphers.yml b/plugins/youtube/ciphers.yml index d6d25fb..eadad85 100644 --- a/plugins/youtube/ciphers.yml +++ b/plugins/youtube/ciphers.yml @@ -97,3 +97,4 @@ en_US-vflbJnZqE: w26 s1 w15 w3 w62 w54 w22 en_US-vflgd5txb: w26 s1 w15 w3 w62 w54 w22 en_US-vflTm330y: w26 s1 w15 w3 w62 w54 w22 en_US-vflnwMARr: s3 r w24 s2 +en_US-vflaTh7jt: w46 r s3 w19 r s2 w15 diff --git a/spec/integration/url_extraction_spec.rb b/spec/integration/url_extraction_spec.rb index 89d5131..632a367 100755 --- a/spec/integration/url_extraction_spec.rb +++ b/spec/integration/url_extraction_spec.rb @@ -97,7 +97,7 @@ def test_bandcamp end def test_facebook - result = `ruby bin/viddl-rb https://www.facebook.com/photo.php?v=101503003357454&set=vb.310080259100701&type=2&theater --url-only` + result = `ruby bin/viddl-rb "https://www.facebook.com/photo.php?v=101503003357454&set=vb.310080259100701&type=2&theater" --url-only` assert_equal $?, 0 can_download_test(result) { |url_output| http_code_grabber(CGI::unescape(url_output), {:method => :get}) } end @@ -131,7 +131,8 @@ def test_dailymotion_hq def can_download_test(result, &grabber) url_output = result.split("\n").last - assert_includes(CGI.unescape(url_output), 'http://') + # Assert url includes http:// or https:// + assert((CGI.unescape(url_output) =~ /https?:\/\//) != nil) code_grabber = grabber || proc { |url_output| http_code_grabber(url_output) } tries = 0 http_response_code = 0 From e59b9dbdffa9d66becddba1d5c6d505d9bcc11b7 Mon Sep 17 00:00:00 2001 From: Mike Bajur Date: Thu, 5 Jun 2014 14:41:34 +0200 Subject: [PATCH 3/6] Remove last line from ciphers.yml --- plugins/youtube/ciphers.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/youtube/ciphers.yml b/plugins/youtube/ciphers.yml index eadad85..d6d25fb 100644 --- a/plugins/youtube/ciphers.yml +++ b/plugins/youtube/ciphers.yml @@ -97,4 +97,3 @@ en_US-vflbJnZqE: w26 s1 w15 w3 w62 w54 w22 en_US-vflgd5txb: w26 s1 w15 w3 w62 w54 w22 en_US-vflTm330y: w26 s1 w15 w3 w62 w54 w22 en_US-vflnwMARr: s3 r w24 s2 -en_US-vflaTh7jt: w46 r s3 w19 r s2 w15 From 8a42e5a3fb7fbe399e10c55348aabe629728ac1d Mon Sep 17 00:00:00 2001 From: Mike Bajur Date: Wed, 4 Jun 2014 16:55:45 +0200 Subject: [PATCH 4/6] Add facebook plugin --- lib/viddl-rb.rb | 6 +++--- plugins/facebook.rb | 25 +++++++++++++++++++++++++ spec/integration/url_extraction_spec.rb | 6 ++++++ 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100755 plugins/facebook.rb diff --git a/lib/viddl-rb.rb b/lib/viddl-rb.rb index 78150d0..a32746f 100755 --- a/lib/viddl-rb.rb +++ b/lib/viddl-rb.rb @@ -26,8 +26,8 @@ def self.io=(io_object) #set the default PluginBase io object to a StringIO instance. #this will suppress any standard output from the plugins. self.io = StringIO.new - - #returns an array of hashes containing the download url(s) and filenames(s) + + #returns an array of hashes containing the download url(s) and filenames(s) #for the specified video url. #if the url does not match any plugin, return nil and if a plugin #throws an error, throw PluginError. @@ -36,7 +36,7 @@ def self.io=(io_object) def self.get_urls_names(url) plugin = PluginBase.registered_plugins.find { |p| p.matches_provider?(url) } - if plugin + if plugin begin #we'll end up with an array of hashes with they keys :url and :name urls_filenames = plugin.get_urls_and_filenames(url) diff --git a/plugins/facebook.rb b/plugins/facebook.rb new file mode 100755 index 0000000..5326b65 --- /dev/null +++ b/plugins/facebook.rb @@ -0,0 +1,25 @@ +require 'uri' +require 'cgi' + +class Facebook < PluginBase + + #this will be called by the main app to check whether this plugin is responsible for the url passed + def self.matches_provider?(url) + url.include?("facebook.com") + end + + def self.get_urls_and_filenames(url, options = {}) + video_page = open(url).read + + title = video_page.scan(/(.+)<\/title>/).flatten.first + + download_url = video_page[/https.*.mp4/].gsub(/\\u([\da-fA-F]{4})/) {|m| [$1].pack("H*").unpack("n*").pack("U*")} + download_url = CGI::unescape(download_url) + download_url = URI::extract(download_url.gsub('\\', '')).first + + file_name = PluginBase.make_filename_safe(title) + '.mp4' + + [{:url => download_url, :name => file_name}] + end +end + diff --git a/spec/integration/url_extraction_spec.rb b/spec/integration/url_extraction_spec.rb index d0c7977..4e23178 100755 --- a/spec/integration/url_extraction_spec.rb +++ b/spec/integration/url_extraction_spec.rb @@ -102,6 +102,12 @@ def test_instagram can_download_test(result) { |url_output| http_code_grabber(CGI::unescape(url_output), {:method => :get}) } end + def test_facebook + result = `ruby bin/viddl-rb "https://www.facebook.com/photo.php?v=101503003357454&set=vb.310080259100701&type=2&theater --url-only"` + assert_equal $?, 0 + can_download_test(result) { |url_output| http_code_grabber(CGI::unescape(url_output), {:method => :get}) } + end + # NOTE: The Metacafe tests are skipped because plugin is currently broken. def test_metacafe From b7bbb575cf3a937b94c633dedd3d82ebdb2fd158 Mon Sep 17 00:00:00 2001 From: Mike Bajur Date: Thu, 5 Jun 2014 14:35:27 +0200 Subject: [PATCH 5/6] Make url_extraction_spec respect SSL video urls --- spec/integration/url_extraction_spec.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spec/integration/url_extraction_spec.rb b/spec/integration/url_extraction_spec.rb index 4e23178..bdbd862 100755 --- a/spec/integration/url_extraction_spec.rb +++ b/spec/integration/url_extraction_spec.rb @@ -103,7 +103,7 @@ def test_instagram end def test_facebook - result = `ruby bin/viddl-rb "https://www.facebook.com/photo.php?v=101503003357454&set=vb.310080259100701&type=2&theater --url-only"` + result = `ruby bin/viddl-rb "https://www.facebook.com/photo.php?v=101503003357454&set=vb.310080259100701&type=2&theater" --url-only` assert_equal $?, 0 can_download_test(result) { |url_output| http_code_grabber(CGI::unescape(url_output), {:method => :get}) } end @@ -137,7 +137,8 @@ def test_dailymotion_hq def can_download_test(result, &grabber) url_output = result.split("\n").last - assert_includes(CGI.unescape(url_output), 'http://') + # Assert url includes http:// or https:// + assert((CGI.unescape(url_output) =~ /https?:\/\//) != nil) code_grabber = grabber || proc { |url_output| http_code_grabber(url_output) } tries = 0 http_response_code = 0 From 54cc4b7fdaa6cfe3cc0837b275734d134f640729 Mon Sep 17 00:00:00 2001 From: Mike Bajur Date: Thu, 5 Jun 2014 14:41:34 +0200 Subject: [PATCH 6/6] Remove last line from ciphers.yml --- plugins/youtube/ciphers.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/youtube/ciphers.yml b/plugins/youtube/ciphers.yml index eadad85..d6d25fb 100644 --- a/plugins/youtube/ciphers.yml +++ b/plugins/youtube/ciphers.yml @@ -97,4 +97,3 @@ en_US-vflbJnZqE: w26 s1 w15 w3 w62 w54 w22 en_US-vflgd5txb: w26 s1 w15 w3 w62 w54 w22 en_US-vflTm330y: w26 s1 w15 w3 w62 w54 w22 en_US-vflnwMARr: s3 r w24 s2 -en_US-vflaTh7jt: w46 r s3 w19 r s2 w15