Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle FB truncating batch #605

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Handle FB truncating batch #605

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1f71dd4
Handle FB truncating batch
dmitchell Apr 17, 2017
18c67f5
Fix other mocks to return correct result size
dmitchell Apr 17, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions lib/koala/api/graph_batch_api.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ def execute(http_options = {})
return [] if batch_calls.empty?

batch_results = []
batch_calls.each_slice(MAX_CALLS) do |batch|
until batch_calls.empty? do
batch = batch_calls.shift(MAX_CALLS)

# Turn the call args collected into what facebook expects
args = {"batch" => batch_args(batch)}
batch.each do |call|
Expand All @@ -55,7 +57,13 @@ def execute(http_options = {})
original_api.graph_call("/", args, "post", http_options) do |response|
raise bad_response if response.nil?

batch_results += generate_results(response, batch)
# FB sometimes truncates the submission batch. dhm thinks it may be limiting create actions
# without erroring
slice_results = generate_results(response, batch)
batch_results += slice_results
if slice_results.length < batch.length
batch_calls.unshift(*batch[slice_results.length .. batch.length])
end
end
end

Expand Down
41 changes: 32 additions & 9 deletions spec/cases/graph_api_batch_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
require 'json' unless Hash.respond_to?(:to_json)

describe "Koala::Facebook::GraphAPI in batch mode" do
DEFAULT_RESPONSE = [{code: 200, headers: [{name: "Content-Type", value: "text/javascript; charset=UTF-8"}], body: "{\"id\":\"1234\"}"}]

before :each do
@api = Koala::Facebook::API.new(@token)
Expand Down Expand Up @@ -270,15 +271,15 @@

describe "GraphAPI batch interface" do
it "returns nothing for a batch operation" do
allow(Koala).to receive(:make_request).and_return(Koala::HTTPService::Response.new(200, "[]", {}))
allow(Koala).to receive(:make_request).and_return(Koala::HTTPService::Response.new(200, DEFAULT_RESPONSE.to_json, {}))
@api.batch do |batch_api|
expect(batch_api.get_object('me')).to be_nil
end
end

describe "#batch" do
before :each do
@fake_response = Koala::HTTPService::Response.new(200, "[]", {})
@fake_response = Koala::HTTPService::Response.new(200, DEFAULT_RESPONSE.to_json, {})
allow(Koala).to receive(:make_request).and_return(@fake_response)
end

Expand All @@ -296,7 +297,11 @@

it "includes the first operation's access token as the main one in the args" do
access_token = "foo"
expect(Koala).to receive(:make_request).with(anything, hash_including("access_token" => access_token), anything, anything).and_return(@fake_response)
expect(Koala).to receive(:make_request).with(anything, hash_including("access_token" => access_token), anything, anything) do |_request, args, _verb, _options|
request_count = JSON.parse(args['batch']).length
Koala::HTTPService::Response.new(200, (DEFAULT_RESPONSE * request_count).to_json, {})
end

Koala::Facebook::API.new(access_token).batch do |batch_api|
batch_api.get_object('me')
batch_api.get_object('me', {}, {'access_token' => 'bar'})
Expand All @@ -308,7 +313,11 @@
access_token = "foo"
app_secret = "baz"
app_secret_digest = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new("sha256"), app_secret, access_token)
expect(Koala).to receive(:make_request).with(anything, hash_including("access_token" => access_token, "appsecret_proof" => app_secret_digest), anything, anything).and_return(@fake_response)
expect(Koala).to receive(:make_request).with(anything, hash_including("access_token" => access_token, "appsecret_proof" => app_secret_digest), anything, anything)do |_request, args, _verb, _options|
request_count = JSON.parse(args['batch']).length
Koala::HTTPService::Response.new(200, (DEFAULT_RESPONSE * request_count).to_json, {})
end

Koala::Facebook::API.new(access_token, app_secret).batch do |batch_api|
batch_api.get_object('me')
batch_api.get_object('me', {}, {'access_token' => 'bar'})
Expand All @@ -324,7 +333,11 @@

# two requests should generate two batch operations
expected = JSON.dump([op.to_batch_params(access_token, nil), op.to_batch_params(access_token, nil)])
expect(Koala).to receive(:make_request).with(anything, hash_including("batch" => expected), anything, anything).and_return(@fake_response)
expect(Koala).to receive(:make_request).with(anything, hash_including("batch" => expected), anything, anything) do |_request, args, _verb, _options|
request_count = JSON.parse(args['batch']).length
Koala::HTTPService::Response.new(200, (DEFAULT_RESPONSE * request_count).to_json, {})
end

Koala::Facebook::API.new(access_token).batch do |batch_api|
batch_api.get_object('me')
batch_api.get_object('me')
Expand All @@ -338,6 +351,8 @@
@key = "file0_0"
@uploadable_io = double("UploadableIO")
batch_op = double("Koala Batch Operation", :files => {@key => @uploadable_io}, :to_batch_params => {}, :access_token => "foo")
allow(batch_op).to receive(:post_processing).and_return(nil)
allow(batch_op).to receive(:http_options).and_return({})
allow(Koala::Facebook::GraphBatchAPI::BatchOperation).to receive(:new).and_return(batch_op)

expect(Koala).to receive(:make_request).with(anything, hash_including(@key => @uploadable_io), anything, anything).and_return(@fake_response)
Expand All @@ -352,7 +367,8 @@
expect(Koala).to receive(:make_request) do |url, args, method, options|
# test the batch operations to make sure they appear in the right order
expect((args ||= {})["batch"]).to match(/.*me\/farglebarg.*otheruser\/bababa/)
@fake_response
request_count = JSON.parse(args['batch']).length
Koala::HTTPService::Response.new(200, (DEFAULT_RESPONSE * request_count).to_json, {})
end
Koala::Facebook::API.new(access_token).batch do |batch_api|
batch_api.get_connections('me', "farglebarg")
Expand Down Expand Up @@ -447,7 +463,10 @@
first_count = 20
second_count = 10

allow(Koala).to receive(:make_request).and_return(@fake_response)
allow(Koala).to receive(:make_request) do |_request, args, _verb, _options|
request_count = JSON.parse(args['batch']).length
Koala::HTTPService::Response.new(200, (DEFAULT_RESPONSE * request_count).to_json, {})
end

thread1 = Thread.new do
@api.batch do |batch_api|
Expand All @@ -474,12 +493,16 @@
end

describe '#big_batches' do
before :all do
@random = Random.new
end
before :each do
payload = [{code: 200, headers: [{name: "Content-Type", value: "text/javascript; charset=UTF-8"}], body: "{\"id\":\"1234\"}"}]
allow(Koala).to receive(:make_request) do |_request, args, _verb, _options|
request_count = JSON.parse(args['batch']).length
expect(request_count).to be <= 50 # check FB's limit
Koala::HTTPService::Response.new(200, (payload * request_count).to_json, {})
# simulate FB's result truncation
response_count = request_count > 35 ? request_count - @random.rand(15) : request_count
Koala::HTTPService::Response.new(200, (DEFAULT_RESPONSE * response_count).to_json, {})
end
end

Expand Down