Having the following, long running rake batch:
class SyncStarredRepo
include Mongoid::Document
def self.update
User.all.map do |user|
if user.email != "admin@gitwatcher.com"
begin
conn = Faraday.new "https://api.github.com"
i = 1
begin
resp = conn.get "/users/#{user.nickname}/starred?page=#{i}"
resp.body.each do |repo|
attempts = 0
begin
existing_repo = user.watchlists.where( html_url: "#{repo['html_url']}" ).batch_size(30)
if existing_repo.exists?
existing_repo.first.update_attributes!(
html_url: "#{repo['html_url']}",
description: "#{repo['description']}"
)
else
user.watchlists.create!(
html_url: "#{repo['html_url']}",
description: "#{repo['description']}"
)
end
rescue Moped::Errors::CursorNotFound => c
attempts += 1
puts "error message: #{c.message}"
if attempts < 3
puts "Retrying... Attempt #: #{attempts.to_s}"
sleep(3)
retry
else
puts "already retried 3 times, go to the next"
next
end
end
end
i += 1
end until resp.body.empty?
rescue TypeError => e
puts "message error (e.inspect): #{e.inspect}"
puts "go to the next user"
next
rescue
p $!, *$@
next
end
end
end
end
end
I aspected to catch Moped::Errors::CursorNotFound exception but it doesn't work.
In the crontab log I just get this message :
rake aborted!
The operation: "GET MORE"
failed with error "cursor 2509315744412059250 not found"
then the batch stops and exit, while I was aspecting the rescue messages and next loop operation.
UPDATE
I'm trying to "extend" catchable exception set to StandardError, by doing this :
rescue
p $!, *$@
next
end
... just in case, even if CursorNotFound inherit from MongoError, which inherit from StandardError, as you can see here.
UPDATE
I just updated previous simplified code, with the full SyncStarredRepo class used by rake, just in case I was missing something into the loop logic, begin/rescue/end and so on ...
UPDATE
follow the stacktrace printed by rake -t :
rake aborted!
The operation: "GET MORE"
failed with error "cursor 5781947091640256877 not found"
/var/www/gitwatcher/vendor/bundle/ruby/1.9.1/gems/moped-1.3.2/lib/moped/node.rb:210:in `get_more'
/var/www/gitwatcher/vendor/bundle/ruby/1.9.1/gems/moped-1.3.2/lib/moped/cursor.rb:44:in `get_more'
/var/www/gitwatcher/vendor/bundle/ruby/1.9.1/gems/moped-1.3.2/lib/moped/cursor.rb:29:in `each'
/var/www/gitwatcher/vendor/bundle/ruby/1.9.1/gems/moped-1.3.2/lib/moped/query.rb:77:in `each'
/var/www/gitwatcher/vendor/bundle/ruby/1.9.1/gems/moped-1.3.2/lib/moped/query.rb:77:in `each'
/var/www/gitwatcher/vendor/bundle/ruby/1.9.1/bundler/gems/mongoid-92cad16e9fbf/lib/mongoid/contextual/mongo.rb:133:in `block in each'
/var/www/gitwatcher/vendor/bundle/ruby/1.9.1/bundler/gems/mongoid-92cad16e9fbf/lib/mongoid/contextual/mongo.rb:605:in `selecting'
/var/www/gitwatcher/vendor/bundle/ruby/1.9.1/bundler/gems/mongoid-92cad16e9fbf/lib/mongoid/contextual/mongo.rb:132:in `each'
/var/www/gitwatcher/vendor/bundle/ruby/1.9.1/bundler/gems/mongoid-92cad16e9fbf/lib/mongoid/contextual.rb:18:in `each'
/var/www/gitwatcher/app/models/sync_starred_repo.rb:15:in `map'
/var/www/gitwatcher/app/models/sync_starred_repo.rb:15:in `update'
/var/www/gitwatcher/lib/tasks/scheduler.rake:9:in `block in <top (required)>'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/task.rb:205:in `call'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/task.rb:205:in `block in execute'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/task.rb:200:in `each'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/task.rb:200:in `execute'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/task.rb:158:in `block in invoke_with_call_chain'
/usr/local/rvm/rubies/ruby-1.9.3-p194/lib/ruby/1.9.1/monitor.rb:211:in `mon_synchronize'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/task.rb:151:in `invoke_with_call_chain'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/task.rb:144:in `invoke'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/application.rb:116:in `invoke_task'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/application.rb:94:in `block (2 levels) in top_level'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/application.rb:94:in `each'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/application.rb:94:in `block in top_level'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/application.rb:133:in `standard_exception_handling'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/application.rb:88:in `top_level'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/application.rb:66:in `block in run'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/application.rb:133:in `standard_exception_handling'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/lib/rake/application.rb:63:in `run'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/gems/rake-0.9.2.2/bin/rake:33:in `<top (required)>'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/bin/rake:23:in `load'
/usr/local/rvm/gems/ruby-1.9.3-p194@rails-3.2.8/bin/rake:23:in `<main>'
Tasks: TOP => update_starred_list_per_user
The error is raised during User.all.map so you have to wrap your begin..rescue block around that whole statement.
Old answer based on initial information:
You're rescuing outside of your loop, so retry doesn't do exactly what you expect (next on the other hand will raise a SyntaxError (1.9) or LocalJumpError (1.8)). In fact, retry inside a rescue just causes the code inside the begin..rescue block to run again.
The right code should look like that:
attempts = 0
begin
i = 1
resp = conn.get "/users/#{user.nickname}/starred?page=#{i}"
resp.body.each do |repo|
begin
existing_repo = user.watchlists.where( html_url: "#{repo['html_url']}" )
if existing_repo.exists?
existing_repo.first.update_attributes!(
html_url: "#{repo['html_url']}",
description: "#{repo['description']}"
)
else
user.watchlists.create!(
html_url: "#{repo['html_url']}",
description: "#{repo['description']}"
)
end
rescue Moped::Errors::CursorNotFound => c
attempts += 1
puts "error message: #{c.message}"
if attempts < 3
puts "Retrying... Attempt #: #{attempts.to_s}"
sleep(3)
retry
else
puts "already retried 3 times, go to the next"
next
end
end
end
i += 1
end until resp.body.empty?