Fix issues with tootctl's parallelization and progress reporting (#12093)
This commit is contained in:
parent
b5be067c88
commit
38b6c34e32
|
@ -15,6 +15,11 @@ module Mastodon
|
|||
end
|
||||
|
||||
def parallelize_with_progress(scope)
|
||||
if options[:concurrency] < 1
|
||||
say('Cannot run with this concurrency setting, must be at least 1', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency]
|
||||
|
||||
progress = create_progress_bar(scope.count)
|
||||
|
@ -27,11 +32,21 @@ module Mastodon
|
|||
|
||||
items.each do |item|
|
||||
futures << Concurrent::Future.execute(executor: pool) do
|
||||
ActiveRecord::Base.connection_pool.with_connection do
|
||||
begin
|
||||
if !progress.total.nil? && progress.progress + 1 > progress.total
|
||||
# The number of items has changed between start and now,
|
||||
# since there is no good way to predict the final count from
|
||||
# here, just change the progress bar to an indeterminate one
|
||||
|
||||
progress.total = nil
|
||||
end
|
||||
|
||||
progress.log("Processing #{item.id}") if options[:verbose]
|
||||
|
||||
result = yield(item)
|
||||
result = ActiveRecord::Base.connection_pool.with_connection do
|
||||
yield(item)
|
||||
end
|
||||
|
||||
aggregate.increment(result) if result.is_a?(Integer)
|
||||
rescue => e
|
||||
progress.log pastel.red("Error processing #{item.id}: #{e}")
|
||||
|
@ -40,13 +55,12 @@ module Mastodon
|
|||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
total.increment(items.size)
|
||||
futures.map(&:value)
|
||||
end
|
||||
|
||||
progress.finish
|
||||
progress.stop
|
||||
|
||||
[total.value, aggregate.value]
|
||||
end
|
||||
|
|
Reference in New Issue