Fix batch importing of channels

This commit is contained in:
Omar Roth 2019-04-04 14:49:32 -05:00
parent 305d636217
commit c728214af7
1 changed files with 26 additions and 15 deletions

View File

@ -23,28 +23,39 @@ struct ChannelVideo
end end
def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10)
active_threads = 0 finished_channel = Channel(String | Nil).new
active_channel = Channel(String | Nil).new
final = [] of String spawn do
channels.map do |ucid| active_threads = 0
active_channel = Channel(Nil).new
channels.each do |ucid|
if active_threads >= max_threads if active_threads >= max_threads
if response = active_channel.receive active_channel.receive
active_threads -= 1 active_threads -= 1
final << response
end
end end
active_threads += 1 active_threads += 1
spawn do spawn do
begin begin
get_channel(ucid, db, refresh, pull_all_videos) get_channel(ucid, db, refresh, pull_all_videos)
active_channel.send(ucid) finished_channel.send(ucid)
rescue ex rescue ex
finished_channel.send(nil)
ensure
active_channel.send(nil) active_channel.send(nil)
end end
end end
end end
end
final = [] of String
channels.size.times do
ucid = finished_channel.receive
if ucid
final << ucid
end
end
return final return final
end end