From c2c224b16f9ee11e5bebeed8901aedbc0e15bfae Mon Sep 17 00:00:00 2001
From: Omar Roth <omarroth@hotmail.com>
Date: Wed, 2 Jan 2019 19:28:01 -0600
Subject: [PATCH] Use fibers to try to speed up importing of channels

---
 src/invidious.cr          | 37 +++++--------------------------------
 src/invidious/channels.cr | 27 +++++++++++++++++++++++++++
 src/invidious/users.cr    | 17 +++++++----------
 3 files changed, 39 insertions(+), 42 deletions(-)

diff --git a/src/invidious.cr b/src/invidious.cr
index 58dcd81f..4be9ee7e 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -1355,7 +1355,7 @@ get "/subscription_manager" do |env|
   subscriptions = [] of InvidiousChannel
   user.subscriptions.each do |ucid|
     begin
-      subscriptions << get_channel(ucid, PG_DB, false)
+      subscriptions << get_channel(ucid, PG_DB, false, false)
     rescue ex
       next
     end
@@ -1475,14 +1475,7 @@ post "/data_control" do |env|
         end
         user.subscriptions.uniq!
 
-        user.subscriptions.select! do |ucid|
-          begin
-            get_channel(ucid, PG_DB, false, false)
-            true
-          rescue ex
-            false
-          end
-        end
+        user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
 
         PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
       when "import_freetube"
@@ -1491,14 +1484,7 @@ post "/data_control" do |env|
         end
         user.subscriptions.uniq!
 
-        user.subscriptions.select! do |ucid|
-          begin
-            get_channel(ucid, PG_DB, false, false)
-            true
-          rescue ex
-            false
-          end
-        end
+        user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
 
         PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
       when "import_newpipe_subscriptions"
@@ -1508,13 +1494,7 @@ post "/data_control" do |env|
         end
         user.subscriptions.uniq!
 
-        user.subscriptions.each do |ucid|
-          begin
-            get_channel(ucid, PG_DB, false, false)
-          rescue ex
-            next
-          end
-        end
+        user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
 
         PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
       when "import_newpipe"
@@ -1533,14 +1513,7 @@ post "/data_control" do |env|
               user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map { |url| url.lchop("https://www.youtube.com/channel/") }
               user.subscriptions.uniq!
 
-              user.subscriptions.select! do |ucid|
-                begin
-                  get_channel(ucid, PG_DB, false, false)
-                  true
-                rescue ex
-                  false
-                end
-              end
+              user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
 
               PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
 
diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr
index ba17ff07..c1915a10 100644
--- a/src/invidious/channels.cr
+++ b/src/invidious/channels.cr
@@ -21,6 +21,33 @@ class ChannelVideo
   })
 end
 
+def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10)
+  active_threads = 0
+  active_channel = Channel(String | Nil).new
+
+  final = [] of String
+  channels.map do |ucid|
+    if active_threads >= max_threads
+      if response = active_channel.receive
+        active_threads -= 1
+        final << response
+      end
+    end
+
+    active_threads += 1
+    spawn do
+      begin
+        get_channel(ucid, db, refresh, pull_all_videos)
+        active_channel.send(ucid)
+      rescue ex
+        active_channel.send(nil)
+      end
+    end
+  end
+
+  return final
+end
+
 def get_channel(id, db, refresh = true, pull_all_videos = true)
   client = make_client(YT_URL)
 
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index 38799a74..dbbe87f8 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -177,19 +177,16 @@ def fetch_user(sid, headers, db)
   feed = XML.parse_html(feed.body)
 
   channels = [] of String
-  feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).each do |channel|
-    if !{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
-      channel_id = channel["href"].lstrip("/channel/")
-
-      begin
-        channel = get_channel(channel_id, db, false, false)
-        channels << channel.id
-      rescue ex
-        next
-      end
+  channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel|
+    if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
+      nil
+    else
+      channel["href"].lstrip("/channel/")
     end
   end
 
+  channels = get_batch_channels(channels, db, false, false)
+
   email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
   if email
     email = email.content.strip