Add support for caching IOS client player requests

When a client requests HLS streams, Invidious will first check the
database to see if the cached video has any HLS streams. If not
we request the IOS client and update the streamingData field with
the now gotten HLS manifest data.

Afterwards, we update the cached video in the database.

src/invidious/routes/watch.cr

@@ -52,7 +52,7 @@ module Invidious::Routes::Watch
     env.params.query.delete_all("listen")
 
     begin
-      video = get_video(id, region: params.region, force_hls: (params.quality == "hls"))
+      video = get_video(id, region: params.region, get_hls: (params.quality == "hls"))
     rescue ex : NotFoundException
       LOGGER.error("get_video not found: #{id} : #{ex.message}")
       return error_template(404, ex)

src/invidious/videos.cr

@@ -294,8 +294,8 @@ struct Video
   predicate_bool upcoming, isUpcoming
 end
 
-def get_video(id, refresh = true, region = nil, force_hls = false, force_refresh = false)
+def get_video(id, refresh = true, region = nil, get_hls = false, force_refresh = false)
-  if (video = Invidious::Database::Videos.select(id)) && !region && !force_hls
+  if (video = Invidious::Database::Videos.select(id)) && !region
     # If record was last updated over 10 minutes ago, or video has since premiered,
     # refresh (expire param in response lasts for 6 hours)
     if (refresh &&
@@ -312,8 +312,21 @@ def get_video(id, refresh = true, region = nil, force_hls = false, force_refresh
       end
     end
   else
-    video = fetch_video(id, region, force_hls)
+    video = fetch_video(id, region)
-    Invidious::Database::Videos.insert(video) if !region && !force_hls
+    Invidious::Database::Videos.insert(video) if !region
+  end
+
+  # The video object we got above could be from a previous request that was not
+  # done through the IOS client. If the users wants HLS we should check if
+  # a manifest exists in the data returned. If not we will rerequest one.
+  if get_hls && !video.hls_manifest_url
+    begin
+      video_with_hls_data = update_video_object_with_hls_data(id, video)
+      return video if !video_with_hls_data
+      Invidious::Database::Videos.update(video_with_hls_data) if !region
+    rescue ex
+      # Use old database video if IOS client request fails
+    end
   end
 
   return video
@@ -323,8 +336,8 @@ rescue DB::Error
   return fetch_video(id, region)
 end
 
-def fetch_video(id, region, force_hls = false)
+def fetch_video(id, region)
-  info = extract_video_info(video_id: id, force_hls: force_hls)
+  info = extract_video_info(video_id: id)
 
   if reason = info["reason"]?
     if reason == "Video unavailable"

src/invidious/videos/parser.cr

@@ -50,7 +50,7 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
   }
 end
 
-def extract_video_info(video_id : String, force_hls : Bool = false)
+def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
 
@@ -101,28 +101,24 @@ def extract_video_info(video_id : String, force_hls : Bool = false)
   params["reason"] = JSON::Any.new(reason) if reason
 
   new_player_response = nil
-  if force_hls
+
-    client_config.client_type = YoutubeAPI::ClientType::IOS
+  # Don't use Android test suite client if po_token is passed because po_token doesn't
+  # work for Android test suite client.
+  if reason.nil? && CONFIG.po_token.nil?
+    # Fetch the video streams using an Android client in order to get the
+    # decrypted URLs and maybe fix throttling issues (#2194). See the
+    # following issue for an explanation about decrypted URLs:
+    # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
+    client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   else
-    # Don't use Android test suite client if po_token is passed because po_token doesn't
+    if reason.nil?
-    # work for Android test suite client.
-    if reason.nil? && CONFIG.po_token.nil?
       # Fetch the video streams using an Android client in order to get the
       # decrypted URLs and maybe fix throttling issues (#2194). See the
       # following issue for an explanation about decrypted URLs:
       # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
       client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
       new_player_response = try_fetch_streaming_data(video_id, client_config)
-    else
-      if reason.nil?
-        # Fetch the video streams using an Android client in order to get the
-        # decrypted URLs and maybe fix throttling issues (#2194). See the
-        # following issue for an explanation about decrypted URLs:
-        # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
-        client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
-        new_player_response = try_fetch_streaming_data(video_id, client_config)
-      end
     end
   end
 
@@ -157,6 +153,24 @@ def extract_video_info(video_id : String, force_hls : Bool = false)
   return params
 end
 
+def update_video_object_with_hls_data(id : String, video : Video)
+  client_config = YoutubeAPI::ClientConfig.new(client_type: YoutubeAPI::ClientType::IOS)
+
+  new_player_response = try_fetch_streaming_data(id, client_config)
+  current_streaming_data = video.info["streamingData"].try &.as_h
+
+  return nil if !new_player_response
+
+  if current_streaming_data && (manifest = new_player_response.dig?("streamingData", "hlsManifestUrl"))
+    current_streaming_data["hlsManifestUrl"] = JSON::Any.new(manifest.as_s)
+    video.info["streamingData"] = JSON::Any.new(current_streaming_data)
+
+    return video
+  end
+
+  return nil
+end
+
 def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
   LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
   response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)