From ffc44b0cff507206880365b17cf09e308f3f18ae Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 10 Nov 2024 23:19:11 -0800 Subject: [PATCH] Add support for caching IOS client player requests When a client requests HLS streams, Invidious will first check the database to see if the cached video has any HLS streams. If not we request the IOS client and update the streamingData field with the now gotten HLS manifest data. Afterwards, we update the cached video in the database. --- src/invidious/routes/watch.cr | 2 +- src/invidious/videos.cr | 25 ++++++++++++++----- src/invidious/videos/parser.cr | 44 ++++++++++++++++++++++------------ 3 files changed, 49 insertions(+), 22 deletions(-) diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 00069545..d0fcf662 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -52,7 +52,7 @@ module Invidious::Routes::Watch env.params.query.delete_all("listen") begin - video = get_video(id, region: params.region, force_hls: (params.quality == "hls")) + video = get_video(id, region: params.region, get_hls: (params.quality == "hls")) rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") return error_template(404, ex) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index e8165f84..6787b957 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -294,8 +294,8 @@ struct Video predicate_bool upcoming, isUpcoming end -def get_video(id, refresh = true, region = nil, force_hls = false, force_refresh = false) - if (video = Invidious::Database::Videos.select(id)) && !region && !force_hls +def get_video(id, refresh = true, region = nil, get_hls = false, force_refresh = false) + if (video = Invidious::Database::Videos.select(id)) && !region # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) if (refresh && @@ -312,8 +312,21 @@ def get_video(id, refresh = true, region = nil, force_hls = false, force_refresh end end else - video = fetch_video(id, region, force_hls) - Invidious::Database::Videos.insert(video) if !region && !force_hls + video = fetch_video(id, region) + Invidious::Database::Videos.insert(video) if !region + end + + # The video object we got above could be from a previous request that was not + # done through the IOS client. If the users wants HLS we should check if + # a manifest exists in the data returned. If not we will rerequest one. + if get_hls && !video.hls_manifest_url + begin + video_with_hls_data = update_video_object_with_hls_data(id, video) + return video if !video_with_hls_data + Invidious::Database::Videos.update(video_with_hls_data) if !region + rescue ex + # Use old database video if IOS client request fails + end end return video @@ -323,8 +336,8 @@ rescue DB::Error return fetch_video(id, region) end -def fetch_video(id, region, force_hls = false) - info = extract_video_info(video_id: id, force_hls: force_hls) +def fetch_video(id, region) + info = extract_video_info(video_id: id) if reason = info["reason"]? if reason == "Video unavailable" diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 7b6605f3..1dba5440 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -50,7 +50,7 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? } end -def extract_video_info(video_id : String, force_hls : Bool = false) +def extract_video_info(video_id : String) # Init client config for the API client_config = YoutubeAPI::ClientConfig.new @@ -101,28 +101,24 @@ def extract_video_info(video_id : String, force_hls : Bool = false) params["reason"] = JSON::Any.new(reason) if reason new_player_response = nil - if force_hls - client_config.client_type = YoutubeAPI::ClientType::IOS + + # Don't use Android test suite client if po_token is passed because po_token doesn't + # work for Android test suite client. + if reason.nil? && CONFIG.po_token.nil? + # Fetch the video streams using an Android client in order to get the + # decrypted URLs and maybe fix throttling issues (#2194). See the + # following issue for an explanation about decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite new_player_response = try_fetch_streaming_data(video_id, client_config) else - # Don't use Android test suite client if po_token is passed because po_token doesn't - # work for Android test suite client. - if reason.nil? && CONFIG.po_token.nil? + if reason.nil? # Fetch the video streams using an Android client in order to get the # decrypted URLs and maybe fix throttling issues (#2194). See the # following issue for an explanation about decrypted URLs: # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite new_player_response = try_fetch_streaming_data(video_id, client_config) - else - if reason.nil? - # Fetch the video streams using an Android client in order to get the - # decrypted URLs and maybe fix throttling issues (#2194). See the - # following issue for an explanation about decrypted URLs: - # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite - new_player_response = try_fetch_streaming_data(video_id, client_config) - end end end @@ -157,6 +153,24 @@ def extract_video_info(video_id : String, force_hls : Bool = false) return params end +def update_video_object_with_hls_data(id : String, video : Video) + client_config = YoutubeAPI::ClientConfig.new(client_type: YoutubeAPI::ClientType::IOS) + + new_player_response = try_fetch_streaming_data(id, client_config) + current_streaming_data = video.info["streamingData"].try &.as_h + + return nil if !new_player_response + + if current_streaming_data && (manifest = new_player_response.dig?("streamingData", "hlsManifestUrl")) + current_streaming_data["hlsManifestUrl"] = JSON::Any.new(manifest.as_s) + video.info["streamingData"] = JSON::Any.new(current_streaming_data) + + return video + end + + return nil +end + def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)