Videos: Convert URL before putting result into cache (#4850)

Closes issue 4837

src/invidious/videos.cr

@@ -26,12 +26,6 @@ struct Video
   @[DB::Field(ignore: true)]
   @captions = [] of Invidious::Videos::Captions::Metadata
 
-  @[DB::Field(ignore: true)]
-  property adaptive_fmts : Array(Hash(String, JSON::Any))?
-
-  @[DB::Field(ignore: true)]
-  property fmt_stream : Array(Hash(String, JSON::Any))?
-
   @[DB::Field(ignore: true)]
   property description : String?
 
@@ -98,72 +92,24 @@ struct Video
 
   # Methods for parsing streaming data
 
-  def convert_url(fmt)
+  def fmt_stream : Array(Hash(String, JSON::Any))
-    if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
+    if formats = info.dig?("streamingData", "formats")
-      sp = cfr["sp"]
+      return formats
-      url = URI.parse(cfr["url"])
+        .as_a.map(&.as_h)
-      params = url.query_params
+        .sort_by! { |f| f["width"]?.try &.as_i || 0 }
-
-      LOGGER.debug("Videos: Decoding '#{cfr}'")
-
-      unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
-      params[sp] = unsig if unsig
     else
-      url = URI.parse(fmt["url"].as_s)
+      return [] of Hash(String, JSON::Any)
-      params = url.query_params
     end
-
-    n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
-    params["n"] = n if n
-
-    if token = CONFIG.po_token
-      params["pot"] = token
-    end
-
-    params["host"] = url.host.not_nil!
-    if region = self.info["region"]?.try &.as_s
-      params["region"] = region
-    end
-
-    url.query_params = params
-    LOGGER.trace("Videos: new url is '#{url}'")
-
-    return url.to_s
-  rescue ex
-    LOGGER.debug("Videos: Error when parsing video URL")
-    LOGGER.trace(ex.inspect_with_backtrace)
-    return ""
   end
 
-  def fmt_stream
+  def adaptive_fmts : Array(Hash(String, JSON::Any))
-    return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
+    if formats = info.dig?("streamingData", "adaptiveFormats")
-
+      return formats
-    fmt_stream = info.dig?("streamingData", "formats")
+        .as_a.map(&.as_h)
-      .try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+        .sort_by! { |f| f["width"]?.try &.as_i || 0 }
-
+    else
-    fmt_stream.each do |fmt|
+      return [] of Hash(String, JSON::Any)
-      fmt["url"] = JSON::Any.new(self.convert_url(fmt))
     end
-
-    fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
-    @fmt_stream = fmt_stream
-    return @fmt_stream.as(Array(Hash(String, JSON::Any)))
-  end
-
-  def adaptive_fmts
-    return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
-
-    fmt_stream = info.dig("streamingData", "adaptiveFormats")
-      .try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
-
-    fmt_stream.each do |fmt|
-      fmt["url"] = JSON::Any.new(self.convert_url(fmt))
-    end
-
-    fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
-    @adaptive_fmts = fmt_stream
-
-    return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
   end
 
   def video_streams

src/invidious/videos/parser.cr

@@ -142,10 +142,21 @@ def extract_video_info(video_id : String)
     params.delete("reason")
   end
 
-  {"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f|
+  {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
     params[f] = player_response[f] if player_response[f]?
   end
 
+  # Convert URLs, if those are present
+  if streaming_data = player_response["streamingData"]?
+    %w[formats adaptiveFormats].each do |key|
+      streaming_data.as_h[key]?.try &.as_a.each do |format|
+        format.as_h["url"] = JSON::Any.new(convert_url(format))
+      end
+    end
+
+    params["streamingData"] = streaming_data
+  end
+
   # Data structure version, for cache control
   params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
 
@@ -454,3 +465,35 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
 
   return params
 end
+
+private def convert_url(fmt)
+  if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
+    sp = cfr["sp"]
+    url = URI.parse(cfr["url"])
+    params = url.query_params
+
+    LOGGER.debug("convert_url: Decoding '#{cfr}'")
+
+    unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
+    params[sp] = unsig if unsig
+  else
+    url = URI.parse(fmt["url"].as_s)
+    params = url.query_params
+  end
+
+  n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
+  params["n"] = n if n
+
+  if token = CONFIG.po_token
+    params["pot"] = token
+  end
+
+  url.query_params = params
+  LOGGER.trace("convert_url: new url is '#{url}'")
+
+  return url.to_s
+rescue ex
+  LOGGER.debug("convert_url: Error when parsing video URL")
+  LOGGER.trace(ex.inspect_with_backtrace)
+  return ""
+end