mirror of https://github.com/iv-org/invidious.git
Channels: Fix "Youtube API returned error 400" (#5059)
This PR also adds sort option to the channel "shorts" tab. Thanks to iBicha for the original fix of the "livestreams" tab. Closes 4029, 5021 and 5029
This commit is contained in:
commit
1480e0089f
|
@ -1,78 +1,3 @@
|
|||
def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
|
||||
object_inner_2 = {
|
||||
"2:0:embedded" => {
|
||||
"1:0:varint" => 0_i64,
|
||||
},
|
||||
"5:varint" => 50_i64,
|
||||
"6:varint" => 1_i64,
|
||||
"7:varint" => (page * 30).to_i64,
|
||||
"9:varint" => 1_i64,
|
||||
"10:varint" => 0_i64,
|
||||
}
|
||||
|
||||
object_inner_2_encoded = object_inner_2
|
||||
.try { |i| Protodec::Any.cast_json(i) }
|
||||
.try { |i| Protodec::Any.from_json(i) }
|
||||
.try { |i| Base64.urlsafe_encode(i) }
|
||||
.try { |i| URI.encode_www_form(i) }
|
||||
|
||||
content_type_numerical =
|
||||
case content_type
|
||||
when "videos" then 15
|
||||
when "livestreams" then 14
|
||||
else 15 # Fallback to "videos"
|
||||
end
|
||||
|
||||
sort_by_numerical =
|
||||
case sort_by
|
||||
when "newest" then 1_i64
|
||||
when "popular" then 2_i64
|
||||
when "oldest" then 4_i64
|
||||
else 1_i64 # Fallback to "newest"
|
||||
end
|
||||
|
||||
object_inner_1 = {
|
||||
"110:embedded" => {
|
||||
"3:embedded" => {
|
||||
"#{content_type_numerical}:embedded" => {
|
||||
"1:embedded" => {
|
||||
"1:string" => object_inner_2_encoded,
|
||||
},
|
||||
"2:embedded" => {
|
||||
"1:string" => "00000000-0000-0000-0000-000000000000",
|
||||
},
|
||||
"3:varint" => sort_by_numerical,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
object_inner_1_encoded = object_inner_1
|
||||
.try { |i| Protodec::Any.cast_json(i) }
|
||||
.try { |i| Protodec::Any.from_json(i) }
|
||||
.try { |i| Base64.urlsafe_encode(i) }
|
||||
.try { |i| URI.encode_www_form(i) }
|
||||
|
||||
object = {
|
||||
"80226972:embedded" => {
|
||||
"2:string" => ucid,
|
||||
"3:string" => object_inner_1_encoded,
|
||||
"35:string" => "browse-feed#{ucid}videos102",
|
||||
},
|
||||
}
|
||||
|
||||
continuation = object.try { |i| Protodec::Any.cast_json(i) }
|
||||
.try { |i| Protodec::Any.from_json(i) }
|
||||
.try { |i| Base64.urlsafe_encode(i) }
|
||||
.try { |i| URI.encode_www_form(i) }
|
||||
|
||||
return continuation
|
||||
end
|
||||
|
||||
def make_initial_content_ctoken(ucid, content_type, sort_by) : String
|
||||
return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by)
|
||||
end
|
||||
|
||||
module Invidious::Channel::Tabs
|
||||
extend self
|
||||
|
||||
|
@ -101,7 +26,7 @@ module Invidious::Channel::Tabs
|
|||
end
|
||||
|
||||
def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
|
||||
continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by)
|
||||
continuation ||= make_initial_videos_ctoken(ucid, sort_by)
|
||||
initial_data = YoutubeAPI.browse(continuation: continuation)
|
||||
|
||||
return extract_items(initial_data, author, ucid)
|
||||
|
@ -130,14 +55,10 @@ module Invidious::Channel::Tabs
|
|||
# Shorts
|
||||
# -------------------
|
||||
|
||||
def get_shorts(channel : AboutChannel, continuation : String? = nil)
|
||||
if continuation.nil?
|
||||
# EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
|
||||
# TODO: try to extract the continuation tokens that allows other sorting options
|
||||
initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
|
||||
else
|
||||
initial_data = YoutubeAPI.browse(continuation: continuation)
|
||||
end
|
||||
def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
|
||||
continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by)
|
||||
initial_data = YoutubeAPI.browse(continuation: continuation)
|
||||
|
||||
return extract_items(initial_data, channel.author, channel.ucid)
|
||||
end
|
||||
|
||||
|
@ -145,9 +66,8 @@ module Invidious::Channel::Tabs
|
|||
# Livestreams
|
||||
# -------------------
|
||||
|
||||
def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest")
|
||||
continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by)
|
||||
|
||||
def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
|
||||
continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by)
|
||||
initial_data = YoutubeAPI.browse(continuation: continuation)
|
||||
|
||||
return extract_items(initial_data, channel.author, channel.ucid)
|
||||
|
@ -171,4 +91,102 @@ module Invidious::Channel::Tabs
|
|||
|
||||
return items, next_continuation
|
||||
end
|
||||
|
||||
# -------------------
|
||||
# C-tokens
|
||||
# -------------------
|
||||
|
||||
private def sort_options_videos_short(sort_by : String)
|
||||
case sort_by
|
||||
when "newest" then return 4_i64
|
||||
when "popular" then return 2_i64
|
||||
when "oldest" then return 5_i64
|
||||
else return 4_i64 # Fallback to "newest"
|
||||
end
|
||||
end
|
||||
|
||||
# Generate the initial "continuation token" to get the first page of the
|
||||
# "videos" tab. The following page requires the ctoken provided in that
|
||||
# first page, and so on.
|
||||
private def make_initial_videos_ctoken(ucid : String, sort_by = "newest")
|
||||
object = {
|
||||
"15:embedded" => {
|
||||
"2:embedded" => {
|
||||
"1:string" => "00000000-0000-0000-0000-000000000000",
|
||||
},
|
||||
"4:varint" => sort_options_videos_short(sort_by),
|
||||
},
|
||||
}
|
||||
|
||||
return channel_ctoken_wrap(ucid, object)
|
||||
end
|
||||
|
||||
# Generate the initial "continuation token" to get the first page of the
|
||||
# "shorts" tab. The following page requires the ctoken provided in that
|
||||
# first page, and so on.
|
||||
private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest")
|
||||
object = {
|
||||
"10:embedded" => {
|
||||
"2:embedded" => {
|
||||
"1:string" => "00000000-0000-0000-0000-000000000000",
|
||||
},
|
||||
"4:varint" => sort_options_videos_short(sort_by),
|
||||
},
|
||||
}
|
||||
|
||||
return channel_ctoken_wrap(ucid, object)
|
||||
end
|
||||
|
||||
# Generate the initial "continuation token" to get the first page of the
|
||||
# "livestreams" tab. The following page requires the ctoken provided in that
|
||||
# first page, and so on.
|
||||
private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest")
|
||||
sort_by_numerical =
|
||||
case sort_by
|
||||
when "newest" then 12_i64
|
||||
when "popular" then 14_i64
|
||||
when "oldest" then 13_i64
|
||||
else 12_i64 # Fallback to "newest"
|
||||
end
|
||||
|
||||
object = {
|
||||
"14:embedded" => {
|
||||
"2:embedded" => {
|
||||
"1:string" => "00000000-0000-0000-0000-000000000000",
|
||||
},
|
||||
"5:varint" => sort_by_numerical,
|
||||
},
|
||||
}
|
||||
|
||||
return channel_ctoken_wrap(ucid, object)
|
||||
end
|
||||
|
||||
# The protobuf structure common between videos/shorts/livestreams
|
||||
private def channel_ctoken_wrap(ucid : String, object)
|
||||
object_inner = {
|
||||
"110:embedded" => {
|
||||
"3:embedded" => object,
|
||||
},
|
||||
}
|
||||
|
||||
object_inner_encoded = object_inner
|
||||
.try { |i| Protodec::Any.cast_json(i) }
|
||||
.try { |i| Protodec::Any.from_json(i) }
|
||||
.try { |i| Base64.urlsafe_encode(i) }
|
||||
.try { |i| URI.encode_www_form(i) }
|
||||
|
||||
object = {
|
||||
"80226972:embedded" => {
|
||||
"2:string" => ucid,
|
||||
"3:string" => object_inner_encoded,
|
||||
},
|
||||
}
|
||||
|
||||
continuation = object.try { |i| Protodec::Any.cast_json(i) }
|
||||
.try { |i| Protodec::Any.from_json(i) }
|
||||
.try { |i| Base64.urlsafe_encode(i) }
|
||||
.try { |i| URI.encode_www_form(i) }
|
||||
|
||||
return continuation
|
||||
end
|
||||
end
|
||||
|
|
|
@ -20,10 +20,11 @@ module Invidious::Routes::Channels
|
|||
sort_by = env.params.query["sort_by"]?.try &.downcase
|
||||
|
||||
if channel.auto_generated
|
||||
sort_by ||= "last"
|
||||
sort_options = {"last", "oldest", "newest"}
|
||||
|
||||
items, next_continuation = fetch_channel_playlists(
|
||||
channel.ucid, channel.author, continuation, (sort_by || "last")
|
||||
channel.ucid, channel.author, continuation, sort_by
|
||||
)
|
||||
|
||||
items.uniq! do |item|
|
||||
|
@ -49,9 +50,11 @@ module Invidious::Routes::Channels
|
|||
end
|
||||
next_continuation = nil
|
||||
else
|
||||
sort_by ||= "newest"
|
||||
sort_options = {"newest", "oldest", "popular"}
|
||||
items, next_continuation = Channel::Tabs.get_videos(
|
||||
channel, continuation: continuation, sort_by: (sort_by || "newest")
|
||||
|
||||
items, next_continuation = Channel::Tabs.get_60_videos(
|
||||
channel, continuation: continuation, sort_by: sort_by
|
||||
)
|
||||
end
|
||||
end
|
||||
|
@ -82,13 +85,12 @@ module Invidious::Routes::Channels
|
|||
end
|
||||
next_continuation = nil
|
||||
else
|
||||
# TODO: support sort option for shorts
|
||||
sort_by = ""
|
||||
sort_options = [] of String
|
||||
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
|
||||
sort_options = {"newest", "oldest", "popular"}
|
||||
|
||||
# Fetch items and continuation token
|
||||
items, next_continuation = Channel::Tabs.get_shorts(
|
||||
channel, continuation: continuation
|
||||
channel, continuation: continuation, sort_by: sort_by
|
||||
)
|
||||
end
|
||||
|
||||
|
|
|
@ -21,6 +21,7 @@ private ITEM_PARSERS = {
|
|||
Parsers::ItemSectionRendererParser,
|
||||
Parsers::ContinuationItemRendererParser,
|
||||
Parsers::HashtagRendererParser,
|
||||
Parsers::LockupViewModelParser,
|
||||
}
|
||||
|
||||
private alias InitialData = Hash(String, JSON::Any)
|
||||
|
@ -467,9 +468,9 @@ private module Parsers
|
|||
# Parses an InnerTube richItemRenderer into a SearchVideo.
|
||||
# Returns nil when the given object isn't a RichItemRenderer
|
||||
#
|
||||
# A richItemRenderer seems to be a simple wrapper for a videoRenderer, used
|
||||
# by the result page for hashtags and for the podcast tab on channels.
|
||||
# It is located inside a continuationItems container for hashtags.
|
||||
# A richItemRenderer seems to be a simple wrapper for a various other types,
|
||||
# used on the hashtags result page and the channel podcast tab. It is located
|
||||
# itself inside a richGridRenderer container.
|
||||
#
|
||||
module RichItemRendererParser
|
||||
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
|
||||
|
@ -482,6 +483,8 @@ private module Parsers
|
|||
child = VideoRendererParser.process(item_contents, author_fallback)
|
||||
child ||= ReelItemRendererParser.process(item_contents, author_fallback)
|
||||
child ||= PlaylistRendererParser.process(item_contents, author_fallback)
|
||||
child ||= LockupViewModelParser.process(item_contents, author_fallback)
|
||||
child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback)
|
||||
return child
|
||||
end
|
||||
|
||||
|
@ -496,6 +499,9 @@ private module Parsers
|
|||
# reelItemRenderer items are used in the new (2022) channel layout,
|
||||
# in the "shorts" tab.
|
||||
#
|
||||
# NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel
|
||||
# TODO: Confirm that hypothesis
|
||||
#
|
||||
module ReelItemRendererParser
|
||||
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
|
||||
if item_contents = item["reelItemRenderer"]?
|
||||
|
@ -582,6 +588,135 @@ private module Parsers
|
|||
end
|
||||
end
|
||||
|
||||
# Parses an InnerTube lockupViewModel into a SearchPlaylist.
|
||||
# Returns nil when the given object is not a lockupViewModel.
|
||||
#
|
||||
# This structure is present since November 2024 on the "podcasts" and
|
||||
# "playlists" tabs of the channel page. It is usually encapsulated in either
|
||||
# a richItemRenderer or a richGridRenderer.
|
||||
#
|
||||
module LockupViewModelParser
|
||||
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
|
||||
if item_contents = item["lockupViewModel"]?
|
||||
return self.parse(item_contents, author_fallback)
|
||||
end
|
||||
end
|
||||
|
||||
private def self.parse(item_contents, author_fallback)
|
||||
playlist_id = item_contents["contentId"].as_s
|
||||
|
||||
thumbnail_view_model = item_contents.dig(
|
||||
"contentImage", "collectionThumbnailViewModel",
|
||||
"primaryThumbnail", "thumbnailViewModel"
|
||||
)
|
||||
|
||||
thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s
|
||||
|
||||
# This complicated sequences tries to extract the following data structure:
|
||||
# "overlays": [{
|
||||
# "thumbnailOverlayBadgeViewModel": {
|
||||
# "thumbnailBadges": [{
|
||||
# "thumbnailBadgeViewModel": {
|
||||
# "text": "430 episodes",
|
||||
# "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT"
|
||||
# }
|
||||
# }]
|
||||
# }
|
||||
# }]
|
||||
#
|
||||
# NOTE: this simplistic `.to_i` conversion might not work on larger
|
||||
# playlists and hasn't been tested.
|
||||
video_count = thumbnail_view_model.dig("overlays").as_a
|
||||
.compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a)
|
||||
.flatten
|
||||
.find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node|
|
||||
{"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) }
|
||||
})
|
||||
.try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false)
|
||||
|
||||
metadata = item_contents.dig("metadata", "lockupMetadataViewModel")
|
||||
title = metadata.dig("title", "content").as_s
|
||||
|
||||
# TODO: Retrieve "updated" info from metadata parts
|
||||
# rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a
|
||||
# parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s)
|
||||
# One of these parts should contain a string like: "Updated 2 days ago"
|
||||
|
||||
# TODO: Maybe add a button to access the first video of the playlist?
|
||||
# item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint")
|
||||
# Available fields: "videoId", "playlistId", "params"
|
||||
|
||||
return SearchPlaylist.new({
|
||||
title: title,
|
||||
id: playlist_id,
|
||||
author: author_fallback.name,
|
||||
ucid: author_fallback.id,
|
||||
video_count: video_count || -1,
|
||||
videos: [] of SearchPlaylistVideo,
|
||||
thumbnail: thumbnail,
|
||||
author_verified: false,
|
||||
})
|
||||
end
|
||||
|
||||
def self.parser_name
|
||||
return {{@type.name}}
|
||||
end
|
||||
end
|
||||
|
||||
# Parses an InnerTube shortsLockupViewModel into a SearchVideo.
|
||||
# Returns nil when the given object is not a shortsLockupViewModel.
|
||||
#
|
||||
# This structure is present since around October 2024 on the "shorts" tab of
|
||||
# the channel page and likely replaces the reelItemRenderer structure. It is
|
||||
# usually (always?) encapsulated in a richItemRenderer.
|
||||
#
|
||||
module ShortsLockupViewModelParser
|
||||
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
|
||||
if item_contents = item["shortsLockupViewModel"]?
|
||||
return self.parse(item_contents, author_fallback)
|
||||
end
|
||||
end
|
||||
|
||||
private def self.parse(item_contents, author_fallback)
|
||||
# TODO: Maybe add support for "oardefault.jpg" thumbnails?
|
||||
# thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s
|
||||
# Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?...
|
||||
|
||||
video_id = item_contents.dig(
|
||||
"onTap", "innertubeCommand", "reelWatchEndpoint", "videoId"
|
||||
).as_s
|
||||
|
||||
title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s
|
||||
|
||||
view_count = short_text_to_number(
|
||||
item_contents.dig("overlayMetadata", "secondaryText", "content").as_s
|
||||
)
|
||||
|
||||
# Approximate to one minute, as "shorts" generally don't exceed that.
|
||||
# NOTE: The actual duration is not provided by Youtube anymore.
|
||||
# TODO: Maybe use -1 as an error value and handle that on the frontend?
|
||||
duration = 60_i32
|
||||
|
||||
SearchVideo.new({
|
||||
title: title,
|
||||
id: video_id,
|
||||
author: author_fallback.name,
|
||||
ucid: author_fallback.id,
|
||||
published: Time.unix(0),
|
||||
views: view_count,
|
||||
description_html: "",
|
||||
length_seconds: duration,
|
||||
premiere_timestamp: Time.unix(0),
|
||||
author_verified: false,
|
||||
badges: VideoBadges::None,
|
||||
})
|
||||
end
|
||||
|
||||
def self.parser_name
|
||||
return {{@type.name}}
|
||||
end
|
||||
end
|
||||
|
||||
# Parses an InnerTube continuationItemRenderer into a Continuation.
|
||||
# Returns nil when the given object isn't a continuationItemRenderer.
|
||||
#
|
||||
|
|
Loading…
Reference in New Issue