From 49b9316b9f2e9ccc6921a2f293abacb37f9805f6 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 13 Sep 2023 23:40:20 +0200 Subject: [PATCH 01/51] Routing: Handle current and future routes more nicely --- src/invidious/routes/channels.cr | 19 +++++++++++++---- src/invidious/routing.cr | 36 +++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 9892ae2a..5500672f 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -217,6 +217,11 @@ module Invidious::Routes::Channels env.redirect "/channel/#{ucid}" end + private KNOWN_TABS = { + "home", "videos", "shorts", "streams", "podcasts", + "releases", "playlists", "community", "channels", "about", + } + # Redirects brand url channels to a normal /channel/:ucid route def self.brand_redirect(env) locale = env.get("preferences").as(Preferences).locale @@ -227,7 +232,10 @@ module Invidious::Routes::Channels yt_url_params = URI::Params.encode(env.params.query.to_h.select(["a", "u", "user"])) # Retrieves URL params that only Invidious uses - invidious_url_params = URI::Params.encode(env.params.query.to_h.select!(["a", "u", "user"])) + invidious_url_params = env.params.query.dup + invidious_url_params.delete_all("a") + invidious_url_params.delete_all("u") + invidious_url_params.delete_all("user") begin resolved_url = YoutubeAPI.resolve_url("https://youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}") @@ -236,14 +244,17 @@ module Invidious::Routes::Channels return error_template(404, translate(locale, "This channel does not exist.")) end - selected_tab = env.request.path.split("/")[-1] - if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab + selected_tab = env.params.url["tab"]? + + if KNOWN_TABS.includes? selected_tab url = "/channel/#{ucid}/#{selected_tab}" else url = "/channel/#{ucid}" end - env.redirect url + url += "?#{invidious_url_params}" if !invidious_url_params.empty? + + return env.redirect url end # Handles redirects for the /profile endpoint diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 9c43171c..5ec7fae3 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -124,22 +124,34 @@ module Invidious::Routing get "/channel/:ucid/community", Routes::Channels, :community get "/channel/:ucid/channels", Routes::Channels, :channels get "/channel/:ucid/about", Routes::Channels, :about + get "/channel/:ucid/live", Routes::Channels, :live get "/user/:user/live", Routes::Channels, :live get "/c/:user/live", Routes::Channels, :live - {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path| - # /c/LinusTechTips - get "/c/:user#{path}", Routes::Channels, :brand_redirect - # /user/linustechtips | Not always the same as /c/ - get "/user/:user#{path}", Routes::Channels, :brand_redirect - # /@LinusTechTips | Handle - get "/@:user#{path}", Routes::Channels, :brand_redirect - # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow - get "/attribution_link#{path}", Routes::Channels, :brand_redirect - # /profile?user=linustechtips - get "/profile/#{path}", Routes::Channels, :profile - end + # Channel catch-all, to redirect future routes to the channel's home + # NOTE: defined last in order to be processed after the other routes + get "/channel/:ucid/*", Routes::Channels, :home + + # /c/LinusTechTips + get "/c/:user", Routes::Channels, :brand_redirect + get "/c/:user/:tab", Routes::Channels, :brand_redirect + + # /user/linustechtips (Not always the same as /c/) + get "/user/:user", Routes::Channels, :brand_redirect + get "/user/:user/:tab", Routes::Channels, :brand_redirect + + # /@LinusTechTips (Handle) + get "/@:user", Routes::Channels, :brand_redirect + get "/@:user/:tab", Routes::Channels, :brand_redirect + + # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow + get "/attribution_link", Routes::Channels, :brand_redirect + get "/attribution_link/:tab", Routes::Channels, :brand_redirect + + # /profile?user=linustechtips + get "/profile", Routes::Channels, :profile + get "/profile/*", Routes::Channels, :profile end def register_watch_routes From 2425c47882feaa56a69f6ba842cf1cb9d5b450e0 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 13 Sep 2023 23:41:31 +0200 Subject: [PATCH 02/51] Routing: Add support for the '/live/' route --- src/invidious/routing.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 5ec7fae3..f6b3aaa6 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -158,6 +158,7 @@ module Invidious::Routing get "/watch", Routes::Watch, :handle post "/watch_ajax", Routes::Watch, :mark_watched get "/watch/:id", Routes::Watch, :redirect + get "/live/:id", Routes::Watch, :redirect get "/shorts/:id", Routes::Watch, :redirect get "/clip/:clip", Routes::Watch, :clip get "/w/:id", Routes::Watch, :redirect From ebee973b2454f3e1adfab798ed8be6b9ef123069 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 16 Sep 2023 00:54:14 +0200 Subject: [PATCH 03/51] Routes: Redirect unknown channel tabs to channel home page --- src/invidious/routes/channels.cr | 6 ++++++ src/invidious/routing.cr | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 5500672f..d744712d 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -1,6 +1,12 @@ {% skip_file if flag?(:api_only) %} module Invidious::Routes::Channels + # Redirection for unsupported routes ("tabs") + def self.redirect_home(env) + ucid = env.params.url["ucid"] + return env.redirect "/channel/#{URI.encode_www_form(ucid)}" + end + def self.home(env) self.videos(env) end diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index f6b3aaa6..8c38aaed 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -131,7 +131,7 @@ module Invidious::Routing # Channel catch-all, to redirect future routes to the channel's home # NOTE: defined last in order to be processed after the other routes - get "/channel/:ucid/*", Routes::Channels, :home + get "/channel/:ucid/*", Routes::Channels, :redirect_home # /c/LinusTechTips get "/c/:user", Routes::Channels, :brand_redirect From e3c365f3d68985a8e3ce35f76d334c7184a548de Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Wed, 19 Jul 2023 12:35:22 -0700 Subject: [PATCH 04/51] Add support for post page --- assets/css/default.css | 2 +- assets/js/comments.js | 174 +++++++++++++++++++++ assets/js/post.js | 3 + assets/js/watch.js | 157 ------------------- src/invidious/channels/community.cr | 39 ++++- src/invidious/comments/youtube.cr | 51 +++++- src/invidious/frontend/comments_youtube.cr | 18 +++ src/invidious/routes/api/v1/channels.cr | 47 ++++++ src/invidious/routes/api/v1/misc.cr | 10 +- src/invidious/routes/channels.cr | 37 +++++ src/invidious/routing.cr | 6 + src/invidious/views/community.ecr | 2 +- src/invidious/views/post.ecr | 31 ++++ src/invidious/views/watch.ecr | 6 +- 14 files changed, 416 insertions(+), 167 deletions(-) create mode 100644 assets/js/comments.js create mode 100644 assets/js/post.js create mode 100644 src/invidious/views/post.ecr diff --git a/assets/css/default.css b/assets/css/default.css index c31b24e5..69fe8d5f 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -392,7 +392,7 @@ p.video-data { margin: 0; font-weight: bold; font-size: 80%; } * Comments & community posts */ -#comments { +.comments { max-width: 800px; margin: auto; } diff --git a/assets/js/comments.js b/assets/js/comments.js new file mode 100644 index 00000000..00a8cae9 --- /dev/null +++ b/assets/js/comments.js @@ -0,0 +1,174 @@ +var video_data = JSON.parse(document.getElementById('video_data').textContent); + +var spinnerHTML = '

'; +var spinnerHTMLwithHR = spinnerHTML + '
'; + +String.prototype.supplant = function (o) { + return this.replace(/{([^{}]*)}/g, function (a, b) { + var r = o[b]; + return typeof r === 'string' || typeof r === 'number' ? r : a; + }); +}; + +function toggle_comments(event) { + var target = event.target; + var body = target.parentNode.parentNode.parentNode.children[1]; + if (body.style.display === 'none') { + target.textContent = '[ − ]'; + body.style.display = ''; + } else { + target.textContent = '[ + ]'; + body.style.display = 'none'; + } +} + +function hide_youtube_replies(event) { + var target = event.target; + + var sub_text = target.getAttribute('data-inner-text'); + var inner_text = target.getAttribute('data-sub-text'); + + var body = target.parentNode.parentNode.children[1]; + body.style.display = 'none'; + + target.textContent = sub_text; + target.onclick = show_youtube_replies; + target.setAttribute('data-inner-text', inner_text); + target.setAttribute('data-sub-text', sub_text); +} + +function show_youtube_replies(event) { + var target = event.target; + + var sub_text = target.getAttribute('data-inner-text'); + var inner_text = target.getAttribute('data-sub-text'); + + var body = target.parentNode.parentNode.children[1]; + body.style.display = ''; + + target.textContent = sub_text; + target.onclick = hide_youtube_replies; + target.setAttribute('data-inner-text', inner_text); + target.setAttribute('data-sub-text', sub_text); +} + +function get_youtube_comments() { + var comments = document.getElementById('comments'); + + var fallback = comments.innerHTML; + comments.innerHTML = spinnerHTML; + + var baseUrl = video_data.base_url || '/api/v1/comments/'+ video_data.id + var url = baseUrl + + '?format=html' + + '&hl=' + video_data.preferences.locale + + '&thin_mode=' + video_data.preferences.thin_mode; + + if (video_data.ucid) { + url += '&ucid=' + video_data.ucid + } + + var onNon200 = function (xhr) { comments.innerHTML = fallback; }; + if (video_data.params.comments[1] === 'youtube') + onNon200 = function (xhr) {}; + + helpers.xhr('GET', url, {retries: 5, entity_name: 'comments'}, { + on200: function (response) { + var commentInnerHtml = ' \ +
\ +

\ + [ − ] \ + {commentsText} \ +

\ + \ + ' + if (video_data.support_reddit) { + commentInnerHtml += ' \ + {redditComments} \ + \ + ' + } + commentInnerHtml += ' \ +
\ +
{contentHtml}
\ +
' + commentInnerHtml = commentInnerHtml.supplant({ + contentHtml: response.contentHtml, + redditComments: video_data.reddit_comments_text, + commentsText: video_data.comments_text.supplant({ + // toLocaleString correctly splits number with local thousands separator. e.g.: + // '1,234,567.89' for user with English locale + // '1 234 567,89' for user with Russian locale + // '1.234.567,89' for user with Portuguese locale + commentCount: response.commentCount.toLocaleString() + }) + }); + comments.innerHTML = commentInnerHtml; + comments.children[0].children[0].children[0].onclick = toggle_comments; + if (video_data.support_reddit) { + comments.children[0].children[1].children[0].onclick = swap_comments; + } + }, + onNon200: onNon200, // declared above + onError: function (xhr) { + comments.innerHTML = spinnerHTML; + }, + onTimeout: function (xhr) { + comments.innerHTML = spinnerHTML; + } + }); +} + +function get_youtube_replies(target, load_more, load_replies) { + var continuation = target.getAttribute('data-continuation'); + + var body = target.parentNode.parentNode; + var fallback = body.innerHTML; + body.innerHTML = spinnerHTML; + var baseUrl = video_data.base_url || '/api/v1/comments/'+ video_data.id + var url = baseUrl + + '?format=html' + + '&hl=' + video_data.preferences.locale + + '&thin_mode=' + video_data.preferences.thin_mode + + '&continuation=' + continuation; + + if (video_data.ucid) { + url += '&ucid=' + video_data.ucid + } + if (load_replies) url += '&action=action_get_comment_replies'; + + helpers.xhr('GET', url, {}, { + on200: function (response) { + if (load_more) { + body = body.parentNode.parentNode; + body.removeChild(body.lastElementChild); + body.insertAdjacentHTML('beforeend', response.contentHtml); + } else { + body.removeChild(body.lastElementChild); + + var p = document.createElement('p'); + var a = document.createElement('a'); + p.appendChild(a); + + a.href = 'javascript:void(0)'; + a.onclick = hide_youtube_replies; + a.setAttribute('data-sub-text', video_data.hide_replies_text); + a.setAttribute('data-inner-text', video_data.show_replies_text); + a.textContent = video_data.hide_replies_text; + + var div = document.createElement('div'); + div.innerHTML = response.contentHtml; + + body.appendChild(p); + body.appendChild(div); + } + }, + onNon200: function (xhr) { + body.innerHTML = fallback; + }, + onTimeout: function (xhr) { + console.warn('Pulling comments failed'); + body.innerHTML = fallback; + } + }); +} \ No newline at end of file diff --git a/assets/js/post.js b/assets/js/post.js new file mode 100644 index 00000000..fcbc9155 --- /dev/null +++ b/assets/js/post.js @@ -0,0 +1,3 @@ +addEventListener('load', function (e) { + get_youtube_comments(); +}); diff --git a/assets/js/watch.js b/assets/js/watch.js index 36506abd..26ad138f 100644 --- a/assets/js/watch.js +++ b/assets/js/watch.js @@ -1,14 +1,4 @@ 'use strict'; -var video_data = JSON.parse(document.getElementById('video_data').textContent); -var spinnerHTML = '

'; -var spinnerHTMLwithHR = spinnerHTML + '
'; - -String.prototype.supplant = function (o) { - return this.replace(/{([^{}]*)}/g, function (a, b) { - var r = o[b]; - return typeof r === 'string' || typeof r === 'number' ? r : a; - }); -}; function toggle_parent(target) { var body = target.parentNode.parentNode.children[1]; @@ -21,18 +11,6 @@ function toggle_parent(target) { } } -function toggle_comments(event) { - var target = event.target; - var body = target.parentNode.parentNode.parentNode.children[1]; - if (body.style.display === 'none') { - target.textContent = '[ − ]'; - body.style.display = ''; - } else { - target.textContent = '[ + ]'; - body.style.display = 'none'; - } -} - function swap_comments(event) { var source = event.target.getAttribute('data-comments'); @@ -43,36 +21,6 @@ function swap_comments(event) { } } -function hide_youtube_replies(event) { - var target = event.target; - - var sub_text = target.getAttribute('data-inner-text'); - var inner_text = target.getAttribute('data-sub-text'); - - var body = target.parentNode.parentNode.children[1]; - body.style.display = 'none'; - - target.textContent = sub_text; - target.onclick = show_youtube_replies; - target.setAttribute('data-inner-text', inner_text); - target.setAttribute('data-sub-text', sub_text); -} - -function show_youtube_replies(event) { - var target = event.target; - - var sub_text = target.getAttribute('data-inner-text'); - var inner_text = target.getAttribute('data-sub-text'); - - var body = target.parentNode.parentNode.children[1]; - body.style.display = ''; - - target.textContent = sub_text; - target.onclick = hide_youtube_replies; - target.setAttribute('data-inner-text', inner_text); - target.setAttribute('data-sub-text', sub_text); -} - var continue_button = document.getElementById('continue'); if (continue_button) { continue_button.onclick = continue_autoplay; @@ -208,111 +156,6 @@ function get_reddit_comments() { }); } -function get_youtube_comments() { - var comments = document.getElementById('comments'); - - var fallback = comments.innerHTML; - comments.innerHTML = spinnerHTML; - - var url = '/api/v1/comments/' + video_data.id + - '?format=html' + - '&hl=' + video_data.preferences.locale + - '&thin_mode=' + video_data.preferences.thin_mode; - - var onNon200 = function (xhr) { comments.innerHTML = fallback; }; - if (video_data.params.comments[1] === 'youtube') - onNon200 = function (xhr) {}; - - helpers.xhr('GET', url, {retries: 5, entity_name: 'comments'}, { - on200: function (response) { - comments.innerHTML = ' \ -
\ -

\ - [ − ] \ - {commentsText} \ -

\ - \ - \ - {redditComments} \ - \ - \ -
\ -
{contentHtml}
\ -
'.supplant({ - contentHtml: response.contentHtml, - redditComments: video_data.reddit_comments_text, - commentsText: video_data.comments_text.supplant({ - // toLocaleString correctly splits number with local thousands separator. e.g.: - // '1,234,567.89' for user with English locale - // '1 234 567,89' for user with Russian locale - // '1.234.567,89' for user with Portuguese locale - commentCount: response.commentCount.toLocaleString() - }) - }); - - comments.children[0].children[0].children[0].onclick = toggle_comments; - comments.children[0].children[1].children[0].onclick = swap_comments; - }, - onNon200: onNon200, // declared above - onError: function (xhr) { - comments.innerHTML = spinnerHTML; - }, - onTimeout: function (xhr) { - comments.innerHTML = spinnerHTML; - } - }); -} - -function get_youtube_replies(target, load_more, load_replies) { - var continuation = target.getAttribute('data-continuation'); - - var body = target.parentNode.parentNode; - var fallback = body.innerHTML; - body.innerHTML = spinnerHTML; - - var url = '/api/v1/comments/' + video_data.id + - '?format=html' + - '&hl=' + video_data.preferences.locale + - '&thin_mode=' + video_data.preferences.thin_mode + - '&continuation=' + continuation; - if (load_replies) url += '&action=action_get_comment_replies'; - - helpers.xhr('GET', url, {}, { - on200: function (response) { - if (load_more) { - body = body.parentNode.parentNode; - body.removeChild(body.lastElementChild); - body.insertAdjacentHTML('beforeend', response.contentHtml); - } else { - body.removeChild(body.lastElementChild); - - var p = document.createElement('p'); - var a = document.createElement('a'); - p.appendChild(a); - - a.href = 'javascript:void(0)'; - a.onclick = hide_youtube_replies; - a.setAttribute('data-sub-text', video_data.hide_replies_text); - a.setAttribute('data-inner-text', video_data.show_replies_text); - a.textContent = video_data.hide_replies_text; - - var div = document.createElement('div'); - div.innerHTML = response.contentHtml; - - body.appendChild(p); - body.appendChild(div); - } - }, - onNon200: function (xhr) { - body.innerHTML = fallback; - }, - onTimeout: function (xhr) { - console.warn('Pulling comments failed'); - body.innerHTML = fallback; - } - }); -} - if (video_data.play_next) { player.on('ended', function () { var url = new URL('https://example.com/watch?v=' + video_data.next_video); diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 791f1641..85ddff35 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -24,7 +24,35 @@ def fetch_channel_community(ucid, cursor, locale, format, thin_mode) return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode) end -def extract_channel_community(items, *, ucid, locale, format, thin_mode) +def fetch_channel_community_post(ucid, postId, locale, format, thin_mode, params : String | Nil = nil) + if params.nil? + object = { + "2:string" => "community", + "25:embedded" => { + "22:string" => postId.to_s, + }, + "45:embedded" => { + "2:varint" => 1_i64, + "3:varint" => 1_i64, + }, + } + params = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + end + + initial_data = YoutubeAPI.browse(ucid, params: params) + + items = [] of JSON::Any + extract_items(initial_data) do |item| + items << item + end + + return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true) +end + +def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false) if message = items[0]["messageRenderer"]? error_message = (message["text"]["simpleText"]? || message["text"]["runs"]?.try &.[0]?.try &.["text"]?) @@ -39,6 +67,9 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode) response = JSON.build do |json| json.object do json.field "authorId", ucid + if is_single_post + json.field "singlePost", true + end json.field "comments" do json.array do items.each do |post| @@ -240,8 +271,10 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode) end end end - if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") - json.field "continuation", extract_channel_community_cursor(cont.as_s) + if !is_single_post + if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") + json.field "continuation", extract_channel_community_cursor(cont.as_s) + end end end end diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr index 1ba1b534..da7f0543 100644 --- a/src/invidious/comments/youtube.cr +++ b/src/invidious/comments/youtube.cr @@ -13,6 +13,51 @@ module Invidious::Comments client_config = YoutubeAPI::ClientConfig.new(region: region) response = YoutubeAPI.next(continuation: ctoken, client_config: client_config) + return parse_youtube(id, response, format, locale, thin_mode, sort_by) + end + + def fetch_community_post_comments(ucid, postId) + object = { + "2:string" => "community", + "25:embedded" => { + "22:string" => postId, + }, + "45:embedded" => { + "2:varint" => 1_i64, + "3:varint" => 1_i64, + }, + "53:embedded" => { + "4:embedded" => { + "6:varint" => 0_i64, + "27:varint" => 1_i64, + "29:string" => postId, + "30:string" => ucid, + }, + "8:string" => "comments-section", + }, + } + + objectParsed = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + + object2 = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => objectParsed, + }, + } + + continuation = object2.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + initial_data = YoutubeAPI.browse(continuation: continuation) + return initial_data + end + + def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", isPost = false) contents = nil if on_response_received_endpoints = response["onResponseReceivedEndpoints"]? @@ -68,7 +113,11 @@ module Invidious::Comments json.field "commentCount", comment_count end - json.field "videoId", id + if isPost + json.field "postId", id + else + json.field "videoId", id + end json.field "comments" do json.array do diff --git a/src/invidious/frontend/comments_youtube.cr b/src/invidious/frontend/comments_youtube.cr index 41f43f04..ecc0bc1b 100644 --- a/src/invidious/frontend/comments_youtube.cr +++ b/src/invidious/frontend/comments_youtube.cr @@ -23,6 +23,24 @@ module Invidious::Frontend::Comments END_HTML + elsif comments["authorId"]? && !comments["singlePost"]? + # for posts we should display a link to the post + replies_count_text = translate_count(locale, + "comments_view_x_replies", + child["replyCount"].as_i64 || 0, + NumberFormatting::Separator + ) + + replies_html = <<-END_HTML +
+
+
+

+ #{replies_count_text} +

+
+
+ END_HTML end if !thin_mode diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index adf05d30..0d2d2eb1 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -343,6 +343,53 @@ module Invidious::Routes::API::V1::Channels end end + def self.post(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + + id = env.params.url["id"].to_s + ucid = env.params.query["ucid"] + + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + + format = env.params.query["format"]? + format ||= "json" + + begin + fetch_channel_community_post(ucid, id, locale, format, thin_mode) + rescue ex + return error_json(500, ex) + end + end + + def self.post_comments(env) + locale = env.get("preferences").as(Preferences).locale + region = env.params.query["region"]? + + env.response.content_type = "application/json" + + id = env.params.url["id"] + + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + + format = env.params.query["format"]? + format ||= "json" + + continuation = env.params.query["continuation"]? + + case continuation + when nil, "" + ucid = env.params.query["ucid"] + comments = Comments.fetch_community_post_comments(ucid, id) + else + comments = YoutubeAPI.browse(continuation: continuation) + end + return Comments.parse_youtube(id, comments, format, locale, thin_mode, isPost: true) + end + def self.channels(env) locale = env.get("preferences").as(Preferences).locale ucid = env.params.url["ucid"] diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index e499f4d6..91a62fa3 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -162,17 +162,23 @@ module Invidious::Routes::API::V1::Misc resolved_url = YoutubeAPI.resolve_url(url.as(String)) endpoint = resolved_url["endpoint"] pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" - if resolved_ucid = endpoint.dig?("watchEndpoint", "videoId") - elsif resolved_ucid = endpoint.dig?("browseEndpoint", "browseId") + if sub_endpoint = endpoint.dig?("watchEndpoint") + resolved_ucid = sub_endpoint.dig?("videoId") + elsif sub_endpoint = endpoint.dig?("browseEndpoint") + resolved_ucid = sub_endpoint.dig?("browseId") elsif pageType == "WEB_PAGE_TYPE_UNKNOWN" return error_json(400, "Unknown url") end + if !sub_endpoint.nil? + params = sub_endpoint.dig?("params") + end rescue ex return error_json(500, ex) end JSON.build do |json| json.object do json.field "ucid", resolved_ucid.try &.as_s || "" + json.field "params", params.try &.as_s json.field "pageType", pageType end end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 9892ae2a..1d02ee08 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -159,6 +159,11 @@ module Invidious::Routes::Channels end locale, user, subscriptions, continuation, ucid, channel = data + # redirect to post page + if lb = env.params.query["lb"]? + env.redirect "/post/#{lb}?ucid=#{ucid}" + end + thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode thin_mode = thin_mode == "true" @@ -187,6 +192,38 @@ module Invidious::Routes::Channels templated "community" end + def self.post(env) + # /post/{postId} + id = env.params.url["id"] + ucid = env.params.query["ucid"]? + + prefs = env.get("preferences").as(Preferences) + + locale = prefs.locale + region = env.params.query["region"]? || prefs.region + + thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode + thin_mode = thin_mode == "true" + + client_config = YoutubeAPI::ClientConfig.new(region: region) + + if !ucid.nil? + ucid = ucid.to_s + post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) + else + # resolve the url to get the author's UCID + response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") + return error_template(400, "Invalid post ID") if response["error"]? + + ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s + params = response.dig("endpoint", "browseEndpoint", "params").as_s + post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode, params: params) + end + + post_response = JSON.parse(post_response) + templated "post" + end + def self.channels(env) data = self.fetch_basic_information(env) return data if !data.is_a?(Tuple) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 9c43171c..8cb49249 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -127,6 +127,7 @@ module Invidious::Routing get "/channel/:ucid/live", Routes::Channels, :live get "/user/:user/live", Routes::Channels, :live get "/c/:user/live", Routes::Channels, :live + get "/post/:id", Routes::Channels, :post {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path| # /c/LinusTechTips @@ -240,6 +241,10 @@ module Invidious::Routing get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} {% end %} + # Posts + get "/api/v1/post/:id", {{namespace}}::Channels, :post + get "/api/v1/post/:id/comments", {{namespace}}::Channels, :post_comments + # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect @@ -249,6 +254,7 @@ module Invidious::Routing get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions get "/api/v1/hashtag/:hashtag", {{namespace}}::Search, :hashtag + # Authenticated # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 24efc34e..d2a305d3 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -26,7 +26,7 @@

<%= error_message %>

<% else %> -
+
<%= IV::Frontend::Comments.template_youtube(items.not_nil!, locale, thin_mode) %>
<% end %> diff --git a/src/invidious/views/post.ecr b/src/invidious/views/post.ecr new file mode 100644 index 00000000..b2cd778c --- /dev/null +++ b/src/invidious/views/post.ecr @@ -0,0 +1,31 @@ +<% content_for "header" do %> +Invidious +<% end %> + +
+ <%= IV::Frontend::Comments.template_youtube(post_response.not_nil!, locale, thin_mode) %> +
+
+
+ + + + \ No newline at end of file diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 498d57a1..62a154a4 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -64,7 +64,8 @@ we're going to need to do it here in order to allow for translations. "premiere_timestamp" => video.premiere_timestamp.try &.to_unix, "vr" => video.is_vr, "projection_type" => video.projection_type, - "local_disabled" => CONFIG.disabled?("local") + "local_disabled" => CONFIG.disabled?("local"), + "support_reddit" => true }.to_pretty_json %> @@ -270,7 +271,7 @@ we're going to need to do it here in order to allow for translations.
<% end %> -
+
<% if nojs %> <%= comment_html %> <% else %> @@ -352,4 +353,5 @@ we're going to need to do it here in order to allow for translations.
<% end %>
+ From 734f1b7764598bd5ff24acd11ab833f831d0f4a7 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Thu, 27 Jul 2023 19:14:34 -0400 Subject: [PATCH 05/51] Simplify resolveUrl api call Co-Authored-By: Samantaz Fox --- src/invidious/channels/community.cr | 4 ++-- src/invidious/comments/youtube.cr | 6 +++--- src/invidious/routes/api/v1/misc.cr | 13 ++++++------- src/invidious/routes/channels.cr | 2 +- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 85ddff35..76dab361 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -24,12 +24,12 @@ def fetch_channel_community(ucid, cursor, locale, format, thin_mode) return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode) end -def fetch_channel_community_post(ucid, postId, locale, format, thin_mode, params : String | Nil = nil) +def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode, params : String | Nil = nil) if params.nil? object = { "2:string" => "community", "25:embedded" => { - "22:string" => postId.to_s, + "22:string" => post_id.to_s, }, "45:embedded" => { "2:varint" => 1_i64, diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr index da7f0543..01c2564f 100644 --- a/src/invidious/comments/youtube.cr +++ b/src/invidious/comments/youtube.cr @@ -16,11 +16,11 @@ module Invidious::Comments return parse_youtube(id, response, format, locale, thin_mode, sort_by) end - def fetch_community_post_comments(ucid, postId) + def fetch_community_post_comments(ucid, post_id) object = { "2:string" => "community", "25:embedded" => { - "22:string" => postId, + "22:string" => post_id, }, "45:embedded" => { "2:varint" => 1_i64, @@ -30,7 +30,7 @@ module Invidious::Comments "4:embedded" => { "6:varint" => 0_i64, "27:varint" => 1_i64, - "29:string" => postId, + "29:string" => post_id, "30:string" => ucid, }, "8:string" => "comments-section", diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 91a62fa3..6118a0d1 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -162,16 +162,15 @@ module Invidious::Routes::API::V1::Misc resolved_url = YoutubeAPI.resolve_url(url.as(String)) endpoint = resolved_url["endpoint"] pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" - if sub_endpoint = endpoint.dig?("watchEndpoint") - resolved_ucid = sub_endpoint.dig?("videoId") - elsif sub_endpoint = endpoint.dig?("browseEndpoint") - resolved_ucid = sub_endpoint.dig?("browseId") + if sub_endpoint = endpoint["watchEndpoint"]? + resolved_ucid = sub_endpoint["videoId"]? + elsif sub_endpoint = endpoint["browseEndpoint"]? + resolved_ucid = sub_endpoint["browseId"]? elsif pageType == "WEB_PAGE_TYPE_UNKNOWN" return error_json(400, "Unknown url") end - if !sub_endpoint.nil? - params = sub_endpoint.dig?("params") - end + + params = sub_endpoint.try &.dig?("params") rescue ex return error_json(500, ex) end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 1d02ee08..8515b910 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -161,7 +161,7 @@ module Invidious::Routes::Channels # redirect to post page if lb = env.params.query["lb"]? - env.redirect "/post/#{lb}?ucid=#{ucid}" + env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}" end thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode From f55b96a53bde8d8c6a24d4db4e9d10f14ffee585 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Mon, 7 Aug 2023 12:46:19 -0700 Subject: [PATCH 06/51] Always craft Community Post params --- src/invidious/channels/community.cr | 32 ++++++++++++++--------------- src/invidious/routes/channels.cr | 3 +-- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 76dab361..49ffd990 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -24,23 +24,21 @@ def fetch_channel_community(ucid, cursor, locale, format, thin_mode) return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode) end -def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode, params : String | Nil = nil) - if params.nil? - object = { - "2:string" => "community", - "25:embedded" => { - "22:string" => post_id.to_s, - }, - "45:embedded" => { - "2:varint" => 1_i64, - "3:varint" => 1_i64, - }, - } - params = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - end +def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode) + object = { + "2:string" => "community", + "25:embedded" => { + "22:string" => post_id.to_s, + }, + "45:embedded" => { + "2:varint" => 1_i64, + "3:varint" => 1_i64, + }, + } + params = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } initial_data = YoutubeAPI.browse(ucid, params: params) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 8515b910..20b02dc1 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -216,8 +216,7 @@ module Invidious::Routes::Channels return error_template(400, "Invalid post ID") if response["error"]? ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s - params = response.dig("endpoint", "browseEndpoint", "params").as_s - post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode, params: params) + post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) end post_response = JSON.parse(post_response) From bb04bcc42c1b135aaf50de8799264f86bc42f4db Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Tue, 29 Aug 2023 19:10:01 -0700 Subject: [PATCH 07/51] Apply suggestions from code review add videoId to resolve_url function Co-Authored-By: Samantaz Fox --- src/invidious/comments/youtube.cr | 4 ++-- src/invidious/routes/api/v1/channels.cr | 11 +++++++++-- src/invidious/routes/api/v1/misc.cr | 10 ++++------ src/invidious/routes/channels.cr | 2 -- src/invidious/views/post.ecr | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr index 01c2564f..185d8e43 100644 --- a/src/invidious/comments/youtube.cr +++ b/src/invidious/comments/youtube.cr @@ -37,14 +37,14 @@ module Invidious::Comments }, } - objectParsed = object.try { |i| Protodec::Any.cast_json(i) } + object_parsed = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i) } object2 = { "80226972:embedded" => { "2:string" => ucid, - "3:string" => objectParsed, + "3:string" => object_parsed, }, } diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 0d2d2eb1..a5ae16a8 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -347,9 +347,8 @@ module Invidious::Routes::API::V1::Channels locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" - id = env.params.url["id"].to_s - ucid = env.params.query["ucid"] + ucid = env.params.query["ucid"]? thin_mode = env.params.query["thin_mode"]? thin_mode = thin_mode == "true" @@ -357,6 +356,14 @@ module Invidious::Routes::API::V1::Channels format = env.params.query["format"]? format ||= "json" + if ucid.nil? + response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") + return error_json(400, "Invalid post ID") if response["error"]? + ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s + else + ucid = ucid.to_s + end + begin fetch_channel_community_post(ucid, id, locale, format, thin_mode) rescue ex diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 6118a0d1..5dfc4afa 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -162,21 +162,19 @@ module Invidious::Routes::API::V1::Misc resolved_url = YoutubeAPI.resolve_url(url.as(String)) endpoint = resolved_url["endpoint"] pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" - if sub_endpoint = endpoint["watchEndpoint"]? - resolved_ucid = sub_endpoint["videoId"]? - elsif sub_endpoint = endpoint["browseEndpoint"]? - resolved_ucid = sub_endpoint["browseId"]? - elsif pageType == "WEB_PAGE_TYPE_UNKNOWN" + if pageType == "WEB_PAGE_TYPE_UNKNOWN" return error_json(400, "Unknown url") end + sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint params = sub_endpoint.try &.dig?("params") rescue ex return error_json(500, ex) end JSON.build do |json| json.object do - json.field "ucid", resolved_ucid.try &.as_s || "" + json.field "ucid", sub_endpoint["browseId"].try &.as_s if sub_endpoint["browseId"]? + json.field "videoId", sub_endpoint["videoId"].try &.as_s if sub_endpoint["videoId"]? json.field "params", params.try &.as_s json.field "pageType", pageType end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 20b02dc1..29995bf6 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -205,8 +205,6 @@ module Invidious::Routes::Channels thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode thin_mode = thin_mode == "true" - client_config = YoutubeAPI::ClientConfig.new(region: region) - if !ucid.nil? ucid = ucid.to_s post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) diff --git a/src/invidious/views/post.ecr b/src/invidious/views/post.ecr index b2cd778c..071d1c88 100644 --- a/src/invidious/views/post.ecr +++ b/src/invidious/views/post.ecr @@ -22,7 +22,7 @@ "comments": ["youtube"] }, "preferences" => prefs, - "base_url" => "/api/v1/post/" + id + "/comments", + "base_url" => "/api/v1/post/#{URI.encode_www_form(id)}/comments", "ucid" => ucid }.to_pretty_json %> From 8781520b8af221e5ab202775a1b58dd5e0e3fd83 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Tue, 18 Jul 2023 08:06:50 -0700 Subject: [PATCH 08/51] Search: Parse channel handle and hide video count when channel handle exists Co-Authored-By: Samantaz Fox --- src/invidious/helpers/serialized_yt_data.cr | 2 ++ src/invidious/views/components/item.ecr | 3 ++- src/invidious/yt_backend/extractors.cr | 10 ++++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index e0bd7279..31a3cf44 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -186,6 +186,7 @@ struct SearchChannel property author_thumbnail : String property subscriber_count : Int32 property video_count : Int32 + property channel_handle : String? property description_html : String property auto_generated : Bool property author_verified : Bool @@ -214,6 +215,7 @@ struct SearchChannel json.field "autoGenerated", self.auto_generated json.field "subCount", self.subscriber_count json.field "videoCount", self.video_count + json.field "channelHandle", self.channel_handle json.field "description", html_to_content(self.description_html) json.field "descriptionHtml", self.description_html diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index c29ec47b..031b46da 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -26,8 +26,9 @@
+ <% if !item.channel_handle.nil? %>

<%= item.channel_handle %>

<% end %>

<%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %>

- <% if !item.auto_generated %>

<%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>

<% end %> + <% if !item.auto_generated && item.channel_handle.nil? %>

<%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>

<% end %>
<%= item.description_html %>
<% when SearchHashtag %> <% if !thin_mode %> diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index aaf7772e..56325cf7 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -175,17 +175,18 @@ private module Parsers # Always simpleText # TODO change default value to nil - subscriber_count = item_contents.dig?("subscriberCountText", "simpleText") + subscriber_count = item_contents.dig?("subscriberCountText", "simpleText").try &.as_s + channel_handle = subscriber_count if (subscriber_count.try &.starts_with? "@") # Since youtube added channel handles, `VideoCountText` holds the number of # subscribers and `subscriberCountText` holds the handle, except when the # channel doesn't have a handle (e.g: some topic music channels). # See https://github.com/iv-org/invidious/issues/3394#issuecomment-1321261688 - if !subscriber_count || !subscriber_count.as_s.includes? " subscriber" - subscriber_count = item_contents.dig?("videoCountText", "simpleText") + if !subscriber_count || !subscriber_count.includes? " subscriber" + subscriber_count = item_contents.dig?("videoCountText", "simpleText").try &.as_s end subscriber_count = subscriber_count - .try { |s| short_text_to_number(s.as_s.split(" ")[0]).to_i32 } || 0 + .try { |s| short_text_to_number(s.split(" ")[0]).to_i32 } || 0 # Auto-generated channels doesn't have videoCountText # Taken from: https://github.com/iv-org/invidious/pull/2228#discussion_r717620922 @@ -200,6 +201,7 @@ private module Parsers author_thumbnail: author_thumbnail, subscriber_count: subscriber_count, video_count: video_count, + channel_handle: channel_handle, description_html: description_html, auto_generated: auto_generated, author_verified: author_verified, From 54fa59cbb0ae90a54136522c944410e2d18c234b Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 24 Aug 2023 14:58:50 -0700 Subject: [PATCH 09/51] Add method to construct WebVTT files Similar to JSON.Build --- spec/helpers/vtt/builder_spec.cr | 64 ++++++++++++++++++++++++++++++ src/invidious/helpers/webvtt.cr | 67 ++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 spec/helpers/vtt/builder_spec.cr create mode 100644 src/invidious/helpers/webvtt.cr diff --git a/spec/helpers/vtt/builder_spec.cr b/spec/helpers/vtt/builder_spec.cr new file mode 100644 index 00000000..69303bab --- /dev/null +++ b/spec/helpers/vtt/builder_spec.cr @@ -0,0 +1,64 @@ +require "../../spec_helper.cr" + +MockLines = [ + { + "start_time": Time::Span.new(seconds: 1), + "end_time": Time::Span.new(seconds: 2), + "text": "Line 1", + }, + + { + "start_time": Time::Span.new(seconds: 2), + "end_time": Time::Span.new(seconds: 3), + "text": "Line 2", + }, +] + +Spectator.describe "WebVTT::Builder" do + it "correctly builds a vtt file" do + result = WebVTT.build do |vtt| + MockLines.each do |line| + vtt.line(line["start_time"], line["end_time"], line["text"]) + end + end + + expect(result).to eq([ + "WEBVTT", + "", + "00:00:01.000 --> 00:00:02.000", + "Line 1", + "", + "00:00:02.000 --> 00:00:03.000", + "Line 2", + "", + "", + ].join('\n')) + end + + it "correctly builds a vtt file with setting fields" do + setting_fields = { + "Kind" => "captions", + "Language" => "en", + } + + result = WebVTT.build(setting_fields) do |vtt| + MockLines.each do |line| + vtt.line(line["start_time"], line["end_time"], line["text"]) + end + end + + expect(result).to eq([ + "WEBVTT", + "Kind: captions", + "Language: en", + "", + "00:00:01.000 --> 00:00:02.000", + "Line 1", + "", + "00:00:02.000 --> 00:00:03.000", + "Line 2", + "", + "", + ].join('\n')) + end +end diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr new file mode 100644 index 00000000..7d9d5f1f --- /dev/null +++ b/src/invidious/helpers/webvtt.cr @@ -0,0 +1,67 @@ +# Namespace for logic relating to generating WebVTT files +# +# Probably not compliant to WebVTT's specs but it is enough for Invidious. +module WebVTT + # A WebVTT builder generates WebVTT files + private class Builder + def initialize(@io : IO) + end + + # Writes an vtt line with the specified time stamp and contents + def line(start_time : Time::Span, end_time : Time::Span, text : String) + timestamp(start_time, end_time) + @io << text + @io << "\n\n" + end + + private def timestamp(start_time : Time::Span, end_time : Time::Span) + add_timestamp_component(start_time) + @io << " --> " + add_timestamp_component(end_time) + + @io << '\n' + end + + private def add_timestamp_component(timestamp : Time::Span) + @io << timestamp.hours.to_s.rjust(2, '0') + @io << ':' << timestamp.minutes.to_s.rjust(2, '0') + @io << ':' << timestamp.seconds.to_s.rjust(2, '0') + @io << '.' << timestamp.milliseconds.to_s.rjust(3, '0') + end + + def document(setting_fields : Hash(String, String)? = nil, &) + @io << "WEBVTT\n" + + if setting_fields + setting_fields.each do |name, value| + @io << "#{name}: #{value}\n" + end + end + + @io << '\n' + + yield + end + end + + # Returns the resulting `String` of writing WebVTT to the yielded WebVTT::Builder + # + # ``` + # string = WebVTT.build do |io| + # vtt.line(Time::Span.new(seconds: 1), Time::Span.new(seconds: 2), "Line 1") + # vtt.line(Time::Span.new(seconds: 2), Time::Span.new(seconds: 3), "Line 2") + # end + # + # string # => "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nLine 1\n\n00:00:02.000 --> 00:00:03.000\nLine 2\n\n" + # ``` + # + # Accepts an optional settings fields hash to add settings attribute to the resulting vtt file. + def self.build(setting_fields : Hash(String, String)? = nil, &) + String.build do |str| + builder = Builder.new(str) + builder.document(setting_fields) do + yield builder + end + end + end +end From 0cb7d0b44137c2cee9b6352969a28dac4e3576c5 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 24 Aug 2023 15:10:50 -0700 Subject: [PATCH 10/51] Refactor Invidious's VTT logic to use WebVtt.build --- src/invidious/routes/api/v1/videos.cr | 39 +++++++------------------ src/invidious/videos/caption.cr | 41 ++++++++------------------- src/invidious/videos/transcript.cr | 40 +++++--------------------- 3 files changed, 29 insertions(+), 91 deletions(-) diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 25e766d2..5c50a804 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -101,20 +101,17 @@ module Invidious::Routes::API::V1::Videos if caption.name.includes? "auto-generated" caption_xml = YT_POOL.client &.get(url).body + settings_field = { + "Kind" => "captions", + "Language" => "#{tlang || caption.language_code}", + } + if caption_xml.starts_with?("/, "") text = text.gsub(/<\/font>/, "") @@ -137,12 +131,7 @@ module Invidious::Routes::API::V1::Videos text = "#{md["text"]}" end - str << <<-END_CUE - #{start_time} --> #{end_time} - #{text} - - - END_CUE + webvtt.line(start_time, end_time, text) end end end @@ -215,11 +204,7 @@ module Invidious::Routes::API::V1::Videos storyboard = storyboard[0] end - String.build do |str| - str << <<-END_VTT - WEBVTT - END_VTT - + WebVTT.build do |vtt| start_time = 0.milliseconds end_time = storyboard[:interval].milliseconds @@ -231,12 +216,8 @@ module Invidious::Routes::API::V1::Videos storyboard[:storyboard_height].times do |j| storyboard[:storyboard_width].times do |k| - str << <<-END_CUE - #{start_time}.000 --> #{end_time}.000 - #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]} - - - END_CUE + current_cue_url = "#{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}" + vtt.line(start_time, end_time, current_cue_url) start_time += storyboard[:interval].milliseconds end_time += storyboard[:interval].milliseconds diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index 256dfcc0..dc58f9a0 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -52,17 +52,13 @@ module Invidious::Videos break end end - result = String.build do |result| - result << <<-END_VTT - WEBVTT - Kind: captions - Language: #{tlang || @language_code} + settings_field = { + "Kind" => "captions", + "Language" => "#{tlang || @language_code}", + } - END_VTT - - result << "\n\n" - + result = WebVTT.build(settings_field) do |vtt| cues.each_with_index do |node, i| start_time = node["t"].to_f.milliseconds @@ -76,29 +72,16 @@ module Invidious::Videos end_time = start_time + duration end - # start_time - result << start_time.hours.to_s.rjust(2, '0') - result << ':' << start_time.minutes.to_s.rjust(2, '0') - result << ':' << start_time.seconds.to_s.rjust(2, '0') - result << '.' << start_time.milliseconds.to_s.rjust(3, '0') - - result << " --> " - - # end_time - result << end_time.hours.to_s.rjust(2, '0') - result << ':' << end_time.minutes.to_s.rjust(2, '0') - result << ':' << end_time.seconds.to_s.rjust(2, '0') - result << '.' << end_time.milliseconds.to_s.rjust(3, '0') - - result << "\n" - - node.children.each do |s| - result << s.content + text = String.build do |io| + node.children.each do |s| + io << s.content + end end - result << "\n" - result << "\n" + + vtt.line(start_time, end_time, text) end end + return result end end diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index f3360a52..cd97cfde 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -34,41 +34,15 @@ module Invidious::Videos # Convert into array of TranscriptLine lines = self.parse(initial_data) + settings_field = { + "Kind" => "captions", + "Language" => target_language + } + # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() - vtt = String.build do |vtt| - vtt << <<-END_VTT - WEBVTT - Kind: captions - Language: #{target_language} - - - END_VTT - - vtt << "\n\n" - + vtt = WebVTT.build(settings_field) do |vtt| lines.each do |line| - start_time = line.start_ms - end_time = line.end_ms - - # start_time - vtt << start_time.hours.to_s.rjust(2, '0') - vtt << ':' << start_time.minutes.to_s.rjust(2, '0') - vtt << ':' << start_time.seconds.to_s.rjust(2, '0') - vtt << '.' << start_time.milliseconds.to_s.rjust(3, '0') - - vtt << " --> " - - # end_time - vtt << end_time.hours.to_s.rjust(2, '0') - vtt << ':' << end_time.minutes.to_s.rjust(2, '0') - vtt << ':' << end_time.seconds.to_s.rjust(2, '0') - vtt << '.' << end_time.milliseconds.to_s.rjust(3, '0') - - vtt << "\n" - vtt << line.line - - vtt << "\n" - vtt << "\n" + vtt.line(line.start_ms, line.end_ms, line.line) end end From d371eb50f27b9d29bc68ec883d8bee54894c79a4 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 24 Aug 2023 15:42:42 -0700 Subject: [PATCH 11/51] WebVTT::Builder: rename #line to #cue --- spec/helpers/vtt/builder_spec.cr | 4 ++-- src/invidious/helpers/webvtt.cr | 8 ++++---- src/invidious/routes/api/v1/videos.cr | 4 ++-- src/invidious/videos/caption.cr | 2 +- src/invidious/videos/transcript.cr | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/spec/helpers/vtt/builder_spec.cr b/spec/helpers/vtt/builder_spec.cr index 69303bab..7b543ddc 100644 --- a/spec/helpers/vtt/builder_spec.cr +++ b/spec/helpers/vtt/builder_spec.cr @@ -18,7 +18,7 @@ Spectator.describe "WebVTT::Builder" do it "correctly builds a vtt file" do result = WebVTT.build do |vtt| MockLines.each do |line| - vtt.line(line["start_time"], line["end_time"], line["text"]) + vtt.cue(line["start_time"], line["end_time"], line["text"]) end end @@ -43,7 +43,7 @@ Spectator.describe "WebVTT::Builder" do result = WebVTT.build(setting_fields) do |vtt| MockLines.each do |line| - vtt.line(line["start_time"], line["end_time"], line["text"]) + vtt.cue(line["start_time"], line["end_time"], line["text"]) end end diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr index 7d9d5f1f..c50d7fa2 100644 --- a/src/invidious/helpers/webvtt.cr +++ b/src/invidious/helpers/webvtt.cr @@ -7,8 +7,8 @@ module WebVTT def initialize(@io : IO) end - # Writes an vtt line with the specified time stamp and contents - def line(start_time : Time::Span, end_time : Time::Span, text : String) + # Writes an vtt cue with the specified time stamp and contents + def cue(start_time : Time::Span, end_time : Time::Span, text : String) timestamp(start_time, end_time) @io << text @io << "\n\n" @@ -48,8 +48,8 @@ module WebVTT # # ``` # string = WebVTT.build do |io| - # vtt.line(Time::Span.new(seconds: 1), Time::Span.new(seconds: 2), "Line 1") - # vtt.line(Time::Span.new(seconds: 2), Time::Span.new(seconds: 3), "Line 2") + # vtt.cue(Time::Span.new(seconds: 1), Time::Span.new(seconds: 2), "Line 1") + # vtt.cue(Time::Span.new(seconds: 2), Time::Span.new(seconds: 3), "Line 2") # end # # string # => "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nLine 1\n\n00:00:02.000 --> 00:00:03.000\nLine 2\n\n" diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 5c50a804..449c9f9b 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -131,7 +131,7 @@ module Invidious::Routes::API::V1::Videos text = "#{md["text"]}" end - webvtt.line(start_time, end_time, text) + webvtt.cue(start_time, end_time, text) end end end @@ -217,7 +217,7 @@ module Invidious::Routes::API::V1::Videos storyboard[:storyboard_height].times do |j| storyboard[:storyboard_width].times do |k| current_cue_url = "#{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}" - vtt.line(start_time, end_time, current_cue_url) + vtt.cue(start_time, end_time, current_cue_url) start_time += storyboard[:interval].milliseconds end_time += storyboard[:interval].milliseconds diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index dc58f9a0..484e61d2 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -78,7 +78,7 @@ module Invidious::Videos end end - vtt.line(start_time, end_time, text) + vtt.cue(start_time, end_time, text) end end diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index cd97cfde..055d96fb 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -42,7 +42,7 @@ module Invidious::Videos # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() vtt = WebVTT.build(settings_field) do |vtt| lines.each do |line| - vtt.line(line.start_ms, line.end_ms, line.line) + vtt.cue(line.start_ms, line.end_ms, line.line) end end From 4e97d8ad0942bd64a23ed4a2ba89e48a97c520aa Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 24 Aug 2023 16:27:06 -0700 Subject: [PATCH 12/51] Update documentation for `WebVTT.build` --- src/invidious/helpers/webvtt.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr index c50d7fa2..52138854 100644 --- a/src/invidious/helpers/webvtt.cr +++ b/src/invidious/helpers/webvtt.cr @@ -44,10 +44,10 @@ module WebVTT end end - # Returns the resulting `String` of writing WebVTT to the yielded WebVTT::Builder + # Returns the resulting `String` of writing WebVTT to the yielded `WebVTT::Builder` # # ``` - # string = WebVTT.build do |io| + # string = WebVTT.build do |vtt| # vtt.cue(Time::Span.new(seconds: 1), Time::Span.new(seconds: 2), "Line 1") # vtt.cue(Time::Span.new(seconds: 2), Time::Span.new(seconds: 3), "Line 2") # end From e9d59a6dfd14fd115f3bfc59ca6f33182a631575 Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Tue, 29 Aug 2023 05:59:08 +0000 Subject: [PATCH 13/51] Update src/invidious/helpers/webvtt.cr Co-authored-by: Samantaz Fox --- src/invidious/helpers/webvtt.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr index 52138854..aace6bb8 100644 --- a/src/invidious/helpers/webvtt.cr +++ b/src/invidious/helpers/webvtt.cr @@ -34,7 +34,7 @@ module WebVTT if setting_fields setting_fields.each do |name, value| - @io << "#{name}: #{value}\n" + @io << name << ": " << value << '\n' end end From a999438ae46739477a6ca5f8515fa70b6b492443 Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 28 Aug 2023 23:14:25 -0700 Subject: [PATCH 14/51] Consistency: rename #add_timestamp_component Removes the add_ prefix for consistency with the other methods in WebVTT::Builder --- src/invidious/helpers/webvtt.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr index aace6bb8..56f761ed 100644 --- a/src/invidious/helpers/webvtt.cr +++ b/src/invidious/helpers/webvtt.cr @@ -15,14 +15,14 @@ module WebVTT end private def timestamp(start_time : Time::Span, end_time : Time::Span) - add_timestamp_component(start_time) + timestamp_component(start_time) @io << " --> " - add_timestamp_component(end_time) + timestamp_component(end_time) @io << '\n' end - private def add_timestamp_component(timestamp : Time::Span) + private def timestamp_component(timestamp : Time::Span) @io << timestamp.hours.to_s.rjust(2, '0') @io << ':' << timestamp.minutes.to_s.rjust(2, '0') @io << ':' << timestamp.seconds.to_s.rjust(2, '0') From be2feba17c2f3b9d8e043825beff57568df46f2e Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 23 Sep 2023 09:57:26 -0400 Subject: [PATCH 15/51] Lint --- src/invidious/videos/transcript.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index 055d96fb..dac00eea 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -35,8 +35,8 @@ module Invidious::Videos lines = self.parse(initial_data) settings_field = { - "Kind" => "captions", - "Language" => target_language + "Kind" => "captions", + "Language" => target_language, } # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() From 4f25069f55b5ee87bb214a975d97522155bffc2c Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Tue, 26 Sep 2023 19:05:37 -0400 Subject: [PATCH 16/51] remove unused variable simplify resolve url remove trailing spaces Co-Authored-By: Samantaz Fox --- assets/js/comments.js | 2 +- src/invidious/routes/api/v1/channels.cr | 1 - src/invidious/routes/api/v1/misc.cr | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/assets/js/comments.js b/assets/js/comments.js index 00a8cae9..35ffa96e 100644 --- a/assets/js/comments.js +++ b/assets/js/comments.js @@ -131,7 +131,7 @@ function get_youtube_replies(target, load_more, load_replies) { '&hl=' + video_data.preferences.locale + '&thin_mode=' + video_data.preferences.thin_mode + '&continuation=' + continuation; - + if (video_data.ucid) { url += '&ucid=' + video_data.ucid } diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index a5ae16a8..67018660 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -373,7 +373,6 @@ module Invidious::Routes::API::V1::Channels def self.post_comments(env) locale = env.get("preferences").as(Preferences).locale - region = env.params.query["region"]? env.response.content_type = "application/json" diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 5dfc4afa..8a92e160 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -173,8 +173,8 @@ module Invidious::Routes::API::V1::Misc end JSON.build do |json| json.object do - json.field "ucid", sub_endpoint["browseId"].try &.as_s if sub_endpoint["browseId"]? - json.field "videoId", sub_endpoint["videoId"].try &.as_s if sub_endpoint["videoId"]? + json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]? + json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]? json.field "params", params.try &.as_s json.field "pageType", pageType end From f77e4378fe1ee69d0cf8adced1c8eef8140896ee Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Wed, 27 Sep 2023 18:40:50 -0400 Subject: [PATCH 17/51] Add support for viewing comments without js Improve stylings --- assets/css/default.css | 8 ++++++++ src/invidious/routes/channels.cr | 11 ++++++++++- src/invidious/views/post.ecr | 25 +++++++++++++++++++++---- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/assets/css/default.css b/assets/css/default.css index 69fe8d5f..b4053b5c 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -397,6 +397,14 @@ p.video-data { margin: 0; font-weight: bold; font-size: 80%; } margin: auto; } +/* + * We don't want the top and bottom margin on the post page. + */ +.comments.post-comments { + margin-bottom: 0; + margin-top: 0; +} + .video-iframe-wrapper { position: relative; height: 0; diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 29995bf6..62b3884e 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -200,11 +200,15 @@ module Invidious::Routes::Channels prefs = env.get("preferences").as(Preferences) locale = prefs.locale - region = env.params.query["region"]? || prefs.region thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode thin_mode = thin_mode == "true" + nojs = env.params.query["nojs"]? + + nojs ||= "0" + nojs = nojs == "1" + if !ucid.nil? ucid = ucid.to_s post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) @@ -218,6 +222,11 @@ module Invidious::Routes::Channels end post_response = JSON.parse(post_response) + + if nojs + comments = Comments.fetch_community_post_comments(ucid, id) + comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, isPost: true))["contentHtml"] + end templated "post" end diff --git a/src/invidious/views/post.ecr b/src/invidious/views/post.ecr index 071d1c88..fb03a44c 100644 --- a/src/invidious/views/post.ecr +++ b/src/invidious/views/post.ecr @@ -2,10 +2,27 @@ Invidious <% end %> -
- <%= IV::Frontend::Comments.template_youtube(post_response.not_nil!, locale, thin_mode) %> -
-
+
+
+ <%= IV::Frontend::Comments.template_youtube(post_response.not_nil!, locale, thin_mode) %> +
+ + <% if nojs %> +
+ <% end %> +
+ +
+ <% if nojs %> + <%= comment_html %> + <% else %> + + <% end %> +