iv-org
/
invidious
mirror of https://github.com/iv-org/invidious.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'iv-org:master' into main

This commit is contained in:
broquemonsieur 2023-09-22 02:13:57 +00:00 committed by GitHub
parent 75cc1899cf bb14f79496
commit cd3256fcff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 305 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/ci.yml vendored
View File

@ -38,11 +38,10 @@ jobs:
matrix:
stable: [true]
crystal:
- 1.4.1
- 1.5.1
- 1.6.2
- 1.7.3
- 1.8.2
- 1.9.2
include:
- crystal: nightly
stable: false
@ -53,7 +52,7 @@ jobs:
submodules: true
- name: Install Crystal
uses: crystal-lang/install-crystal@v1.7.0
uses: crystal-lang/install-crystal@v1.8.0
with:
crystal: ${{ matrix.crystal }}

5
.github/workflows/container-release.yml vendored
View File

@ -25,9 +25,9 @@ jobs:
uses: actions/checkout@v3
- name: Install Crystal
uses: crystal-lang/install-crystal@v1.6.0
uses: crystal-lang/install-crystal@v1.8.0
with:
crystal: 1.5.0
crystal: 1.9.2
- name: Run lint
run: |
@ -77,4 +77,3 @@ jobs:
tags: quay.io/invidious/invidious:${{ github.sha }}-arm64,quay.io/invidious/invidious:latest-arm64
build-args: |
"release=1"

2
.github/workflows/stale.yml vendored
View File

@ -14,7 +14,7 @@ jobs:
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
days-before-stale: 365
days-before-pr-stale: 45 # PRs should be active. Anything that hasn't had activity in more than 45 days should be considered abandoned.
days-before-pr-stale: 90
days-before-close: 30
exempt-pr-labels: blocked
stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.'

13
config/config.example.yml
View File

@ -161,6 +161,19 @@ https_only: false
#force_resolve:
##
## Use Innertube's transcripts API instead of timedtext for closed captions
##
## Useful for larger instances as InnerTube is **not ratelimited**. See https://github.com/iv-org/invidious/issues/2567
##
## Subtitle experience may differ slightly on Invidious.
##
## Accepted values: true, false
## Default: false
##
# use_innertube_for_captions: false
# -----------------------------
# Logging
# -----------------------------

3
src/invidious/config.cr
View File

@ -127,6 +127,9 @@ class Config
# Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property pool_size : Int32 = 100
# Use Innertube's transcripts API instead of timedtext for closed captions
property use_innertube_for_captions : Bool = false
# Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new

2
src/invidious/frontend/watch_page.cr
View File

@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage
getter full_videos : Array(Hash(String, JSON::Any))
getter video_streams : Array(Hash(String, JSON::Any))
getter audio_streams : Array(Hash(String, JSON::Any))
getter captions : Array(Invidious::Videos::Caption)
getter captions : Array(Invidious::Videos::Captions::Metadata)
def initialize(
@full_videos,

5
src/invidious/playlists.cr
View File

@ -89,6 +89,7 @@ struct Playlist
property views : Int64
property updated : Time
property thumbnail : String?
property subtitle : String?
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
@ -100,6 +101,7 @@ struct Playlist
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "subtitle", self.subtitle
json.field "authorThumbnails" do
json.array do
@ -356,6 +358,8 @@ def fetch_playlist(plid : String)
updated = Time.utc
video_count = 0
subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle"))
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
@ -397,6 +401,7 @@ def fetch_playlist(plid : String)
views: views,
updated: updated,
thumbnail: thumbnail,
subtitle: subtitle,
})
end

8
src/invidious/routes/api/v1/videos.cr
View File

@ -87,6 +87,13 @@ module Invidious::Routes::API::V1::Videos
caption = caption[0]
end
if CONFIG.use_innertube_for_captions
params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
initial_data = YoutubeAPI.get_transcript(params)
webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code)
else
# Timedtext API handling
url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
# Auto-generated captions often have cues that aren't aligned properly with the video,
@ -153,6 +160,7 @@ module Invidious::Routes::API::V1::Videos
.gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
end
end
end
if title = env.params.query["title"]?
# https://blog.fastmail.com/2011/06/24/download-non-english-filenames/

6
src/invidious/videos.cr
View File

@ -24,7 +24,7 @@ struct Video
property updated : Time
@[DB::Field(ignore: true)]
@captions = [] of Invidious::Videos::Caption
@captions = [] of Invidious::Videos::Captions::Metadata
@[DB::Field(ignore: true)]
property adaptive_fmts : Array(Hash(String, JSON::Any))?
@ -215,9 +215,9 @@ struct Video
keywords.includes? "YouTube Red"
end
def captions : Array(Invidious::Videos::Caption)
def captions : Array(Invidious::Videos::Captions::Metadata)
if @captions.empty? && @info.has_key?("captions")
@captions = Invidious::Videos::Caption.from_yt_json(info["captions"])
@captions = Invidious::Videos::Captions::Metadata.from_yt_json(info["captions"])
end
return @captions

16
src/invidious/videos/caption.cr
View File

@ -1,21 +1,24 @@
require "json"
module Invidious::Videos
struct Caption
module Captions
struct Metadata
property name : String
property language_code : String
property base_url : String
def initialize(@name, @language_code, @base_url)
property auto_generated : Bool
def initialize(@name, @language_code, @base_url, @auto_generated)
end
# Parse the JSON structure from Youtube
def self.from_yt_json(container : JSON::Any) : Array(Caption)
def self.from_yt_json(container : JSON::Any) : Array(Captions::Metadata)
caption_tracks = container
.dig?("playerCaptionsTracklistRenderer", "captionTracks")
.try &.as_a
captions_list = [] of Caption
captions_list = [] of Captions::Metadata
return captions_list if caption_tracks.nil?
caption_tracks.each do |caption|
@ -25,7 +28,9 @@ module Invidious::Videos
language_code = caption["languageCode"].to_s
base_url = caption["baseUrl"].to_s
captions_list << Caption.new(name, language_code, base_url)
auto_generated = (caption["kind"]? == "asr")
captions_list << Captions::Metadata.new(name, language_code, base_url, auto_generated)
end
return captions_list
@ -96,6 +101,7 @@ module Invidious::Videos
end
return result
end
end
# List of all caption languages available on Youtube.
LANGUAGES = {

103
src/invidious/videos/transcript.cr Normal file
View File

@ -0,0 +1,103 @@
module Invidious::Videos
# Namespace for methods primarily relating to Transcripts
module Transcript
record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String
def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
kind = auto_generated ? "asr" : ""
object = {
"1:0:string" => video_id,
"2:base64" => {
"1:string" => kind,
"2:string" => language_code,
"3:string" => "",
},
"3:varint" => 1_i64,
"5:string" => "engagement-panel-searchable-transcript-search-panel",
"6:varint" => 1_i64,
"7:varint" => 1_i64,
"8:varint" => 1_i64,
}
params = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return params
end
def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String
# Convert into array of TranscriptLine
lines = self.parse(initial_data)
# Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt()
vtt = String.build do |vtt|
vtt << <<-END_VTT
WEBVTT
Kind: captions
Language: #{target_language}
END_VTT
vtt << "\n\n"
lines.each do |line|
start_time = line.start_ms
end_time = line.end_ms
# start_time
vtt << start_time.hours.to_s.rjust(2, '0')
vtt << ':' << start_time.minutes.to_s.rjust(2, '0')
vtt << ':' << start_time.seconds.to_s.rjust(2, '0')
vtt << '.' << start_time.milliseconds.to_s.rjust(3, '0')
vtt << " --> "
# end_time
vtt << end_time.hours.to_s.rjust(2, '0')
vtt << ':' << end_time.minutes.to_s.rjust(2, '0')
vtt << ':' << end_time.seconds.to_s.rjust(2, '0')
vtt << '.' << end_time.milliseconds.to_s.rjust(3, '0')
vtt << "\n"
vtt << line.line
vtt << "\n"
vtt << "\n"
end
end
return vtt
end
private def self.parse(initial_data : Hash(String, JSON::Any))
body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
"content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer",
"initialSegments").as_a
lines = [] of TranscriptLine
body.each do |line|
# Transcript section headers. They are not apart of the captions and as such we can safely skip them.
if line.as_h.has_key?("transcriptSectionHeaderRenderer")
next
end
line = line["transcriptSegmentRenderer"]
start_ms = line["startMs"].as_s.to_i.millisecond
end_ms = line["endMs"].as_s.to_i.millisecond
text = extract_text(line["snippet"]) || ""
lines << TranscriptLine.new(start_ms, end_ms, text)
end
return lines
end
end
end

5
src/invidious/views/playlist.ecr
View File

@ -70,7 +70,12 @@
</b>
<% else %>
<b>
<% if !author.empty? %>
<a href="/channel/<%= playlist.ucid %>"><%= author %></a> |
<% elsif !playlist.subtitle.nil? %>
<% subtitle = playlist.subtitle || "" %>
<span><%= HTML.escape(subtitle[0..subtitle.rindex(" • ") || subtitle.size]) %></span> |
<% end %>
<%= translate_count(locale, "generic_videos_count", playlist.video_count) %> |
<%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %>
</b>

2
src/invidious/views/user/preferences.ecr
View File

@ -89,7 +89,7 @@
<label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label>
<% preferences.captions.each_with_index do |caption, index| %>
<select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]">
<% Invidious::Videos::Caption::LANGUAGES.each do |option| %>
<% Invidious::Videos::Captions::LANGUAGES.each do |option| %>
<option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option>
<% end %>
</select>

24
src/invidious/yt_backend/youtube_api.cr
View File

@ -557,6 +557,30 @@ module YoutubeAPI
return self._post_json("/youtubei/v1/search", data, client_config)
end
####################################################################
# get_transcript(params, client_config?)
#
# Requests the youtubei/v1/get_transcript endpoint with the required headers
# and POST data in order to get a JSON reply.
#
# The requested data is a specially encoded protobuf string that denotes the specific language requested.
#
# An optional ClientConfig parameter can be passed, too (see
# `struct ClientConfig` above for more details).
#
def get_transcript(
params : String,
client_config : ClientConfig | Nil = nil
) : Hash(String, JSON::Any)
data = {
"context" => self.make_context(client_config),
"params" => params,
}
return self._post_json("/youtubei/v1/get_transcript", data, client_config)
end
####################################################################
# _post_json(endpoint, data, client_config?)
#