From d300797e229e12973559334cc53a17f79a27ac90 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 7 Oct 2021 22:17:24 +0200 Subject: [PATCH] Move the YoutubeConnectionPool code to its own file --- src/invidious/helpers/utils.cr | 81 --------------------- src/invidious/yt_backend/connection_pool.cr | 81 +++++++++++++++++++++ 2 files changed, 81 insertions(+), 81 deletions(-) create mode 100644 src/invidious/yt_backend/connection_pool.cr diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 68ba76f9..6100d403 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,4 +1,3 @@ -require "lsquic" require "db" def add_yt_headers(request) @@ -16,55 +15,6 @@ def add_yt_headers(request) end end -struct YoutubeConnectionPool - property! url : URI - property! capacity : Int32 - property! timeout : Float64 - property pool : DB::Pool(QUIC::Client | HTTP::Client) - - def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) - @url = url - @pool = build_pool(use_quic) - end - - def client(region = nil, &block) - if region - conn = make_client(url, region) - response = yield conn - else - conn = pool.checkout - begin - response = yield conn - rescue ex - conn.close - conn = QUIC::Client.new(url) - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - response = yield conn - ensure - pool.release(conn) - end - end - - response - end - - private def build_pool(use_quic) - DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - if use_quic - conn = QUIC::Client.new(url) - else - conn = HTTP::Client.new(url) - end - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - conn - end - end -end - # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html def ci_lower_bound(pos, n) if n == 0 @@ -85,37 +35,6 @@ def elapsed_text(elapsed) "#{(millis * 1000).round(2)}µs" end -def make_client(url : URI, region = nil) - # TODO: Migrate any applicable endpoints to QUIC - client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - if region - PROXY_LIST[region]?.try &.sample(40).each do |proxy| - begin - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - client.set_proxy(proxy) - break - rescue ex - end - end - end - - return client -end - -def make_client(url : URI, region = nil, &block) - client = make_client(url, region) - begin - yield client - ensure - client.close - end -end - def decode_length_seconds(string) length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i length_seconds = [0] * (3 - length_seconds.size) + length_seconds diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr new file mode 100644 index 00000000..505f2cf6 --- /dev/null +++ b/src/invidious/yt_backend/connection_pool.cr @@ -0,0 +1,81 @@ +require "lsquic" + +struct YoutubeConnectionPool + property! url : URI + property! capacity : Int32 + property! timeout : Float64 + property pool : DB::Pool(QUIC::Client | HTTP::Client) + + def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) + @url = url + @pool = build_pool(use_quic) + end + + def client(region = nil, &block) + if region + conn = make_client(url, region) + response = yield conn + else + conn = pool.checkout + begin + response = yield conn + rescue ex + conn.close + conn = QUIC::Client.new(url) + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC + conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + response = yield conn + ensure + pool.release(conn) + end + end + + response + end + + private def build_pool(use_quic) + DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + if use_quic + conn = QUIC::Client.new(url) + else + conn = HTTP::Client.new(url) + end + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC + conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + conn + end + end +end + +def make_client(url : URI, region = nil) + # TODO: Migrate any applicable endpoints to QUIC + client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) + client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + client.read_timeout = 10.seconds + client.connect_timeout = 10.seconds + + if region + PROXY_LIST[region]?.try &.sample(40).each do |proxy| + begin + proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) + client.set_proxy(proxy) + break + rescue ex + end + end + end + + return client +end + +def make_client(url : URI, region = nil, &block) + client = make_client(url, region) + begin + yield client + ensure + client.close + end +end