Merge pull request #4008 from syeopite/remove_lsquic

2023-08-04 17:44:49 +02:00 · 2023-08-04 17:44:49 +02:00 · 701f03d5e8
parent d956b1826e a8ba02051b
commit 701f03d5e8
12 changed files with 36 additions and 246 deletions
--- a/.github/workflows/container-release.yml
+++ b/.github/workflows/container-release.yml
@ -52,7 +52,7 @@ jobs:
          username: ${{ secrets.QUAY_USERNAME }}
          password: ${{ secrets.QUAY_PASSWORD }}

-      - name: Build and push Docker AMD64 image without QUIC for Push Event
+      - name: Build and push Docker AMD64 image for Push Event
        if: github.ref == 'refs/heads/master'
        uses: docker/build-push-action@v3
        with:
@ -64,9 +64,8 @@ jobs:
          tags: quay.io/invidious/invidious:${{ github.sha }},quay.io/invidious/invidious:latest
          build-args: |
            "release=1"
-            "disable_quic=1"

-      - name: Build and push Docker ARM64 image without QUIC for Push Event
+      - name: Build and push Docker ARM64 image for Push Event
        if: github.ref == 'refs/heads/master'
        uses: docker/build-push-action@v3
        with:
@ -78,28 +77,4 @@ jobs:
          tags: quay.io/invidious/invidious:${{ github.sha }}-arm64,quay.io/invidious/invidious:latest-arm64
          build-args: |
            "release=1"
-            "disable_quic=1"
        
-      - name: Build and push Docker AMD64 image with QUIC for Push Event
-        if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: docker/Dockerfile
-          platforms: linux/amd64
-          labels: quay.expires-after=12w
-          push: true
-          tags: quay.io/invidious/invidious:${{ github.sha }}-quic,quay.io/invidious/invidious:latest-quic
-          build-args: release=1
-
-      - name: Build and push Docker ARM64 image with QUIC for Push Event
-        if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: docker/Dockerfile.arm64
-          platforms: linux/arm64/v8
-          labels: quay.expires-after=12w
-          push: true
-          tags: quay.io/invidious/invidious:${{ github.sha }}-arm64-quic,quay.io/invidious/invidious:latest-arm64-quic
-          build-args: release=1
--- a/6
+++ b/6
@ -5,7 +5,6 @@
 RELEASE  := 1
 STATIC   := 0

-DISABLE_QUIC := 1
 NO_DBG_SYMBOLS := 0


@ -27,10 +26,6 @@ else
  FLAGS += --debug
 endif

-ifeq ($(DISABLE_QUIC), 1)
-  FLAGS += -Ddisable_quic
-endif
-
 ifeq ($(API_ONLY), 1)
  FLAGS += -Dapi_only
 endif
@ -115,7 +110,6 @@ help:
 	@echo "  STATIC           Link libraries statically       (Default: 0)"
 	@echo ""
 	@echo "  API_ONLY         Build invidious without a GUI   (Default: 0)"
-	@echo "  DISABLE_QUIC     Disable support for QUIC        (Default: 0)"
 	@echo "  NO_DBG_SYMBOLS   Strip debug symbols             (Default: 0)"


--- a/config/config.example.yml
+++ b/config/config.example.yml
@ -140,27 +140,6 @@ https_only: false
 ##
 #pool_size: 100

-##
-## Enable/Disable the use of QUIC (HTTP/3) when connecting
-## to the youtube API and websites ('youtube.com', 'ytimg.com').
-## QUIC's main advantages are its lower latency and lower bandwidth
-## use, compared to its predecessors. However, the current version
-## of QUIC used in invidious is still based on the IETF draft 31,
-## meaning that the underlying library may still not be fully
-## optimized. You can read more about QUIC at the link below:
-## https://datatracker.ietf.org/doc/html/draft-ietf-quic-transport-31
-##
-## Note: you should try both options and see what is the best for your
-## instance. In general QUIC is recommended for public instances. Your
-## mileage may vary.
-##
-## Note 2: Using QUIC prevents some captcha challenges from appearing.
-## See: https://github.com/iv-org/invidious/issues/957#issuecomment-576424042
-##
-## Accepted values: true, false
-## Default: false
-##
-#use_quic: false

 ##
 ## Additional cookies to be sent when requesting the youtube API.
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -2,15 +2,12 @@ FROM crystallang/crystal:1.4.1-alpine AS builder
 RUN apk add --no-cache sqlite-static yaml-static

 ARG release
-ARG disable_quic

 WORKDIR /invidious
 COPY ./shard.yml ./shard.yml
 COPY ./shard.lock ./shard.lock
 RUN shards install --production

-COPY --from=quay.io/invidious/lsquic-compiled /root/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a
-
 COPY ./src/ ./src/
 # TODO: .git folder is required for building – this is destructive.
 # See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION.
@ -24,13 +21,7 @@ COPY ./videojs-dependencies.yml ./videojs-dependencies.yml
 RUN crystal spec --warnings all \
    --link-flags "-lxml2 -llzma"

-RUN if [[ "${release}" == 1 && "${disable_quic}" == 1 ]] ; then \
-        crystal build ./src/invidious.cr \
-        --release \
-        -Ddisable_quic \
-        --static --warnings all \
-        --link-flags "-lxml2 -llzma"; \
-    elif [[ "${release}" == 1 ]] ; then \
+RUN if [[ "${release}" == 1 ]] ; then \
        crystal build ./src/invidious.cr \
        --release \
        --static --warnings all \
--- a/docker/Dockerfile.arm64
+++ b/docker/Dockerfile.arm64
@ -2,15 +2,12 @@ FROM alpine:3.16 AS builder
 RUN apk add --no-cache 'crystal=1.4.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev

 ARG release
-ARG disable_quic

 WORKDIR /invidious
 COPY ./shard.yml ./shard.yml
 COPY ./shard.lock ./shard.lock
 RUN shards install --production

-COPY --from=quay.io/invidious/lsquic-compiled /root/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a
-
 COPY ./src/ ./src/
 # TODO: .git folder is required for building – this is destructive.
 # See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION.
@ -24,13 +21,7 @@ COPY ./videojs-dependencies.yml ./videojs-dependencies.yml
 RUN crystal spec --warnings all \
    --link-flags "-lxml2 -llzma"

-RUN if [[ "${release}" == 1 && "${disable_quic}" == 1 ]] ; then \
-        crystal build ./src/invidious.cr \
-        --release \
-        -Ddisable_quic \
-        --static --warnings all \
-        --link-flags "-lxml2 -llzma"; \
-    elif [[ "${release}" == 1 ]] ; then \
+RUN if [[ "${release}" == 1 ]] ; then \
        crystal build ./src/invidious.cr \
        --release \
        --static --warnings all \
--- a/shard.lock
+++ b/shard.lock
@ -24,10 +24,6 @@ shards:
    git: https://github.com/jeromegn/kilt.git
    version: 0.6.1

-  lsquic:
-    git: https://github.com/iv-org/lsquic.cr.git
-    version: 2.18.1-2
-
  pg:
    git: https://github.com/will/crystal-pg.git
    version: 0.24.0
--- a/shard.yml
+++ b/shard.yml
@ -25,9 +25,6 @@ dependencies:
  protodec:
    github: iv-org/protodec
    version: ~> 0.1.5
-  lsquic:
-    github: iv-org/lsquic.cr
-    version: ~> 2.18.1-2
  athena-negotiation:
    github: athena-framework/negotiation
    version: ~> 0.1.1
--- a/src/invidious.cr
+++ b/src/invidious.cr
@ -90,7 +90,7 @@ SOFTWARE = {
  "branch"  => "#{CURRENT_BRANCH}",
 }

-YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size, use_quic: CONFIG.use_quic)
+YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)

 # CLI
 Kemal.config.extra_options do |parser|
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@ -126,8 +126,6 @@ class Config
  property host_binding : String = "0.0.0.0"
  # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
  property pool_size : Int32 = 100
-  # Use quic transport for youtube api
-  property use_quic : Bool = false

  # Saved cookies in "name1=value1; name2=value2..." format
  @[YAML::Field(converter: Preferences::StringToCookies)]
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@ -3,17 +3,7 @@ module Invidious::Routes::Images
  def self.ggpht(env)
    url = env.request.path.lchop("/ggpht")

-    headers = (
-      {% unless flag?(:disable_quic) %}
-        if CONFIG.use_quic
-          HTTP::Headers{":authority" => "yt3.ggpht.com"}
-        else
-          HTTP::Headers.new
-        end
-      {% else %}
-        HTTP::Headers.new
-      {% end %}
-    )
+    headers = HTTP::Headers.new

    REQUEST_HEADERS_WHITELIST.each do |header|
      if env.request.headers[header]?
@ -42,22 +32,9 @@ module Invidious::Routes::Images
    }

    begin
-      {% unless flag?(:disable_quic) %}
-        if CONFIG.use_quic
-          YT_POOL.client &.get(url, headers) do |resp|
-            return request_proc.call(resp)
-          end
-        else
      HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
        return request_proc.call(resp)
      end
-        end
-      {% else %}
-        # This can likely be optimized into a (small) pool sometime in the future.
-        HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
-          return request_proc.call(resp)
-        end
-      {% end %}
    rescue ex
    end
  end
@ -78,10 +55,6 @@ module Invidious::Routes::Images

    headers = HTTP::Headers.new

-    {% unless flag?(:disable_quic) %}
-      headers[":authority"] = "#{authority}.ytimg.com"
-    {% end %}
-
    REQUEST_HEADERS_WHITELIST.each do |header|
      if env.request.headers[header]?
        headers[header] = env.request.headers[header]
@ -107,22 +80,9 @@ module Invidious::Routes::Images
    }

    begin
-      {% unless flag?(:disable_quic) %}
-        if CONFIG.use_quic
-          YT_POOL.client &.get(url, headers) do |resp|
-            return request_proc.call(resp)
-          end
-        else
      HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
        return request_proc.call(resp)
      end
-        end
-      {% else %}
-        # This can likely be optimized into a (small) pool sometime in the future.
-        HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
-          return request_proc.call(resp)
-        end
-      {% end %}
    rescue ex
    end
  end
@ -133,17 +93,7 @@ module Invidious::Routes::Images
    name = env.params.url["name"]
    url = env.request.resource

-    headers = (
-      {% unless flag?(:disable_quic) %}
-        if CONFIG.use_quic
-          HTTP::Headers{":authority" => "i9.ytimg.com"}
-        else
-          HTTP::Headers.new
-        end
-      {% else %}
-        HTTP::Headers.new
-      {% end %}
-    )
+    headers = HTTP::Headers.new

    REQUEST_HEADERS_WHITELIST.each do |header|
      if env.request.headers[header]?
@ -169,22 +119,9 @@ module Invidious::Routes::Images
    }

    begin
-      {% unless flag?(:disable_quic) %}
-        if CONFIG.use_quic
-          YT_POOL.client &.get(url, headers) do |resp|
-            return request_proc.call(resp)
-          end
-        else
      HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
        return request_proc.call(resp)
      end
-        end
-      {% else %}
-        # This can likely be optimized into a (small) pool sometime in the future.
-        HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
-          return request_proc.call(resp)
-        end
-      {% end %}
    rescue ex
    end
  end
@ -223,41 +160,16 @@ module Invidious::Routes::Images
    id = env.params.url["id"]
    name = env.params.url["name"]

-    headers = (
-      {% unless flag?(:disable_quic) %}
-        if CONFIG.use_quic
-          HTTP::Headers{":authority" => "i.ytimg.com"}
-        else
-          HTTP::Headers.new
-        end
-      {% else %}
-        HTTP::Headers.new
-      {% end %}
-    )
+    headers = HTTP::Headers.new

    if name == "maxres.jpg"
      build_thumbnails(id).each do |thumb|
        thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
-        # Logic here is short enough that manually typing them out should be fine.
-        {% unless flag?(:disable_quic) %}
-          if CONFIG.use_quic
-            if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200
-              name = thumb[:url] + ".jpg"
-              break
-            end
-          else
-            if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
-              name = thumb[:url] + ".jpg"
-              break
-            end
-          end
-        {% else %}
        # This can likely be optimized into a (small) pool sometime in the future.
        if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
          name = thumb[:url] + ".jpg"
          break
        end
-        {% end %}
      end
    end

@ -287,22 +199,10 @@ module Invidious::Routes::Images
    }

    begin
-      {% unless flag?(:disable_quic) %}
-        if CONFIG.use_quic
-          YT_POOL.client &.get(url, headers) do |resp|
-            return request_proc.call(resp)
-          end
-        else
-          HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
-            return request_proc.call(resp)
-          end
-        end
-      {% else %}
      # This can likely be optimized into a (small) pool sometime in the future.
      HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
        return request_proc.call(resp)
      end
-      {% end %}
    rescue ex
    end
  end
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@ -1,11 +1,3 @@
-{% unless flag?(:disable_quic) %}
-  require "lsquic"
-
-  alias HTTPClientType = QUIC::Client | HTTP::Client
-{% else %}
-  alias HTTPClientType = HTTP::Client
-{% end %}
-
 def add_yt_headers(request)
  if request.headers["User-Agent"] == "Crystal"
    request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
@ -26,11 +18,11 @@ struct YoutubeConnectionPool
  property! url : URI
  property! capacity : Int32
  property! timeout : Float64
-  property pool : DB::Pool(HTTPClientType)
+  property pool : DB::Pool(HTTP::Client)

-  def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true)
+  def initialize(url : URI, @capacity = 5, @timeout = 5.0)
    @url = url
-    @pool = build_pool(use_quic)
+    @pool = build_pool()
  end

  def client(region = nil, &block)
@ -43,11 +35,7 @@ struct YoutubeConnectionPool
        response = yield conn
      rescue ex
        conn.close
-        {% unless flag?(:disable_quic) %}
-          conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url)
-        {% else %}
        conn = HTTP::Client.new(url)
-        {% end %}

        conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
        conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
@ -61,19 +49,9 @@ struct YoutubeConnectionPool
    response
  end

-  private def build_pool(use_quic)
-    DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
-      conn = nil # Declare
-      {% unless flag?(:disable_quic) %}
-        if use_quic
-          conn = QUIC::Client.new(url)
-        else
+  private def build_pool
+    DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
      conn = HTTP::Client.new(url)
-        end
-      {% else %}
-        conn = HTTP::Client.new(url)
-      {% end %}
-
      conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
      conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
      conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@ -83,7 +61,6 @@ struct YoutubeConnectionPool
 end

 def make_client(url : URI, region = nil)
-  # TODO: Migrate any applicable endpoints to QUIC
  client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
  client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
  client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
--- a/src/invidious/yt_backend/youtube_api.cr
+++ b/src/invidious/yt_backend/youtube_api.cr
@ -595,19 +595,11 @@ module YoutubeAPI
    LOGGER.trace("YoutubeAPI: POST data: #{data}")

    # Send the POST request
-    if {{ !flag?(:disable_quic) }} && CONFIG.use_quic
-      # Using QUIC client
-      body = YT_POOL.client(client_config.proxy_region,
-        &.post(url, headers: headers, body: data.to_json)
-      ).body
-    else
-      # Using HTTP client
    body = YT_POOL.client(client_config.proxy_region) do |client|
      client.post(url, headers: headers, body: data.to_json) do |response|
        self._decompress(response.body_io, response.headers["Content-Encoding"]?)
      end
    end
-    end

    # Convert result to Hash
    initial_data = JSON.parse(body).as_h