iv-org
/
invidious
mirror of https://github.com/iv-org/invidious.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Clean up channel feed

This commit is contained in:
Omar Roth 2018-08-10 08:38:31 -05:00
parent 9fbab6125a
commit 09d4d46fcd
3 changed files with 108 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
src/invidious.cr
View File

@ -1279,33 +1279,31 @@ get "/feed/channel/:ucid" do |env|
ucid = env.params.url["ucid"]
client = make_client(YT_URL)
if !ucid.match(/UC[a-zA-Z0-9_-]{22}/)
rss = client.get("/feeds/videos.xml?user=#{ucid}").body
rss = XML.parse_html(rss)
if !ucid.match(/UC[a-zA-Z0-9_-]{22}/)
rss = client.get("/feeds/videos.xml?user=#{ucid}")
rss = XML.parse_html(rss.body)
ucid = rss.xpath_node("//feed/channelid")
if ucid
ucid = ucid.content
else
env.response.content_type = "application/json"
next {"error" => "User does not exist"}.to_json
if !ucid
error_message = "User does not exist."
halt env, status_code: 404, response: error_message
end
next env.redirect "/channel/#{ucid}"
end
url = produce_videos_url(ucid)
response = client.get(url)
response = JSON.parse(response.body)
if !response["content_html"]?
error_message = "This channel does not exist."
halt env, status_code: 404, response: error_message
end
content_html = response["content_html"].as_s
document = XML.parse_html(content_html)
channel = get_channel(ucid, client, PG_DB, pull_all_videos: false)
json = JSON.parse(response.body)
if !json["content_html"]? || json["content_html"].as_s.empty?
error_message = "This channel does not exist or has no videos."
next templated "error"
end
content_html = json["content_html"].as_s
document = XML.parse_html(content_html)
host_url = make_host_url(Kemal.config.ssl || CONFIG.https_only, env.request.headers["Host"]?)
path = env.request.path
@ -1323,77 +1321,30 @@ get "/feed/channel/:ucid" do |env|
xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" }
end
document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])).each do |node|
anchor = node.xpath_node(%q(.//h3[contains(@class,"yt-lockup-title")]/a))
if !anchor
next
end
if anchor["href"].starts_with? "https://www.googleadservices.com"
next
end
title = anchor.content.strip
video_id = anchor["href"].lchop("/watch?v=")
metadata = node.xpath_nodes(%q(.//div[contains(@class,"yt-lockup-meta")]/ul/li))
if metadata.size == 0
next
elsif metadata.size == 1
view_count = metadata[0].content.split(" ")[0].delete(",").to_i64
published = Time.now
else
published = decode_date(metadata[0].content)
view_count = metadata[1].content.split(" ")[0]
if view_count == "No"
view_count = 0_i64
else
view_count = view_count.delete(",").to_i64
end
end
description_html = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-description")]))
if !description_html
description = ""
description_html = ""
else
description_html = description_html.to_s
description = description_html.gsub("<br>", "\n")
description = description.gsub("<br/>", "\n")
description = XML.parse_html(description).content.strip("\n ")
end
length_seconds = node.xpath_node(%q(.//span[@class="video-time"]))
if length_seconds
length_seconds = decode_length_seconds(length_seconds.content)
else
length_seconds = -1
end
extract_channel_videos(document, channel.author, ucid).each do |video|
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{video_id}" }
xml.element("yt:videoId") { xml.text video_id }
xml.element("id") { xml.text "yt:video:#{video.id}" }
xml.element("yt:videoId") { xml.text video.id }
xml.element("yt:channelId") { xml.text ucid }
xml.element("title") { xml.text title }
xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{video_id}")
xml.element("title") { xml.text video.title }
xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{video.id}")
xml.element("author") do
xml.element("name") { xml.text channel.author }
xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" }
end
xml.element("published") { xml.text published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("published") { xml.text video.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text title }
xml.element("media:thumbnail", url: "https://i.ytimg.com/vi/#{video_id}/hqdefault.jpg",
width: "480", height: "360")
xml.element("media:description") { xml.text description }
xml.element("media:title") { xml.text video.title }
xml.element("media:thumbnail", url: "https://i.ytimg.com/vi/#{video.id}/mqdefault.jpg",
width: "320", height: "180")
xml.element("media:description") { xml.text video.description }
end
xml.element("media:community") do
xml.element("media:statistics", views: view_count)
xml.element("media:statistics", views: video.views)
end
end
end
@ -1492,8 +1443,8 @@ get "/feed/private" do |env|
xml.element("media:group") do
xml.element("media:title") { xml.text video.title }
xml.element("media:thumbnail", url: "https://i.ytimg.com/vi/#{video.id}/hqdefault.jpg",
width: "480", height: "360")
xml.element("media:thumbnail", url: "https://i.ytimg.com/vi/#{video.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end

66
src/invidious/channels.cr
View File

@ -130,3 +130,69 @@ def fetch_channel(ucid, client, db, pull_all_videos = true)
return channel
end
def extract_channel_videos(document, author, ucid)
channel_videos = [] of Video
document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])).each do |node|
anchor = node.xpath_node(%q(.//h3[contains(@class,"yt-lockup-title")]/a))
if !anchor
next
end
if anchor["href"].starts_with? "https://www.googleadservices.com"
next
end
title = anchor.content.strip
id = anchor["href"].lchop("/watch?v=")
metadata = node.xpath_nodes(%q(.//div[contains(@class,"yt-lockup-meta")]/ul/li))
if metadata.size == 0
next
elsif metadata.size == 1
view_count = metadata[0].content.split(" ")[0].delete(",").to_i64
published = Time.now
else
published = decode_date(metadata[0].content)
view_count = metadata[1].content.split(" ")[0]
if view_count == "No"
view_count = 0_i64
else
view_count = view_count.delete(",").to_i64
end
end
description_html = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-description")]))
description = description_html_to_description(description_html)
length_seconds = node.xpath_node(%q(.//span[@class="video-time"]))
if length_seconds
length_seconds = decode_length_seconds(length_seconds.content)
else
length_seconds = -1
end
info = HTTP::Params.parse("length_seconds=#{length_seconds}")
channel_videos << Video.new(
id,
info,
Time.now,
title,
view_count,
0, # Like count
0, # Dislike count
0.0, # Wilson score
published,
description,
"", # Language,
author,
ucid,
[] of String, # Allowed regions
true, # Is family friendly
"" # Genre
)
end
return channel_videos
end

14
src/invidious/helpers/helpers.cr
View File

@ -272,3 +272,17 @@ def generate_captcha(key)
return {challenge: challenge, token: token}
end
def description_html_to_description(description_html)
if !description_html
description = ""
description_html = ""
else
description_html = description_html.to_s
description = description_html.gsub("<br>", "\n")
description = description.gsub("<br/>", "\n")
description = XML.parse_html(description).content.strip("\n ")
end
return description
end