Get videos from search

This commit is contained in:
Omar Roth 2018-01-21 11:04:58 -06:00
parent daba77d665
commit 02b6990240
1 changed files with 33 additions and 8 deletions

View File

@ -21,23 +21,45 @@ POOL = Deque.new(30) do
client client
end end
# Refresh all the connections in the pool by crawling recommended # Refresh connections by crawling YT
spawn do spawn do
# Arbitrary start value # Start video
id = Deque.new(50, "0xjKNDMgE54") id = Deque.new(10, "_wbqqI0IgY8")
client = get_client
random = Random.new
html = client.get("https://www.youtube.com/results?q=#{random.base64(3)}&sp=EgIQAVAU").body
html = XML.parse_html(html)
html.xpath_nodes(%q(//ol[@class="item-section"]/li)).each do |item|
root = item.xpath_node(%q(div[contains(@class,"yt-lockup-video")]/div))
if root
link = root.xpath_node(%q(div[contains(@class,"yt-lockup-thumbnail")]/a/@href))
if link
id << link.content.split("=")[1]
id.shift
end
end
end
POOL << client
loop do loop do
if rand(600) < 1
client = get_client client = get_client
if rand(100) < 1
client = HTTP::Client.new(URL, CONTEXT) client = HTTP::Client.new(URL, CONTEXT)
client.connect_timeout = Time::Span.new(0, 0, 0, 5) client.connect_timeout = Time::Span.new(0, 0, 0, 5)
POOL << client
end end
time = Time.now time = Time.now
begin begin
video = get_video(id[rand(id.size)], false) i = id[rand(id.size)]
video = get_video(i, false)
id.delete(i)
rescue ex rescue ex
puts ex puts ex
POOL << client
next next
end end
@ -50,13 +72,16 @@ spawn do
rvs.each do |rv| rvs.each do |rv|
if rv.has_key?("id") if rv.has_key?("id")
if !PG_DB.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", rv["id"], as: Bool)
id << rv["id"] id << rv["id"]
if id.size == 50
id.shift id.shift
end end
end end
end
end
POOL << client puts "#{Time.now} 200 GET www.youtube.com/watch?v=#{video.id} #{elapsed_text(Time.now - time)}"
puts "#{Time.now} 200 GET youtube.com/watch?v=#{video.id} #{elapsed_text(Time.now - time)}"
end end
end end