upstreams
/
ua
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

scraper-yggtorrent: bugfix

rrix
Simon Lipp 2017-08-12 19:30:11 +02:00
parent 2d296bb526
commit f077c13882
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapers/ua-scraper-yggtorrent
View File

@ -19,7 +19,7 @@ class YggTorrent(scrapy.Spider):
def parse_item(self, response, title):
print((json.dumps({
'title': title,
'title': title.strip(),
'body': '%s<p><a href="%s">View torrent</a></p>' % (response.css('#description').extract()[0], response.url),
'id': response.url,
'host': 'yggtorrent.com',
@ -27,10 +27,12 @@ class YggTorrent(scrapy.Spider):
})))
def parse(self, response):
def details_request(url, title):
return scrapy.Request(url, lambda response: self.parse_item(response, title))
for item in response.css('a.torrent-name'):
url = urllib.parse.urljoin(response.url, item.css('::attr("href")').extract_first())
title = item.css('::text').extract_first()
yield scrapy.Request(url, lambda response: self.parse_item(response, title))
yield details_request(url, title)
if __name__ == "__main__":
from scrapy.commands.runspider import Command