Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import scrapy
- from lesson_03_scraping.items import Lesson03ScrapingItem
- from scrapy.loader import ItemLoader
- class QuotesSpider(scrapy.Spider):
- name = "quotes_pt4"
- start_urls = ['http://quotes.toscrape.com/page/1/']
- def parse(self, response):
- for quote in response.css('div.quote'):
- loader = ItemLoader(item=Lesson03ScrapingItem(), selector=quote)
- loader.add_css('quote_content', 'span.text::text')
- loader.add_css('author', 'small.author::text')
- loader.add_css('tags', 'div.tags a.tag::text')
- quote_item = loader.load_item()
- yield quote_item
- next_page = response.css('li.next a::attr(href)').get()
- if next_page is not None:
- next_page = response.urljoin(next_page)
- yield scrapy.Request(next_page, callback=self.parse)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement