# -*- coding: utf-8 -*-
import scrapy
import re
from urllib.parse import urljoin
from ..items import ScrapyItem
class DocsScrapySpider(scrapy.Spider):
name = 'docs.scrapy'
allowed_domains = ['docs.scrapy']
start_urls = ['https://docs.scrapy/en/latest/index.html']
def parse(self, response):
docs = response.xpath('//div[@class="document"]').extract_first()
print(docs)
scrapy_docs = ScrapyItem()
scrapy_docs["docs_scrapy"] = docs
yield scrapy_docs
next_page = response.xpath('//a[@rel="next"]/@href').extract_first()
if next_page:
next_page = urljoin(response.url, next_page)
print(next_page)
yield response.follow(next_page, self.parse)
urljoin
- 引入urllib.parse模块下的urljoin,
- 获取当前页的链接response.url
- 将当前页的地址和下一页的相对路径地址拼接,从而获取下一页的链接
转载于:https://wwwblogs/chenliang0309/p/10030723.html
更多推荐
通过相对路径获取下一页的链接
发布评论