#!/usr/bin/env python3

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class BaiduPanSpider(CrawlSpider):

    name = 'yigushi'
    allowed_domains = ['yigushi.com']
    start_urls = ['http://www.yigushi.com/soft/']

    rules = [
        Rule(LinkExtractor(restrict_xpaths='//div[@class="head-txt"]//a[text()="更多"]'), follow=True),
        Rule(LinkExtractor(allow=r'/\d+\.html', restrict_css='.catlist>.pages'), follow=True),
        Rule(LinkExtractor(allow=r'/\d+\.html', restrict_css='.catlist>ul'), follow=False, callback='parse_item'),
    ]

    def parse_item(self, response):
        return {
            'url': response.url,
            'name': response.css('h1#title::text').get(),
            'link': response.css('#content a[href*="pan.baidu.com"]::attr(href)').get() or response.css('#content a[href^="ed2k://"]::attr(href)').get(),
            'code': response.xpath('//div[@id="content"]').re_first(r'提取(?:密)?码[^a-z0-9]*([a-z0-9]{4})'),
        }