DOWNLOADER_MIDDLEWARES = {

    'dbdm.middlewares.DbdmSpiderMiddleware': 543,

}

上面需要加载selenium 
from selenium import webdriver
#........省略部分代码 
@classmethod

    def process_request(cls, request, spider):

        #if request.meta.has_key('PhantomJS'):

        driver = webdriver.PhantomJS('E:\\p_python\\Scripts\\phantomjs\\bin\\phantomjs.exe')

        driver.get(request.url)

        if request.url=='https://movie.douban.com/tag':

            driver.find_element_by_xpath('//*[@id="app"]/div/div[1]/div[1]/ul[1]/li[5]/span').click()

            time.sleep(5)

            if driver.find_element_by_xpath('//*[@id="app"]/div/div[1]/a'):

                click_more(driver)

        content = driver.page_source.encode('utf-8')

        #print content

        #file = open(path.join(d, '1.txt'),'w')

        #file.write(content)

        #file.close()

        driver.quit()

        return HtmlResponse(request.url, encoding='utf-8', body=content, request=request)

def click_more(driver,i=1):

    driver.find_element_by_xpath('//*[@id="app"]/div/div[1]/a').click()

    print str(i)+'  click'

    time.sleep(5)

    i = i+1

    try:

        more_btn = driver.find_element_by_xpath('//*[@id="app"]/div/div[1]/a')

        if more_btn:

            click_more(driver,i)

    except:

        print 'click Over!!'

上面需要加载selenium 
from selenium import webdriver 

#........省略部分代码 

@classmethod

    def process_request(cls, request, spider):

        if request.meta.has_key('PhantomJS'):

            driver = webdriver.PhantomJS('E:\\p_python\\Scripts\\phantomjs\\bin\\phantomjs.exe')

            driver.get(request.url)

            content = driver.page_source.encode('utf-8')

            driver.quit()

            return HtmlResponse(request.url, encoding='utf-8', body=content, request=request)

import scrapy

from scrapy.linkextractors import LinkExtractor

from scrapy.spiders import CrawlSpider, Rule

from phantomjs_test.items import PhantomscrapyItem

class PhantomjsTestSpider(CrawlSpider):

    name = 'phantomjs_test'

    allowed_domains = ['book.com']

    start_urls = ['http://book.com/']

    #all_urls = []   去重似乎不需要

     rules = (

        ###获取所有的分页列表

        Rule(LinkExtractor(allow=r'/story/p/[2-9]*'), callback='parse', follow=True),

        ###获取里面所有的详情页

        #Rule(LinkExtractor(allow=r'/detail/p/[2-9]*'), callback = 'parse_item',follow=True),

    )

    ###从分页页面获取所有的文章url

    def parse(self, response):

        url_list = response.xpath('/a/@href').extract()

        for url in url_list:

            request = Request(url=url, callback=self.parse_item, dont_filter=True)

            request.meta['PhantomJS'] = True

            yield request

    def parse_item(self, response):

        item = PhantomscrapyItem()

        #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()

        #i['name'] = response.xpath('//div[@id="name"]').extract()

        #i['description'] = response.xpath('//div[@id="description"]').extract()

        item['bookName'] = response.xpath()

        items = []

        items.append(item)

        return items