项目5 爬取图书网站数据

spiders/SpiderBook.py

import scrapy
from scrapy.selector import Selector
from SpiderBook.items import BookItem
import os

class MySpider(scrapy.Spider):
    name = "mySpider"
    def start_requests(self):
        #初始化图像文件夹，删除原有的图像
        try:
            if not os.path.exists("download"):
                os.mkdir("download")
            images = os.listdir("download")
            for img in images:
                s = os.path.join("download",img)
                os.remove(s)
        except Exception as err:
            print(err)
        #url ='http://127.0.0.1:5000'
        url="http://www.rj-jwb.com.cn/spiderbook/chapter5/"
        self.ID=0
        yield scrapy.Request(url=url,callback=self.parse)

    def getFileExt(self, url):
        p = url.rindex(".")
        if p >= 0:
            s = url[p:]
            if s[len(s) - 1] == "/":
                s = s[0:len(s) - 1]
        else:
            s = ""
        return s
    
    def parse(self, response):
        selector=Selector(text=response.body.decode())
        lis=selector.xpath("//ul/li")
        for li in lis:
            self.ID+=1
            ID="%06d"%self.ID
            div=li.xpath(".//div[@class='info']")
            d=div.xpath(".//div[@class='title']")
            bTitle=d.xpath("./h3/text()").extract_first()
            d=div.xpath(".//div[@class='author']")
            bAuthor=d.xpath("./span[@class='attrs']/text()").extract_first()
            d=div.xpath(".//div[@class='publisher']")
            bPublisher=d.xpath("./span[@class='attrs']/text()").extract_first()
            d=div.xpath(".//div[@class='date']")
            bDate=d.xpath("./span[@class='attrs']/text()").extract_first()
            d=div.xpath(".//div[@class='price']")
            bPrice=d.xpath("./span[@class='attrs']/text()").extract_first()
            bDetail=div.xpath(".//div[@class='detail']/text()").extract_first()
            src=li.xpath(".//div[@class='pic']//img/@src").extract_first()
            if src:
                src=response.urljoin(src).strip("/")
                bImage=ID+self.getFileExt(src)
                request=scrapy.Request(url=src,callback=self.download)
                request.meta["bImage"]=bImage
                yield request
            else:
                bImage=""
            #创建BookItem对象
            item=BookItem()
            item["ID"]=ID
            item["bTitle"]=bTitle.strip() if bTitle else ""
            item["bAuthor"]=bAuthor.strip()  if bAuthor else ""
            item["bPublisher"]=bPublisher.strip()  if bPublisher else ""
            item["bDate"]=bDate.strip()  if bDate else ""
            item["bPrice"]=bPrice.strip()  if bPrice else ""
            item["bDetail"]=bDetail.strip()  if bDetail else ""
            item["bImage"]=bImage
            yield item
        
        href=selector.xpath("//div[@class='paging']//a[position()=last()-1]/@href").extract_first().strip()
        if href!="#":
            href=response.urljoin(href)
            yield scrapy.Request(url=href,callback=self.parse)
        

    def download(self,response):
        bImage = response.meta["bImage"]
        #print("downloading",bImage)
        f=open("download\\"+bImage,"wb")
        f.write(response.body)
        f.close()
        print("downloaded",bImage)

items.py

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy
class BookItem(scrapy.Item):
    # define the fields for your item here like:
    ID=scrapy.Field()
    bTitle = scrapy.Field()
    bAuthor = scrapy.Field()
    bPublisher = scrapy.Field()
    bDate = scrapy.Field()
    bPrice = scrapy.Field()
    bDetail = scrapy.Field()
    bImage = scrapy.Field()

pipeline.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import sqlite3
import os

class BookPipeline(object):
    def open_spider(self,spider):
        print("open_spider")
        self.con=sqlite3.connect("books.db")
        self.cursor=self.con.cursor()
        try:
            self.cursor.execute("drop table books")
        except:
            pass
        sql="create table books (ID varchar(8) primary key,bTitle varchar(256),bAuthor varchar(256),bPublisher varchar(256),bDate varchar(256),bPrice varchar(256),bDetail text,bImage varchar(256))"
        self.cursor.execute(sql)
        self.count=0
        if not os.path.exists("download"):
            os.mkdir("download")

    def close_spider(self,spider):
        print("close_spider")
        print("Total ",self.count)
        #self.showDB()
        self.con.commit()
        self.con.close()

    def insertDB(self,ID,bTitle,bAuthor,bPublisher,bDate,bPrice,bDetail,bImage):
        try:
            sql="insert into books (ID,bTitle,bAuthor,bPublisher,bDate,bPrice,bDetail,bImage) values (?,?,?,?,?,?,?,?)"
            self.cursor.execute(sql,[ID,bTitle,bAuthor,bPublisher,bDate,bPrice,bDetail,bImage])
        except Exception as err:
            print(err)

    def showDB(self):
        self.cursor.execute("select ID,bTitle,bAuthor,bPublisher,bDate,bPrice,bDetail,bImage from books")
        rows=self.cursor.fetchall()
        for row in rows:
            for r in row:
                print(r)

    def process_item(self, item, spider):
        self.count += 1
        ID=item["ID"]
        bTitle=item["bTitle"]
        bAuthor=item["bAuthor"]
        bPublisher=item["bPublisher"]
        bDate=item["bDate"]
        bPrice=item["bPrice"]
        bDetail=item["bDetail"]
        bImage=item["bImage"]
        print(ID,bTitle)
        self.insertDB(ID,bTitle,bAuthor,bPublisher,bDate,bPrice,bDetail,bImage)
        return item

settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for SpiderBook project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'SpiderBook'

SPIDER_MODULES = ['SpiderBook.spiders']
NEWSPIDER_MODULE = 'SpiderBook.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'SpiderBook (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'SpiderBook.middlewares.SpiderbookSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'SpiderBook.middlewares.SpiderbookDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'SpiderBook.pipelines.BookPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

show.py

import sqlite3

try:
    con=sqlite3.connect("books.db")
    cursor=con.cursor()
    cursor.execute("select ID,bTitle from books")
    rows=cursor.fetchall()
    for row in rows:
        print(row[0],"   ",row[1])
    print("Total ",len(rows))
    con.close()
except Exception as err:
    print(err)

runSpider.py

from scrapy import cmdline
cmdline.execute("scrapy crawl mySpider -s LOG_ENABLED=False".split())