项目6 爬取商城网站数据

spider.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import urllib.request
import threading
import sqlite3
import os
import time

class MySpider:
    #设置Use-Agenet值，模拟浏览器
    headers = {
        "User-Agent": "Mozilla/6.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre"}
    #下载图像的路径
    imagePath = "download"

    def startUp(self):
        #初始化函数
        # Initializing Chrome browser
        options = Options()
        options.add_argument('--headless')
        options.add_argument('--disable-gpu')
        self.driver = webdriver.Chrome(options=options)
        self.driver.maximize_window()

        # Initializing variables
        #初始化线程列表
        self.threads = []
        self.No = 0

        # Initializing database
        #初始化数据库，建立phone表
        try:
            self.con = sqlite3.connect("phones.db")
            self.cursor = self.con.cursor()
            try:
                self.cursor.execute("drop table phones")
            except:
                pass
            try:
                sql = "create table phones (mNo varchar(8) primary key,mMark varchar(256),mPrice varchar(64),mNote varchar(1024),mFile varchar(32))"
                self.cursor.execute(sql)
            except:
                pass
        except Exception as err:
            print(err)

        # Initializing images folder
        #初始化图像文件夹，删除原有的图像
        try:
            if not os.path.exists(MySpider.imagePath):
                os.mkdir(MySpider.imagePath)
            images = os.listdir(MySpider.imagePath)
            for img in images:
                s = os.path.join(MySpider.imagePath, img)
                os.remove(s)
        except Exception as err:
            print(err)

    def closeUp(self):
        #结束函数，关闭数据库与浏览器
        try:
            self.con.commit()
            self.con.close()
            self.driver.close()
        except Exception as err:
            print(err);

    def insertDB(self, mNo, mMark, mPrice, mNote, mFile):
        #插入数据记录
        try:
            sql = "insert into phones (mNo,mMark,mPrice,mNote,mFile) values (?,?,?,?,?)"
            self.cursor.execute(sql, (mNo, mMark, mPrice, mNote, mFile))
        except Exception as err:
            print(err)

    def showDB(self):
        #显示爬取数据
        try:
            print("%8s %16s %8s %16s %s" % ("No", "Mark", "Price", "Image", "Note"))
            self.cursor.execute("select mNo,mMark,mPrice,mFile,mNote from phones order by mNo")
            rows = self.cursor.fetchall()
            for row in rows:
                print("%8s %16s %16s %16s %s" % (row[0], row[1], row[2], row[3], row[4]))
        except Exception as err:
            print(err)

    def download (self, src,mFile):
        #下载图像
        try:
            #print("Downloading " , src)
            req = urllib.request.Request(src, headers=MySpider.headers)
            data = urllib.request.urlopen(req, timeout=400)
            data = data.read()
            fobj = open(MySpider.imagePath + "\\" + mFile, "wb")
            fobj.write(data)
            fobj.close()
            print("Downloaded " , src)
        except Exception as err:
            print(mFile + " " + str(err) + " src=" + src)

    def processSpider(self):
        #爬取过程函数
        try:
            print(self.driver.current_url)
            divs = self.driver.find_elements(by=By.XPATH,value="//div[@class='phone']")
            for div in divs:
                #爬取数据
                src = div.find_element(by=By.XPATH,value=".//div[position()=2]//img").get_attribute("src")
                src = urllib.request.urljoin(self.driver.current_url, src)
                mark = div.find_element(by=By.XPATH,value=".//div[position()=3]").text
                price = div.find_element(by=By.XPATH,value=".//div[position()=4]").text
                note = div.find_element(by=By.XPATH,value=".//div[position()=5]").text

                self.No = self.No + 1
                no = str(self.No)
                while len(no) < 6:
                    no = "0" + no
                p=src.rfind(".")
                mFile=no+src[p:]
                self.insertDB(no, mark, price, note, mFile)
                
                #启动下载图像子线程
                T=threading.Thread(target=self.download,args=[src,mFile])
                T.start()
                self.threads.append(T)
            
            #实现翻页
            try:
                link = self.driver.find_elements(by=By.XPATH,value="//div[@class='paging']//input[@type='button']")[-2]
                if link.is_enabled():
                    link.click()
                    time.sleep(0.1)
                    self.processSpider()
            except Exception as err:
                print(err)
        except Exception as err:
            print(err)

    def executeSpider(self, url):
        #爬取函数
        print("Spider starting......")
        self.startUp()
        self.driver.get(url)
        self.processSpider()
        #等待所有线程结束
        for T in self.threads:
            T.join()
        print("Spider completed......")
        self.showDB()
        self.closeUp()

#主程序
spider = MySpider()
#spider.executeSpider("http://127.0.0.1:5000")
spider.executeSpider("http://www.rj-jwb.com.cn/spiderbook/chapter6/")