项目3 爬取天气网站数据

from lxml import etree
import urllib.request
import sqlite3

class MySpider:
    def openDB(self):
        #初始化数据库
        self.con = sqlite3.connect("weathers.db")
        self.cursor = self.con.cursor()
        try:
            self.cursor.execute("drop table weathers")
        except:
            pass
        sql='''
        create table weathers (
        ID INTEGER PRIMARY KEY AUTOINCREMENT ,
        city varchar(256),
        date varchar(256),
        weather varchar(256),
        maxTemp varchar(256),
        minTemp varchar(256),
        winDay varchar(256),
        winNight varchar(256),
        winStrength varchar(256))
        '''
        self.cursor.execute(sql)
        self.con.commit()

    def insert(self,city,date,weather,maxTemp,minTemp,winDay,winNight,winStrength):
        sql="insert into weathers (city,date,weather,maxTemp,minTemp,winDay,\
        winNight,winStrength) values (?,?,?,?,?,?,?,?)"
        self.cursor.execute(sql,[city,date,weather,maxTemp,minTemp,winDay,\
                                 winNight,winStrength])
        self.cursor.execute("select ID from weathers order by ID desc limit 1")
        row=self.cursor.fetchone()
        ID=row[0]
        return ID

    def closeDB(self):
        #关闭数据库
        self.con.commit()
        self.con.close()

    def spider(self,url):
        #爬虫函数
        try:
            resp=urllib.request.urlopen(url)
            html=resp.read().decode()
            root=etree.HTML(html)
            lis=root.xpath("//ul//li")
            for li in lis:
                city="深圳"
                date=li.xpath("./h5/text()")[0]
                weather=li.xpath("./p[@class='wea']/text()")[0]
                maxTemp=li.xpath("./p[@class='tem']/span/text()")[0]
                minTemp=li.xpath("./p[@class='tem']/i/text()")[0]
                winDay=li.xpath("./p[@class='win']//span[position()=1]/@title")[0]
                winNight=li.xpath("./p[@class='win']//span[position()=2]/@title")[0]
                winStrength=li.xpath("./p[@class='win']//i/text()")[0]
                self.insert(city,date,weather,maxTemp,minTemp,winDay,winNight,winStrength)
        except Exception as err:
            print(err)

    def show(self):
        #显示函数
        self.cursor.execute("select ID,city,date,weather,maxTemp,minTemp,winDay,\
                            winNight,winStrength from weathers")
        rows=self.cursor.fetchall()
        for row in rows:
            ID=row[0]
            city=row[1]
            date=row[2]
            weather=row[3]
            maxTemp=row[4]
            minTemp=row[5]
            winDay=row[6]
            winNight=row[7]
            winStrength=row[8]
            print("ID:",ID)
            print("city:",city)
            print("date:",date)
            print("weather:",weather)
            print("maxTemp:",maxTemp)
            print("minTemp:",minTemp)
            print("winDay:",winDay)
            print("winNight:",winNight)
            print("winStrength:",winStrength)
            print("-"*40)
        print("Total ",len(rows))

#主程序
spider=MySpider()
spider.openDB()
#spider.spider("http://127.0.0.1:5000")
spider.spider("http://www.rj-jwb.com.cn/spiderbook/chapter3/")
spider.show()
spider.closeDB()