项目4 爬取旅游网站数据

database.py

import sqlite3
class Database:
    def open(self):
        self.con=sqlite3.connect("travels.db")
        self.cursor=self.con.cursor()
    
    def close(self):
        self.con.commit()
        self.con.close()

    def initialize(self):
        try:
            self.cursor.execute("drop table travels")
            self.con.commit()
        except:
            pass 
        sql="""
        create table travels (
            ID varchar(8) primary key,
            Title varchar(256),
            Date varchar(256),
            Ext  varchar(256),
            Content text 
        )
        """
        self.cursor.execute(sql)
        self.con.commit()

    def insert(self,ID,Title,Date,Ext,Content):
        try:
            sql="insert into travels (ID,Title,Date,Ext,Content) values (?,?,?,?,?)"
            self.cursor.execute(sql,[ID,Title,Date,Ext,Content])
        except:
            pass 
    
    def updateContent(self,ID,Content):
        try:
            sql="update tavels set Content=? whete ID=?"
            self.cursor.execute(sql,[Content,ID])
        except:
            pass 
 
    def show(self):
        sql="select ID,Title,Date,Ext,Content from travels"
        self.cursor.execute(sql)
        rows=self.cursor.fetchall()
        for row in rows:
            print(row[0],row[1],row[2],row[3],row[4][:20])
        print("Total ",len(rows),"items")

spider.py

import urllib.response
from bs4 import BeautifulSoup
import urllib.request
import os
import time 
import threading
from database import Database

def download(src):
    try:
        p=src.rfind("/")
        name=src[p+1:]
        data=urllib.request.urlopen(src,timeout=100)
        data=data.read()
        if len(data)>0:
            fobj=open("downloadImages/"+name,"wb")
            fobj.write(data)
            fobj.close()
            print("downloaded ",src)
    except Exception as err:
        print(err)

def getContent(url):
    try:
        data=urllib.request.urlopen(url)
        data=data.read()
        data=data.decode()
        soup=BeautifulSoup(data,"lxml")
        return soup.select_one("pre").text 
    except Exception as err:
        print(err)
        return ""

def initializeDownload():
    #初始化downloadImages文件夹
    if not os.path.exists("downloadImages"):
        os.mkdir("downloadImages")
    fs=os.listdir("downloadImages")
    for f in fs:
        os.remove("downloadImages\\"+f)
   
def spider(url):
    global TS,db
    print(url)
    try:
        data=urllib.request.urlopen(url)
        data=data.read()
        data=data.decode()
        soup=BeautifulSoup(data,"lxml")
        spans=soup.select("div span[class='second']")
        for span in spans:
            link=span.select_one("div a")
            title=link.text
            ID=span.select_one("div b").text
            date=span.select("div")[-1].text
            url=urllib.request.urljoin(start_url,link["href"])
            content=getContent(url)
            #span的父元素是div，div的第一个span是图片
            div=span.parent
            image=div.select_one("span[class='first'] img")
            src=urllib.request.urljoin(start_url,image["src"])
            p=src.rfind(".")
            ext=src[p:]
            db.insert(ID,title,date,ext,content)
            print(ID,title)
            T=threading.Thread(target=download,args=(src,))
            T.start()
            TS.append(T)
        links=soup.select("div[id='pagnation'] a")
        url=""
        for link in links:
            if link.text=="下一页":
                url=urllib.request.urljoin(start_url,link["href"])
                break  
        if url!="":
            spider(url)
    except Exception as err:
        print(err)

if os.path.exists("downloadImages")==False:
    os.mkdir("downloadImages")

TS=[]
db=None
start_url="http://127.0.0.1:5000"
while True:
    print("1.Spider")
    print("2.Show")
    print("3.Exit")
    choice=input("Enter your choice(1,2,3): ")
    if choice=="1":
        startTime=time.time()
        initializeDownload()
        db=Database()
        db.open()
        db.initialize()
        spider(start_url)
        db.close()
        for T in TS:
            T.join()
        endTime=time.time()
        print("time used: %.2f seconds"%(endTime-startTime))
    elif choice=="2":
        db=Database()
        db.open()
        db.show()
        db.close()
    elif choice=="3":
        break
    else:
        print("Invalid choice")