【】

實驗環(huán)境

1.安裝Python 3.7

2.安裝requests, bs4，pymysql 模塊

實驗步驟1.安裝環(huán)境及模塊

可參考https://www.jb51.net/article/194104.htm

2.編寫代碼# 51cto 博客頁面數(shù)據(jù)插入mysql數(shù)據(jù)庫

# 導(dǎo)入模塊

import re

import bs4

import pymysql

import requests

# 連接數(shù)據(jù)庫賬號密碼

db = pymysql.connect(host='172.171.13.229',

user='root', passwd='abc123',

db='test', port=3306,

charset='utf8')

# 獲取游標(biāo)

cursor = db.cursor()

def open_url(url):

# 連接模擬網(wǎng)頁訪問

headers = {

'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '

'Chrome/57.0.2987.98 Safari/537.36'}

res = requests.get(url, headers=headers)

return res

# 爬取網(wǎng)頁內(nèi)容

def find_text(res):

soup = bs4.BeautifulSoup(res.text, 'html.parser')

# 博客名

titles = []

targets = soup.find_all("a", class_="tit")

for each in targets:

each = each.text.strip()

if "置頂" in each:

each = each.split(' ')[0]

titles.append(each)

# 閱讀量

reads = []

read1 = soup.find_all("p", class_="read fl on")

read2 = soup.find_all("p", class_="read fl")

for each in read1:

reads.append(each.text)

for each in read2:

reads.append(each.text)

# 評論數(shù)

comment = []

targets = soup.find_all("p", class_='comment fl')

for each in targets:

comment.append(each.text)

# 收藏

collects = []

targets = soup.find_all("p", class_='collect fl')

for each in targets:

collects.append(each.text)

# 發(fā)布時間

dates=[]

targets = soup.find_all("a", class_='time fl')

for each in targets:

each = each.text.split('：')[1]

dates.append(each)

# 插入sql 語句

sql = """insert into blog (blog_title,read_number,comment_number, collect, dates)

values( '%s', '%s', '%s', '%s', '%s');"""

# 替換頁面 \xa0

for titles, reads, comment, collects, dates in zip(titles, reads, comment, collects, dates):

reads = re.sub('\s', '', reads)

comment = re.sub('\s', '', comment)

collects = re.sub('\s', '', collects)

cursor.execute(sql % (titles, reads, comment, collects，dates))

db.commit()

pass

# 統(tǒng)計總頁數(shù)

def find_depth(res):

soup = bs4.BeautifulSoup(res.text, 'html.parser')

depth = soup.find('li', class_='next').previous_sibling.previous_sibling.text

return int(depth)

# 主函數(shù)

def main():

host = "https://blog.51cto.com/13760351"

res = open_url(host) # 打開首頁鏈接

depth = find_depth(res) # 獲取總頁數(shù)

# 爬取其他頁面信息

for i in range(1, depth + 1):

url = host + '/p' + str(i) # 完整鏈接

res = open_url(url) # 打開其他鏈接

find_text(res) # 爬取數(shù)據(jù)

# 關(guān)閉游標(biāo)

cursor.close()

# 關(guān)閉數(shù)據(jù)庫連接

db.close()

if __name__ == '__main__':

main()

3..MySQL創(chuàng)建對應(yīng)的表CREATE TABLE `blog` (

`row_id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主鍵',

`blog_title` varchar(52) DEFAULT NULL COMMENT '博客標(biāo)題',

`read_number` varchar(26) DEFAULT NULL COMMENT '閱讀數(shù)量',

`comment_number` varchar(16) DEFAULT NULL COMMENT '評論數(shù)量',

`collect` varchar(16) DEFAULT NULL COMMENT '收藏數(shù)量',

`dates` varchar(16) DEFAULT NULL COMMENT '發(fā)布日期',

PRIMARY KEY (`row_id`)

) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

4.運(yùn)行代碼，查看效果:

改進(jìn)版：

改進(jìn)內(nèi)容：

1.數(shù)據(jù)庫里面的某些字段只保留數(shù)字即可

2.默認(rèn)爬取的內(nèi)容都是字符串，存放數(shù)據(jù)庫的某些字段，最好改為整型，方便后面數(shù)據(jù)庫操作

1.代碼如下：import re

import bs4

import pymysql

import requests

# 連接數(shù)據(jù)庫

db = pymysql.connect(host='172.171.13.229',

user='root', passwd='abc123',

db='test', port=3306,

charset='utf8')

# 獲取游標(biāo)

cursor = db.cursor()

def open_url(url):

# 連接模擬網(wǎng)頁訪問

headers = {

'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '

'Chrome/57.0.2987.98 Safari/537.36'}

res = requests.get(url, headers=headers)

return res

# 爬取網(wǎng)頁內(nèi)容

def find_text(res):

soup = bs4.BeautifulSoup(res.text, 'html.parser')

# 博客標(biāo)題

titles = []

targets = soup.find_all("a", class_="tit")

for each in targets:

each = each.text.strip()

if "置頂" in each:

each = each.split(' ')[0]

titles.append(each)

# 閱讀量

reads = []

read1 = soup.find_all("p", class_="read fl on")

read2 = soup.find_all("p", class_="read fl")

for each in read1:

reads.append(each.text)

for each in read2:

reads.append(each.text)

# 評論數(shù)

comment = []

targets = soup.find_all("p", class_='comment fl')

for each in targets:

comment.append(each.text)

# 收藏

collects = []

targets = soup.find_all("p", class_='collect fl')

for each in targets:

collects.append(each.text)

# 發(fā)布時間

dates=[]

targets = soup.find_all("a", class_='time fl')

for each in targets:

each = each.text.split('：')[1]

dates.append(each)

# 插入sql 語句

sql = """insert into blogs (blog_title,read_number,comment_number, collect, dates)

values( '%s', '%s', '%s', '%s', '%s');"""

# 替換頁面 \xa0

for titles, reads, comment, collects, dates in zip(titles, reads, comment, collects, dates):

reads = re.sub('\s', '', reads)

reads=int(re.sub('\D', "", reads)) #匹配數(shù)字，轉(zhuǎn)換為整型

comment = re.sub('\s', '', comment)

comment = int(re.sub('\D', "", comment)) #匹配數(shù)字，轉(zhuǎn)換為整型

collects = re.sub('\s', '', collects)

collects = int(re.sub('\D', "", collects)) #匹配數(shù)字，轉(zhuǎn)換為整型

dates = re.sub('\s', '', dates)

cursor.execute(sql % (titles, reads, comment, collects,dates))

db.commit()

pass

# 統(tǒng)計總頁數(shù)

def find_depth(res):

soup = bs4.BeautifulSoup(res.text, 'html.parser')

depth = soup.find('li', class_='next').previous_sibling.previous_sibling.text

return int(depth)

# 主函數(shù)

def main():

host = "https://blog.51cto.com/13760351"

res = open_url(host) # 打開首頁鏈接

depth = find_depth(res) # 獲取總頁數(shù)

# 爬取其他頁面信息

for i in range(1, depth + 1):

url = host + '/p' + str(i) # 完整鏈接

res = open_url(url) # 打開其他鏈接

find_text(res) # 爬取數(shù)據(jù)

# 關(guān)閉游標(biāo)

cursor.close()

# 關(guān)閉數(shù)據(jù)庫連接

db.close()

#主程序入口

if __name__ == '__main__':

main()

2.創(chuàng)建對應(yīng)表CREATE TABLE `blogs` (

`row_id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主鍵',

`blog_title` varchar(52) DEFAULT NULL COMMENT '博客標(biāo)題',

`read_number` int(26) DEFAULT NULL COMMENT '閱讀數(shù)量',

`comment_number` int(16) DEFAULT NULL COMMENT '評論數(shù)量',

`collect` int(16) DEFAULT NULL COMMENT '收藏數(shù)量',

`dates` varchar(16) DEFAULT NULL COMMENT '發(fā)布日期',

PRIMARY KEY (`row_id`)

) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

3.運(yùn)行代碼，驗證

升級版

為了能讓小白就可以使用這個程序，可以把這個項目打包成exe格式的文件，讓其他人，使用電腦就可以運(yùn)行代碼，這樣非常方便！

1.改進(jìn)代碼：#末尾修改為：

if __name__ == '__main__':

main()

print("\n\t\t所有數(shù)據(jù)已成功存放數(shù)據(jù)庫！!! \n")

time.sleep(5)

2.安裝打包模塊pyinstaller(cmd安裝)

pip install pyinstaller -i https://pypi.tuna.tsinghua.edu.cn/simple/

3.Python代碼打包

1.切換到需要打包代碼的路徑下面

2.在cmd窗口運(yùn)行 pyinstaller -F test03.py (test03為項目名稱)

4.查看exe包

在打包后會出現(xiàn)dist目錄，打好包就在這個目錄里面

5.運(yùn)行exe包，查看效果

檢查數(shù)據(jù)庫

總結(jié)

以上是生活随笔為你收集整理的用python爬取网页数据并存入数据库中源代码_Python爬取51cto数据并存入MySQL方法详解...的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。