第一次實戰(zhàn)爬蟲，爬取了新浪國內(nèi)的最新的首頁新聞，附效果截圖：

附代碼：

import requests from bs4 import BeautifulSoup import json import re from datetime import datetime import pandas import sqlite3 import osurl = 'https://feed.sina.com.cn/api/roll/get?pageid=121&lid=1356&num=20&\ versionNumber=1.2.4&page={}&encode=utf-8&callback=feedCardJsonpCallback&_=1568108608385'commentUrl = 'https://comment.sina.com.cn/page/info?version=1&format=json\ &channel=gn&newsid=comos-{}&group=undefined&compress=0&ie=utf-8&oe=utf-8\ &page=1&page_size=3&t_size=3&h_size=3&thread=1&uid=3537461634' #爬取網(wǎng)頁內(nèi)詳細信息 def getNewsDetail(newsurl):result = {}res = requests.get(newsurl)res.encoding = 'utf=8'soup = BeautifulSoup(res.text,'html.parser')result['title'] = soup.select('.main-title')[0].texttimesource = soup.select('.date-source')[0].select('.date')[0].text#dt = datetime.strptime(timesource,'%Y年%m月%d日 %H:%M')result['time'] = timesourceresult['url'] = newsurlresult['origin'] = soup.select('.date-source a')[0].textresult['article'] = ' '.join([p.text.strip() for p in soup.select('#article p')[:-1]])result['author'] = soup.select('.show_author')[0].text.lstrip('責(zé)任編輯：')result['comments'] = getCommentCounts(newsurl)return result #爬取評論數(shù)量 def getCommentCounts(newsulr):m = re.search('doc-i(.+).shtml',newsulr)newsId = m.group(1)commentURL = commentUrl.format(newsId)comments = requests.get(commentURL)jd = json.loads(comments.text)return jd['result']['count']['total']#獲取每個分頁的所有新聞的URL，然后取得詳細信息 def parseListLinks(url):newsdetails = []res = requests.get(url)res = res.text.split("try{feedCardJsonpCallback(")[1].split(");}catch(e){};")[0]jd = json.loads(res)for ent in jd['result']['data']:newsdetails.append(getNewsDetail(ent['url']))return newsdetailsnews_total = [] #取得指定分頁的新聞信息 for i in range(1,2):#取第一頁的新聞信息newsurl = url.format(i)newsary = parseListLinks(newsurl)news_total.extend(newsary)df = pandas.DataFrame(news_total) #指定生成的列順序 cols=['title','author','time','origin','article','comments','url'] df=df.loc[:,cols] #df.head(10) #存儲到sqlite數(shù)據(jù)庫中 with sqlite3.connect('news.sqlite') as db:df.to_sql('newsDetail', con=db) #讀取數(shù)據(jù)庫中的信息 with sqlite3.connect('news.sqlite') as db1:df2 = pandas.read_sql_query('SELECT * FROM newsDetail', con=db1) #保存新聞信息到excle表格中 df2.to_excel('newsDetail.xlsx') df2

?

總結(jié)

以上是生活随笔為你收集整理的pyhton爬虫实战-爬取新浪国内新闻的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。