import requests import pandas as pd from bs4 import BeautifulSoup import json import pandas def comments(newsurl): #獲取評論信息commentsurl = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&\ channel=gn&newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20' x = newsurl.split('/')[-1].lstrip('doc-i').rstrip('.shtml')jd = json.loads(requests.get(commentsurl.format(x)).text.strip('var data='))return jd['result']['count']['total']def getNewsDetail(newsurl): #獲取新聞內(nèi)容result = {}sample = requests.get(newsurl)sample.encoding = 'utf-8'soup = BeautifulSoup(sample.text,'html.parser')result['title'] = soup.select('.page-header')[0].textresult['dt'] = soup.select('.time-source')[0].contents[0].strip()result['article'] = ' '.join([p.text.strip() for p in soup.select('#artibody p')[0:-1]])result['resouce'] = soup.select('.time-source')[0].contents[1].textresult['editor'] = soup.select('.article-editor')[0].text.lstrip('責(zé)任編輯：')result['comments'] = comments(newsurl)return resultdef getnews(url): #獲取主頁面新聞連接url.encoding='utf-8'alllist = []soup = BeautifulSoup(url.text,'html.parser')for a in soup.select('.news-item'):if len(a.select('h2')) > 0:newsurl = a.select('a')[0]['href']alllist.append(getNewsDetail(newsurl))df = pandas.DataFrame(alllist)df.to_excel('news.xlsx',sheet_name='Random Data') #導(dǎo)出數(shù)據(jù)到excel中url = requests.get('http://news.sina.com.cn/china/') getnews(url) #新浪國內(nèi)新聞---------------------------------------------------------------------------------------------------------------------------------------------- 將主頁面通過get方法拿到--通過BeautifulSoup進(jìn)行剖析--利用開發(fā)者工具找出標(biāo)題、內(nèi)文、編輯、評論等信息所在位置--利用BeautifulSoup中的select方法對所需信息進(jìn)行提取，并且

存入字典resul{}中--通過循環(huán)將每則新聞信息append到最后的列表alllist--使用pandas函數(shù)、DataFrame方法將列表整理，最終導(dǎo)出為excel表格

?

轉(zhuǎn)載于:https://www.cnblogs.com/missmissmiss/p/7136261.html

總結(jié)

以上是生活随笔為你收集整理的第一个爬虫的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。