看到網(wǎng)上其他同學(xué)的思路是爬取所有歌單，然后篩選出評(píng)論過(guò)萬(wàn)的歌曲。但我覺(jué)得不同歌單之間會(huì)有交叉，這種方式可能效率不高，而且可能會(huì)有漏網(wǎng)之魚(yú)。所以我準(zhǔn)備爬取所有歌手，再爬取他們的熱門(mén)50單曲，從中篩選評(píng)論過(guò)萬(wàn)的歌曲。現(xiàn)階段幾乎沒(méi)有歌手有超過(guò)50首評(píng)論過(guò)萬(wàn)的歌曲，所以這種方法目前是可行的。

查看歌手頁(yè)面，歌手被分成了華語(yǔ)男歌手、華語(yǔ)女歌手、歐美男歌手……共計(jì)15個(gè)板塊，板塊代號(hào)如下：

group = ['1001', '1002', '1003', '2001', '2002', '2003', '6001', '6002', '6003', '7001', '7002', '7003', '4001', '4002', '4003']

而每個(gè)板塊又按照首字母分成了27個(gè)子頁(yè)面（包括熱門(mén)歌手頁(yè)面），子頁(yè)面代號(hào)如下：

initial = ['0'] for i in range(65, 91):initial.append(str(i))

15*27=405，我們要爬取405個(gè)歌手子頁(yè)面，可以利用上述代號(hào)拼接出這405個(gè)歌手子頁(yè)面鏈接：

urls = [] for g in group:for i in initial:url = 'http://music.163.com/discover/artist/cat?id=' + g + '&initial=' + iurls.append(url)

然后就是用爬蟲(chóng)從這些頁(yè)面上爬取歌手的id：

def get_artist(url):k = 0t = []while True:try:resp = request.urlopen(url)html = resp.read().decode('utf-8')soup = bs(html, 'html.parser')l = soup.find_all('a', class_='nm nm-icn f-thide s-fc0')p = r'\s*\/[a-z]+\?[a-z]+\=([0-9]+)'for i in l:t.append(re.match(p, i['href']).group(1))return texcept Exception as e:print(e)k += 1if k > 10:print('頁(yè)面' + url + '發(fā)生錯(cuò)誤')return Nonet = []continue

獲得歌手id以后，再讓爬蟲(chóng)爬取歌手的個(gè)人頁(yè)面，獲取熱門(mén)50單曲的歌曲id：

def get_song(artist_id):k = 0t = []while True:url = 'http://music.163.com/artist?id=' + artist_idtry:req = request.Request(url)req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4399.400 QQBrowser/9.7.12777.400')req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')resp = request.urlopen(req)html = resp.read().decode('utf-8')soup = bs(html, 'html.parser')except Exception as e:k += 1if k > 10:print('歌手' + artist_id + '發(fā)生錯(cuò)誤')print(e)return Nonecontinuetry:a = soup.find('ul', class_='f-hide')l = a.childrenp = r'\s*\/[a-z]+\?[a-z]+\=([0-9]+)'for i in l:music_id = re.match(p, i.a['href']).group(1)data = (music_id, artist_id)t.append(data)return texcept Exception as e:print(e)print('歌手' + artist_id + '發(fā)生錯(cuò)誤')return None

利用歌曲id訪問(wèn)歌曲頁(yè)面，獲取歌曲評(píng)論數(shù)，這里遇到了難點(diǎn)。評(píng)論信息都是動(dòng)態(tài)加載的，直接獲取評(píng)論數(shù)的結(jié)果總是0，所以這里借鑒了知乎用戶平胸小仙女的回答，方法如下：

# -*- coding: utf-8 -*- from Crypto.Cipher import AES import base64 import requests import json import codecs import time import random#代理ip proxy_host = '122.72.18.35' proxy = {'http':proxy_host}# 頭部信息 headers={'Host':'music.163.com','Accept':'*/*','Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3','Accept-Encoding':'gzip, deflate','Content-Type':'application/x-www-form-urlencoded','Referer':'http://music.163.com/song?id=347597','Content-Length':'484','Cookie':'__s_=1; _ntes_nnid=f17890f7160fd145486752ebbf2066df,1505221478108; _ntes_nuid=f17890f7160fd145486752ebbf2066df; JSESSIONID-WYYY=Z99pE%2BatJVOAGco1d%2FJpojOK94Xe9GHqe0epcCOj23nqP2SlHt1XwzWQ2FXTwaM2xgIN628qJGj8%2BikzfYkv%2FXAUo%2FSzwMxjdyO9oeQlGKBvH6nYoFpJpVlA%2F8eP57fkZAVEsuB9wqkVgdQc2cjIStE1vyfE6SxKAlA8r0sAgOnEun%2BV%3A1512200032388; _iuqxldmzr_=32; __utma=94650624.1642739310.1512184312.1512184312.1512184312.1; __utmc=94650624; __utmz=94650624.1512184312.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); playerid=10841206','Connection':'keep-alive','User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'}# offset的取值為:(評(píng)論頁(yè)數(shù)-1)*20,total第一頁(yè)為true，其余頁(yè)為false first_param = '{rid:"", offset:"0", total:"true", limit:"20", csrf_token:""}' second_param = "010001" third_param = "00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7" forth_param = "0CoJUm6Qyw8W8jud"# 獲取參數(shù) def get_params(page): # page為傳入頁(yè)數(shù)iv = "0102030405060708"first_key = forth_paramsecond_key = 16 * 'F'if(page == 1): # 如果為第一頁(yè)first_param = '{rid:"", offset:"0", total:"true", limit:"20", csrf_token:""}'h_encText = AES_encrypt(first_param, first_key, iv)else:offset = str((page-1)*20)first_param = '{rid:"", offset:"%s", total:"%s", limit:"20", csrf_token:""}' %(offset,'false')h_encText = AES_encrypt(first_param, first_key, iv)h_encText = AES_encrypt(h_encText, second_key, iv)return h_encText# 獲取 encSecKey def get_encSecKey():encSecKey = "257348aecb5e556c066de214e531faadd1c55d814f9be95fd06d6bff9f4c7a41f831f6394d5a3fd2e3881736d94a02ca919d952872e7d0a50ebfa1769a7a62d512f5f1ca21aec60bc3819a9c3ffca5eca9a0dba6d6f7249b06f5965ecfff3695b54e1c28f3f624750ed39e7de08fc8493242e26dbc4484a01c76f739e135637c"return encSecKey# 解密過(guò)程 def AES_encrypt(text, key, iv):pad = 16 - len(text) % 16text = text + pad * chr(pad)encryptor = AES.new(key, AES.MODE_CBC, iv)encrypt_text = encryptor.encrypt(text)encrypt_text = base64.b64encode(encrypt_text)encrypt_text = str(encrypt_text, encoding="utf-8") #注意一定要加上這一句，沒(méi)有這一句則出現(xiàn)錯(cuò)誤return encrypt_textdef get_json(url, params, encSecKey):data = {"params": params,"encSecKey": encSecKey}response = requests.post(url, headers=headers, data=data, proxies=proxy)return response.content#外部調(diào)用方法 def get_comments_total(id):url = 'http://music.163.com/weapi/v1/resource/comments/R_SO_4_'+str(id)+'?csrf_token='params = get_params(1)encSecKey = get_encSecKey()json_text = get_json(url,params,encSecKey)json_dict = json.loads(json_text)comments_num = int(json_dict['total'])return comments_num

最后再將獲得的數(shù)據(jù)逐條寫(xiě)入數(shù)據(jù)庫(kù)就可以了
總的代碼如下：

# _*_ coding: utf-8 _*_ from urllib import request import requests import json from bs4 import BeautifulSoup as bs from Crypto.Cipher import AES import base64 import re import mysql.connector import get_comments_total as gct import threadinggroup = ['1001', '1002', '1003', '2001', '2002', '2003', '6001', '6002', '6003', '7001', '7002', '7003', '4001', '4002','4003']initial = ['0'] for i in range(65, 91):initial.append(str(i))urls = [] for g in group:for i in initial:url = 'http://music.163.com/discover/artist/cat?id=' + g + '&initial=' + iurls.append(url)#寫(xiě)入數(shù)據(jù)庫(kù) def write(L):try:conn = mysql.connector.connect(user='root', password='lixiao187.', database='cloudmusic', charset='utf8')cursor = conn.cursor()for l in L:try:cursor.execute('insert into music(music_id, music_name, artist_id, artist_name, comments) values (%s, %s, %s, %s, %s)',l)conn.commit()except Exception as e:print(e)print(l)continuecursor.close()conn.close()except Exception as e:print(e)print(L)# 獲得某字母頁(yè)面上的歌手id列表 def get_artist(url):k = 0t = []while True:try:resp = request.urlopen(url)html = resp.read().decode('utf-8')soup = bs(html, 'html.parser')l = soup.find_all('a', class_='nm nm-icn f-thide s-fc0')p = r'\s*\/[a-z]+\?[a-z]+\=([0-9]+)'for i in l:t.append(re.match(p, i['href']).group(1))return texcept Exception as e:print(e)k += 1if k > 10:print('頁(yè)面' + url + '發(fā)生錯(cuò)誤')return Nonet = []continue# 獲得某歌手的熱門(mén)歌曲id列表 def get_song(artist_id):k = 0t = []while True:url = 'http://music.163.com/artist?id=' + artist_idtry:req = request.Request(url)req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4399.400 QQBrowser/9.7.12777.400')req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')resp = request.urlopen(req)html = resp.read().decode('utf-8')soup = bs(html, 'html.parser')except Exception as e:k += 1if k > 10:print('歌手' + artist_id + '發(fā)生錯(cuò)誤')print(e)return Nonecontinuetry:a = soup.find('ul', class_='f-hide')l = a.childrenp = r'\s*\/[a-z]+\?[a-z]+\=([0-9]+)'for i in l:music_id = re.match(p, i.a['href']).group(1)data = (music_id, artist_id)t.append(data)return texcept Exception as e:print(e)print('歌手' + artist_id + '發(fā)生錯(cuò)誤')return None# 獲得全部所需信息 def get_data(music_id, artist_id):k = 0while True:try:comments = gct.get_comments_total(music_id)print('歌曲'+music_id+'，評(píng)論數(shù)：'+str(comments))if comments < 10000:return Noneurl = 'http://music.163.com/song?id=' + music_iddata = []req = request.Request(url)req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4399.400 QQBrowser/9.7.12777.400')req.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')resp = request.urlopen(req)html = resp.read().decode('utf-8')soup = bs(html, 'html.parser')d = soup.find('div', class_='tit')p = soup.find('p', class_='des s-fc4')s = soup.find('span', class_='j-flag')music_name = d.em.textartist_name = p.span['title']data.append(music_id)data.append(music_name)data.append(artist_id)data.append(artist_name)data.append(comments)return dataexcept Exception as e:k += 1if k > 10:print('歌曲' + music_id + '發(fā)生錯(cuò)誤')return Nonecontinue# 逐條寫(xiě)入 def get_and_write(artists, name):data = []for a in artists:songs = get_song(a)if songs == None:continuefor s in songs:d = get_data(s[0], a)if d == None:continuedata.append(d)if len(data) > 0:write(data)# 歌手子頁(yè)面爬取線程 def crawl(url, name):L = []artists = get_artist(url)if artists == None:returnfor a in artists:L.append(a)if len(L) > 9:t = threading.Thread(target=get_and_write, args=(L, ''))t.start()L = []t = threading.Thread(target=get_and_write, args=(L, ''))t.start()# 總方法 def threads_crawl(start, end):L = []for i in range(start - 1, end):t = threading.Thread(target=crawl, args=(urls[i], ''))L.append(t)for t in L:t.start()for t in L:t.join()threads_crawl(1, 405)

總結(jié)

以上是生活随笔為你收集整理的爬取网易云音乐评论过万歌曲的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。