1.簡(jiǎn)單網(wǎng)頁(yè)

#!/usr/bin/env python # -*- coding:utf-8 -*-import urllib.requestfile = urllib.request.urlopen("https://www.jd.com") data = file.read() # dataline = file.readline() print(data)fhandle = open("E:/python/1_1.html", "wb") fhandle.write(data) fhandle.close()# filename = urllib.request.urlretrieve("http://edu.51cto.com", filename="E:/python/2.html") # filename2 = urllib.request.urlretrieve("http://www.jd.com", filename="E:/python/3.html")print(file.getcode()) print(file.geturl())

2.模擬瀏覽器

#!/usr/bin/env python # -*- coding:utf-8 -*-import urllib.requesturl = "https://blog.csdn.net/java_zhangshuai/article/details/81749208" headers = ("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36") opener = urllib.request.build_opener() opener.addheaders = [headers] data = opener.open(url).read() print(data) fhandle = open("E:/python/2_1.html", "wb") fhandle.write(data) fhandle.close()

3.http請(qǐng)求

#!/usr/bin/env python # -*- coding:utf-8 -*-import urllib.requestkeywd = "hello" # 中文等不符合url標(biāo)準(zhǔn)的,需要編碼 keywd = urllib.request.quote(keywd) url = "http://www.baidu.com/s?wd=" + keywd req = urllib.request.Request(url) data = urllib.request.urlopen(req).read()fhandle = open("E:/python/3_1.html", "wb") fhandle.write(data) fhandle.close()import urllib.parseurl = "http://www.iqianyue.com/mypost" data = {"name": "zhangsan", "pass": "zhangsanpass"} postdata = urllib.parse.urlencode(data).encode("utf-8")for x in range(1, 3):try:req = urllib.request.Request(url, postdata)req.add_header("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36")data = urllib.request.urlopen(req).read()fhandle = open("E:/python/3_2.html", "wb")fhandle.write(data)fhandle.close()print(len(data))except Exception as e:print("出現(xiàn)異常--->"+str(e))

4.爬取某電商商品列表下的圖片集合

#!/usr/bin/env python # -*- coding:utf-8 -*-import urllib.request import redef craw(url, page):html1 = urllib.request.urlopen(url).read()html1 = str(html1)pat1 = '<div id="plist".+? <div class="page clearfix">'# 根據(jù)pat1,過(guò)濾出圖片部分result1 = re.compile(pat1).findall(html1)result1 = result1[0]pat2 = '<img width="220" height="220" data-img="1" src="//(.+?\.jpg)">'# 根據(jù)pat2,過(guò)濾出圖片listimagelist = re.compile(pat2).findall(result1)x = 1for imageurl in imagelist:print(imageurl)imagename = "E:/python/爬蟲(chóng)/" + str(page) + str(x) + ".jpg"imageurl = "https://" + imageurltry:# 將圖片鏈接是imageurl的圖片存在路徑為imagename的地方urllib.request.urlretrieve(imageurl, filename=imagename)except:x += 1x += 1for i in range(1, 10):url = "https://list.jd.com/list.html?cat=9192,12632,12633&page=" + str(i)craw(url, i)

總結(jié)

以上是生活随笔為你收集整理的[python]---从java到python(03)---爬虫的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。