抓取的網(wǎng)站是 吉林水文信息網(wǎng)，都是公開數(shù)據(jù)。本文主要講的是如何使用selenium提取頁面表格，下面***號間的代碼就是提取表格的代碼，只需要在開頭的地方獲取tr和td兩個元素就可以了。原理是通過tr元素個數(shù)獲知表格行數(shù)，td元素中的colspan、rowspan屬性獲知單元格總個數(shù)，從而計算出表格的列數(shù)。然后構建值為0的對應行列數(shù)表格，逐一將td元素中的文本填入表格，即可得到頁面的表格。

from selenium.webdriver.common.keys import Keys
from selenium import webdriver
import time
import os
from selenium.webdriver.common.action_chains import ActionChains
import random
import sys
import urllib.request
import urllib.parse
import socket
import xlsxwriter

def output_excel(file_name, report_data1): #輸出Excel

today = time.strftime('%Y%m%d%H%M',time.localtime(time.time())); #time.struct_time(tm_year=2018, tm_mon=7, tm_mday=23, tm_hour=12, tm_min=50, tm_sec=30, tm_wday=0, tm_yday=204, tm_isdst=0) #新建文件 filename = (os.getcwd()+'\\'+file_name+today+'.xlsx') test_book = xlsxwriter.Workbook(filename)#添加sheet1 worksheet = test_book.add_worksheet(file_name) #往文件里面添加sheet #worksheet.freeze_panes(1, 4)#設置窗口凍結區(qū)域 距離頂部1行，距離左邊2列 bold0 = worksheet.set_column(0, 0, 10.7) #列寬 x=-1; for report in report_data1:if len(report)==0:continuey=-1;x=x+1;for rep in report:if rep=='nan':rep=''y=y+1;try:worksheet.write(x,y,rep)except:worksheet.write(x,y,rep);#關閉文件 test_book.close()

def get_data():
os.system(‘taskkill /F /IM chrome.exe’);os.system(‘taskkill /F /IM chromedriver.exe’);

#url = r'http://httpbin.org/get?show_env=1' #url = r'http://httpbin.org/post?show_env=1' url = r'http://www.hydrojl.net/pages/fb/W01FirstPageNew.aspx'while(True):try:obj = webdriver.Chrome()#obj = webdriver.Firefox()obj.set_page_load_timeout(60)time.sleep(6)obj.get(url)time.sleep(10)obj.maximize_window()breakexcept:print('啟動瀏覽器異常')os.system('taskkill /F /IM chrome.exe');os.system('taskkill /F /IM chromedriver.exe');time.sleep(20) try:#滾動頁面obj.execute_script("window.scrollBy(0,300)") #向下滾動time.sleep(3)#obj.execute_script("window.scrollBy(0,document.body.scrollHeight/5*2)")#time.sleep(2)#切換框架obj.switch_to.frame("indexFrame") #用name來定位time.sleep(2)#print(obj.page_source)#單擊 跳轉頁面btn = obj.find_elements_by_xpath('//form[@id="form1"]/div/div/a/img')[0]ActionChains(obj).move_to_element(btn).perform() #窗口移動到元素time.sleep(2)btn.click() #單擊#obj.implicitly_wait(60)time.sleep(10)#切換框架obj.switch_to.frame("iframeContent") #用name來定位time.sleep(2)#抓數(shù)據(jù)#**************************************************************************trlist = obj.find_elements_by_xpath('//div[@id="chg"]//tbody/tr') #行tdlist = obj.find_elements_by_xpath('//div[@id="chg"]//tbody/tr/td') #列#行：rowspan="2"#列：colspan="14"trn = len(trlist) #總行數(shù)temlist = []for td in tdlist:try:temlist.append(int(td.get_attribute('colspan')))except:temlist.append(1)try:temlist.append(int(td.get_attribute('rowspan'))-1)except:passtdn = sum(temlist)tdn = int(tdn/trn) #總列數(shù)#構建表格table = []for j in range(0,trn):table.append([0]*tdn)#tdlist 所有元素for i in range(0,trn): #行for j in range(0,tdn): #列if table[i][j]==0:td = tdlist.pop(0) #獲取數(shù)據(jù)table[i][j] = td.texttry:aa = int(td.get_attribute('colspan'))#列for a in range(1,aa):table[i][j+a] = ''except:passtry:aa = int(td.get_attribute('rowspan'))#行for a in range(1,aa):table[i+a][j] = ''except:passif len(tdlist)==0:breakif len(tdlist)==0:breakfor j in table:print(j)#**************************************************************************datalist = tabletime.sleep(6)#退出程序obj.close()obj.quit()print('ok')return datalist except:os.system('taskkill /F /IM chrome.exe');os.system('taskkill /F /IM chromedriver.exe');print('no')return []

def main():

datalist = get_data()output_excel('雨情信息', datalist)

if name==‘main’:

main()time.sleep(6)os.system('taskkill /F /IM chrome.exe');os.system('taskkill /F /IM chromedriver.exe') #os.system('pause')

總結

以上是生活随笔為你收集整理的selenium抓取页面表格的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內容還不錯，歡迎將生活随笔推薦給好友。