根據

爬蟲 spider07——爬取騰訊娛樂新聞

https://blog.csdn.net/qq_41946557/article/details/102566143

進行修改！

代碼：?

import com.alibaba.fastjson.JSON; import com.offcn.springcloud.jedis.JedisPoolUtil; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import org.junit.Test; import redis.clients.jedis.Jedis;import java.io.IOException; import java.util.Iterator; import java.util.List;public class TestQQ {private final String key="bigdata:spider:url";@Test //根據標簽名稱public void jsoupParse01() throws IOException {String url="https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=ent&page=1&callback=__jp6";CloseableHttpClient closeableHttpClient = HttpClients.createDefault();HttpGet httpGet=new HttpGet(url);//執行請求 user center pageCloseableHttpResponse closeableHttpResponse = closeableHttpClient.execute(httpGet);HttpEntity entity = closeableHttpResponse.getEntity();//響應實體轉換成字符串String content = EntityUtils.toString(entity, "GB2312");//把字符串截取成json格式數據String jsonContent= StringToJson(content);//把json轉換成java對象，并以list形式返回List<News> news=jsonToObject(jsonContent);System.out.println("源數據量:"+news.size());//獲取不重復的數據List<News> newsList = cleanNews(news);System.out.println("新數據量:"+newsList.size());}//清洗爬取重復的數據public List<News> cleanNews(List<News> listNews){//通過迭代器遍歷新聞listIterator<News> iteratorNew = listNews.iterator();while(iteratorNew.hasNext()){News news = iteratorNew.next();//獲取redis連接對象Jedis jedis = JedisPoolUtil.getJedis();//與redis庫比較數據Boolean isExist = jedis.hexists(key, news.getUrl());//true 存在 false 不存在if(!isExist){//false 不存在System.out.println("不存在:"+news.getUrl());//存儲數據到redisjedis.hset(key,news.getUrl(),objectToJson(news));}else {//true 存在System.out.println("存在:"+news.getUrl());//刪除list中重復的數據iteratorNew.remove();}}return listNews;}//java對象轉換成jsonpublic String objectToJson(Object object){return JSON.toJSONString(object);}/*** string轉換成json工具方法* @param srcJson* @return*/public String StringToJson(String srcJson){int start=srcJson.indexOf('(')+1;int end=srcJson.lastIndexOf(')');String jsonContent= srcJson.substring(start,end);return jsonContent;}public List<News> jsonToObject(String srcJson){//通過fastjson把json字符串轉換成java對象NewsCollection newsCollection = JSON.parseObject(srcJson, NewsCollection.class);List<News> news = newsCollection.getData();for (News newsMessage : news) {System.out.println(newsMessage);System.out.println("---------------------------------------------------------");}return news;} }

結果展示：

?

?

?

總結

以上是生活随笔為你收集整理的爬虫 spider08——爬取腾讯娱乐新闻【使用redis去重】的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。