添加maven依賴：

<dependencies><dependency><groupId>com.alibaba</groupId><artifactId>fastjson</artifactId><version>1.2.60</version></dependency><dependency><groupId>org.apache.spark</groupId><artifactId>spark-core_2.11</artifactId><version>2.3.3</version></dependency></dependencies>

數(shù)據(jù)地址如下（真實(shí)數(shù)據(jù)）

https://pan.baidu.com/s/1GYr_sR4CBXCmd1AIaBbvPg

1、json工具類，解析json數(shù)據(jù)，并且拿到我們想要的字段（有標(biāo)簽的字段）

import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject;public class ReviewTags {//86913510 {"reviewPics":null,"extInfoList":null,"expenseList":null,"reviewIndexes":[2],"scoreList":[{"score":5,"title":"口味","desc":""},{"score":5,"title":"服務(wù)","desc":""},{"score":5,"title":"環(huán)境","desc":""}]}public static String extractTags(String jsonStr){JSONObject object = JSON.parseObject(jsonStr);if(object == null || !object.containsKey("extInfoList")){return "";}JSONArray array = object.getJSONArray("extInfoList");if(array == null) {return "";}StringBuilder sb = new StringBuilder();for (int i = 0; i < array.size(); i++) {JSONObject obj = array.getJSONObject(i);if (obj != null && obj.containsKey("title") && obj.getString("title").equals("contentTags") && obj.containsKey("values")) {JSONArray arr = obj.getJSONArray("values");if(arr == null){continue;}boolean begin = true;for (int j = 0; j < arr.size(); j++) {if (begin) {begin = false;} else {sb.append(",");}sb.append(arr.getString(j));}}}return sb.toString();}/*** 測(cè)試類，用于拿出extInfoList的內(nèi)容，并json解析* @param args*/public static void main(String[] args){String s = "{\"reviewPics\":[{\"picId\":2405538806,\"url\":\"http://p0.where.net/shaitu/7c10019c62947d01ded80cc698c77c90217708.jpg\",\"status\":1},{\"picId\":2405442602,\"url\":\"http://p0.meituan.net/shaitu/d41ef06f5d16d5d3cbc871765ff93130270451.jpg\",\"status\":1}],\"extInfoList\":[{\"title\":\"contentTags\",\"values\":[\"回頭客\",\"上菜快\",\"環(huán)境優(yōu)雅\",\"性價(jià)比高\(yùn)",\"菜品不錯(cuò)\"],\"desc\":\"\",\"defineType\":0},{\"title\":\"tagIds\",\"values\":[\"493\",\"232\",\"24\",\"300\",\"1\"],\"desc\":\"\",\"defineType\":0}],\"expenseList\":null,\"reviewIndexes\":[1,2],\"scoreList\":null}";//上面定義的是靜態(tài)方法，所以直接調(diào)用System.out.println(extractTags(s));System.out.println(extractTags(""));System.out.println(extractTags(null));} }

2、標(biāo)簽統(tǒng)計(jì)：

import com.fengrui.taggen.ReviewTags import org.apache.spark.rdd.RDD import org.apache.spark.{SparkConf, SparkContext}/*** 團(tuán)購(gòu)網(wǎng)站的標(biāo)簽生成應(yīng)用*/ object TagGenerator {def main(args: Array[String]): Unit = {val conf: SparkConf = new SparkConf().setAppName("TagGenerator").setMaster("local[*]")val sc = new SparkContext(conf)//86913510 {"reviewPics":null,"extInfoList":null,"expenseList":null,"reviewIndexes":[2],"scoreList":[{"score":5,"title":"口味","desc":""},{"score":5,"title":"服務(wù)","desc":""},{"score":5,"title":"環(huán)境","desc":""}]}val lines: RDD[String] = sc.textFile("file:///F:\\ideaworkspace\\temptag\\src\\main\\resources\\temptags.txt")//將數(shù)據(jù)用制表符切割，并且過(guò)濾出長(zhǎng)度等于2的所有滿足條件的數(shù)據(jù)（原始數(shù)據(jù)制表符切割以后// 左面是商店ID，右面是客戶反饋信息）val filter: RDD[Array[String]] = lines.map(_.split("\t")).filter(x => x.length == 2)//將過(guò)濾完的數(shù)據(jù)進(jìn)行映射，x(0) -> ReviewTags.extractTags(x(1))的意思就是（86913510,環(huán)境好,好吃,..）（86913510,）//利用json工具類拿出出第二個(gè)元素，有的是空，有的會(huì)有數(shù)據(jù)val useJson: RDD[(String, String)] = filter.map(x => x(0) -> ReviewTags.extractTags(x(1))) // useJson.take(50).foreach(println)//因?yàn)閡seJson中的二元組的value值有的是空，所以將useJson中第二個(gè)元素（value）長(zhǎng)度大于零的二元組過(guò)濾出來(lái)（取出）val ujf: RDD[(String, String)] = useJson.filter(x => x._2.length > 0)//還是映射，變成 86913510 -> [環(huán)境好，好吃...]val ujfm: RDD[(String, Array[String])] = ujf.map(x => x._1 -> x._2.split(","))//將value值壓平，變?yōu)?#xff08;86913510,環(huán)境好）（86913510,好吃）....val fmv: RDD[(String, String)] = ujfm.flatMapValues(x => x) // s.take(50).foreach(println)//給fmv中元祖都拼1，((86913510,環(huán)境好),1) ((86913510,好吃),1)....val fmvm: RDD[((String, String), Int)] = fmv.map(x => (x._1,x._2) -> 1) // a.take(20).foreach(println)//進(jìn)行聚合，聚合完以后為 ((78824187,干凈衛(wèi)生),7) ((77373671,分量少),1) ((84270191,體驗(yàn)好),2)val reduceByKey: RDD[((String, String), Int)] = fmvm.reduceByKey(_+_) // reduceByKey.foreach(println)//變成(78824187,List((干凈衛(wèi)生,7))) (77373671,List((分量少,1)))...val rsm: RDD[(String, List[(String, Int)])] = reduceByKey.map(x => x._1._1 -> List((x._1._2,x._2))) // rsm.foreach(println)//:::為數(shù)組的聚合，變?yōu)?83644298,List((體驗(yàn)好,1), (性價(jià)比高,1), (服務(wù)熱情,1), (價(jià)格實(shí)惠,1), (味道贊,1)))//(71039150,List((團(tuán)建,1), (價(jià)格實(shí)惠,1), (朋友聚會(huì),1), (環(huán)境優(yōu)雅,1), (體驗(yàn)好,1)))....val rsmr: RDD[(String, List[(String, Int)])] = rsm.reduceByKey(_:::_) // rsmr.foreach(println)//reverse是倒序排列//最后變?yōu)?83644298,味道贊:1,價(jià)格實(shí)惠:1,服務(wù)熱情:1,性價(jià)比高:1,體驗(yàn)好:1)val res: RDD[(String, String)] = rsmr.map(x => x._1 -> x._2.sortBy(_._2).reverse.take(10).map(a => a._1 + ":" + a._2.toString).mkString(",")) // res.foreach(println)//寫出存儲(chǔ)res.map(x => x._1 + "\t" + x._2).coalesce(1).saveAsTextFile("file:///G:/res1.txt")println("----success----")}}

收集到的數(shù)據(jù)：

總結(jié)

以上是生活随笔為你收集整理的大数据项目——团购网标签的生成的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。