# -*- coding: utf-8 -*- from __future__ import print_function from pyspark.sql import SparkSession from pyspark.sql import Rowif __name__ == "__main__":# 初始化SparkSessionspark = SparkSession \.builder \.appName("RDD_and_DataFrame") \.config("spark.some.config.option", "some-value") \.getOrCreate()sc = spark.sparkContextlines = sc.textFile("employee.txt")parts = lines.map(lambda l: l.split(","))employee = parts.map(lambda p: Row(name=p[0], salary=int(p[1])))#RDD轉換成DataFrameemployee_temp = spark.createDataFrame(employee)#顯示DataFrame數據employee_temp.show()#創建視圖employee_temp.createOrReplaceTempView("employee")#過濾數據employee_result = spark.sql("SELECT name,salary FROM employee WHERE salary >= 14000 AND salary <= 20000")# DataFrame轉換成RDDresult = employee_result.rdd.map(lambda p: "name: " + p.name + " salary: " + str(p.salary)).collect()#打印RDD數據for n in result:print(n)

總結

以上是生活随笔為你收集整理的pyspark DataFrame 转RDD的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。