以使用pyspark的隨機森林作為例子：

#! /usr/bin/python3
#-*-coding:utf-8-*-

from pyspark import SparkContext,SparkConf
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.classification import LogisticRegressionWithLBFGS
from pyspark.mllib.tree import RandomForest
from pyspark.sql import SQLContext

# Configuration if you use spark-submit?
conf = SparkConf().setAppName("Test Application")
conf = conf.setMaster("local[10]")
sc = SparkContext(conf=conf)
sqlCtx = SQLContext(sc)

def create_label_point(line):
? ? line=line.strip().split(',')
? ? return LabeledPoint(int(line[-1]), [float(x) for x in line[:-1]])

train=sc.textFile("file:///home/hujianqiu/20eg/BLOGGER/kohkiloyeh_train").map(create_label_point)
test=sc.textFile("file:///home/hujianqiu/20eg/BLOGGER/kohkiloyeh_test").map(create_label_point)

#print("rf start")
model = RandomForest.trainClassifier(train, numClasses=2,
? ? ? ? ? ? ? ? ? ? ? ? ? ? categoricalFeaturesInfo={0:3,1:3,2:5,3:2,4:2},
? ? ? ? ? ? ? ? ? ? ? ? ? ? numTrees=50,
? ? ? ? ? ? ? ? ? ? ? ? ? ? featureSubsetStrategy="auto",
? ? ? ? ? ? ? ? ? ? ? ? ? ? impurity="gini",
? ? ? ? ? ? ? ? ? ? ? ? ? ? maxDepth=5,
? ? ? ? ? ? ? ? ? ? ? ? ? ? maxBins=100,
? ? ? ? ? ? ? ? ? ? ? ? ? ? seed=12345)

predictions = model.predict(test.map(lambda x: x.features))
labels_and_preds = test.map(lambda p: p.label).zip(predictions)

# Confusion Matrix
testErr_11 = labels_and_preds.filter(lambda (v, p): (v, p) == (1, 1)).count()
testErr_10 = labels_and_preds.filter(lambda (v, p): (v, p) == (1, 0)).count()
testErr_01 = labels_and_preds.filter(lambda (v, p): (v, p) == (0, 1)).count()
testErr_00 = labels_and_preds.filter(lambda (v, p): (v, p) == (0, 0)).count()

accuracy=(float(testErr_11)+float(testErr_00))/(float(testErr_11)+float(testErr_10)+float(testErr_01)+float(testErr_00))
recall=float(testErr_11)/(float(testErr_11)+float(testErr_10))
precision=float(testErr_11)/(float(testErr_11)+float(testErr_01))
F1_measure=2*precision*recall/(precision+recall)

with open('/home/hujianqiu/20eg/BLOGGER/result.txt','w') as f:
? ? f.write('testErr_11:\t%d\n'%testErr_11)
? ? f.write('testErr_10:\t%d\n'%testErr_10)
? ? f.write('testErr_01:\t%d\n'%testErr_01)
? ? f.write('testErr_00:\t%d\n'%testErr_00)
? ? f.write('accuracy:\t%f\n'%accuracy)
? ? f.write('recall:\t\t%f\n'%recall)
? ? f.write('precision:\t%f\n'%precision)
? ? f.write('F1_measure:\t%f\n'%F1_measure)
? ? #f.write(model.toDebugString())
# Save and load model
# model.save(sc, "/home/air/hjq/proofread-randomforesst/myRandomForestClassificationModel")
# sameModel = RandomForestModel.load(sc, "target/tmp/myRandomForestClassificationModel")
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
其中可以看到RandomForest.trainClassifier中有一個參數為categoricalFeaturesInfo，這其實是一個字典?
如果沒有分類型變量，需要設categoricalFeaturesInfo={}?
如果有分類型變量， 則該字典需要有值：?
- key為自變量的位置，value為自變量的分類數；?
- 第一個自變量為0，第二個自變量為1，以此類推；?
- 數據需要事先碼值，編碼為0,1,…,N-1，N為該自變量的分類數，要與categoricalFeaturesInfo對應好

kohkiloyeh_test
0,2,1,0,0,0
0,2,2,0,0,0
1,1,3,0,0,0
1,0,0,0,1,0
0,0,0,0,0,0
0,2,2,0,0,1
0,2,2,0,0,0
0,0,2,0,0,0
1,2,1,0,0,0
0,2,2,0,0,0
1,2,3,0,0,0
2,2,0,0,1,1
0,2,3,0,0,0
0,2,2,0,0,0
2,0,1,1,1,0
1,2,4,0,0,1
2,2,4,0,0,1
1,0,0,0,0,1
0,2,1,0,1,0
1,2,0,0,1,0
1,1,4,0,1,0
1,1,1,1,0,1
1,0,4,0,0,1
0,2,2,0,0,1
1,0,3,0,0,1
1,1,3,0,0,0
2,0,1,0,1,1
1,0,3,0,0,1
1,1,3,0,0,0
1,1,3,0,0,0
0,2,2,0,0,0
1,0,0,0,1,0
1,0,0,0,0,0
2,0,3,1,0,1
0,2,2,0,0,0
1,2,2,0,0,0
0,0,1,0,0,1
1,2,2,0,0,0
1,2,3,0,0,0
1,0,2,1,0,1
2,0,1,1,1,0
0,0,2,0,0,0
0,2,1,1,1,0
1,2,4,0,0,1
0,0,2,0,0,0
2,0,2,0,0,0
1,1,1,1,0,1
1,2,1,1,1,0
1,2,3,1,0,0
1,2,0,0,1,0
2,1,1,0,1,1
1,2,1,0,0,0

kohkiloyeh_test
1,1,0,0,0,0
0,2,2,0,0,0
1,1,3,0,0,0
0,2,2,0,0,0
0,0,2,0,1,0
0,0,2,0,1,1
1,2,3,0,1,0
2,0,3,1,0,1
1,2,1,0,0,0
1,2,2,0,0,0
0,0,0,0,0,1
0,0,2,1,0,1
0,0,2,0,0,0
0,2,1,1,1,0
0,0,2,0,0,0
2,0,2,0,0,0
1,2,1,1,1,0
1,2,3,1,0,0
0,2,2,0,0,0
1,2,0,0,1,0
2,1,1,0,1,1
1,2,1,0,0,0
0,2,2,0,0,0
0,2,2,0,0,0
1,1,0,0,0,0
0,2,2,0,0,0
0,0,2,0,1,0
0,0,2,0,1,1
1,2,3,0,1,0
0,2,1,0,0,1
1,2,1,0,0,0
0,0,2,0,0,0
1,2,2,0,0,0
2,2,0,0,1,1
0,2,3,0,0,0
0,2,2,0,0,0
2,2,4,0,0,1
1,0,0,0,0,1
0,2,1,0,1,0
1,2,0,0,1,0
1,1,1,0,1,0
1,0,4,0,0,1
0,2,2,0,0,1
0,2,2,0,0,0
1,0,3,0,0,1
1,1,1,0,0,0
2,0,1,0,1,1
1,0,3,0,0,1
?

創作挑戰賽新人創作獎勵來咯，堅持創作打卡瓜分現金大獎

總結

以上是生活随笔為你收集整理的spark中使用categoricalFeaturesInfo来标记分类型变量的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。