GET /log.gif?t=item.010001&m=UA-J2011-1&pin=-&uid=1679790178&sid=1679790178|12&v=je=1$sc=24-bit$sr=1600x900$ul=zh-cn$cs=GBK$dt=【云南白藥套裝】云南白藥 牙膏 180g×3 (留蘭香型)【行情 報價 價格 評測】-京東$hn=item.jd.com$fl=16.0 r0$os=win$br=chrome$bv=39.0.2171.95$wb=1437269412$xb=1449548587$yb=1456186252$zb=12$cb=4$usc=direct$ucp=-$umd=none$uct=-$ct=1456186505411$lt=0$tad=-$sku=1326523$cid1=1316$cid2=1384$cid3=1405$brand=20583$pinid=-&ref=&rm=1456186505411 HTTP/1.1

時間戳

IP地址

Cookie

Session

請求URL

Referal

2012-01-01 12:31:12

101.0.0.1

User01

S001

/a/...

somesite.com

2012-01-01 12:31:16

201.0.0.2

User02

S002

/a/...

-

2012-01-01 12:33:06

101.0.0.2

User03

S002

/b/...

baidu.com

2012-01-01 15:16:39

234.0.0.3

User01

S003

/c/...

google.com

2012-01-01 15:17:11

101.0.0.1

User01

S004

/d/...

/c/...

2012-01-01 15:19:23

101.0.0.1

User01

S004

/e/...

/d/....

Session

userid

時間

訪問頁面URL

停留時長

第幾步

S001

User01

2012-01-01 12:31:12

/a/....

30

1

S002

User02

2012-01-01 12:31:16

/a/....

10

1

S002

User02

2012-01-01 12:33:06

/b/....

110

2

S002

User02

2012-01-01 12:35:06

/e/....

30

3

Session

起始時間

結束時間

進入頁面

離開頁面

訪問頁面數

IP

cookie

referal

S001

2012-01-01 12:31:12

2012-01-01 12:31:12

/a/...

/a/...

1

101.0.0.1

User01

somesite.com

S002

2012-01-01 12:31:16

2012-01-01 12:35:06

/a/...

/e/...

3

201.0.0.2

User02

-

S003

2012-01-01 12:35:42

2012-01-01 12:35:42

/c/...

/c/...

1

234.0.0.3

User03

baidu.com

S003

2012-01-01 15:16:39

2012-01-01 15:19:23

/c/...

/e/...

3

101.0.0.1

User01

google.com

……

……

……

……

……

……

……

……

……

網站的眼睛

網站的神經

網站的大腦

訪問者來自哪里？

訪問者在尋找什么？

哪些頁面最受歡迎？

訪問者從哪里進入？

網頁布局合理嗎？

網站導航清晰嗎？

哪些功能存在問題

網站內容有效嗎

轉化路徑靠譜嗎？

如何分解目標？

如何分配廣告預算？

如何衡量產品表現？

哪些產品需要優化？

哪些指標需要關注？

58.215.204.118 - - [18/Sep/2013:06:51:35 +0000] "GET /wp-includes/js/jquery/jquery.js?ver=1.10.2 HTTP/1.1" 304 0 "http://blog.fens.me/nodejs-socketio-chat/" "Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0"

log4j.appender.logDailyFile = org.apache.log4j.DailyRollingFileAppender

log4j.appender.logDailyFile.layout = org.apache.log4j.PatternLayout

log4j.appender.logDailyFile.layout.ConversionPattern = [%-5p][%-22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n

log4j.appender.logDailyFile.Threshold = DEBUG

log4j.appender.logDailyFile.ImmediateFlush = TRUE

log4j.appender.logDailyFile.Append = TRUE

log4j.appender.logDailyFile.File = /var/logs/access_log?

log4j.appender.logDailyFile.DatePattern = '.'yyyy-MM-dd-HH-mm'.log'

log4j.appender.logDailyFile.Encoding = UTF-8

agent1.sources = source1

agent1.sinks = sink1

agent1.channels = channel1

# Describe/configure spooldir source1

#agent1.sources.source1.type = spooldir

#agent1.sources.source1.spoolDir = /var/logs/nginx/

#agent1.sources.source1.fileHeader = false

# Describe/configure tail -F source1

#使用exec作為數據源source組件

agent1.sources.source1.type = exec

#使用tail -F命令實時收集新產生的日志數據

agent1.sources.source1.command = tail -F /var/logs/nginx/access_log

agent1.sources.source1.channels = channel1

#configure host for source

#配置一個攔截器插件

agent1.sources.source1.interceptors = i1

agent1.sources.source1.interceptors.i1.type = host

#使用攔截器插件獲取agent所在服務器的主機名

agent1.sources.source1.interceptors.i1.hostHeader = hostname

#配置sink組件為hdfs

agent1.sinks.sink1.type = hdfs

#a1.sinks.k1.channel = c1

#agent1.sinks.sink1.hdfs.path=hdfs://hdp-node-01:9000/weblog/flume-collection/%y-%m-%d/%H%M%S

#指定文件sink到hdfs上的路徑

agent1.sinks.sink1.hdfs.path=

hdfs://hdp-node-01:9000/weblog/flume-collection/%y-%m-%d/%H-%M_%hostname

#指定文件名前綴

agent1.sinks.sink1.hdfs.filePrefix = access_log

agent1.sinks.sink1.hdfs.maxOpenFiles = 5000

#指定每批下沉數據的記錄條數

agent1.sinks.sink1.hdfs.batchSize= 100

agent1.sinks.sink1.hdfs.fileType = DataStream

agent1.sinks.sink1.hdfs.writeFormat =Text

#指定下沉文件按1G大小滾動

agent1.sinks.sink1.hdfs.rollSize = 1024*1024*1024

#指定下沉文件按1000000條數滾動

agent1.sinks.sink1.hdfs.rollCount = 1000000

#指定下沉文件按30分鐘滾動

agent1.sinks.sink1.hdfs.rollInterval = 30

#agent1.sinks.sink1.hdfs.round = true

#agent1.sinks.sink1.hdfs.roundValue = 10

#agent1.sinks.sink1.hdfs.roundUnit = minute

agent1.sinks.sink1.hdfs.useLocalTimeStamp = true

# Use a channel which buffers events in memory

#使用memory類型channel

agent1.channels.channel1.type = memory

agent1.channels.channel1.keep-alive = 120

agent1.channels.channel1.capacity = 500000

agent1.channels.channel1.transactionCapacity = 600

# Bind the source and sink to the channel

agent1.sources.source1.channels = channel1

agent1.sinks.sink1.channel = channel1

public class WeblogPreProcess {

static class WeblogPreProcessMapper extends Mapper<LongWritable, Text, Text, NullWritable> {

Text k = new Text();

NullWritable v = NullWritable.get();

@Override

protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

String line = value.toString();

WebLogBean webLogBean = WebLogParser.parser(line);

//WebLogBean productWebLog = WebLogParser.parser2(line);

//WebLogBean bbsWebLog = WebLogParser.parser3(line);

//WebLogBean cuxiaoBean = WebLogParser.parser4(line);

if (!webLogBean.isValid())

return;

k.set(webLogBean.toString());

context.write(k, v);

//k.set(productWebLog);

//context.write(k, v);

}

}

public static void main(String[] args) throws Exception {

Configuration conf = new Configuration();

Job job = Job.getInstance(conf);

job.setJarByClass(WeblogPreProcess.class);

job.setMapperClass(WeblogPreProcessMapper.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(NullWritable.class);

FileInputFormat.setInputPaths(job, new Path(args[0]));

FileOutputFormat.setOutputPath(job, new Path(args[1]));

job.waitForCompletion(true);

}

}

hadoop jar weblog.jar ?cn.itcast.bigdata.hive.mr.WeblogPreProcess /weblog/input /weblog/preout

代碼見工程

hadoop jar weblogpreprocess.jar ?\

cn.itcast.bigdata.hive.mr.ClickStreamThree ??\

/user/hive/warehouse/dw_click.db/test_ods_weblog_origin/datestr=2013-09-20/ /test-click/pageviews/

代碼見工程

hadoop jar weblogpreprocess.jar cn.itcast.bigdata.hive.mr.ClickStreamVisit /weblog/sessionout /weblog/visitout

drop table if exist click_stream_visit;

create table click_stream_visit(

session ????string,

remote_addr string,

inTime ?????string,

outTime ????string,

inPage ?????string,

outPage ????string,

referal ????string,

pageVisits ?int)

partitioned by (datestr string);

load data inpath '/weblog/visitout' into table click_stream_visit partition(datestr='2013-09-18');

原始數據表:t_origin_weblog

valid

string

是否有效

remote_addr

string

訪客ip

remote_user

string

訪客用戶信息

time_local

string

請求時間

request

string

請求url

status

string

響應碼

body_bytes_sent

string

響應字節數

http_referer

string

來源url

http_user_agent

string

訪客終端信息

ETL中間表：t_etl_referurl

valid

string

是否有效

remote_addr

string

訪客ip

remote_user

string

訪客用戶信息

time_local

string

請求時間

request

string

請求url

request_host

string

請求的域名

status

string

響應碼

body_bytes_sent

string

響應字節數

http_referer

string

來源url

http_user_agent

string

訪客終端信息

valid

string

是否有效

remote_addr

string

訪客ip

remote_user

string

訪客用戶信息

time_local

string

請求時間

request

string

請求url

status

string

響應碼

body_bytes_sent

string

響應字節數

http_referer

string

外鏈url

http_user_agent

string

訪客終端信息

host

string

外鏈url的域名

path

string

外鏈url的路徑

query

string

外鏈url的參數

query_id

string

外鏈url的參數值

訪問日志明細寬表：t_ods_access_detail

valid

string

是否有效

remote_addr

string

訪客ip

remote_user

string

訪客用戶信息

time_local

string

請求時間

request

string

請求url整串

request_level1

string

請求的一級欄目

request_level2

string

請求的二級欄目

request_level3

string

請求的三級欄目

status

string

響應碼

body_bytes_sent

string

響應字節數

http_referer

string

來源url

http_user_agent

string

訪客終端信息

valid

string

是否有效

remote_addr

string

訪客ip

remote_user

string

訪客用戶信息

time_local

string

請求時間

request

string

請求url

status

string

響應碼

body_bytes_sent

string

響應字節數

http_referer

string

外鏈url

http_user_agent

string

訪客終端信息整串

http_user_agent_browser

string

訪客終端瀏覽器

http_user_agent_sys

string

訪客終端操作系統

http_user_agent_dev

string

訪客終端設備

host

string

外鏈url的域名

path

string

外鏈url的路徑

query

string

外鏈url的參數

query_id

string

外鏈url的參數值

daystr

string

日期整串

tmstr

string

時間整串

month

string

月份

day

string

日

hour

string

時

minute

string

分

##

##

##

mm

string

分區字段--月

dd

string

分區字段--日

時間維度 v_year_month_date

year

month

day

hour

minute

訪客地域維度t_dim_area

北京

上海

廣州

深圳

河北

河南

終端類型維度t_dim_termination

uc

firefox

chrome

safari

ios

android

網站欄目維度 t_dim_section

跳蚤市場

房租信息

休閑娛樂

建材裝修

本地服務

人才市場

drop table if exists ods_weblog_origin;

create table ods_weblog_origin(

valid string,

remote_addr string,

remote_user string,

time_local string,

request string,

status string,

body_bytes_sent string,

http_referer string,

http_user_agent string)

partitioned by (datestr string)

row format delimited

fields terminated by '\001';

drop table if exists ods_click_pageviews;

create table ods_click_pageviews(

Session string,

remote_addr string,

time_local string,

request string,

visit_step string,

page_staylong string,

http_referer string,

http_user_agent string,

body_bytes_sent string,

status string)

partitioned by (datestr string)

row format delimited

fields terminated by '\001';

drop table dim_time if exists ods_click_pageviews;

create table dim_time(

year string,

month string,

day string,

hour string)

row format delimited

fields terminated by ',';

導入清洗結果數據到貼源數據表ods_weblog_origin

load data inpath '/weblog/preprocessed/16-02-24-16/' overwrite into table ods_weblog_origin partition(datestr='2013-09-18');

0: jdbc:hive2://localhost:10000> show partitions ods_weblog_origin;

+-------------------+--+

| ????partition ????|

+-------------------+--+

| timestr=20151203 ?|

+-------------------+--+

0: jdbc:hive2://localhost:10000> select count(*) from ods_origin_weblog;

+--------+--+

| ?_c0 ??|

+--------+--+

| 11347 ?|

+--------+--+

導入點擊流模型pageviews數據到ods_click_pageviews表

0: jdbc:hive2://hdp-node-01:10000> load data inpath '/weblog/clickstream/pageviews' overwrite into table ods_click_pageviews partition(datestr='2013-09-18');

0: jdbc:hive2://hdp-node-01:10000> select count(1) from ods_click_pageviews;

+------+--+

| _c0 ?|

+------+--+

| 66 ??|

+------+--+

導入點擊流模型visit數據到ods_click_visit表

導入時間維表：

load data inpath '/dim_time.txt' into table dim_time;

drop table ods_weblog_detail;

create table ods_weblog_detail(

valid ??????????string, --有效標識

remote_addr ????string, --來源IP

remote_user ????string, --用戶標識

time_local ?????string, --訪問完整時間

daystr ?????????string, --訪問日期

timestr ????????string, --訪問時間

month ??????????string, --訪問月

day ????????????string, --訪問日

hour ???????????string, --訪問時

request ????????string, --請求的url

status ?????????string, --響應碼

body_bytes_sent string, --傳輸字節數

http_referer ???string, --來源url

ref_host ???????string, --來源的host

ref_path ???????string, --來源的路徑

ref_query ??????string, --來源參數query

ref_query_id ???string, --來源參數query的值

http_user_agent string --客戶終端標識

)

partitioned by(datestr string);

drop table if exists t_ods_tmp_referurl;

create table t_ ods _tmp_referurl as

SELECT a.*,b.*

FROM ods_origin_weblog a LATERAL VIEW parse_url_tuple(regexp_replace(http_referer, "\"", ""), 'HOST', 'PATH','QUERY', 'QUERY:id') b as host, path, query, query_id;?

drop table if exists t_ods_tmp_detail;

create table t_ods_tmp_detail as

select b.*,substring(time_local,0,10) as daystr,

substring(time_local,11) as tmstr,

substring(time_local,5,2) as month,

substring(time_local,8,2) as day,

substring(time_local,11,2) as hour

From?t_?ods?_tmp_referurl b;

insert into table zs.ods_weblog_detail partition(datestr='$day_01')

select c.valid,c.remote_addr,c.remote_user,c.time_local,

substring(c.time_local,0,10) as daystr,

substring(c.time_local,12) as tmstr,

substring(c.time_local,6,2) as month,

substring(c.time_local,9,2) as day,

substring(c.time_local,11,3) as hour,

c.request,c.status,c.body_bytes_sent,c.http_referer,c.ref_host,c.ref_path,c.ref_query,c.ref_query_id,c.http_user_agent

from

(SELECT

a.valid,a.remote_addr,a.remote_user,a.time_local,

a.request,a.status,a.body_bytes_sent,a.http_referer,a.http_user_agent,b.ref_host,b.ref_path,b.ref_query,b.ref_query_id

FROM zs.ods_weblog_origin a LATERAL VIEW parse_url_tuple(regexp_replace(http_referer, "\"", ""), 'HOST', 'PATH','QUERY', 'QUERY:id') b as ref_host, ref_path, ref_query, ref_query_id) c

"

0: jdbc:hive2://localhost:10000> show partitions ods_weblog_detail;

+---------------------+--+

| ?????partition ?????|

+---------------------+--+

| dd=18%2FSep%2F2013 ?|

+---------------------+--+

1 row selected (0.134 seconds)

create table v_time(year string,month string,day string,hour string)

row format delimited

fields terminated by ',';

load data local inpath '/home/hadoop/v_time.txt' into table v_time;

--計算指定的某個小時pvs

select count(*),month,day,hour from dw_click.ods_weblog_detail group by month,day,hour;

--計算該處理批次(一天)中的各小時pvs

drop table dw_pvs_hour;

create table dw_pvs_hour(month string,day string,hour string,pvs bigint) partitioned by(datestr string);

insert into table dw_pvs_hour partition(datestr='2016-03-18')

select a.month as month,a.day as day,a.hour as hour,count(1) as pvs from ods_weblog_detail a

where ?a.datestr='2016-03-18' ?group by a.month,a.day,a.hour;

或者用時間維表關聯

drop table dw_pvs_day;

create table dw_pvs_day(pvs bigint,month string,day string);

insert into table dw_pvs_day

select count(1) as pvs,a.month as month,a.day as day ?from dim_time a

join ods_weblog_detail b

on b.dd='18/Sep/2013' and a.month=b.month and a.day=b.day

group by a.month,a.day;

--或者，從之前算好的小時結果中統計

Insert into table dw_pvs_day

Select sum(pvs) as pvs,month,day from dw_pvs_hour group by month,day having day='18';

結果如下：

drop table t_display_pv_month;

create table t_display_pv_month (pvs bigint,month string);

insert into table t_display_pv_month

select count(*) as pvs,a.month from t_dim_time a

join t_ods_detail_prt b on a.month=b.month group by a.month;

select distinct(http_user_agent) from ods_weblog_detail where http_user_agent like '%Mozilla%' limit 200;

drop table t_display_pv_terminal_uc;

create table t_display_pv_ terminal_uc (pvs bigint,mm string,dd string,hh string);

drop table t_display_pv_terminal_chrome;

create table t_display_pv_ terminal_ chrome (pvs bigint,mm string,dd string,hh string);

drop table t_display_pv_terminal_safari;

create table t_display_pv_ terminal_ safari (pvs bigint,mm string,dd string,hh string);

drop table dw_avgpv_user_d;

create table dw_avgpv_user_d(

day string,

avgpv string);

insert into table dw_avgpv_user_d

select '2013-09-18',sum(b.pvs)/count(b.remote_addr) from

(select remote_addr,count(1) as pvs from ods_weblog_detail where datestr='2013-09-18' group by remote_addr) b;

drop table dw_pvs_referer_h;

create table dw_pvs_referer_h(referer_url string,referer_host string,month string,day string,hour string,pv_referer_cnt bigint) partitioned by(datestr string);

insert into table dw_pvs_referer_h partition(datestr='2016-03-18')

select http_referer,ref_host,month,day,hour,count(1) as pv_referer_cnt

from ods_weblog_detail

group by http_referer,ref_host,month,day,hour

having ref_host is not null

order by hour asc,day asc,month asc,pv_referer_cnt desc;

drop table dw_ref_host_visit_cnts_h;

create table dw_ref_host_visit_cnts_h(ref_host string,month string,day string,hour string,ref_host_cnts bigint) partitioned by(datestr string);

insert into table dw_ref_host_visit_cnts_h partition(datestr='2016-03-18')

select ref_host,month,day,hour,count(1) as ref_host_cnts

from ods_weblog_detail

group by ref_host,month,day,hour

having ref_host is not null

order by hour asc,day asc,month asc,ref_host_cnts desc;

drop table dw_pvs_refhost_topn_h;

create table dw_pvs_refhost_topn_h(

hour string,

toporder string,

ref_host string,

ref_host_cnts string

) partitioned by(datestr string);

insert into table zs.dw_pvs_refhost_topn_h partition(datestr='2016-03-18')

select t.hour,t.od,t.ref_host,t.ref_host_cnts from

?(select ref_host,ref_host_cnts,concat(month,day,hour) as hour,

row_number() over (partition by concat(month,day,hour) order by ref_host_cnts desc) as od

from zs.dw_ref_host_visit_cnts_h) t where od<=3;

drop table dw_pvs_d;

create table dw_pvs_d(day string,url string,pvs string);

insert into table dw_pvs_d

select '2013-09-18',a.request,a.request_counts from

(select request as request,count(request) as request_counts from ods_weblog_detail where datestr='2013-09-18' group by request having request is not null) a

order by a.request_counts desc limit 10;

結果如下：

drop table dw_user_dstc_ip_h;

create table dw_user_dstc_ip_h(

remote_addr string,

pvs ?????bigint,

hour ????string);

insert into table dw_user_dstc_ip_h

select remote_addr,count(1) as pvs,concat(month,day,hour) as hour

from ods_weblog_detail

Where datestr='2013-09-18'

group by concat(month,day,hour),remote_addr;

select count(1) as dstc_ip_cnts,hour from dw_user_dstc_ip_h group by hour;

練習：

統計每小時請求次數topn的獨立訪客

select remote_addr,count(1) as counts,month

from ods_weblog_detail

group by month,remote_addr;

select remote_addr,count(1) as counts,concat(month,day) as day

from ods_weblog_detail

Where dd='18/Sep/2013'

group by concat(month,day),remote_addr;

--歷日去重訪客累積表

drop table dw_user_dsct_history;

create table dw_user_dsct_history(

day string,

ip string

)

partitioned by(datestr string);

--每日新用戶追加到累計表

drop table dw_user_dsct_history;

create table dw_user_dsct_history(

day string,

ip string

)

partitioned by(datestr string);

--每日新用戶追加到累計表

insert into table dw_user_dsct_history partition(datestr='2013-09-19')

select tmp.day as day,tmp.today_addr as new_ip from

(

select today.day as day,today.remote_addr as today_addr,old.ip as old_addr

from

(select distinct remote_addr as remote_addr,"2013-09-19" as day from ods_weblog_detail where datestr="2013-09-19") today

left outer join

dw_user_dsct_history old

on today.remote_addr=old.ip

) tmp

where tmp.old_addr is null;

select count(distinct remote_addr) from ods_weblog_detail;

-- 1005

select count(1) from dw_user_dsct_history where prtflag_day='18/Sep/2013';

--845

select count(1) from dw_user_dsct_history where prtflag_day='19/Sep/2013';

--160

類似日粒度算法

drop table dw_user_returning;

create table dw_user_returning(

day string,

remote_addr string,

acc_cnt string)

partitioned by (datestr string);

insert overwrite table dw_user_returning partition(datestr='2013-09-18')

select tmp.day,tmp.remote_addr,tmp.acc_cnt

from

(select '2013-09-18' as day,remote_addr,count(session) as acc_cnt from click_stream_visit group by remote_addr) tmp

where tmp.acc_cnt>1;

select sum(pagevisits)/count(distinct remote_addr) from click_stream_visit partition(datestr='2013-09-18');

注：代碼略長，見項目工程代碼

[hadoop@hdp-node-01 ~]$ hadoop jar weblog.jar cn.itcast.bigdata.hive.mr.UserStayTime /weblog/input /weblog/stayout4

--導入hive表("t_display_access_info")中

drop table ods_access_info;

create table ods_access_info(remote_addr string,firt_req_time string,last_req_time string,stay_long string)

partitioned by(prtflag_day string)

row format delimited

fields terminated by '\t';

load data inpath '/weblog/stayout4' into table ods_access_info partition(prtflag_day='18/Sep/2013');

創建表時stay_long使用的string類型，但是在后續過程中發現還是用bigint更好，進行表修改

alter table ods_access_info change column stay_long stay_long bigint;

drop table dw_access_info;

create table dw_access_info(remote_addr string,firt_req_time string,last_req_time string,stay_long string)

partitioned by(prtflag_day string);

insert into table dw_access_info partition(prtflag_day='19/Sep/2013')

select remote_addr,firt_req_time,last_req_time,

case stay_long

when 0 then 30000

else stay_long

end as stay_long

from ods_access_info

where prtflag_day='18/Sep/2013';

drop table dw_access_info_htip;

create table dw_access_info_htip(remote_addr string, firt_req_time string, last_req_time string, stay_long string,acc_counts string)

partitioned by(prtflag_day string);

insert into table dw_access_info_htip partition(prtflag_day='18/Sep/2013')

select b.remote_addr,b.firt_req_time,b.last_req_time,b.stay_long,a.acc_counts from

(select remote_addr,count(remote_addr) as acc_counts from dw_access_info where prtflag_day='18/Sep/2013' group by remote_addr having acc_counts>1) a

join

dw_access_info b

on a.remote_addr = b.remote_addr;

drop table dw_access_info_dcip;

create table dw_access_info_dcip(remote_addr string, firt_req_time string, last_req_time string, stay_long string,acc_counts string)

partitioned by(prtflag_day string);

insert into table dw_access_dcip partition(prtflag_day='18/Sep/2013')

select b.remote_addr,b.firt_req_time,b.last_req_time,b.stay_long,a.acc_counts from

(select remote_addr,count(remote_addr) as acc_counts from dw_access_info where prtflag_day='18/Sep/2013' group by remote_addr having acc_counts<2) a

join

dw_access_info b

on a.remote_addr = b.remote_addr;

drop table dw_htpercent_d;

create table dw_htpercent_d(day string,ht_percent float);

Insert into table dw_htpercent_d

select '18/Sep/2013',(tmp_ht.ht/tmp_all.amount)*100 from

(select count( distinct a.remote_addr) as ht from dw_access_info_htip a where prtflag_day='18/Sep/2013') tmp_ht

Join

(select count(distinct b.remote_addr) as amount from dw_access_info b where prtflag_day='18/Sep/2013') tmp_all;

select avg(user_times.counts) as user_access_freq

from

(select remote_addr,counts from t_display_htip

union all

select remote_addr,counts from t_display_access_dcip) user_times;

--或直接從 訪問信息表 t_display_access_info 中查詢

select avg(a.acc_cts) from

(select remote_addr,count(*) as acc_cts from dw_access_info group by remote_addr) a;

create table route_numbs as

select 'step1' as step,count(distinct remote_addr) ?as numbs from ods_click_pageviews where request like '/item%'

union

select 'step2' as step,count(distinct remote_addr) as numbs from ods_click_pageviews where request like '/category%'

union

select 'step3' as step,count(distinct remote_addr) as numbs from ods_click_pageviews where request like '/order%'

union

select 'step4' as step,count(distinct remote_addr) ?as numbs from ods_click_pageviews where request like '/index%';

select rn.step as rnstep,rn.numbs as rnnumbs,rr.step as rrstep,rr.numbs as rrnumbs ?from route_num rn

inner join

route_num rr

select tmp.rnstep,tmp.rnnumbs/tmp.rrnumbs as ratio

from

(

select rn.step as rnstep,rn.numbs as rnnumbs,rr.step as rrstep,rr.numbs as rrnumbs ?from route_num rn

inner join

route_num rr) tmp

where tmp.rrstep='step1';

select tmp.rrstep as rrstep,tmp.rrnumbs/tmp.rnnumbs as ration

from

(

select rn.step as rnstep,rn.numbs as rnnumbs,rr.step as rrstep,rr.numbs as rrnumbs ?from route_num rn

inner join

route_num rr) tmp

where cast(substr(tmp.rnstep,5,1) as int)=cast(substr(tmp.rrstep,5,1) as int)-1

select abs.step,abs.numbs,abs.ratio as abs_ratio,rel.ratio as rel_ratio

from

(

select tmp.rnstep as step,tmp.rnnumbs as numbs,tmp.rnnumbs/tmp.rrnumbs as ratio

from

(

select rn.step as rnstep,rn.numbs as rnnumbs,rr.step as rrstep,rr.numbs as rrnumbs ?from route_num rn

inner join

route_num rr) tmp

where tmp.rrstep='step1'

) abs

left outer join

(

select tmp.rrstep as step,tmp.rrnumbs/tmp.rnnumbs as ratio

from

(

select rn.step as rnstep,rn.numbs as rnnumbs,rr.step as rrstep,rr.numbs as rrnumbs ?from route_num rn

inner join

route_num rr) tmp

where cast(substr(tmp.rnstep,5,1) as int)=cast(substr(tmp.rrstep,5,1) as int)-1

) rel

on abs.step=rel.step

sqoop export \

--connect jdbc:mysql://hdp-node-01:3306/webdb --username root --password root ?\

--table click_stream_visit ?\

--export-dir /user/hive/warehouse/dw_click.db/click_stream_visit/datestr=2013-09-18 \

--input-fields-terminated-by '\001'

<workflow-app name="weblogpreprocess" xmlns="uri:oozie:workflow:0.4">

<start to="firstjob"/>

<action name="firstjob">

<map-reduce>

<job-tracker>${jobTracker}</job-tracker>

<name-node>${nameNode}</name-node>

<prepare>

<delete path="${nameNode}/${outpath}"/>

</prepare>

<configuration>

<property>

<name>mapreduce.job.map.class</name>

<value>cn.itcast.bigdata.hive.mr.WeblogPreProcess$WeblogPreProcessMapper</value>

</property>

<property>

<name>mapreduce.job.output.key.class</name>

<value>org.apache.hadoop.io.Text</value>

</property>

<property>

<name>mapreduce.job.output.value.class</name>

<value>org.apache.hadoop.io.NullWritable</value>

</property>

<property>

<name>mapreduce.input.fileinputformat.inputdir</name>

<value>${inpath}</value>

</property>

<property>

<name>mapreduce.output.fileoutputformat.outputdir</name>

<value>${outpath}</value>

</property>

<property>

<name>mapred.mapper.new-api</name>

<value>true</value>

</property>

<property>

<name>mapred.reducer.new-api</name>

<value>true</value>

</property>

</configuration>

</map-reduce>

<ok to="end"/>

<error to="kill"/>

<workflow-app xmlns="uri:oozie:workflow:0.5" name="hive2-wf">

<start to="hive2-node"/>

<action name="hive2-node">

<hive2 xmlns="uri:oozie:hive2-action:0.1">

<job-tracker>${jobTracker}</job-tracker>

<name-node>${nameNode}</name-node>

<configuration>

<property>

<name>mapred.job.queue.name</name>

<value>${queueName}</value>

</property>

</configuration>

<jdbc-url>jdbc:hive2://hdp-node-01:10000</jdbc-url>

<script>script.q</script>

<param>input=/weblog/outpre2</param>

</hive2>

<ok to="end"/>

<error to="fail"/>

</action>

<kill name="fail">

<message>Hive2 (Beeline) action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>

</kill>

<end name="end"/>

</workflow-app>

create database if not exists dw_weblog;

use dw_weblog;

drop table if exists t_orgin_weblog;

create table t_orgin_weblog(valid string,remote_addr string,

remote_user string,

time_local string,

request string,

status string,

body_bytes_sent string,

http_referer string,

http_user_agent string)

row format delimited

fields terminated by '\001';

load data inpath '/weblog/preout' overwrite into table t_orgin_weblog;

drop table if exists t_ods_detail_tmp_referurl;

create table t_ods_detail_tmp_referurl as

SELECT a.*,b.*

FROM t_orgin_weblog a

LATERAL VIEW parse_url_tuple(regexp_replace(http_referer, "\"", ""), 'HOST', 'PATH','QUERY', 'QUERY:id') b as host, path, query, query_id;

drop table if exists t_ods_detail;

create table t_ods_detail as

select b.*,substring(time_local,0,11) as daystr,

substring(time_local,13) as tmstr,

substring(time_local,4,3) as month,

substring(time_local,0,2) as day,

substring(time_local,13,2) as hour

from t_ods_detail_tmp_referurl b;

drop table t_ods_detail_prt;

create table t_ods_detail_prt(

valid ?????????????????string,

remote_addr ???????????string,

remote_user ???????????string,

time_local ????????????string,

request ???????????????string,

status ????????????????string,

body_bytes_sent ???????string,

http_referer ??????????string,

http_user_agent ???????string,

host ??????????????????string,

path ??????????????????string,

query ?????????????????string,

query_id ??????????????string,

daystr ????????????????string,

tmstr ?????????????????string,

month ?????????????????string,

day ???????????????????string,

hour ??????????????????string)

partitioned by (mm string,dd string);

insert into table t_ods_detail_prt partition(mm='Sep',dd='18')

select * from t_ods_detail where daystr='18/Sep/2013';

insert into table t_ods_detail_prt partition(mm='Sep',dd='19')

select * from t_ods_detail where daystr='19/Sep/2013';

[hadoop@hdp-node-01 wf-oozie]$ hadoop fs -put hive2-etl /user/hadoop/oozie/myapps/

[hadoop@hdp-node-01 wf-oozie]$ hadoop fs -put hive2-dw /user/hadoop/oozie/myapps/

[hadoop@hdp-node-01 wf-oozie]$ ll

total 12

drwxrwxr-x. 2 hadoop hadoop 4096 Nov 23 16:32 hive2-dw

drwxrwxr-x. 2 hadoop hadoop 4096 Nov 23 16:32 hive2-etl

drwxrwxr-x. 3 hadoop hadoop 4096 Nov 23 11:24 weblog

[hadoop@hdp-node-01 wf-oozie]$ export OOZIE_URL=http://localhost:11000/oozie

[hadoop@hdp-node-01 hive2-etl]$ ll

total 28

-rw-rw-r--. 1 hadoop hadoop ?265 Nov 13 16:39 config-default.xml

-rw-rw-r--. 1 hadoop hadoop ?512 Nov 26 16:43 coordinator.xml

-rw-rw-r--. 1 hadoop hadoop ?382 Nov 26 16:49 job.properties

drwxrwxr-x. 2 hadoop hadoop 4096 Nov 27 11:26 lib

-rw-rw-r--. 1 hadoop hadoop 1910 Nov 23 17:49 script.q

-rw-rw-r--. 1 hadoop hadoop ?687 Nov 23 16:32 workflow.xml

<configuration>

<property>

<name>jobTracker</name>

<value>hdp-node-01:8032</value>

</property>

<property>

<name>nameNode</name>

<value>hdfs://hdp-node-01:9000</value>

</property>

<property>

<name>queueName</name>

<value>default</value>

</property>

</configuration>

user.name=hadoop

oozie.use.system.libpath=true

oozie.libpath=hdfs://hdp-node-01:9000/user/hadoop/share/lib

oozie.wf.application.path=hdfs://hdp-node-01:9000/user/hadoop/oozie/myapps/hive2-etl/

<workflow-app xmlns="uri:oozie:workflow:0.5" name="hive2-wf">

<start to="hive2-node"/>

<action name="hive2-node">

<hive2 xmlns="uri:oozie:hive2-action:0.1">

<job-tracker>${jobTracker}</job-tracker>

<name-node>${nameNode}</name-node>

<configuration>

<property>

<name>mapred.job.queue.name</name>

<value>${queueName}</value>

</property>

</configuration>

<jdbc-url>jdbc:hive2://hdp-node-01:10000</jdbc-url>

<script>script.q</script>

<param>input=/weblog/outpre2</param>

</hive2>

<ok to="end"/>

<error to="fail"/>

</action>

<kill name="fail">

<message>Hive2 (Beeline) action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>

</kill>

<end name="end"/>

</workflow-app>

<coordinator-app name="cron-coord" frequency="${coord:minutes(5)}" start="${start}" end="${end}" timezone="Asia/Shanghai" xmlns="uri:oozie:coordinator:0.2">

<action>

<workflow>

<app-path>${workflowAppUri}</app-path>

<configuration>

<property>

<name>jobTracker</name>

<value>${jobTracker}</value>

</property>

<property>

<name>nameNode</name>

<value>${nameNode}</value>

</property>

<property>

<name>queueName</name>

<value>${queueName}</value>

</property>

</configuration>

</workflow>

</action>

</coordinator-app>

頁面

HTML + JQUERY + ECHARTS

Controller

SpringMVC

Service

Service

DAO

Mybatis

數據庫

Mysql

@Service("chartService")

public class ChartServiceImpl implements IChartService {

@Autowired

IEchartsDao iEchartsDao;

public EchartsData getChartsData() {

List<Integer> xAxiesList = iEchartsDao.getXAxiesList("");

List<Integer> pointsDataList = iEchartsDao.getPointsDataList("");

EchartsData data = new EchartsData();

ToolBox toolBox = EchartsOptionUtil.getToolBox();

Serie serie = EchartsOptionUtil.getSerie(pointsDataList);

ArrayList<Serie> series = new ArrayList<Serie>();

series.add(serie);

List<XAxi> xAxis = EchartsOptionUtil.getXAxis(xAxiesList);

List<YAxi> yAxis = EchartsOptionUtil.getYAxis();

HashMap<String, String> title = new HashMap<String, String>();

title.put("text", "pvs");

title.put("subtext", "超級pvs");

HashMap<String, String> tooltip = new HashMap<String, String>();

tooltip.put("trigger", "axis");

HashMap<String, String[]> legend = new HashMap<String, String[]>();

legend.put("data", new String[]{"pv統計"});

data.setTitle(title);

data.setTooltip(tooltip);

data.setLegend(legend);

data.setToolbox(toolBox);

data.setCalculable(true);

data.setxAxis(xAxis);

data.setyAxis(yAxis);

data.setSeries(series);

return data;

}

public List<HashMap<String, Integer>> getGaiKuangList(String date) throws ParseException{

HashMap<String, Integer> gaiKuangToday = iEchartsDao.getGaiKuang(date);

SimpleDateFormat sf = new SimpleDateFormat("MMdd");

Date parse = sf.parse(date);

Calendar calendar = Calendar.getInstance();

calendar.setTime(parse);

calendar.add(Calendar.DAY_OF_MONTH, -1);

Date before = calendar.getTime();

String beforeString = sf.format(before);

System.out.println(beforeString);

HashMap<String, Integer> gaiKuangBefore = iEchartsDao.getGaiKuang(beforeString);

ArrayList<HashMap<String, Integer>> gaiKuangList = new ArrayList<HashMap<String, Integer>>();

gaiKuangList.add(gaiKuangToday);

gaiKuangList.add(gaiKuangBefore);

return gaiKuangList;

}

public static void main(String[] args) {

ChartServiceImpl chartServiceImpl = new ChartServiceImpl();

EchartsData chartsData = chartServiceImpl.getChartsData();

Gson gson = new Gson();

String json = gson.toJson(chartsData);

System.out.println(json);

}

}