Skip to content

张浩 / BrBigDataTest

Go to a project
Toggle navigation
Toggle navigation pinning
Commit bc2af192 bc2af192021737534661770f80839e72a61176f3 by zhanghao
Options

commit

1 parent c23aa6f9
Inline Side-by-side
Showing 6 changed files with 23 additions and 6 deletions
...@@ -9,11 +9,14 @@ hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLApp /input/data/ ...@@ -9,11 +9,14 @@ hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLApp /input/data/
9 sleep 3 9 sleep 3
10 10
11 hive -f ./shell/load_data.hql 11 hive -f ./shell/load_data.hql
12 hive -f ./shell/method_count.hql
12 hive -f ./shell/biz_data_count.hql 13 hive -f ./shell/biz_data_count.hql
13 14
14 hadoop fs -rm /input/data/$time 15 hadoop fs -rm /input/data/$time
15 rm ../$time 16 rm ../$time
16 17
18 sqoop export --connect jdbc:mysql://127.0.0.1:3306/bigdata --username root --password zidonghua --table method_count --export-dir /user/hive/warehouse/testdb.db/method_count --input-fields-terminated-by '\t'
19 sleep 3
17 sqoop export --connect jdbc:mysql://127.0.0.1:3306/bigdata --username root --password zidonghua --table biz_data_count --export-dir /user/hive/warehouse/testdb.db/biz_data_count --input-fields-terminated-by '\t' 20 sqoop export --connect jdbc:mysql://127.0.0.1:3306/bigdata --username root --password zidonghua --table biz_data_count --export-dir /user/hive/warehouse/testdb.db/biz_data_count --input-fields-terminated-by '\t'
18 sleep 3 21 sleep 3
19 sqoop export --connect jdbc:mysql://127.0.0.1:3306/bigdata --username root --password zidonghua --table biz_data_response --export-dir /user/hive/warehouse/testdb.db/biz_data_response --input-fields-terminated-by '\t' 22 sqoop export --connect jdbc:mysql://127.0.0.1:3306/bigdata --username root --password zidonghua --table biz_data_response --export-dir /user/hive/warehouse/testdb.db/biz_data_response --input-fields-terminated-by '\t'
......
...@@ -4,9 +4,4 @@ INSERT INTO TABLE biz_data_count ...@@ -4,9 +4,4 @@ INSERT INTO TABLE biz_data_count
4 select method,biz_data,count(1) as count,max(timestamp) as timestamp 4 select method,biz_data,count(1) as count,max(timestamp) as timestamp
5 from qa_log 5 from qa_log
6 where method='com.ai.handler.plan.detail.index.info' 6 where method='com.ai.handler.plan.detail.index.info'
7 group by method,biz_data;
8
9 INSERT INTO TABLE biz_data_response
10 select method,biz_data,uid,timestamp,response
11 from qa_log
12 where method='com.ai.handler.plan.detail.index.info';
...\ No newline at end of file ...\ No newline at end of file
7 group by method,biz_data;
...\ No newline at end of file ...\ No newline at end of file
......
1 use testdb;
2
3 INSERT INTO TABLE biz_data_response
4 select method,biz_data,uid,timestamp,response
5 from qa_log
6 where method='com.ai.handler.plan.detail.index.info';
...\ No newline at end of file ...\ No newline at end of file
...@@ -14,6 +14,12 @@ trace string, ...@@ -14,6 +14,12 @@ trace string,
14 timestamp string 14 timestamp string
15 ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'; 15 ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
16 16
17 CREATE TABLE method_count(
18 method string,
19 count string,
20 timestamp string
21 ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
22
17 CREATE TABLE biz_data_count( 23 CREATE TABLE biz_data_count(
18 method string, 24 method string,
19 biz_data string, 25 biz_data string,
......
1 use testdb; 1 use testdb;
2 truncate table qa_log; 2 truncate table qa_log;
3 truncate table method_count;
3 truncate table biz_data_count; 4 truncate table biz_data_count;
4 truncate table biz_data_response; 5 truncate table biz_data_response;
5 LOAD DATA INPATH 'hdfs://localhost:8020/input/etl' OVERWRITE INTO TABLE qa_log; 6 LOAD DATA INPATH 'hdfs://localhost:8020/input/etl' OVERWRITE INTO TABLE qa_log;
...\ No newline at end of file ...\ No newline at end of file
......
1 use testdb;
2
3 INSERT INTO TABLE method_count
4 select method,count(1) as count,max(timestamp) as timestamp
5 from qa_log
6 group by method;
...\ No newline at end of file ...\ No newline at end of file