Skip to content

张浩 / BrBigDataTest

Go to a project
Toggle navigation
Toggle navigation pinning
Commit e0b7e9d3 e0b7e9d311d575fb9b586858f698129e3e970d5d by zhanghao
Options

commit

1 parent 8489f1b7
Inline Side-by-side
Showing 5 changed files with 13 additions and 6 deletions
...@@ -4,10 +4,12 @@ hadoop fs -put /home/test/logs/app.out /input/debug/data/app.out ...@@ -4,10 +4,12 @@ hadoop fs -put /home/test/logs/app.out /input/debug/data/app.out
4 hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLAppDebug /input/debug/data/app.out /input/debug/etl/ 4 hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLAppDebug /input/debug/data/app.out /input/debug/etl/
5 sleep 3 5 sleep 3
6 6
7 maxTimestamp=$(hive -f ./shell/max_timestamp.hql)
8
7 hive -f ./shell_debug/load_data.hql 9 hive -f ./shell_debug/load_data.hql
8 hive -f ./shell_debug/method_count.hql 10 $(hive -hivevar maxTimestamp=$maxTimestamp -f ./shell_debug/method_count.hql)
9 hive -f ./shell_debug/biz_data_count.hql 11 $(hive -hivevar maxTimestamp=$maxTimestamp -f ./shell_debug/biz_data_count.hql)
10 hive -f ./shell_debug/biz_data_response.hql 12 $(hive -hivevar maxTimestamp=$maxTimestamp -f ./shell_debug/biz_data_response.hql)
11 13
12 hadoop fs -rm /input/debug/data/app.out 14 hadoop fs -rm /input/debug/data/app.out
13 15
......
...@@ -3,5 +3,5 @@ use testdb; ...@@ -3,5 +3,5 @@ use testdb;
3 INSERT OVERWRITE TABLE biz_data_count_debug 3 INSERT OVERWRITE TABLE biz_data_count_debug
4 select method,biz_data,count(1) as count,max(timestamp) as timestamp 4 select method,biz_data,count(1) as count,max(timestamp) as timestamp
5 from qa_log_debug 5 from qa_log_debug
6 where method is not null and method!='N/A' 6 where method is not null and method!='N/A' and timestamp>${hivevar:maxTimestamp}
7 group by method,biz_data; 7 group by method,biz_data;
...\ No newline at end of file ...\ No newline at end of file
......
...@@ -3,4 +3,4 @@ use testdb; ...@@ -3,4 +3,4 @@ use testdb;
3 INSERT OVERWRITE TABLE biz_data_response_debug 3 INSERT OVERWRITE TABLE biz_data_response_debug
4 select method,biz_data,uid,timestamp,response 4 select method,biz_data,uid,timestamp,response
5 from qa_log_debug 5 from qa_log_debug
6 where method is not null and method!='N/A';
...\ No newline at end of file ...\ No newline at end of file
6 where method is not null and method!='N/A' and timestamp>${hivevar:maxTimestamp};
...\ No newline at end of file ...\ No newline at end of file
......
1 use testdb;
2
3 select max(timestamp) as timestamp
4 from qa_log_debug
5 where method is not null and method!='N/A'
...\ No newline at end of file ...\ No newline at end of file
...@@ -3,5 +3,5 @@ use testdb; ...@@ -3,5 +3,5 @@ use testdb;
3 INSERT OVERWRITE TABLE method_count_debug 3 INSERT OVERWRITE TABLE method_count_debug
4 select method,count(1) as count,max(timestamp) as timestamp 4 select method,count(1) as count,max(timestamp) as timestamp
5 from qa_log_debug 5 from qa_log_debug
6 where method is not null and method!='N/A' 6 where method is not null and method!='N/A' and timestamp>${hivevar:maxTimestamp}
7 group by method; 7 group by method;
...\ No newline at end of file ...\ No newline at end of file
......