commit
Showing
5 changed files
with
9 additions
and
12 deletions
| ... | @@ -4,14 +4,12 @@ hadoop fs -put /home/test/logs/app.out /input/debug/data/app.out | ... | @@ -4,14 +4,12 @@ hadoop fs -put /home/test/logs/app.out /input/debug/data/app.out |
| 4 | hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLAppDebug /input/debug/data/app.out /input/debug/etl/ | 4 | hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLAppDebug /input/debug/data/app.out /input/debug/etl/ |
| 5 | sleep 3 | 5 | sleep 3 |
| 6 | 6 | ||
| 7 | #maxTimestamp='2022-02-10 17:13:21.308' | 7 | hive -f ./shell_debug/max_timestamp.hql |
| 8 | maxTimestamp=$(hive -f ./shell_debug/max_timestamp.hql) | ||
| 9 | |||
| 10 | hive -f ./shell_debug/load_data.hql | 8 | hive -f ./shell_debug/load_data.hql |
| 11 | $(hive -hivevar myTimestamp=$maxTimestamp -f ./shell_debug/method_count.hql) | 9 | hive -f ./shell_debug/method_count.hql |
| 12 | $(hive -hivevar myTimestamp=$maxTimestamp -f ./shell_debug/biz_data_count.hql) | 10 | hive -f ./shell_debug/biz_data_count.hql |
| 13 | $(hive -hivevar myTimestamp=$maxTimestamp -f ./shell_debug/biz_data_response.hql) | 11 | hive -f ./shell_debug/biz_data_response.hql |
| 14 | $(hive -hivevar myTimestamp=$maxTimestamp -f ./shell_debug/huo_dong_ru_kou.hql) | 12 | hive -f ./shell_debug/huo_dong_ru_kou.hql |
| 15 | 13 | ||
| 16 | hadoop fs -rm /input/debug/data/app.out | 14 | hadoop fs -rm /input/debug/data/app.out |
| 17 | 15 | ... | ... |
| ... | @@ -2,5 +2,5 @@ use testdb; | ... | @@ -2,5 +2,5 @@ use testdb; |
| 2 | 2 | ||
| 3 | INSERT OVERWRITE TABLE biz_data_count_debug | 3 | INSERT OVERWRITE TABLE biz_data_count_debug |
| 4 | select method,biz_data,count(1) as count,max(timestamp) as timestamp | 4 | select method,biz_data,count(1) as count,max(timestamp) as timestamp |
| 5 | from (select * from qa_log_debug where method is not null and method!='N/A' and timestamp>'${hivevar:myTimestamp}') my_debug | 5 | from (select * from qa_log_debug where method is not null and method!='N/A') my_debug |
| 6 | group by method,biz_data; | 6 | group by method,biz_data; |
| ... | \ No newline at end of file | ... | \ No newline at end of file | ... | ... |
| ... | @@ -3,4 +3,4 @@ use testdb; | ... | @@ -3,4 +3,4 @@ use testdb; |
| 3 | INSERT OVERWRITE TABLE biz_data_response_debug | 3 | INSERT OVERWRITE TABLE biz_data_response_debug |
| 4 | select method,biz_data,uid,timestamp,response | 4 | select method,biz_data,uid,timestamp,response |
| 5 | from qa_log_debug | 5 | from qa_log_debug |
| 6 | where method is not null and method!='N/A' and timestamp>'${hiveconf:myTimestamp}'; | ||
| ... | \ No newline at end of file | ... | \ No newline at end of file |
| 6 | where method is not null and method!='N/A'; | ||
| ... | \ No newline at end of file | ... | \ No newline at end of file | ... | ... |
| ... | @@ -4,5 +4,4 @@ INSERT OVERWRITE TABLE huo_dong_ru_kou_debug | ... | @@ -4,5 +4,4 @@ INSERT OVERWRITE TABLE huo_dong_ru_kou_debug |
| 4 | select method,biz_data,uid,timestamp,response | 4 | select method,biz_data,uid,timestamp,response |
| 5 | from qa_log_debug | 5 | from qa_log_debug |
| 6 | where method is not null and method!='N/A' | 6 | where method is not null and method!='N/A' |
| 7 | and (method='com.lejane.handler.common.sidebar.operate.query.new.v3' or method='com.lejane.handler.common.sidebar.operate.query.list') | ||
| 8 | and timestamp>'${hivevar:myTimestamp}'; | ||
| ... | \ No newline at end of file | ... | \ No newline at end of file |
| 7 | and (method='com.lejane.handler.common.sidebar.operate.query.new.v3' or method='com.lejane.handler.common.sidebar.operate.query.list'); | ||
| ... | \ No newline at end of file | ... | \ No newline at end of file | ... | ... |
| ... | @@ -2,5 +2,5 @@ use testdb; | ... | @@ -2,5 +2,5 @@ use testdb; |
| 2 | 2 | ||
| 3 | INSERT OVERWRITE TABLE method_count_debug | 3 | INSERT OVERWRITE TABLE method_count_debug |
| 4 | select method,count(1) as count,max(timestamp) as timestamp | 4 | select method,count(1) as count,max(timestamp) as timestamp |
| 5 | from (select * from qa_log_debug where method is not null and method!='N/A' and timestamp>'${hivevar:myTimestamp}') my_debug | 5 | from (select * from qa_log_debug where method is not null and method!='N/A') my_debug |
| 6 | group by method; | 6 | group by method; |
| ... | \ No newline at end of file | ... | \ No newline at end of file | ... | ... |
-
Please register or sign in to post a comment