Skip to content
Toggle navigation
Toggle navigation
This project
Loading...
Sign in
张浩
/
BrBigDataTest
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Network
Create a new issue
Builds
Commits
Issue Boards
Files
Commits
Network
Compare
Branches
Tags
Commit
668fa882
...
668fa8822b9737d285314a6ff94497ddb8f56985
authored
2022-02-16 10:53:35 +0800
by
zhanghao
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
commit
1 parent
5f392017
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
9 additions
and
12 deletions
run_linux_debug.sh
shell_debug/biz_data_count.hql
shell_debug/biz_data_response.hql
shell_debug/huo_dong_ru_kou.hql
shell_debug/method_count.hql
run_linux_debug.sh
View file @
668fa88
...
...
@@ -4,14 +4,12 @@ hadoop fs -put /home/test/logs/app.out /input/debug/data/app.out
hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLAppDebug /input/debug/data/app.out /input/debug/etl/
sleep 3
#maxTimestamp='2022-02-10 17:13:21.308'
maxTimestamp
=
$(
hive -f ./shell_debug/max_timestamp.hql
)
hive -f ./shell_debug/max_timestamp.hql
hive -f ./shell_debug/load_data.hql
$(
hive -hivevar
myTimestamp
=
$maxTimestamp
-f ./shell_debug/method_count.hql
)
$(
hive -hivevar
myTimestamp
=
$maxTimestamp
-f ./shell_debug/biz_data_count.hql
)
$(
hive -hivevar
myTimestamp
=
$maxTimestamp
-f ./shell_debug/biz_data_response.hql
)
$(
hive -hivevar
myTimestamp
=
$maxTimestamp
-f ./shell_debug/huo_dong_ru_kou.hql
)
hive -f ./shell_debug/method_count.hql
hive -f ./shell_debug/biz_data_count.hql
hive -f ./shell_debug/biz_data_response.hql
hive -f ./shell_debug/huo_dong_ru_kou.hql
hadoop fs -rm /input/debug/data/app.out
...
...
shell_debug/biz_data_count.hql
View file @
668fa88
...
...
@@ -2,5 +2,5 @@ use testdb;
INSERT OVERWRITE TABLE biz_data_count_debug
select method,biz_data,count(1) as count,max(timestamp) as timestamp
from (select * from qa_log_debug where method is not null and method!='N/A'
and timestamp>'${hivevar:myTimestamp}'
) my_debug
from (select * from qa_log_debug where method is not null and method!='N/A') my_debug
group by method,biz_data;
\ No newline at end of file
...
...
shell_debug/biz_data_response.hql
View file @
668fa88
...
...
@@ -3,4 +3,4 @@ use testdb;
INSERT OVERWRITE TABLE biz_data_response_debug
select method,biz_data,uid,timestamp,response
from qa_log_debug
where method is not null and method!='N/A' and timestamp>'${hiveconf:myTimestamp}';
\ No newline at end of file
where method is not null and method!='N/A';
\ No newline at end of file
...
...
shell_debug/huo_dong_ru_kou.hql
View file @
668fa88
...
...
@@ -4,5 +4,4 @@ INSERT OVERWRITE TABLE huo_dong_ru_kou_debug
select method,biz_data,uid,timestamp,response
from qa_log_debug
where method is not null and method!='N/A'
and (method='com.lejane.handler.common.sidebar.operate.query.new.v3' or method='com.lejane.handler.common.sidebar.operate.query.list')
and timestamp>'${hivevar:myTimestamp}';
\ No newline at end of file
and (method='com.lejane.handler.common.sidebar.operate.query.new.v3' or method='com.lejane.handler.common.sidebar.operate.query.list');
\ No newline at end of file
...
...
shell_debug/method_count.hql
View file @
668fa88
...
...
@@ -2,5 +2,5 @@ use testdb;
INSERT OVERWRITE TABLE method_count_debug
select method,count(1) as count,max(timestamp) as timestamp
from (select * from qa_log_debug where method is not null and method!='N/A'
and timestamp>'${hivevar:myTimestamp}'
) my_debug
from (select * from qa_log_debug where method is not null and method!='N/A') my_debug
group by method;
\ No newline at end of file
...
...
Please
register
or
sign in
to post a comment