Skip to content

张浩 / BrBigDataTest

Go to a project
Toggle navigation
Toggle navigation pinning
Commit c23aa6f9 c23aa6f9d4e8949d6532dc6c7c21293c6dd2aedf by zhanghao
Options

commit

1 parent a8b5846a
Inline Side-by-side
Showing 4 changed files with 21 additions and 8 deletions
#!/bin/bash
time=log-$(date -d "1 hour ago" +%Y-%m-%d.%H).log
scp -P 61702 -r test@172.17.19.238:/home/data/logs/logstash/litetuo/asst-gateway/$time ../
hadoop fs -rm /input/data/$time
hadoop fs -put ../$time /input/data/$time
hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLApp /input/data/$time /input/etl/
sleep 3
hive -f ./shell/load_data.hql
hive -f ./shell/biz_data_count.hql
hadoop fs -rm /input/data/$time
rm ../$time
sqoop export --connect jdbc:mysql://127.0.0.1:3306/bigdata --username root --password zidonghua --table biz_data_count --export-dir /user/hive/warehouse/testdb.db/biz_data_count --input-fields-terminated-by '\t'
sleep 3
sqoop export --connect jdbc:mysql://127.0.0.1:3306/bigdata --username root --password zidonghua --table biz_data_response --export-dir /user/hive/warehouse/testdb.db/biz_data_response --input-fields-terminated-by '\t'
use testdb;
drop table ${hivevar:name};
create table ${hivevar:name} as select * from qa_log;
\ No newline at end of file
use testdb;
LOAD DATA INPATH 'hdfs://localhost:8020/input/etl' OVERWRITE INTO TABLE qa_log;
truncate table qa_log;
truncate table biz_data_count;
truncate table biz_data_response;
LOAD DATA INPATH 'hdfs://localhost:8020/input/etl' OVERWRITE INTO TABLE qa_log;
\ No newline at end of file
......
use testdb;
select count(1) from qa_log where requestUri like concat('%','gateway.do','%');
select count(1) from qa_log where requestUri like concat('%','gateway.do','%') and (status like '200' and code like '0');
select count(1) from qa_log where requestUri like concat('%','gateway.do','%') and (status not like '200' or code not like '0');
\ No newline at end of file