commit

@@ -10,11 +10,11 @@ hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLApp /input/data/
 sleep 3
 
 hive -f ./shell/load_data.hql
+hive -f ./shell/biz_data_count.hql
 
-total=$(hive -f ./shell/select.hql)
-result=$(echo $total | tr " " ","),$time
-curl http://192.168.8.113:9090/tool/hadoopTotal?total=${result}
-
-$(hive -hivevar name=$backupname -f ./shell/back_up.hql)
 hadoop fs -rm /input/data/$time
-rm ../$time
\ No newline at end of file
+rm ../$time
+
+sqoop export --connect jdbc:mysql://127.0.0.1:3306/bigdata --username root --password zidonghua --table biz_data_count --export-dir /user/hive/warehouse/testdb.db/biz_data_count --input-fields-terminated-by '\t'
+sleep 3
+sqoop export --connect jdbc:mysql://127.0.0.1:3306/bigdata --username root --password zidonghua --table biz_data_response --export-dir /user/hive/warehouse/testdb.db/biz_data_response --input-fields-terminated-by '\t'

+use testdb;
+
+INSERT INTO TABLE biz_data_count
+select method,biz_data,count(1) as count,max(timestamp) as timestamp
+from qa_log
+where method='com.ai.handler.plan.detail.index.info'
+group by method,biz_data;
+
+INSERT INTO TABLE biz_data_response
+select method,biz_data,uid,timestamp,response
+from qa_log
+where method='com.ai.handler.plan.detail.index.info';
\ No newline at end of file

 CREATE DATABASE IF NOT EXISTS testdb;
 use testdb;
-CREATE EXTERNAL TABLE qa_log(
+CREATE TABLE qa_log(
 rest string,
 requestUri string,
 request string,
@@ -12,4 +12,19 @@ biz_data string,
 code string,
 trace string,
 timestamp string
+) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
+
+CREATE TABLE biz_data_count(
+method string,
+biz_data string,
+count string,
+timestamp string
+) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
+
+CREATE TABLE biz_data_response(
+method string,
+biz_data string,
+uid string,
+timestamp string,
+response string
 ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
\ No newline at end of file

 use testdb;
-LOAD DATA INPATH 'hdfs://localhost:8020/input/etl' OVERWRITE INTO TABLE qa_log;
\ No newline at end of file
+LOAD DATA INPATH 'hdfs://localhost:8020/input/etl' OVERWRITE INTO TABLE qa_log;
+truncate table biz_data_count;
+truncate table biz_data_response;
\ No newline at end of file