commit

+#!/bin/bash
+hadoop fs -rm /input/debug/data/app.out
+hadoop fs -put /home/test/logs/app.out /input/debug/data/app.out
+hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLApp /input/debug/data/app.out /input/debug/etl/
+sleep 3
+
+hive -f ./shell_debug/load_data.hql
+hive -f ./shell_debug/method_count.hql
+hive -f ./shell_debug/biz_data_count.hql
+hive -f ./shell_debug/biz_data_response.hql
+
+hadoop fs -rm /input/debug/data/app.out
+
+sqoop export --connect "jdbc:mysql://127.0.0.1:3306/bigdata" --username root --password zidonghua --table method_count_debug --export-dir /user/hive/warehouse/testdb.db/method_count_debug --input-fields-terminated-by '\t'
+sleep 3
+sqoop export --connect "jdbc:mysql://127.0.0.1:3306/bigdata" --username root --password zidonghua --table biz_data_count_debug --export-dir /user/hive/warehouse/testdb.db/biz_data_count_debug --input-fields-terminated-by '\t'
+sleep 3
+sqoop export --connect "jdbc:mysql://127.0.0.1:3306/bigdata" --username root --password zidonghua --table biz_data_response_debug --export-dir /user/hive/warehouse/testdb.db/biz_data_response_debug --input-fields-terminated-by '\t'

+use testdb;
+
+INSERT OVERWRITE TABLE biz_data_count_debug
+select method,biz_data,count(1) as count,max(timestamp) as timestamp
+from qa_log_debug
+where method is not null and method!='N/A'
+group by method,biz_data;
\ No newline at end of file

+use testdb;
+
+INSERT OVERWRITE TABLE biz_data_response_debug
+select method,biz_data,uid,timestamp,response
+from qa_log_debug
+where method is not null and method!='N/A';
\ No newline at end of file

+CREATE DATABASE IF NOT EXISTS testdb;
+
+use testdb;
+
+CREATE TABLE qa_log_debug(
+rest string,
+requestUri string,
+request string,
+response string,
+status string,
+method string,
+uid string,
+biz_data string,
+code string,
+trace string,
+timestamp string
+) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
+
+CREATE TABLE method_count_debug(
+method string,
+count string,
+timestamp string
+) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
+
+CREATE TABLE biz_data_count_debug(
+method string,
+biz_data string,
+count string,
+timestamp string
+) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
+
+CREATE TABLE biz_data_response_debug(
+method string,
+biz_data string,
+uid string,
+timestamp string,
+response string
+) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
\ No newline at end of file

+use testdb;
+
+LOAD DATA INPATH 'hdfs://localhost:8020/input/debug/etl' OVERWRITE INTO TABLE qa_log_debug;
\ No newline at end of file

+use testdb;
+
+INSERT OVERWRITE TABLE method_count_debug
+select method,count(1) as count,max(timestamp) as timestamp
+from qa_log_debug
+where method is not null and method!='N/A'
+group by method;
\ No newline at end of file

+use testdb;
+
+select method from qa_log_debug
+where method is not null and method!='N/A'
+group by method;
\ No newline at end of file