commit

@@ -4,10 +4,12 @@ hadoop fs -put /home/test/logs/app.out /input/debug/data/app.out
 hadoop jar ./target/BrBigDataTest-1.0.0.jar com.bigdata.test.ETLAppDebug /input/debug/data/app.out /input/debug/etl/
 sleep 3
 
+maxTimestamp=$(hive -f ./shell/max_timestamp.hql)
+
 hive -f ./shell_debug/load_data.hql
-hive -f ./shell_debug/method_count.hql
-hive -f ./shell_debug/biz_data_count.hql
-hive -f ./shell_debug/biz_data_response.hql
+$(hive -hivevar maxTimestamp=$maxTimestamp -f ./shell_debug/method_count.hql)
+$(hive -hivevar maxTimestamp=$maxTimestamp -f ./shell_debug/biz_data_count.hql)
+$(hive -hivevar maxTimestamp=$maxTimestamp -f ./shell_debug/biz_data_response.hql)
 
 hadoop fs -rm /input/debug/data/app.out
 

@@ -3,5 +3,5 @@ use testdb;
 INSERT OVERWRITE TABLE biz_data_count_debug
 select method,biz_data,count(1) as count,max(timestamp) as timestamp
 from qa_log_debug
-where method is not null and method!='N/A'
+where method is not null and method!='N/A' and timestamp>${hivevar:maxTimestamp}
 group by method,biz_data;
\ No newline at end of file

@@ -3,4 +3,4 @@ use testdb;
 INSERT OVERWRITE TABLE biz_data_response_debug
 select method,biz_data,uid,timestamp,response
 from qa_log_debug
-where method is not null and method!='N/A';
\ No newline at end of file
+where method is not null and method!='N/A' and timestamp>${hivevar:maxTimestamp};
\ No newline at end of file

+use testdb;
+
+select max(timestamp) as timestamp
+from qa_log_debug
+where method is not null and method!='N/A'
\ No newline at end of file

@@ -3,5 +3,5 @@ use testdb;
 INSERT OVERWRITE TABLE method_count_debug
 select method,count(1) as count,max(timestamp) as timestamp
 from qa_log_debug
-where method is not null and method!='N/A'
+where method is not null and method!='N/A' and timestamp>${hivevar:maxTimestamp}
 group by method;
\ No newline at end of file