Skip to content

张浩 / BrBigDataTest

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 8f84ead0 8f84ead08691574b0257717da9a4c86ae8397a53 by zhanghao
Options

commit

1 parent 0630c5d3
Inline Side-by-side
Showing 1 changed file with 70 additions and 73 deletions
...@@ -17,31 +17,31 @@ import java.io.IOException; ...@@ -17,31 +17,31 @@ import java.io.IOException;
17 public class ETLAppDebug { 17 public class ETLAppDebug {
18 18
19 public static void main(String[] args) throws Exception{ 19 public static void main(String[] args) throws Exception{
20 // System.setProperty("HADOOP_USER_NAME","test"); 20 System.setProperty("HADOOP_USER_NAME","test");
21 Configuration configuration = new Configuration(); 21 Configuration configuration = new Configuration();
22 // configuration.set("fs.defaultFS","hdfs://localhost:8020"); 22 configuration.set("fs.defaultFS","hdfs://localhost:8020");
23 23
24 FileSystem fileSystem = FileSystem.get(configuration); 24 FileSystem fileSystem = FileSystem.get(configuration);
25 // Path outputPath = new Path(args[1]); 25 Path outputPath = new Path(args[1]);
26 Path outputPath = new Path("./input/etl"); 26 // Path outputPath = new Path("./input/etl");
27 if(fileSystem.exists(outputPath)) { 27 if(fileSystem.exists(outputPath)) {
28 fileSystem.delete(outputPath,true); 28 fileSystem.delete(outputPath,true);
29 } 29 }
30 30
31 Job job = Job.getInstance(configuration); 31 Job job = Job.getInstance(configuration);
32 job.setJarByClass(ETLAppDebug.class); 32 job.setJarByClass(ETLAppDebug.class);
33 job.addArchiveToClassPath(new Path("./input/data/fastjson-1.2.79.jar")); 33 // job.addArchiveToClassPath(new Path("./input/data/fastjson-1.2.79.jar"));
34 // job.addArchiveToClassPath(new Path("/jar/fastjson-1.2.79.jar")); 34 job.addArchiveToClassPath(new Path("/jar/fastjson-1.2.79.jar"));
35 35
36 job.setMapperClass(MyMapper.class); 36 job.setMapperClass(MyMapper.class);
37 37
38 job.setMapOutputKeyClass(NullWritable.class); 38 job.setMapOutputKeyClass(NullWritable.class);
39 job.setMapOutputValueClass(Text.class); 39 job.setMapOutputValueClass(Text.class);
40 40
41 // FileInputFormat.setInputPaths(job, new Path(args[0])); 41 FileInputFormat.setInputPaths(job, new Path(args[0]));
42 // FileOutputFormat.setOutputPath(job, new Path(args[1])); 42 FileOutputFormat.setOutputPath(job, new Path(args[1]));
43 FileInputFormat.setInputPaths(job, new Path("./input/data/app.out")); 43 // FileInputFormat.setInputPaths(job, new Path("./input/data/app.out"));
44 FileOutputFormat.setOutputPath(job, new Path("./input/etl")); 44 // FileOutputFormat.setOutputPath(job, new Path("./input/etl"));
45 45
46 job.waitForCompletion(true); 46 job.waitForCompletion(true);
47 } 47 }
...@@ -66,72 +66,69 @@ public class ETLAppDebug { ...@@ -66,72 +66,69 @@ public class ETLAppDebug {
66 if(StringUtil.isMessyCode(log)) { 66 if(StringUtil.isMessyCode(log)) {
67 rest = "messy"; 67 rest = "messy";
68 } else { 68 } else {
69 JSONObject obj = JSONObject.parseObject(log); 69 timestamp = "";
70 70 rest = "";
71 timestamp = obj.getString("timestamp"); 71 trace = "";
72 timestamp = StringUtil.format(timestamp); 72 if(log != null && log.contains("com.brty.gateway.filters.groovy.LoggerPostFilter")) {
73 if(StringUtil.isMessyCode(timestamp)) { 73 String[] split_log = log.split("] \\[");
74 timestamp = ""; 74 if(split_log != null && split_log.length == 11) {
75 } 75 rest = "ok";
76 rest = obj.getString("rest"); 76 if(split_log[0] != null) {
77 rest = StringUtil.format(rest); 77 String[] timechar = split_log[0].split(" INFO \\[http");
78 if(StringUtil.isMessyCode(rest)) { 78 if(timechar != null && timechar.length == 2) {
79 rest = ""; 79 timestamp = timechar[0];
80 } 80 timestamp = StringUtil.format(timestamp);
81 trace = obj.getString("trace"); 81 if(StringUtil.isMessyCode(timestamp)) {
82 trace = StringUtil.format(trace); 82 timestamp = "";
83 if(StringUtil.isMessyCode(trace)) { 83 }
84 trace = "";
85 }
86
87 String[] split_rest = rest.split("] \\[");
88 if(split_rest != null && split_rest.length == 11) {
89 rest = "ok";
90 requestUri = split_rest[2] != null ? split_rest[2].replace("requestUri:","") : StringUtil.isNull;
91 requestUri = StringUtil.format(requestUri);
92 if(StringUtil.isMessyCode(requestUri)) {
93 requestUri = "";
94 }
95 status = split_rest[5] != null ? split_rest[5].replace("status:","") : StringUtil.isNull;
96 status = StringUtil.format(status);
97 if(StringUtil.isMessyCode(status)) {
98 status = "";
99 }
100 request = split_rest[8] != null ? split_rest[8].replace("request:","") : StringUtil.isNull;
101 request = StringUtil.format(request);
102 if(StringUtil.isMessyCode(request)) {
103 request = "messy";
104 status = "messy";
105 }
106 response = split_rest[9] != null ? split_rest[9].replace("response:","") : StringUtil.isNull;
107 response = StringUtil.format(response);
108 if(StringUtil.isMessyCode(response)) {
109 response = "";
110 }
111 if(request != null && !"".equals(request)) {
112 String[] requests = request.split("&");
113 if(requests != null && requests.length == 10) {
114 method = requests[1] != null ? requests[1].replace("method=","") : StringUtil.isNull;
115 method = StringUtil.format(method);
116 if(StringUtil.isMessyCode(method)) {
117 method = "";
118 } 84 }
119 uid = requests[2] != null ? requests[2].replace("uid=","") : StringUtil.isNull; 85 }
120 uid = StringUtil.format(uid); 86 requestUri = split_log[2] != null ? split_log[2].replace("requestUri:","") : StringUtil.isNull;
121 if(StringUtil.isMessyCode(uid)) { 87 requestUri = StringUtil.format(requestUri);
122 uid = ""; 88 if(StringUtil.isMessyCode(requestUri)) {
123 } 89 requestUri = "";
124 biz_data = requests[6] != null ? requests[6].replace("biz_data=","") : StringUtil.isNull; 90 }
125 biz_data = StringUtil.format(biz_data); 91 status = split_log[5] != null ? split_log[5].replace("status:","") : StringUtil.isNull;
126 if(StringUtil.isMessyCode(biz_data)) { 92 status = StringUtil.format(status);
127 biz_data = ""; 93 if(StringUtil.isMessyCode(status)) {
94 status = "";
95 }
96 request = split_log[8] != null ? split_log[8].replace("request:","") : StringUtil.isNull;
97 request = StringUtil.format(request);
98 if(StringUtil.isMessyCode(request)) {
99 request = "messy";
100 status = "messy";
101 }
102 response = split_log[9] != null ? split_log[9].replace("response:","") : StringUtil.isNull;
103 response = StringUtil.format(response);
104 if(StringUtil.isMessyCode(response)) {
105 response = "";
106 }
107 if(request != null && !"".equals(request)) {
108 String[] requests = request.split("&");
109 if(requests != null && requests.length == 10) {
110 method = requests[1] != null ? requests[1].replace("method=","") : StringUtil.isNull;
111 method = StringUtil.format(method);
112 if(StringUtil.isMessyCode(method)) {
113 method = "";
114 }
115 uid = requests[2] != null ? requests[2].replace("uid=","") : StringUtil.isNull;
116 uid = StringUtil.format(uid);
117 if(StringUtil.isMessyCode(uid)) {
118 uid = "";
119 }
120 biz_data = requests[6] != null ? requests[6].replace("biz_data=","") : StringUtil.isNull;
121 biz_data = StringUtil.format(biz_data);
122 if(StringUtil.isMessyCode(biz_data)) {
123 biz_data = "";
124 }
128 } 125 }
129 } 126 }
130 } 127 if(response != null && !"".equals(response) && JsonUtil.isJson(response)) {
131 if(response != null && !"".equals(response) && JsonUtil.isJson(response)) { 128 code = StringUtil.format(JSONObject.parseObject(response).getString("code"));
132 code = StringUtil.format(JSONObject.parseObject(response).getString("code")); 129 if(StringUtil.isMessyCode(code)) {
133 if(StringUtil.isMessyCode(code)) { 130 code = "";
134 code = ""; 131 }
135 } 132 }
136 } 133 }
137 } 134 }
......