Skip to content

张浩 / BrBigDataTest

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 8f84ead0 8f84ead08691574b0257717da9a4c86ae8397a53 by zhanghao
Options

commit

1 parent 0630c5d3
Inline Side-by-side
Showing 1 changed file with 26 additions and 29 deletions
...@@ -17,31 +17,31 @@ import java.io.IOException; ...@@ -17,31 +17,31 @@ import java.io.IOException;
17 public class ETLAppDebug { 17 public class ETLAppDebug {
18 18
19 public static void main(String[] args) throws Exception{ 19 public static void main(String[] args) throws Exception{
20 // System.setProperty("HADOOP_USER_NAME","test"); 20 System.setProperty("HADOOP_USER_NAME","test");
21 Configuration configuration = new Configuration(); 21 Configuration configuration = new Configuration();
22 // configuration.set("fs.defaultFS","hdfs://localhost:8020"); 22 configuration.set("fs.defaultFS","hdfs://localhost:8020");
23 23
24 FileSystem fileSystem = FileSystem.get(configuration); 24 FileSystem fileSystem = FileSystem.get(configuration);
25 // Path outputPath = new Path(args[1]); 25 Path outputPath = new Path(args[1]);
26 Path outputPath = new Path("./input/etl"); 26 // Path outputPath = new Path("./input/etl");
27 if(fileSystem.exists(outputPath)) { 27 if(fileSystem.exists(outputPath)) {
28 fileSystem.delete(outputPath,true); 28 fileSystem.delete(outputPath,true);
29 } 29 }
30 30
31 Job job = Job.getInstance(configuration); 31 Job job = Job.getInstance(configuration);
32 job.setJarByClass(ETLAppDebug.class); 32 job.setJarByClass(ETLAppDebug.class);
33 job.addArchiveToClassPath(new Path("./input/data/fastjson-1.2.79.jar")); 33 // job.addArchiveToClassPath(new Path("./input/data/fastjson-1.2.79.jar"));
34 // job.addArchiveToClassPath(new Path("/jar/fastjson-1.2.79.jar")); 34 job.addArchiveToClassPath(new Path("/jar/fastjson-1.2.79.jar"));
35 35
36 job.setMapperClass(MyMapper.class); 36 job.setMapperClass(MyMapper.class);
37 37
38 job.setMapOutputKeyClass(NullWritable.class); 38 job.setMapOutputKeyClass(NullWritable.class);
39 job.setMapOutputValueClass(Text.class); 39 job.setMapOutputValueClass(Text.class);
40 40
41 // FileInputFormat.setInputPaths(job, new Path(args[0])); 41 FileInputFormat.setInputPaths(job, new Path(args[0]));
42 // FileOutputFormat.setOutputPath(job, new Path(args[1])); 42 FileOutputFormat.setOutputPath(job, new Path(args[1]));
43 FileInputFormat.setInputPaths(job, new Path("./input/data/app.out")); 43 // FileInputFormat.setInputPaths(job, new Path("./input/data/app.out"));
44 FileOutputFormat.setOutputPath(job, new Path("./input/etl")); 44 // FileOutputFormat.setOutputPath(job, new Path("./input/etl"));
45 45
46 job.waitForCompletion(true); 46 job.waitForCompletion(true);
47 } 47 }
...@@ -66,44 +66,40 @@ public class ETLAppDebug { ...@@ -66,44 +66,40 @@ public class ETLAppDebug {
66 if(StringUtil.isMessyCode(log)) { 66 if(StringUtil.isMessyCode(log)) {
67 rest = "messy"; 67 rest = "messy";
68 } else { 68 } else {
69 JSONObject obj = JSONObject.parseObject(log); 69 timestamp = "";
70 70 rest = "";
71 timestamp = obj.getString("timestamp"); 71 trace = "";
72 if(log != null && log.contains("com.brty.gateway.filters.groovy.LoggerPostFilter")) {
73 String[] split_log = log.split("] \\[");
74 if(split_log != null && split_log.length == 11) {
75 rest = "ok";
76 if(split_log[0] != null) {
77 String[] timechar = split_log[0].split(" INFO \\[http");
78 if(timechar != null && timechar.length == 2) {
79 timestamp = timechar[0];
72 timestamp = StringUtil.format(timestamp); 80 timestamp = StringUtil.format(timestamp);
73 if(StringUtil.isMessyCode(timestamp)) { 81 if(StringUtil.isMessyCode(timestamp)) {
74 timestamp = ""; 82 timestamp = "";
75 } 83 }
76 rest = obj.getString("rest");
77 rest = StringUtil.format(rest);
78 if(StringUtil.isMessyCode(rest)) {
79 rest = "";
80 } 84 }
81 trace = obj.getString("trace");
82 trace = StringUtil.format(trace);
83 if(StringUtil.isMessyCode(trace)) {
84 trace = "";
85 } 85 }
86 86 requestUri = split_log[2] != null ? split_log[2].replace("requestUri:","") : StringUtil.isNull;
87 String[] split_rest = rest.split("] \\[");
88 if(split_rest != null && split_rest.length == 11) {
89 rest = "ok";
90 requestUri = split_rest[2] != null ? split_rest[2].replace("requestUri:","") : StringUtil.isNull;
91 requestUri = StringUtil.format(requestUri); 87 requestUri = StringUtil.format(requestUri);
92 if(StringUtil.isMessyCode(requestUri)) { 88 if(StringUtil.isMessyCode(requestUri)) {
93 requestUri = ""; 89 requestUri = "";
94 } 90 }
95 status = split_rest[5] != null ? split_rest[5].replace("status:","") : StringUtil.isNull; 91 status = split_log[5] != null ? split_log[5].replace("status:","") : StringUtil.isNull;
96 status = StringUtil.format(status); 92 status = StringUtil.format(status);
97 if(StringUtil.isMessyCode(status)) { 93 if(StringUtil.isMessyCode(status)) {
98 status = ""; 94 status = "";
99 } 95 }
100 request = split_rest[8] != null ? split_rest[8].replace("request:","") : StringUtil.isNull; 96 request = split_log[8] != null ? split_log[8].replace("request:","") : StringUtil.isNull;
101 request = StringUtil.format(request); 97 request = StringUtil.format(request);
102 if(StringUtil.isMessyCode(request)) { 98 if(StringUtil.isMessyCode(request)) {
103 request = "messy"; 99 request = "messy";
104 status = "messy"; 100 status = "messy";
105 } 101 }
106 response = split_rest[9] != null ? split_rest[9].replace("response:","") : StringUtil.isNull; 102 response = split_log[9] != null ? split_log[9].replace("response:","") : StringUtil.isNull;
107 response = StringUtil.format(response); 103 response = StringUtil.format(response);
108 if(StringUtil.isMessyCode(response)) { 104 if(StringUtil.isMessyCode(response)) {
109 response = ""; 105 response = "";
...@@ -136,6 +132,7 @@ public class ETLAppDebug { ...@@ -136,6 +132,7 @@ public class ETLAppDebug {
136 } 132 }
137 } 133 }
138 } 134 }
135 }
139 136
140 StringBuilder builder = new StringBuilder(); 137 StringBuilder builder = new StringBuilder();
141 builder.append(rest).append("\t"); 138 builder.append(rest).append("\t");
......