Skip to content

张浩 / BrBigDataTest

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 0630c5d3 0630c5d3ccd02c4fc13d613c9cce763149eca196 by zhanghao
Options

commit

1 parent af8f0e61
Inline Side-by-side
Showing 1 changed file with 158 additions and 0 deletions
1 package com.bigdata.test;
2
3 import com.alibaba.fastjson.JSONObject;
4 import org.apache.hadoop.conf.Configuration;
5 import org.apache.hadoop.fs.FileSystem;
6 import org.apache.hadoop.fs.Path;
7 import org.apache.hadoop.io.LongWritable;
8 import org.apache.hadoop.io.NullWritable;
9 import org.apache.hadoop.io.Text;
10 import org.apache.hadoop.mapreduce.Job;
11 import org.apache.hadoop.mapreduce.Mapper;
12 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
13 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
14
15 import java.io.IOException;
16
17 public class ETLAppDebug {
18
19 public static void main(String[] args) throws Exception{
20 // System.setProperty("HADOOP_USER_NAME","test");
21 Configuration configuration = new Configuration();
22 // configuration.set("fs.defaultFS","hdfs://localhost:8020");
23
24 FileSystem fileSystem = FileSystem.get(configuration);
25 // Path outputPath = new Path(args[1]);
26 Path outputPath = new Path("./input/etl");
27 if(fileSystem.exists(outputPath)) {
28 fileSystem.delete(outputPath,true);
29 }
30
31 Job job = Job.getInstance(configuration);
32 job.setJarByClass(ETLAppDebug.class);
33 job.addArchiveToClassPath(new Path("./input/data/fastjson-1.2.79.jar"));
34 // job.addArchiveToClassPath(new Path("/jar/fastjson-1.2.79.jar"));
35
36 job.setMapperClass(MyMapper.class);
37
38 job.setMapOutputKeyClass(NullWritable.class);
39 job.setMapOutputValueClass(Text.class);
40
41 // FileInputFormat.setInputPaths(job, new Path(args[0]));
42 // FileOutputFormat.setOutputPath(job, new Path(args[1]));
43 FileInputFormat.setInputPaths(job, new Path("./input/data/app.out"));
44 FileOutputFormat.setOutputPath(job, new Path("./input/etl"));
45
46 job.waitForCompletion(true);
47 }
48
49 static class MyMapper extends Mapper<LongWritable, Text, NullWritable, Text> {
50
51 @Override
52 protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
53 String timestamp = StringUtil.isNull;
54 String rest = StringUtil.isNull;
55 String trace = StringUtil.isNull;
56 String requestUri = StringUtil.isNull;
57 String status = StringUtil.isNull;
58 String request = StringUtil.isNull;
59 String response = StringUtil.isNull;
60 String method = StringUtil.isNull;
61 String uid = StringUtil.isNull;
62 String biz_data = StringUtil.isNull;
63 String code = StringUtil.isNull;
64
65 String log = value.toString();
66 if(StringUtil.isMessyCode(log)) {
67 rest = "messy";
68 } else {
69 JSONObject obj = JSONObject.parseObject(log);
70
71 timestamp = obj.getString("timestamp");
72 timestamp = StringUtil.format(timestamp);
73 if(StringUtil.isMessyCode(timestamp)) {
74 timestamp = "";
75 }
76 rest = obj.getString("rest");
77 rest = StringUtil.format(rest);
78 if(StringUtil.isMessyCode(rest)) {
79 rest = "";
80 }
81 trace = obj.getString("trace");
82 trace = StringUtil.format(trace);
83 if(StringUtil.isMessyCode(trace)) {
84 trace = "";
85 }
86
87 String[] split_rest = rest.split("] \\[");
88 if(split_rest != null && split_rest.length == 11) {
89 rest = "ok";
90 requestUri = split_rest[2] != null ? split_rest[2].replace("requestUri:","") : StringUtil.isNull;
91 requestUri = StringUtil.format(requestUri);
92 if(StringUtil.isMessyCode(requestUri)) {
93 requestUri = "";
94 }
95 status = split_rest[5] != null ? split_rest[5].replace("status:","") : StringUtil.isNull;
96 status = StringUtil.format(status);
97 if(StringUtil.isMessyCode(status)) {
98 status = "";
99 }
100 request = split_rest[8] != null ? split_rest[8].replace("request:","") : StringUtil.isNull;
101 request = StringUtil.format(request);
102 if(StringUtil.isMessyCode(request)) {
103 request = "messy";
104 status = "messy";
105 }
106 response = split_rest[9] != null ? split_rest[9].replace("response:","") : StringUtil.isNull;
107 response = StringUtil.format(response);
108 if(StringUtil.isMessyCode(response)) {
109 response = "";
110 }
111 if(request != null && !"".equals(request)) {
112 String[] requests = request.split("&");
113 if(requests != null && requests.length == 10) {
114 method = requests[1] != null ? requests[1].replace("method=","") : StringUtil.isNull;
115 method = StringUtil.format(method);
116 if(StringUtil.isMessyCode(method)) {
117 method = "";
118 }
119 uid = requests[2] != null ? requests[2].replace("uid=","") : StringUtil.isNull;
120 uid = StringUtil.format(uid);
121 if(StringUtil.isMessyCode(uid)) {
122 uid = "";
123 }
124 biz_data = requests[6] != null ? requests[6].replace("biz_data=","") : StringUtil.isNull;
125 biz_data = StringUtil.format(biz_data);
126 if(StringUtil.isMessyCode(biz_data)) {
127 biz_data = "";
128 }
129 }
130 }
131 if(response != null && !"".equals(response) && JsonUtil.isJson(response)) {
132 code = StringUtil.format(JSONObject.parseObject(response).getString("code"));
133 if(StringUtil.isMessyCode(code)) {
134 code = "";
135 }
136 }
137 }
138 }
139
140 StringBuilder builder = new StringBuilder();
141 builder.append(rest).append("\t");
142 builder.append(requestUri).append("\t");
143 builder.append(request).append("\t");
144 builder.append(response).append("\t");
145 builder.append(status).append("\t");
146 builder.append(method).append("\t");
147 builder.append(uid).append("\t");
148 builder.append(biz_data).append("\t");
149 builder.append(code).append("\t");
150 builder.append(trace).append("\t");
151 builder.append(timestamp);
152
153 context.write(NullWritable.get(), new Text(builder.toString()));
154 }
155 }
156 }
157
158