Skip to content

张浩 / BrBigDataTest

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 8f84ead0 8f84ead08691574b0257717da9a4c86ae8397a53 by zhanghao
Options

commit

1 parent 0630c5d3
Inline Side-by-side
Showing 1 changed file with 70 additions and 73 deletions
......@@ -17,31 +17,31 @@ import java.io.IOException;
public class ETLAppDebug {
public static void main(String[] args) throws Exception{
// System.setProperty("HADOOP_USER_NAME","test");
System.setProperty("HADOOP_USER_NAME","test");
Configuration configuration = new Configuration();
// configuration.set("fs.defaultFS","hdfs://localhost:8020");
configuration.set("fs.defaultFS","hdfs://localhost:8020");
FileSystem fileSystem = FileSystem.get(configuration);
// Path outputPath = new Path(args[1]);
Path outputPath = new Path("./input/etl");
Path outputPath = new Path(args[1]);
// Path outputPath = new Path("./input/etl");
if(fileSystem.exists(outputPath)) {
fileSystem.delete(outputPath,true);
}
Job job = Job.getInstance(configuration);
job.setJarByClass(ETLAppDebug.class);
job.addArchiveToClassPath(new Path("./input/data/fastjson-1.2.79.jar"));
// job.addArchiveToClassPath(new Path("/jar/fastjson-1.2.79.jar"));
// job.addArchiveToClassPath(new Path("./input/data/fastjson-1.2.79.jar"));
job.addArchiveToClassPath(new Path("/jar/fastjson-1.2.79.jar"));
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(NullWritable.class);
job.setMapOutputValueClass(Text.class);
// FileInputFormat.setInputPaths(job, new Path(args[0]));
// FileOutputFormat.setOutputPath(job, new Path(args[1]));
FileInputFormat.setInputPaths(job, new Path("./input/data/app.out"));
FileOutputFormat.setOutputPath(job, new Path("./input/etl"));
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
// FileInputFormat.setInputPaths(job, new Path("./input/data/app.out"));
// FileOutputFormat.setOutputPath(job, new Path("./input/etl"));
job.waitForCompletion(true);
}
......@@ -66,72 +66,69 @@ public class ETLAppDebug {
if(StringUtil.isMessyCode(log)) {
rest = "messy";
} else {
JSONObject obj = JSONObject.parseObject(log);
timestamp = obj.getString("timestamp");
timestamp = StringUtil.format(timestamp);
if(StringUtil.isMessyCode(timestamp)) {
timestamp = "";
}
rest = obj.getString("rest");
rest = StringUtil.format(rest);
if(StringUtil.isMessyCode(rest)) {
rest = "";
}
trace = obj.getString("trace");
trace = StringUtil.format(trace);
if(StringUtil.isMessyCode(trace)) {
trace = "";
}
String[] split_rest = rest.split("] \\[");
if(split_rest != null && split_rest.length == 11) {
rest = "ok";
requestUri = split_rest[2] != null ? split_rest[2].replace("requestUri:","") : StringUtil.isNull;
requestUri = StringUtil.format(requestUri);
if(StringUtil.isMessyCode(requestUri)) {
requestUri = "";
}
status = split_rest[5] != null ? split_rest[5].replace("status:","") : StringUtil.isNull;
status = StringUtil.format(status);
if(StringUtil.isMessyCode(status)) {
status = "";
}
request = split_rest[8] != null ? split_rest[8].replace("request:","") : StringUtil.isNull;
request = StringUtil.format(request);
if(StringUtil.isMessyCode(request)) {
request = "messy";
status = "messy";
}
response = split_rest[9] != null ? split_rest[9].replace("response:","") : StringUtil.isNull;
response = StringUtil.format(response);
if(StringUtil.isMessyCode(response)) {
response = "";
}
if(request != null && !"".equals(request)) {
String[] requests = request.split("&");
if(requests != null && requests.length == 10) {
method = requests[1] != null ? requests[1].replace("method=","") : StringUtil.isNull;
method = StringUtil.format(method);
if(StringUtil.isMessyCode(method)) {
method = "";
timestamp = "";
rest = "";
trace = "";
if(log != null && log.contains("com.brty.gateway.filters.groovy.LoggerPostFilter")) {
String[] split_log = log.split("] \\[");
if(split_log != null && split_log.length == 11) {
rest = "ok";
if(split_log[0] != null) {
String[] timechar = split_log[0].split(" INFO \\[http");
if(timechar != null && timechar.length == 2) {
timestamp = timechar[0];
timestamp = StringUtil.format(timestamp);
if(StringUtil.isMessyCode(timestamp)) {
timestamp = "";
}
}
uid = requests[2] != null ? requests[2].replace("uid=","") : StringUtil.isNull;
uid = StringUtil.format(uid);
if(StringUtil.isMessyCode(uid)) {
uid = "";
}
biz_data = requests[6] != null ? requests[6].replace("biz_data=","") : StringUtil.isNull;
biz_data = StringUtil.format(biz_data);
if(StringUtil.isMessyCode(biz_data)) {
biz_data = "";
}
requestUri = split_log[2] != null ? split_log[2].replace("requestUri:","") : StringUtil.isNull;
requestUri = StringUtil.format(requestUri);
if(StringUtil.isMessyCode(requestUri)) {
requestUri = "";
}
status = split_log[5] != null ? split_log[5].replace("status:","") : StringUtil.isNull;
status = StringUtil.format(status);
if(StringUtil.isMessyCode(status)) {
status = "";
}
request = split_log[8] != null ? split_log[8].replace("request:","") : StringUtil.isNull;
request = StringUtil.format(request);
if(StringUtil.isMessyCode(request)) {
request = "messy";
status = "messy";
}
response = split_log[9] != null ? split_log[9].replace("response:","") : StringUtil.isNull;
response = StringUtil.format(response);
if(StringUtil.isMessyCode(response)) {
response = "";
}
if(request != null && !"".equals(request)) {
String[] requests = request.split("&");
if(requests != null && requests.length == 10) {
method = requests[1] != null ? requests[1].replace("method=","") : StringUtil.isNull;
method = StringUtil.format(method);
if(StringUtil.isMessyCode(method)) {
method = "";
}
uid = requests[2] != null ? requests[2].replace("uid=","") : StringUtil.isNull;
uid = StringUtil.format(uid);
if(StringUtil.isMessyCode(uid)) {
uid = "";
}
biz_data = requests[6] != null ? requests[6].replace("biz_data=","") : StringUtil.isNull;
biz_data = StringUtil.format(biz_data);
if(StringUtil.isMessyCode(biz_data)) {
biz_data = "";
}
}
}
}
if(response != null && !"".equals(response) && JsonUtil.isJson(response)) {
code = StringUtil.format(JSONObject.parseObject(response).getString("code"));
if(StringUtil.isMessyCode(code)) {
code = "";
if(response != null && !"".equals(response) && JsonUtil.isJson(response)) {
code = StringUtil.format(JSONObject.parseObject(response).getString("code"));
if(StringUtil.isMessyCode(code)) {
code = "";
}
}
}
}
......