Skip to content
Toggle navigation
Toggle navigation
This project
Loading...
Sign in
张浩
/
BrBigDataTest
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Network
Create a new issue
Builds
Commits
Issue Boards
Files
Commits
Network
Compare
Branches
Tags
Commit
d109de81
...
d109de81a65aafdfad02d036ffd3c0fad0b36e54
authored
2022-02-28 10:36:00 +0800
by
zhanghao
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
commit
1 parent
051f8e4d
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
62 additions
and
2 deletions
shell/biz_data_response.hql
shell/create_table.hql
shell/tc_response.hql
src/main/java/com/bigdata/test/ETLApp.java
shell/biz_data_response.hql
View file @
d109de8
use testdb;
INSERT OVERWRITE TABLE biz_data_response
select method,biz_data,uid,timestamp,response
select method,biz_data,uid,
platform,deviceInfo,version,imei,app_id,
timestamp,response
from qa_log
where method is not null and method!='N/A';
\ No newline at end of file
...
...
shell/create_table.hql
View file @
d109de8
...
...
@@ -11,6 +11,11 @@ status string,
method string,
uid string,
biz_data string,
platform string,
deviceInfo string,
version string,
imei string,
app_id string,
code string,
trace string,
timestamp string
...
...
@@ -33,6 +38,11 @@ CREATE TABLE biz_data_response(
method string,
biz_data string,
uid string,
platform string,
deviceInfo string,
version string,
imei string,
app_id string,
timestamp string,
response string
) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
...
...
@@ -41,6 +51,11 @@ CREATE TABLE tc_response(
method string,
biz_data string,
uid string,
platform string,
deviceInfo string,
version string,
imei string,
app_id string,
timestamp string,
response string
) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t';
\ No newline at end of file
...
...
shell/tc_response.hql
View file @
d109de8
use testdb;
INSERT OVERWRITE TABLE tc_response
select method,biz_data,uid,timestamp,response
select method,biz_data,uid,
platform,deviceInfo,version,imei,app_id,
timestamp,response
from qa_log
where method is not null and method!='N/A'
and (method='com.lejane.handler.common.sidebar.operate.query.new.v3'
...
...
src/main/java/com/bigdata/test/ETLApp.java
View file @
d109de8
...
...
@@ -59,6 +59,11 @@ public class ETLApp {
String
method
=
StringUtil
.
isNull
;
String
uid
=
StringUtil
.
isNull
;
String
biz_data
=
StringUtil
.
isNull
;
String
platform
=
StringUtil
.
isNull
;
String
deviceInfo
=
StringUtil
.
isNull
;
String
version
=
StringUtil
.
isNull
;
String
imei
=
StringUtil
.
isNull
;
String
app_id
=
StringUtil
.
isNull
;
String
code
=
StringUtil
.
isNull
;
String
log
=
value
.
toString
();
...
...
@@ -130,11 +135,46 @@ public class ETLApp {
if
(
StringUtil
.
isMessyCode
(
biz_data
))
{
biz_data
=
""
;
}
}
else
if
(
requests
[
i
].
contains
(
"platform="
))
{
platform
=
requests
[
i
].
replace
(
"platform="
,
""
);
platform
=
StringUtil
.
format
(
platform
);
if
(
StringUtil
.
isMessyCode
(
platform
))
{
platform
=
""
;
}
}
else
if
(
requests
[
i
].
contains
(
"deviceInfo="
))
{
deviceInfo
=
requests
[
i
].
replace
(
"deviceInfo="
,
""
);
deviceInfo
=
StringUtil
.
format
(
deviceInfo
);
if
(
StringUtil
.
isMessyCode
(
deviceInfo
))
{
deviceInfo
=
""
;
}
}
else
if
(
requests
[
i
].
contains
(
"version="
))
{
version
=
requests
[
i
].
replace
(
"version="
,
""
);
version
=
StringUtil
.
format
(
version
);
if
(
StringUtil
.
isMessyCode
(
version
))
{
version
=
""
;
}
}
else
if
(
requests
[
i
].
contains
(
"imei="
))
{
imei
=
requests
[
i
].
replace
(
"imei="
,
""
);
imei
=
StringUtil
.
format
(
imei
);
if
(
StringUtil
.
isMessyCode
(
imei
))
{
imei
=
""
;
}
}
else
if
(
requests
[
i
].
contains
(
"app_id="
))
{
app_id
=
requests
[
i
].
replace
(
"app_id="
,
""
);
app_id
=
StringUtil
.
format
(
app_id
);
if
(
StringUtil
.
isMessyCode
(
app_id
))
{
app_id
=
""
;
}
}
}
else
{
uid
=
StringUtil
.
isNull
;
method
=
StringUtil
.
isNull
;
biz_data
=
StringUtil
.
isNull
;
platform
=
StringUtil
.
isNull
;
deviceInfo
=
StringUtil
.
isNull
;
version
=
StringUtil
.
isNull
;
imei
=
StringUtil
.
isNull
;
app_id
=
StringUtil
.
isNull
;
}
}
}
...
...
@@ -157,6 +197,11 @@ public class ETLApp {
builder
.
append
(
method
).
append
(
"\t"
);
builder
.
append
(
uid
).
append
(
"\t"
);
builder
.
append
(
biz_data
).
append
(
"\t"
);
builder
.
append
(
platform
).
append
(
"\t"
);
builder
.
append
(
deviceInfo
).
append
(
"\t"
);
builder
.
append
(
version
).
append
(
"\t"
);
builder
.
append
(
imei
).
append
(
"\t"
);
builder
.
append
(
app_id
).
append
(
"\t"
);
builder
.
append
(
code
).
append
(
"\t"
);
builder
.
append
(
trace
).
append
(
"\t"
);
builder
.
append
(
timestamp
);
...
...
Please
register
or
sign in
to post a comment