1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
======
最后三次活跃日期，话单 --- 如何存储 【标签由存储数据可以计算得到】
新增用户时间
新增用户：只要中间有6个月停止活跃
scan 'UserActiveDate' , {LIMIT=>3}
限制保留3个版本
alter 'user', NAME=>'lastActive', VERSIONS => 3
存活时间限制7天 【6个月，ttl是以插入时间 】
alter 'user' ,{NAME=> 'lastNewActive', TTL=>604800}
预分区
SPLITS => ['1000','2000','3000','4000']
{NUMREGIONS => 15, SPLITALGO => 'HexStringSplit'}
create 'UserActiveDate', 'lastActive', 'lastNewActive'
create 'UserActiveDate', {NAME=>'lastActive', VERSIONS=>3,COMPRESSION => 'SNAPPY' }
put 'UserActiveDate', '13800138000', 'lastActive:pc', '20210101'
put 'UserActiveDate', '13800138000', 'lastActive:pc', '20210201'
put 'UserActiveDate', '13800138000', 'lastActive:pc', '20210301'
put 'UserActiveDate', '13800138000','lastActive:app','20210107'
put 'UserActiveDate', '13800138000','lastActive:app','20210117'
put 'UserActiveDate', '13800138000','lastActive:app','20210127'
put 'UserActiveDate', '13800138000','lastActive:app','20210137'
scan 'UserActiveDate',{COLUMN=>'lastActive:pc' , VERSIONS=>3}
scan 'UserActiveDate', {LIMIT => 20, VERSIONS=>3}
scan 'UserActiveDate',{COLUMNS=>['lastActive:pc','lastActive:app'], VERSIONS=>3}
select count(*) from (select distinct servnumber, media_id from vgop.vgop_huadan_hcy_operation_behavior_71201 where year = '2021' and month = '10' and day = '20' ) a
(1)删除单元格数据：delete 'tableName' , 'rowName' , 'colFamily:column'
(2)删除一行所有单元格：deleteall 'tableName' ,'rowName'
put 'UserActiveDate', '198xxx','lastActive:5','20210808'
get 'UserActiveDate','13800138000'
hbase> get 't1', 'r1'
hbase> get 't1', 'r1', {TIMERANGE => [ts1, ts2]}
hbase> get 'UserActiveDate','32812020891', {COLUMN => 'lastActive'}
hbase> get 'UserActiveDate','19802021823', {COLUMN => 'lastActive'}
hbase> get 'UserActiveDate','19802021823', {COLUMN => 'lastActive', VERSIONS => 3}
hbase> get 't1', 'r1', {COLUMN => ['c1', 'c2', 'c3']}
hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1}
hbase> get 't1', 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4}
hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4}
hbase> get 't1', 'r1', 'c1'
hbase> get 't1', 'r1', 'c1', 'c2'
hbase> get 't1', 'r1', ['c1', 'c2']
truncate 'UserActiveDate' (丢失分region信息)
disable 'UserActiveDate'
enable 'UserActiveDate'
====================
count of record
10670843
10169654 9603651
11693287 10651234
insert overwrite directory '/tmp/active' row format delimited fields terminated by ',' select distinct servnumber, media_id from vgop.vgop_huadan_hcy_operation_behavior_71201 where year = '2021' and month = '10' and day = '20'
insert overwrite directory '/tmp/blog' row format delimited fields terminated by ',' select * from dw.dwd_buryingpoint_hcy_active_log_day where concat(year,month,day) = '20210721' and phoneNumber='19802021823' and platform='android';
==============
测试数据
[hadoop@LGJF-ZYC6-HCY-SVR556 tmp]$ hdfs dfs -get /hive/warehouse/dw.db/tb_mk_caiyun_active_user_day/year=2021/month=11/day=25/part-00000-0656cff2-56bf-4a65-8c29-90be83aaecd7.c000.snappy.orc
/etc/host.conf: line 1: bad command `nospoof on'
[hadoop@LGJF-ZYC6-HCY-SVR556 tmp]$ hdfs dfs -get /hive/warehouse/dw.db/tb_mk_caiyun_active_user_day/year=2021/month=11/day=24/part-00000-f5982b26-8ce7-453a-9470-4da2e86f26bb.c000.snappy.orc
put 'UserActiveDate', 'u1','lastActive:5','20210101',1609430400000
put 'UserActiveDate', 'u1','lastActive:5','20210301',1614528000000
put 'UserActiveDate', 'u1','lastActive:3','20211001',1633017600000
put 'UserActiveDate', 'u2','lastActive:3','20211001',1633017600000
put 'UserActiveDate', 'u2','lastActive:5','20211001',1633017600000
9-11月
scan 'UserActiveDate', {COLUMNS => 'lastActive', TIMERANGE => [1630425600000, 1638288000000]}
scan 'UserActiveDate', {COLUMNS => 'lastActive', TIMERANGE => [1609430400000, 1612108800000]}
scan 'UserActiveDate', {COLUMNS => 'lastActive', TIMERANGE => [1609430400000, 1614528000001]}
scan 'UserActiveDate', {COLUMNS => 'lastActive',VERSIONS => 1, TIMERANGE => [1609430400000, 1614528000001]}
scan 'UserActiveDate', {COLUMNS => 'lastActive', TIMERANGE => [1609430400000, 1612108800000], RAW => true,VERSIONS => 1}
get 'UserActiveDate',
==== rename table
重命名主要通过hbase的快照功能。
1.停止表继续插入
hbase shell> disable 'UserActiveDate'
2。制作快照
hbase shell> snapshot 'UserActiveDate', 'UserActiveDate_old'
3.克隆快照为新的名字
hbase shell> clone_snapshot 'tableSnapshot', 'newTableName'
4.删除快照
hbase shell> delete_snapshot 'tableSnapshot'
5.删除原来表
hbase shell> drop 'tableName'
|
