By

Contents
  1. 1. 读取文件为参数
  2. 2. if-else
  3. 3. for loop
  4. 4. ftp
  5. 5. 传FTP文件脚本
  6. 6. sftp nickname tool
  7. 7. 脚本加密
  8. 8. loop file in dir
  9. 9. tar
  10. 10. 常用日期
  11. 11. 目录下最新日期文件
  12. 12. lsof
  13. 13. 端口使用分析
  14. 14. 端口测试
  15. 15. 同步文件
  16. 16. 异常处理
    1. 16.1. jps – process information unavailable
    2. 16.2. fork: retry: Resource temporarily unavailable
  17. 17. Bash路径通配符
  18. 18. 文件切割
  19. 19. 编码转换
  20. 20. bash shortcut
  21. 21. vi shortcut
  22. 22. telnet批量测试
  23. 23. hdfs目录检查
  24. 24. awk生成命令
  25. 25. screen
  26. 26. zsh
    1. 26.1. 安装提示插件：
  27. 27. find 查找创建时间早于xx的目录

读取文件为参数

1
2
ans_lines=`sed -n 1p  ${verf_file}`
ans_md5=`sed -n 2p ${verf_file}`

if-else

1
2
3
4
5
6
7
8
9
10
11
12
13
14

# value equal
if [ ${ans_lines} -eq ${lines} ]
then
    echo "lines is ok"
else
    echo "lines is not ok"
fi

# file exist
hdfs dfs -test -d ${HDFS_path}/wap
if [ $? == 0 ]; then
    hdfs dfs -rmr ${HDFS_path}/wap
fi

for loop

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
for i in {1..10}; do echo $i; done
echo "**************"
END=10
for i in $(seq 1 $END); do echo $i; done
echo "**************"
INTERVAL=2
for i in $(seq 1 $INTERVAL $END); do echo $i; done



for i in {2..5}; do 
tablesuffix=$i
echo $tablesuffix
sleep 1; 
done

ftp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
sftp_get_wap()
{
    if [ -d ${sftpLocal_wap} ]; then
       rm -rf ${sftpLocal_wap}
    fi
    mkdir -p ${sftpLocal_wap}
    expect <<- EOF
    set timeout 120
    spawn sftp $sftp_user@$sftp_IP
    expect {
        "(yes/no)?" {send "yes\r"; expect_continue }
        "*assword:" {send "$sftp_password\n"; exp_continue}
        "*Permission*" {exit -1}
        "sftp>" {send "pwd\n"}
    }
    expect "sftp>"
    send "cd $sftpRemote_wap\n"
    expect "sftp>"
    send "lcd $sftpLocal_wap\n"
    expect "sftp>"
    set timeout -1
    send "mget *$DT_time*log\n"
    expect "sftp>"
    set timeout -1
    send "mget *$DT_time*verf.txt\n"
    expect "sftp>"
    send "bye\n"
EOF
}

传FTP文件脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
sftp_IP=10.xx
port=3030
sftp_user=biuser
sftp_password=xx
sftpRemote=/hcycdnsrc/biapp/
sftpLocal=/data/tmp
putfiles=ndmc_ispace_disk_attr_brwspace.csv

    expect <<- EOF
    set timeout 5
    spawn sftp -P $port $sftp_user@$sftp_IP
    expect {
        "(yes/no*" {send "yes\r"; exp_continue }
        "*assword:" {send "$sftp_password\n"}
    }
    expect "sftp>"
    send "cd $sftpRemote\n"
    expect "sftp>"
    send "lcd $sftpLocal\n"
    expect "sftp>"
    set timeout -1
    send "mput $putfiles\n"
    expect "sftp>"
    send "ls -l \n"
    expect "sftp>"
    send "bye\n"

sftp nickname tool

echo “StrictHostKeyChecking no” >~/.ssh/config

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
#!/bin/bash

  

# transfer.sh

encrypt() {

    local str=$1

    # Use PBKDF2 with 10000 iterations for better security

    echo "$str" | openssl enc -aes-256-cbc -a -pbkdf2 -iter 100 \

        -S "1234567890ABCDEF" \

        -iv "0123456789ABCDEF0123456789ABCDEF" \

        -pass pass:"${ENCRYPTION_KEY:-default_key}"

}

  

decrypt() {

    local str=$1

    # Use PBKDF2 with 10000 iterations for better security

    echo "$str" | openssl enc -aes-256-cbc -a -d -pbkdf2 -iter 100 \

        -S "1234567890ABCDEF" \

        -iv "0123456789ABCDEF0123456789ABCDEF" \

        -pass pass:"${ENCRYPTION_KEY:-default_key}"

}

  

CONFIG_FILE="config.dat"

ENCRYPTION_KEY="your_encryption_key_here"  # Change this to your secure key

SEPARATOR="|"  # Separator for config file entries

  

# Function to parse user@host format

parse_user_host() {

    local input=$1

    local username=$(echo "$input" | cut -d'@' -f1)

    local hostname=$(echo "$input" | cut -d'@' -f2)

    echo "$username|$hostname"

}

  

# Function to add new SFTP configuration

add_config() {

    echo "Enter connection string (format: user@host):"

    read -r connection_string

  

    if [[ ! "$connection_string" =~ ^[^@]+@[^@]+$ ]]; then

        echo "Invalid format. Please use user@host format"

        exit 1

    fi

  

    # Parse username and hostname

    parsed=$(parse_user_host "$connection_string")

    username=$(echo "$parsed" | cut -d'|' -f1)

    host=$(echo "$parsed" | cut -d'|' -f2)

  

    # Set default nickname as username if not provided

    echo "Enter nickname (press Enter to use '$username' as default):"

    read -r nickname

    nickname=${nickname:-$username}

  

    # Set default port as 22 if not provided

    echo "Enter port (press Enter to use default port 22):"

    read -r port

    port=${port:-22}

  

    echo "Enter password:"

    read -rs password

    echo

  

    # Encrypt sensitive data

    encrypted_user=$(encrypt "$username")

    encrypted_pass=$(encrypt "$password")

    # Add to config file (format: nickname|host|port|encrypted_user|encrypted_pass)

    echo "$nickname$SEPARATOR$host$SEPARATOR$port$SEPARATOR$encrypted_user$SEPARATOR$encrypted_pass" >> "$CONFIG_FILE"

    echo "Configuration added successfully!"

}

  

# Alternative add_config function that accepts direct parameters

add_config_direct() {

    if [ "$#" -ne 4 ]; then

        echo "Usage: $0 add-direct <port> <host> <user> <password>"

        exit 1

    fi

    local port=$1

    local host=$2

    local username=$3

    local password=$4

    local nickname=${username}

    # If nickname already exists, append a random number

    if [ -f "$CONFIG_FILE" ]; then

        if grep -q "^$nickname$SEPARATOR" "$CONFIG_FILE"; then

            nickname="${nickname}_$(date +%s)"

            echo "Nickname already exists. Using '$nickname' as nickname."

        fi

    fi

    # Encrypt sensitive data

    encrypted_user=$(encrypt "$username")

    encrypted_pass=$(encrypt "$password")

    # Add to config file

    echo "$nickname$SEPARATOR$host$SEPARATOR$port$SEPARATOR$encrypted_user$SEPARATOR$encrypted_pass" >> "$CONFIG_FILE"

    echo "Configuration added successfully!"

}

  

# Function to get configuration by nickname

get_config() {

    local nickname=$1

    if [ ! -f "$CONFIG_FILE" ]; then

        echo "Config file not found!"

        exit 1

    fi

  

    # Get the line for this nickname

    local config_line=$(grep "^$nickname$SEPARATOR" "$CONFIG_FILE")

    if [ -z "$config_line" ]; then

        echo "Nickname not found!"

        exit 1

    fi

  

    # Parse the line using separator

    host=$(echo "$config_line" | cut -d"$SEPARATOR" -f2)

    port=$(echo "$config_line" | cut -d"$SEPARATOR" -f3)

    encrypted_user=$(echo "$config_line" | cut -d"$SEPARATOR" -f4)

    encrypted_pass=$(echo "$config_line" | cut -d"$SEPARATOR" -f5)

    # Decrypt sensitive data

    user=$(decrypt "$encrypted_user")

    password=$(decrypt "$encrypted_pass")

}

  

# Function to list all configured nicknames

list_configs() {

    if [ ! -f "$CONFIG_FILE" ]; then

        echo "No configurations found."

        return

    fi

    echo "Available SFTP configurations:"

    while IFS="$SEPARATOR" read -r nickname host port encrypted_user _; do

        if [ ! -z "$nickname" ]; then

            user=$(decrypt "$encrypted_user")

            echo "  $nickname -> $user@$host:$port"

        fi

    done < "$CONFIG_FILE"

}

  

# Function to list remote directory

sftp_ls() {

    local nickname=$1

    local remote_path=${2:-.}

    get_config "$nickname"

    sshpass -p "$password" sftp -P "$port" "$user@$host" << EOL

cd "$remote_path"

ls

bye

EOL

}

  

# Function to download files/directories

sftp_get() {

    local nickname=$1

    local remote_path=$2

    local local_path=${3:-.}

    get_config "$nickname"

    if [[ "$remote_path" == */ ]]; then

        # It's a directory

        sshpass -p "$password" sftp -P "$port" "$user@$host" << EOL

get -r "$remote_path" "$local_path"

bye

EOL

    else

        # It's a file

        sshpass -p "$password" sftp -P "$port" "$user@$host" << EOL

get "$remote_path" "$local_path"

bye

EOL

    fi

}

  

# Function to upload files/directories

sftp_put() {

    local nickname=$1

    local local_path=$2

    local remote_path=${3:-.}

    get_config "$nickname"

    if [ -d "$local_path" ]; then

        # It's a directory

        sshpass -p "$password" sftp -P "$port" "$user@$host" << EOL

put -r "$local_path" "$remote_path"

bye

EOL

    else

        # It's a file

        sshpass -p "$password" sftp -P "$port" "$user@$host" << EOL

put "$local_path" "$remote_path"

bye

EOL

    fi

}

  

# Main menu

case "$1" in

    "add")

        add_config

        ;;

    "add-direct")

        shift  # Remove first argument

        add_config_direct "$@"

        ;;

    "list")

        list_configs

        ;;

    "ls")

        if [ -z "$2" ]; then

            echo "Usage: $0 ls <nickname> [remote_path]"

            exit 1

        fi

        sftp_ls "$2" "$3"

        ;;

    "get")

        if [ -z "$3" ]; then

            echo "Usage: $0 get <nickname> <remote_path> [local_path]"

            exit 1

        fi

        sftp_get "$2" "$3" "$4"

        ;;

    "put")

        if [ -z "$3" ]; then

            echo "Usage: $0 put <nickname> <local_path> [remote_path]"

            exit 1

        fi

        sftp_put "$2" "$3" "$4"

        ;;

    *)

        echo "Usage: $0 {add|list|ls|get|put} [arguments]"

        echo "Commands:"

        echo "  add                     Add new SFTP configuration"

        echo "  add-direct              Add with direct parameters <port> <host> <user> <password>"

        echo "  list                    List all configured SFTP connections"

        echo "  ls <nickname> [path]    List remote directory contents"

        echo "  get <nickname> <remote_path> [local_path]   Download file/directory"

        echo "  put <nickname> <local_path> [remote_path]   Upload file/directory"

        exit 1

        ;;

esac

# ./sftp_tool.sh add-direct '22' '192.168.x' 'xx' 'xx!xx'

脚本加密

1
2
3
4
5
6
7
8
apt-get install -y shc
shc -v -f sftp_tool.sh
输出C语言源代码和可执行文件

自行编译方式： 
cc test.sh.x.c -o test.sh.x
strip sftp_tool.sh.x
chmod +x test.sh.x

loop file in dir

1
2
3
4
5
6
7
fileList=`ls *$DT_time*.zip`
fileArr=($fileList)
for fileName in ${fileArr[@]}
do
    unzip  $unzip_name
done

tar

打包、排除指定文件、删除源文件

1
tar -zcf wap.tar.gz wap --exclude *.zip --remove-files

常用日期

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
ymd=${date}
year=${ymd:0:4}
month=${ymd:4:2}
day=${ymd:6:2}

nowdate=`date +%Y%m01` #本月第一天
startdate=`date -d"$nowdate last month" +%Y%m%d` #上个月第一天
enddate=`date -d"$nowdate last day" +%Y%m%d` #上个月最后一天

daytime=20220315
year=${daytime:0:4}
month=${daytime:4:2}
nowdate=${year}${month}01
nextmonth=`date -d"$nowdate +1 month" +%Y%m%d`
enddate=`date -d"$nextmonth last day" +%Y%m%d` #本月最后一天

目录下最新日期文件

find /opt/folder -type f -print0 | xargs -0 stat --format '%Y :%y %n' | sort -nr | cut -d: -f2- | head

lsof

端口查进程
lsof -i:<port>

进程查端口
lsof -Pp 123674 |grep IPv |grep LISTEN

进程的线程信息
ps -mp 11139 -o THREAD

本机所有进程监听端口
lsof -nP -iTCP -sTCP:LISTEN

端口使用分析

netstat -an //查看所有
netstat -plntu //查询当前处于监听状态的端口
netstat -plnt -4 //查询当前处于监听状态的tcp4端口

netstat -ntlp -4 | head |awk ‘{print}’

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
# 这个方法得不到shell命令的输出

import commands,os
# os.system('netstat -ntlp -4')
retCode,output = commands.getstatusoutput('netstat -ntlp')

tmp = set()
lines = output.split('\n')
for line in lines:
    if str(line).startswith('tcp'):
        items = str(line).split()
        if not items[3].startswith('127.0.0.1') :
            if items[3].startswith('0.0.0.0') or items[3].startswith(':::') or items[3].split(':')[0] != items[4].split(':')[0]:
                print('%s\t\t%s\t\t%s' %(items[3], items[4], items[6]))
                tmp.add(int(items[3].split(':')[-1]))

listeningPort=list(tmp)
listeningPort.sort()
listeningPort

# listeningPort.sort()
for p in listeningPort:
   print(p)

#########
inUsePortInfo = {}
retCode,output = commands.getstatusoutput('netstat -ntp -4 |grep ESTABLISHED')
lines = output.split('\n')
for line in lines:
    if str(line).startswith('tcp'):
        items = str(line).split()
        if items[3].split(':')[0] != items[4].split(':')[0]:
            port = items[3].split(':')[1] + ' ' + items[6]
            # print(line)
            tmp = inUsePortInfo.get(port, list())
            tmp.append(items[4])
            tmp.sort()
            inUsePortInfo[port] = tmp


# for k,v in inUsePortInfo.items():
#     print('%s \t %s' %(k, v))

for i in sorted(inUsePortInfo):
    print('%s \t %s' %(i, inUsePortInfo[i]))


listeningPort.sort()
listeningPort


Expected Output：
port, foreign address list

Filter
local server 127.0.0.1
local 和 remoter 都是本机

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
# iftop/iptraf - IP连接分析

nohup sudo iftop -n -N -t -s 10 -L 1000 -o destination > /data/tmp/iftop20231211 &

nohup sudo iftop -n -N -t -s 86400 -L 5000 -o destination > /data/tmp/iftop20231211 &
cat  /data/tmp/iftop20231211 |grep '<=' |awk '{print $1}' |grep -v :

 sudo iptraf -i eth0 -L /var/log/traffic_log -B



# crontab监控重启

写脚本时注意环境变量问题，建议程序使用全路径执行

```shell
#!/bin/bash

# */5 * * * * /bin/sh /data/dolphinscheduler-1.3.6/ds-watcher.sh

restart_worker()
{
#pid=`pgrep -f org.apache.dolphinscheduler.server.worker.WorkerServer`
/usr/bin/pkill -f org.apache.dolphinscheduler.server.worker.WorkerServer
rm -f /data/dolphinscheduler-1.3.6/pid/dolphinscheduler-worker-server.pid
sleep 3s
su dolphinscheduler -c "sh /data/dolphinscheduler-1.3.6/bin/dolphinscheduler-daemon.sh start worker-server"
}

# worker
ret=`/data/java/bin/java -cp /data/dolphinscheduler-1.3.6/ds-watcher.jar  ds.ZkCheckNode master:2181,slave1:2181,slave2:2181 /dolphinscheduler/nodes/worker/default/`

if [ $ret = 'false' ] ;then
  restart_worker
fi

端口测试

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
#!/bin/bash

# Define IP addresses array
ip_list=(
    "192.168.1.10"
    "10.0.0.15"
    "172.16.0.20"
)

# Port to check
PORT=1019

# Function to check port connectivity
check_port() {
    local ip=$1
    result=$(timeout 3 telnet "$ip" $PORT 2>&1)
    if echo "$result" | grep -q "Connected to"; then
        return 0
    else
        return 1
    fi
}

echo "Checking port $PORT connectivity..."
echo "Unreachable IPs:"

# Check each IP
for ip in "${ip_list[@]}"; do
    if ! check_port "$ip"; then
        echo "$ip"
    fi
done

同步文件

集群服务器 增减同步，注意目的地址写父目录
rsync -av –delete /data/soft/spark/jars LGJF-ZYC6-SJZT-SVR04:/data/soft/spark/

异常处理

jps – process information unavailable

问题：无法显示进程名称

解决：删除目录

1
rm -rf /tmp/hsperfdata_*

fork: retry: Resource temporarily unavailable

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/etc/security/limits.d/90-nproc.conf

*          soft    nproc    65535
hadoop     soft    nproc     257529
root       soft    nproc     unlimited

修改后重启进程.  

在控制台执行(修改这个不需要重启)

   ulimit -u 655350

检查下是否生效，在控制台切到该用户下

   ulimit -u

其他信息查看命令：

ps h -Led -o user | sort | uniq -c | sort -n

ps -o nlwp,pid,lwp,args -u hadoop | sort -n

第一列NLWP为线程数

Bash路径通配符

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
?：匹配任意单个字符  
*：匹配任意个任意字符  
[]：中括号通配。包括下面几种模式：  
  [abc]：匹配中括号中任意单个字符，即匹配a或b或c均可  
  [^abc]和`[!abc]`：匹配非中括号中的任意单个字符  
  [a-z] [^a-z]：匹配abc...z，但和Locale环境的排序规则有关。  
         Locale C环境下，[a-d]表示abcd，字典排序规则的[a-d]表示aBbCcDd  
  字符类：  
    [:alpha:]、[:alnum:]、[:ascii:]、[:blank:]、[:cntrl:]、  
    [:digit:]、[:graph:]、[:lower:]、[:print:]、[:punct:]、  
    [:space:]、[:upper:]、[:word:]、[:xdigit:]

{abc,def} 枚举，支持连续字符，支持嵌套；
    - echo {j{p,pe}g,png}
    - echo {0{2..9},{1..2}{0..9},3{0..1}}

每月6日至次月5日
statis_date=2024{06{0[^1-5],1*,2*,3*},070[1-5]}

文件切割

100w每个文件，3位数字序号，前缀cloudid_part
split -l 1000000 -d -a 3 cloudid533379830 cloudid_part

编码转换

iconv -f 'gbk' -t 'utf-8' 199801.txt > 199801_utf8.txt

iconv -f 'utf-8' -t 'gbk' 199801.txt > 199801_utf8.txt

bash shortcut

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
CTRL+A              # 移动到行首，同 <Home>
CTRL+E              # 移动到行末，同 <End>
CTRL+B              # 向后移动，同 <Left>
CTRL+F              # 向前移动，同 <Right>

CTRL+D              # 删除光标前的字符，同 <Delete> ，或者没有内容时，退出会话
CTRL+G              # 退出当前编辑（比如正在 CTRL+R 搜索历史时）
CTRL+H              # 删除光标左边的字符，同 <Backspace>
CTRL+K              # 删除光标位置到行末的内容
CTRL+L              # 清屏并重新显示
CTRL+N              # 移动到命令历史的下一行，同 <Down>
CTRL+O              # 类似回车，但是会显示下一行历史
CTRL+P              # 移动到命令历史的上一行，同 <Up>
CTRL+R              # 历史命令反向搜索，使用 CTRL+G 退出搜索
CTRL+S              # 历史命令正向搜索，使用 CTRL+G 退出搜索
CTRL+T              # 交换前后两个字符
CTRL+U              # 删除字符到行首
CTRL+V              # 输入字符字面量，先按 CTRL+V 再按任意键
CTRL+W              # 删除光标左边的一个单词
CTRL+X              # 列出可能的补全
CTRL+Y              # 粘贴前面 CTRL+u/k/w 删除过的内容
CTRL+Z              # 暂停前台进程返回 bash，需要时可用 fg 将其切换回前台
CTRL+_              # 撤销（undo），有的终端将 CTRL+_ 映射为 CTRL+/ 或 CTRL+7

ALT+b               # 向后（左边）移动一个单词
ALT+f               # 向前（右边）移动一个单词

ALT+d               # 删除光标后（右边）一个单词
ALT+t               # 交换字符
ALT+BACKSPACE       # 删除光标前面一个单词，类似 CTRL+W，但不影响剪贴板

CTRL+X CTRL+X       # 连续按两次 CTRL+X，光标在当前位置和行首来回跳转 
CTRL+X CTRL+E       # 用你指定的编辑器，编辑当前命令

vi shortcut

跳转 备注
b 上一单词 w 下一单词 e 词尾
B 上一单词 W 下一单词 小写的可以标点、空格分隔，大写的必须空格分隔
( 上一句首 ) 下一句首 句号后面的第一个字符为句首
0 或 | 行首 $ 行末 ^ 行首非零字符
- 上一行首 + 下一行首
{ 段首 } 段末 o的操作算段
H 屏幕顶行 L 屏幕底行
? 向前搜索 / 向后搜索
N 查找上一处 n 查找下一处
编辑
~ 大小写转换
o 分段（后） O 分段（前）
p 粘贴（后） P 粘贴（前）
i 插入模式 I 插入到行首
d 删除 D 删除到行末 dd 删除行
x 删除 X 删除前面字符，退格
s 删除并插入 S 删除行并插入
c C 修改至行末 cc 删除行并编辑
r 替换 R 替换模式
< 反缩进 > 缩进 <<, >>
y 复制 Y 复制行
u 撤销
J 与下一行合并 
1
2
3
4
5
ZZ 保存退出 
ZQ 不保存退出



telnet批量测试

1
2
3
4
5
6
7
8
for i in {14..23}; 
do  
  a=`(sleep 1;echo 'quit') |telnet 10.27.48.${i} 22 |grep OpenS |wc -l`; 
  if [ ${a} == 0 ]; 
    then     echo "10.27.48.${i} failed"; 
    else     echo "10.27.48.${i} ok"; 
  fi; 
done

hdfs目录检查

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
#!/bin/bash


HDFS_PATH=/hive/warehouse/bdoc.db/lg_dim_db01_dim_public_section_no_df/dtime=20250121
MIN_FILE_COUNT=0

if ! hdfs dfs -test -d "$HDFS_PATH"; then
    echo "Error: Path does not exist: $HDFS_PATH"
    exit 1
fi

if hdfs dfs -ls "$HDFS_PATH" | grep -q ".hive-staging_"; then
    echo "Error: Found .hive-staging_ files/directories in path"
    exit 1
fi

FILE_COUNT=$(hdfs dfs -ls "$HDFS_PATH" | grep -v Found | grep -v "^d" | grep -v "/\." | wc -l)

if [ "$FILE_COUNT" -le "$MIN_FILE_COUNT" ]; then
    echo "Error: File count ($FILE_COUNT) is not greater than minimum required ($MIN_FILE_COUNT)"
    exit 1
fi

echo "Success: All conditions met!"
echo "- Path exists: $HDFS_PATH"
echo "- No .hive-staging_ files found"
echo "- File count: $FILE_COUNT (minimum required: $MIN_FILE_COUNT)"
exit 0

awk生成命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# case 1
cat mblock | grep '/hive/warehouse/dw.db/dwm_huadan_user_act_summary_d/year=2025/month=01/' | awk '{print "/Current20250214" $2, $2}' | xargs -P 5 -n 2 -I {} echo "hdfs dfs -mv {}"  |bash


# case 2
snapshot_prefix="/hive/warehouse/.snapshot/hive_warehouse_20250213"
target_prefix="/hive/warehouse"

cat mblock | grep 'dwd_huadan_share_person_d' |grep 'year=2025' |awk -v snapshot="$snapshot_prefix" -v target="$target_prefix" '{
    file_path = $2
    source_path = snapshot substr(file_path, length(target) + 1)
    target_path = file_path
    print "hdfs dfs -cp " source_path " " target_path
}'

screen

1
2
3
4
5
6
7
8
9
10
11
12
13
14
screen 创建会话
screen -ls
screen -S xxx 指定名称创建会话
screen -r xxx 指定名称恢复会话
screen -R xxx 指定名称恢复或创建会话

`Ctrl+a`，再按:
- d：保存会话，后台运行改虚拟终端
- k：关闭对话，等同输入：`exit`
- c：新建一个虚拟终端
- ?：显示所有绑定键盘

exit 虚拟终端内退出
screen -R [pid/Name] -X quit 主终端内结束虚拟终端

zsh

1
2
3
4
5
# 安装zsh
apt install zsh

#安装ohmyzsh
sh -c "$(curl -fsSL https://install.ohmyz.sh)"

安装提示插件：

  1. 把插件下载到本地的 ~/.oh-my-zsh/custom/plugins 目录：
1
git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions
  1. 在 .zshrc 中，把 zsh-autosuggestions 加入插件列表：
1
2
3
4
plugins=(
    # other plugins...
    zsh-autosuggestions  # 插件之间使用空格隔开
)
  1. 开启新的 Shell 或执行 source ~/.zshrc，就可以开始体验插件。

更多插件Plugins Overview · ohmyzsh/ohmyzsh Wiki

find 查找创建时间早于xx的目录

1
2
3
4
5
6
7
8
9
10
# + 表示将所有匹配的路径作为参数传递给一次命令调用，而不是每次调用一次命令
# 修改时间参数为 newermt 

find . -maxdepth 1 -type d ! -newerct "2025-01-01" |head

find . -maxdepth 1 -type d ! -newerct "2025-01-01" -exec ls -ld {} + 

find . -maxdepth 1 -type d ! -newerct "2025-01-01" -exec rm -r {} +