By

Contents
  1. 1. 读取文件为参数
  2. 2. if-else
  3. 3. for loop
  4. 4. ftp
  5. 5. 传FTP文件脚本
  6. 6. loop file in dir
  7. 7. tar
  8. 8. 常用日期
  9. 9. 目录下最新日期文件
  10. 10. lsof
  11. 11. 端口使用分析
  12. 12. 同步文件
  13. 13. 异常处理
    1. 13.1. jps – process information unavailable
    2. 13.2. fork: retry: Resource temporarily unavailable
  14. 14. Bash路径通配规则

读取文件为参数

1
2
ans_lines=`sed -n 1p  ${verf_file}`
ans_md5=`sed -n 2p ${verf_file}`

if-else

1
2
3
4
5
6
7
8
9
10
11
12
13
14

# value equal
if [ ${ans_lines} -eq ${lines} ]
then
    echo "lines is ok"
else
    echo "lines is not ok"
fi

# file exist
hdfs dfs -test -d ${HDFS_path}/wap
if [ $? == 0 ]; then
    hdfs dfs -rmr ${HDFS_path}/wap
fi

for loop

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
for i in {1..10}; do echo $i; done
echo "**************"
END=10
for i in $(seq 1 $END); do echo $i; done
echo "**************"
INTERVAL=2
for i in $(seq 1 $INTERVAL $END); do echo $i; done



for i in {2..5}; do 
tablesuffix=$i
echo $tablesuffix
sleep 1; 
done

ftp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
sftp_get_wap()
{
    if [ -d ${sftpLocal_wap} ]; then
       rm -rf ${sftpLocal_wap}
    fi
    mkdir -p ${sftpLocal_wap}
    expect <<- EOF
    set timeout 120
    spawn sftp $sftp_user@$sftp_IP
    expect {
        "(yes/no)?" {send "yes\r"; expect_continue }
        "*assword:" {send "$sftp_password\n"; exp_continue}
        "*Permission*" {exit -1}
        "sftp>" {send "pwd\n"}
    }
    expect "sftp>"
    send "cd $sftpRemote_wap\n"
    expect "sftp>"
    send "lcd $sftpLocal_wap\n"
    expect "sftp>"
    set timeout -1
    send "mget *$DT_time*log\n"
    expect "sftp>"
    set timeout -1
    send "mget *$DT_time*verf.txt\n"
    expect "sftp>"
    send "bye\n"
EOF
}

传FTP文件脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
sftp_IP=10.xx
port=3030
sftp_user=biuser
sftp_password=xx
sftpRemote=/hcycdnsrc/biapp/
sftpLocal=/data/tmp
putfiles=ndmc_ispace_disk_attr_brwspace.csv

    expect <<- EOF
    set timeout 5
    spawn sftp -P $port $sftp_user@$sftp_IP
    expect {
        "(yes/no*" {send "yes\r"; exp_continue }
        "*assword:" {send "$sftp_password\n"}
    }
    expect "sftp>"
    send "cd $sftpRemote\n"
    expect "sftp>"
    send "lcd $sftpLocal\n"
    expect "sftp>"
    set timeout -1
    send "mput $putfiles\n"
    expect "sftp>"
    send "ls -l \n"
    expect "sftp>"
    send "bye\n"

loop file in dir

1
2
3
4
5
6
7
fileList=`ls *$DT_time*.zip`
fileArr=($fileList)
for fileName in ${fileArr[@]}
do
    unzip  $unzip_name
done

tar

打包、排除指定文件、删除源文件

1
tar -zcf wap.tar.gz wap --exclude *.zip --remove-files

常用日期

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
ymd=${date}
year=${ymd:0:4}
month=${ymd:4:2}
day=${ymd:6:2}

nowdate=`date +%Y%m01` #本月第一天
startdate=`date -d"$nowdate last month" +%Y%m%d` #上个月第一天
enddate=`date -d"$nowdate last day" +%Y%m%d` #上个月最后一天

daytime=20220315
year=${daytime:0:4}
month=${daytime:4:2}
nowdate=${year}${month}01
nextmonth=`date -d"$nowdate +1 month" +%Y%m%d`
enddate=`date -d"$nextmonth last day" +%Y%m%d` #本月最后一天

目录下最新日期文件

find /opt/folder -type f -print0 | xargs -0 stat --format '%Y :%y %n' | sort -nr | cut -d: -f2- | head

lsof

端口查进程
lsof -i:<port>

进程查端口
lsof -Pp 123674 |grep IPv |grep LISTEN

进程的线程信息
ps -mp 11139 -o THREAD

本机所有进程监听端口
lsof -nP -iTCP -sTCP:LISTEN

端口使用分析

netstat -an //查看所有
netstat -plntu //查询当前处于监听状态的端口
netstat -plnt -4 //查询当前处于监听状态的tcp4端口

netstat -ntlp -4 | head |awk ‘{print}’

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
# 这个方法得不到shell命令的输出

import commands,os
# os.system('netstat -ntlp -4')
retCode,output = commands.getstatusoutput('netstat -ntlp')

tmp = set()
lines = output.split('\n')
for line in lines:
    if str(line).startswith('tcp'):
        items = str(line).split()
        if not items[3].startswith('127.0.0.1') :
            if items[3].startswith('0.0.0.0') or items[3].startswith(':::') or items[3].split(':')[0] != items[4].split(':')[0]:
                print('%s\t\t%s\t\t%s' %(items[3], items[4], items[6]))
                tmp.add(int(items[3].split(':')[-1]))

listeningPort=list(tmp)
listeningPort.sort()
listeningPort

# listeningPort.sort()
for p in listeningPort:
   print(p)

#########
inUsePortInfo = {}
retCode,output = commands.getstatusoutput('netstat -ntp -4 |grep ESTABLISHED')
lines = output.split('\n')
for line in lines:
    if str(line).startswith('tcp'):
        items = str(line).split()
        if items[3].split(':')[0] != items[4].split(':')[0]:
            port = items[3].split(':')[1] + ' ' + items[6]
            # print(line)
            tmp = inUsePortInfo.get(port, list())
            tmp.append(items[4])
            tmp.sort()
            inUsePortInfo[port] = tmp


# for k,v in inUsePortInfo.items():
#     print('%s \t %s' %(k, v))

for i in sorted(inUsePortInfo):
    print('%s \t %s' %(i, inUsePortInfo[i]))


listeningPort.sort()
listeningPort


Expected Output：
port, foreign address list

Filter
local server 127.0.0.1
local 和 remoter 都是本机

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
# iftop/iptraf - IP连接分析

nohup sudo iftop -n -N -t -s 10 -L 1000 -o destination > /data/tmp/iftop20231211 &

nohup sudo iftop -n -N -t -s 86400 -L 5000 -o destination > /data/tmp/iftop20231211 &
cat  /data/tmp/iftop20231211 |grep '<=' |awk '{print $1}' |grep -v :

 sudo iptraf -i eth0 -L /var/log/traffic_log -B



# crontab监控重启

写脚本时注意环境变量问题，建议程序使用全路径执行

```shell
#!/bin/bash

# */5 * * * * /bin/sh /data/dolphinscheduler-1.3.6/ds-watcher.sh

restart_worker()
{
#pid=`pgrep -f org.apache.dolphinscheduler.server.worker.WorkerServer`
/usr/bin/pkill -f org.apache.dolphinscheduler.server.worker.WorkerServer
rm -f /data/dolphinscheduler-1.3.6/pid/dolphinscheduler-worker-server.pid
sleep 3s
su dolphinscheduler -c "sh /data/dolphinscheduler-1.3.6/bin/dolphinscheduler-daemon.sh start worker-server"
}

# worker
ret=`/data/java/bin/java -cp /data/dolphinscheduler-1.3.6/ds-watcher.jar  ds.ZkCheckNode master:2181,slave1:2181,slave2:2181 /dolphinscheduler/nodes/worker/default/`

if [ $ret = 'false' ] ;then
  restart_worker
fi

同步文件

集群服务器 增减同步，注意目的地址写父目录
rsync -av –delete /data/soft/spark/jars LGJF-ZYC6-SJZT-SVR04:/data/soft/spark/

异常处理

jps – process information unavailable

问题：无法显示进程名称

解决：删除目录

1
rm -rf /tmp/hsperfdata_*

fork: retry: Resource temporarily unavailable

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/etc/security/limits.d/90-nproc.conf

*          soft    nproc    65535
hadoop     soft    nproc     257529
root       soft    nproc     unlimited

修改后重启进程.  

在控制台执行(修改这个不需要重启)

   ulimit -u 655350

检查下是否生效，在控制台切到该用户下

   ulimit -u

其他信息查看命令：

ps h -Led -o user | sort | uniq -c | sort -n

ps -o nlwp,pid,lwp,args -u hadoop | sort -n

第一列NLWP为线程数

Bash路径通配规则

1
2
3
4
5
6
7
8
9
10
11
12
13
*：匹配任意个任意字符  
?：匹配任意单个字符  
[]：中括号通配。包括下面几种模式：  
  [abc]：匹配中括号中任意单个字符，即匹配a或b或c均可  
  [^abc]和`[!abc]`：匹配非中括号中的任意单个字符  
  [a-z]：匹配abc...z，但和Locale环境的排序规则有关。  
         Locale C环境下，[a-d]表示abcd，字典排序规则的[a-d]表示aBbCcDd  
  字符类：  
    [:alpha:]、[:alnum:]、[:ascii:]、[:blank:]、[:cntrl:]、  
    [:digit:]、[:graph:]、[:lower:]、[:print:]、[:punct:]、  
    [:space:]、[:upper:]、[:word:]、[:xdigit:]

{abc,def} 枚举