dfs dfs -appendToFile - HDFSfile
# 按ctrl+C结束写入

export JAVA_HOME=/opt/soft/jdk
export HADOOP_PID_DIR=${HADOOP_HOME}

<property>
    <name>fs.defaultFS</name>
    <value>hdfs://localhost:9000</value>
</property>

<property>
    <name>dfs.namenode.name.dir</name>
    <value>/opt/soft/data/hdfs/name</value>
</property>
<property>
    <name>dfs.datanode.data.dir</name>
    <value>/opt/soft/data/hdfs/data</value>
</property>  
<property>
    <name>dfs.replication</name>
    <value>1</value>
</property>

<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>
<property>
    <name>yarn.nodemanager.env-whitelist</name>
    <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
</property>

<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>
<property>
    <name>mapreduce.application.classpath</name>
    <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
</property>

bin/hdfs namenode -format
sbin/start-dfs.sh

hadoop distcp hdfs://ip:9000/hive/spark-jars/metrics-core-4.1.1.jar /
hadoop distcp /spark/lib/HikariCP-2.5.1.jar  hdfs://ip:9000/

curl -i  "http://10.17.41.126:50070/webhdfs/v1/opendata?op=LISTSTATUS"

curl -i -X PUT "http://10.17.41.126:50070/webhdfs/v1/opendata/testFile?op=CREATE&overwrite=false&replication=1"

curl -i -X PUT -T <LOCAL_FILE> "http://LGJF-ZYC6-HCY-SVR553.cmic:50075/webhdfs/v1/opendata/testFile?&op=CREATE&namenoderpcaddress=LGJF-ZYC6-HCY-SVR553:9000&createflag=&createparent=true&overwrite=false&replication=1"

<property> 
<name>net.topology.script.file.name</name> 
<value>/home/rack_topology.sh</value> 
</property> 

TOPO=/mnt/d/topology.data

while [ $# -gt 0 ] ; do
  nodeArg=$1
  result=`awk -v var=${nodeArg} '$1 == var {print $2}' ${TOPO}`
  shift
  if [ -z "$result" ] ; then
    echo -n "/default/rack "
  else
    echo -n "$result "
  fi
done

hadoopdata1.com     /dc1/rack1
hadoopdata1         /dc1/rack1
10.1.1.1            /dc1/rack2

import sys
rack = {dn178.tj":"rack1",
        dn187.tj":"rack2"
        "192.168.1.178":"rack1",
        "192.168.1.187":"rack2",
        }
if __name__=="__main__":
    print "/" + rack.get(sys.argv[1],"rack0")

export HADOOP_CONF_DIR=/data/soft/hadoop-2.10.2-bdoc/etc/hadoop
export HADOOP_CLASSPATH=`hadoop classpath

check:  yarn app -list && hdfs dfs -ls .

core-site.xml
```xml
  <configuration  xmlns:xi="http://www.w3.org/2001/XInclude">
  <!--需与目标集群一致，不得自定义名称-->
    <property>
      <name>fs.defaultFS</name>
      <value>hdfs://ns1</value>
    </property>
<!--kerberos认证环境-->
    <property>
      <name>hadoop.security.authentication</name>
      <value>kerberos</value>
    </property>
<!-- 用于支持同时访问kerberos和simple证环境   -->
    <property>
      <name>ipc.client.fallback-to-simple-auth-allowed</name>
      <value>true</value>
    </property>
  </configuration>

hadoop archive -archiveName foo.har -p <src-parent-dir> [-r <replication factor>] <src>* <dest>

整个目录打包
hadoop archive -archiveName zoo.har -p /foo/bar -r 3 /outputdir

<property>
 <name>fs.defaultFS</name>
 <value>hdfs://nameservice1</value>
</property>

<configuration>
<!-- services -->
<!-- local sevice and remote service -->
<property>
      <name>dfs.nameservices</name>
      <value>ns1,ns8</value>
</property>
<!-- clusters that Datanode will report to -->
<property>
      <name>dfs.internal.nameservices </name>
      <value>ns1</value>
</property>

<!-- remote namespace ns8 -->
<!-- service ns8 -->
<property>
	<name>dfs.ha.namenodes.ns8</name>
	<value>nn1,nn2</value>
</property>
<property>
	<name>dfs.namenode.rpc-address.ns8.nn1</name>
	<value>192.168.100.1:8020</value>
</property>
<property>
	<name>dfs.namenode.rpc-address.ns8.nn2</name>
	<value>192.168.100.2:8020</value>
</property>
<property>
	<name>dfs.namenode.http-address.ns8.nn1</name>
	<value>192.168.100.1:50070</value>
</property>
<property>
	<name>dfs.namenode.http-address.ns8.nn2</name>
	<value>192.168.100.2:50070</value>
</property>
<property>
    <name>dfs.client.failover.proxy.provider.ns8</name>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
 
 
<!-- local namespace ns1 -->
<!-- service ns1 -->
<property>
      <name>dfs.internal.nameservices</name>
      <value>ns1</value>
</property>
<property>
    <name>dfs.ha.namenodes.ns1</name>
    <value>nn1,nn2</value>
</property>
<property>
    <name>dfs.namenode.rpc-address.ns1.nn1</name>
    <value>dev01:8020</value>
</property>
<property>
    <name>dfs.namenode.rpc-address.ns1.nn2</name>
    <value>dev02:8020</value>
</property>
<property>
    <name>dfs.namenode.http-address.ns1.nn1</name>
    <value>dev01:50070</value>
</property>
<property>
    <name>dfs.namenode.http-address.ns1.nn2</name>
    <value>dev02:50070</value>
</property>
<property>
    <name>dfs.client.failover.proxy.provider.ns1</name>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
</configuration>

[-exclude [-f <hosts-file> | <comma-separated list of hosts>]]
[-include [-f <hosts-file> | <comma-separated list of hosts>]]
[-source [-f <hosts-file> | <comma-separated list of hosts>]]
[-blockpools <comma-separated list of blockpool ids>]
[-idleiterations <idleiterations>]

# 设置用户名，密码 生成 db文件
htpasswd -c /usr/local/nginx/passwd.db username password
# 查看生成的db文件内容
cat /usr/nginx/conf/htpasswd.users

vi /usr/local/nginx/conf/nginx.conf
    server {
            listen 50070;
            server_name localhost;

            location / {
                    auth_basic "hadoop001"; # 虚拟主机认证命名
                    auth_basic_user_file /usr/local/nginx/passwd.db; # 虚拟主机用户名密码认证数据库
                    proxy_pass http://127.0.0.1:9870; # hadoop 访问
            }
    }

dfs.permissions.enabled = true -> hdfs-site 默认值true

dfs.permissions.superusergroup = supergroup -> hdfs-site 默认值supergroup

fs.permissions.umask-mode = 022 -> core-site 默认值022

# hadoop-policy.xml
yarn rmadmin -refreshServiceAcl
hadoop dfsadmin -refreshServiceAcl

基础开关
hadoop.security.authorization=true
控制提交任务/ JobTracker
security.job.client.protocol.acl
访问HDFS /NameNode
security.client.protocol.acl


# yarn-site

开启yarn ACLs:
     hadoop: core-site.xml
hadoop.security.authorization=true  #开启服务级别验证,否则hadoop组件的acl设置不生效
     yarn: yarn-site.xml
<property>
<name>yarn.acl.enable</name>
<value>true</value>
</property>
<property>
<name>yarn.admin.acl</name>
<value>hdfs</value>
</property>

<property>
<name>yarn.nodemanager.default-container-executor.log-dirs.permissions</name>
<value>755</value>
</property>
$ vi $HADOOP_CONF_DIR/capacity-scheduler.xml

$ yarn rmadmin -refreshQueues
yarn.scheduler.capacity.root.<queue-path>.acl_submit_applications

# modified 20230303
KNOWN_FILE="^/data$"
KNOWN_FILE=$KNOWN_FILE"|^/dolphinscheduler$"
KNOWN_FILE=$KNOWN_FILE"|^/flink$"
KNOWN_FILE=$KNOWN_FILE"|^/hive$"
KNOWN_FILE=$KNOWN_FILE"|^/spark$"
KNOWN_FILE=$KNOWN_FILE"|^/tmp$"
KNOWN_FILE=$KNOWN_FILE"|^/user$"
KNOWN_FILE=$KNOWN_FILE"|^/yarn$"
hdfs dfs -ls -C /  |grep -Ev $KNOWN_FILE

KNOWN_FILE="^/data/huadan$"
KNOWN_FILE=$KNOWN_FILE"|^/data/env$"
hdfs dfs -ls -C /data  |grep -Ev $KNOWN_FILE

KNOWN_FILE="^/dolphinscheduler/hadoop$"
hdfs dfs -ls -C /dolphinscheduler  |grep -Ev $KNOWN_FILE

KNOWN_FILE="^/flink/completed-jobs$"
hdfs dfs -ls -C /flink  |grep -Ev $KNOWN_FILE

KNOWN_FILE="^/spark/warehouse$"
KNOWN_FILE=$KNOWN_FILE"|^/spark/spark-history$"
KNOWN_FILE=$KNOWN_FILE"|^/spark/spark-jar.zip$"
hdfs dfs -ls -C /spark  |grep -Ev $KNOWN_FILE

KNOWN_FILE="^/hive/warehouse$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/tmp$"
hdfs dfs -ls -C /hive  |grep -Ev $KNOWN_FILE


KNOWN_FILE="^/hive/warehouse/ads.db$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/warehouse/default.db$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/warehouse/dim.db$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/warehouse/dm.db$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/warehouse/dw.db$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/warehouse/dwd.db$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/warehouse/lg.db$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/warehouse/mid.db$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/warehouse/ods.db$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/warehouse/oracle_orc_fromdatax.db$"
KNOWN_FILE=$KNOWN_FILE"|^/hive/warehouse/tmp.db$"
hdfs dfs -ls -C /hive/warehouse  |grep -Ev $KNOWN_FILE

dfs.namenode.audit.log.debug.cmdlist=getfileinfo,listStatus

export HDFS_AUDIT_LOGGER=INFO,RFAAUDIT

hdfs dfs -ls /hive/warehouse/tmp.db/ |awk '{print $8}' |awk -F'/' '{print $5}' |awk -F'_' '{print $1}' | sort |uniq -c |sort -h

$HADOOP_HOME/sbin/hadoop-daemon.sh stop namenode
$HADOOP_HOME/sbin/hadoop-daemon.sh start namenode
$HADOOP_HOME/sbin/hadoop-daemon.sh stop datanode
$HADOOP_HOME/sbin/hadoop-daemon.sh start datanode
$HADOOP_HOME/sbin/hadoop-daemon.sh stop nodemanager
$HADOOP_HOME/sbin/hadoop-daemon.sh start nodemanager

[Hadoop开启关闭调试信息]
开启：export HADOOP_ROOT_LOGGER=DEBUG,console

关闭：export HADOOP_ROOT_LOGGER=INFO,console

# 写入测试
hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-3.2.4.jar org.apache.hadoop.fs.TestDFSIO -D test.build.data=/user/ydy_bi_user48/benchmarks/TestDFSIO -write -nrFiles 10 -size 1000MB 

hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.10.2.jar TestDFSIO -D test.build.data=/tmp/benchmarks/TestDFSIO -write -nrFiles 10 -size 1000MB 


# 读取测试
hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-3.2.4.jar org.apache.hadoop.fs.TestDFSIO -D test.build.data=/user/ydy_bi_user48/benchmarks/TestDFSIO -read -nrFiles 10 -size 100MB

hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.10.2.jar TestDFSIO -D test.build.data=/tmp/benchmarks/TestDFSIO -read -nrFiles 10 -size 100MB


# 清理测试数据
hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-3.2.4.jar org.apache.hadoop.fs.TestDFSIO -D test.build.data=/user/ydy_bi_user48/benchmarks/TestDFSIO -clean

hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.10.2.jar TestDFSIO -D test.build.data=/tmp/benchmarks/TestDFSIO -clean