hadoop
1
2
3
4
5
6
7
8
9
10
11
12
13
| cd /data/soft
hdfs dfs -get /spark/pkg/hadoop-3.2.4.tar.gz && tar -xf hadoop-3.2.4.tar.gz && rm hadoop-3.2.4.tar.gz && cd hadoop-3.2.4/etc
hdfs dfs -get /spark/pkg/bdoc/ydy_hadoop_conf /spark/pkg/bdoc/tc_hadoop_conf .
mkdir ~/.env
cd ~/.env
hdfs dfs -get /spark/pkg/bdoc/ydy_bi_user48.keytab /spark/pkg/bdoc/tc_ycpfxs_user03.keytab .
hdfs dfs -get /spark/pkg/bdoc/bdoc_ydy /spark/pkg/bdoc/bdoc_tc .
hdfs dfs -cat /spark/pkg/bdoc/krb5.conf
/etc/hosts
|
spark
[[Thrift server#部署spark(3.5.6)]]
1
2
3
4
5
| export SPARK_CONF_DIR=
或
--config xxx
需要 spark-defaults.conf hive-site.xml
|
conf_ydy
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
| spark.master=yarn
spark.driver.host 10.27.48.6
spark.yarn.archive hdfs://ns1/user/ydy_bi_user48/spark/spark-jar356-20260630.zip
spark.yarn.queue root.bdoc.ydy_bi_yarn27
spark.kerberos.keytab /home/hadoop/.env/ydy_bi_user48.keytab
spark.kerberos.principal ydy_bi_user48@GZHLWYJQ
spark.serializer=org.apache.spark.serializer.KryoSerializer
spark.sql.sources.default=orc
spark.eventLog.enabled true
spark.eventLog.dir hdfs:///user/ydy_bi_user48/spark/spark-history
spark.eventLog.compress true
spark.history.fs.logDirectory hdfs:///user/ydy_bi_user48/spark/spark-history
spark.driver.extraJavaOptions=-Dsun.security.krb5.debug=false
|
conf_tc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
| spark.master=yarn
spark.driver.host 10.27.48.6
spark.yarn.archive hdfs://gdhlwtc/user/tc_ycpfxs_user03/spark/spark-jar356-20260630.zip
spark.yarn.queue root.bdoc.tc_ycpfxs_yarn03
spark.kerberos.keytab /home/hadoop/.env/tc_ycpfxs_user03.keytab
spark.kerberos.principal tc_ycpfxs_user03@GZHLWYJQ
spark.serializer=org.apache.spark.serializer.KryoSerializer
spark.sql.sources.default=orc
spark.eventLog.enabled true
spark.eventLog.dir hdfs:///user/tc_ycpfxs_user03/spark/spark-history
spark.eventLog.compress true
spark.history.fs.logDirectory hdfs:///user/tc_ycpfxs_user03/spark/spark-history
spark.driver.extraJavaOptions=-Dsun.security.krb5.debug=false
|
hdfs dfs -ls /spark/pkg/bdoc/thrift
快捷环境变量
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
| . ~/.env/bdoc_ydy
. ~/.env/bdoc_tc
source /etc/profile
vi bdoc_ydy
export PATH=/sbin:/bin:/usr/sbin:/usr/bin
export JAVA_HOME=/data/soft/jdk
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH
export HADOOP_HOME=/data/soft/hadoop-3.2.4
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/ydy_hadoop_conf
unset HADOOP_CLASSPATH
export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
export SPARK_HOME=/data/soft/spark
export SPARK_CONF_DIR=$SPARK_HOME/conf_ydy
export PATH=$SPARK_HOME/bin:$PATH
export KRB5CCNAME=/tmp/krb5cc_ydy
kinit -kt /home/hadoop/.env/ydy_bi_user48.keytab ydy_bi_user48@GZHLWYJQ
vi bdoc_tc
export PATH=/sbin:/bin:/usr/sbin:/usr/bin
export JAVA_HOME=/data/soft/jdk
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH
export HADOOP_HOME=/data/soft/hadoop-3.2.4
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/tc_hadoop_conf
unset HADOOP_CLASSPATH
export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
export KRB5CCNAME=/tmp/krb5cc_tc
kinit -kt /home/hadoop/.env/tc_ycpfxs_user03.keytab tc_ycpfxs_user03@GZHLWYJQ
|
