展示网站模板下载,白市驿网站建设,简易手工小制作,整站seo排名外包使用该脚本注意事项
安装hive之前确定机子有网络。或者yum 更改为本地源#xff0c;因为会使用epel仓库下载一个pv的软件使用该脚本前提是自行安装好mysql数据库准备好tomcat软件包#xff0c;该脚本使用tomcat9.x版本测试过能正常执行安装成功#xff0c;其他版本没有测试…使用该脚本注意事项
安装hive之前确定机子有网络。或者yum 更改为本地源因为会使用epel仓库下载一个pv的软件使用该脚本前提是自行安装好mysql数据库准备好tomcat软件包该脚本使用tomcat9.x版本测试过能正常执行安装成功其他版本没有测试过不知道有没这个问题hive安装脚本中自动更换成tez引擎了需要准备好tez安装软件测试时使用的是tez-0.9.2版本安装之前请自行更改脚本内容前面定义的software_dir(所有安装软件包存放目录)、install_dir(安装软件目录)、mysql_user、mysql_password 这个几个变量根据自已情况进行更改使用该脚本之前需要把hadoop先安装好并且把hadoop集群启动起来hadoop集群未启动切勿执行该脚本
#!/bin/bashsoftware_dir/root/hadoop/
install_dir/opt/
hadoop_path$(which hadoop|sed s|/bin.*||)
tomcat_path${install_dir}tomcat
mysql_userroot
mysql_password1234yum repolist |grep epel
if [ $? -ne 0 ]
thenwget -O /etc/yum.repos.d/epel.repo https://mirrors.aliyun.com/repo/epel-7.repo /dev/null
firpm -qa |grep pv
if [ $? -ne 0 ]
thenyum install -y pv /dev/nullrpm -qa |grep pv[ $? -ne 0 ] echo Please resolve the network problem !!! exit 1
fiecho Start Hive Configuration
hive_path${install_dir}hive
hive_conf_path$hive_path/conf
tez_path${install_dir}tezrm -rf $tez_path
pv ${software_dir}*tez* |tar zxf - -C $install_dir
mv ${install_dir}*tez* $tez_path
chown -R root:root $tez_pathrm -rf $hive_path
pv ${software_dir}*hive* |tar zxf - -C $install_dir
mv ${install_dir}*hive* ${install_dir}hiveif [ -f $hive_conf_path/hive-log4j2.properties.template ]
then mv $hive_conf_path/hive-log4j2.properties.template $hive_conf_path/hive-log4j2.properties
fiif [ -f $hive_conf_path/hive-exec-log4j2.properties.template ]
thenmv $hive_conf_path/hive-exec-log4j2.properties.template $hive_conf_path/hive-exec-log4j2.properties
fi
cp ${software_dir}mysql-connector-java-5.1.44-bin.jar $hive_path/lib
sed -i /property.hive.log.dir/c\property.hive.log.dir$hive_path/logs $hive_conf_path/hive-log4j2.properties
sed -i /property.hive.log.dir/c\property.hive.log.dir$hive_path/logs $hive_conf_path/hive-exec-log4j2.propertiescat $hive_conf_path/hive-site.xml EOF
?xml version1.0 encodingUTF-8 standaloneno?
?xml-stylesheet typetext/xsl hrefconfiguration.xsl?configuration!-- 配置hive元数据在hdfs的存储路径 --propertynamehive.metastore.warehouse.dir/namevalue/hive/database/value/property!-- 对于小数据量自动使用本地模式执行 MR job 加快执行过程 默认是false --propertynamehive.exec.mode.local.auto/namevaluetrue/value/property!-- 取消列名前面的表名 --propertynamehive.resultset.use.unique.column.names/namevaluefalse/value/property!-- 更换计算引擎 默认MR -- propertynamehive.execution.engine/namevaluetez/value/property!-- 关闭元数据校验 --propertynamehive.metastore.schema.verification/namevaluefalse/value/propertypropertynamejavax.jdo.option.ConnectionURL/namevaluejdbc:mysql://${HOSTNAME}:3306/hive?createDatabaseIfNotExisttrueamp;useUnicodetrueamp;characterEncodeingUTF-8amp;useSSLfalse/value/propertypropertynamejavax.jdo.option.ConnectionDriverName/namevaluecom.mysql.jdbc.Driver/value/propertypropertynamejavax.jdo.option.ConnectionUserName/namevalue$mysql_user/value/propertypropertynamejavax.jdo.option.ConnectionPassword/namevalue$mysql_password/value/property!-- 下面三个设置解决 tez-ui中hive queries不显示数据的问题--propertynamehive.exec.pre.hooks/namevalueorg.apache.hadoop.hive.ql.hooks.ATSHook/value/propertypropertynamehive.exec.post.hooks/namevalueorg.apache.hadoop.hive.ql.hooks.ATSHook/value/propertypropertynamehive.exec.failure.hooks/namevalueorg.apache.hadoop.hive.ql.hooks.ATSHook/value/property/configuration
EOF
rm -rf ${hive_path}/lib/log4j-slf4j-impl-*.jar
docker exec -it mysql mysql -u $mysql_user -p$mysql_password -e drop database if exists hive; /dev/null
schematool -dbType mysql -initSchemahdfs dfs -rm -r /tez
hdfs dfs -mkdir /tez
tez_name$(ls $tez_path/share)
hdfs dfs -put $tez_path/share/$tez_name /tez
rm -rf ${tez_path}/lib/slf4j-log4j12-*.jarcat $hive_conf_path/tez-site.xml EOF
?xml version1.0 encodingUTF-8 standaloneno?
?xml-stylesheet typetext/xsl hrefconfiguration.xsl?
configurationpropertynametez.lib.uris/namevalue\${fs.defaultFS}/tez/$tez_name/value/propertypropertynametez.use.cluster.hadoop-libs/namevaluetrue/value/propertypropertynametez.history.logging.service.class/namevalueorg.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService/value/propertypropertynametez.tez-ui.history-url.base/namevaluehttp://localhost:8080/tez-ui//value/property
/configuration
EOFmv $hive_conf_path/hive-env.sh.template $hive_conf_path/hive-env.shcat $hive_conf_path/hive-env.sh EOF
export TEZ_HOME$tez_path
export HADOOP_CLASSPATH\$HADOOP_CLASSPATH:\$TEZ_HOME/*:\$TEZ_HOME/lib/*
EOFsed -i /^\/configuration$/d $hadoop_path/etc/hadoop/yarn-site.xml
sed -i /!-- 开启timelineserver --/,$d $hadoop_path/etc/hadoop/yarn-site.xml
cat $hadoop_path/etc/hadoop/yarn-site.xml EOF!-- 开启timelineserver --propertynameyarn.timeline-service.enabled/namevaluetrue/value/property!-- 配置timeline安装在指定节点上 --propertynameyarn.timeline-service.hostname/namevalue$HOSTNAME/value/property!-- 开启跨域 --propertynameyarn.timeline-service.http-cross-origin.enabled/namevaluetrue/value/property!-- 发送RM的系统指标到timeline服务上 --propertynameyarn.resourcemanager.system-metrics-publisher.enabled/namevaluetrue/value/property!-- 开启通用历史服务 -- propertynameyarn.timeline-service.generic-application-history.enabled/namevaluetrue/value/property!-- 设置RPC请求的处理程序线程数 默认 10 --propertynameyarn.timeline-service.handler-thread-count/namevalue24/value/property/configuration
EOFxsync $hadoop_path/etc/hadoop/yarn-site.xml
jps |egrep NodeManager|ResourceManager
if [ $? -eq 0 ]
thenstop-yarn.shsleep 2start-yarn.sh
elsestart-all.sh
fijps |grep ApplicationHistoryServer
if [ $? -eq 0 ]
then yarn-daemon.sh stop timelineserver
fi
yarn-daemon.sh start timelineserverif [ -x $tomcat_path/bin/shutdown.sh ]
then$tomcat_path/bin/shutdown.sh /dev/nullrm -rf $tomcat_path
fipv ${software_dir}*tomcat* |tar zxf - -C $install_dir
mv ${install_dir}*tomcat* $tomcat_pathmkdir -p $tomcat_path/webapps/tez-ui/
unzip -d $tomcat_path/webapps/tez-ui/ $tez_path/*.war /dev/null
sed -i s|//timeline: http://localhost:8188|timeline: http://master:8188| $tomcat_path/webapps/tez-ui/config/configs.env
sed -i s|//rm: http://localhost:8088|rm: http://node1:8088| $tomcat_path/webapps/tez-ui/config/configs.env
sed -i s|//timeZone: UTC|timeZone: Asia/Shanghai| $tomcat_path/webapps/tez-ui/config/configs.envcat /usr/lib/systemd/system/tomcat.service EOF
[Unit]
Descriptiontomcat
Aftersyslog.target network.target remote-fs.target nss-lookup.target[Service]
TypeforkingEnvironmentJAVA_HOME/opt/jdk1.8.0_201PIDFile$tomcat_path/tomcat.pid
ExecStart$tomcat_path/bin/startup.sh
ExecStop$tomcat_path/bin/shutdown.sh
ExecReload/bin/kill -s HUP $MAINPID
PrivateTmptrue[Install]
WantedBymulti-user.targetEOFecho CATALINA_PID\$CATALINA_BASE/tomcat.pid $tomcat_path/bin/setenv.shsystemctl daemon-reload
systemctl enable --now tomcat
