[
https://issues.apache.org/jira/browse/HIVE-25647?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sylwester Lachiewicz resolved HIVE-25647.
-----------------------------------------
Resolution: Invalid
> hadoop memo
> -----------
>
> Key: HIVE-25647
> URL: https://issues.apache.org/jira/browse/HIVE-25647
> Project: Hive
> Issue Type: Wish
> Components: Configuration
> Affects Versions: 3.1.2
> Environment: hadoop 2.7.3
> Reporter: St Li
> Assignee: St Li
> Priority: Major
> Fix For: All Versions
>
> Attachments: worldip.csv
>
>
> master,slave1,slave2master,slave1,slave2//opt represent wechat hadoop bigdata
> dev//2019 :bigdata competitionhadoop 50070hbase 16010storm 8080
> #hostnamehostnamectl set-hostname master && bash hostname master && bash
> hostname slave1/slave2 && bash vim /etc/hostname master/slave1/slave2vim
> /etc/hosts ip master ip slave1 ipslave2
> #yumcd /etc/yum.repos.d && rm -rf *wget
> http://172.16.47.240/bigdata/repofile/bigdata.repoyum clean all
> #firewallsystemctl stop firewalldsystemctl status firewalld
> #timezonetzselect 5-9-1-1echo "TZ='Asia/Shanghai'; export TZ" >>
> /etc/profile && source /etc/profile
> #ntpyum install -y ntpvim /etc/ntp.conf//#server 0~3.centos.pool.ntp.org
> iburstserver 127.127.1.0fudge 127.127.1.0 stratum 10/bin/systemctl restart
> ntpd.servicentpdate master (slave1,slave2)
> #crontabservice crond status/sbin/service crond startcrontab -e*/30 8-17 * *
> * /usr/sbin/ntpdate mastercrontab –l
> #ssh passwordssh-keygen -t dsa -P '' -f ~/.ssh/id_dsacat
> /root/.ssh/id_dsa.pub >> /root/.ssh/authorized_keysscp ~/.ssh/authorized_keys
> root@slave1:~/.ssh/scp ~/.ssh/authorized_keys root@slave2:~/.ssh/
> ssh-copy-id masterssh-copy-id slave1ssh-copy-id slave2
> #install jdkmkdir -p /usr/javatar -zxvf jdk-8u171-linux-x64.tar.gz -C
> /usr/java/
> vim /etc/profileexport JAVA_HOME=/usr/java/jdk1.8.0_171export
> CLASSPATH=$JAVA_HOME/lib/export PATH=$PATH:$JAVA_HOME/bin
> source /etc/profile && java -version
> scp -r /usr/java root@slave1:/usr/scp -r /usr/java root@slave2:/usr/
> #install hadoopmkdir -p /usr/hadoop && cd /usr/hadooptar -zxvf
> /usr/hadoop/hadoop-2.7.3.tar.gz -C /usr/hadoop/rm -rf
> /usr/hadoop/hadoop-2.7.3.tar.gzvim /etc/profileexport
> HADOOP_HOME=/usr/hadoop/hadoop-2.7.3export
> PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbinhadoop //test
> hadoop-env.sh/mapred-env.sh/yarn-en.shexport JAVA_HOME=/usr/java/jdk1.8.0_171
> ##vim core-site.xmlfs.default.name \{hdfs://master:9000}hadoop.tmp.dir
> \{/usr/hadoop/hadoop-2.7.3/hdfs/tmp}io.file.buffer.size
> \{131072}fs.checkpoint.period \{60}fs.checkpoint.size \{67108864}
> ##hdfs-site.xmldfs.replication \{2}dfs.namenode.name.dir
> \{file:/usr/hadoop/hadoop-2.7.3/hdfs/name}dfs.datanode.data.dir
> \{file:/usr/hadoop/hadoop-2.7.3/hdfs/data}
> ##vim yarn-env.shyarn.resourcemanager.address
> \{master:18040}yarn.resourcemanager.scheduler.address
> \{master:18030}yarn.resourcemanager.webapp.address
> \{master:18088}yarn.resourcemanager.resource-tracker.address
> \{18025}yarn.resourcemanager.admin.address
> \{master:18141}yarn.nodemanager.aux-services
> \{mapreduce_shuffle}yarn.nodemanager.auxservices.mapreduce.shuffle.class
> \{org.apache.hadoop.mapred.ShuffleHandler}
> #vim mapred-site.xmlmapreduce.framework.name \{yarn}
> #slaves fileecho master > master && echo slave1 > slaves && echo slave2 >>
> slaves
> #hadoop formathadoop namenode -format (master) //has been
> successfully#start hadoopstart-all.shmaster
> :NameNode,SecondaryNameNode,ResourceManagerslave1~2:DataNode,NodeManager
> start-dfs.shstart-yarn.shhadoop-daemon.sh start namenodehadoop-daemon.sh
> start datanodehadoop-daemon.sh start secondarynamenodehadoop-daemon.sh start
> resourcemanagerhadoop-daemon.sh start nodemanager
> test hdfs& mapreducehadoop fs -mkdir /inputhadoop fs -put
> $HADOOP_HOME/README.txt /input
> http://master:50070hadoop jar
> $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.8.4.jar
> #install hiveyum -y install mysql-community-server slave2:mysqlserver
> slave1:hiveserver master:hiveclientsystemctl daemon-reloadsystemctl start
> mysqldcat /var/log/mysqld.log grep "temporary password"mysql -uroot -pset
> global validate_password_policy=0;set global validate_password_length=4;alter
> user 'root'@'localhost' identified by '123456';mysql -uroot -p123456create
> user 'root'@'%' identified by '123456';grant all privileges on *.* to
> 'root'@'%' with grant option;flush privileges;
> mkdir -p /usr/hive tar -zxvf /usr/hive/apache-hive-2.1.1-bin.tar.gz -C
> /usr/hive/
> vim /etc/profile //for hiveexport
> HIVE_HOME=/usr/hive/apache-hive-2.1.1-binexport
> PATH=$PATH:$HIVE_HOME/binsource /etc/profile
> vim hive-env.shcd $HIVE_HOME/conf && vim hive-env.shexport
> HADOOP_HOME=/usr/hadoop/hadoop-2.7.3export
> HIVE_CONF_DIR=/usr/hive/apache-hive-2.1.1-bin/confexport
> HIVE_AUX_JARS_PATH=/usr/hive/apache-hive-2.1.1-bin/lib
> cp $HIVE_HOME/lib/jline-2.12.jar $HADOOP_HOME/share/hadoop/yarn/lib/
> ##slave1 hive-servercd $HIVE_HOME/lib && wget or cp
> mysql-connector-java-5.1.47-bin.jar
> hive-site.xml (hive-server)hive.metastore.warehouse.dir
> \{/user/hive_remote/warehouse}javax.jdo.option.ConnectionDriverName
> \{com.mysql.jdbc.Driver}javax.jdo.option.ConnectionURL
> \{jdbc:mysql://slave2:3306/hive?createDatabaseIfNotExist=true&useSSL=false}javax.jdo.option.ConnectionUserName
> \{root}javax.jdo.option.ConnectionPassword \{123456}
> hive-site.xml (hive client)hive.metastore.warehouse.dir
> \{/user/hive_remote/warehouse}hive.metastore.local
> \{false}hive.metastore.uris \{thrift://slave1:9083}
> //opt hadoop fs -mkdir /tmp hadoop fs -mkdir -p /user/hive/warehousehadoop
> fs -chmod g+w /tmp& /user/hive/warehouse//opt modify log locationvim
> hive-log4j2.propertiesproperty.hive.log.dir=/home/hadoop
> //opt hive-site.xmlhive.cli.print.current.db \{true}hive.cli.print.header
> \{true}
> beeline connect shoud config core-site.xmlhadoop.proxyuser.hadoop.hosts
> \{*}..groups \{*}bin/beeline->!connect jdbc:hive2://centos01:10000or
> bin/beeline -u jdbc:hive2://centos01:10000 -n hadoop
> desc table/desc database/desc database extended testdb;
> hive-serverschematool -dbType mysql -initSchemahive --service metastore
> hive-clienthive \ncreate database hongyaa;
> #install zookeepermkdir -p /usr/zookeeper && cd /usr/zookeepertar -zxvf
> /usr/zookeeper/zookeeper-3.4.10.tar.gz -C /usr/zookeeper/vi /etc/profile
> //for zookeeperexport
> ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10PATH=$PATH:$ZOOKEEPER_HOME/binsource
> /etc/profile
> cd /usr/zookeeper/zookeeper-3.4.10/conf/ && mv zoo_sample.cfg zoo.cfg &&
> vimzoo.cfgtickTime=2000initLimit=10syncLimit=5dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdataclientPort=2181dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalogserver.1=master:2888:3888server.2=slave1:2888:3888server.3=slave2:2888:3888
> cd /usr/zookeeper/zookeeper-3.4.10 && mkdir zkdata zkdatalogcd
> /usr/zookeeper/zookeeper-3.4.10/zkdata && echo "1" >> myid
> zkServer.sh start(master,slave1,slave2)netstat -antp|grep 2181
> #install hbase//opttar -zxf hbase-1.2.6.1-bin.tar.gz -C /opt/modules/export
> JAVA_HOME=/usr/java/jdk1.8.0_171export HBASE_MANAGES_ZK=true
> vim hbase-site.xmlhbase.rootdir
> \{hdfs://centos01:9000}hbase.cluster.distributed
> \{true}hbase.zookeeper.quorum
> \{centos01,centos02,centos03}hbase.zookeeper.property.dataDir
> \{/opt/modules/hbase-1.2.6/zkData}
> #install kafkatar -zxvf kafka_2.11-2.0.0.tgz -C /opt/modulesvim
> config/server.propertiesbroker.id=1num.partitions=2default.replication.factor=1lisheners=PLAINTEXT://centos01:9092log.dirs=/opt/modules/kafka_2.11-2.0.0/kafka-logszookeeper.connect=centos01:2181,centos02:2181,centos03:2181scp
> to centos02,centos03 and modify id
> bin/kafka-server-start.sh -daemon
> config/server.properties/kafka-server-start.sh -daemon
> config/server.properties
> create topicbin/kafka-topics.sh \--create \--zookeeper
> centos01:2181,centos02:2181,centos03:2181 \--replication-factor 2
> \--partitions 2 \--topic topictest
> bin/kafka-topics.sh \--list \--zookeeper centos01:2181
> bin/kafka-topics.sh \--describe \--zookeeper
> centos01:2181//生产者bin/kafka-console-producer.sh \--broker-list
> centos01:9092,centos02:9092,centos03:9092 \--topic
> topictest//消费者bin/kafka-console-consumer.sh \--bootstrap-server
> centos01:9092,centos02:9092,centos03:9092 \--topic topictest
> java -Xms256m -Xmx512m -classpath .:./lib/*
> com.cmcc.hsd.loader.ScannerV2p9Loader -t 6 -d 202008250000
> #install stormtar zxvf apache-storm-1.1.0.tar.gz -C /opt/modulesmv
> apache-storm-1.1.0/ storm-1.1.0vim /etc/profileexport
> STORM_HOME=/opt/modules/storm-1.1.0export PATH=$PATH:$STORM_HOME/bin
> storm-env.shexport JAVA_HOME=/opt/softwares/jdk1.8.0_201export
> STORM_CONF_DIR="/opt/modules/storm-1.1.0/conf"vim storm.yaml
> storm.zookeeper.servers: - "centos01" - "centos02" - "centos03"
> supervisor.slots.ports: - 6700 - 6701 - 6702 - 6703
> storm.zookeeper.port: 2181 storm.local.dir: "/opt/modules/storm-1.1.0/data"
> scp -r to centos01,centos02,centos03 storm nimbus > /dev/null 2>&1 &
> (master)storm ui > /dev/null 2>&1 & storm supervisor > /dev/null 2>&1 &
> (slave)
> jpsnimbus/Supervisor
> #install sparktar zxvf spark-2.4.0-bin-hadoop2.7.tgz -C /opt/modulesvim
> conf/slavescentos02centos03vim spark-env.shexport
> JAVA_HOME=/opt/softwares/jdk1.8.0_201export SPARK_MASTER_IP=centos01export
> SPARK_MASTER_PORT=7077
> export HADOOP_HOME=/opt/modules/hadoop-2.8.4export
> HADOOP_CONF_DIR=$HADOOP_HOME/etc/haoop
> scp -r to other node
> spark ha delete SPARK_MASTER_IP add export
> SPARK_DAEMON_JAVA_OPT="-Dspark.deploy.recoveryMode=ZOOKEEPER-Dspark.deploy.zookeeper.url=centos01:2181,centos02:2181,centos03:2181-Dspark.deploy.zookeepr.dir=/spark"
> bin/spark-submit \--class org.apache.spark.examples.SparkPi \--master yarn
> \--deploy-mode cluster
> \/opt/modules/spark-2.4.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.4.0.jar
> bin/spark-shell \--master spark://centos02:7077
> bin/spark-submit \--class org.apache.spark.examples.SparkPi \--master yarn
> \--deploy-mode cluster \--drive-memory 512m \--executor-memory 1g
> \--executor-cores 2
> \/opt/modules/spark-2.4.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.4.0.jar
> yarn-site.xml ofen kill by yarnpmem-check-enablevmem-check-enable
> #analysis data *****create database hive;use hive;create table bike
> (duration int,startdate timestamp,enddate timestamp,type string) row format
> delimitedfields terminated by ',';
> LOAD DATA LOCAL INPATH './load.csv' INTO TABLE loan;insert overwrite local
> directory '/home/hadoop/college001/'row format delimited fields terminated by
> 't'select count(*) from bike;##about hiveDateType ,UDAF UDTFselect
> from_unixtime cast(concat(startdate,':00') as timestamp),'yyyyMMdd000000')
> from bike limit 10;select date_format('2019-10-07', 'yyyyMMdd000000')select
> from_unixtime(unix_timestamp('2019-10-07', 'yyyy-MM-dd'),
> 'yyyyMMdd000000')select unix_timestamp('2019-10-07 13:24:20','yyyy-MM-dd
> HH:mm:ss')
> JSON.stringify(option)
> iconv -f gbk -t utf8 a.txt –o b.txt
> mapreduce output -rw-r--r-- 2 hadoop supergroup 0 2021-10-26 07:06
> /output2/_SUCCESS-rw-r--r-- 2 hadoop supergroup 1306 2021-10-26 07:06
> /output2/part-r-00000
> #add node slave3yum install -y ntp && ntpdate mastervim /etc/hostsslave3scp
> ~/.ssh/authorized_keys root@slave3:~/.ssh/yum install -y ntp && ntpdate
> mastervim /etc/profile && source /etc/profileexport JAVA...export
> HADOOP...vim $HADOOP_HOME/etc/hadoop/slaves add slave3
> $HADOOP_HOME/sbin/hadoop-daemon.sh start
> datanode$HADOOP_HOME/sbin/yarn-daemon.sh start nodemanagerhdfs dfsadmin
> -refreshNodessbin/start-balancer.shhdfs dfsadmin -report
> echo "slave3" > excludes
> hdfs-site.xml (master)
> dfs.hosts.exclude \{/usr/hadoop/hadoop-2.7.3/etc/hadoop/excludes}
> hdfs dfsadmin -refreshNodeshdfs dfsadmin -reporthadoop-daemon.sh stop
> datanodeyarn-daemon.sh stop nodemanagerhdfs dfsadmin
> -reportsbin/start-balancer.sh
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
