Hi,
I'm able to deploy the 'data-analytics-with-sql-like' bundle successfully.
When it's deployed I can run the hdfs and hive smoke tests described
here (
https://api.jujucharms.com/v4/bundle/data-analytics-with-sql-like-5/archive/README.md)
successfully.
I then need to deploy the Spark charm and add relations connecting
spark-master to yarn-hdfs-master. I do this by deploying spark-master
to the same machine where yarn-hdfs-master is located (see my
previous email).
However, once I have deployed the spark charm and added relations
from yarn-hdfs-master to spark-master, I can no longer connect
to hdfs.
root@adminuser-VirtualBox:~# juju ssh yarn-hdfs-master/0
ubuntu@ip-172-31-27-83:~$ sudo su $HDFS_USER
hdfs@ip-172-31-27-83:/home/ubuntu$
hdfs@ip-172-31-27-83:/home/ubuntu$ hdfs dfs -ls /
ls: Incomplete HDFS URI, no host: hdfs://TODO-NAMENODE-HOSTNAME:PORT
hdfs@ip-172-31-27-83:/home/ubuntu$
It seems that adding relations between yarn-hdfs-master and
spark overwrites the hdfs configuration so it cannot connect anymore.
This is how I deploy the spark-charm and the relations I add,
root@adminuser-VirtualBox:~# cd ~
root@adminuser-VirtualBox:~# mkdir charms
root@adminuser-VirtualBox:~# mkdir charms/trusty
root@adminuser-VirtualBox:~# cd charms/trusty
root@adminuser-VirtualBox:~# git clone
https://github.com/Archethought/spark-charm spark
// Deploy spark to specific node(s) in the cluster
// in particular, deploy spark-master to same node as yarn-hdfs-master
juju status // which node machine is yarn-hdfs-master on ?
// e.g. say machine: "4"
// deploy spark-master to same node as yarn-hdfs-master
juju deploy --repository=charms local:trusty/spark --to 4 spark-master
juju deploy --repository=charms local:trusty/spark spark-slave
juju add-relation spark-master:master spark-slave:slave
// add relations - connect YARN to SPARK
juju add-relation yarn-hdfs-master:resourcemanager spark-master
juju add-relation yarn-hdfs-master:namenode spark-master
juju status
Attached is my 'juju status'.
Any help is very appreciated,
Ken
root@adminuser-VirtualBox:~# juju status
environment: amazon
machines:
"0":
agent-state: started
agent-version: 1.21.1
dns-name: 54.152.135.200
instance-id: i-a0b24a5a
instance-state: running
series: trusty
hardware: arch=amd64 cpu-cores=1 cpu-power=100 mem=1740M root-disk=8192M
state-server-member-status: has-vote
"1":
agent-state: started
agent-version: 1.21.1
dns-name: 54.164.136.34
instance-id: i-4026e7af
instance-state: running
series: trusty
hardware: arch=amd64 cpu-cores=1 cpu-power=100 mem=1740M root-disk=8192M
"2":
agent-state: started
agent-version: 1.21.1
dns-name: 54.152.183.122
instance-id: i-d81b7429
instance-state: running
series: trusty
hardware: arch=amd64 cpu-cores=1 cpu-power=100 mem=1740M root-disk=8192M
"3":
agent-state: started
agent-version: 1.21.1
dns-name: 54.152.37.83
instance-id: i-68b74f92
instance-state: running
series: trusty
hardware: arch=amd64 cpu-cores=1 cpu-power=100 mem=1740M root-disk=8192M
"4":
agent-state: started
agent-version: 1.21.1
dns-name: 54.152.140.119
instance-id: i-532aebbc
instance-state: running
series: trusty
hardware: arch=amd64 cpu-cores=1 cpu-power=100 mem=1740M root-disk=8192M
"5":
agent-state: started
agent-version: 1.21.1
dns-name: 54.152.233.201
instance-id: i-ec3c521d
instance-state: running
series: trusty
hardware: arch=amd64 cpu-cores=1 cpu-power=100 mem=1740M root-disk=8192M
services:
compute-node:
charm: cs:trusty/hdp-hadoop-4
exposed: false
relations:
datanode:
- yarn-hdfs-master
nodemanager:
- yarn-hdfs-master
units:
compute-node/0:
agent-state: started
agent-version: 1.21.1
machine: "1"
open-ports:
- 8010/tcp
- 8025/tcp
- 8030/tcp
- 8050/tcp
- 8088/tcp
- 8141/tcp
- 8480/tcp
- 10020/tcp
- 19888/tcp
- 50010/tcp
- 50075/tcp
public-address: 54.164.136.34
hdphive:
charm: cs:trusty/hdp-hive-2
exposed: false
relations:
db:
- mysql
namenode:
- yarn-hdfs-master
resourcemanager:
- yarn-hdfs-master
units:
hdphive/0:
agent-state: started
agent-version: 1.21.1
machine: "2"
open-ports:
- 10000/tcp
public-address: 54.152.183.122
juju-gui:
charm: cs:trusty/juju-gui-17
exposed: true
units:
juju-gui/0:
agent-state: started
agent-version: 1.21.1
machine: "0"
open-ports:
- 80/tcp
- 443/tcp
public-address: 54.152.135.200
mysql:
charm: cs:trusty/mysql-4
exposed: false
relations:
cluster:
- mysql
db:
- hdphive
units:
mysql/0:
agent-state: started
agent-version: 1.21.1
machine: "3"
public-address: 54.152.37.83
spark-master:
charm: local:trusty/spark-0
exposed: false
relations:
master:
- spark-slave
namenode:
- yarn-hdfs-master
resourcemanager:
- yarn-hdfs-master
units:
spark-master/0:
agent-state: started
agent-version: 1.21.1
machine: "4"
open-ports:
- 4040/tcp
- 7077/tcp
- 8080/tcp
- 18080/tcp
public-address: 54.152.140.119
spark-slave:
charm: local:trusty/spark-1
exposed: false
relations:
slave:
- spark-master
units:
spark-slave/0:
agent-state: started
agent-version: 1.21.1
machine: "5"
open-ports:
- 8081/tcp
public-address: 54.152.233.201
yarn-hdfs-master:
charm: cs:trusty/hdp-hadoop-4
exposed: false
relations:
namenode:
- compute-node
- hdphive
- spark-master
resourcemanager:
- compute-node
- hdphive
- spark-master
units:
yarn-hdfs-master/0:
agent-state: error
agent-state-info: 'hook failed: "namenode-relation-joined" for
spark-master:namenode'
agent-version: 1.21.1
machine: "4"
open-ports:
- 8010/tcp
- 8020/tcp
- 8025/tcp
- 8030/tcp
- 8050/tcp
- 8088/tcp
- 8141/tcp
- 8480/tcp
- 10020/tcp
- 19888/tcp
- 50070/tcp
- 50075/tcp
- 50470/tcp
public-address: 54.152.140.119
root@adminuser-VirtualBox:~#
--
Juju mailing list
Juju@lists.ubuntu.com
Modify settings or unsubscribe at:
https://lists.ubuntu.com/mailman/listinfo/juju
