This is an automated email from the ASF dual-hosted git repository.
janardhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 0cf5125 [SYSTEMDS-2926][1/3] Update readme with administrative setup
0cf5125 is described below
commit 0cf5125ab6fdc74abd07cfc9779ca4eba7890b69
Author: Janardhan Pulivarthi <[email protected]>
AuthorDate: Sun Apr 11 07:04:05 2021 +0530
[SYSTEMDS-2926][1/3] Update readme with administrative setup
Closes #1222.
---
scripts/aws/README.md | 110 +++++++++++++++++++++++++++++++--
scripts/aws/spinup_systemds_cluster.sh | 14 +++--
scripts/aws/systemds_cluster.config | 1 +
3 files changed, 115 insertions(+), 10 deletions(-)
diff --git a/scripts/aws/README.md b/scripts/aws/README.md
index 3040101..90fc6fd 100644
--- a/scripts/aws/README.md
+++ b/scripts/aws/README.md
@@ -17,13 +17,113 @@ limitations under the License.
{% endcomment %}
-->
-Instructions:
+## Administrator setup
-1. Create aws account / use your existing aws account
-2. Install aws-cli on your system
+### With [`Cloud Shell`](https://console.aws.amazon.com/cloudshell/home):
+
+Assumed variables,
+
+| Name | Value |
+| --- | --- |
+| `UserName` | `systemds-bot` |
+| `GroupName` | `systemds-group` |
+
+#### 1. Create a user and a group
+
+Create a user and a group, and join user to the created group.
+
+[`create-user`](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/iam/create-user.html)
+```sh
+[cloudshell-user@host ~]$ aws iam create-user --user-name systemds-bot
+{
+ "User": {
+ "Path": "/",
+ "UserName": "systemds-bot",
+ "UserId": "AIDAQSHHX7DDAODFXYZ3",
+ "Arn": "arn:aws:iam::12345:user/systemds-bot",
+ "CreateDate": "2021-04-10T20:36:59+00:00"
+ }
+}
+```
+
+[`create-group`](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/iam/create-group.html)
+
+```sh
+[cloudshell-user@host ~]$aws iam create-group --group-name systemds-group
+{
+ "Group": {
+ "Path": "/",
+ "GroupName": "systemds-group",
+ "GroupId": "AGPAQSHHX7DDB3XYZABCW",
+ "Arn": "arn:aws:iam::12345:group/systemds-group",
+ "CreateDate": "2021-04-10T20:41:58+00:00"
+ }
+}
+```
+
+#### 2. Attach roles to the group
+
+[`attach-group-policy`](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/iam/attach-group-policy.html)
+
+```sh
+aws iam attach-group-policy --policy-arn
arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole --group-name
systemds-group
+aws iam attach-group-policy --policy-arn
arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role
--group-name systemds-group
+aws iam attach-group-policy --policy-arn
arn:aws:iam::aws:policy/AmazonElasticMapReduceFullAccess --group-name
systemds-group
+aws iam attach-group-policy --policy-arn
arn:aws:iam::aws:policy/AWSKeyManagementServicePowerUser --group-name
systemds-group
+aws iam attach-group-policy --policy-arn
arn:aws:iam::aws:policy/IAMUserSSHKeys --group-name systemds-group
+
+# Grant cloud shell access too.
+aws iam attach-group-policy --policy-arn
arn:aws:iam::aws:policy/AWSCloudShellFullAccess --group-name systemds-group
+
+# To create EC2 keys
+aws iam attach-group-policy --policy-arn
arn:aws:iam::aws:policy/AmazonEC2FullAccess --group-name systemds-group
+```
+
+#### 3. Add user to the group
-(https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html)
+```sh
+aws iam add-user-to-group --user-name systemds-bot --group-name systemds-group
+```
+
+#### 4. Create the login-profile with credentials
+
+```sh
+$ aws iam create-login-profile --generate-cli-skeleton > login-profile.json
+```
+
+`login-profile.json` contains
+
+```json
+{
+ "LoginProfile": {
+ "UserName": "",
+ "Password": "",
+ "PasswordResetRequired": false
+ }
+}
+```
+
+Create the credentials manually by editing `login-profile.json`.
+
+| Name | Value |
+| --- | --- |
+| `UserName` | `systemds-bot` |
+| `Password` | For example, `9U*tYP` |
+| `PasswordResetRequired` | `false` |
+
+Now, create the login profile.
+
+```sh
+aws iam create-login-profile --cli-input-json file://login-profile.json
+```
+
+---
+### With [`AWS
CLI`](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html):
+
+1. Create aws account / use your existing aws account
+
+2. Install `aws-cli` specific to your Operating System.
3. Create a user
@@ -45,6 +145,8 @@ Instructions:
4. Configure your aws-cli
(https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration)
+## User Setup
+
5. Spin up an EMR cluster with SystemDS
* Put your SystemDS artifacts (dml-scripts, jars, config-file) in the
directory systemds
diff --git a/scripts/aws/spinup_systemds_cluster.sh
b/scripts/aws/spinup_systemds_cluster.sh
index a9a1b8e..8c93a63 100755
--- a/scripts/aws/spinup_systemds_cluster.sh
+++ b/scripts/aws/spinup_systemds_cluster.sh
@@ -44,17 +44,18 @@ set_config "SPARK_NUM_EXECUTORS" $CORE_INSTANCES_COUNT
set_config "SPARK_EXECUTOR_CORES" $SPARK_EXECUTOR_CORES
set_config "SPARK_EXECUTOR_MEMORY" $SPARK_EXECUTOR_MEMORY
set_config "SPARK_DRIVER_MEMORY" "1G"
+set_config "BUCKET" $BUCKET-$(((RANDOM % 999) + 1000))
#Create systemDS bucket
-aws s3api create-bucket --bucket system-ds-bucket --region $REGION &> /dev/null
-aws s3api create-bucket --bucket system-ds-logs-bucket --region $REGION &>
/dev/null
+aws s3api create-bucket --bucket $BUCKET --region $REGION &> /dev/null
+aws s3api create-bucket --bucket $BUCKET-logs --region $REGION &> /dev/null
# Upload Jar and scripts to s3
-aws s3 sync $SYSTEMDS_TARGET_DIRECTORY s3://system-ds-bucket/ --exclude "*"
--include "*.dml" --include "*config.xml" --include "*DS.jar*"
+aws s3 sync $SYSTEMDS_TARGET_DIRECTORY s3://$BUCKET --exclude "*" --include
"*.dml" --include "*config.xml" --include "*DS.jar*"
# Create keypair
if [ ! -f ${KEYPAIR_NAME}.pem ]; then
- aws ec2 create-key-pair --key-name $KEYPAIR_NAME --query "KeyMaterial"
--output text > "$KEYPAIR_NAME.pem"
+ aws ec2 create-key-pair --key-name $KEYPAIR_NAME --region $REGION --query
"KeyMaterial" --output text > "$KEYPAIR_NAME.pem"
chmod 700 "${KEYPAIR_NAME}.pem"
echo "${KEYPAIR_NAME}.pem private key created!"
fi
@@ -67,7 +68,7 @@ CLUSTER_INFO=$(aws emr create-cluster \
--service-role EMR_DefaultRole \
--enable-debugging \
--release-label $EMR_VERSION \
- --log-uri "s3n://system-ds-logs/" \
+ --log-uri "s3://$BUCKET-logs/" \
--name "SystemDS cluster" \
--instance-groups '[{"InstanceCount":'${MASTER_INSTANCES_COUNT}',
"InstanceGroupType":"MASTER",
@@ -102,6 +103,7 @@ aws emr wait cluster-running --cluster-id $CLUSTER_ID
echo "Cluster info:"
export CLUSTER_URL=$(aws emr describe-cluster --cluster-id $CLUSTER_ID | jq
.Cluster.MasterPublicDnsName | tr -d '"')
-aws emr ssh --cluster-id $CLUSTER_ID --key-pair-file ${KEYPAIR_NAME}.pem
--command 'aws s3 cp s3://system-ds-bucket/target . --recursive --exclude "*"
--include "*DS.jar*"'
+aws emr ssh --cluster-id $CLUSTER_ID --key-pair-file ${KEYPAIR_NAME}.pem
--region $REGION \
+ --command 'aws s3 cp s3://system-ds-bucket/target . --recursive --exclude
"*" --include "*DS.jar*"'
echo "Spinup finished."
diff --git a/scripts/aws/systemds_cluster.config
b/scripts/aws/systemds_cluster.config
index 6fce6f0..a254bbc 100644
--- a/scripts/aws/systemds_cluster.config
+++ b/scripts/aws/systemds_cluster.config
@@ -23,6 +23,7 @@
KEYPAIR_NAME="SystemDSkeynamex"
REGION="us-east-1"
+BUCKET="systemds-bucket"
EMR_VERSION="emr-5.28.0"
INSTANCES_TYPE="m5.xlarge"
