This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit ce5c99d8dbe445f2d738b10716ded1136bf9346a
Author: Matthias Boehm <mboe...@gmail.com>
AuthorDate: Thu Jul 4 12:34:34 2024 +0200
[SYSTEMDS-1780] Fix missing licenses and merge conflict artifacts
---
scripts/aws/run_systemds_script.sh | 4 ---
scripts/aws/spinup_systemds_cluster.sh | 32 ----------------------
scripts/aws/systemds_cluster.config | 10 -------
.../resource/cost/CostEstimationException.java | 19 +++++++++++++
.../apache/sysds/resource/cost/IOCostUtils.java | 19 +++++++++++++
.../org/apache/sysds/resource/cost/RDDStats.java | 19 +++++++++++++
6 files changed, 57 insertions(+), 46 deletions(-)
diff --git a/scripts/aws/run_systemds_script.sh
b/scripts/aws/run_systemds_script.sh
index 52de5d9227..48f5a59f66 100755
--- a/scripts/aws/run_systemds_script.sh
+++ b/scripts/aws/run_systemds_script.sh
@@ -57,9 +57,5 @@ echo "Waiting for the step to finish"
aws emr wait step-complete --cluster-id $CLUSTER_ID --step-id $STEP_ID
--region $REGION
aws emr ssh --cluster-id $CLUSTER_ID --key-pair-file ${KEYPAIR_NAME}.pem
--region $REGION --command "cat /mnt/var/log/hadoop/steps/$STEP_ID/stderr"
-<<<<<<< Upstream, based on 8e1e53bacbf444625272462c90e1e9c2bfaf206a
aws emr ssh --cluster-id $CLUSTER_ID --key-pair-file ${KEYPAIR_NAME}.pem
--region $REGION --command "cat /mnt/var/log/hadoop/steps/$STEP_ID/stdout"
-=======
-aws emr ssh --cluster-id $CLUSTER_ID --key-pair-file ${KEYPAIR_NAME}.pem
--region $REGION --command "cat /mnt/var/log/hadoop/steps/$STEP_ID/stdout"
->>>>>>> 8f04cfd Improve CP cost estimation
diff --git a/scripts/aws/spinup_systemds_cluster.sh
b/scripts/aws/spinup_systemds_cluster.sh
index 353f3c061a..72775e99fb 100755
--- a/scripts/aws/spinup_systemds_cluster.sh
+++ b/scripts/aws/spinup_systemds_cluster.sh
@@ -49,25 +49,11 @@ set_config "BUCKET" $BUCKET-$(((RANDOM % 999) + 1000))
#Source again to update the changes for the current session
source systemds_cluster.config
-<<<<<<< Upstream, based on 8e1e53bacbf444625272462c90e1e9c2bfaf206a
-<<<<<<< HEAD
#Create systemDS bucket
#LocationConstraint configuration required regions outside of us-east-1
if [ "$REGION" = "us-east-1" ]; then LOCATION_CONSTRAINT=""; else
LOCATION_CONSTRAINT="--create-bucket-configuration LocationConstraint=$REGION";
fi
aws s3api create-bucket --bucket $BUCKET --region $REGION $LOCATION_CONSTRAINT
&> /dev/null
aws s3api create-bucket --bucket $BUCKET-logs --region $REGION
$LOCATION_CONSTRAINT &> /dev/null
-=======
-#Create systemDS bucket (LocationConstraint configuration required regions
outside of us-east-1)
-aws s3api create-bucket --bucket $BUCKET --region $REGION
--create-bucket-configuration LocationConstraint=$REGION &> /dev/null
-aws s3api create-bucket --bucket $BUCKET-logs --region $REGION
--create-bucket-configuration LocationConstraint=$REGION &> /dev/null
->>>>>>> ef2ced4379 (Fixing AWS scripts for emr-7.0.0)
-=======
-#Create systemDS bucket
-#LocationConstraint configuration required regions outside of us-east-1
-if [ "$REGION" = "us-east-1" ]; then LOCATION_CONSTRAINT=""; else
LOCATION_CONSTRAINT="--create-bucket-configuration LocationConstraint=$REGION";
fi
-aws s3api create-bucket --bucket $BUCKET --region $REGION $LOCATION_CONSTRAINT
&> /dev/null
-aws s3api create-bucket --bucket $BUCKET-logs --region $REGION
$LOCATION_CONSTRAINT &> /dev/null
->>>>>>> 8f04cfd Improve CP cost estimation
# Upload Jar and scripts to s3
aws s3 sync $SYSTEMDS_TARGET_DIRECTORY s3://$BUCKET --exclude "*" --include
"*.dml" --include "*config.xml" --include "*DS.jar*"
@@ -79,31 +65,13 @@ if [ ! -f ${KEYPAIR_NAME}.pem ]; then
echo "${KEYPAIR_NAME}.pem private key created!"
fi
-<<<<<<< Upstream, based on 8e1e53bacbf444625272462c90e1e9c2bfaf206a
-<<<<<<< HEAD
#Get the first available subnet in the default VPC of the configured region
DEFAULT_SUBNET=$(aws ec2 describe-subnets --region $REGION \
--filter "Name=defaultForAz,Values=true" --query "Subnets[0].SubnetId"
--output text)
-#Create the cluster
-#Note: Ganglia not available since emr-6.15.0: exchanged with
AmazonCloudWatchAgent
-=======
-#Get the first available subnet in the default VPC of the region
-DEFAULT_SUBNET=$(aws ec2 describe-subnets --region eu-central-1 \
-=======
-#Get the first available subnet in the default VPC of the configured region
-DEFAULT_SUBNET=$(aws ec2 describe-subnets --region $REGION \
->>>>>>> f3b58ef AWS scripts for emr-7.0.0 update
- --filter "Name=defaultForAz,Values=true" --query "Subnets[0].SubnetId"
--output text)
#Create the cluster
-<<<<<<< Upstream, based on 8e1e53bacbf444625272462c90e1e9c2bfaf206a
-#Note: Ganglia not available since emr-6.15.0: exchange with
AmazonCloudWatchAgent
-#Note: '--availability-zone ANY' enforce assigning a default subnet to the
cluster
->>>>>>> ef2ced4379 (Fixing AWS scripts for emr-7.0.0)
-=======
#Note: Ganglia not available since emr-6.15.0: exchanged with
AmazonCloudWatchAgent
->>>>>>> f3b58ef AWS scripts for emr-7.0.0 update
CLUSTER_INFO=$(aws emr create-cluster \
--applications Name=AmazonCloudWatchAgent Name=Spark \
--ec2-attributes '{"KeyName":"'${KEYPAIR_NAME}'",
diff --git a/scripts/aws/systemds_cluster.config
b/scripts/aws/systemds_cluster.config
index 60465f00c9..8afed8d2bb 100644
--- a/scripts/aws/systemds_cluster.config
+++ b/scripts/aws/systemds_cluster.config
@@ -22,18 +22,8 @@
# Configuration
KEYPAIR_NAME="SystemDSkeynamex"
-<<<<<<< Upstream, based on 8e1e53bacbf444625272462c90e1e9c2bfaf206a
-<<<<<<< HEAD
REGION="us-east-1"
BUCKET=systemds-bucket
-=======
-REGION="eu-central-1"
-BUCKET=systemds-bucket-1612-1480-1991-1609
->>>>>>> ef2ced4379 (Fixing AWS scripts for emr-7.0.0)
-=======
-REGION="us-east-1"
-BUCKET=systemds-bucket
->>>>>>> 8f04cfd Improve CP cost estimation
EMR_VERSION="emr-7.0.0"
INSTANCES_TYPE="m5.xlarge"
diff --git
a/src/main/java/org/apache/sysds/resource/cost/CostEstimationException.java
b/src/main/java/org/apache/sysds/resource/cost/CostEstimationException.java
index ea04d5540a..39e750d4a3 100644
--- a/src/main/java/org/apache/sysds/resource/cost/CostEstimationException.java
+++ b/src/main/java/org/apache/sysds/resource/cost/CostEstimationException.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.sysds.resource.cost;
/**
diff --git a/src/main/java/org/apache/sysds/resource/cost/IOCostUtils.java
b/src/main/java/org/apache/sysds/resource/cost/IOCostUtils.java
index c0c94bfb36..1f92828afc 100644
--- a/src/main/java/org/apache/sysds/resource/cost/IOCostUtils.java
+++ b/src/main/java/org/apache/sysds/resource/cost/IOCostUtils.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.sysds.resource.cost;
import org.apache.sysds.common.Types;
diff --git a/src/main/java/org/apache/sysds/resource/cost/RDDStats.java
b/src/main/java/org/apache/sysds/resource/cost/RDDStats.java
index 468849fee8..d4091bac81 100644
--- a/src/main/java/org/apache/sysds/resource/cost/RDDStats.java
+++ b/src/main/java/org/apache/sysds/resource/cost/RDDStats.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.sysds.resource.cost;
import org.apache.sysds.conf.ConfigurationManager;
