This is an automated email from the ASF dual-hosted git repository. bertty pushed a commit to branch s3 in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit f5c9cdd900edae982acf7b53eb5333002cdb35c8 Author: Bertty Contreras-Rojas <[email protected]> AuthorDate: Tue Jul 5 11:57:51 2022 +0000 [WAYANG-S3] corrections Signed-off-by: bertty <[email protected]> --- .gitignore | 1 + .../main/java/org/apache/wayang/apps/sgd/SGDImpl.java | 7 ++++--- wayang-benchmark/pom.xml | 10 ++++++++++ .../org/apache/wayang/core/util/fs/S3FileSystem.java | 17 +++++++++++++++++ .../apache/wayang/spark/execution/SparkExecutor.java | 1 + 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3a9f55c4..210c8e3d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .idea .idea/ **/*.iml +**/maven-wrapper.jar # Maven **/target/ diff --git a/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java b/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java index d5331c64..7585e735 100644 --- a/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java +++ b/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java @@ -119,15 +119,16 @@ class Transform implements FunctionDescriptor.SerializableFunction<String, doubl @Override public double[] apply(String line) { - String[] pointStr = line.split(" "); + String[] pointStr = line.split(","); double[] point = new double[features + 1]; point[0] = Double.parseDouble(pointStr[0]); for (int i = 1; i < pointStr.length; i++) { - if (pointStr[i].equals("")) { +/* if (pointStr[i].equals("")) { continue; } String kv[] = pointStr[i].split(":", 2); - point[Integer.parseInt(kv[0]) - 1] = Double.parseDouble(kv[1]); + point[Integer.parseInt(kv[0]) - 1] = Double.parseDouble(kv[1]);*/ + point[i] = Double.parseDouble(pointStr[i]); } return point; } diff --git a/wayang-benchmark/pom.xml b/wayang-benchmark/pom.xml index 7df47adf..5cace142 100644 --- a/wayang-benchmark/pom.xml +++ b/wayang-benchmark/pom.xml @@ -64,6 +64,16 @@ <artifactId>hadoop-aws</artifactId> <version>3.1.2</version> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>3.2.3</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>3.1.2</version> + </dependency> </dependencies> <modules> diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java index 99fe3cb8..d175e632 100644 --- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java +++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.wayang.core.util.fs; import com.amazonaws.auth.AWSStaticCredentialsProvider; diff --git a/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java b/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java index 9aab425a..7100de44 100644 --- a/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java +++ b/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java @@ -195,6 +195,7 @@ public class SparkExecutor extends PushExecutorTemplate { @Override public void dispose() { super.dispose(); + this.sparkContextReference.get().stop(); this.sparkContextReference.noteDiscardedReference(true); }
