Repository: giraph Updated Branches: refs/heads/trunk 931569d58 -> 63b8c412f
GIRAPH-991 Update versions of guava and swift Summary: Changes in WorkerProgress are needed because newer version of thrift requires thrift classes to be final. Test Plan: mvn clean verify + run a bunch of test jobs Reviewers: maja.kabiljo Differential Revision: https://reviews.facebook.net/D32757 Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/63b8c412 Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/63b8c412 Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/63b8c412 Branch: refs/heads/trunk Commit: 63b8c412fbe731c7fc0a9d7dceb92ae955b553ae Parents: 931569d Author: Sergey Edunov <[email protected]> Authored: Tue Feb 3 14:47:15 2015 -0800 Committer: Sergey Edunov <[email protected]> Committed: Wed Feb 4 10:04:42 2015 -0800 ---------------------------------------------------------------------- CHANGELOG | 2 + .../giraph/job/CombinedWorkerProgress.java | 9 +- .../apache/giraph/worker/WorkerProgress.java | 57 ++----------- .../giraph/worker/WorkerProgressStats.java | 86 ++++++++++++++++++++ pom.xml | 2 +- 5 files changed, 101 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/63b8c412/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index 2e8e8bf..8da1efe 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ Giraph Change Log Release 1.2.0 - unreleased + GIRAPH-991: Update versions of guava and swift (edunov) + GIRAPH-987: Improve naming for ReduceOperation (ikabiljo via majakabiljo) GIRAPH-986: Add no-arg constructor to BasicSet (ikabiljo via edunov) http://git-wip-us.apache.org/repos/asf/giraph/blob/63b8c412/giraph-core/src/main/java/org/apache/giraph/job/CombinedWorkerProgress.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/job/CombinedWorkerProgress.java b/giraph-core/src/main/java/org/apache/giraph/job/CombinedWorkerProgress.java index e5fef8a..e931a99 100644 --- a/giraph-core/src/main/java/org/apache/giraph/job/CombinedWorkerProgress.java +++ b/giraph-core/src/main/java/org/apache/giraph/job/CombinedWorkerProgress.java @@ -18,22 +18,21 @@ package org.apache.giraph.job; +import com.google.common.collect.Iterables; import org.apache.giraph.conf.FloatConfOption; import org.apache.giraph.worker.WorkerProgress; +import org.apache.giraph.worker.WorkerProgressStats; import org.apache.hadoop.conf.Configuration; -import com.google.common.collect.Iterables; - -import java.text.DecimalFormat; - import javax.annotation.concurrent.NotThreadSafe; +import java.text.DecimalFormat; /** * Class which combines multiple workers' progresses to get overall * application progress */ @NotThreadSafe -public class CombinedWorkerProgress extends WorkerProgress { +public class CombinedWorkerProgress extends WorkerProgressStats { /** Decimal format which rounds numbers to two decimal places */ public static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("#.##"); /** http://git-wip-us.apache.org/repos/asf/giraph/blob/63b8c412/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgress.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgress.java b/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgress.java index 3c25cfe..eb543cd 100644 --- a/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgress.java +++ b/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgress.java @@ -18,10 +18,9 @@ package org.apache.giraph.worker; -import org.apache.giraph.utils.MemoryUtils; - import com.facebook.swift.codec.ThriftField; import com.facebook.swift.codec.ThriftStruct; +import org.apache.giraph.utils.MemoryUtils; import javax.annotation.concurrent.ThreadSafe; @@ -31,56 +30,16 @@ import javax.annotation.concurrent.ThreadSafe; */ @ThreadSafe @ThriftStruct -public class WorkerProgress { +public final class WorkerProgress extends WorkerProgressStats { /** Singleton instance for everyone to use */ private static final WorkerProgress INSTANCE = new WorkerProgress(); - /** Superstep which worker is executing, Long.MAX_VALUE if it's output */ - protected long currentSuperstep = -1; - - /** How many vertices were loaded until now */ - protected long verticesLoaded = 0; - /** How many vertex input splits were loaded until now */ - protected int vertexInputSplitsLoaded = 0; - /** Whether worker finished loading vertices */ - protected boolean loadingVerticesDone = false; - /** How many edges were loaded */ - protected long edgesLoaded = 0; - /** How many edge input splits were loaded until now */ - protected int edgeInputSplitsLoaded = 0; - /** Whether worker finished loading edges until now */ - protected boolean loadingEdgesDone = false; - - /** How many vertices are there to compute in current superstep */ - protected long verticesToCompute = 0; - /** How many vertices were computed in current superstep until now */ - protected long verticesComputed = 0; - /** How many partitions are there to compute in current superstep */ - protected int partitionsToCompute = 0; - /** How many partitions were computed in current superstep until now */ - protected int partitionsComputed = 0; - - /** Whether all compute supersteps are done */ - protected boolean computationDone = false; - - /** How many vertices are there to store */ - protected long verticesToStore = 0; - /** How many vertices were stored until now */ - protected long verticesStored = 0; - /** How many partitions are there to store */ - protected int partitionsToStore = 0; - /** How many partitions were stored until now */ - protected int partitionsStored = 0; - /** Whether worker finished storing data */ - protected boolean storingDone = false; - - /** Id of the mapper */ - protected int taskId; - - /** Free memory */ - protected double freeMemoryMB; - /** Fraction of memory that's free */ - protected double freeMemoryFraction; + /** + * Public constructor for thrift to create us. + * Please use WorkerProgress.get() to get the static instance. + */ + public WorkerProgress() { + } /** * Get singleton instance of WorkerProgress. http://git-wip-us.apache.org/repos/asf/giraph/blob/63b8c412/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgressStats.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgressStats.java b/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgressStats.java new file mode 100644 index 0000000..04ed2ea --- /dev/null +++ b/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgressStats.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.giraph.worker; + +import javax.annotation.concurrent.NotThreadSafe; + +/** + * Stats about a worker's progress + */ +@NotThreadSafe +public class WorkerProgressStats { + /** Superstep which worker is executing, Long.MAX_VALUE if it's output */ + protected long currentSuperstep = -1; + + /** How many vertices were loaded until now */ + protected long verticesLoaded = 0; + /** How many vertex input splits were loaded until now */ + protected int vertexInputSplitsLoaded = 0; + /** Whether worker finished loading vertices */ + protected boolean loadingVerticesDone = false; + /** How many edges were loaded */ + protected long edgesLoaded = 0; + /** How many edge input splits were loaded until now */ + protected int edgeInputSplitsLoaded = 0; + /** Whether worker finished loading edges until now */ + protected boolean loadingEdgesDone = false; + + /** How many vertices are there to compute in current superstep */ + protected long verticesToCompute = 0; + /** How many vertices were computed in current superstep until now */ + protected long verticesComputed = 0; + /** How many partitions are there to compute in current superstep */ + protected int partitionsToCompute = 0; + /** How many partitions were computed in current superstep until now */ + protected int partitionsComputed = 0; + + /** Whether all compute supersteps are done */ + protected boolean computationDone = false; + + /** How many vertices are there to store */ + protected long verticesToStore = 0; + /** How many vertices were stored until now */ + protected long verticesStored = 0; + /** How many partitions are there to store */ + protected int partitionsToStore = 0; + /** How many partitions were stored until now */ + protected int partitionsStored = 0; + /** Whether worker finished storing data */ + protected boolean storingDone = false; + + /** Id of the mapper */ + protected int taskId; + + /** Free memory */ + protected double freeMemoryMB; + /** Fraction of memory that's free */ + protected double freeMemoryFraction; + + public boolean isInputSuperstep() { + return currentSuperstep == -1; + } + + public boolean isComputeSuperstep() { + return currentSuperstep >= 0 && currentSuperstep < Long.MAX_VALUE; + } + + public boolean isOutputSuperstep() { + return currentSuperstep == Long.MAX_VALUE; + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/63b8c412/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index cf0e1f9..87792a3 100644 --- a/pom.xml +++ b/pom.xml @@ -291,7 +291,7 @@ under the License. <dep.commons-logging.version>1.1.1</dep.commons-logging.version> <dep.commons-io.version>2.1</dep.commons-io.version> <dep.commons-net.version>3.1</dep.commons-net.version> - <dep.facebook-swift.version>0.13.1</dep.facebook-swift.version> + <dep.facebook-swift.version>0.14.0</dep.facebook-swift.version> <dep.fasterxml-jackson.version>2.1.2</dep.fasterxml-jackson.version> <dep.fastutil.version>6.5.4</dep.fastutil.version> <dep.google.findbugs.version>2.0.2</dep.google.findbugs.version>
