This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 645d33ea3e8 [SPARK-42685][CORE] Optimize Utils.bytesToString routines
645d33ea3e8 is described below
commit 645d33ea3e8e9ba41409f15b63544ea6b078fba4
Author: Alkis Evlogimenos <alkis.evlogime...@databricks.com>
AuthorDate: Fri Mar 10 11:27:41 2023 -0600
[SPARK-42685][CORE] Optimize Utils.bytesToString routines
### What changes were proposed in this pull request?
Optimize `Utils.bytesToString`. Arithmetic ops on `BigInt` and `BigDecimal`
are order(s) of magnitude slower than the ops on primitive types. Division is
an especially slow operation and it is used en masse here.
To avoid heating up the Earth while formatting byte counts for human
consumption we observe that most formatting operations are not in the 10s of
EiBs but on counts that fit in 64-bits and use (fastpath) 64-bit operations to
format them.
### Why are the changes needed?
Use of `Utils.bytesToString` is prevalent through the codebase and they are
mainly used in logs. If the logs are emitted then this becomes a heavyweight
operation.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
Closes #40301 from alkis/faster-byte-to-string.
Authored-by: Alkis Evlogimenos <alkis.evlogime...@databricks.com>
Signed-off-by: Sean Owen <sro...@gmail.com>
---
.../main/scala/org/apache/spark/util/Utils.scala | 43 ++++++++--------------
1 file changed, 16 insertions(+), 27 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 510486bc56b..d81e85ffe08 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -1305,41 +1305,30 @@ private[spark] object Utils extends Logging {
(JavaUtils.byteStringAsBytes(str) / 1024 / 1024).toInt
}
+ private[this] val siByteSizes =
+ Array(1L << 60, 1L << 50, 1L << 40, 1L << 30, 1L << 20, 1L << 10, 1)
+ private[this] val siByteSuffixes =
+ Array("EiB", "PiB", "TiB", "GiB", "MiB", "KiB", "B")
/**
* Convert a quantity in bytes to a human-readable string such as "4.0 MiB".
*/
- def bytesToString(size: Long): String = bytesToString(BigInt(size))
+ def bytesToString(size: Long): String = {
+ var i = 0
+ while (i < siByteSizes.length - 1 && size < 2 * siByteSizes(i)) i += 1
+ "%.1f %s".formatLocal(Locale.US, size.toDouble / siByteSizes(i),
siByteSuffixes(i))
+ }
def bytesToString(size: BigInt): String = {
val EiB = 1L << 60
- val PiB = 1L << 50
- val TiB = 1L << 40
- val GiB = 1L << 30
- val MiB = 1L << 20
- val KiB = 1L << 10
-
- if (size >= BigInt(1L << 11) * EiB) {
+ if (size.isValidLong) {
+ // Common case, most sizes fit in 64 bits and all ops on BigInt are
order(s) of magnitude
+ // slower than Long/Double.
+ bytesToString(size.toLong)
+ } else if (size < BigInt(2L << 10) * EiB) {
+ "%.1f EiB".formatLocal(Locale.US, BigDecimal(size) / EiB)
+ } else {
// The number is too large, show it in scientific notation.
BigDecimal(size, new MathContext(3, RoundingMode.HALF_UP)).toString() +
" B"
- } else {
- val (value, unit) = {
- if (size >= 2 * EiB) {
- (BigDecimal(size) / EiB, "EiB")
- } else if (size >= 2 * PiB) {
- (BigDecimal(size) / PiB, "PiB")
- } else if (size >= 2 * TiB) {
- (BigDecimal(size) / TiB, "TiB")
- } else if (size >= 2 * GiB) {
- (BigDecimal(size) / GiB, "GiB")
- } else if (size >= 2 * MiB) {
- (BigDecimal(size) / MiB, "MiB")
- } else if (size >= 2 * KiB) {
- (BigDecimal(size) / KiB, "KiB")
- } else {
- (BigDecimal(size), "B")
- }
- }
- "%.1f %s".formatLocal(Locale.US, value, unit)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
