(hbase) branch branch-2.6 updated: HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL writer (#5529)(#5539)(#5582)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new 634be120df9 HBASE-28031 TestClusterScopeQuotaThrottle is still failing
with broken WAL writer (#5529)(#5539)(#5582)
634be120df9 is described below
commit 634be120df992022c276a847f069c68057913ef3
Author: Duo Zhang
AuthorDate: Mon Nov 20 10:41:35 2023 +0800
HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL
writer (#5529)(#5539)(#5582)
Limit the scope for EnvironmentEdge injection
Add timeout for ThrottleQuotaTestUtil.triggerCacheRefresh
Signed-off-by: Guanghao Zhang
Signed-off-by: Bryan Beaudreault
(cherry picked from commit b1eccb364b30b14bdf475c339dbea12daa6804e2)
---
.../util/EnvironmentEdgeManagerTestHelper.java | 32 ++
.../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java | 107 +
2 files changed, 101 insertions(+), 38 deletions(-)
diff --git
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
index 684247248dc..73e7f1623ef 100644
---
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
+++
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java
@@ -33,4 +33,36 @@ public final class EnvironmentEdgeManagerTestHelper {
public static void injectEdge(EnvironmentEdge edge) {
EnvironmentEdgeManager.injectEdge(edge);
}
+
+ private static final class PackageEnvironmentEdgeWrapper implements
EnvironmentEdge {
+
+private final EnvironmentEdge delegate;
+
+private final String packageName;
+
+PackageEnvironmentEdgeWrapper(EnvironmentEdge delegate, String
packageName) {
+ this.delegate = delegate;
+ this.packageName = packageName;
+}
+
+@Override
+public long currentTime() {
+ StackTraceElement[] elements = new Exception().getStackTrace();
+ // the first element is us, the second one is EnvironmentEdgeManager, so
let's check the third
+ // one
+ if (elements.length > 2 &&
elements[2].getClassName().startsWith(packageName)) {
+return delegate.currentTime();
+ } else {
+return System.currentTimeMillis();
+ }
+}
+ }
+
+ /**
+ * Inject a {@link EnvironmentEdge} which only takes effect when calling
directly from the classes
+ * in the given package.
+ */
+ public static void injectEdgeForPackage(EnvironmentEdge edge, String
packageName) {
+injectEdge(new PackageEnvironmentEdgeWrapper(edge, packageName));
+ }
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
index d349fc25aa1..baf11df848c 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Objects;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
@@ -42,7 +43,9 @@ public final class ThrottleQuotaTestUtil {
private final static int REFRESH_TIME = 30 * 6;
static {
envEdge.setValue(EnvironmentEdgeManager.currentTime());
-EnvironmentEdgeManagerTestHelper.injectEdge(envEdge);
+// only active the envEdge for quotas package
+EnvironmentEdgeManagerTestHelper.injectEdgeForPackage(envEdge,
+ ThrottleQuotaTestUtil.class.getPackage().getName());
}
private ThrottleQuotaTestUtil() {
@@ -135,51 +138,79 @@ public final class ThrottleQuotaTestUtil {
RegionServerRpcQuotaManager quotaManager =
rst.getRegionServer().getRegionServerRpcQuotaManager();
QuotaCache quotaCache = quotaManager.getQuotaCache();
-
quotaCache.triggerCacheRefresh();
- // sleep for cache update
Thread.sleep(250);
-
- for (TableName table : tables) {
-quotaCache.getTableLimiter(table);
- }
-
- boolean isUpdated = false;
- while (!isUpdated) {
-quotaCache.triggerCacheRefresh();
-isUpdated = true;
-for (TableName table : tables) {
- boolean isBypass = true;
- if (userLimiter) {
-isBypass = quotaCache.getUserLimiter(User.getCurrent().getUGI(),
table).isBypass();
+ testUtil.waitFor(6, 250, new ExplainingPredicate() {
+
+@Override
+public boolean evaluate() throws Exception {
+
(hbase) branch branch-2 updated: HBASE-27814 Add support for dump and process metrics servlet in REST InfoServer (#5215)
This is an automated email from the ASF dual-hosted git repository.
nihaljain pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new a419b390cfa HBASE-27814 Add support for dump and process metrics
servlet in REST InfoServer (#5215)
a419b390cfa is described below
commit a419b390cfac90a91ca7082b6af6ddb8b189ecf1
Author: Nihal Jain
AuthorDate: Sun Jan 14 11:59:26 2024 +0530
HBASE-27814 Add support for dump and process metrics servlet in REST
InfoServer (#5215)
Other changes:
- Ensure info server stops during stop()
- Extract header and footer. This would fix the log level page layout for
rest web UI (See HBASE-20693)
- Add hostname in the landing page instead of just port similar to other
web UIs
Signed-off-by: Nick Dimiduk
(cherry picked from commit a683fcfbe8c5c84e58fa8670f4414f9d01ff43ed)
---
.../apache/hadoop/hbase/rest/RESTDumpServlet.java | 80 +
.../org/apache/hadoop/hbase/rest/RESTServer.java | 39 -
.../main/resources/hbase-webapps/rest/footer.jsp | 32
.../main/resources/hbase-webapps/rest/header.jsp | 74 +
.../resources/hbase-webapps/rest/processRest.jsp | 184 +
.../src/main/resources/hbase-webapps/rest/rest.jsp | 80 ++---
6 files changed, 422 insertions(+), 67 deletions(-)
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java
new file mode 100644
index 000..8bb306f7829
--- /dev/null
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rest;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.Date;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.http.HttpServer;
+import org.apache.hadoop.hbase.monitoring.StateDumpServlet;
+import org.apache.hadoop.hbase.util.LogMonitoring;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class RESTDumpServlet extends StateDumpServlet {
+ private static final long serialVersionUID = 1L;
+ private static final String LINE =
"===";
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response)
throws IOException {
+if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
request, response)) {
+ return;
+}
+
+RESTServer restServer = (RESTServer)
getServletContext().getAttribute(RESTServer.REST_SERVER);
+assert restServer != null : "No REST Server in context!";
+
+response.setContentType("text/plain");
+OutputStream os = response.getOutputStream();
+try (PrintWriter out = new PrintWriter(os)) {
+
+ out.println("REST Server status for " + restServer.getServerName() + "
as of " + new Date());
+
+ out.println("\n\nVersion Info:");
+ out.println(LINE);
+ dumpVersionInfo(out);
+
+ out.println("\n\nStacks:");
+ out.println(LINE);
+ out.flush();
+ PrintStream ps = new PrintStream(response.getOutputStream(), false,
"UTF-8");
+ Threads.printThreadInfo(ps, "");
+ ps.flush();
+
+ out.println("\n\nREST Server configuration:");
+ out.println(LINE);
+ Configuration conf = restServer.conf;
+ out.flush();
+ conf.writeXml(os);
+ os.flush();
+
+ out.println("\n\nLogs");
+ out.println(LINE);
+ long tailKb = getTailKbParam(request);
+ LogMonitoring.dumpTailOfLogs(out, tailKb);
+
+ out.flush();
+}
+ }
+}
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
index acf1c669d33..4479b8f4dc2 100644
---
(hbase) branch branch-3 updated: HBASE-27814 Add support for dump and process metrics servlet in REST InfoServer (#5215)
This is an automated email from the ASF dual-hosted git repository.
nihaljain pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 5129cbe3a6a HBASE-27814 Add support for dump and process metrics
servlet in REST InfoServer (#5215)
5129cbe3a6a is described below
commit 5129cbe3a6aa90e8e5112b65d007893bcec126a8
Author: Nihal Jain
AuthorDate: Sun Jan 14 11:59:26 2024 +0530
HBASE-27814 Add support for dump and process metrics servlet in REST
InfoServer (#5215)
Other changes:
- Ensure info server stops during stop()
- Extract header and footer. This would fix the log level page layout for
rest web UI (See HBASE-20693)
- Add hostname in the landing page instead of just port similar to other
web UIs
Signed-off-by: Nick Dimiduk
(cherry picked from commit a683fcfbe8c5c84e58fa8670f4414f9d01ff43ed)
---
.../apache/hadoop/hbase/rest/RESTDumpServlet.java | 80 +
.../org/apache/hadoop/hbase/rest/RESTServer.java | 39 -
.../main/resources/hbase-webapps/rest/footer.jsp | 32
.../main/resources/hbase-webapps/rest/header.jsp | 74 +
.../resources/hbase-webapps/rest/processRest.jsp | 184 +
.../src/main/resources/hbase-webapps/rest/rest.jsp | 80 ++---
6 files changed, 422 insertions(+), 67 deletions(-)
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java
new file mode 100644
index 000..8bb306f7829
--- /dev/null
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rest;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.Date;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.http.HttpServer;
+import org.apache.hadoop.hbase.monitoring.StateDumpServlet;
+import org.apache.hadoop.hbase.util.LogMonitoring;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class RESTDumpServlet extends StateDumpServlet {
+ private static final long serialVersionUID = 1L;
+ private static final String LINE =
"===";
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response)
throws IOException {
+if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
request, response)) {
+ return;
+}
+
+RESTServer restServer = (RESTServer)
getServletContext().getAttribute(RESTServer.REST_SERVER);
+assert restServer != null : "No REST Server in context!";
+
+response.setContentType("text/plain");
+OutputStream os = response.getOutputStream();
+try (PrintWriter out = new PrintWriter(os)) {
+
+ out.println("REST Server status for " + restServer.getServerName() + "
as of " + new Date());
+
+ out.println("\n\nVersion Info:");
+ out.println(LINE);
+ dumpVersionInfo(out);
+
+ out.println("\n\nStacks:");
+ out.println(LINE);
+ out.flush();
+ PrintStream ps = new PrintStream(response.getOutputStream(), false,
"UTF-8");
+ Threads.printThreadInfo(ps, "");
+ ps.flush();
+
+ out.println("\n\nREST Server configuration:");
+ out.println(LINE);
+ Configuration conf = restServer.conf;
+ out.flush();
+ conf.writeXml(os);
+ os.flush();
+
+ out.println("\n\nLogs");
+ out.println(LINE);
+ long tailKb = getTailKbParam(request);
+ LogMonitoring.dumpTailOfLogs(out, tailKb);
+
+ out.flush();
+}
+ }
+}
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
index 886c81dc668..42c00480526 100644
---
(hbase) branch master updated: HBASE-27814 Add support for dump and process metrics servlet in REST InfoServer (#5215)
This is an automated email from the ASF dual-hosted git repository.
nihaljain pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new a683fcfbe8c HBASE-27814 Add support for dump and process metrics
servlet in REST InfoServer (#5215)
a683fcfbe8c is described below
commit a683fcfbe8c5c84e58fa8670f4414f9d01ff43ed
Author: Nihal Jain
AuthorDate: Sun Jan 14 11:59:26 2024 +0530
HBASE-27814 Add support for dump and process metrics servlet in REST
InfoServer (#5215)
Other changes:
- Ensure info server stops during stop()
- Extract header and footer. This would fix the log level page layout for
rest web UI (See HBASE-20693)
- Add hostname in the landing page instead of just port similar to other
web UIs
Signed-off-by: Nick Dimiduk
---
.../apache/hadoop/hbase/rest/RESTDumpServlet.java | 80 +
.../org/apache/hadoop/hbase/rest/RESTServer.java | 39 -
.../main/resources/hbase-webapps/rest/footer.jsp | 32
.../main/resources/hbase-webapps/rest/header.jsp | 74 +
.../resources/hbase-webapps/rest/processRest.jsp | 184 +
.../src/main/resources/hbase-webapps/rest/rest.jsp | 80 ++---
6 files changed, 422 insertions(+), 67 deletions(-)
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java
new file mode 100644
index 000..8bb306f7829
--- /dev/null
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rest;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.Date;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.http.HttpServer;
+import org.apache.hadoop.hbase.monitoring.StateDumpServlet;
+import org.apache.hadoop.hbase.util.LogMonitoring;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class RESTDumpServlet extends StateDumpServlet {
+ private static final long serialVersionUID = 1L;
+ private static final String LINE =
"===";
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response)
throws IOException {
+if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
request, response)) {
+ return;
+}
+
+RESTServer restServer = (RESTServer)
getServletContext().getAttribute(RESTServer.REST_SERVER);
+assert restServer != null : "No REST Server in context!";
+
+response.setContentType("text/plain");
+OutputStream os = response.getOutputStream();
+try (PrintWriter out = new PrintWriter(os)) {
+
+ out.println("REST Server status for " + restServer.getServerName() + "
as of " + new Date());
+
+ out.println("\n\nVersion Info:");
+ out.println(LINE);
+ dumpVersionInfo(out);
+
+ out.println("\n\nStacks:");
+ out.println(LINE);
+ out.flush();
+ PrintStream ps = new PrintStream(response.getOutputStream(), false,
"UTF-8");
+ Threads.printThreadInfo(ps, "");
+ ps.flush();
+
+ out.println("\n\nREST Server configuration:");
+ out.println(LINE);
+ Configuration conf = restServer.conf;
+ out.flush();
+ conf.writeXml(os);
+ os.flush();
+
+ out.println("\n\nLogs");
+ out.println(LINE);
+ long tailKb = getTailKbParam(request);
+ LogMonitoring.dumpTailOfLogs(out, tailKb);
+
+ out.flush();
+}
+ }
+}
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
index 886c81dc668..42c00480526 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
+++
(hbase) branch branch-2.6 updated: HBASE-27402 Clone Scan in ClientScanner to avoid errors with Scan re-used (#5614)
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new cf6f9f9670e HBASE-27402 Clone Scan in ClientScanner to avoid errors
with Scan re-used (#5614)
cf6f9f9670e is described below
commit cf6f9f9670e15850b565fa3e37ae348b37e49039
Author: Bryan Beaudreault
AuthorDate: Sat Jan 13 15:03:23 2024 -0500
HBASE-27402 Clone Scan in ClientScanner to avoid errors with Scan re-used
(#5614)
Signed-off-by: Duo Zhang
---
.../hbase/client/ClientAsyncPrefetchScanner.java | 10 -
.../apache/hadoop/hbase/client/ClientScanner.java | 13 ---
.../hadoop/hbase/client/ClientSimpleScanner.java | 10 -
.../hbase/client/ConnectionImplementation.java | 9
.../org/apache/hadoop/hbase/client/HTable.java | 26 +-
.../hadoop/hbase/client/ReversedClientScanner.java | 6 ++---
.../hadoop/hbase/client/TestClientScanner.java | 2 +-
.../hbase/client/TestScannersFromClientSide.java | 19
8 files changed, 64 insertions(+), 31 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
index abd1267ffc4..0b892349b80 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
@@ -62,15 +62,15 @@ public class ClientAsyncPrefetchScanner extends
ClientSimpleScanner {
private final Condition notEmpty = lock.newCondition();
private final Condition notFull = lock.newCondition();
- public ClientAsyncPrefetchScanner(Configuration configuration, Scan scan,
TableName name,
-ClusterConnection connection, RpcRetryingCallerFactory rpcCallerFactory,
+ public ClientAsyncPrefetchScanner(Configuration configuration, Scan scan,
Scan scanForMetrics,
+TableName name, ClusterConnection connection, RpcRetryingCallerFactory
rpcCallerFactory,
RpcControllerFactory rpcControllerFactory, ExecutorService pool, int
scanReadRpcTimeout,
int scannerTimeout, int replicaCallTimeoutMicroSecondScan,
ConnectionConfiguration connectionConfiguration, Map
requestAttributes)
throws IOException {
-super(configuration, scan, name, connection, rpcCallerFactory,
rpcControllerFactory, pool,
- scanReadRpcTimeout, scannerTimeout, replicaCallTimeoutMicroSecondScan,
- connectionConfiguration, requestAttributes);
+super(configuration, scan, scanForMetrics, name, connection,
rpcCallerFactory,
+ rpcControllerFactory, pool, scanReadRpcTimeout, scannerTimeout,
+ replicaCallTimeoutMicroSecondScan, connectionConfiguration,
requestAttributes);
exceptionsQueue = new ConcurrentLinkedQueue<>();
final Context context = Context.current();
final Runnable runnable = context.wrap(new PrefetchRunnable());
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index ef8e4b0404f..df7f900830e 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -61,6 +61,12 @@ public abstract class ClientScanner extends
AbstractClientScanner {
private static final Logger LOG =
LoggerFactory.getLogger(ClientScanner.class);
protected final Scan scan;
+ // We clone the original client Scan to avoid modifying user object from
scan internals.
+ // The below scanForMetrics is the client's object, which we mutate only for
returning
+ // ScanMetrics.
+ // See https://issues.apache.org/jira/browse/HBASE-27402.
+ private final Scan scanForMetrics;
+
protected boolean closed = false;
// Current region scanner is against. Gets cleared if current region goes
// wonky: e.g. if it splits on us.
@@ -101,12 +107,13 @@ public abstract class ClientScanner extends
AbstractClientScanner {
* @param tableName The table that we wish to scan
* @param connection Connection identifying the cluster
*/
- public ClientScanner(final Configuration conf, final Scan scan, final
TableName tableName,
-ClusterConnection connection, RpcRetryingCallerFactory rpcFactory,
+ public ClientScanner(final Configuration conf, final Scan scan, final Scan
scanForMetrics,
+final TableName tableName, ClusterConnection connection,
RpcRetryingCallerFactory rpcFactory,
RpcControllerFactory controllerFactory, ExecutorService pool, int
scanReadRpcTimeout,
int scannerTimeout, int primaryOperationTimeout,
ConnectionConfiguration connectionConfiguration, Map
requestAttributes)
throws
(hbase) branch branch-2 updated: HBASE-27402 Clone Scan in ClientScanner to avoid errors with Scan re-used (#5614)
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new e6c01849fa8 HBASE-27402 Clone Scan in ClientScanner to avoid errors
with Scan re-used (#5614)
e6c01849fa8 is described below
commit e6c01849fa87ff6334ea27655cfd9fa9ef60c304
Author: Bryan Beaudreault
AuthorDate: Sat Jan 13 15:03:23 2024 -0500
HBASE-27402 Clone Scan in ClientScanner to avoid errors with Scan re-used
(#5614)
---
.../hbase/client/ClientAsyncPrefetchScanner.java | 10 -
.../apache/hadoop/hbase/client/ClientScanner.java | 13 ---
.../hadoop/hbase/client/ClientSimpleScanner.java | 10 -
.../hbase/client/ConnectionImplementation.java | 9
.../org/apache/hadoop/hbase/client/HTable.java | 26 +-
.../hadoop/hbase/client/ReversedClientScanner.java | 6 ++---
.../hadoop/hbase/client/TestClientScanner.java | 2 +-
.../hbase/client/TestScannersFromClientSide.java | 19
8 files changed, 64 insertions(+), 31 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
index abd1267ffc4..0b892349b80 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientAsyncPrefetchScanner.java
@@ -62,15 +62,15 @@ public class ClientAsyncPrefetchScanner extends
ClientSimpleScanner {
private final Condition notEmpty = lock.newCondition();
private final Condition notFull = lock.newCondition();
- public ClientAsyncPrefetchScanner(Configuration configuration, Scan scan,
TableName name,
-ClusterConnection connection, RpcRetryingCallerFactory rpcCallerFactory,
+ public ClientAsyncPrefetchScanner(Configuration configuration, Scan scan,
Scan scanForMetrics,
+TableName name, ClusterConnection connection, RpcRetryingCallerFactory
rpcCallerFactory,
RpcControllerFactory rpcControllerFactory, ExecutorService pool, int
scanReadRpcTimeout,
int scannerTimeout, int replicaCallTimeoutMicroSecondScan,
ConnectionConfiguration connectionConfiguration, Map
requestAttributes)
throws IOException {
-super(configuration, scan, name, connection, rpcCallerFactory,
rpcControllerFactory, pool,
- scanReadRpcTimeout, scannerTimeout, replicaCallTimeoutMicroSecondScan,
- connectionConfiguration, requestAttributes);
+super(configuration, scan, scanForMetrics, name, connection,
rpcCallerFactory,
+ rpcControllerFactory, pool, scanReadRpcTimeout, scannerTimeout,
+ replicaCallTimeoutMicroSecondScan, connectionConfiguration,
requestAttributes);
exceptionsQueue = new ConcurrentLinkedQueue<>();
final Context context = Context.current();
final Runnable runnable = context.wrap(new PrefetchRunnable());
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
index ef8e4b0404f..df7f900830e 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
@@ -61,6 +61,12 @@ public abstract class ClientScanner extends
AbstractClientScanner {
private static final Logger LOG =
LoggerFactory.getLogger(ClientScanner.class);
protected final Scan scan;
+ // We clone the original client Scan to avoid modifying user object from
scan internals.
+ // The below scanForMetrics is the client's object, which we mutate only for
returning
+ // ScanMetrics.
+ // See https://issues.apache.org/jira/browse/HBASE-27402.
+ private final Scan scanForMetrics;
+
protected boolean closed = false;
// Current region scanner is against. Gets cleared if current region goes
// wonky: e.g. if it splits on us.
@@ -101,12 +107,13 @@ public abstract class ClientScanner extends
AbstractClientScanner {
* @param tableName The table that we wish to scan
* @param connection Connection identifying the cluster
*/
- public ClientScanner(final Configuration conf, final Scan scan, final
TableName tableName,
-ClusterConnection connection, RpcRetryingCallerFactory rpcFactory,
+ public ClientScanner(final Configuration conf, final Scan scan, final Scan
scanForMetrics,
+final TableName tableName, ClusterConnection connection,
RpcRetryingCallerFactory rpcFactory,
RpcControllerFactory controllerFactory, ExecutorService pool, int
scanReadRpcTimeout,
int scannerTimeout, int primaryOperationTimeout,
ConnectionConfiguration connectionConfiguration, Map
requestAttributes)
throws IOException {
+this.scanForMetrics =
(hbase-site) branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 9b333f164c7 INFRA-10751 Empty commit 9b333f164c7 is described below commit 9b333f164c758b39e0d3168f79cc50f3bf87bab7 Author: jenkins AuthorDate: Sat Jan 13 14:43:31 2024 + INFRA-10751 Empty commit
(hbase) branch branch-3 updated: HBASE-28305 Add "Uncompressed StoreFileSize" column to the table.jsp (#5620)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new d864f753522 HBASE-28305 Add "Uncompressed StoreFileSize" column to the
table.jsp (#5620)
d864f753522 is described below
commit d864f753522a45cba915398804a2e705579c7b6b
Author: haosen chen <99318736+haosenc...@users.noreply.github.com>
AuthorDate: Sat Jan 13 17:42:57 2024 +0800
HBASE-28305 Add "Uncompressed StoreFileSize" column to the table.jsp (#5620)
Co-authored-by: Haosen Chen
Signed-off-by: Duo Zhang
(cherry picked from commit e3a0174e20542e661f787a874870b629a274daf5)
---
.../main/resources/hbase-webapps/master/table.jsp| 20
1 file changed, 20 insertions(+)
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 1d48a7561e1..f0599b7aa64 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -315,6 +315,7 @@
Region Server
ReadRequests
WriteRequests
+Uncompressed StoreFileSize
StorefileSize
Num.Storefiles
MemSize
@@ -338,6 +339,7 @@
String hostAndPort = "";
String readReq = "N/A";
String writeReq = "N/A";
+ String fileSizeUncompressed = ZEROMB;
String fileSize = ZEROMB;
String fileCount = "N/A";
String memSize = ZEROMB;
@@ -356,6 +358,10 @@
if (rSize > 0) {
fileSize = StringUtils.byteDesc((long) rSize);
}
+double rSizeUncompressed =
load.getUncompressedStoreFileSize().get(Size.Unit.BYTE);
+if (rSizeUncompressed > 0) {
+fileSizeUncompressed = StringUtils.byteDesc((long)
rSizeUncompressed);
+}
fileCount = String.format("%,1d",
load.getStoreFileCount());
double mSize = load.getMemStoreSize().get(Size.Unit.BYTE);
if (mSize > 0) {
@@ -370,6 +376,7 @@
<%=
StringEscapeUtils.escapeHtml4(hostAndPort) %>
<%= readReq%>
<%= writeReq%>
+<%= fileSizeUncompressed%>
<%= fileSize%>
<%= fileCount%>
<%= memSize%>
@@ -834,6 +841,7 @@
<%
long totalReadReq = 0;
long totalWriteReq = 0;
+ long totalSizeUncompressed = 0;
long totalSize = 0;
long totalStoreFileCount = 0;
long totalMemSize = 0;
@@ -844,6 +852,7 @@
long totalBlocksLocalWithSsdWeight = 0;
String totalCompactionProgress = "";
String totalMemSizeStr = ZEROMB;
+ String totalSizeUncompressedStr = ZEROMB;
String totalSizeStr = ZEROMB;
String totalLocality = "";
String totalLocalityForSsd = "";
@@ -865,6 +874,7 @@
if (regionMetrics != null) {
totalReadReq += regionMetrics.getReadRequestCount();
totalWriteReq += regionMetrics.getWriteRequestCount();
+ totalSizeUncompressed +=
regionMetrics.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE);
totalSize +=
regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE);
totalStoreFileCount += regionMetrics.getStoreFileCount();
totalMemSize +=
regionMetrics.getMemStoreSize().get(Size.Unit.MEGABYTE);
@@ -890,6 +900,9 @@
if (totalSize > 0) {
totalSizeStr = StringUtils.byteDesc(totalSize*1024l*1024);
}
+ if (totalSizeUncompressed > 0){
+totalSizeUncompressedStr =
StringUtils.byteDesc(totalSizeUncompressed*1024l*1024);
+ }
if (totalMemSize > 0) {
totalMemSizeStr = StringUtils.byteDesc(totalMemSize*1024l*1024);
}
@@ -920,6 +933,7 @@
Region Server
ReadRequests(<%= String.format("%,1d",
totalReadReq)%>)
WriteRequests(<%= String.format("%,1d",
totalWriteReq)%>)
+Uncompressed StoreFileSize(<%= totalSizeUncompressedStr
%>)
StorefileSize(<%= totalSizeStr %>)
Num.Storefiles(<%= String.format("%,1d",
totalStoreFileCount)%>)
MemSize(<%= totalMemSizeStr %>)
@@ -944,6 +958,7 @@
RegionMetrics load = hriEntry.getValue();
String readReq = "N/A";
String writeReq = "N/A";
+String regionSizeUncompressed = ZEROMB;
String regionSize = ZEROMB;
String fileCount = "N/A";
String memSize = ZEROMB;
@@ -951,6 +966,10 @@
if (load != null) {
readReq = String.format("%,1d", load.getReadRequestCount());
writeReq = String.format("%,1d", load.getWriteRequestCount());
+ double rSizeUncompressed =
(hbase) branch branch-3 updated: HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 5b88d160623 HBASE-28287 MOB HFiles are expired earlier than their
reference data (#5599)
5b88d160623 is described below
commit 5b88d160623da98beb9dfc092f319e310d3b1aba
Author: liuwenjing17 <154434091+liuwenjin...@users.noreply.github.com>
AuthorDate: Sat Jan 13 17:39:28 2024 +0800
HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
Co-authored-by: liuwenjing3
Signed-off-by: Duo Zhang
(cherry picked from commit 4d2b8f8de668f0b5938c7665c9730133cd48b8da)
---
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java | 1 +
.../java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java| 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index e04d67a0aaa..60f0f126ab6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -280,6 +280,7 @@ public final class MobUtils {
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
+calendar.set(Calendar.MILLISECOND, 0);
Date expireDate = calendar.getTime();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
index f282c6f9d8f..4bbc8868129 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
@@ -140,7 +140,8 @@ public class TestExpiredMobFileCleaner {
assertEquals("Before cleanup without delay 1", 1, firstFiles.length);
String firstFile = firstFiles[0].getPath().getName();
-ts = EnvironmentEdgeManager.currentTime() - 1 * secondsOfDay() * 1000; //
1 day before
+// 1.5 day before
+ts = (long) (EnvironmentEdgeManager.currentTime() - 1.5 * secondsOfDay() *
1000);
putKVAndFlush(table, row2, dummyData, ts);
FileStatus[] secondFiles =
TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
// now there are 2 mob files
(hbase) branch branch-2 updated: HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 25cbea2d210 HBASE-28287 MOB HFiles are expired earlier than their
reference data (#5599)
25cbea2d210 is described below
commit 25cbea2d210145e8f8c9acf2e59c3e75e6e6e670
Author: liuwenjing17 <154434091+liuwenjin...@users.noreply.github.com>
AuthorDate: Sat Jan 13 17:39:28 2024 +0800
HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
Co-authored-by: liuwenjing3
Signed-off-by: Duo Zhang
(cherry picked from commit 4d2b8f8de668f0b5938c7665c9730133cd48b8da)
---
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java | 1 +
.../java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java| 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index 25800c9ce86..b6b8be9d179 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -280,6 +280,7 @@ public final class MobUtils {
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
+calendar.set(Calendar.MILLISECOND, 0);
Date expireDate = calendar.getTime();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
index 3594190d065..6aeab33893f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
@@ -141,7 +141,8 @@ public class TestExpiredMobFileCleaner {
assertEquals("Before cleanup without delay 1", 1, firstFiles.length);
String firstFile = firstFiles[0].getPath().getName();
-ts = EnvironmentEdgeManager.currentTime() - 1 * secondsOfDay() * 1000; //
1 day before
+// 1.5 day before
+ts = (long) (EnvironmentEdgeManager.currentTime() - 1.5 * secondsOfDay() *
1000);
putKVAndFlush(table, row2, dummyData, ts);
FileStatus[] secondFiles =
TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
// now there are 2 mob files
(hbase) branch branch-2.6 updated: HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new 94ec75653ee HBASE-28287 MOB HFiles are expired earlier than their
reference data (#5599)
94ec75653ee is described below
commit 94ec75653eed12af1cdba094b1c7672b0c7a62d5
Author: liuwenjing17 <154434091+liuwenjin...@users.noreply.github.com>
AuthorDate: Sat Jan 13 17:39:28 2024 +0800
HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
Co-authored-by: liuwenjing3
Signed-off-by: Duo Zhang
(cherry picked from commit 4d2b8f8de668f0b5938c7665c9730133cd48b8da)
---
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java | 1 +
.../java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java| 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index 25800c9ce86..b6b8be9d179 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -280,6 +280,7 @@ public final class MobUtils {
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
+calendar.set(Calendar.MILLISECOND, 0);
Date expireDate = calendar.getTime();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
index 3594190d065..6aeab33893f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
@@ -141,7 +141,8 @@ public class TestExpiredMobFileCleaner {
assertEquals("Before cleanup without delay 1", 1, firstFiles.length);
String firstFile = firstFiles[0].getPath().getName();
-ts = EnvironmentEdgeManager.currentTime() - 1 * secondsOfDay() * 1000; //
1 day before
+// 1.5 day before
+ts = (long) (EnvironmentEdgeManager.currentTime() - 1.5 * secondsOfDay() *
1000);
putKVAndFlush(table, row2, dummyData, ts);
FileStatus[] secondFiles =
TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
// now there are 2 mob files
(hbase) branch branch-2.5 updated: HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 37aeb197893 HBASE-28287 MOB HFiles are expired earlier than their
reference data (#5599)
37aeb197893 is described below
commit 37aeb197893a17d2a22cd9c0fc7f9d17550d6092
Author: liuwenjing17 <154434091+liuwenjin...@users.noreply.github.com>
AuthorDate: Sat Jan 13 17:39:28 2024 +0800
HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
Co-authored-by: liuwenjing3
Signed-off-by: Duo Zhang
(cherry picked from commit 4d2b8f8de668f0b5938c7665c9730133cd48b8da)
---
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java | 1 +
.../java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java| 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index 25800c9ce86..b6b8be9d179 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -280,6 +280,7 @@ public final class MobUtils {
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
+calendar.set(Calendar.MILLISECOND, 0);
Date expireDate = calendar.getTime();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
index 3594190d065..6aeab33893f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
@@ -141,7 +141,8 @@ public class TestExpiredMobFileCleaner {
assertEquals("Before cleanup without delay 1", 1, firstFiles.length);
String firstFile = firstFiles[0].getPath().getName();
-ts = EnvironmentEdgeManager.currentTime() - 1 * secondsOfDay() * 1000; //
1 day before
+// 1.5 day before
+ts = (long) (EnvironmentEdgeManager.currentTime() - 1.5 * secondsOfDay() *
1000);
putKVAndFlush(table, row2, dummyData, ts);
FileStatus[] secondFiles =
TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
// now there are 2 mob files
(hbase) branch branch-2.4 updated: HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 38e61e915b6 HBASE-28287 MOB HFiles are expired earlier than their
reference data (#5599)
38e61e915b6 is described below
commit 38e61e915b6ea76d6ec8a0c428ce05c272859ccd
Author: liuwenjing17 <154434091+liuwenjin...@users.noreply.github.com>
AuthorDate: Sat Jan 13 17:39:28 2024 +0800
HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
Co-authored-by: liuwenjing3
Signed-off-by: Duo Zhang
(cherry picked from commit 4d2b8f8de668f0b5938c7665c9730133cd48b8da)
---
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java | 1 +
.../java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java| 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index be6af9fe572..962c99db851 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -296,6 +296,7 @@ public final class MobUtils {
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
+calendar.set(Calendar.MILLISECOND, 0);
Date expireDate = calendar.getTime();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
index bd9a872101a..14c6dd555fc 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
@@ -140,7 +140,8 @@ public class TestExpiredMobFileCleaner {
assertEquals("Before cleanup without delay 1", 1, firstFiles.length);
String firstFile = firstFiles[0].getPath().getName();
-ts = System.currentTimeMillis() - 1 * secondsOfDay() * 1000; // 1 day
before
+// 1.5 day before
+ts = (long) (System.currentTimeMillis() - 1.5 * secondsOfDay() * 1000);
putKVAndFlush(table, row2, dummyData, ts);
FileStatus[] secondFiles =
TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
// now there are 2 mob files
(hbase) branch master updated (4d2b8f8de66 -> e3a0174e205)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git from 4d2b8f8de66 HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599) add e3a0174e205 HBASE-28305 Add "Uncompressed StoreFileSize" column to the table.jsp (#5620) No new revisions were added by this update. Summary of changes: .../main/resources/hbase-webapps/master/table.jsp| 20 1 file changed, 20 insertions(+)
(hbase) branch master updated: HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 4d2b8f8de66 HBASE-28287 MOB HFiles are expired earlier than their
reference data (#5599)
4d2b8f8de66 is described below
commit 4d2b8f8de668f0b5938c7665c9730133cd48b8da
Author: liuwenjing17 <154434091+liuwenjin...@users.noreply.github.com>
AuthorDate: Sat Jan 13 17:39:28 2024 +0800
HBASE-28287 MOB HFiles are expired earlier than their reference data (#5599)
Co-authored-by: liuwenjing3
Signed-off-by: Duo Zhang
---
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java | 1 +
.../java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java| 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index e04d67a0aaa..60f0f126ab6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -280,6 +280,7 @@ public final class MobUtils {
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
+calendar.set(Calendar.MILLISECOND, 0);
Date expireDate = calendar.getTime();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
index f282c6f9d8f..4bbc8868129 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
@@ -140,7 +140,8 @@ public class TestExpiredMobFileCleaner {
assertEquals("Before cleanup without delay 1", 1, firstFiles.length);
String firstFile = firstFiles[0].getPath().getName();
-ts = EnvironmentEdgeManager.currentTime() - 1 * secondsOfDay() * 1000; //
1 day before
+// 1.5 day before
+ts = (long) (EnvironmentEdgeManager.currentTime() - 1.5 * secondsOfDay() *
1000);
putKVAndFlush(table, row2, dummyData, ts);
FileStatus[] secondFiles =
TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
// now there are 2 mob files
