[
https://issues.apache.org/jira/browse/PHOENIX-5435?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17247445#comment-17247445
]
ASF GitHub Bot commented on PHOENIX-5435:
-----------------------------------------
ChinmaySKulkarni commented on a change in pull request #913:
URL: https://github.com/apache/phoenix/pull/913#discussion_r540433388
##########
File path:
phoenix-core/src/it/java/org/apache/phoenix/end2end/WALAnnotationIT.java
##########
@@ -0,0 +1,550 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.coprocessor.BaseWALObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.WALCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALKey;
+import org.apache.phoenix.compat.hbase.HbaseCompatCapabilities;
+import org.apache.phoenix.compat.hbase.coprocessor.CompatIndexRegionObserver;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.execute.MutationState;
+import org.apache.phoenix.hbase.index.IndexRegionObserver;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.PhoenixTestBuilder;
+import org.apache.phoenix.query.PhoenixTestBuilder.SchemaBuilder;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.util.MetaDataUtil;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Properties;
+
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHANGE_DETECTION_ENABLED;
+import static org.junit.Assert.assertNotNull;
+
+@RunWith(Parameterized.class)
+@Category(NeedsOwnMiniClusterTest.class)
+public class WALAnnotationIT extends BaseUniqueNamesOwnClusterIT {
+ private final boolean isImmutable;
+ private final boolean isMultiTenant;
+
+ // name is used by failsafe as file name in reports
+ @Parameterized.Parameters(name =
"WALAnnotationIT_isImmutable={0}_isMultiTenant={1}")
+ public static synchronized Collection<Object[]> data() {
+ return Arrays.asList(new Object[]{true, true}, new Object[]{true,
false},
+ new Object[]{false, true}, new Object[]{false, false});
+ }
+
+ public WALAnnotationIT(boolean isImmutable, boolean isMultiTenant) {
+ this.isImmutable = isImmutable;
+ this.isMultiTenant = isMultiTenant;
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = new HashMap<>(2);
+ props.put("hbase.coprocessor.wal.classes",
+ AnnotatedWALObserver.class.getName());
+ props.put(IndexRegionObserver.PHOENIX_APPEND_METADATA_TO_WAL, "true");
+ props.put(QueryServices.ENABLE_SERVER_UPSERT_SELECT, "true");
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Test
+ public void testSimpleUpsertAndDelete() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.hasPreWALAppend());
+ SchemaBuilder builder = new SchemaBuilder(getUrl());
+ boolean createGlobalIndex = false;
+ long ddlTimestamp = upsertAndDeleteHelper(builder, createGlobalIndex);
+ assertAnnotation(2, builder.getPhysicalTableName(false), null,
+ builder.getTableOptions().getSchemaName(),
+ builder.getDataOptions().getTableName(), PTableType.TABLE,
ddlTimestamp);
+ }
+
+ @Test
+ public void testNoAnnotationsIfChangeDetectionDisabled() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.hasPreWALAppend());
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(true);
+ SchemaBuilder builder = new SchemaBuilder(getUrl());
+ SchemaBuilder.TableOptions tableOptions = getTableOptions();
+ tableOptions.setChangeDetectionEnabled(false);
+ builder.withTableOptions(tableOptions).build();
+ PTable table = PhoenixRuntime.getTableNoCache(conn,
builder.getEntityTableName());
+ Assert.assertFalse("Change detection is enabled when it shouldn't
be!",
+ table.isChangeDetectionEnabled());
+ String upsertSql = "UPSERT INTO " + builder.getEntityTableName() +
" VALUES" +
+ " ('a', 'b', '2', 'bc', '3')";
+ conn.createStatement().execute(upsertSql);
+ List<Map<String, byte[]>> entries =
+
getEntriesForTable(TableName.valueOf(builder.getPhysicalTableName(false)));
+ Assert.assertEquals(0, entries.size());
+ //now flip to TRUE so we can test disabling it
+ String enableSql =
+ "ALTER TABLE " + builder.getEntityTableName() +
+ " SET " + CHANGE_DETECTION_ENABLED + "=TRUE";
+ conn.createStatement().execute(enableSql);
+ table = PhoenixRuntime.getTableNoCache(conn,
builder.getEntityTableName());
+ Assert.assertTrue("Change detection is disabled when it should be
enabled!",
+ table.isChangeDetectionEnabled());
+ //set to FALSE
+ String disableSql =
+ "ALTER TABLE " + builder.getEntityTableName() +
+ " SET " + CHANGE_DETECTION_ENABLED + "=FALSE";
+ conn.createStatement().execute(disableSql);
+ table = PhoenixRuntime.getTableNoCache(conn,
builder.getEntityTableName());
+ Assert.assertFalse("Change detection is enabled when it should be
disabled!",
+ table.isChangeDetectionEnabled());
+ //now upsert again
+ conn.createStatement().execute(upsertSql);
+ //check that we still didn't annotate anything
+ entries =
getEntriesForTable(TableName.valueOf(builder.getPhysicalTableName(false)));
+ Assert.assertEquals(0, entries.size());
+ }
+ }
+
+ @Test
+ public void testCantSetChangeDetectionOnIndex() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.hasPreWALAppend());
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ SchemaBuilder builder = new SchemaBuilder(getUrl());
+ builder.withTableDefaults().build();
+ try {
+ String badIndexSql =
+ "CREATE INDEX IDX_SHOULD_FAIL" + " ON " +
builder.getEntityTableName() +
+ "(COL1) "
+ + CHANGE_DETECTION_ENABLED + "=TRUE";
+ conn.createStatement().execute(badIndexSql);
+ Assert.fail("Didn't throw a SQLException for setting change
detection on an " +
+ "index at create time!");
+ } catch (SQLException se) {
+ TestUtil.assertSqlExceptionCode(
+
SQLExceptionCode.CHANGE_DETECTION_SUPPORTED_FOR_TABLES_AND_VIEWS_ONLY, se);
+ }
+ }
+ }
+
+ @Test
+ public void testUpsertAndDeleteWithGlobalIndex() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.hasPreWALAppend());
+ SchemaBuilder builder = new SchemaBuilder(getUrl());
+ boolean createGlobalIndex = true;
+ long ddlTimestamp = upsertAndDeleteHelper(builder, createGlobalIndex);
+ assertAnnotation(2, builder.getPhysicalTableName(false), null,
+ builder.getTableOptions().getSchemaName(),
+ builder.getDataOptions().getTableName(), PTableType.TABLE,
ddlTimestamp);
+ assertAnnotation(0, builder.getPhysicalTableIndexName(false),
+ null, builder.getTableOptions().getSchemaName(),
+
SchemaUtil.getTableNameFromFullName(builder.getEntityTableIndexName()),
+ PTableType.INDEX,
+ ddlTimestamp);
+ }
+
+ //Note that local secondary indexes aren't supported because they go in
the same WALEdit as the
Review comment:
Ok, got it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Annotate HBase WALs with Phoenix Metadata
> -----------------------------------------
>
> Key: PHOENIX-5435
> URL: https://issues.apache.org/jira/browse/PHOENIX-5435
> Project: Phoenix
> Issue Type: Sub-task
> Reporter: Geoffrey Jacoby
> Assignee: Geoffrey Jacoby
> Priority: Major
> Attachments: PHOENIX-5435-4.x.patch
>
>
> HBase write-ahead-logs (WALs) drive not only failure recovery, but HBase
> replication and some HBase backup frameworks. The WALs contain HBase-level
> metadata such as table and region, but lack Phoenix-level metadata. That
> means that it's quite difficult to build correct logic that needs to know
> about Phoenix-level constructs such as multi-tenancy, views, or indexes.
> HBASE-22622 and HBASE-22623 add the capacity for coprocessors to annotate
> extra key/value pairs of metadata into the HBase WAL. We should have the
> option to annotate the tuple <tenant_id, table-or-view-name, timestamp>, or
> some hashed way to reconstruct that tuple into the WAL. It should have a
> feature toggle so operators who don't need it don't bear the slight extra
> storage cost.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
