keith-turner commented on code in PR #4415: URL: https://github.com/apache/accumulo/pull/4415#discussion_r1552593384
########## test/src/main/java/org/apache/accumulo/test/ample/TestAmple.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.ample; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedMap; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.ConditionalWriter; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.NewTableConfiguration; +import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletInformation; +import org.apache.accumulo.core.client.admin.TimeType; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.conf.SiteConfiguration; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.iterators.user.WholeRowIterator; +import org.apache.accumulo.core.metadata.AccumuloTable; +import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataTime; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.util.ColumnFQ; +import org.apache.accumulo.manager.Manager; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.metadata.ConditionalTabletsMutatorImpl; +import org.apache.accumulo.server.metadata.ServerAmpleImpl; +import org.apache.accumulo.server.metadata.TabletsMutatorImpl; +import org.apache.hadoop.io.Text; +import org.easymock.EasyMock; + +import com.google.common.base.Preconditions; +import com.google.common.collect.MoreCollectors; + +public class TestAmple { + + public static Ample create(ServerContext context, Map<DataLevel,String> tables) { + return new TestServerAmpleImpl(context, tables); + } + + public static class TestServerAmpleImpl extends ServerAmpleImpl { + + private final Map<DataLevel,String> tables; + + public TestServerAmpleImpl(ServerContext context, final Map<DataLevel,String> tables) { + super(context); Review Comment: This posting is for discussion, not sure if its actually workable so not suggesting we do this. Wondering if we added a constructor like the following to ServerAmpleImpl that abstracts the mapping if that would simplify things for TestAmple. Thinking this mapping function could flow into all the sub objects created by AmpleImple and ServerAmpleImpl. I am not sure if doing that would be less code than the protected methods that are being overridden or an improvement over the protected methods. ```java public ServerAmpleImpl(ServerContext context, Function<DataLevel, String> tableMapper) { super(context, tableMapper); this.context = context; } public ServerAmpleImpl(ServerContext context) { this(context, DataLevel::metaTable) } ``` If we had something like the above then could do the following in TestAmple ```suggestion super(context, dataLevel->{ Preconditions.checkArgument(dataLevel != ROOT); // not sure about this // TODO how to copy tables? return tables.get(dataLevel); }); ``` ########## test/src/main/java/org/apache/accumulo/test/ample/TestAmple.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.ample; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedMap; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.ConditionalWriter; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.NewTableConfiguration; +import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletInformation; +import org.apache.accumulo.core.client.admin.TimeType; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.conf.SiteConfiguration; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.iterators.user.WholeRowIterator; +import org.apache.accumulo.core.metadata.AccumuloTable; +import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataTime; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.util.ColumnFQ; +import org.apache.accumulo.manager.Manager; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.metadata.ConditionalTabletsMutatorImpl; +import org.apache.accumulo.server.metadata.ServerAmpleImpl; +import org.apache.accumulo.server.metadata.TabletsMutatorImpl; +import org.apache.hadoop.io.Text; +import org.easymock.EasyMock; + +import com.google.common.base.Preconditions; +import com.google.common.collect.MoreCollectors; + +public class TestAmple { + + public static Ample create(ServerContext context, Map<DataLevel,String> tables) { + return new TestServerAmpleImpl(context, tables); + } + + public static class TestServerAmpleImpl extends ServerAmpleImpl { + + private final Map<DataLevel,String> tables; + + public TestServerAmpleImpl(ServerContext context, final Map<DataLevel,String> tables) { + super(context); + this.tables = Map.copyOf(tables); + Preconditions.checkArgument(tables.containsKey(DataLevel.USER)); + } + + @Override + public TabletsMutator mutateTablets() { + return new TabletsMutatorImpl(getContext()) { + @Override + protected String getMetadataTableName() { + return TestServerAmpleImpl.this.getMetadataTableName(); + } + }; + } + + @Override + public TabletsMetadata.TableOptions readTablets() { + return TabletsMetadata.builder(getContext(), getMetadataTableName()); + } + + /** + * Create default metadata for a Table + * + * TODO: Add a way to pass in options for config + * + * @param tableId The id of the table to create metadata for + */ + public void createMetadata(TableId tableId) { + try (var tabletsMutator = mutateTablets()) { + var extent = new KeyExtent(tableId, null, null); + var tabletMutator = tabletsMutator.mutateTablet(extent); + String dirName = ServerColumnFamily.DEFAULT_TABLET_DIR_NAME; + tabletMutator.putPrevEndRow(extent.prevEndRow()); + tabletMutator.putDirName(dirName); + tabletMutator.putTime(new MetadataTime(0, TimeType.MILLIS)); + tabletMutator.putTabletAvailability(TabletAvailability.HOSTED); + tabletMutator.mutate(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + /** + * Create metadata for a table by copying existing metadata for the table from the metadata + * table in an existing Accumulo instance + * + * TODO: Add config parents (such as a way to include/exclude what is copied, etc) + * + * @param client The client to scan the existing accumulo metadata table + * @param tableId The id of the table to create metadata for + * @throws Exception thrown for any error on metadata creation + */ + public void createMetadataFromExisting(AccumuloClient client, TableId tableId) + throws Exception { + createMetadataFromExisting(client, tableId, Set.of()); + } + + public void createMetadataFromExisting(AccumuloClient client, TableId tableId, + Set<ColumnFQ> excludedColumnFq) throws Exception { + try (Scanner scanner = + client.createScanner(AccumuloTable.METADATA.tableName(), Authorizations.EMPTY)) { + scanner.setRange(TabletsSection.getRange(tableId)); + IteratorSetting iterSetting = new IteratorSetting(100, WholeRowIterator.class); + scanner.addScanIterator(iterSetting); + + try (BatchWriter bw = client.createBatchWriter(getMetadataTableName())) { + for (Entry<Key,Value> entry : scanner) { + final SortedMap<Key,Value> decodedRow = + WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()); + Text row = decodedRow.firstKey().getRow(); + Mutation m = new Mutation(row); + + decodedRow.entrySet().stream() + .filter(e -> !excludedColumnFq.contains(new ColumnFQ(e.getKey()))).forEach(e -> { + m.put(e.getKey().getColumnFamily(), e.getKey().getColumnQualifier(), + e.getKey().getColumnVisibilityParsed(), e.getKey().getTimestamp(), + e.getValue()); + }); + bw.addMutation(m); + } + } + } + } + + public ConditionalTabletsMutator + conditionallyMutateTablets(ConditionalWriterInterceptor interceptor) { Review Comment: Having this interceptor will be great for testing. Need some way to use it when code only has a handle to the Ample interface. Wonder if we could pass something like `Supplier< ConditionalWriterInterceptor> interceptorFactory` to the TestAmple constructor. Then could do the following in TestAmple ```java @Override public ConditionalTabletsMutator conditionallyMutateTablets() { return conditionallyMutateTablets(interceptorFactory.get()); } ``` ########## test/src/main/java/org/apache/accumulo/test/ample/TestAmple.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.ample; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedMap; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.ConditionalWriter; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.NewTableConfiguration; +import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletInformation; +import org.apache.accumulo.core.client.admin.TimeType; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.conf.SiteConfiguration; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.iterators.user.WholeRowIterator; +import org.apache.accumulo.core.metadata.AccumuloTable; +import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataTime; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.util.ColumnFQ; +import org.apache.accumulo.manager.Manager; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.metadata.ConditionalTabletsMutatorImpl; +import org.apache.accumulo.server.metadata.ServerAmpleImpl; +import org.apache.accumulo.server.metadata.TabletsMutatorImpl; +import org.apache.hadoop.io.Text; +import org.easymock.EasyMock; + +import com.google.common.base.Preconditions; +import com.google.common.collect.MoreCollectors; + +public class TestAmple { + + public static Ample create(ServerContext context, Map<DataLevel,String> tables) { + return new TestServerAmpleImpl(context, tables); + } + + public static class TestServerAmpleImpl extends ServerAmpleImpl { + + private final Map<DataLevel,String> tables; + + public TestServerAmpleImpl(ServerContext context, final Map<DataLevel,String> tables) { + super(context); + this.tables = Map.copyOf(tables); + Preconditions.checkArgument(tables.containsKey(DataLevel.USER)); + } + + @Override + public TabletsMutator mutateTablets() { + return new TabletsMutatorImpl(getContext()) { + @Override + protected String getMetadataTableName() { + return TestServerAmpleImpl.this.getMetadataTableName(); + } + }; + } + + @Override + public TabletsMetadata.TableOptions readTablets() { + return TabletsMetadata.builder(getContext(), getMetadataTableName()); + } + + /** + * Create default metadata for a Table + * + * TODO: Add a way to pass in options for config + * + * @param tableId The id of the table to create metadata for + */ + public void createMetadata(TableId tableId) { + try (var tabletsMutator = mutateTablets()) { + var extent = new KeyExtent(tableId, null, null); + var tabletMutator = tabletsMutator.mutateTablet(extent); + String dirName = ServerColumnFamily.DEFAULT_TABLET_DIR_NAME; + tabletMutator.putPrevEndRow(extent.prevEndRow()); + tabletMutator.putDirName(dirName); + tabletMutator.putTime(new MetadataTime(0, TimeType.MILLIS)); + tabletMutator.putTabletAvailability(TabletAvailability.HOSTED); + tabletMutator.mutate(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + /** + * Create metadata for a table by copying existing metadata for the table from the metadata + * table in an existing Accumulo instance + * + * TODO: Add config parents (such as a way to include/exclude what is copied, etc) Review Comment: Could pass a `BiPredicate<Key, Value>` to the function. Not sure if I have ever used BiPredicate, this seems like the perfect place for it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@accumulo.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
