keith-turner commented on code in PR #4691:
URL: https://github.com/apache/accumulo/pull/4691#discussion_r1676218729
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -209,4 +223,37 @@ static void upgradeDataFileCF(final Key key, final Value
value, final Mutation m
}
}
+ public void removeScanServerRange(ServerContext context, String tableName) {
+ log.info("Removing Scan Server Range from table {}", tableName);
+ try (BatchDeleter batchDeleter =
+ context.createBatchDeleter(tableName, Authorizations.EMPTY, 4)) {
+ batchDeleter.setRanges(OLD_SCAN_SERVERS_RANGES);
+ batchDeleter.delete();
+ } catch (TableNotFoundException | MutationsRejectedException e) {
+ throw new RuntimeException(e);
+ }
+ log.info("Scan Server Ranges {} removed from table {}",
OLD_SCAN_SERVERS_RANGES, tableName);
+ }
+
+ public void createScanServerRefTable(ServerContext context) {
+ ZooKeeperInitializer zkInit = new ZooKeeperInitializer();
+ zkInit.initScanRefTableState(context);
+
+ try {
+ FileSystemInitializer initializer = new FileSystemInitializer(
+ new InitialConfiguration(context.getHadoopConf(),
context.getSiteConfiguration()));
+ FileSystemInitializer.InitialTablet scanRefTablet =
initializer.createScanRefTablet(context);
+ // Add references to the Metadata Table
+ try (BatchWriter writer =
context.createBatchWriter(AccumuloTable.METADATA.tableName())) {
+ writer.addMutation(scanRefTablet.createMutation());
+ writer.flush();
Review Comment:
Could drop the flush, it will happen when try block closes the writer.
```suggestion
```
##########
server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java:
##########
@@ -62,81 +60,109 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-class FileSystemInitializer {
+public class FileSystemInitializer {
private static final String TABLE_TABLETS_TABLET_DIR = "table_info";
private static final Logger log =
LoggerFactory.getLogger(FileSystemInitializer.class);
+ private static final Text SPLIT_POINT =
+ MetadataSchema.TabletsSection.getRange().getEndKey().getRow();
// config only for root table
private final InitialConfiguration initConfig;
- FileSystemInitializer(InitialConfiguration initConfig, ZooReaderWriter zoo,
InstanceId uuid) {
+ public FileSystemInitializer(InitialConfiguration initConfig) {
this.initConfig = initConfig;
}
- private static class Tablet {
+ public static class InitialTablet {
TableId tableId;
String dirName;
- Text prevEndRow, endRow;
+ Text prevEndRow, endRow, extent;
String[] files;
- Tablet(TableId tableId, String dirName, Text prevEndRow, Text endRow,
String... files) {
+ InitialTablet(TableId tableId, String dirName, Text prevEndRow, Text
endRow, String... files) {
this.tableId = tableId;
this.dirName = dirName;
this.prevEndRow = prevEndRow;
this.endRow = endRow;
this.files = files;
+ this.extent = new
Text(MetadataSchema.TabletsSection.encodeRow(this.tableId, this.endRow));
}
+
+ private TreeMap<Key,Value> createEntries() {
+ TreeMap<Key,Value> sorted = new TreeMap<>();
+ Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue();
+ sorted.put(new Key(this.extent, DIRECTORY_COLUMN.getColumnFamily(),
+ DIRECTORY_COLUMN.getColumnQualifier(), 0), new Value(this.dirName));
+ sorted.put(
+ new Key(this.extent, TIME_COLUMN.getColumnFamily(),
TIME_COLUMN.getColumnQualifier(), 0),
+ new Value(new MetadataTime(0, TimeType.LOGICAL).encode()));
+ sorted.put(
+ new Key(this.extent, PREV_ROW_COLUMN.getColumnFamily(),
+ PREV_ROW_COLUMN.getColumnQualifier(), 0),
+
MetadataSchema.TabletsSection.TabletColumnFamily.encodePrevEndRow(this.prevEndRow));
+ for (String file : this.files) {
+ var col =
+ new
ColumnFQ(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME, new
Text(file));
+ sorted.put(new Key(extent, col.getColumnFamily(),
col.getColumnQualifier()), EMPTY_SIZE);
Review Comment:
need to set the 0 timestamp, otherwise I think it defaults to max long
creating an entry that can never be deleted or overwritten
```suggestion
sorted.put(new Key(extent, col.getColumnFamily(),
col.getColumnQualifier(),0), EMPTY_SIZE);
```
##########
server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java:
##########
@@ -194,28 +215,19 @@ private void createMetadataFile(VolumeManager volmanager,
String fileName,
.forFile(file, fs, fs.getConf(),
cs).withTableConfiguration(conf).build();
tabletWriter.startDefaultLocalityGroup();
- for (Map.Entry<Key,Value> entry : sorted.entrySet()) {
- tabletWriter.append(entry.getKey(), entry.getValue());
+ for (InitialTablet initialTablet : initialTablets) {
+ // sort file contents in memory, then play back to the file
+ for (Map.Entry<Key,Value> entry :
initialTablet.createEntries().entrySet()) {
+ tabletWriter.append(entry.getKey(), entry.getValue());
+ }
Review Comment:
Still need to sort the key values prior to append them to file. The file
expects data appended to be in sorted order and will throw an exception if its
not. The order of the InitialTablets list may or may not produce sorted data.
It probably is producing sorted data at the moment if init is running w/o
issue.
```suggestion
TreeMap<Key,Value> sorted = new TreeMap<>();
for (InitialTablet initialTablet : initialTablets) {
// sort file contents in memory, then play back to the file
sorted.putAll( initialTablet.createEntries());
}
for (Map.Entry<Key,Value> entry : sorted.entrySet()) {
tabletWriter.append(entry.getKey(), entry.getValue());
}
```
##########
server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java:
##########
@@ -62,81 +60,109 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-class FileSystemInitializer {
+public class FileSystemInitializer {
private static final String TABLE_TABLETS_TABLET_DIR = "table_info";
private static final Logger log =
LoggerFactory.getLogger(FileSystemInitializer.class);
+ private static final Text SPLIT_POINT =
+ MetadataSchema.TabletsSection.getRange().getEndKey().getRow();
// config only for root table
private final InitialConfiguration initConfig;
- FileSystemInitializer(InitialConfiguration initConfig, ZooReaderWriter zoo,
InstanceId uuid) {
+ public FileSystemInitializer(InitialConfiguration initConfig) {
this.initConfig = initConfig;
}
- private static class Tablet {
+ public static class InitialTablet {
TableId tableId;
String dirName;
- Text prevEndRow, endRow;
+ Text prevEndRow, endRow, extent;
String[] files;
- Tablet(TableId tableId, String dirName, Text prevEndRow, Text endRow,
String... files) {
+ InitialTablet(TableId tableId, String dirName, Text prevEndRow, Text
endRow, String... files) {
this.tableId = tableId;
this.dirName = dirName;
this.prevEndRow = prevEndRow;
this.endRow = endRow;
this.files = files;
+ this.extent = new
Text(MetadataSchema.TabletsSection.encodeRow(this.tableId, this.endRow));
}
+
+ private TreeMap<Key,Value> createEntries() {
+ TreeMap<Key,Value> sorted = new TreeMap<>();
+ Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue();
+ sorted.put(new Key(this.extent, DIRECTORY_COLUMN.getColumnFamily(),
+ DIRECTORY_COLUMN.getColumnQualifier(), 0), new Value(this.dirName));
+ sorted.put(
+ new Key(this.extent, TIME_COLUMN.getColumnFamily(),
TIME_COLUMN.getColumnQualifier(), 0),
+ new Value(new MetadataTime(0, TimeType.LOGICAL).encode()));
+ sorted.put(
+ new Key(this.extent, PREV_ROW_COLUMN.getColumnFamily(),
+ PREV_ROW_COLUMN.getColumnQualifier(), 0),
+
MetadataSchema.TabletsSection.TabletColumnFamily.encodePrevEndRow(this.prevEndRow));
+ for (String file : this.files) {
+ var col =
+ new
ColumnFQ(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME, new
Text(file));
+ sorted.put(new Key(extent, col.getColumnFamily(),
col.getColumnQualifier()), EMPTY_SIZE);
+ }
+ return sorted;
+ }
+
+ public Mutation createMutation() {
+ Mutation mutation = new Mutation(this.extent);
+ for (Map.Entry<Key,Value> entry : createEntries().entrySet()) {
+ mutation.put(entry.getKey().getColumnFamily(),
entry.getKey().getColumnQualifier(),
+ entry.getKey().getTimestamp(), entry.getValue());
Review Comment:
Should not set the timestamp here. When its not set the server side will
set it using the latest timestamp for the tablet. Each table has a timestamp
for the next insert that it increments on each insert.
```suggestion
mutation.put(entry.getKey().getColumnFamily(),
entry.getKey().getColumnQualifier(),
entry.getValue());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
