ivandika3 commented on code in PR #4585:
URL: https://github.com/apache/ozone/pull/4585#discussion_r1325865352
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java:
##########
@@ -147,4 +157,158 @@ public long getLastSequenceNumberFromDB() {
public boolean isOmTablesInitialized() {
return omTablesInitialized;
}
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<OmVolumeArgs> listVolumes(String startKey,
+ int maxKeys) throws IOException {
+ List<OmVolumeArgs> result = Lists.newArrayList();
+
+ String volumeName;
+ OmVolumeArgs omVolumeArgs;
+
+ boolean startKeyIsEmpty = Strings.isNullOrEmpty(startKey);
+
+ // Unlike in {@link OmMetadataManagerImpl}, the volumes are queried
directly
+ // from the volume table (not through cache) since Recon does not use
+ // Table cache.
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmVolumeArgs>>
+ iterator = getVolumeTable().iterator()) {
+
+ while (iterator.hasNext() && result.size() < maxKeys) {
+ Table.KeyValue<String, OmVolumeArgs> kv = iterator.next();
+ omVolumeArgs = kv.getValue();
+ volumeName = omVolumeArgs.getVolume();
+
+
+ if (!startKeyIsEmpty) {
+ if (volumeName.equals(startKey)) {
+ startKeyIsEmpty = true;
+ }
+ continue;
+ }
+
+ result.add(omVolumeArgs);
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * Return all volumes in the file system.
+ * This method can be optimized by using username as a filter.
+ * @return a list of volume names under the system
+ */
+ @Override
+ public List<OmVolumeArgs> listVolumes() throws IOException {
+ return listVolumes(null, Integer.MAX_VALUE);
+ }
+
+ @Override
+ public boolean volumeExists(String volName) throws IOException {
+ String volDBKey = getVolumeKey(volName);
+ return getVolumeTable().getSkipCache(volDBKey) != null;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<OmBucketInfo> listBucketsUnderVolume(final String volumeName,
+ final String startBucket, final int maxNumOfBuckets) throws IOException
{
+ List<OmBucketInfo> result = new ArrayList<>();
+
+ // List all buckets if volume is empty
+ if (Strings.isNullOrEmpty(volumeName)) {
+ // startBucket requires the knowledge of the volume, for example
+ // there might be buckets with the same names under different
+ // volumes. Hence, startBucket will be ignored if
+ // volumeName is not specified
+ return listAllBuckets(maxNumOfBuckets);
+ }
+
+ if (!volumeExists(volumeName)) {
+ return result;
+ }
+
+ String startKey;
+ boolean skipStartKey = false;
+ if (StringUtil.isNotBlank(startBucket)) {
+ startKey = getBucketKey(volumeName, startBucket);
+ skipStartKey = true;
+ } else {
+ startKey = getBucketKey(volumeName, null);
+ }
+
+ String seekPrefix = getVolumeKey(volumeName + OM_KEY_PREFIX);
+
+ int currentCount = 0;
+
+ // Unlike in {@link OmMetadataManagerImpl}, the buckets are queried
directly
+ // from the volume table (not through cache) since Recon does not use
+ // Table cache.
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmBucketInfo>>
+ iterator = getBucketTable().iterator(seekPrefix)) {
+
+ while (currentCount < maxNumOfBuckets && iterator.hasNext()) {
+ Table.KeyValue<String, OmBucketInfo> kv =
+ iterator.next();
+
+ String key = kv.getKey();
+ OmBucketInfo omBucketInfo = kv.getValue();
+
+ if (omBucketInfo != null) {
+ if (key.equals(startKey) && skipStartKey) {
+ continue;
+ }
+
+ // We should return only the keys, whose keys match with prefix and
+ // the keys after the startBucket.
+ if (key.startsWith(seekPrefix) && key.compareTo(startKey) >= 0) {
+ result.add(omBucketInfo);
+ currentCount++;
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * List all buckets under a volume, if volume name is null, return all
buckets
+ * under the system.
+ * @param volumeName volume name without protocol prefix
+ * @return buckets under volume or all buckets if volume is null
+ * @throws IOException IOE
+ */
+ public List<OmBucketInfo> listBucketsUnderVolume(final String volumeName)
+ throws IOException {
+ return listBucketsUnderVolume(volumeName, null,
Review Comment:
> Hence, currently, my current suggestion is that we can have a drop down
that will set the limit parameter and initialize it to a good default (e.g.
5000) to bound the number of buckets/volumes fetched
As I mentioned in the earlier thread, currently it's currently very
difficult to implement pagination on top of RocksDB table without sacrificing
sortability and searchability of different fields. So currently to bound the
maximum number of objects returned, we set the `limit` when fetching from the
Recon API.
The pagination implementation can be implemented in different ticket (due to
the large change in this patch), but my current idea is to write a task to get
all the volumes / buckets from RocksDB tables, copying them to the separate
derby tables for volumes and buckets, and finally wrapping it in JPA
(pagination supported natively) / JOOQ (I'm not familiar with it, might need to
write sql for pagination). In that way, we can achieve the expected pagination
capabilities easily using known SQL methods (e.g. OFFSET and LIMIT) while still
retaining the ability to sort and search.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
