ivandika3 commented on code in PR #4585:
URL: https://github.com/apache/ozone/pull/4585#discussion_r1325865352
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java:
##########
@@ -147,4 +157,158 @@ public long getLastSequenceNumberFromDB() {
public boolean isOmTablesInitialized() {
return omTablesInitialized;
}
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<OmVolumeArgs> listVolumes(String startKey,
+ int maxKeys) throws IOException {
+ List<OmVolumeArgs> result = Lists.newArrayList();
+
+ String volumeName;
+ OmVolumeArgs omVolumeArgs;
+
+ boolean startKeyIsEmpty = Strings.isNullOrEmpty(startKey);
+
+ // Unlike in {@link OmMetadataManagerImpl}, the volumes are queried
directly
+ // from the volume table (not through cache) since Recon does not use
+ // Table cache.
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmVolumeArgs>>
+ iterator = getVolumeTable().iterator()) {
+
+ while (iterator.hasNext() && result.size() < maxKeys) {
+ Table.KeyValue<String, OmVolumeArgs> kv = iterator.next();
+ omVolumeArgs = kv.getValue();
+ volumeName = omVolumeArgs.getVolume();
+
+
+ if (!startKeyIsEmpty) {
+ if (volumeName.equals(startKey)) {
+ startKeyIsEmpty = true;
+ }
+ continue;
+ }
+
+ result.add(omVolumeArgs);
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * Return all volumes in the file system.
+ * This method can be optimized by using username as a filter.
+ * @return a list of volume names under the system
+ */
+ @Override
+ public List<OmVolumeArgs> listVolumes() throws IOException {
+ return listVolumes(null, Integer.MAX_VALUE);
+ }
+
+ @Override
+ public boolean volumeExists(String volName) throws IOException {
+ String volDBKey = getVolumeKey(volName);
+ return getVolumeTable().getSkipCache(volDBKey) != null;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<OmBucketInfo> listBucketsUnderVolume(final String volumeName,
+ final String startBucket, final int maxNumOfBuckets) throws IOException
{
+ List<OmBucketInfo> result = new ArrayList<>();
+
+ // List all buckets if volume is empty
+ if (Strings.isNullOrEmpty(volumeName)) {
+ // startBucket requires the knowledge of the volume, for example
+ // there might be buckets with the same names under different
+ // volumes. Hence, startBucket will be ignored if
+ // volumeName is not specified
+ return listAllBuckets(maxNumOfBuckets);
+ }
+
+ if (!volumeExists(volumeName)) {
+ return result;
+ }
+
+ String startKey;
+ boolean skipStartKey = false;
+ if (StringUtil.isNotBlank(startBucket)) {
+ startKey = getBucketKey(volumeName, startBucket);
+ skipStartKey = true;
+ } else {
+ startKey = getBucketKey(volumeName, null);
+ }
+
+ String seekPrefix = getVolumeKey(volumeName + OM_KEY_PREFIX);
+
+ int currentCount = 0;
+
+ // Unlike in {@link OmMetadataManagerImpl}, the buckets are queried
directly
+ // from the volume table (not through cache) since Recon does not use
+ // Table cache.
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmBucketInfo>>
+ iterator = getBucketTable().iterator(seekPrefix)) {
+
+ while (currentCount < maxNumOfBuckets && iterator.hasNext()) {
+ Table.KeyValue<String, OmBucketInfo> kv =
+ iterator.next();
+
+ String key = kv.getKey();
+ OmBucketInfo omBucketInfo = kv.getValue();
+
+ if (omBucketInfo != null) {
+ if (key.equals(startKey) && skipStartKey) {
+ continue;
+ }
+
+ // We should return only the keys, whose keys match with prefix and
+ // the keys after the startBucket.
+ if (key.startsWith(seekPrefix) && key.compareTo(startKey) >= 0) {
+ result.add(omBucketInfo);
+ currentCount++;
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * List all buckets under a volume, if volume name is null, return all
buckets
+ * under the system.
+ * @param volumeName volume name without protocol prefix
+ * @return buckets under volume or all buckets if volume is null
+ * @throws IOException IOE
+ */
+ public List<OmBucketInfo> listBucketsUnderVolume(final String volumeName)
+ throws IOException {
+ return listBucketsUnderVolume(volumeName, null,
Review Comment:
> Hence, currently, my current suggestion is that we can have a drop down
that will set the limit parameter and initialize it to a good default (e.g.
5000) to bound the number of buckets/volumes fetched
As I mentioned in the earlier thread, currently it's very hard to implement
pagination in RocksDB without sacrificing sortability of different fields. So
to bound the number of objects returned, we set the `limit` when fetching from
the Recon API.
The pagination implementation can be implemented in different ticket, but I
think the most straightaway solution is to write a task to get all he volumes
/ buckets from the RocksDB, creating separate derby tables for volumes and
buckets, and probably wrapping it in JPA (supported natively) / JOOQ (might
need to write sql for pagination). In that way, we can achieve the expected
pagination capabilities easily using known SQL methods (e.g. offset and limit)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
