GeorgeJahad commented on code in PR #3746:
URL: https://github.com/apache/ozone/pull/3746#discussion_r984119772
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java:
##########
@@ -45,165 +41,71 @@
* add the current directory's objectID to the parent object's childDir field.
*
* Process() will write all OMDB updates to RocksDB.
- * The write logic is the same as above. For update action, we will treat it as
+ * Write logic is the same as above. For update action, we will treat it as
* delete old value first, and write updated value then.
*/
-public abstract class NSSummaryTask implements ReconOmTask {
+public class NSSummaryTask implements ReconOmTask {
private static final Logger LOG =
LoggerFactory.getLogger(NSSummaryTask.class);
- private final ReconNamespaceSummaryManager reconNamespaceSummaryManager;
+ private ReconNamespaceSummaryManager reconNamespaceSummaryManager;
+ private ReconOMMetadataManager reconOMMetadataManager;
+ private NSSummaryTaskWithFSO nsSummaryTaskWithFSO;
+ private NSSummaryTaskWithLegacy nsSummaryTaskWithLegacy;
@Inject
public NSSummaryTask(ReconNamespaceSummaryManager
- reconNamespaceSummaryManager) {
+ reconNamespaceSummaryManager,
+ ReconOMMetadataManager
+ reconOMMetadataManager) {
this.reconNamespaceSummaryManager = reconNamespaceSummaryManager;
+ this.reconOMMetadataManager = reconOMMetadataManager;
+ this.nsSummaryTaskWithFSO = new NSSummaryTaskWithFSO(
+ reconNamespaceSummaryManager, reconOMMetadataManager);
+ this.nsSummaryTaskWithLegacy = new NSSummaryTaskWithLegacy(
+ reconNamespaceSummaryManager, reconOMMetadataManager);
}
public ReconNamespaceSummaryManager getReconNamespaceSummaryManager() {
return reconNamespaceSummaryManager;
}
- public abstract String getTaskName();
-
- public abstract Pair<String, Boolean> process(OMUpdateEventBatch events);
-
- public abstract Pair<String, Boolean> reprocess(
- OMMetadataManager omMetadataManager);
-
- protected void writeNSSummariesToDB(Map<Long, NSSummary> nsSummaryMap)
- throws IOException {
- try (RDBBatchOperation rdbBatchOperation = new RDBBatchOperation()) {
- nsSummaryMap.keySet().forEach((Long key) -> {
- try {
- reconNamespaceSummaryManager.batchStoreNSSummaries(rdbBatchOperation,
- key, nsSummaryMap.get(key));
- } catch (IOException e) {
- LOG.error("Unable to write Namespace Summary data in Recon DB.",
- e);
- }
- });
- reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation);
- }
- }
-
- protected void handlePutKeyEvent(OmKeyInfo keyInfo, Map<Long,
- NSSummary> nsSummaryMap) throws IOException {
- long parentObjectId = keyInfo.getParentObjectID();
- // Try to get the NSSummary from our local map that maps NSSummaries to IDs
- NSSummary nsSummary = nsSummaryMap.get(parentObjectId);
- if (nsSummary == null) {
- // If we don't have it in this batch we try to get it from the DB
- nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
- }
- if (nsSummary == null) {
- // If we don't have it locally and in the DB we create a new instance
- // as this is a new ID
- nsSummary = new NSSummary();
- }
- int numOfFile = nsSummary.getNumOfFiles();
- long sizeOfFile = nsSummary.getSizeOfFiles();
- int[] fileBucket = nsSummary.getFileSizeBucket();
- nsSummary.setNumOfFiles(numOfFile + 1);
- long dataSize = keyInfo.getDataSize();
- nsSummary.setSizeOfFiles(sizeOfFile + dataSize);
- int binIndex = ReconUtils.getBinIndex(dataSize);
-
- ++fileBucket[binIndex];
- nsSummary.setFileSizeBucket(fileBucket);
- nsSummaryMap.put(parentObjectId, nsSummary);
+ public ReconOMMetadataManager getReconOMMetadataManager() {
+ return reconOMMetadataManager;
}
- protected void handlePutDirEvent(OmDirectoryInfo directoryInfo,
- Map<Long, NSSummary> nsSummaryMap)
- throws IOException {
- long parentObjectId = directoryInfo.getParentObjectID();
- long objectId = directoryInfo.getObjectID();
- // write the dir name to the current directory
- String dirName = directoryInfo.getName();
- // Try to get the NSSummary from our local map that maps NSSummaries to IDs
- NSSummary curNSSummary = nsSummaryMap.get(objectId);
- if (curNSSummary == null) {
- // If we don't have it in this batch we try to get it from the DB
- curNSSummary = reconNamespaceSummaryManager.getNSSummary(objectId);
- }
- if (curNSSummary == null) {
- // If we don't have it locally and in the DB we create a new instance
- // as this is a new ID
- curNSSummary = new NSSummary();
- }
- curNSSummary.setDirName(dirName);
- nsSummaryMap.put(objectId, curNSSummary);
-
- // Write the child dir list to the parent directory
- // Try to get the NSSummary from our local map that maps NSSummaries to IDs
- NSSummary nsSummary = nsSummaryMap.get(parentObjectId);
- if (nsSummary == null) {
- // If we don't have it in this batch we try to get it from the DB
- nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
- }
- if (nsSummary == null) {
- // If we don't have it locally and in the DB we create a new instance
- // as this is a new ID
- nsSummary = new NSSummary();
- }
- nsSummary.addChildDir(objectId);
- nsSummaryMap.put(parentObjectId, nsSummary);
+ @Override
+ public String getTaskName() {
+ return "NSSummaryTask";
}
- protected void handleDeleteKeyEvent(OmKeyInfo keyInfo,
- Map<Long, NSSummary> nsSummaryMap)
- throws IOException {
- long parentObjectId = keyInfo.getParentObjectID();
- // Try to get the NSSummary from our local map that maps NSSummaries to IDs
- NSSummary nsSummary = nsSummaryMap.get(parentObjectId);
- if (nsSummary == null) {
- // If we don't have it in this batch we try to get it from the DB
- nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
- }
-
- // Just in case the OmKeyInfo isn't correctly written.
- if (nsSummary == null) {
- LOG.error("The namespace table is not correctly populated.");
- return;
+ @Override
+ public Pair<String, Boolean> process(OMUpdateEventBatch events) {
Review Comment:
Also, please sure the test coverage results
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java:
##########
@@ -45,165 +41,71 @@
* add the current directory's objectID to the parent object's childDir field.
*
* Process() will write all OMDB updates to RocksDB.
- * The write logic is the same as above. For update action, we will treat it as
+ * Write logic is the same as above. For update action, we will treat it as
* delete old value first, and write updated value then.
*/
-public abstract class NSSummaryTask implements ReconOmTask {
+public class NSSummaryTask implements ReconOmTask {
private static final Logger LOG =
LoggerFactory.getLogger(NSSummaryTask.class);
- private final ReconNamespaceSummaryManager reconNamespaceSummaryManager;
+ private ReconNamespaceSummaryManager reconNamespaceSummaryManager;
+ private ReconOMMetadataManager reconOMMetadataManager;
+ private NSSummaryTaskWithFSO nsSummaryTaskWithFSO;
+ private NSSummaryTaskWithLegacy nsSummaryTaskWithLegacy;
@Inject
public NSSummaryTask(ReconNamespaceSummaryManager
- reconNamespaceSummaryManager) {
+ reconNamespaceSummaryManager,
+ ReconOMMetadataManager
+ reconOMMetadataManager) {
this.reconNamespaceSummaryManager = reconNamespaceSummaryManager;
+ this.reconOMMetadataManager = reconOMMetadataManager;
+ this.nsSummaryTaskWithFSO = new NSSummaryTaskWithFSO(
+ reconNamespaceSummaryManager, reconOMMetadataManager);
+ this.nsSummaryTaskWithLegacy = new NSSummaryTaskWithLegacy(
+ reconNamespaceSummaryManager, reconOMMetadataManager);
}
public ReconNamespaceSummaryManager getReconNamespaceSummaryManager() {
return reconNamespaceSummaryManager;
}
- public abstract String getTaskName();
-
- public abstract Pair<String, Boolean> process(OMUpdateEventBatch events);
-
- public abstract Pair<String, Boolean> reprocess(
- OMMetadataManager omMetadataManager);
-
- protected void writeNSSummariesToDB(Map<Long, NSSummary> nsSummaryMap)
- throws IOException {
- try (RDBBatchOperation rdbBatchOperation = new RDBBatchOperation()) {
- nsSummaryMap.keySet().forEach((Long key) -> {
- try {
- reconNamespaceSummaryManager.batchStoreNSSummaries(rdbBatchOperation,
- key, nsSummaryMap.get(key));
- } catch (IOException e) {
- LOG.error("Unable to write Namespace Summary data in Recon DB.",
- e);
- }
- });
- reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation);
- }
- }
-
- protected void handlePutKeyEvent(OmKeyInfo keyInfo, Map<Long,
- NSSummary> nsSummaryMap) throws IOException {
- long parentObjectId = keyInfo.getParentObjectID();
- // Try to get the NSSummary from our local map that maps NSSummaries to IDs
- NSSummary nsSummary = nsSummaryMap.get(parentObjectId);
- if (nsSummary == null) {
- // If we don't have it in this batch we try to get it from the DB
- nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
- }
- if (nsSummary == null) {
- // If we don't have it locally and in the DB we create a new instance
- // as this is a new ID
- nsSummary = new NSSummary();
- }
- int numOfFile = nsSummary.getNumOfFiles();
- long sizeOfFile = nsSummary.getSizeOfFiles();
- int[] fileBucket = nsSummary.getFileSizeBucket();
- nsSummary.setNumOfFiles(numOfFile + 1);
- long dataSize = keyInfo.getDataSize();
- nsSummary.setSizeOfFiles(sizeOfFile + dataSize);
- int binIndex = ReconUtils.getBinIndex(dataSize);
-
- ++fileBucket[binIndex];
- nsSummary.setFileSizeBucket(fileBucket);
- nsSummaryMap.put(parentObjectId, nsSummary);
+ public ReconOMMetadataManager getReconOMMetadataManager() {
+ return reconOMMetadataManager;
}
- protected void handlePutDirEvent(OmDirectoryInfo directoryInfo,
- Map<Long, NSSummary> nsSummaryMap)
- throws IOException {
- long parentObjectId = directoryInfo.getParentObjectID();
- long objectId = directoryInfo.getObjectID();
- // write the dir name to the current directory
- String dirName = directoryInfo.getName();
- // Try to get the NSSummary from our local map that maps NSSummaries to IDs
- NSSummary curNSSummary = nsSummaryMap.get(objectId);
- if (curNSSummary == null) {
- // If we don't have it in this batch we try to get it from the DB
- curNSSummary = reconNamespaceSummaryManager.getNSSummary(objectId);
- }
- if (curNSSummary == null) {
- // If we don't have it locally and in the DB we create a new instance
- // as this is a new ID
- curNSSummary = new NSSummary();
- }
- curNSSummary.setDirName(dirName);
- nsSummaryMap.put(objectId, curNSSummary);
-
- // Write the child dir list to the parent directory
- // Try to get the NSSummary from our local map that maps NSSummaries to IDs
- NSSummary nsSummary = nsSummaryMap.get(parentObjectId);
- if (nsSummary == null) {
- // If we don't have it in this batch we try to get it from the DB
- nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
- }
- if (nsSummary == null) {
- // If we don't have it locally and in the DB we create a new instance
- // as this is a new ID
- nsSummary = new NSSummary();
- }
- nsSummary.addChildDir(objectId);
- nsSummaryMap.put(parentObjectId, nsSummary);
+ @Override
+ public String getTaskName() {
+ return "NSSummaryTask";
}
- protected void handleDeleteKeyEvent(OmKeyInfo keyInfo,
- Map<Long, NSSummary> nsSummaryMap)
- throws IOException {
- long parentObjectId = keyInfo.getParentObjectID();
- // Try to get the NSSummary from our local map that maps NSSummaries to IDs
- NSSummary nsSummary = nsSummaryMap.get(parentObjectId);
- if (nsSummary == null) {
- // If we don't have it in this batch we try to get it from the DB
- nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
- }
-
- // Just in case the OmKeyInfo isn't correctly written.
- if (nsSummary == null) {
- LOG.error("The namespace table is not correctly populated.");
- return;
+ @Override
+ public Pair<String, Boolean> process(OMUpdateEventBatch events) {
Review Comment:
Also, please share the test coverage results
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
