arjun4084346 commented on code in PR #3563:
URL: https://github.com/apache/gobblin/pull/3563#discussion_r973392017
##########
gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/TimeAwareRecursiveCopyableDataset.java:
##########
@@ -134,9 +135,37 @@ protected List<FileStatus> getFilesAtPath(FileSystem fs,
Path path, PathFilter f
return recursivelyGetFilesAtDatePath(fs, path, "", fileFilter, 1,
startDate, endDate, formatter);
}
+ public Boolean checkPathDateTimeValidity(LocalDateTime startDate,
LocalDateTime endDate, String traversedDatePath) {
+ int[] startDateSplit = new int[] { startDate.getYear(),
startDate.getMonthOfYear(), startDate.getDayOfMonth(),
+ startDate.getHourOfDay(), startDate.getMinuteOfHour(),
startDate.getSecondOfMinute(), startDate.getMillisOfSecond() };
+ int[] endDateSplit = new int[] { endDate.getYear(),
endDate.getMonthOfYear(), endDate.getDayOfMonth(),
+ endDate.getHourOfDay(), endDate.getMinuteOfHour(),
endDate.getSecondOfMinute(), endDate.getMillisOfSecond() };
+
+ String[] traversedDatePathSplit = traversedDatePath.split("/");
+
+ // Only check the number of parameters that the traversedDatePath has
traversed through so far
+ for (int index = 0; index < traversedDatePathSplit.length; index++) {
+ try {
+ if (Integer.parseInt(traversedDatePathSplit[index]) <
startDateSplit[index] ||
+ Integer.parseInt(traversedDatePathSplit[index]) >
endDateSplit[index]) {
+ return false;
+ }
+ } catch (Exception e) {
+ return false;
+ }
+ }
+ return true;
+ }
+
private List<FileStatus> recursivelyGetFilesAtDatePath(FileSystem fs, Path
path, String traversedDatePath, PathFilter fileFilter,
int level, LocalDateTime startDate, LocalDateTime endDate,
DateTimeFormatter formatter) throws IOException {
List<FileStatus> fileStatuses = Lists.newArrayList();
+ if (!Objects.equals(traversedDatePath, "")) {
Review Comment:
Or for sake of uniformity, you can do `traversedDatePath.isEmpty()` as done
later in this method.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscr...@gobblin.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
