[
https://issues.apache.org/jira/browse/GOBBLIN-1001?focusedWorklogId=360428&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-360428
]
ASF GitHub Bot logged work on GOBBLIN-1001:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 16/Dec/19 19:01
Start Date: 16/Dec/19 19:01
Worklog Time Spent: 10m
Work Description: autumnust commented on pull request #2846:
[GOBBLIN-1001] Implement TimePartitionGlobFinder
URL: https://github.com/apache/incubator-gobblin/pull/2846#discussion_r358398316
##########
File path:
gobblin-data-management/src/main/java/org/apache/gobblin/data/management/dataset/TimePartitionGlobFinder.java
##########
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.data.management.dataset;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import lombok.extern.slf4j.Slf4j;
+
+import
org.apache.gobblin.data.management.retention.profile.ConfigurableGlobDatasetFinder;
+import org.apache.gobblin.dataset.DatasetsFinder;
+import org.apache.gobblin.dataset.FileSystemDataset;
+import org.apache.gobblin.time.TimeIterator;
+import org.apache.gobblin.util.PathUtils;
+
+
+/**
+ * A {@link TimePartitionGlobFinder} finds all dataset time partitions within
time window
+ * [current time - look back time, current time]. It derives an efficient
dataset partition pattern based
+ * on the time window and a supported {@value #TIME_FORMAT}.
+ *
+ * <p> If {@value #ENABLE_VIRTUAL_PARTITION} is set, it will create virtual
{@link SimpleFileSystemDataset}
+ * instances if a partition within the time window doesn't exist
+ */
+@Slf4j
+public class TimePartitionGlobFinder implements
DatasetsFinder<FileSystemDataset> {
+ private static final String CONF_PREFIX = "timePartitionGlobFinder.";
+
+ public static final String PARTITION_PREFIX = CONF_PREFIX +
"partitionPrefix";
+ public static final String TIME_FORMAT = CONF_PREFIX + "timeFormat";
+ public static final String ENABLE_VIRTUAL_PARTITION = CONF_PREFIX +
"enableVirtualPartition";
+ /**
+ * Options are enumerated in {@link
org.apache.gobblin.time.TimeIterator.Granularity}
+ */
+ public static final String GRANULARITY = CONF_PREFIX + "granularity";
+ public static final String TIME_ZONE = CONF_PREFIX + "timeZone";
+ public static final String LOOKBACK_SPEC = CONF_PREFIX + "lookbackSpec";
+
+ private static final String DEFAULT_TIME_ZONE = "America/Los_Angeles";
+
+ private static final Pattern SUPPORTED_TIME_FORMAT =
Pattern.compile("(yyyy/MM(/.*)*)|(yyyy-MM(-.*)*)");
+
+ private final String datasetPattern;
+ private final String datasetPartitionPattern;
+ private final String partitionPrefix;
+ private final DateTimeFormatter timeFormatter;
+ private final boolean enableVirtualPartition;
+
+ private final ZonedDateTime startTime;
+ private final ZonedDateTime endTime;
+ private final TimeIterator.Granularity granularity;
+
+ private final Properties props;
+ private final FileSystem fs;
+
+ public TimePartitionGlobFinder(FileSystem fs, Properties properties) {
+ this(fs, properties,
+ ZonedDateTime.now(ZoneId.of(properties.getProperty(TIME_ZONE,
DEFAULT_TIME_ZONE))));
+ }
+
+ @VisibleForTesting
Review comment:
From what I learned before, this annotation is not helping you testing
methods and it doesn't even make the annotated method loaded in JVM:
https://stackoverflow.com/questions/24051476/guava-visiblefortesting-help-me-with-a-complete-example
it is mostly used for documentation purpose.
Can you double-check if you need it?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 360428)
Time Spent: 2h 10m (was: 2h)
> Implement TimePartitionGlobFinder
> ---------------------------------
>
> Key: GOBBLIN-1001
> URL: https://issues.apache.org/jira/browse/GOBBLIN-1001
> Project: Apache Gobblin
> Issue Type: Task
> Reporter: Zhixiong Chen
> Assignee: Zhixiong Chen
> Priority: Major
> Time Spent: 2h 10m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
