[
https://issues.apache.org/jira/browse/APEXMALHAR-1701?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15403622#comment-15403622
]
ASF GitHub Bot commented on APEXMALHAR-1701:
--------------------------------------------
Github user bhupeshchawda commented on a diff in the pull request:
https://github.com/apache/apex-malhar/pull/335#discussion_r73115095
--- Diff:
library/src/main/java/org/apache/apex/malhar/lib/dedup/DeduperTimeBasedPOJOImpl.java
---
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.lib.dedup;
+
+import javax.validation.constraints.NotNull;
+
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+import org.apache.apex.malhar.lib.state.managed.TimeBucketAssigner;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.Context.PortContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.Operator.ActivationListener;
+import com.datatorrent.api.StreamCodec;
+import com.datatorrent.api.annotation.InputPortFieldAnnotation;
+import com.datatorrent.lib.util.PojoUtils;
+import com.datatorrent.lib.util.PojoUtils.Getter;
+import com.datatorrent.netlet.util.Slice;
+
+@Evolving
+public class DeduperTimeBasedPOJOImpl extends AbstractDeduper<Object>
implements ActivationListener<Context>
+{
+
+ // Required properties
+ @NotNull
+ private String keyExpression;
+
+ private String timeExpression;
+
+ @NotNull
+ private long bucketSpan;
+
+ @NotNull
+ private long expireBefore;
+
+ // Optional
+ private long referenceInstant = -1;
+
+ private transient Class<?> pojoClass;
+
+ private transient Getter<Object, Long> timeGetter;
+
+ private transient Getter<Object, Object> keyGetter;
+
+ @InputPortFieldAnnotation(schemaRequired = true)
+ public final transient DefaultInputPort<Object> input = new
DefaultInputPort<Object>()
+ {
+ @Override
+ public void setup(PortContext context)
+ {
+ pojoClass = context.getAttributes().get(PortContext.TUPLE_CLASS);
+ }
+
+ @Override
+ public void process(Object tuple)
+ {
+ processTuple(tuple);
+ }
+
+ @Override
+ public StreamCodec<Object> getStreamCodec()
+ {
+ return getDeduperStreamCodec();
+ }
+ };
+
+ @Override
+ protected long getTime(Object tuple)
+ {
+ if (timeGetter != null) {
+ return timeGetter.get(tuple);
+ }
+ return System.currentTimeMillis();
+ }
+
+ @Override
+ protected Slice getKey(Object tuple)
+ {
+ Object key = keyGetter.get(tuple);
+ return new Slice(key.toString().getBytes());
+ }
+
+ protected StreamCodec<Object> getDeduperStreamCodec()
+ {
+ return new DeduperStreamCodec(keyExpression);
+ }
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ TimeBucketAssigner timeBucketAssigner = new TimeBucketAssigner();
+ timeBucketAssigner.setBucketSpan(Duration.standardSeconds(bucketSpan));
+
timeBucketAssigner.setExpireBefore(Duration.standardSeconds(expireBefore));
+ if (referenceInstant == -1) {
+ timeBucketAssigner.setReferenceInstant(new Instant());
--- End diff --
Have set the default in the declaration. Will remove the if statement.
> Deduper : create a deduper backed by Managed State
> --------------------------------------------------
>
> Key: APEXMALHAR-1701
> URL: https://issues.apache.org/jira/browse/APEXMALHAR-1701
> Project: Apache Apex Malhar
> Issue Type: Task
> Components: algorithms
> Reporter: Chandni Singh
> Assignee: Chandni Singh
>
> Need a de-deduplicator operator that is based on Managed State.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
