subject:"\[GitHub\] nifi pull request #2671\: NiFi\-5102 \- Adding Processors for MarkLogic DB"

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191618134
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,399 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.SystemResource;
+import org.apache.nifi.annotation.behavior.SystemResourceConsideration;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic", "Put", "Write", "Insert"})
+@CapabilityDescription("Write batches of FlowFiles as documents to a 
MarkLogic server using the " +
+"MarkLogic Data Movement SDK (DMSDK)")
+@SystemResourceConsideration(resource = SystemResource.MEMORY)
+@TriggerWhenEmpty
+public class PutMarkLogic extends AbstractMarkLogicProcessor {
+
+class FlowFileInfo {
+FlowFile flowFile;
+ProcessSession session;
+FlowFileInfo(FlowFile flowFile, ProcessSession session) {
+this.flowFile = flowFile;
+this.session = session;
+}
+}
+private Map uriFlowFileMap = new HashMap<>();
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
+.name("Collections")
+.displayName("Collections")
+.description("Comma-delimited sequence of collections to add to 
each document")
+.required(false)
+.addValidator(Validator.VALID)
+.build();
+
+public static final PropertyDescriptor FORMAT = new 
PropertyDescriptor.Builder()
+.name("Format")
+.displayName("Format")
+.description("Format for each document; if not specified, 
MarkLogic will determine the format" +
+" based on the URI")
+.allowableValues(Format.JSON.name(), Format.XML.name(), 
Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name())
+.required(false)
+.addValidator(Validator.VALID)
+.build();
+
+public static final PropertyDescriptor JOB_ID = new 
PropertyDescriptor.Builder()
+.name("Job ID")

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191616334
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java
 ---
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.DatabaseClient;
+import com.marklogic.nifi.controller.MarkLogicDatabaseClientService;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Defines common properties for MarkLogic processors.
+ */
+public abstract class AbstractMarkLogicProcessor extends 
AbstractSessionFactoryProcessor {
+
+protected List properties;
+protected Set relationships;
+
+public static final PropertyDescriptor DATABASE_CLIENT_SERVICE = new 
PropertyDescriptor.Builder()
+.name("DatabaseClient Service")
+.displayName("DatabaseClient Service")
+.required(true)
+.description("The DatabaseClient Controller Service that provides 
the MarkLogic connection")
+.identifiesControllerService(MarkLogicDatabaseClientService.class)
+.build();
+
+public static final PropertyDescriptor BATCH_SIZE = new 
PropertyDescriptor.Builder()
+.name("Batch size")
+.displayName("Batch size")
+.required(true)
+.defaultValue("100")
+.description("The number of documents per batch - sets the batch 
size on the Batcher")
+.addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor THREAD_COUNT = new 
PropertyDescriptor.Builder()
+.name("Thread count")
+.displayName("Thread count")
+.required(false)
+.description("The number of threads - sets the thread count on the 
Batcher")
+.addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
--- End diff --

probably makes sense to include a default value here for people to 
undersatnd how many threads this will have by default


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191615698
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt ---
@@ -0,0 +1,2106 @@
+MarkLogic NiFi Processors
+
+
+Copyright ï¿½ 2018 MarkLogic Corporation.
+
+This project is licensed under the Apache License, Version 2.0 (the 
"License"); you may not use this project except in compliance with the License. 
You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0 
+
+Unless required by applicable law or agreed to in writing, software 
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 
License for the specific language governing permissions and limitations under 
the License.
+
+
+Please direct questions, comments and requests to 
fossrev...@marklogic.com. 
+
+Open source software required to be made available under license is 
included herein. In the event you are unable to obtain a copy of such open 
source software, please contact fossrev...@marklogic.com and a copy will be 
made available to you.
+
+
+The following software may be included in this project (last updated May 
1, 2018):
+
+Apache Commons Codecï¿½ 1.7 
+Attribution Statements
+http://commons.apache.org/codec/
+
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson (kev...@gnu.org)
+
+The content of package org.apache.commons.codec.language.bm has been 
translated
+from the original php source code available at 
http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
+
+Copyright Statements
+Copyright 2002-2016 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
http://commons.apache.org/proper/commons-codec/source-repository.html 
+
+
+Apache Commons Langï¿½ 3.4 
+Attribution Statements
+http://commons.apache.org/proper/commons-lang/
+
+This product includes software from the Spring Framework,
+under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+Copyright Statements
+Copyright 2001-2017 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
https://github.com/apache/commons-lang 
+
+
+Apache Commons Loggingï¿½ 1.1.1 
+Attribution Statements
+http://commons.apache.org/logging
+
+Copyright Statements
+Copyright 2003-2016 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
https://github.com/apache/commons-logging 
+
+
+Apache Derby 10.13.1.1 
+Attribution Statements
+https://db.apache.org/derby/releases/release-10.13.1.1.cgi
+
+Portions of Derby were originally developed by
+International Business Machines Corporation and are
+licensed to the Apache Software Foundation under the
+"Software Grant and Corporate Contribution License Agreement",
+informally known as the "Derby CLA".
+The following copyright notice(s) were affixed to portions of the code
+with which this file is now or was at one time distributed
+and are placed here unaltered.
+
+(C) Copyright 1997,2004 International Business Machines Corporation.  All 
rights reserved.
+
+(C) Copyright IBM Corp. 2003. 
+
+
+=
+
+
+The portion of the functionTests under 'nist' was originally 
+developed by the National Institute of Standards and Technology (NIST), 
+an agency of the United States Department of Commerce, and adapted by
+International Business Machines Corporation in accordance with the NIST
+Software Acknowledgment and Redistribution document at
+http://www.itl.nist.gov/div897/ctg/sql_form.htm
+
+
+
+=
+
+
+Derby uses the  SerialBlob and SerialClob implementations from the Apache
+Harmony project. The following notice covers the Harmony sources:
+
+Portions of Harmony were originally developed by
+Intel Corporation and are licensed to the

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191615567
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt ---
@@ -0,0 +1,2106 @@
+MarkLogic NiFi Processors
+
+
+Copyright ï¿½ 2018 MarkLogic Corporation.
+
+This project is licensed under the Apache License, Version 2.0 (the 
"License"); you may not use this project except in compliance with the License. 
You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0 
+
+Unless required by applicable law or agreed to in writing, software 
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 
License for the specific language governing permissions and limitations under 
the License.
+
+
+Please direct questions, comments and requests to 
fossrev...@marklogic.com. 
+
+Open source software required to be made available under license is 
included herein. In the event you are unable to obtain a copy of such open 
source software, please contact fossrev...@marklogic.com and a copy will be 
made available to you.
+
+
+The following software may be included in this project (last updated May 
1, 2018):
+
+Apache Commons Codecï¿½ 1.7 
+Attribution Statements
+http://commons.apache.org/codec/
+
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson (kev...@gnu.org)
+
+The content of package org.apache.commons.codec.language.bm has been 
translated
+from the original php source code available at 
http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
+
+Copyright Statements
+Copyright 2002-2016 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
http://commons.apache.org/proper/commons-codec/source-repository.html 
+
+
+Apache Commons Langï¿½ 3.4 
+Attribution Statements
+http://commons.apache.org/proper/commons-lang/
+
+This product includes software from the Spring Framework,
+under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+Copyright Statements
+Copyright 2001-2017 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
https://github.com/apache/commons-lang 
+
+
+Apache Commons Loggingï¿½ 1.1.1 
+Attribution Statements
+http://commons.apache.org/logging
+
+Copyright Statements
+Copyright 2003-2016 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
https://github.com/apache/commons-logging 
+
+
+Apache Derby 10.13.1.1 
+Attribution Statements
+https://db.apache.org/derby/releases/release-10.13.1.1.cgi
+
+Portions of Derby were originally developed by
+International Business Machines Corporation and are
+licensed to the Apache Software Foundation under the
+"Software Grant and Corporate Contribution License Agreement",
+informally known as the "Derby CLA".
+The following copyright notice(s) were affixed to portions of the code
+with which this file is now or was at one time distributed
+and are placed here unaltered.
+
+(C) Copyright 1997,2004 International Business Machines Corporation.  All 
rights reserved.
+
+(C) Copyright IBM Corp. 2003. 
+
+
+=
+
+
+The portion of the functionTests under 'nist' was originally 
+developed by the National Institute of Standards and Technology (NIST), 
+an agency of the United States Department of Commerce, and adapted by
+International Business Machines Corporation in accordance with the NIST
+Software Acknowledgment and Redistribution document at
+http://www.itl.nist.gov/div897/ctg/sql_form.htm
+
+
+
+=
+
+
+Derby uses the  SerialBlob and SerialClob implementations from the Apache
+Harmony project. The following notice covers the Harmony sources:
+
+Portions of Harmony were originally developed by
+Intel Corporation and are licensed to the

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191615452
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt ---
@@ -0,0 +1,2106 @@
+MarkLogic NiFi Processors
+
+
+Copyright ï¿½ 2018 MarkLogic Corporation.
+
+This project is licensed under the Apache License, Version 2.0 (the 
"License"); you may not use this project except in compliance with the License. 
You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0 
+
+Unless required by applicable law or agreed to in writing, software 
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 
License for the specific language governing permissions and limitations under 
the License.
+
+
+Please direct questions, comments and requests to 
fossrev...@marklogic.com. 
+
+Open source software required to be made available under license is 
included herein. In the event you are unable to obtain a copy of such open 
source software, please contact fossrev...@marklogic.com and a copy will be 
made available to you.
+
+
+The following software may be included in this project (last updated May 
1, 2018):
+
+Apache Commons Codecï¿½ 1.7 
+Attribution Statements
+http://commons.apache.org/codec/
+
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson (kev...@gnu.org)
+
+The content of package org.apache.commons.codec.language.bm has been 
translated
+from the original php source code available at 
http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
+
+Copyright Statements
+Copyright 2002-2016 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
http://commons.apache.org/proper/commons-codec/source-repository.html 
+
+
+Apache Commons Langï¿½ 3.4 
+Attribution Statements
+http://commons.apache.org/proper/commons-lang/
+
+This product includes software from the Spring Framework,
+under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+Copyright Statements
+Copyright 2001-2017 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
https://github.com/apache/commons-lang 
+
+
+Apache Commons Loggingï¿½ 1.1.1 
+Attribution Statements
+http://commons.apache.org/logging
+
+Copyright Statements
+Copyright 2003-2016 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
https://github.com/apache/commons-logging 
+
+
+Apache Derby 10.13.1.1 
--- End diff --

i dont see apache derby in your nars.  do you?


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191615429
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt ---
@@ -0,0 +1,2106 @@
+MarkLogic NiFi Processors
+
+
+Copyright ï¿½ 2018 MarkLogic Corporation.
+
+This project is licensed under the Apache License, Version 2.0 (the 
"License"); you may not use this project except in compliance with the License. 
You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0 
+
+Unless required by applicable law or agreed to in writing, software 
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 
License for the specific language governing permissions and limitations under 
the License.
+
+
+Please direct questions, comments and requests to 
fossrev...@marklogic.com. 
+
+Open source software required to be made available under license is 
included herein. In the event you are unable to obtain a copy of such open 
source software, please contact fossrev...@marklogic.com and a copy will be 
made available to you.
+
+
+The following software may be included in this project (last updated May 
1, 2018):
+
+Apache Commons Codecï¿½ 1.7 
+Attribution Statements
+http://commons.apache.org/codec/
+
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson (kev...@gnu.org)
+
+The content of package org.apache.commons.codec.language.bm has been 
translated
+from the original php source code available at 
http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
+
+Copyright Statements
+Copyright 2002-2016 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
http://commons.apache.org/proper/commons-codec/source-repository.html 
+
+
+Apache Commons Langï¿½ 3.4 
+Attribution Statements
+http://commons.apache.org/proper/commons-lang/
+
+This product includes software from the Spring Framework,
+under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+Copyright Statements
+Copyright 2001-2017 The Apache Software Foundation
+
+License Text (http://spdx.org/licenses/Apache-2.0)
+Made available under the Apache License 2.0. See Appendix for full text.
+
+Source materials are available for download at: 
https://github.com/apache/commons-lang 
+
+
+Apache Commons Loggingï¿½ 1.1.1 
--- End diff --

i dont see commons logging in your nars.  Where do you see it?


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191615146
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt ---
@@ -0,0 +1,2106 @@
+MarkLogic NiFi Processors
+
+
+Copyright ï¿½ 2018 MarkLogic Corporation.
+
+This project is licensed under the Apache License, Version 2.0 (the 
"License"); you may not use this project except in compliance with the License. 
You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0 
+
+Unless required by applicable law or agreed to in writing, software 
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 
License for the specific language governing permissions and limitations under 
the License.
+
+
+Please direct questions, comments and requests to 
fossrev...@marklogic.com. 
+
+Open source software required to be made available under license is 
included herein. In the event you are unable to obtain a copy of such open 
source software, please contact fossrev...@marklogic.com and a copy will be 
made available to you.
+
+
+The following software may be included in this project (last updated May 
1, 2018):
+
+Apache Commons Codecï¿½ 1.7 
--- End diff --

Please model all of these dependency references off examples you can find 
in the existing source tree.  I dont see any new dependencies here that you 
would have trouble finding examples of how to reference.  If you have questions 
on how to find any of these please let me know.  Formatting is important to 
help this be readable/consistent/maintainable.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191614275
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt ---
@@ -0,0 +1,2106 @@
+MarkLogic NiFi Processors
+
+
+Copyright ï¿½ 2018 MarkLogic Corporation.
+
+This project is licensed under the Apache License, Version 2.0 (the 
"License"); you may not use this project except in compliance with the License. 
You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0 
+
+Unless required by applicable law or agreed to in writing, software 
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 
License for the specific language governing permissions and limitations under 
the License.
+
+
+Please direct questions, comments and requests to 
fossrev...@marklogic.com. 
+
+Open source software required to be made available under license is 
included herein. In the event you are unable to obtain a copy of such open 
source software, please contact fossrev...@marklogic.com and a copy will be 
made available to you.
+
+
+The following software may be included in this project (last updated May 
1, 2018):
+
+Apache Commons Codecï¿½ 1.7 
--- End diff --

Please have this follow formatting similar to what is shown in this example


https://github.com/apache/nifi/blob/master/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-nar/src/main/resources/META-INF/NOTICE#L30



---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191613968
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt ---
@@ -0,0 +1,2106 @@
+MarkLogic NiFi Processors
+
+
+Copyright ï¿½ 2018 MarkLogic Corporation.
+
+This project is licensed under the Apache License, Version 2.0 (the 
"License"); you may not use this project except in compliance with the License. 
You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0 
+
+Unless required by applicable law or agreed to in writing, software 
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 
License for the specific language governing permissions and limitations under 
the License.
+
+
+Please direct questions, comments and requests to 
fossrev...@marklogic.com. 
+
+Open source software required to be made available under license is 
included herein. In the event you are unable to obtain a copy of such open 
source software, please contact fossrev...@marklogic.com and a copy will be 
made available to you.
+
+
+The following software may be included in this project (last updated May 
1, 2018):
--- End diff --

This line can be removed.  The NOTICE needs to reflect what is actually 
bundled when this NAR is built and should account for what is in it that 
required NOTICE entries no more no less.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191612648
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt ---
@@ -0,0 +1,2106 @@
+MarkLogic NiFi Processors
+
+
+Copyright ï¿½ 2018 MarkLogic Corporation.
+
+This project is licensed under the Apache License, Version 2.0 (the 
"License"); you may not use this project except in compliance with the License. 
You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0 
+
+Unless required by applicable law or agreed to in writing, software 
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 
License for the specific language governing permissions and limitations under 
the License.
+
+
+Please direct questions, comments and requests to 
fossrev...@marklogic.com. 
--- End diff --

This is not appropriate for a NOTICE within an Apache project.  Same 
applies to the above statement.  If these are important to Marklogic it is 
important to consider that you don't have to contribute your code/Nar to the 
apache project.  You could simply cancel the PR and JIRA and have this code 
available on your own company website/etc..

Finally, consider that all contributions to Apache projects are as 
individuals.  This is not a contribution of Marklogic the corporation or else 
we cannot accept it.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191612470
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt ---
@@ -0,0 +1,2106 @@
+MarkLogic NiFi Processors
+
+
+Copyright ï¿½ 2018 MarkLogic Corporation.
+
+This project is licensed under the Apache License, Version 2.0 (the 
"License"); you may not use this project except in compliance with the License. 
You may obtain a copy of the License at
--- End diff --

Please see other NOTICE files for example.  
https://github.com/apache/nifi/blob/master/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-nar/src/main/resources/META-INF/NOTICE

All of this ALv2 reference information does not belong here.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191612335
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt ---
@@ -0,0 +1,2106 @@
+MarkLogic NiFi Processors
+
+
+Copyright ï¿½ 2018 MarkLogic Corporation.
--- End diff --

This notice file should be at the same location under META-INF just like 
this notice file is for the azure nar


https://github.com/apache/nifi/blob/master/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-nar/src/main/resources/META-INF/NOTICE

Also, it isn't Copyright MarkLogic if you're submitting this as a pull 
request to the Apache Software Foundation.  The notice header then should read
---
nifi-marklogic-nar
Copyright 2015-2018 The Apache Software Foundation

This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
--

Then be followed by that which is legally required by the binary 
dependencies of this nar bundle - no more no less.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191612034
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/LICENSE.txt ---
@@ -0,0 +1,72 @@
+Apache License 
--- End diff --

Also unless you're altering the license by appending items to it as would 
be necessary was there any copied source from somewhere and that source was 
from the ASF category A you dont need to include it as it will be pulled in 
automatically.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-29 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r191611907
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/LICENSE.txt ---
@@ -0,0 +1,72 @@
+Apache License 
--- End diff --

This license file should be located in the NAR under meta-inf just like the 
following example.  Further, it should contain the same text/same format of 
text for the ALv2 license.


https://github.com/apache/nifi/blob/master/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-nar/src/main/resources/META-INF/LICENSE




---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-08 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r186803533
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml ---
@@ -0,0 +1,88 @@
+
+
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+
+4.0.0
+
+
+org.apache.nifi
+nifi-marklogic-bundle
+1.7.0-SNAPSHOT
+
+
+nifi-marklogic-processors
+
+
+
+org.apache.nifi
+nifi-api
+${nifi.version}
+
+
+org.apache.nifi
+nifi-utils
+${nifi.version}
--- End diff --

i'm editing this and other findings on a local branch.  I'll share them in 
a patch


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-07 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r186576405
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic", "Put", "Write", "Insert"})
+@CapabilityDescription("Write batches of FlowFiles as documents to a 
MarkLogic server using the " +
+"MarkLogic Data Movement SDK (DMSDK)")
+@TriggerWhenEmpty
+public class PutMarkLogic extends AbstractMarkLogicProcessor {
+
+class FlowFileInfo {
+FlowFile flowFile;
+ProcessSession session;
+FlowFileInfo(FlowFile flowFile, ProcessSession session) {
+this.flowFile = flowFile;
+this.session = session;
+}
+}
+private Map uriFlowFileMap = new HashMap<>();
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
+.name("Collections")
+.displayName("Collections")
+.description("Comma-delimited sequence of collections to add to 
each document")
+.required(false)
+.addValidator(Validator.VALID)
+.build();
+
+public static final PropertyDescriptor FORMAT = new 
PropertyDescriptor.Builder()
+.name("Format")
+.displayName("Format")
+.description("Format for each document; if not specified, 
MarkLogic will determine the format" +
+" based on the URI")
+.allowableValues(Format.JSON.name(), Format.XML.name(), 
Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name())
+.required(false)
+.addValidator(Validator.VALID)
+.build();
+
+public static final PropertyDescriptor JOB_ID = new 
PropertyDescriptor.Builder()
+.name("Job ID")
+.displayName("Job ID")
+.description("ID for the WriteBatcher job")
+.required(false)
+.addValidator(Validator.VALID)
+

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-07 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r186527597
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml ---
@@ -0,0 +1,88 @@
+
+
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+
+4.0.0
+
+
+org.apache.nifi
+nifi-marklogic-bundle
+1.7.0-SNAPSHOT
+
+
+nifi-marklogic-processors
+
+
+
+org.apache.nifi
+nifi-api
+${nifi.version}
+
+
+org.apache.nifi
+nifi-utils
+${nifi.version}
--- End diff --

i strongly recommend avoiding use of ${} notation for version of components 
being built within this reactor/multi-module build of nifi.  It can create 
problems during the release process and our release plugins take care of 
changing the strings for us.  Please use '1.7.0-SNAPSHOT' instead.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-07 Thread joewitt

Github user joewitt commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r186527567
  
--- Diff: nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml 
---
@@ -0,0 +1,56 @@
+
+
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+
+4.0.0
+
+
+org.apache.nifi
+nifi-marklogic-bundle
+1.7.0-SNAPSHOT
+
+
+nifi-marklogic-nar
+
+nar
+
+
+true
+true
+
+
+
+
+org.apache.nifi
+nifi-marklogic-services-api-nar
+${project.parent.version}
--- End diff --

i strongly recommend avoiding use of ${} notation for version of components 
being built within this reactor/multi-module build of nifi.  It can create 
problems during the release process and our release plugins take care of 
changing the strings for us.  Please use '1.7.0-SNAPSHOT' instead.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread vivekmuniyandi

Github user vivekmuniyandi commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185979858
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java
 ---
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.ExportListener;
+import com.marklogic.client.ext.datamovement.job.SimpleQueryBatcherJob;
+import com.marklogic.client.io.BytesHandle;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+@Tags({"MarkLogic"})
+@CapabilityDescription("Creates FlowFiles from batches of documents, 
matching the given criteria," +
+" retrieved from a MarkLogic server using the MarkLogic Data Movement 
SDK (DMSDK)")
+public class QueryMarkLogic extends AbstractMarkLogicProcessor {
--- End diff --

We have added this as a task in our sprint and accommodate in the coming 
weeks. 


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185738926
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java
 ---
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.DatabaseClient;
+import com.marklogic.nifi.controller.DatabaseClientService;
+import com.marklogic.nifi.controller.DatabaseClientService;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Defines common properties for MarkLogic processors.
+ */
+public abstract class AbstractMarkLogicProcessor extends 
AbstractSessionFactoryProcessor {
+
+protected List properties;
+protected Set relationships;
+
+// NiFi requires a validator for every property, even those that don't 
need any validation
+protected static Validator NO_VALIDATION_VALIDATOR = new Validator() {
+@Override
+public ValidationResult validate(String subject, String input, 
ValidationContext context) {
+return new ValidationResult.Builder().valid(true).build();
+}
+};
+
+public static final PropertyDescriptor DATABASE_CLIENT_SERVICE = new 
PropertyDescriptor.Builder()
+.name("DatabaseClient Service")
+.displayName("DatabaseClient Service")
+.required(true)
+.description("The DatabaseClient Controller Service that provides 
the MarkLogic connection")
+.identifiesControllerService(DatabaseClientService.class)
+.build();
+
+public static final PropertyDescriptor BATCH_SIZE = new 
PropertyDescriptor.Builder()
+.name("Batch size")
+.displayName("Batch size")
+.required(true)
+.defaultValue("100")
+.description("The number of documents per batch - sets the batch 
size on the Batcher")
+.addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor THREAD_COUNT = new 
PropertyDescriptor.Builder()
+.name("Thread count")
+.displayName("Thread count")
+.required(true)
+.defaultValue("16")
--- End diff --

NiFi uses a lot of threads, so you might want to think about lowering this 
default so it doesn't risk getting greedy.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185739906
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java
 ---
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.ExportListener;
+import com.marklogic.client.ext.datamovement.job.SimpleQueryBatcherJob;
+import com.marklogic.client.io.BytesHandle;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+@Tags({"MarkLogic"})
+@CapabilityDescription("Creates FlowFiles from batches of documents, 
matching the given criteria," +
+" retrieved from a MarkLogic server using the MarkLogic Data Movement 
SDK (DMSDK)")
+public class QueryMarkLogic extends AbstractMarkLogicProcessor {
--- End diff --

You might want to think about setting `@InputRequirement` to ALLOWED so 
that you can add an incoming relationship for having the queries provided by 
flowfile properties or content. There are examples of how to do this in 
`GetMongo` and `ExecuteSQL`. When looking there, take note of the top of their 
`onTrigger` methods to see how the presence of a connection is detected.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185742208
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic"})
+@CapabilityDescription("Write batches of FlowFiles as documents to a 
MarkLogic server using the " +
+"MarkLogic Data Movement SDK (DMSDK)")
+@TriggerWhenEmpty
+public class PutMarkLogic extends AbstractMarkLogicProcessor {
+
+class FlowFileInfo {
+FlowFile flowFile;
+ProcessSession session;
+FlowFileInfo(FlowFile flowFile, ProcessSession session) {
+this.flowFile = flowFile;
+this.session = session;
+}
+}
+private Map URIFlowFileMap = new HashMap<>();
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
+.name("Collections")
+.displayName("Collections")
+.description("Comma-delimited sequence of collections to add to 
each document")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor FORMAT = new 
PropertyDescriptor.Builder()
+.name("Format")
+.displayName("Format")
+.description("Format for each document; if not specified, 
MarkLogic will determine the format" +
+" based on the URI")
+.allowableValues(Format.JSON.name(), Format.XML.name(), 
Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name())
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_ID = new 
PropertyDescriptor.Builder()
+.name("Job ID")
+.displayName("Job ID")
+.description("ID for the WriteBatcher job")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_NAME = new 
PropertyDescriptor.Builder()
+.name("Job Name")
+

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185743318
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic"})
+@CapabilityDescription("Write batches of FlowFiles as documents to a 
MarkLogic server using the " +
+"MarkLogic Data Movement SDK (DMSDK)")
+@TriggerWhenEmpty
+public class PutMarkLogic extends AbstractMarkLogicProcessor {
+
+class FlowFileInfo {
+FlowFile flowFile;
+ProcessSession session;
+FlowFileInfo(FlowFile flowFile, ProcessSession session) {
+this.flowFile = flowFile;
+this.session = session;
+}
+}
+private Map URIFlowFileMap = new HashMap<>();
--- End diff --

This variable name violates standard Java conventions.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185740310
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java
 ---
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.ExportListener;
+import com.marklogic.client.ext.datamovement.job.SimpleQueryBatcherJob;
+import com.marklogic.client.io.BytesHandle;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+@Tags({"MarkLogic"})
+@CapabilityDescription("Creates FlowFiles from batches of documents, 
matching the given criteria," +
+" retrieved from a MarkLogic server using the MarkLogic Data Movement 
SDK (DMSDK)")
+public class QueryMarkLogic extends AbstractMarkLogicProcessor {
+
+public static final PropertyDescriptor CONSISTENT_SNAPSHOT = new 
PropertyDescriptor.Builder()
+.name("Consistent snapshot")
+.displayName("Consistent snapshot")
+.defaultValue("true")
+.description("Boolean used to indicate that the matching documents 
were retrieved from a " +
+"consistent snapshot")
+.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
+.name("Collections")
+.displayName("Collections")
+.description("Comma-separated list of collections to query from a 
MarkLogic server")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor URIS_QUERY = new 
PropertyDescriptor.Builder()
+.name("URIs query")
+.displayName("URIs query")
+.description("CTS URI Query for retrieving documents from a 
MarkLogic server")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor URI_PATTERN = new 
PropertyDescriptor.Builder()
+.name("URI pattern")
+.displayName("URI pattern")
+.description("URI pattern for retrieving documents from a 
MarkLogic server")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+protected static final Relationship SUCCESS = new 
Relationship.Builder()
+.name("SUCCESS")
+.description("All FlowFiles that are created from documents read 
from MarkLogic are routed to" +
+" this success relationship")
+.build();
+
+@Override
+public void init(ProcessorInitializationContext context) {
+super.init(context);
+
+List list = new ArrayList<>();
+list.addAll(properties);
+list.add(CONSISTENT_SNAPSHOT);
+list.add(COLLECTIONS);
+list.add(URIS_QUERY);
+list.add(URI_PATTERN);
+properties = Collections.unmodifiableList(list);
+Set set = new HashSet<>();
+set.add(SUCCESS);
+relationships = Collections.unmodifiableSet(set);
+}
+
+@Override
+public final void

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185738577
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java
 ---
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.DatabaseClient;
+import com.marklogic.nifi.controller.DatabaseClientService;
+import com.marklogic.nifi.controller.DatabaseClientService;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Defines common properties for MarkLogic processors.
+ */
+public abstract class AbstractMarkLogicProcessor extends 
AbstractSessionFactoryProcessor {
+
+protected List properties;
+protected Set relationships;
+
+// NiFi requires a validator for every property, even those that don't 
need any validation
+protected static Validator NO_VALIDATION_VALIDATOR = new Validator() {
--- End diff --

`Validator.VALID` does the same thing.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185741806
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic"})
+@CapabilityDescription("Write batches of FlowFiles as documents to a 
MarkLogic server using the " +
+"MarkLogic Data Movement SDK (DMSDK)")
+@TriggerWhenEmpty
+public class PutMarkLogic extends AbstractMarkLogicProcessor {
+
+class FlowFileInfo {
+FlowFile flowFile;
+ProcessSession session;
+FlowFileInfo(FlowFile flowFile, ProcessSession session) {
+this.flowFile = flowFile;
+this.session = session;
+}
+}
+private Map URIFlowFileMap = new HashMap<>();
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
--- End diff --

Rule of thumb for all of these:

1. Add an explicit call to `required(boolean)` so it's obvious what your 
intent is there.
2. Switch `NO_VALIDATION_VALIDATOR` to Validator.VALID.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185740483
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java
 ---
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.ExportListener;
+import com.marklogic.client.ext.datamovement.job.SimpleQueryBatcherJob;
+import com.marklogic.client.io.BytesHandle;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+@Tags({"MarkLogic"})
+@CapabilityDescription("Creates FlowFiles from batches of documents, 
matching the given criteria," +
+" retrieved from a MarkLogic server using the MarkLogic Data Movement 
SDK (DMSDK)")
+public class QueryMarkLogic extends AbstractMarkLogicProcessor {
+
+public static final PropertyDescriptor CONSISTENT_SNAPSHOT = new 
PropertyDescriptor.Builder()
+.name("Consistent snapshot")
+.displayName("Consistent snapshot")
+.defaultValue("true")
+.description("Boolean used to indicate that the matching documents 
were retrieved from a " +
+"consistent snapshot")
+.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
+.name("Collections")
+.displayName("Collections")
+.description("Comma-separated list of collections to query from a 
MarkLogic server")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor URIS_QUERY = new 
PropertyDescriptor.Builder()
+.name("URIs query")
+.displayName("URIs query")
+.description("CTS URI Query for retrieving documents from a 
MarkLogic server")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor URI_PATTERN = new 
PropertyDescriptor.Builder()
+.name("URI pattern")
+.displayName("URI pattern")
+.description("URI pattern for retrieving documents from a 
MarkLogic server")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+protected static final Relationship SUCCESS = new 
Relationship.Builder()
+.name("SUCCESS")
+.description("All FlowFiles that are created from documents read 
from MarkLogic are routed to" +
+" this success relationship")
+.build();
+
+@Override
+public void init(ProcessorInitializationContext context) {
+super.init(context);
+
+List list = new ArrayList<>();
+list.addAll(properties);
+list.add(CONSISTENT_SNAPSHOT);
+list.add(COLLECTIONS);
+list.add(URIS_QUERY);
+list.add(URI_PATTERN);
+properties = Collections.unmodifiableList(list);
+Set set = new HashSet<>();
+set.add(SUCCESS);
+relationships = Collections.unmodifiableSet(set);
+}
+
+@Override
+public final void

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185740763
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic"})
--- End diff --

Should add a few more descriptive tags to help people find it.


---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185742105
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic"})
+@CapabilityDescription("Write batches of FlowFiles as documents to a 
MarkLogic server using the " +
+"MarkLogic Data Movement SDK (DMSDK)")
+@TriggerWhenEmpty
+public class PutMarkLogic extends AbstractMarkLogicProcessor {
+
+class FlowFileInfo {
+FlowFile flowFile;
+ProcessSession session;
+FlowFileInfo(FlowFile flowFile, ProcessSession session) {
+this.flowFile = flowFile;
+this.session = session;
+}
+}
+private Map URIFlowFileMap = new HashMap<>();
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
+.name("Collections")
+.displayName("Collections")
+.description("Comma-delimited sequence of collections to add to 
each document")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor FORMAT = new 
PropertyDescriptor.Builder()
+.name("Format")
+.displayName("Format")
+.description("Format for each document; if not specified, 
MarkLogic will determine the format" +
+" based on the URI")
+.allowableValues(Format.JSON.name(), Format.XML.name(), 
Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name())
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_ID = new 
PropertyDescriptor.Builder()
+.name("Job ID")
+.displayName("Job ID")
+.description("ID for the WriteBatcher job")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_NAME = new 
PropertyDescriptor.Builder()
+.name("Job Name")
+

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185742052
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic"})
+@CapabilityDescription("Write batches of FlowFiles as documents to a 
MarkLogic server using the " +
+"MarkLogic Data Movement SDK (DMSDK)")
+@TriggerWhenEmpty
+public class PutMarkLogic extends AbstractMarkLogicProcessor {
+
+class FlowFileInfo {
+FlowFile flowFile;
+ProcessSession session;
+FlowFileInfo(FlowFile flowFile, ProcessSession session) {
+this.flowFile = flowFile;
+this.session = session;
+}
+}
+private Map URIFlowFileMap = new HashMap<>();
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
+.name("Collections")
+.displayName("Collections")
+.description("Comma-delimited sequence of collections to add to 
each document")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor FORMAT = new 
PropertyDescriptor.Builder()
+.name("Format")
+.displayName("Format")
+.description("Format for each document; if not specified, 
MarkLogic will determine the format" +
+" based on the URI")
+.allowableValues(Format.JSON.name(), Format.XML.name(), 
Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name())
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_ID = new 
PropertyDescriptor.Builder()
+.name("Job ID")
+.displayName("Job ID")
+.description("ID for the WriteBatcher job")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_NAME = new 
PropertyDescriptor.Builder()
+.name("Job Name")
+

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185745410
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic"})
+@CapabilityDescription("Write batches of FlowFiles as documents to a 
MarkLogic server using the " +
+"MarkLogic Data Movement SDK (DMSDK)")
+@TriggerWhenEmpty
+public class PutMarkLogic extends AbstractMarkLogicProcessor {
+
+class FlowFileInfo {
+FlowFile flowFile;
+ProcessSession session;
+FlowFileInfo(FlowFile flowFile, ProcessSession session) {
+this.flowFile = flowFile;
+this.session = session;
+}
+}
+private Map URIFlowFileMap = new HashMap<>();
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
+.name("Collections")
+.displayName("Collections")
+.description("Comma-delimited sequence of collections to add to 
each document")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor FORMAT = new 
PropertyDescriptor.Builder()
+.name("Format")
+.displayName("Format")
+.description("Format for each document; if not specified, 
MarkLogic will determine the format" +
+" based on the URI")
+.allowableValues(Format.JSON.name(), Format.XML.name(), 
Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name())
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_ID = new 
PropertyDescriptor.Builder()
+.name("Job ID")
+.displayName("Job ID")
+.description("ID for the WriteBatcher job")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_NAME = new 
PropertyDescriptor.Builder()
+.name("Job Name")
+

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185741393
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic"})
+@CapabilityDescription("Write batches of FlowFiles as documents to a 
MarkLogic server using the " +
+"MarkLogic Data Movement SDK (DMSDK)")
+@TriggerWhenEmpty
+public class PutMarkLogic extends AbstractMarkLogicProcessor {
+
+class FlowFileInfo {
+FlowFile flowFile;
+ProcessSession session;
+FlowFileInfo(FlowFile flowFile, ProcessSession session) {
+this.flowFile = flowFile;
+this.session = session;
+}
+}
+private Map URIFlowFileMap = new HashMap<>();
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
+.name("Collections")
+.displayName("Collections")
+.description("Comma-delimited sequence of collections to add to 
each document")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor FORMAT = new 
PropertyDescriptor.Builder()
+.name("Format")
+.displayName("Format")
+.description("Format for each document; if not specified, 
MarkLogic will determine the format" +
+" based on the URI")
+.allowableValues(Format.JSON.name(), Format.XML.name(), 
Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name())
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_ID = new 
PropertyDescriptor.Builder()
+.name("Job ID")
+.displayName("Job ID")
+.description("ID for the WriteBatcher job")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_NAME = new 
PropertyDescriptor.Builder()
+.name("Job Name")
+

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-03 Thread MikeThomsen

Github user MikeThomsen commented on a diff in the pull request:

https://github.com/apache/nifi/pull/2671#discussion_r185740990
  
--- Diff: 
nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java
 ---
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.nifi.processor;
+
+import com.marklogic.client.datamovement.DataMovementManager;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.WriteEvent;
+import com.marklogic.client.datamovement.impl.WriteEventImpl;
+import com.marklogic.client.document.ServerTransform;
+import com.marklogic.client.io.BytesHandle;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.Format;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.ProcessorInitializationContext;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.stream.io.StreamUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * The TriggerWhenEmpty annotation is used so that this processor has a 
chance to flush the WriteBatcher when no
+ * flowfiles are ready to be received.
+ */
+@Tags({"MarkLogic"})
+@CapabilityDescription("Write batches of FlowFiles as documents to a 
MarkLogic server using the " +
+"MarkLogic Data Movement SDK (DMSDK)")
+@TriggerWhenEmpty
+public class PutMarkLogic extends AbstractMarkLogicProcessor {
+
+class FlowFileInfo {
+FlowFile flowFile;
+ProcessSession session;
+FlowFileInfo(FlowFile flowFile, ProcessSession session) {
+this.flowFile = flowFile;
+this.session = session;
+}
+}
+private Map URIFlowFileMap = new HashMap<>();
+public static final PropertyDescriptor COLLECTIONS = new 
PropertyDescriptor.Builder()
+.name("Collections")
+.displayName("Collections")
+.description("Comma-delimited sequence of collections to add to 
each document")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor FORMAT = new 
PropertyDescriptor.Builder()
+.name("Format")
+.displayName("Format")
+.description("Format for each document; if not specified, 
MarkLogic will determine the format" +
+" based on the URI")
+.allowableValues(Format.JSON.name(), Format.XML.name(), 
Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name())
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_ID = new 
PropertyDescriptor.Builder()
+.name("Job ID")
+.displayName("Job ID")
+.description("ID for the WriteBatcher job")
+.addValidator(NO_VALIDATION_VALIDATOR)
+.build();
+
+public static final PropertyDescriptor JOB_NAME = new 
PropertyDescriptor.Builder()
+.name("Job Name")
+

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

2018-05-02 Thread vivekmuniyandi

GitHub user vivekmuniyandi opened a pull request:

https://github.com/apache/nifi/pull/2671

NiFi-5102 - Adding Processors for MarkLogic DB

â¦tMarkLogic

Thank you for submitting a contribution to Apache NiFi.

In order to streamline the review of the contribution we ask you
to ensure the following steps have been taken:

### For all changes:
- [x] Is there a JIRA ticket associated with this PR? Is it referenced 
 in the commit message?
Yes, there is a JIRA ticket - 
https://issues.apache.org/jira/browse/NIFI-5102

- [x] Does your PR title start with NIFI- where  is the JIRA number 
you are trying to resolve? Pay particular attention to the hyphen "-" character.
Yes

- [x] Has your PR been rebased against the latest commit within the target 
branch (typically master)?
Yes

- [x] Is your initial contribution a single, squashed commit?
No

### For code changes:
- [x] Have you ensured that the full suite of tests is executed via mvn 
-Pcontrib-check clean install at the root nifi folder?
Yes, but certain tests from other processors fail - such as from grpc 
processors were failing with an unapproved license error for auto-generated 
files in the target folder.

- [x] Have you written or updated unit tests to verify your changes?
Yes

- [x] If adding new dependencies to the code, are these dependencies 
licensed in a way that is compatible for inclusion under [ASF 
2.0](http://www.apache.org/legal/resolved.html#category-a)? 
Yes

- [x] If applicable, have you updated the LICENSE file, including the main 
LICENSE file under nifi-assembly?
No
- [x] If applicable, have you updated the NOTICE file, including the main 
NOTICE file found under nifi-assembly?
Yes

Have added a NOTICE file and a LICENSE file under the nifi-marklogic-bundle 
project

- [x] If adding new Properties, have you added .displayName in addition to 
.name (programmatic access) for each of the new properties?
Yes

### For documentation related changes:
- [x] Have you ensured that format looks appropriate for the output in 
which it is rendered?
Yes

### Note:
Please ensure that once the PR is submitted, you check travis-ci for build 
issues and submit an update to your PR as soon as possible.


You can merge this pull request into a Git repository by running:

$ git pull https://github.com/marklogic/nifi nifi-5102

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/nifi/pull/2671.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #2671


commit 16f8ffef0e0a853535c5af63b7e28441b4674c47
Author: Vivek Siddharthan Muniyandi 
Date:   2018-05-03T03:01:25Z

NIFI-5102 Adding MarkLogic DB NiFi processors - QueryMarkLogic and 
PutMarkLogic

commit e7b4bee69b6bd6a5274613ff884625a9d30b5ffb
Author: Vivek Siddharthan Muniyandi 
Date:   2018-05-03T03:26:14Z

NIFI-5102 Added displaynames for Properties




---

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

[GitHub] nifi pull request #2671: NiFi-5102 - Adding Processors for MarkLogic DB

34 matches

Site Navigation

Mail list logo

Footer information