Repository: nifi Updated Branches: refs/heads/master e97387479 -> 937dc71ae
NIFI-2417: Addressing code review comments Signed-off-by: Matt Burgess <mattyb...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/937dc71a Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/937dc71a Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/937dc71a Branch: refs/heads/master Commit: 937dc71aec107cdd0367d75d91435eaa5c7e73d9 Parents: 00412f6 Author: Joe Gresock <joseph.gres...@lmco.com> Authored: Thu Sep 22 12:24:35 2016 +0000 Committer: Matt Burgess <mattyb...@apache.org> Committed: Thu Sep 22 15:00:06 2016 -0400 ---------------------------------------------------------------------- .../elasticsearch/FetchElasticsearchHttp.java | 13 +++++---- .../elasticsearch/QueryElasticsearchHttp.java | 30 +++++++++++++------- .../elasticsearch/ScrollElasticsearchHttp.java | 28 +++++++++++------- 3 files changed, 46 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/937dc71a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/FetchElasticsearchHttp.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/FetchElasticsearchHttp.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/FetchElasticsearchHttp.java index 9ce1510..ac598bf 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/FetchElasticsearchHttp.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/FetchElasticsearchHttp.java @@ -104,7 +104,8 @@ public class FetchElasticsearchHttp extends AbstractElasticsearchHttpProcessor { public static final PropertyDescriptor INDEX = new PropertyDescriptor.Builder() .name("fetch-es-index") .displayName("Index") - .description("The name of the index to read from") + .description("The name of the index to read from. If the property is set " + + "to _all, the query will match across all indexes.") .required(true) .expressionLanguageSupported(true) .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) @@ -113,8 +114,8 @@ public class FetchElasticsearchHttp extends AbstractElasticsearchHttpProcessor { public static final PropertyDescriptor TYPE = new PropertyDescriptor.Builder() .name("fetch-es-type") .displayName("Type") - .description("The (optional) type of this document, used by Elasticsearch for indexing and searching. If the property is empty or set " - + "to _all, the first document matching the identifier across all types will be retrieved.") + .description("The (optional) type of this document, used by Elasticsearch for indexing and searching. If the property is empty, " + + "the first document matching the identifier across all types will be retrieved.") .required(false) .expressionLanguageSupported(true) .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) @@ -297,8 +298,10 @@ public class FetchElasticsearchHttp extends AbstractElasticsearchHttpProcessor { throw new MalformedURLException("Base URL cannot be null"); } HttpUrl.Builder builder = HttpUrl.parse(baseUrl).newBuilder(); - builder.addPathSegment(index); - builder.addPathSegment((StringUtils.isEmpty(type)) ? "_all" : type); + builder.addPathSegment((StringUtils.isEmpty(index)) ? "_all" : index); + if (!StringUtils.isEmpty(type)) { + builder.addPathSegment(type); + } builder.addPathSegment(docId); if (!StringUtils.isEmpty(fields)) { String trimmedFields = Stream.of(fields.split(",")).map(String::trim).collect(Collectors.joining(",")); http://git-wip-us.apache.org/repos/asf/nifi/blob/937dc71a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/QueryElasticsearchHttp.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/QueryElasticsearchHttp.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/QueryElasticsearchHttp.java index ba3969f..f921323 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/QueryElasticsearchHttp.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/QueryElasticsearchHttp.java @@ -68,6 +68,7 @@ import okhttp3.ResponseBody; + "To retrieve more records, use the ScrollElasticsearchHttp processor.") @WritesAttributes({ @WritesAttribute(attribute = "filename", description = "The filename attribute is set to the document identifier"), + @WritesAttribute(attribute = "es.id", description = "The Elasticsearch document identifier"), @WritesAttribute(attribute = "es.index", description = "The Elasticsearch index containing the document"), @WritesAttribute(attribute = "es.type", description = "The Elasticsearch document type"), @WritesAttribute(attribute = "es.result.*", description = "If Target is 'Flow file attributes', the JSON attributes of " @@ -106,13 +107,14 @@ public class QueryElasticsearchHttp extends AbstractElasticsearchHttpProcessor { public static final PropertyDescriptor QUERY = new PropertyDescriptor.Builder() .name("query-es-query").displayName("Query") - .description("The Lucene-style query to run against ElasticSearch").required(true) + .description("The Lucene-style query to run against ElasticSearch (e.g., genre:blues AND -artist:muddy)").required(true) .expressionLanguageSupported(true).addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); public static final PropertyDescriptor INDEX = new PropertyDescriptor.Builder() .name("query-es-index").displayName("Index") - .description("The name of the index to read from").required(true) + .description("The name of the index to read from. If the property is set " + + "to _all, the query will match across all indexes.").required(true) .expressionLanguageSupported(true).addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); @@ -120,10 +122,12 @@ public class QueryElasticsearchHttp extends AbstractElasticsearchHttpProcessor { .name("query-es-type") .displayName("Type") .description( - "The (optional) type of this document, used by Elasticsearch for indexing and searching. If the property is empty or set " - + "to _all, the first document matching the identifier across all types will be retrieved.") - .required(false).expressionLanguageSupported(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR).build(); + "The (optional) type of this query, used by Elasticsearch for indexing and searching. If the property is empty, " + + "the the query will match across all types.") + .required(false) + .expressionLanguageSupported(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); public static final PropertyDescriptor FIELDS = new PropertyDescriptor.Builder() .name("query-es-fields") @@ -131,8 +135,10 @@ public class QueryElasticsearchHttp extends AbstractElasticsearchHttpProcessor { .description( "A comma-separated list of fields to retrieve from the document. If the Fields property is left blank, " + "then the entire document's source will be retrieved.") - .required(false).expressionLanguageSupported(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR).build(); + .required(false) + .expressionLanguageSupported(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); public static final PropertyDescriptor SORT = new PropertyDescriptor.Builder() .name("query-es-sort") @@ -331,11 +337,13 @@ public class QueryElasticsearchHttp extends AbstractElasticsearchHttpProcessor { } JsonNode source = hit.get("_source"); + documentFlowFile = session.putAttribute(documentFlowFile, "es.id", retrievedId); documentFlowFile = session.putAttribute(documentFlowFile, "es.index", retrievedIndex); documentFlowFile = session.putAttribute(documentFlowFile, "es.type", retrievedType); if (targetIsContent) { documentFlowFile = session.putAttribute(documentFlowFile, "filename", retrievedId); + documentFlowFile = session.putAttribute(documentFlowFile, "mime.type", "application/json"); documentFlowFile = session.write(documentFlowFile, out -> { out.write(source.toString().getBytes()); }); @@ -390,8 +398,10 @@ public class QueryElasticsearchHttp extends AbstractElasticsearchHttpProcessor { throw new MalformedURLException("Base URL cannot be null"); } HttpUrl.Builder builder = HttpUrl.parse(baseUrl).newBuilder(); - builder.addPathSegment(index); - builder.addPathSegment((StringUtils.isEmpty(type)) ? "_all" : type); + builder.addPathSegment((StringUtils.isEmpty(index)) ? "_all" : index); + if (!StringUtils.isEmpty(type)) { + builder.addPathSegment(type); + } builder.addPathSegment("_search"); builder.addQueryParameter(QUERY_QUERY_PARAM, query); builder.addQueryParameter(SIZE_QUERY_PARAM, String.valueOf(pageSize)); http://git-wip-us.apache.org/repos/asf/nifi/blob/937dc71a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/ScrollElasticsearchHttp.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/ScrollElasticsearchHttp.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/ScrollElasticsearchHttp.java index accdca9..3d897cf 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/ScrollElasticsearchHttp.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/ScrollElasticsearchHttp.java @@ -100,7 +100,7 @@ public class ScrollElasticsearchHttp extends AbstractElasticsearchHttpProcessor public static final PropertyDescriptor QUERY = new PropertyDescriptor.Builder() .name("scroll-es-query").displayName("Query") - .description("The Lucene-style query to run against ElasticSearch").required(true) + .description("The Lucene-style query to run against ElasticSearch (e.g., genre:blues AND -artist:muddy)").required(true) .expressionLanguageSupported(true).addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); @@ -116,19 +116,24 @@ public class ScrollElasticsearchHttp extends AbstractElasticsearchHttpProcessor .build(); public static final PropertyDescriptor INDEX = new PropertyDescriptor.Builder() - .name("scroll-es-index").displayName("Index") - .description("The name of the index to read from").required(true) - .expressionLanguageSupported(true).addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .name("scroll-es-index") + .displayName("Index") + .description("The name of the index to read from. If the property is set " + + "to _all, the query will match across all indexes.") + .required(true) + .expressionLanguageSupported(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); public static final PropertyDescriptor TYPE = new PropertyDescriptor.Builder() .name("scroll-es-type") .displayName("Type") .description( - "The (optional) type of this document, used by Elasticsearch for indexing and searching. If the property is empty or set " - + "to _all, the first document matching the identifier across all types will be retrieved.") - .required(false).expressionLanguageSupported(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR).build(); + "The (optional) type of this query, used by Elasticsearch for indexing and searching. If the property is empty, " + + "the the query will match across all types.") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(false) + .expressionLanguageSupported(true).build(); public static final PropertyDescriptor FIELDS = new PropertyDescriptor.Builder() .name("scroll-es-fields") @@ -303,6 +308,7 @@ public class ScrollElasticsearchHttp extends AbstractElasticsearchHttpProcessor JsonNode source = hit.get("_source"); flowFile = session.putAttribute(flowFile, "es.index", retrievedIndex); flowFile = session.putAttribute(flowFile, "es.type", retrievedType); + flowFile = session.putAttribute(flowFile, "mime.type", "application/json"); builder.append(source.toString()); if (i < hits.size() - 1) { @@ -394,8 +400,10 @@ public class ScrollElasticsearchHttp extends AbstractElasticsearchHttpProcessor builder.addPathSegment("scroll"); builder.addQueryParameter(SCROLL_ID_QUERY_PARAM, scrollId); } else { - builder.addPathSegment(index); - builder.addPathSegment((StringUtils.isEmpty(type)) ? "_all" : type); + builder.addPathSegment((StringUtils.isEmpty(index)) ? "_all" : index); + if (!StringUtils.isEmpty(type)) { + builder.addPathSegment(type); + } builder.addPathSegment("_search"); builder.addQueryParameter(QUERY_QUERY_PARAM, query); builder.addQueryParameter(SIZE_QUERY_PARAM, String.valueOf(pageSize));
