This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new b0063fa ATLAS-3100 Enhance Full Text Search backend implementation to
use SOLR request handler.
b0063fa is described below
commit b0063fa54e59088a5ef72604113c55c6284ab337
Author: Sridhar K <[email protected]>
AuthorDate: Tue May 7 15:28:10 2019 +0200
ATLAS-3100 Enhance Full Text Search backend implementation to use SOLR
request handler.
Signed-off-by: Sarath Subramanian <[email protected]>
---
.../patches/003-base_model_add_searchWeight.json | 82 ++++
addons/models/1000-Hadoop/1090-impala_model.json | 9 +-
.../1000-Hadoop/patches/009-add_searchweight.json | 412 +++++++++++++++++++++
.../2000-RDBMS/patches/004-add_searchweight.json | 126 +++++++
.../patches/002-cloud_model_add_searchweight.json | 82 ++++
.../org/apache/atlas/repository/Constants.java | 8 +-
distro/src/conf/solr/schema.xml | 26 ++
distro/src/conf/solr/solrconfig.xml | 22 +-
.../atlas/repository/graphdb/AtlasGraph.java | 31 ++
.../repository/graphdb/AtlasGraphIndexClient.java | 31 ++
.../repository/graphdb/AtlasGraphManagement.java | 8 +
.../graphdb/AtlasIndexQueryParameter.java | 38 ++
.../graphdb/GraphIndexQueryParameters.java | 61 +++
.../repository/graphdb/janus/AtlasJanusGraph.java | 70 +++-
.../graphdb/janus/AtlasJanusGraphManagement.java | 8 +
.../janus/AtlasJanusGraphSolrIndexClient.java | 61 +++
.../janus/AtlasJanusIndexQueryParameter.java | 54 +++
.../janusgraph/diskstorage/solr/Solr6Index.java | 34 +-
.../org/apache/atlas/ApplicationProperties.java | 30 +-
.../apache/atlas/model/typedef/AtlasStructDef.java | 48 ++-
.../v1/model/typedef/AttributeDefinition.java | 32 +-
.../atlas/discovery/FreeTextSearchProcessor.java | 178 +++++++++
.../org/apache/atlas/discovery/SearchContext.java | 38 +-
.../repository/converters/TypeConverterUtil.java | 4 +-
.../atlas/repository/graph/FullTextMapperV2.java | 46 ++-
.../repository/graph/GraphBackedSearchIndexer.java | 84 ++++-
.../bootstrap/AtlasTypeDefStoreInitializer.java | 104 +++++-
.../store/graph/v2/AtlasStructDefStoreV2.java | 9 +
.../store/graph/v2/EntityGraphMapper.java | 36 +-
webapp/pom.xml | 6 +
30 files changed, 1647 insertions(+), 131 deletions(-)
diff --git
a/addons/models/0000-Area0/patches/003-base_model_add_searchWeight.json
b/addons/models/0000-Area0/patches/003-base_model_add_searchWeight.json
new file mode 100644
index 0000000..d0be88f
--- /dev/null
+++ b/addons/models/0000-Area0/patches/003-base_model_add_searchWeight.json
@@ -0,0 +1,82 @@
+{
+ "patches": [
+ {
+ "id": "TYPEDEF_PATCH_0000_033",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "Referenceable",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "qualifiedName",
+ "params": {
+ "searchWeight": 10
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_0000_034",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "Asset",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "name",
+ "params": {
+ "searchWeight": 10
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_0000_035",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "Asset",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "description",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_0000_036",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "Asset",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "owner",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_0000_037",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "AtlasServer",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "name",
+ "params": {
+ "searchWeight": 10
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_0000_038",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "AtlasServer",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "displayName",
+ "params": {
+ "searchWeight": 10
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_0000_039",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "AtlasServer",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "fullName",
+ "params": {
+ "searchWeight": 10
+ }
+ }
+ ]
+}
diff --git a/addons/models/1000-Hadoop/1090-impala_model.json
b/addons/models/1000-Hadoop/1090-impala_model.json
index dad50ff..24eb5db 100644
--- a/addons/models/1000-Hadoop/1090-impala_model.json
+++ b/addons/models/1000-Hadoop/1090-impala_model.json
@@ -49,7 +49,8 @@
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
- "isUnique": false
+ "isUnique": false,
+ "searchWeight": 10
},
{
"name": "queryPlan",
@@ -82,7 +83,8 @@
"isIndexable": false,
"isOptional": true,
"includeInNotification": true,
- "isUnique": false
+ "isUnique": false,
+ "searchWeight": 10
},
{
"name": "queryGraph",
@@ -150,7 +152,8 @@
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
- "isUnique": false
+ "isUnique": false,
+ "searchWeight": 10
},
{
"name": "queryGraph",
diff --git a/addons/models/1000-Hadoop/patches/009-add_searchweight.json
b/addons/models/1000-Hadoop/patches/009-add_searchweight.json
new file mode 100644
index 0000000..1392803
--- /dev/null
+++ b/addons/models/1000-Hadoop/patches/009-add_searchweight.json
@@ -0,0 +1,412 @@
+{
+ "patches": [
+ {
+ "id": "TYPEDEF_PATCH_1000_101",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hive_process",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "userName",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_102",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hive_process",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "queryText",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_103",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hive_process",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "clusterName",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_104",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hive_table",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "comment",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_105",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hive_table",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "viewOriginalText",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_106",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hive_table",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "viewExpandedText",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_107",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hive_storagedesc",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "location",
+ "params": {
+ "searchWeight": 5
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_108",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hive_db",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "location",
+ "params": {
+ "searchWeight": 5
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_109",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hive_db",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "clusterName",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_110",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hive_column",
+ "applyToVersion": "1.4",
+ "updateToVersion": "1.5",
+ "attributeName": "comment",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_111",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "sqoop_process",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "userName",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_112",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "falcon_cluster",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "colo",
+ "params": {
+ "searchWeight": 5
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_113",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "falcon_feed",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "groups",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_114",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "falcon_process",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "pipelines",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_115",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hbase_namespace",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "clusterName",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_116",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hbase_table",
+ "applyToVersion": "1.4",
+ "updateToVersion": "1.5",
+ "attributeName": "uri",
+ "params": {
+ "searchWeight": 5
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_117",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "avro_field",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "doc",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_118",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "avro_record",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "doc",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_119",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "avro_record",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "avro_notation",
+ "params": {
+ "searchWeight": 5
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_120",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "avro_record",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "avroClass",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_121",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "avro_schema",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "namespace",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_122",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "kafka_topic",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "topic",
+ "params": {
+ "searchWeight": 10
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_123",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "kafka_topic",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "uri",
+ "params": {
+ "searchWeight": 5
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_124",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "kafka_topic",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "contactInfo",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_125",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "kafka_topic",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "type",
+ "params": {
+ "searchWeight": 10
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_126",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "kafka_topic",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "keyClassname",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_127",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "jms_topic",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "topic",
+ "params": {
+ "searchWeight": 10
+ }
+ } , {
+ "id": "TYPEDEF_PATCH_1000_128",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "jms_topic",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "uri",
+ "params": {
+ "searchWeight": 10
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_129",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "storm_topology",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "clusterName",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_130",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "storm_node",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "name",
+ "params": {
+ "searchWeight": 10
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_131",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "storm_node",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "description",
+ "params": {
+ "searchWeight": 10
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_132",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "storm_node",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "driverClass",
+ "params": {
+ "searchWeight": 8
+ }
+ } , {
+ "id": "TYPEDEF_PATCH_1000_133",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "fs_path",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "path",
+ "params": {
+ "searchWeight": 5
+ }
+ } , {
+ "id": "TYPEDEF_PATCH_1000_134",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "fs_path",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "group",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_135",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "fs_path",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "posixPermissions",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_136",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hdfs_path",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "clusterName",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_1000_137",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "hdfs_path",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "nameServiceId",
+ "params": {
+ "searchWeight": 10
+ }
+ }
+ ]
+}
diff --git a/addons/models/2000-RDBMS/patches/004-add_searchweight.json
b/addons/models/2000-RDBMS/patches/004-add_searchweight.json
new file mode 100644
index 0000000..16832ba
--- /dev/null
+++ b/addons/models/2000-RDBMS/patches/004-add_searchweight.json
@@ -0,0 +1,126 @@
+{
+ "patches": [
+ {
+ "id": "TYPEDEF_PATCH_2000_127",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_instance",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "rdbms_type",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_2000_128",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_instance",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "platform",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_2000_129",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_instance",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "contact_info",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_2000_130",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_instance",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "hostname",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_2000_131",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_instance",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "comment",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_2000_132",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_db",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "contact_info",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_2000_133",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_table",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "name_path",
+ "params": {
+ "searchWeight": 5
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_2000_134",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_table",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "comment",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_2000_135",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_table",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "contact_info",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_2000_136",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_column",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "attributeName": "comment",
+ "params": {
+ "searchWeight": 9
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_2000_137",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "rdbms_index",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "comment",
+ "params": {
+ "searchWeight": 9
+ }
+ }
+ ]
+}
diff --git
a/addons/models/3000-Cloud/patches/002-cloud_model_add_searchweight.json
b/addons/models/3000-Cloud/patches/002-cloud_model_add_searchweight.json
new file mode 100644
index 0000000..5dfe169
--- /dev/null
+++ b/addons/models/3000-Cloud/patches/002-cloud_model_add_searchweight.json
@@ -0,0 +1,82 @@
+{
+ "patches": [
+ {
+ "id": "TYPEDEF_PATCH_3000_138",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "aws_s3_pseudo_dir",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "subject",
+ "params": {
+ "searchWeight": 10
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_3000_139",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "aws_s3_pseudo_dir",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "objectPrefix",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_3000_140",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "aws_s3_pseudo_dir",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "currentStorageLocation",
+ "params": {
+ "searchWeight": 5
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_3000_141",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "aws_s3_pseudo_dir",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "subdomain",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_3000_142",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "aws_s3_object",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "partner",
+ "params": {
+ "searchWeight": 5
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_3000_143",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "aws_s3_object",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "domain",
+ "params": {
+ "searchWeight": 8
+ }
+ }, {
+ "id": "TYPEDEF_PATCH_3000_144",
+ "description": "Adding search Weights",
+ "action": "UPDATE_ATTRIBUTE_METADATA",
+ "typeName": "aws_s3_object",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "attributeName": "region",
+ "params": {
+ "searchWeight": 8
+ }
+ }
+ ]
+}
diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java
b/common/src/main/java/org/apache/atlas/repository/Constants.java
index 130456d..c0851fe 100644
--- a/common/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/common/src/main/java/org/apache/atlas/repository/Constants.java
@@ -81,9 +81,11 @@ public final class Constants {
public static final String TRAIT_NAMES_PROPERTY_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames");
public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames");
- public static final String VERSION_PROPERTY_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version");
- public static final String STATE_PROPERTY_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
- public static final String CREATED_BY_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
+ public static final String VERSION_PROPERTY_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version");
+ public static final String STATE_PROPERTY_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
+ public static final String CREATED_BY_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
+ public static final String CLASSIFICATION_TEXT_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
+
public static final String MODIFIED_BY_KEY =
encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
/**
diff --git a/distro/src/conf/solr/schema.xml b/distro/src/conf/solr/schema.xml
index 1d445b1..3f82ed7 100644
--- a/distro/src/conf/solr/schema.xml
+++ b/distro/src/conf/solr/schema.xml
@@ -524,11 +524,37 @@
class="solr.UUIDField"
indexed="true" />
+ <fieldType name="freetext" class="solr.TextField" omitNorms="true"
+ omitTermFreqAndPositions="true">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.EdgeNGramFilterFactory" minGramSize="2"
maxGramSize="16" />
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.KeywordTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+
<dynamicField name="*_uuid" type="uuid" indexed="true" stored="true"/>
<!-- TTL -->
<field name="ttl" type="string" indexed="true" stored="true" />
<field name="expire_at" type="date" indexed="true" stored="true" />
<field name="timestamp" type="date" indexed="true" stored="true" />
+ <field name="allt1s" type="freetext" multiValued="true" indexed="true"
stored="false"/>
+ <field name="allt2s" type="freetext" multiValued="true" indexed="true"
stored="false"/>
+ <field name="allt3s" type="freetext" multiValued="true" indexed="true"
stored="false"/>
+ <field name="allt4s" type="freetext" multiValued="true" indexed="true"
stored="false"/>
+ <field name="allt5s" type="freetext" multiValued="true" indexed="true"
stored="false"/>
+ <field name="allt6s" type="freetext" multiValued="true" indexed="true"
stored="false"/>
+ <field name="allt7s" type="freetext" multiValued="true" indexed="true"
stored="false"/>
+ <field name="allt8s" type="freetext" multiValued="true" indexed="true"
stored="false"/>
+ <field name="allt9s" type="freetext" multiValued="true" indexed="true"
stored="false"/>
+ <field name="allt10s" type="freetext" multiValued="true" indexed="true"
stored="false"/>
+
</schema>
diff --git a/distro/src/conf/solr/solrconfig.xml
b/distro/src/conf/solr/solrconfig.xml
index 1d414f7..a83e03b 100644
--- a/distro/src/conf/solr/solrconfig.xml
+++ b/distro/src/conf/solr/solrconfig.xml
@@ -110,9 +110,13 @@
When ManagedIndexSchemaFactory is specified with mutable = true, schema
modification REST API calls will be allowed; otherwise, error responses
will be
sent back for these requests.
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
-->
- <schemaFactory class="ClassicIndexSchemaFactory"/>
+ <schemaFactory class="ManagedIndexSchemaFactory">
+ <bool name="mutable">true</bool>
+ <str name="managedSchemaResourceName">managed-schema</str>
+ </schemaFactory>
<!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Index Config - These settings control low-level behavior of indexing
@@ -474,6 +478,22 @@
</lst>
</requestHandler>
+ <requestHandler name="/freetext" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="defType">edismax</str>
+ <int name="rows">100</int>
+ <str name="lowercaseOperators">true</str>
+ <str name="qf">
+ allt10s^10 allt9s^9 allt8s^8 allt7s^7 allt6s^6 allt5s^5 allt4s^4
allt3s^3 allt2s^2 allt1s^1
+ </str>
+ <str name="hl.fl">*</str>
+ <str name="hl.requireFieldMatch">true</str>
+ <str name="lowercaseOperators">true</str>
+ <str name="facet.limit">5</str>
+ </lst>
+ </requestHandler>
+
+
<!--
The export request handler is used to export full sorted result sets.
Do not change these defaults.
diff --git
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
index d282c99..53a5a7a 100644
---
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
+++
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
@@ -17,6 +17,7 @@
*/
package org.apache.atlas.repository.graphdb;
+import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.type.AtlasType;
@@ -177,6 +178,14 @@ public interface AtlasGraph<V, E> {
AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString, int
offset);
/**
+ * Creates an index query.
+ *
+ * @param indexQueryParameters the parameterObject containing the
information needed for creating the index.
+ *
+ */
+ AtlasIndexQuery<V, E> indexQuery(GraphIndexQueryParameters
indexQueryParameters);
+
+ /**
* Gets the management object associated with this graph and opens a
transaction
* for changes that are made.
* @return
@@ -329,4 +338,26 @@ public interface AtlasGraph<V, E> {
* @return
*/
boolean isMultiProperty(String name);
+
+ /**
+ * return the encoded name used for the attribute identified by property
key and index name.
+ * @param propertyKey the property key of attributed
+ * @param indexName the name of the index containing the property.
+ * @return the encoded name of the property.
+ */
+ String getIndexFieldName(AtlasPropertyKey propertyKey, String indexName);
+ /**
+ * Create Index query parameter for use with atlas graph.
+ * @param parameterName the name of the parameter that needs to be passed
to index layer.
+ * @param parameterValue the value of the paratemeter that needs to be
passed to the index layer.
+ * @return
+ */
+ AtlasIndexQueryParameter indexQueryParameter(String parameterName, String
parameterValue);
+
+ /**
+ * Implementors should return graph index client.
+ * @return the graph index client
+ * @throws AtlasException when error encountered in creating the client.
+ */
+ AtlasGraphIndexClient getGraphIndexClient()throws AtlasException;
}
diff --git
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
new file mode 100644
index 0000000..482e6f6
--- /dev/null
+++
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb;
+
+/**
+ * Represents a graph client work with indices used by Jansgraph.
+ */
+public interface AtlasGraphIndexClient {
+ /**
+ * The implementers should create a mapping from source propertyName to
mapping field name.
+ * @param indexName the name of index that needs to be worked with.
+ * @param sourcePropertyName the name of the source attribute.
+ * @param targetPropertyName the name of the target attribute to which the
mapping is getting done.
+ */
+ void createCopyField(String indexName, String sourcePropertyName, String
targetPropertyName);
+}
diff --git
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
index b1e2c5d..6fe3460 100644
---
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
+++
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
@@ -157,4 +157,12 @@ public interface AtlasGraphManagement {
* @param propertyKey
*/
void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
+
+ /**
+ * Gets the index field name for the vertex property.
+ * @param indexName
+ * @param propertyKey
+ * @return the encoded name for the index
+ */
+ String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey);
}
diff --git
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java
new file mode 100644
index 0000000..347f216
--- /dev/null
+++
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb;
+
+import java.util.Iterator;
+
+/**
+ * Represents an index query parameter for use with AtlasGraph queries.
+ */
+public interface AtlasIndexQueryParameter {
+
+ /**
+ * @return the name of the parameter for index search.
+ */
+ public String getParameterName();
+
+ /**
+ * @return the value of the parameter for index search.
+ */
+ public String getParameterValue();
+
+}
diff --git
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphIndexQueryParameters.java
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphIndexQueryParameters.java
new file mode 100644
index 0000000..d9a22e3
--- /dev/null
+++
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphIndexQueryParameters.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb;
+
+import java.util.List;
+
+/**
+ * A class wrapping the parameters that need to create a graph index query.
+ */
+public class GraphIndexQueryParameters {
+ final String indexName;
+ final String graphQueryString;
+ final int offset;
+ final List<AtlasIndexQueryParameter> indexQueryParameters;
+
+ /**
+ *
+ * @param indexName the name of the index on which the query is being made
+ * @param graphQueryString the graph query string.
+ * @param offset the offset to be used for data fetch.
+ * @param indexQueryParameters any index system specific parameters for
use in query.
+ */
+ public GraphIndexQueryParameters(String indexName, String
graphQueryString, int offset, List<AtlasIndexQueryParameter>
indexQueryParameters) {
+ this.indexName = indexName;
+ this.graphQueryString = graphQueryString;
+ this.offset = offset;
+ this.indexQueryParameters = indexQueryParameters;
+ }
+
+
+ public String getIndexName() {
+ return indexName;
+ }
+
+ public String getGraphQueryString() {
+ return graphQueryString;
+ }
+
+ public int getOffset() {
+ return offset;
+ }
+
+ public List<AtlasIndexQueryParameter> getIndexQueryParameters() {
+ return indexQueryParameters;
+ }
+}
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index 840ebca..e973f72 100644
---
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -25,15 +25,7 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
-import org.apache.atlas.repository.graphdb.AtlasEdge;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
-import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
-import org.apache.atlas.repository.graphdb.AtlasGraphTraversal;
-import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
-import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.repository.graphdb.GremlinVersion;
+import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.graphdb.janus.query.AtlasJanusGraphQuery;
import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
import org.apache.atlas.type.AtlasType;
@@ -59,20 +51,18 @@ import org.janusgraph.core.PropertyKey;
import org.janusgraph.core.SchemaViolationException;
import org.janusgraph.core.schema.JanusGraphIndex;
import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.Parameter;
import org.janusgraph.diskstorage.BackendException;
import org.janusgraph.graphdb.database.StandardJanusGraph;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.io.IOException;
import java.io.OutputStream;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@@ -84,12 +74,13 @@ import static
org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase.
* Janus implementation of AtlasGraph.
*/
public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex,
AtlasJanusEdge> {
-
+ private static final Logger LOG =
LoggerFactory.getLogger(AtlasJanusGraph.class);
private static Configuration APPLICATION_PROPERTIES = null;
private final ConvertGremlinValueFunction
GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction();
private final Set<String> multiProperties
= new HashSet<>();
private final StandardJanusGraph janusGraph;
+ private final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0];
public AtlasJanusGraph() {
this(getGraphInstance());
@@ -198,6 +189,21 @@ public class AtlasJanusGraph implements
AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
@Override
+ public AtlasIndexQueryParameter indexQueryParameter(String parameterName,
String parameterValue) {
+ return new AtlasJanusIndexQueryParameter(parameterName,
parameterValue);
+ }
+
+ @Override
+ public AtlasGraphIndexClient getGraphIndexClient() throws AtlasException {
+ try {
+ return new AtlasJanusGraphSolrIndexClient();
+ } catch (Exception e) {
+ LOG.error("Error encountered in creating Graph Index Client.", e);
+ throw new AtlasException(e);
+ }
+ }
+
+ @Override
public void commit() {
getGraph().tx().commit();
}
@@ -208,15 +214,34 @@ public class AtlasJanusGraph implements
AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
@Override
- public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String
fulltextIndex, String graphQuery) {
- return indexQuery(fulltextIndex, graphQuery, 0);
+ public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String
indexName, String graphQuery) {
+ return indexQuery(indexName, graphQuery, 0, null);
}
@Override
- public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String
fulltextIndex, String graphQuery, int offset) {
+ public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String
indexName, String graphQuery, int offset) {
+ return indexQuery(indexName, graphQuery, offset, null);
+ }
+
+ /**
+ * Creates an index query.
+ *
+ * @param indexQueryParameters the parameterObject containing the
information needed for creating the index.
+ *
+ */
+ public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge>
indexQuery(GraphIndexQueryParameters indexQueryParameters) {
+ return indexQuery(indexQueryParameters.getIndexName(),
indexQueryParameters.getGraphQueryString(), indexQueryParameters.getOffset(),
indexQueryParameters.getIndexQueryParameters());
+ }
+
+ private AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge>
indexQuery(String indexName, String graphQuery, int offset,
List<AtlasIndexQueryParameter> indexQueryParameterList) {
String prefix = getIndexQueryPrefix();
- JanusGraphIndexQuery query = getGraph().indexQuery(fulltextIndex,
graphQuery).setElementIdentifier(prefix).offset(offset);
+ JanusGraphIndexQuery query = getGraph().indexQuery(indexName,
graphQuery).setElementIdentifier(prefix).offset(offset);
+ if(indexQueryParameterList != null && indexQueryParameterList.size() >
0) {
+ for(AtlasIndexQueryParameter indexQueryParameter:
indexQueryParameterList) {
+ query = query.addParameter(new
Parameter(indexQueryParameter.getParameterName(),
indexQueryParameter.getParameterValue()));
+ }
+ }
return new AtlasJanusIndexQuery(this, query);
}
@@ -497,4 +522,9 @@ public class AtlasJanusGraph implements
AtlasGraph<AtlasJanusVertex, AtlasJanusE
return convertGremlinValue(input);
}
}
+
+ public String getIndexFieldName(AtlasPropertyKey propertyKey, String
indexName) {
+ PropertyKey janusKey =
AtlasJanusObjectFactory.createPropertyKey(propertyKey);
+ return janusGraph.getIndexSerializer().getDefaultFieldName(janusKey,
EMPTY_PARAMETER_ARRAY, indexName);
+ }
}
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
index 4e18432..feee036 100644
---
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
@@ -200,6 +200,14 @@ public class AtlasJanusGraphManagement implements
AtlasGraphManagement {
JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
management.addIndexKey(vertexIndex, janusKey);
+ String encodedName = graph.getIndexFieldName(propertyKey,
vertexIndex.getBackingIndex());
+ LOG.info("property '{}' is encoded to '{}'.", propertyKey.getName(),
encodedName);
+ }
+
+ @Override
+ public String getIndexFieldName(String indexName, AtlasPropertyKey
propertyKey) {
+ JanusGraphIndex index = management.getGraphIndex(indexName);
+ return graph.getIndexFieldName(propertyKey, index.getBackingIndex());
}
@Override
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java
new file mode 100644
index 0000000..f8f8061
--- /dev/null
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse;
+import org.janusgraph.diskstorage.solr.Solr6Index;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class AtlasJanusGraphSolrIndexClient implements AtlasGraphIndexClient {
+ private static final Logger LOG =
LoggerFactory.getLogger(AtlasJanusGraphSolrIndexClient.class);
+ private final SolrClient solrClient;
+
+ public AtlasJanusGraphSolrIndexClient() throws Exception {
+ //well, this is temp hack to get solr client using same settings as
that of Janus Graph
+ solrClient = Solr6Index.getSolrClient();
+ if(solrClient == null) {
+ LOG.warn("The indexing system is not solr based. Non SOLR based
indexing systems are not supported yet.");
+ }
+ }
+
+ @Override
+ public void createCopyField(String collectionName, String srcFieldName,
String mappedCopyFieldName) {
+ if(solrClient == null) {
+ LOG.error("The indexing system is not solr based. Copy fields can
not be created in non SOLR based indexing systems. This request will be treated
as no op.");
+ return;
+ }
+ SchemaRequest.AddCopyField addCopyFieldRequest =
+ new SchemaRequest.AddCopyField(srcFieldName,
Arrays.asList(mappedCopyFieldName));
+ SchemaResponse.UpdateResponse addCopyFieldResponse = null;
+ try {
+ addCopyFieldResponse = addCopyFieldRequest.process(solrClient,
collectionName);
+ } catch (SolrServerException | IOException e) {
+ String msg = String.format("Error encountered in creating the copy
field from %s to %s for collection %s.", srcFieldName, mappedCopyFieldName,
collectionName);
+ LOG.error(msg);
+ throw new RuntimeException(msg, e);
+ }
+ }
+}
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQueryParameter.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQueryParameter.java
new file mode 100644
index 0000000..b719b89
--- /dev/null
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQueryParameter.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasIndexQueryParameter;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.janusgraph.core.JanusGraphIndexQuery;
+import org.janusgraph.core.JanusGraphVertex;
+
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Janus implementation of AtlasIndexQueryParameter.
+ */
+public class AtlasJanusIndexQueryParameter implements AtlasIndexQueryParameter
{
+
+ private final String parameterName;
+ private final String parameterValue;
+
+ public AtlasJanusIndexQueryParameter(String parameterName, String
parameterValue) {
+ this.parameterName = parameterName;
+ this.parameterValue = parameterValue;
+ }
+
+ @Override
+ public String getParameterName() {
+ return parameterName;
+ }
+
+ @Override
+ public String getParameterValue() {
+ return parameterValue;
+ }
+}
diff --git
a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
index b300668..0d5ae4f 100644
---
a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
+++
b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
@@ -135,9 +135,9 @@ public class Solr6Index implements IndexProvider {
private static final Logger logger =
LoggerFactory.getLogger(Solr6Index.class);
-
private static final String DEFAULT_ID_FIELD = "id";
private static final char CHROOT_START_CHAR = '/';
+ private static Configuration config;
private enum Mode {
HTTP, CLOUD;
@@ -181,7 +181,7 @@ public class Solr6Index implements IndexProvider {
public Solr6Index(final Configuration config) throws BackendException {
// Add Kerberos-enabled SolrHttpClientBuilder
HttpClientUtil.setHttpClientBuilder(new
Krb5HttpClientBuilder().getBuilder());
-
+ initConfiguration(config);
Preconditions.checkArgument(config!=null);
configuration = config;
mode = Mode.parse(config.get(SOLR_MODE));
@@ -200,23 +200,39 @@ public class Solr6Index implements IndexProvider {
logger.debug("KERBEROS_ENABLED name is " +
KERBEROS_ENABLED.getName() + " and it is" + (KERBEROS_ENABLED.isOption() ? " "
: " not") + " an option.");
logger.debug("KERBEROS_ENABLED type is " +
KERBEROS_ENABLED.getType().name());
}
+ solrClient = getSolrClient();
+
+ }
+
+ private static void initConfiguration(Configuration config) {
+ if(Solr6Index.config == null) {
+ Solr6Index.config = config;
+ }
+ }
+
+ public static SolrClient getSolrClient() {
final ModifiableSolrParams clientParams = new ModifiableSolrParams();
+ SolrClient solrClient = null;
+ if(Solr6Index.config == null) {
+ logger.error("The solr client is not being used for the indexing
purposes.");
+ return null;
+ }
+ Configuration config = Solr6Index.config;
+ Mode mode = Mode.parse(config.get(SOLR_MODE));
switch (mode) {
case CLOUD:
- /* ATLAS-2920: Update JanusGraph Solr clients to use all
zookeeper entries – start */
- final List<String> zookeeperUrls = getZookeeperURLs(config);
- /* ATLAS-2920: end */
final CloudSolrClient cloudServer = new
CloudSolrClient.Builder()
.withLBHttpSolrClientBuilder(
new LBHttpSolrClient.Builder()
.withHttpSolrClientBuilder(new
HttpSolrClient.Builder().withInvariantParams(clientParams))
.withBaseSolrUrls(config.get(HTTP_URLS))
)
- .withZkHost(zookeeperUrls)
+ .withZkHost(getZookeeperURLs(config))
.sendUpdatesOnlyToShardLeaders()
.build();
cloudServer.connect();
solrClient = cloudServer;
+ logger.info("Created solr client using Cloud based
configuration.");
break;
case HTTP:
clientParams.add(HttpClientUtil.PROP_ALLOW_COMPRESSION,
config.get(HTTP_ALLOW_COMPRESSION).toString());
@@ -228,12 +244,12 @@ public class Solr6Index implements IndexProvider {
.withHttpClient(client)
.withBaseSolrUrls(config.get(HTTP_URLS))
.build();
-
-
+ logger.info("Created solr client using HTTP based
configuration.");
break;
default:
throw new IllegalArgumentException("Unsupported Solr operation
mode: " + mode);
}
+ return solrClient;
}
private void configureSolrClientsForKerberos() throws
PermanentBackendException {
@@ -1146,7 +1162,7 @@ public class Solr6Index implements IndexProvider {
}
/* ATLAS-2920: Update JanusGraph Solr clients to use all zookeeper entries
– start */
- private List<String> getZookeeperURLs(Configuration config) {
+ private static List<String> getZookeeperURLs(Configuration config) {
List<String> ret = null;
String[] zkHosts = config.get(ZOOKEEPER_URLS);
diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
index 3f43496..6d0153f 100644
--- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
+++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
@@ -42,20 +42,22 @@ public final class ApplicationProperties extends
PropertiesConfiguration {
public static final String ATLAS_CONFIGURATION_DIRECTORY_PROPERTY =
"atlas.conf";
private static final Logger LOG =
LoggerFactory.getLogger(ApplicationProperties.class);
- public static final String APPLICATION_PROPERTIES =
"atlas-application.properties";
-
- public static final String GRAPHDB_BACKEND_CONF =
"atlas.graphdb.backend";
- public static final String STORAGE_BACKEND_CONF =
"atlas.graph.storage.backend";
- public static final String INDEX_BACKEND_CONF =
"atlas.graph.index.search.backend";
- public static final String INDEX_MAP_NAME_CONF =
"atlas.graph.index.search.map-name";
- public static final String SOLR_WAIT_SEARCHER_CONF =
"atlas.graph.index.search.solr.wait-searcher";
- public static final String GRAPHBD_BACKEND_JANUS = "janus";
- public static final String STORAGE_BACKEND_HBASE = "hbase";
- public static final String STORAGE_BACKEND_HBASE2 = "hbase2";
- public static final String INDEX_BACKEND_SOLR = "solr";
- public static final String DEFAULT_GRAPHDB_BACKEND =
GRAPHBD_BACKEND_JANUS;
- public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = true;
- public static final boolean DEFAULT_INDEX_MAP_NAME = false;
+ public static final String APPLICATION_PROPERTIES =
"atlas-application.properties";
+
+ public static final String GRAPHDB_BACKEND_CONF =
"atlas.graphdb.backend";
+ public static final String STORAGE_BACKEND_CONF =
"atlas.graph.storage.backend";
+ public static final String INDEX_BACKEND_CONF =
"atlas.graph.index.search.backend";
+ public static final String INDEX_MAP_NAME_CONF =
"atlas.graph.index.search.map-name";
+ public static final String FREE_TEXT_INDEX_ENABLED =
"atlas.freetext.index.enabled";
+ public static final String SOLR_WAIT_SEARCHER_CONF =
"atlas.graph.index.search.solr.wait-searcher";
+ public static final String GRAPHBD_BACKEND_JANUS = "janus";
+ public static final String STORAGE_BACKEND_HBASE = "hbase";
+ public static final String STORAGE_BACKEND_HBASE2 = "hbase2";
+ public static final String INDEX_BACKEND_SOLR = "solr";
+ public static final String DEFAULT_GRAPHDB_BACKEND =
GRAPHBD_BACKEND_JANUS;
+ public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = true;
+ public static final boolean DEFAULT_INDEX_MAP_NAME = false;
+ public static final boolean DEFAULT_FREE_TEXT_INDEX_ENABLED = false;
public static final SimpleEntry<String, String> DB_CACHE_CONF
= new SimpleEntry<>("atlas.graph.cache.db-cache", "true");
public static final SimpleEntry<String, String> DB_CACHE_CLEAN_WAIT_CONF
= new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20");
diff --git
a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
index d244388..7b5b675 100644
--- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
+++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
@@ -259,10 +259,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef
implements Serializable {
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
public static class AtlasAttributeDef implements Serializable {
- private static final long serialVersionUID = 1L;
-
+ private static final long serialVersionUID = 1L;
+ public static final int DEFAULT_SEARCHWEIGHT = -1;
+ public static final int DEFAULT_SEARCHWEIGHT_FOR_STRINGS = 3;
+ public static final String SEARCH_WEIGHT_ATTR_NAME =
"searchWeight";
public static final String ATTRDEF_OPTION_SOFT_REFERENCE =
"isSoftReference";
- private final String STRING_TRUE = "true";
+ private final String STRING_TRUE = "true";
/**
* single-valued attribute or multi-valued attribute.
@@ -282,6 +284,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef
implements Serializable {
private boolean includeInNotification;
private String defaultValue;
private String description;
+ private int searchWeight = DEFAULT_SEARCHWEIGHT;
private List<AtlasConstraintDef> constraints;
private Map<String, String> options;
@@ -289,21 +292,36 @@ public class AtlasStructDef extends AtlasBaseTypeDef
implements Serializable {
public AtlasAttributeDef() { this(null, null); }
public AtlasAttributeDef(String name, String typeName) {
- this(name, typeName, false, Cardinality.SINGLE);
+ this(name, typeName, DEFAULT_SEARCHWEIGHT);
+ }
+
+ public AtlasAttributeDef(String name, String typeName, int
searchWeight) {
+ this(name, typeName, false, Cardinality.SINGLE, searchWeight);
}
public AtlasAttributeDef(String name, String typeName, boolean
isOptional, Cardinality cardinality) {
- this(name, typeName, isOptional, cardinality, COUNT_NOT_SET,
COUNT_NOT_SET, false, false, false, null);
+ this(name, typeName, isOptional, cardinality,
DEFAULT_SEARCHWEIGHT);
}
+ private AtlasAttributeDef(String name, String typeName, boolean
isOptional, Cardinality cardinality, int searchWeight) {
+ this(name, typeName, isOptional, cardinality, COUNT_NOT_SET,
COUNT_NOT_SET, false, false, false, null, searchWeight);
+ }
+
+
+
public AtlasAttributeDef(String name, String typeName, boolean
isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount,
boolean isUnique, boolean isIndexable, boolean includeInNotification,
List<AtlasConstraintDef> constraints) {
- this(name, typeName, isOptional, cardinality, valuesMinCount,
valuesMaxCount, isUnique, isIndexable, includeInNotification, null,
constraints, null, null);
+ this(name, typeName, isOptional, cardinality, valuesMinCount,
valuesMaxCount, isUnique, isIndexable, includeInNotification, constraints,
DEFAULT_SEARCHWEIGHT);
+ }
+
+ private AtlasAttributeDef(String name, String typeName, boolean
isOptional, Cardinality cardinality,
+ int valuesMinCount, int valuesMaxCount,
boolean isUnique, boolean isIndexable, boolean includeInNotification,
List<AtlasConstraintDef> constraints, int searchWeight) {
+ this(name, typeName, isOptional, cardinality, valuesMinCount,
valuesMaxCount, isUnique, isIndexable, includeInNotification, null,
constraints, null, null, searchWeight);
}
public AtlasAttributeDef(String name, String typeName, boolean
isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount,
boolean isUnique, boolean isIndexable, boolean includeInNotification, String
defaultValue,
- List<AtlasConstraintDef> constraints,
Map<String,String> options, String description) {
+ List<AtlasConstraintDef> constraints,
Map<String,String> options, String description, int searchWeight) {
setName(name);
setTypeName(typeName);
setIsOptional(isOptional);
@@ -317,6 +335,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef
implements Serializable {
setConstraints(constraints);
setOptions(options);
setDescription(description);
+ setSearchWeight(searchWeight);
}
public AtlasAttributeDef(AtlasAttributeDef other) {
@@ -334,9 +353,18 @@ public class AtlasStructDef extends AtlasBaseTypeDef
implements Serializable {
setConstraints(other.getConstraints());
setOptions(other.getOptions());
setDescription((other.getDescription()));
+ setSearchWeight(other.getSearchWeight());
}
}
+ public int getSearchWeight() {
+ return searchWeight;
+ }
+
+ public void setSearchWeight(int searchWeight) {
+ this.searchWeight = searchWeight;
+ }
+
public String getName() {
return name;
}
@@ -481,6 +509,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef
implements Serializable {
sb.append(",
includeInNotification=").append(includeInNotification);
sb.append(", defaultValue=").append(defaultValue);
sb.append(", options='").append(options).append('\'');
+ sb.append(", searchWeight='").append(searchWeight).append('\'');
sb.append(", constraints=[");
if (CollectionUtils.isNotEmpty(constraints)) {
int i = 0;
@@ -515,12 +544,13 @@ public class AtlasStructDef extends AtlasBaseTypeDef
implements Serializable {
Objects.equals(defaultValue, that.defaultValue) &&
Objects.equals(description, that.description) &&
Objects.equals(constraints, that.constraints) &&
- Objects.equals(options, that.options);
+ Objects.equals(options, that.options) &&
+ Objects.equals(searchWeight, that.searchWeight);
}
@Override
public int hashCode() {
- return Objects.hash(name, typeName, isOptional, cardinality,
valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification,
defaultValue, constraints, options, description);
+ return Objects.hash(name, typeName, isOptional, cardinality,
valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification,
defaultValue, constraints, options, description, searchWeight);
}
@Override
diff --git
a/intg/src/main/java/org/apache/atlas/v1/model/typedef/AttributeDefinition.java
b/intg/src/main/java/org/apache/atlas/v1/model/typedef/AttributeDefinition.java
index e353856..d9a3a81 100644
---
a/intg/src/main/java/org/apache/atlas/v1/model/typedef/AttributeDefinition.java
+++
b/intg/src/main/java/org/apache/atlas/v1/model/typedef/AttributeDefinition.java
@@ -32,6 +32,7 @@ import java.util.Objects;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+import static
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY,
fieldVisibility=NONE)
@@ -40,7 +41,6 @@ import static
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_
@XmlAccessorType(XmlAccessType.PROPERTY)
public class AttributeDefinition implements Serializable {
private static final long serialVersionUID = 1L;
-
private String name;
private String dataTypeName;
private Multiplicity multiplicity;
@@ -51,22 +51,34 @@ public class AttributeDefinition implements Serializable {
private String defaultValue;
private String description;
private Map<String, String> options;
+ private int searchWeight = DEFAULT_SEARCHWEIGHT;
public AttributeDefinition() {
}
public AttributeDefinition(String name, String dataTypeName, Multiplicity
multiplicity) {
- this(name, dataTypeName, multiplicity, false, false, true, null, null);
+ this(name, dataTypeName, multiplicity, false, false, true, null, null,
DEFAULT_SEARCHWEIGHT);
}
public AttributeDefinition(String name, String dataTypeName, Multiplicity
multiplicity, boolean isComposite,
String reverseAttributeName) {
- this(name, dataTypeName, multiplicity, isComposite, false, false,
reverseAttributeName, null);
+ this(name, dataTypeName, multiplicity, isComposite,
reverseAttributeName, DEFAULT_SEARCHWEIGHT);
+ }
+
+ public AttributeDefinition(String name, String dataTypeName, Multiplicity
multiplicity, boolean isComposite,
+ String reverseAttributeName, int searchWeight) {
+ this(name, dataTypeName, multiplicity, isComposite, false, false,
reverseAttributeName, null, searchWeight);
}
public AttributeDefinition(String name, String dataTypeName, Multiplicity
multiplicity, boolean isComposite,
boolean isUnique, boolean isIndexable, String
reverseAttributeName,
Map<String, String> options) {
+ this(name, dataTypeName, multiplicity, isComposite, isUnique,
isIndexable,reverseAttributeName, options, DEFAULT_SEARCHWEIGHT);
+ }
+
+ public AttributeDefinition(String name, String dataTypeName, Multiplicity
multiplicity, boolean isComposite,
+ boolean isUnique, boolean isIndexable, String
reverseAttributeName,
+ Map<String, String> options, int searchWeight) {
this.name = name;
this.dataTypeName = dataTypeName;
this.multiplicity = multiplicity;
@@ -75,6 +87,7 @@ public class AttributeDefinition implements Serializable {
this.isIndexable = isIndexable;
this.reverseAttributeName = reverseAttributeName;
this.options = options;
+ this.searchWeight = searchWeight;
}
@@ -183,12 +196,21 @@ public class AttributeDefinition implements Serializable {
Objects.equals(defaultValue, that.defaultValue) &&
Objects.equals(description, that.description) &&
Objects.equals(reverseAttributeName, that.reverseAttributeName)
&&
- Objects.equals(options, that.options);
+ Objects.equals(options, that.options) &&
+ Objects.equals(searchWeight, that.searchWeight);
}
@Override
public int hashCode() {
return Objects.hash(name, dataTypeName, multiplicity, isComposite,
isUnique, isIndexable,
- reverseAttributeName, defaultValue, description,
options);
+ reverseAttributeName, defaultValue, description,
options, searchWeight);
}
+
+ public void setSearchWeight(int searchWeight) {
+ this.searchWeight = searchWeight;
+ }
+
+ public int getSearchWeight() {
+ return searchWeight;
+ }
}
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
new file mode 100644
index 0000000..b1a84c7
--- /dev/null
+++
b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
+import org.apache.atlas.utils.AtlasPerfTracer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * This class is equivalent to legacy FullTextSearchProcessor--except that it
uses a better search techniques using SOLR
+ * than going through Janus Graph index apis.
+ */
+public class FreeTextSearchProcessor extends SearchProcessor {
+ private static final Logger LOG =
LoggerFactory.getLogger(FreeTextSearchProcessor.class);
+ private static final Logger PERF_LOG =
AtlasPerfTracer.getPerfLogger("FreeTextSearchProcessor");
+ public static final String SOLR_QT_PARAMETER = "qt";
+ public static final String SOLR_REQUEST_HANDLER_NAME = "/freetext";
+ private static final int MAX_TYPES_STRING_SIZE = 1000;
+
+ private final AtlasIndexQuery indexQuery;
+
+ public FreeTextSearchProcessor(SearchContext context) {
+ super(context);
+
+ SearchParameters searchParameters = context.getSearchParameters();
+ StringBuilder queryString = new StringBuilder();
+
+ queryString.append(searchParameters.getQuery());
+
+ String queryFields = null;
+ // if search includes entity-type criteria, adding a filter here can
help avoid unnecessary
+ // processing (and rejection) by subsequent EntitySearchProcessor
+ if (context.getEntityType() != null) {
+ String typeString =
context.getEntityType().getTypeAndAllSubTypesQryStr();
+ if (typeString.length() > MAX_TYPES_STRING_SIZE) {
+ LOG.info("Dropping the use of types string optimization as
there are too many types {} for select type {}.", typeString,
context.getEntityType().getTypeName());
+ } else {
+ LOG.debug("Using the use of types string optimization as there
are too many types {} for select type {}.", typeString,
context.getEntityType().getTypeName());
+
+ final Set<String> types =
context.getEntityType().getTypeAndAllSubTypes();
+ final AtlasGraphManagement managementSystem =
context.getGraph().getManagementSystem();
+ AtlasPropertyKey entityTypeNamePropertyKey =
managementSystem.getPropertyKey(AtlasGraphUtilsV2.encodePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY));
+ String encodedPropertyName =
managementSystem.getIndexFieldName(Constants.VERTEX_INDEX,
entityTypeNamePropertyKey);
+
+
+
+ StringBuilder typesStringBuilder = new StringBuilder();
+ for(String typeName: types) {
+ typesStringBuilder.append(" ").append(typeName);
+ }
+ //append the query with type and substypes listed in it
+ String typesString = typesStringBuilder.toString();
+ queryString.append(" AND
+").append(encodedPropertyName).append(":[");
+ queryString.append(typesStringBuilder.toString());
+ queryString.append("]");
+ }
+ }
+
+ //just use the query string as is
+ LOG.debug("Using query string '{}'.", queryString);
+ indexQuery =
context.getGraph().indexQuery(prepareGraphIndexQueryParameters(context,
queryString));
+ }
+
+ private GraphIndexQueryParameters
prepareGraphIndexQueryParameters(SearchContext context, StringBuilder
queryString) {
+ List<AtlasIndexQueryParameter> parameters = new
ArrayList<AtlasIndexQueryParameter>();
+
parameters.add(context.getGraph().indexQueryParameter(SOLR_QT_PARAMETER,
SOLR_REQUEST_HANDLER_NAME));
+ return new GraphIndexQueryParameters(Constants.VERTEX_INDEX,
queryString.toString(), 0, parameters);
+ }
+
+ @Override
+ public List<AtlasVertex> execute() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> FullTextSearchProcessorUsingFreeText.execute({})",
context);
+ }
+
+ List<AtlasVertex> ret = new ArrayList<>();
+
+ AtlasPerfTracer perf = null;
+
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG,
"FullTextSearchProcessorUsingFreeText.execute(" + context + ")");
+ }
+
+ try {
+ final int startIdx =
context.getSearchParameters().getOffset();
+ final int limit =
context.getSearchParameters().getLimit();
+ final boolean activeOnly =
context.getSearchParameters().getExcludeDeletedEntities();
+
+ // query to start at 0, even though startIdx can be higher -
because few results in earlier retrieval could
+ // have been dropped: like vertices of non-entity or
non-active-entity
+ //
+ // first 'startIdx' number of entries will be ignored
+ int qryOffset = 0;
+ int resultIdx = qryOffset;
+
+ final List<AtlasVertex> entityVertices = new ArrayList<>();
+ try {
+ for (; ret.size() < limit; qryOffset += limit) {
+ entityVertices.clear();
+
+ if (context.terminateSearch()) {
+ LOG.warn("query terminated: {}",
context.getSearchParameters());
+
+ break;
+ }
+
+ Iterator<AtlasIndexQuery.Result> idxQueryResult =
indexQuery.vertices(qryOffset, limit);
+
+ final boolean isLastResultPage;
+ int resultCount = 0;
+
+ while (idxQueryResult.hasNext()) {
+ AtlasVertex vertex = idxQueryResult.next().getVertex();
+
+ resultCount++;
+
+ // skip non-entity vertices
+ if (!AtlasGraphUtilsV2.isEntityVertex(vertex)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FullTextSearchProcessor.execute():
ignoring non-entity vertex (id={})", vertex.getId());
+ }
+
+ continue;
+ }
+
+ if (activeOnly && AtlasGraphUtilsV2.getState(vertex)
!= AtlasEntity.Status.ACTIVE) {
+ continue;
+ }
+
+ entityVertices.add(vertex);
+ }
+
+ isLastResultPage = resultCount < limit;
+
+ super.filter(entityVertices);
+
+ resultIdx = collectResultVertices(ret, startIdx, limit,
resultIdx, entityVertices);
+
+ if (isLastResultPage) {
+ break;
+ }
+ }
+ } catch (Throwable t) {
+ throw t;
+ }
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== FullTextSearchProcessor.execute({}):
ret.size()={}", context, ret.size());
+ }
+
+ return ret;
+ }
+}
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 393ce38..50edb9f 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -18,7 +18,9 @@
package org.apache.atlas.discovery;
+import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
@@ -39,6 +41,8 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashSet;
@@ -56,6 +60,7 @@ import static
org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIF
* possible chaining of processor(s)
*/
public class SearchContext {
+ private static final Logger LOG =
LoggerFactory.getLogger(SearchContext.class);
private final SearchParameters searchParameters;
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph;
@@ -65,11 +70,21 @@ public class SearchContext {
private final AtlasClassificationType classificationType;
private SearchProcessor searchProcessor;
private boolean terminateSearch = false;
+ private static boolean isIndexSolrBased = false;
public final static AtlasClassificationType
MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new
AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED
= new AtlasClassificationType(new
AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED
= new AtlasClassificationType(new
AtlasClassificationDef(NO_CLASSIFICATIONS));
+ static {
+ try {
+ isIndexSolrBased =
ApplicationProperties.INDEX_BACKEND_SOLR.equalsIgnoreCase(ApplicationProperties.get().getString(ApplicationProperties.INDEX_BACKEND_CONF));
+ } catch (AtlasException e) {
+ String msg = String.format("Error encountered in verifying the
backend index mode.");
+ LOG.error(msg, e);
+ throw new RuntimeException(msg, e);
+ };
+ }
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry
typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws
AtlasBaseException {
String classificationName = searchParameters.getClassification();
@@ -109,13 +124,20 @@ public class SearchContext {
}
if (needFullTextProcessor()) {
- addProcessor(new FullTextSearchProcessor(this));
+ if(!isFreeTextIndexEnabled()) {
+ LOG.info("Using Full Text index based search.");
+ addProcessor(new FullTextSearchProcessor(this));
+ }else {
+ LOG.info("Using Free Text index based search.");
+ addProcessor(new FreeTextSearchProcessor(this));
+ }
}
if (needClassificationProcessor()) {
addProcessor(new ClassificationSearchProcessor(this));
}
+
if (needEntityProcessor()) {
addProcessor(new EntitySearchProcessor(this));
}
@@ -262,4 +284,18 @@ public class SearchContext {
private AtlasEntityType getTermEntityType() {
return
typeRegistry.getEntityTypeByName(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ENTITY_TYPE);
}
+
+ public static boolean isFreeTextIndexEnabled() {
+ try {
+ return isIndexSolrBased() &&
ApplicationProperties.get().getBoolean(ApplicationProperties.FREE_TEXT_INDEX_ENABLED,
ApplicationProperties.DEFAULT_FREE_TEXT_INDEX_ENABLED);
+ } catch (AtlasException e) {
+ String msg = String.format("Error encountered in fetching the
configuration %s.", ApplicationProperties.FREE_TEXT_INDEX_ENABLED);
+ LOG.error(msg, e);
+ throw new RuntimeException(msg, e);
+ }
+ }
+
+ public static boolean isIndexSolrBased() {
+ return isIndexSolrBased;
+ }
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/converters/TypeConverterUtil.java
b/repository/src/main/java/org/apache/atlas/repository/converters/TypeConverterUtil.java
index caa63ee..a248de6 100644
---
a/repository/src/main/java/org/apache/atlas/repository/converters/TypeConverterUtil.java
+++
b/repository/src/main/java/org/apache/atlas/repository/converters/TypeConverterUtil.java
@@ -307,7 +307,7 @@ public final class TypeConverterUtil {
}
public static AtlasAttributeDef toAtlasAttributeDef(final
AttributeDefinition attrDefinition) {
- AtlasAttributeDef ret = new
AtlasAttributeDef(attrDefinition.getName(), attrDefinition.getDataTypeName());
+ AtlasAttributeDef ret = new
AtlasAttributeDef(attrDefinition.getName(), attrDefinition.getDataTypeName(),
attrDefinition.getSearchWeight());
ret.setIsIndexable(attrDefinition.getIsIndexable());
ret.setIsUnique(attrDefinition.getIsUnique());
@@ -362,7 +362,7 @@ public final class TypeConverterUtil {
AttributeDefinition oldAttrDef =
AtlasStructDefStoreV2.toAttributeDefinition(attribute);
- ret.add(new AttributeDefinition(oldAttrDef.getName(),
oldAttrDef.getDataTypeName(), new Multiplicity(oldAttrDef.getMultiplicity()),
oldAttrDef.getIsComposite(), oldAttrDef.getIsUnique(),
oldAttrDef.getIsIndexable(), oldAttrDef.getReverseAttributeName(),
oldAttrDef.getOptions()));
+ ret.add(new AttributeDefinition(oldAttrDef.getName(),
oldAttrDef.getDataTypeName(), new Multiplicity(oldAttrDef.getMultiplicity()),
oldAttrDef.getIsComposite(), oldAttrDef.getIsUnique(),
oldAttrDef.getIsIndexable(), oldAttrDef.getReverseAttributeName(),
oldAttrDef.getOptions(), oldAttrDef.getSearchWeight()));
}
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
index 77cecff..7de7b29 100644
---
a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
@@ -106,7 +106,7 @@ public class FullTextMapperV2 {
sb.append(classification.getTypeName()).append(FULL_TEXT_DELIMITER);
- mapAttributes(classificationType,
classification.getAttributes(), entityWithExtInfo, sb, new HashSet<String>(),
excludeAttributes);
+ mapAttributes(classificationType,
classification.getAttributes(), entityWithExtInfo, sb, new HashSet<String>(),
excludeAttributes, false); //false because of full text context.
}
}
@@ -138,7 +138,7 @@ public class FullTextMapperV2 {
if (entity != null) {
StringBuilder sb = new StringBuilder();
- map(entity, entityExtInfo, sb, new HashSet<String>());
+ map(entity, entityExtInfo, sb, new HashSet<String>(), false);
ret = sb.toString();
}
@@ -150,7 +150,22 @@ public class FullTextMapperV2 {
return ret;
}
- private void map(AtlasEntity entity, AtlasEntityExtInfo entityExtInfo,
StringBuilder sb, Set<String> processedGuids) throws AtlasBaseException {
+ public String getClassificationTextForEntity(AtlasEntity entity) throws
AtlasBaseException {
+ String ret = null;
+
+ if (entity != null) {
+ StringBuilder sb = new StringBuilder();
+ map(entity, null, sb, new HashSet<String>(), true);
+ ret = sb.toString();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.info("FullTextMapperV2.getClassificationTextForEntity({}):
{}", entity.getGuid(), ret);
+ }
+ return ret;
+ }
+
+ private void map(AtlasEntity entity, AtlasEntityExtInfo entityExtInfo,
StringBuilder sb, Set<String> processedGuids, boolean isClassificationOnly)
throws AtlasBaseException {
if (entity == null || processedGuids.contains(entity.getGuid())) {
return;
}
@@ -159,10 +174,11 @@ public class FullTextMapperV2 {
final Set<String> excludeAttributes =
getExcludeAttributesForIndexText(entity.getTypeName());
processedGuids.add(entity.getGuid());
+ if(!isClassificationOnly) {
+ sb.append(entity.getTypeName()).append(FULL_TEXT_DELIMITER);
- sb.append(entity.getTypeName()).append(FULL_TEXT_DELIMITER);
-
- mapAttributes(entityType, entity.getAttributes(), entityExtInfo, sb,
processedGuids, excludeAttributes);
+ mapAttributes(entityType, entity.getAttributes(), entityExtInfo,
sb, processedGuids, excludeAttributes, isClassificationOnly);
+ }
final List<AtlasClassification> classifications =
entity.getClassifications();
if (CollectionUtils.isNotEmpty(classifications)) {
@@ -173,13 +189,13 @@ public class FullTextMapperV2 {
sb.append(classification.getTypeName()).append(FULL_TEXT_DELIMITER);
- mapAttributes(classificationType,
classification.getAttributes(), entityExtInfo, sb, processedGuids,
excludeClassificationAttributes);
+ mapAttributes(classificationType,
classification.getAttributes(), entityExtInfo, sb, processedGuids,
excludeClassificationAttributes, isClassificationOnly);
}
}
}
private void mapAttributes(AtlasStructType structType, Map<String, Object>
attributes, AtlasEntityExtInfo entityExtInfo, StringBuilder sb,
- Set<String> processedGuids, Set<String>
excludeAttributes) throws AtlasBaseException {
+ Set<String> processedGuids, Set<String>
excludeAttributes, boolean isClassificationOnly) throws AtlasBaseException {
if (MapUtils.isEmpty(attributes)) {
return;
}
@@ -212,32 +228,32 @@ public class FullTextMapperV2 {
sb.append(attribKey).append(FULL_TEXT_DELIMITER);
- mapAttribute(attrValue, entityExtInfo, sb, processedGuids);
+ mapAttribute(attrValue, entityExtInfo, sb, processedGuids,
isClassificationOnly);
}
}
- private void mapAttribute(Object value, AtlasEntityExtInfo entityExtInfo,
StringBuilder sb, Set<String> processedGuids) throws AtlasBaseException {
+ private void mapAttribute(Object value, AtlasEntityExtInfo entityExtInfo,
StringBuilder sb, Set<String> processedGuids, boolean isClassificationOnly)
throws AtlasBaseException {
if (value instanceof AtlasObjectId) {
if (followReferences && entityExtInfo != null) {
AtlasObjectId objectId = (AtlasObjectId) value;
AtlasEntity entity =
entityExtInfo.getEntity(objectId.getGuid());
if (entity != null) {
- map(entity, entityExtInfo, sb, processedGuids);
+ map(entity, entityExtInfo, sb, processedGuids,
isClassificationOnly);
}
}
} else if (value instanceof List) {
List valueList = (List) value;
for (Object listElement : valueList) {
- mapAttribute(listElement, entityExtInfo, sb, processedGuids);
+ mapAttribute(listElement, entityExtInfo, sb, processedGuids,
isClassificationOnly);
}
} else if (value instanceof Map) {
Map valueMap = (Map) value;
for (Object key : valueMap.keySet()) {
- mapAttribute(key, entityExtInfo, sb, processedGuids);
- mapAttribute(valueMap.get(key), entityExtInfo, sb,
processedGuids);
+ mapAttribute(key, entityExtInfo, sb, processedGuids,
isClassificationOnly);
+ mapAttribute(valueMap.get(key), entityExtInfo, sb,
processedGuids, isClassificationOnly);
}
} else if (value instanceof Enum) {
Enum enumValue = (Enum) value;
@@ -248,7 +264,7 @@ public class FullTextMapperV2 {
for (Map.Entry<String, Object> entry :
atlasStruct.getAttributes().entrySet()) {
sb.append(entry.getKey()).append(FULL_TEXT_DELIMITER);
- mapAttribute(entry.getValue(), entityExtInfo, sb,
processedGuids);
+ mapAttribute(entry.getValue(), entityExtInfo, sb,
processedGuids, isClassificationOnly);
}
} else {
sb.append(String.valueOf(value)).append(FULL_TEXT_DELIMITER);
diff --git
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index eee0037..8f3d382 100755
---
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
+import org.apache.atlas.discovery.SearchContext;
import org.apache.atlas.discovery.SearchIndexer;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration;
@@ -36,23 +37,9 @@ import
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.RepositoryException;
-import org.apache.atlas.repository.graphdb.AtlasCardinality;
-import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
-import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
-import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
-import org.apache.atlas.type.AtlasArrayType;
-import org.apache.atlas.type.AtlasClassificationType;
-import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasEnumType;
-import org.apache.atlas.type.AtlasMapType;
-import org.apache.atlas.type.AtlasRelationshipType;
-import org.apache.atlas.type.AtlasStructType;
-import org.apache.atlas.type.AtlasType;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.atlas.type.*;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
@@ -72,6 +59,8 @@ import java.util.List;
import java.util.Set;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
+import static
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
+import static
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET;
@@ -104,12 +93,14 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
// Added for type lookup when indexing the new typedefs
private final AtlasTypeRegistry typeRegistry;
+ private final AtlasGraphIndexClient atlasGraphIndexClient;
+
//allows injection of a dummy graph for testing
private IAtlasGraphProvider provider;
private boolean recomputeIndexedKeys = true;
private Set<String> vertexIndexKeys = new HashSet<>();
-
+ private final String[] mappedCopyFieldNames;
public enum UniqueKind { NONE, GLOBAL_UNIQUE, PER_TYPE_UNIQUE }
@Inject
@@ -122,6 +113,18 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
throws IndexException, RepositoryException {
this.provider = provider;
this.typeRegistry = typeRegistry;
+
+ mappedCopyFieldNames = new String[11];
+ for(int i=0; i < mappedCopyFieldNames.length; i++) {
+ mappedCopyFieldNames[i] = String.format("allt%ds", i);
+ }
+
+ try {
+ atlasGraphIndexClient = provider.get().getGraphIndexClient();
+ } catch (Exception e) {
+ LOG.error("Error encountered in creating solr client.", e);
+ throw new RepositoryException("Error encountered in creating solr
client.", e);
+ }
if (!HAConfiguration.isHAEnabled(configuration)) {
initialize(provider.get());
}
@@ -280,6 +283,7 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
createVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY,
UniqueKind.NONE, Long.class, SINGLE, false, false);
createVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE,
String.class, SINGLE, false, false);
createVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE,
String.class, SINGLE, false, false);
+ createVertexIndex(management, CLASSIFICATION_TEXT_KEY,
UniqueKind.NONE, String.class, SINGLE, false, false, 10);
createVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE,
String.class, SINGLE, false, false);
createVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY,
UniqueKind.NONE, String.class, SET, true, true);
createVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY,
UniqueKind.NONE, String.class, LIST, true, true);
@@ -331,6 +335,10 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
}
}
+ private static boolean isStringAttribute(AtlasStructDef.AtlasAttributeDef
attributeDef) {
+ return
AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeDef.getTypeName());
+ }
+
private void createIndexForAttribute(AtlasGraphManagement management,
String typeName, AtlasAttributeDef attributeDef) {
final String propertyName =
AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + attributeDef.getName());
AtlasCardinality cardinality =
toAtlasCardinality(attributeDef.getCardinality());
@@ -341,7 +349,22 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
boolean isArrayType = isArrayType(attribTypeName);
boolean isMapType = isMapType(attribTypeName);
final String uniqPropName = isUnique ?
AtlasGraphUtilsV2.encodePropertyKey(typeName + "." +
UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + attributeDef.getName()) : null;
+ int searchWeight = attributeDef.getSearchWeight();
+
+ if(attributeDef.getSearchWeight() == DEFAULT_SEARCHWEIGHT) {
+ if (isStringAttribute(attributeDef)) {
+ //We will use default search weight of 3 for string attributes.
+ //this will make the string data searchable like in
FullTextIndex Searcher using Free Text searcher.
+ LOG.info("Applying default search weight of {} for
attribute{}.", DEFAULT_SEARCHWEIGHT_FOR_STRINGS, propertyName);
+ searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
+ }
+ }
+ if(!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) {
+ String msg = String.format("Invalid search weight '%d' was
provided for property %s.", searchWeight, propertyName);
+ LOG.error(msg);
+ throw new RuntimeException(msg);
+ }
try {
AtlasType atlasType = typeRegistry.getType(typeName);
@@ -380,7 +403,7 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
if (isRelationshipType(atlasType)) {
createEdgeIndex(management, propertyName,
getPrimitiveClass(attribTypeName), cardinality, false);
} else {
- createVertexIndex(management, propertyName,
UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable,
false);
+ createVertexIndex(management, propertyName,
UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable,
false, searchWeight);
if (uniqPropName != null) {
createVertexIndex(management, uniqPropName,
UniqueKind.PER_TYPE_UNIQUE, getPrimitiveClass(attribTypeName), cardinality,
isIndexable, true);
@@ -506,6 +529,10 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
public void createVertexIndex(AtlasGraphManagement management, String
propertyName, UniqueKind uniqueKind, Class propertyClass,
AtlasCardinality cardinality, boolean
createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes) {
+ createVertexIndex(management, propertyName, uniqueKind, propertyClass,
cardinality, createCompositeIndex, createCompositeIndexWithTypeAndSuperTypes,
-1);
+ }
+ private void createVertexIndex(AtlasGraphManagement management, String
propertyName, UniqueKind uniqueKind, Class propertyClass,
+ AtlasCardinality cardinality, boolean
createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes, int
searchWeight) {
if (propertyName != null) {
AtlasPropertyKey propertyKey =
management.getPropertyKey(propertyName);
@@ -518,11 +545,21 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
}
management.addMixedIndex(VERTEX_INDEX, propertyKey);
-
LOG.info("Created backing index for vertex property {} of
type {} ", propertyName, propertyClass.getName());
+
}
}
+ //for now, we will support free text based enhancements for SOLR
backed index systems only.
+ //WARNING--it is suggested that we don't change the search weight
once is assigned with a number in the range 1 to 10.
+ //otherwise, we might end up mapping the index to multiple copy
fields...unwanted to space usage.
+ if( searchWeight != -1 && SearchContext.isIndexSolrBased()) {
+ String encodedPropertyName =
management.getIndexFieldName(VERTEX_INDEX, propertyKey);
+ String mappedCopyFieldName =
mappedCopyFieldNames[searchWeight];
+ atlasGraphIndexClient.createCopyField(VERTEX_INDEX,
encodedPropertyName, mappedCopyFieldName);
+ LOG.info("Created copy field from {} to {} for collection {}
for property {}.", encodedPropertyName, mappedCopyFieldName, VERTEX_INDEX,
propertyName);
+ }
+
if (propertyKey != null) {
if (createCompositeIndex || uniqueKind ==
UniqueKind.GLOBAL_UNIQUE || uniqueKind == UniqueKind.PER_TYPE_UNIQUE) {
createVertexCompositeIndex(management, propertyClass,
propertyKey, uniqueKind == UniqueKind.GLOBAL_UNIQUE);
@@ -806,4 +843,13 @@ public class GraphBackedSearchIndexer implements
SearchIndexer, ActiveStateChang
addIndexForType(management, typeDef);
LOG.info("Index creation for type {} complete", typeDef.getName());
}
+
+ public static boolean isValidSearchWeight(int searchWeight) {
+ if(searchWeight != -1 ) {
+ if(searchWeight < 1 || searchWeight > 10) {
+ return false;
+ }
+ }
+ return true;
+ }
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
index ad5c60d..adf1972 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -40,6 +40,8 @@ import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.patches.AtlasPatchRegistry;
import org.apache.atlas.store.AtlasTypeDefStore;
@@ -66,11 +68,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
@@ -425,7 +423,8 @@ public class AtlasTypeDefStoreInitializer implements
ActiveStateChangeHandler {
new UpdateAttributePatchHandler(typeDefStore,
typeRegistry),
new RemoveLegacyRefAttributesPatchHandler(typeDefStore,
typeRegistry),
new UpdateTypeDefOptionsPatchHandler(typeDefStore,
typeRegistry),
- new SetServiceTypePatchHandler(typeDefStore, typeRegistry)
+ new SetServiceTypePatchHandler(typeDefStore, typeRegistry),
+ new UpdateAttributeMetadataHandler(typeDefStore,
typeRegistry)
};
Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
@@ -506,6 +505,7 @@ public class AtlasTypeDefStoreInitializer implements
ActiveStateChangeHandler {
private List<AtlasAttributeDef> attributeDefs;
private Map<String, String> typeDefOptions;
private String serviceType;
+ private String attributeName;
public String getId() {
return id;
@@ -586,6 +586,10 @@ public class AtlasTypeDefStoreInitializer implements
ActiveStateChangeHandler {
public void setServiceType(String serviceType) {
this.serviceType = serviceType;
}
+
+ public String getAttributeName() { return attributeName; }
+
+ public void setAttributeName(String attributeName) {
this.attributeName = attributeName; }
}
/**
@@ -962,4 +966,92 @@ public class AtlasTypeDefStoreInitializer implements
ActiveStateChangeHandler {
return ret;
}
}
+
+ class UpdateAttributeMetadataHandler extends PatchHandler {
+ public UpdateAttributeMetadataHandler(AtlasTypeDefStore typeDefStore,
AtlasTypeRegistry typeRegistry) {
+ super(typeDefStore, typeRegistry, new String[] {
"UPDATE_ATTRIBUTE_METADATA" });
+ }
+
+ @Override
+ public PatchStatus applyPatch(TypeDefPatch patch) throws
AtlasBaseException {
+ String typeName = patch.getTypeName();
+ AtlasBaseTypeDef typeDef =
typeRegistry.getTypeDefByName(typeName);
+ PatchStatus ret;
+
+ if (typeDef == null) {
+ throw new
AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(),
typeName);
+ }
+
+ if (isPatchApplicable(patch, typeDef)) {
+ String attributeNameFromPatch = patch.getAttributeName();
+ if (typeDef.getClass().equals(AtlasEntityDef.class)) {
+ AtlasEntityDef entityDef = new
AtlasEntityDef((AtlasEntityDef)typeDef);
+
+ updateAttributeMetadata(patch,
entityDef.getAttributeDefs());
+
+ entityDef.setTypeVersion(patch.getUpdateToVersion());
+
+ typeDefStore.updateEntityDefByName(typeName, entityDef);
+
+ ret = APPLIED;
+ } else if (typeDef.getClass().equals(AtlasStructDef.class)) {
+ AtlasStructDef updatedDef = new
AtlasStructDef((AtlasStructDef)typeDef);
+
+ updateAttributeMetadata(patch,
updatedDef.getAttributeDefs());
+
+ updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+ typeDefStore.updateStructDefByName(typeName, updatedDef);
+
+ ret = APPLIED;
+ } else {
+ throw new
AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE,
patch.getAction(), typeDef.getClass().getSimpleName());
+ }
+ } else {
+ LOG.info("patch skipped: typeName={}; applyToVersion={};
updateToVersion={}",
+ patch.getTypeName(), patch.getApplyToVersion(),
patch.getUpdateToVersion());
+ ret = SKIPPED;
+ }
+
+ return ret;
+ }
+
+ private void updateAttributeMetadata(TypeDefPatch patch,
List<AtlasAttributeDef> attributeDefsFromEntity) {
+ for(AtlasAttributeDef attributeDef: attributeDefsFromEntity) {
+
if(attributeDef.getName().equalsIgnoreCase(patch.getAttributeName())) {
+ updateAttribute(attributeDef, patch.getParams());
+ }
+ }
+ }
+
+ private void updateAttribute(AtlasAttributeDef atlasAttributeDef,
Map<String, Object> params) {
+ if(!params.isEmpty()) {
+ for(Map.Entry<String, Object> entry: params.entrySet()) {
+ try {
+ if
(AtlasAttributeDef.SEARCH_WEIGHT_ATTR_NAME.equalsIgnoreCase(entry.getKey())) {
+ Number number = (Number) entry.getValue();
+ int searchWeight = number.intValue();
+
if(!GraphBackedSearchIndexer.isValidSearchWeight(number.intValue())) {
+ String msg = String.format("Invalid search
weight '%d' was provided for property %s.", searchWeight,
atlasAttributeDef.getName());
+ LOG.error(msg);
+ throw new RuntimeException(msg);
+ }
+ atlasAttributeDef.setSearchWeight(searchWeight);
+ LOG.info("Updating Model attribute {}'s property{}
to {}.", atlasAttributeDef.getName(), entry.getKey(), entry.getValue());
+ } else {
+ //sanity exception
+ //more attributes can be added as needed.
+ String msg = String.format("Received unknown
property{} for attribute {}'s ", entry.getKey(), atlasAttributeDef.getName());
+ LOG.error(msg);
+ throw new RuntimeException(msg);
+ }
+ } catch (Exception e) {
+ String msg = String.format("Error encountered in
updating Model attribute %s's property '%s' to %s.",
atlasAttributeDef.getName(), entry.getKey(), entry.getValue().toString());
+ LOG.info(msg, e);
+ throw new RuntimeException(msg, e);
+ }
+ }
+ }
+ }
+ }
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
index 9ecc6e3..b93faf0 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
@@ -539,6 +539,7 @@ public class AtlasStructDefStoreV2 extends
AtlasAbstractDefStoreV2<AtlasStructDe
attribInfo.put("reverseAttributeName",
attribute.getInverseRefAttributeName());
attribInfo.put("defaultValue", attributeDef.getDefaultValue());
attribInfo.put("description", attributeDef.getDescription());
+ attribInfo.put("searchWeight", attributeDef.getSearchWeight());
if(attributeDef.getOptions() != null) {
attribInfo.put("options",
AtlasType.toJson(attributeDef.getOptions()));
@@ -627,6 +628,12 @@ public class AtlasStructDefStoreV2 extends
AtlasAbstractDefStoreV2<AtlasStructDe
ret.setValuesMaxCount(maxCount.intValue());
}
+ Number searchWeight = (Number) attribInfo.get("searchWeight");
+ if( searchWeight != null ) {
+ ret.setSearchWeight(searchWeight.intValue());
+ } else {
+ ret.setSearchWeight(-1);
+ }
return ret;
}
@@ -644,7 +651,9 @@ public class AtlasStructDefStoreV2 extends
AtlasAbstractDefStoreV2<AtlasStructDe
ret.setReverseAttributeName(attribute.getInverseRefAttributeName());
ret.setDescription(attrDef.getDescription());
ret.setDefaultValue(attrDef.getDefaultValue());
+ ret.setSearchWeight(attrDef.getSearchWeight());
return ret;
}
}
+
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 4f93c8f..66eeded 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -38,6 +38,7 @@ import
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.converters.AtlasInstanceConverter;
+import org.apache.atlas.repository.graph.FullTextMapperV2;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
@@ -79,28 +80,7 @@ import static
org.apache.atlas.model.instance.EntityMutations.EntityOperation.DE
import static
org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE;
import static
org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
-import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
-import static
org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
-import static
org.apache.atlas.repository.Constants.CLASSIFICATION_VALIDITY_PERIODS_KEY;
-import static
org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
-import static
org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_REMOVE_PROPAGATIONS_KEY;
-import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
-import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.GUID_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.HOME_ID_KEY;
-import static org.apache.atlas.repository.Constants.IS_PROXY_KEY;
-import static
org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
-import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY;
-import static
org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.SUPER_TYPES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
-import static
org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.*;
import static
org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
@@ -136,11 +116,12 @@ public class EntityGraphMapper {
private final AtlasEntityChangeNotifier entityChangeNotifier;
private final AtlasInstanceConverter instanceConverter;
private final EntityGraphRetriever entityRetriever;
+ private final FullTextMapperV2 fullTextMapperV2;
@Inject
public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate,
AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph,
AtlasRelationshipStore relationshipStore,
AtlasEntityChangeNotifier entityChangeNotifier,
- AtlasInstanceConverter instanceConverter) {
+ AtlasInstanceConverter instanceConverter,
FullTextMapperV2 fullTextMapperV2) {
this.deleteDelegate = deleteDelegate;
this.typeRegistry = typeRegistry;
this.graph = atlasGraph;
@@ -148,6 +129,7 @@ public class EntityGraphMapper {
this.entityChangeNotifier = entityChangeNotifier;
this.instanceConverter = instanceConverter;
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
+ this.fullTextMapperV2 = fullTextMapperV2;
}
public AtlasVertex createVertex(AtlasEntity entity) {
@@ -1588,6 +1570,8 @@ public class EntityGraphMapper {
AtlasEntity entity =
instanceConverter.getAndCacheEntity(entityGuid);
List<AtlasClassification> addedClassifications =
StringUtils.equals(entityGuid, guid) ? addClassifications :
propagations.get(vertex);
+
+ vertex.setProperty(CLASSIFICATION_TEXT_KEY,
fullTextMapperV2.getClassificationTextForEntity(entity));
if (CollectionUtils.isNotEmpty(addedClassifications)) {
entityChangeNotifier.onClassificationAddedToEntity(entity,
addedClassifications);
}
@@ -1691,10 +1675,12 @@ public class EntityGraphMapper {
updateModificationMetadata(entityVertex);
for (Map.Entry<AtlasVertex, List<AtlasClassification>> entry :
removedClassifications.entrySet()) {
- String guid =
GraphHelper.getGuid(entry.getKey());
+ AtlasVertex vertex =
entry.getKey();
+ String guid =
GraphHelper.getGuid(vertex);
List<AtlasClassification> deletedClassificationNames =
entry.getValue();
AtlasEntity entity =
instanceConverter.getAndCacheEntity(guid);
+ vertex.setProperty(CLASSIFICATION_TEXT_KEY,
fullTextMapperV2.getClassificationTextForEntity(entity));
entityChangeNotifier.onClassificationDeletedFromEntity(entity,
deletedClassificationNames);
}
}
@@ -1854,6 +1840,7 @@ public class EntityGraphMapper {
AtlasEntity entity =
instanceConverter.getAndCacheEntity(entityGuid);
if (isActive(entity)) {
+ vertex.setProperty(CLASSIFICATION_TEXT_KEY,
fullTextMapperV2.getClassificationTextForEntity(entity));
entityChangeNotifier.onClassificationUpdatedToEntity(entity,
updatedClassifications);
}
}
@@ -1866,6 +1853,7 @@ public class EntityGraphMapper {
AtlasEntity entity =
instanceConverter.getAndCacheEntity(entityGuid);
if (isActive(entity)) {
+ vertex.setProperty(CLASSIFICATION_TEXT_KEY,
fullTextMapperV2.getClassificationTextForEntity(entity));
entityChangeNotifier.onClassificationDeletedFromEntity(entity,
removedClassifications);
}
}
diff --git a/webapp/pom.xml b/webapp/pom.xml
index e62b2fe..f877714 100755
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -137,6 +137,12 @@
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-janusgraph-hbase2</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.noggit</groupId>
+ <artifactId>noggit</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>