atlas git commit: ATLAS-2827: fix to handle failure in saving indexable string property of large size - #2
Repository: atlas
Updated Branches:
refs/heads/branch-0.8 fe4486b41 -> 647d2e27f
ATLAS-2827: fix to handle failure in saving indexable string property of large
size - #2
(cherry picked from commit 4c6f1d167e4b41dab62f2f0fa9e153e6ceac8621)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/647d2e27
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/647d2e27
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/647d2e27
Branch: refs/heads/branch-0.8
Commit: 647d2e27f2552352e729bbf3d04343a5e2810aa6
Parents: fe4486b
Author: Madhan Neethiraj
Authored: Thu Sep 6 13:15:51 2018 -0700
Committer: Madhan Neethiraj
Committed: Thu Sep 6 13:16:13 2018 -0700
--
.../org/apache/atlas/AtlasConfiguration.java| 2 ++
.../store/graph/v1/EntityGraphMapper.java | 37 +++-
.../java/org/apache/atlas/RequestContext.java | 18 ++
.../notification/NotificationHookConsumer.java | 3 ++
4 files changed, 52 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/647d2e27/common/src/main/java/org/apache/atlas/AtlasConfiguration.java
--
diff --git a/common/src/main/java/org/apache/atlas/AtlasConfiguration.java
b/common/src/main/java/org/apache/atlas/AtlasConfiguration.java
index bd2bf7f..ace8a0f 100644
--- a/common/src/main/java/org/apache/atlas/AtlasConfiguration.java
+++ b/common/src/main/java/org/apache/atlas/AtlasConfiguration.java
@@ -38,6 +38,8 @@ public enum AtlasConfiguration {
NOTIFICATION_SPLIT_MESSAGE_SEGMENTS_WAIT_TIME_SECONDS("atlas.notification.split.message.segments.wait.time.seconds",
15 * 60),
NOTIFICATION_SPLIT_MESSAGE_BUFFER_PURGE_INTERVAL_SECONDS("atlas.notification.split.message.buffer.purge.interval.seconds",
5 * 60),
+
GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH("atlas.graphstore.indexed.string.safe.length",
Short.MAX_VALUE), // based on
org.apache.hadoop.hbase.client.Mutation.checkRow()
+
//search configuration
SEARCH_MAX_LIMIT("atlas.search.maxlimit", 1),
SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100);
http://git-wip-us.apache.org/repos/asf/atlas/blob/647d2e27/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
index a085ca5..d38fbb3 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
@@ -18,8 +18,10 @@
package org.apache.atlas.repository.store.graph.v1;
+import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
+import org.apache.atlas.RequestContext;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
@@ -70,8 +72,8 @@ import static
org.apache.atlas.repository.graph.GraphHelper.string;
public class EntityGraphMapper {
private static final Logger LOG =
LoggerFactory.getLogger(EntityGraphMapper.class);
-private static final String SOFT_REF_FORMAT = "%s:%s";
-private static final intINDEXED_STR_MAX_ALLOWED_LEN = 33482223;
+private static final String SOFT_REF_FORMAT = "%s:%s";
+private static final intINDEXED_STR_SAFE_LEN =
AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
private final GraphHelper graphHelper = GraphHelper.getInstance();
private final AtlasGraphgraph;
@@ -437,13 +439,32 @@ public class EntityGraphMapper {
// Titan bug, when an indexed string attribute has a value longer than
a certain length then the reverse indexed key generated by JanusGraph
// exceeds the HBase row length's hard limit (Short.MAX). This
trimming and hashing procedure is to circumvent that limitation
if (ret != null && isIndexableStrAttr) {
-String value = (String) ctx.getValue();
+String value = ret.toString();
+
+if (value.length() > INDEXED_STR_SAFE_LEN) {
+RequestContext requestContext = RequestContext.get();
+
+final int trimmedLength;
+
+if (requestContext.getAttemptCount() <= 1) { // if this is the
first attempt, try saving as it is; trim on retry
+trimmedLength = value.length();
+} else if (requestContext.getAttemptCount() >=
requestContext.getMaxAttempts()) { // if thi
atlas git commit: ATLAS-2827: fix to handle failure in saving indexable string property of large size - #2
Repository: atlas
Updated Branches:
refs/heads/branch-1.0 70088b3c2 -> a19b3906b
ATLAS-2827: fix to handle failure in saving indexable string property of large
size - #2
(cherry picked from commit 4c6f1d167e4b41dab62f2f0fa9e153e6ceac8621)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/a19b3906
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/a19b3906
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/a19b3906
Branch: refs/heads/branch-1.0
Commit: a19b3906b229f4adddf1c0c934d718ff319e3f4f
Parents: 70088b3
Author: Madhan Neethiraj
Authored: Thu Sep 6 02:40:32 2018 -0700
Committer: Madhan Neethiraj
Committed: Thu Sep 6 13:00:01 2018 -0700
--
.../org/apache/atlas/AtlasConfiguration.java| 2 ++
.../store/graph/v2/EntityGraphMapper.java | 35
.../java/org/apache/atlas/RequestContext.java | 19 +++
.../notification/NotificationHookConsumer.java | 3 ++
4 files changed, 52 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/a19b3906/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
--
diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
index bd2bf7f..ace8a0f 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
@@ -38,6 +38,8 @@ public enum AtlasConfiguration {
NOTIFICATION_SPLIT_MESSAGE_SEGMENTS_WAIT_TIME_SECONDS("atlas.notification.split.message.segments.wait.time.seconds",
15 * 60),
NOTIFICATION_SPLIT_MESSAGE_BUFFER_PURGE_INTERVAL_SECONDS("atlas.notification.split.message.buffer.purge.interval.seconds",
5 * 60),
+
GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH("atlas.graphstore.indexed.string.safe.length",
Short.MAX_VALUE), // based on
org.apache.hadoop.hbase.client.Mutation.checkRow()
+
//search configuration
SEARCH_MAX_LIMIT("atlas.search.maxlimit", 1),
SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100);
http://git-wip-us.apache.org/repos/asf/atlas/blob/a19b3906/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index c0a868c..e0bfdc4 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.store.graph.v2;
+import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
@@ -120,7 +121,8 @@ import static
org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelation
@Component
public class EntityGraphMapper {
private static final Logger LOG =
LoggerFactory.getLogger(EntityGraphMapper.class);
-private static final int INDEXED_STR_MAX_ALLOWED_LEN = 33482223;
+
+private static final int INDEXED_STR_SAFE_LEN =
AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
private final GraphHelper graphHelper =
GraphHelper.getInstance();
private final AtlasGraphgraph;
@@ -652,13 +654,32 @@ public class EntityGraphMapper {
// Janus bug, when an indexed string attribute has a value longer than
a certain length then the reverse indexed key generated by JanusGraph
// exceeds the HBase row length's hard limit (Short.MAX). This
trimming and hashing procedure is to circumvent that limitation
if (ret != null && isIndexableStrAttr) {
-String value = (String) ctx.getValue();
+String value = ret.toString();
+
+if (value.length() > INDEXED_STR_SAFE_LEN) {
+RequestContext requestContext = RequestContext.get();
+
+final int trimmedLength;
+
+if (requestContext.getAttemptCount() <= 1) { // if this is the
first attempt, try saving as it is; trim on retry
+trimmedLength = value.length();
+} else if (requestContext.getAttemptCount() >=
requestContext.getMaxAttempts()) { // if this is the last attempt, set to
'safe_len'
+trimmedLength = INDEXED_STR_SAFE_LEN;
+} else if (requestContext.getAttemptCount() == 2) { // based
on experimentation, string length of 4 times 'safe_len' su
atlas git commit: ATLAS-2827: fix to handle failure in saving indexable string property of large size - #2
Repository: atlas
Updated Branches:
refs/heads/master f31a7c631 -> 4c6f1d167
ATLAS-2827: fix to handle failure in saving indexable string property of large
size - #2
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/4c6f1d16
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/4c6f1d16
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/4c6f1d16
Branch: refs/heads/master
Commit: 4c6f1d167e4b41dab62f2f0fa9e153e6ceac8621
Parents: f31a7c6
Author: Madhan Neethiraj
Authored: Thu Sep 6 02:40:32 2018 -0700
Committer: Madhan Neethiraj
Committed: Thu Sep 6 12:59:29 2018 -0700
--
.../org/apache/atlas/AtlasConfiguration.java| 2 ++
.../store/graph/v2/EntityGraphMapper.java | 36 +++-
.../java/org/apache/atlas/RequestContext.java | 19 +++
.../notification/NotificationHookConsumer.java | 3 ++
4 files changed, 52 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/4c6f1d16/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
--
diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
index bd2bf7f..ace8a0f 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
@@ -38,6 +38,8 @@ public enum AtlasConfiguration {
NOTIFICATION_SPLIT_MESSAGE_SEGMENTS_WAIT_TIME_SECONDS("atlas.notification.split.message.segments.wait.time.seconds",
15 * 60),
NOTIFICATION_SPLIT_MESSAGE_BUFFER_PURGE_INTERVAL_SECONDS("atlas.notification.split.message.buffer.purge.interval.seconds",
5 * 60),
+
GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH("atlas.graphstore.indexed.string.safe.length",
Short.MAX_VALUE), // based on
org.apache.hadoop.hbase.client.Mutation.checkRow()
+
//search configuration
SEARCH_MAX_LIMIT("atlas.search.maxlimit", 1),
SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100);
http://git-wip-us.apache.org/repos/asf/atlas/blob/4c6f1d16/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 0d9cb05..4cd9909 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.store.graph.v2;
+import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
@@ -35,7 +36,6 @@ import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
-import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.converters.AtlasInstanceConverter;
import org.apache.atlas.repository.graph.GraphHelper;
@@ -87,7 +87,8 @@ import static
org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelation
@Component
public class EntityGraphMapper {
private static final Logger LOG =
LoggerFactory.getLogger(EntityGraphMapper.class);
-private static final int INDEXED_STR_MAX_ALLOWED_LEN = 33482223;
+
+private static final int INDEXED_STR_SAFE_LEN =
AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
private final GraphHelper graphHelper =
GraphHelper.getInstance();
private final AtlasGraphgraph;
@@ -619,13 +620,32 @@ public class EntityGraphMapper {
// Janus bug, when an indexed string attribute has a value longer than
a certain length then the reverse indexed key generated by JanusGraph
// exceeds the HBase row length's hard limit (Short.MAX). This
trimming and hashing procedure is to circumvent that limitation
if (ret != null && isIndexableStrAttr) {
-String value = (String) ctx.getValue();
+String value = ret.toString();
+
+if (value.length() > INDEXED_STR_SAFE_LEN) {
+RequestContext requestContext = RequestContext.get();
+
+final int trimmedLength;
+
+if (requestContext.getAttemptCount() <= 1) { // if
