This is an automated email from the ASF dual-hosted git repository.

krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new d3bbb03d001 HIVE-28409: Column lineage when creating view is missing 
if atlas HiveHook is set (Krisztian Kasa, reviewed by Denys Kuzmenko, Zsolt 
Miskolczi)
d3bbb03d001 is described below

commit d3bbb03d0015898ea760d6edfd3a2bc36766fcb0
Author: Krisztian Kasa <[email protected]>
AuthorDate: Mon Aug 26 09:51:38 2024 +0200

    HIVE-28409: Column lineage when creating view is missing if atlas HiveHook 
is set (Krisztian Kasa, reviewed by Denys Kuzmenko, Zsolt Miskolczi)
---
 .../java/org/apache/hadoop/hive/conf/HiveConf.java |  10 +-
 .../org/apache/hadoop/hive/ql/QueryProperties.java |   9 ++
 .../apache/hadoop/hive/ql/optimizer/Optimizer.java |   2 +-
 .../hive/ql/optimizer/SimpleFetchOptimizer.java    |   3 +-
 .../hive/ql/optimizer/lineage/Generator.java       |  79 ++++++++++++---
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java     |   5 +-
 .../hive/ql/optimizer/lineage/TestGenerator.java   | 106 +++++++++++++++++++++
 ql/src/test/queries/clientpositive/lineage6.q      |  13 +++
 .../results/clientpositive/llap/lineage6.q.out     |  28 ++++++
 9 files changed, 235 insertions(+), 20 deletions(-)

diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index d33d60e7b0f..af3a7576429 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3863,8 +3863,16 @@ public class HiveConf extends Configuration {
         "partition columns or non-partition columns while displaying columns 
in describe\n" +
         "table. From 0.12 onwards, they are displayed separately. This flag 
will let you\n" +
         "get old behavior, if desired. See, test-case in patch for 
HIVE-6689."),
+    @Deprecated
     HIVE_LINEAGE_INFO("hive.lineage.hook.info.enabled", false,
-        "Whether Hive provides lineage information to hooks."),
+        "Whether Hive provides lineage information to hooks." +
+            "Deprecated: use hive.lineage.statement.filter instead."),
+    HIVE_LINEAGE_STATEMENT_FILTER("hive.lineage.statement.filter", "ALL",
+        "Whether Hive provides lineage information to hooks for the specified 
statements only, " +
+            "the value is a comma-separated list (ex.: 
CREATE_MATERIALIZED_VIEW," +
+            "CREATE_TABLE,CREATE_TABLE_AS_SELECT). Possible values are: 
CREATE_TABLE, CREATE_TABLE_AS_SELECT, " +
+            "CREATE_VIEW, CREATE_MATERIALIZED_VIEW, LOAD, QUERY, ALL, NONE." +
+            " ALL means lineage information is always provided, NONE and empty 
string means never."),
 
     HIVE_SSL_PROTOCOL_BLACKLIST("hive.ssl.protocol.blacklist", "SSLv2,SSLv3",
         "SSL Versions to disable for all Hive Servers"),
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java 
b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
index d9d338c824f..eec7f548b19 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
@@ -69,6 +69,7 @@ public class QueryProperties {
 
   // True if this statement creates or replaces a materialized view
   private boolean isMaterializedView;
+  private boolean isView;
 
   public boolean isQuery() {
     return query;
@@ -285,6 +286,14 @@ public class QueryProperties {
     this.isMaterializedView = isMaterializedView;
   }
 
+  public boolean isView() {
+    return isView;
+  }
+
+  public void setView(boolean view) {
+    isView = view;
+  }
+
   public void clear() {
     query = false;
     analyzeCommand = false;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index 3341be88fd1..c483168d62e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -79,7 +79,7 @@ public class Optimizer {
         || 
postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.PostExecutePrinter")
         || 
postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.LineageLogger")
         || postExecHooks.contains("org.apache.atlas.hive.hook.HiveHook")) {
-      transformations.add(new Generator(postExecHooks));
+      transformations.add(Generator.fromConf(hiveConf));
     }
 
     // Try to transform OR predicates in Filter into simpler IN clauses first
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
index a4e236b4c33..f86e027e59a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
@@ -105,7 +105,8 @@ public class SimpleFetchOptimizer extends Transform {
   @Override
   public ParseContext transform(ParseContext pctx) throws SemanticException {
     Map<String, TableScanOperator> topOps = pctx.getTopOps();
-    if (pctx.getQueryProperties().isQuery() && 
!pctx.getQueryProperties().isAnalyzeCommand()
+    if ((pctx.getQueryProperties().isQuery() || 
pctx.getQueryProperties().isView())
+        && !pctx.getQueryProperties().isAnalyzeCommand()
         && topOps.size() == 1) {
       // no join, no groupby, no distinct, no lateral view, no subq,
       // no CTAS or insert, not analyze command, and single sourced.
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
index 3a4522fffd9..20889eb849a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
@@ -19,10 +19,15 @@
 package org.apache.hadoop.hive.ql.optimizer.lineage;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Predicate;
 
+import org.apache.commons.lang3.EnumUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
 import org.apache.hadoop.hive.ql.exec.GroupByOperator;
@@ -59,11 +64,62 @@ public class Generator extends Transform {
 
   private static final Logger LOG = LoggerFactory.getLogger(Generator.class);
 
-  private final Set<String> hooks;
-  private static final String ATLAS_HOOK_CLASSNAME = 
"org.apache.atlas.hive.hook.HiveHook";
+  private final Predicate<ParseContext> statementFilter;
 
-  public Generator(Set<String> hooks) {
-    this.hooks = hooks;
+  enum LineageInfoFilter {
+    CREATE_TABLE(parseContext -> parseContext.getCreateTable() != null),
+    CREATE_TABLE_AS_SELECT(parseContext -> 
parseContext.getQueryProperties().isCTAS()),
+    CREATE_VIEW(parseContext -> parseContext.getQueryProperties().isView()),
+    CREATE_MATERIALIZED_VIEW(parseContext -> 
parseContext.getQueryProperties().isMaterializedView()),
+    LOAD(parseContext -> !(parseContext.getLoadTableWork() == null || 
parseContext.getLoadTableWork().isEmpty())),
+    QUERY(parseContext -> parseContext.getQueryProperties().isQuery()),
+    ALL(parseContext -> true),
+    NONE(parseContext -> false);
+
+    final Predicate<ParseContext> predicate;
+
+    LineageInfoFilter(Predicate<ParseContext> predicate) {
+      this.predicate = predicate;
+    }
+  }
+
+  public static Generator fromConf(HiveConf conf) {
+    return new Generator(createFilterPredicateFromConf(conf));
+  }
+
+  static Predicate<ParseContext> createFilterPredicateFromConf(Configuration 
conf) {
+    Set<LineageInfoFilter> operations = new HashSet<>();
+    boolean noneSpecified = false;
+    for (String valueText : 
conf.getStringCollection(HiveConf.ConfVars.HIVE_LINEAGE_STATEMENT_FILTER.varname))
 {
+      LineageInfoFilter enumValue = 
EnumUtils.getEnumIgnoreCase(LineageInfoFilter.class, valueText);
+      if (enumValue == null) {
+        throw new EnumConstantNotPresentException(LineageInfoFilter.class, 
valueText);
+      }
+
+      if (LineageInfoFilter.NONE == enumValue) {
+        noneSpecified = true;
+        continue;
+      }
+
+      operations.add(enumValue);
+    }
+
+    if (noneSpecified) {
+      if (!operations.isEmpty()) {
+        throw new IllegalArgumentException(
+            "No other value can be specified when " + 
LineageInfoFilter.NONE.name() + " is present!");
+      }
+      else {
+        return parseContext -> false;
+      }
+    }
+
+    return parseContext ->
+        operations.stream().anyMatch(lineageInfoFilter -> 
lineageInfoFilter.predicate.test(parseContext));
+  }
+
+  public Generator(Predicate<ParseContext> statementFilter) {
+    this.statementFilter = statementFilter;
   }
 
   /* (non-Javadoc)
@@ -72,18 +128,11 @@ public class Generator extends Transform {
   @Override
   public ParseContext transform(ParseContext pctx) throws SemanticException {
 
-    if (hooks != null && hooks.contains(ATLAS_HOOK_CLASSNAME)) {
-      // Atlas would be interested in lineage information for 
insert,load,create etc.
-      if (!pctx.getQueryProperties().isCTAS()
-          && !pctx.getQueryProperties().isMaterializedView()
-          && pctx.getQueryProperties().isQuery()
-          && pctx.getCreateTable() == null
-          && pctx.getCreateViewDesc() == null
-          && (pctx.getLoadTableWork() == null || 
pctx.getLoadTableWork().isEmpty())) {
-        LOG.debug("Not evaluating lineage");
-        return pctx;
-      }
+    if (!statementFilter.test(pctx)) {
+      LOG.debug("Not evaluating lineage");
+      return pctx;
     }
+
     Index index = pctx.getQueryState().getLineageState().getIndex();
     if (index == null) {
       index = new Index();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index a44def6927c..ca9d5996d34 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -13226,7 +13226,7 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
           || postExecHooks.contains("org.apache.atlas.hive.hook.HiveHook")) {
         List<Transform> transformations = new ArrayList<Transform>();
         transformations.add(new HiveOpConverterPostProc());
-        transformations.add(new Generator(postExecHooks));
+        transformations.add(Generator.fromConf(conf));
         for (Transform t : transformations) {
           pCtx = t.transform(pCtx);
         }
@@ -15665,7 +15665,7 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
 
   private void copyInfoToQueryProperties(QueryProperties queryProperties) {
     if (qb != null) {
-      queryProperties.setQuery(qb.getIsQuery());
+      queryProperties.setQuery(qb.getIsQuery() && !forViewCreation);
       queryProperties.setAnalyzeCommand(qb.getParseInfo().isAnalyzeCommand());
       
queryProperties.setNoScanAnalyzeCommand(qb.getParseInfo().isNoScanAnalyzeCommand());
       queryProperties.setAnalyzeRewrite(qb.isAnalyzeRewrite());
@@ -15673,6 +15673,7 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
       queryProperties.setHasOuterOrderBy(!qb.getParseInfo().getIsSubQ() &&
           !qb.getParseInfo().getDestToOrderBy().isEmpty());
       
queryProperties.setOuterQueryLimit(qb.getParseInfo().getOuterQueryLimit());
+      queryProperties.setView(forViewCreation);
       queryProperties.setMaterializedView(qb.isMaterializedView());
     }
   }
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/lineage/TestGenerator.java 
b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/lineage/TestGenerator.java
new file mode 100644
index 00000000000..38eff316d0d
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/lineage/TestGenerator.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.lineage;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryProperties;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.junit.Test;
+
+import java.util.function.Predicate;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.*;
+
+public class TestGenerator {
+  @Test
+  public void testCreateFilterPredicateFromConf() {
+    HiveConf conf = new HiveConf();
+    conf.set(HiveConf.ConfVars.HIVE_LINEAGE_STATEMENT_FILTER.varname,
+        "CREATE_tABLE_AS_sELECT," + HiveOperation.QUERY.name());
+
+    Predicate<ParseContext> predicate = 
Generator.createFilterPredicateFromConf(conf);
+
+    ParseContext parseContext = new ParseContext();
+    QueryProperties queryProperties = new QueryProperties();
+    queryProperties.setCTAS(true);
+    parseContext.setQueryProperties(queryProperties);
+    assertThat(predicate.test(parseContext), is(true));
+
+    parseContext = new ParseContext();
+    queryProperties = new QueryProperties();
+    queryProperties.setQuery(true);
+    parseContext.setQueryProperties(queryProperties);
+    assertThat(predicate.test(parseContext), is(true));
+
+    parseContext = new ParseContext();
+    queryProperties = new QueryProperties();
+    queryProperties.setView(true);
+    parseContext.setQueryProperties(queryProperties);
+    assertThat(predicate.test(parseContext), is(false));
+  }
+
+  @Test
+  public void 
testCreateFilterPredicateFromConfReturnsAlwaysTrueWhenSettingIsNotPresent() {
+    HiveConf conf = new HiveConf();
+
+    Predicate<ParseContext> predicate = 
Generator.createFilterPredicateFromConf(conf);
+
+    assertThat(predicate.test(new ParseContext()), is(true));
+  }
+
+  @Test
+  public void 
testCreateFilterPredicateFromConfReturnsAlwaysFalseWhenSettingValueIsEmptyString()
 {
+    HiveConf conf = new HiveConf();
+    conf.set(HiveConf.ConfVars.HIVE_LINEAGE_STATEMENT_FILTER.varname, "");
+
+    Predicate<ParseContext> predicate = 
Generator.createFilterPredicateFromConf(conf);
+
+    assertThat(predicate.test(new ParseContext()), is(false));
+  }
+
+  @Test
+  public void 
testCreateFilterPredicateFromConfThrowsExceptionWhenInputStringIsInvalid() {
+    HiveConf conf = new HiveConf();
+    conf.set(HiveConf.ConfVars.HIVE_LINEAGE_STATEMENT_FILTER.varname, 
"Invalid");
+
+    EnumConstantNotPresentException exception = assertThrows(
+        EnumConstantNotPresentException.class,
+        () -> Generator.createFilterPredicateFromConf(conf)
+    );
+
+    assertThat(exception.getMessage(), is(
+        
"org.apache.hadoop.hive.ql.optimizer.lineage.Generator$LineageInfoFilter.Invalid"));
+  }
+
+  @Test
+  public void 
testCreateFilterPredicateFromConfThrowsExceptionWhenNoneAndAnyOtherConstantPresent()
 {
+    HiveConf conf = new HiveConf();
+    conf.set(HiveConf.ConfVars.HIVE_LINEAGE_STATEMENT_FILTER.varname, "None," 
+ HiveOperation.QUERY.name());
+
+    IllegalArgumentException exception = assertThrows(
+        IllegalArgumentException.class,
+        () -> Generator.createFilterPredicateFromConf(conf)
+    );
+
+    assertThat(exception.getMessage(), is("No other value can be specified 
when NONE is present!"));
+  }
+}
\ No newline at end of file
diff --git a/ql/src/test/queries/clientpositive/lineage6.q 
b/ql/src/test/queries/clientpositive/lineage6.q
new file mode 100644
index 00000000000..1dc9613a023
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/lineage6.q
@@ -0,0 +1,13 @@
+set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.LineageLogger;
+set hive.lineage.statement.filter=Query;
+
+create table table_1_qegkz (id int, first_name string);
+create table table_2_gkvuw (id int, last_name string);
+
+select table_1_qegkz.id, concat_ws(' ' , table_1_qegkz.first_name, 
table_2_gkvuw.last_name) full_name from table_1_qegkz, table_2_gkvuw where 
table_1_qegkz.id = table_2_gkvuw.id;
+
+create view view_fcuyp as (select table_1_qegkz.id, concat_ws(' ' , 
table_1_qegkz.first_name, table_2_gkvuw.last_name) full_name from 
table_1_qegkz, table_2_gkvuw where table_1_qegkz.id = table_2_gkvuw.id);
+
+set hive.lineage.statement.filter=Query,CREATE_VIEW;
+
+create view view_fcuyp2 as (select table_1_qegkz.id, concat_ws(' ' , 
table_1_qegkz.first_name, table_2_gkvuw.last_name) full_name from 
table_1_qegkz, table_2_gkvuw where table_1_qegkz.id = table_2_gkvuw.id);
diff --git a/ql/src/test/results/clientpositive/llap/lineage6.q.out 
b/ql/src/test/results/clientpositive/llap/lineage6.q.out
new file mode 100644
index 00000000000..41daca1bcd6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/lineage6.q.out
@@ -0,0 +1,28 @@
+PREHOOK: query: create table table_1_qegkz (id int, first_name string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table_1_qegkz
+PREHOOK: query: create table table_2_gkvuw (id int, last_name string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table_2_gkvuw
+PREHOOK: query: select table_1_qegkz.id, concat_ws(' ' , 
table_1_qegkz.first_name, table_2_gkvuw.last_name) full_name from 
table_1_qegkz, table_2_gkvuw where table_1_qegkz.id = table_2_gkvuw.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table_1_qegkz
+PREHOOK: Input: default@table_2_gkvuw
+#### A masked pattern was here ####
+{"version":"1.0","engine":"tez","database":"default","hash":"35144a690834a6399b85caf27dbf8a3c","queryText":"select
 table_1_qegkz.id, concat_ws(' ' , table_1_qegkz.first_name, 
table_2_gkvuw.last_name) full_name from table_1_qegkz, table_2_gkvuw where 
table_1_qegkz.id = 
table_2_gkvuw.id","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3,4],"targets":[1],"expression":"concat_ws('
 ', table_1_qegkz.first_name, 
table_2_gkvuw.last_name)","edgeType":"PROJECTION"},{"sou [...]
+PREHOOK: query: create view view_fcuyp as (select table_1_qegkz.id, 
concat_ws(' ' , table_1_qegkz.first_name, table_2_gkvuw.last_name) full_name 
from table_1_qegkz, table_2_gkvuw where table_1_qegkz.id = table_2_gkvuw.id)
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@table_1_qegkz
+PREHOOK: Input: default@table_2_gkvuw
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view_fcuyp
+{"version":"1.0","engine":"tez","database":"default","hash":"26e597ea7b3ca43b008466502b235b52","queryText":"create
 view view_fcuyp as (select table_1_qegkz.id, concat_ws(' ' , 
table_1_qegkz.first_name, table_2_gkvuw.last_name) full_name from 
table_1_qegkz, table_2_gkvuw where table_1_qegkz.id = 
table_2_gkvuw.id)","edges":[],"vertices":[]}
+PREHOOK: query: create view view_fcuyp2 as (select table_1_qegkz.id, 
concat_ws(' ' , table_1_qegkz.first_name, table_2_gkvuw.last_name) full_name 
from table_1_qegkz, table_2_gkvuw where table_1_qegkz.id = table_2_gkvuw.id)
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@table_1_qegkz
+PREHOOK: Input: default@table_2_gkvuw
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view_fcuyp2
+{"version":"1.0","engine":"tez","database":"default","hash":"3c6019537df2f6001cce07de1aa0e352","queryText":"create
 view view_fcuyp2 as (select table_1_qegkz.id, concat_ws(' ' , 
table_1_qegkz.first_name, table_2_gkvuw.last_name) full_name from 
table_1_qegkz, table_2_gkvuw where table_1_qegkz.id = 
table_2_gkvuw.id)","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3,4],"targets":[1],"expression":"concat_ws('
 ', table_1_qegkz.first_name, table_2_gkvuw.last_name)"," [...]

Reply via email to