This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit db2a22c56705ade98f6b05d3851ca2b9db61ed72
Author: Wang gang <gwa...@ebay.com>
AuthorDate: Mon Jun 22 12:20:26 2020 +0800
KYLIN-4594 Send out notification email when triggering cube building job
and finding out table schema changes
---
.../org/apache/kylin/rest/service/JobService.java | 49 +++-
.../rest/service/TableSchemaUpdateChecker.java | 139 +++++++++--
.../mail_templates/HIVE_SCHEMA_CHANGED.ftl | 264 +++++++++++++++++++++
3 files changed, 417 insertions(+), 35 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index a214c12..f9880bd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -33,7 +33,6 @@ import java.util.TimeZone;
import javax.annotation.Nullable;
-import com.google.common.collect.Maps;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.lock.DistributedLock;
@@ -51,8 +50,8 @@ import
org.apache.kylin.engine.mr.BatchOptimizeJobCheckpointBuilder;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.LookupSnapshotBuildJob;
import org.apache.kylin.engine.mr.LookupSnapshotJobBuilder;
-import org.apache.kylin.engine.mr.common.CubeJobLockUtil;
import org.apache.kylin.engine.mr.StreamingCubingEngine;
+import org.apache.kylin.engine.mr.common.CubeJobLockUtil;
import org.apache.kylin.engine.mr.common.JobInfoConverter;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.JobInstance;
@@ -70,21 +69,31 @@ import org.apache.kylin.job.execution.CheckpointExecutable;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.Output;
+import org.apache.kylin.job.lock.zookeeper.ZookeeperJobLock;
import org.apache.kylin.job.util.MailNotificationUtil;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableExtDesc;
+import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
+import org.apache.kylin.rest.response.ResponseCode;
import org.apache.kylin.rest.util.AclEvaluate;
+import org.apache.kylin.shaded.com.google.common.base.Function;
+import org.apache.kylin.shaded.com.google.common.base.Predicate;
+import org.apache.kylin.shaded.com.google.common.base.Predicates;
+import org.apache.kylin.shaded.com.google.common.collect.FluentIterable;
+import org.apache.kylin.shaded.com.google.common.collect.Lists;
+import org.apache.kylin.shaded.com.google.common.collect.Maps;
+import org.apache.kylin.shaded.com.google.common.collect.Sets;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceManager;
import org.apache.kylin.source.SourcePartition;
-import org.apache.kylin.job.lock.zookeeper.ZookeeperJobLock;
import org.apache.kylin.source.hive.MRHiveDictUtil;
import org.apache.kylin.stream.coordinator.Coordinator;
import org.apache.kylin.stream.core.model.SegmentBuildState;
@@ -92,17 +101,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
-import org.apache.kylin.shaded.com.google.common.base.Function;
-import org.apache.kylin.shaded.com.google.common.base.Predicate;
-import org.apache.kylin.shaded.com.google.common.base.Predicates;
-import org.apache.kylin.shaded.com.google.common.collect.FluentIterable;
-import org.apache.kylin.shaded.com.google.common.collect.Lists;
-import org.apache.kylin.shaded.com.google.common.collect.Sets;
-
/**
* @author ysong1
*/
@@ -116,6 +119,10 @@ public class JobService extends BasicService implements
InitializingBean {
@Autowired
private AclEvaluate aclEvaluate;
+ @Autowired
+ @Qualifier("tableService")
+ private TableService tableService;
+
/*
* (non-Javadoc)
*
@@ -248,6 +255,7 @@ public class JobService extends BasicService implements
InitializingBean {
if (buildType == CubeBuildTypeEnum.BUILD || buildType ==
CubeBuildTypeEnum.REFRESH) {
checkAllowParallelBuilding(cube);
+ checkSourceSchemaUpdate(cube);
}
DefaultChainedExecutable job;
@@ -293,6 +301,27 @@ public class JobService extends BasicService implements
InitializingBean {
return jobInstance;
}
+ private void checkSourceSchemaUpdate(CubeInstance cube) {
+ String[] tables =
cube.getModel().getAllTables().stream().map(TableRef::getTableIdentity)
+ .toArray(String[]::new);
+ String project =
getProjectManager().getProjectOfModel(cube.getModel().getName()).getName();
+ try {
+ List<Pair<TableDesc, TableExtDesc>> allMeta =
tableService.extractHiveTableMeta(tables, project);
+ // do schema check
+ TableSchemaUpdateChecker checker = new
TableSchemaUpdateChecker(getTableManager(), getCubeManager(),
+ getDataModelManager());
+ for (Pair<TableDesc, TableExtDesc> pair : allMeta) {
+ TableDesc tableDesc = pair.getFirst();
+ TableSchemaUpdateChecker.CheckResult result =
checker.allowReload(tableDesc, project);
+ result.raiseExceptionAndNotifyWhenInvalid();
+ }
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException(e.getMessage(),
ResponseCode.CODE_UNDEFINED, e);
+ } catch (Exception e) {
+ logger.warn("Can not connect to hive, table schema check
skipped!");
+ }
+ }
+
public Pair<JobInstance, List<JobInstance>> submitOptimizeJob(CubeInstance
cube, Set<Long> cuboidsRecommend,
String submitter) throws IOException, JobException {
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
b/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
index 3a45f49..c2307ff 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
@@ -18,19 +18,26 @@
package org.apache.kylin.rest.service;
-import static
org.apache.kylin.shaded.com.google.common.base.Preconditions.checkNotNull;
import static java.lang.String.format;
+import static
org.apache.kylin.shaded.com.google.common.base.Preconditions.checkNotNull;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Matcher;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.MailService;
+import org.apache.kylin.common.util.MailTemplateProvider;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.job.util.MailNotificationUtil;
import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -38,62 +45,90 @@ import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.ModelDimensionDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-
+import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.shaded.com.google.common.base.Preconditions;
import org.apache.kylin.shaded.com.google.common.base.Predicate;
import org.apache.kylin.shaded.com.google.common.collect.ImmutableList;
import org.apache.kylin.shaded.com.google.common.collect.Iterables;
import org.apache.kylin.shaded.com.google.common.collect.Lists;
+import org.apache.kylin.shaded.com.google.common.collect.Maps;
import org.apache.kylin.shaded.com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TableSchemaUpdateChecker {
+ private static final Logger logger =
LoggerFactory.getLogger(TableSchemaUpdateChecker.class);
+
private final TableMetadataManager metadataManager;
private final CubeManager cubeManager;
private final DataModelManager dataModelManager;
public static class CheckResult {
+ private final String tableName;
private final boolean valid;
private final String reason;
+ private final Set<CubeInstance> impactedCubes;
- private CheckResult(boolean valid, String reason) {
+ private CheckResult(String tableName, boolean valid, String reason) {
+ this(tableName, valid, reason, Sets.<CubeInstance> newHashSet());
+ }
+
+ private CheckResult(String tableName, boolean valid, String reason,
Set<CubeInstance> impactedCubes) {
+ this.tableName = tableName;
this.valid = valid;
this.reason = reason;
+ this.impactedCubes = impactedCubes;
}
public void raiseExceptionWhenInvalid() {
+ raiseExceptionAndNotifyWhenInvalid(false);
+ }
+
+ public void raiseExceptionAndNotifyWhenInvalid() {
+ raiseExceptionAndNotifyWhenInvalid(true);
+ }
+
+ private void raiseExceptionAndNotifyWhenInvalid(boolean ifNotify) {
if (!valid) {
- throw new RuntimeException(reason);
+ if (ifNotify) {
+ notifyInvalid(tableName, reason, impactedCubes);
+ }
+ throw new IllegalArgumentException(reason);
}
}
static CheckResult validOnFirstLoad(String tableName) {
- return new CheckResult(true, format(Locale.ROOT, "Table '%s'
hasn't been loaded before", tableName));
+ return new CheckResult(tableName, true,
+ format(Locale.ROOT, "Table '%s' hasn't been loaded
before", tableName));
}
static CheckResult validOnCompatibleSchema(String tableName) {
- return new CheckResult(true,
+ return new CheckResult(tableName, true,
format(Locale.ROOT, "Table '%s' is compatible with all
existing cubes", tableName));
}
static CheckResult invalidOnFetchSchema(String tableName, Exception e)
{
- return new CheckResult(false,
+ return new CheckResult(tableName, false,
format(Locale.ROOT, "Failed to fetch metadata of '%s':
%s", tableName, e.getMessage()));
}
- static CheckResult invalidOnIncompatibleSchema(String tableName,
List<String> reasons) {
+ static CheckResult invalidOnIncompatibleSchema(String tableName,
List<String> reasons,
+ Set<CubeInstance> impactedCubes) {
StringBuilder buf = new StringBuilder();
for (String reason : reasons) {
buf.append("- ").append(reason).append("\n");
}
- return new CheckResult(false,
+ return new CheckResult(tableName, false,
format(Locale.ROOT,
"Found %d issue(s) with '%s':%n%s Please disable
and " + "purge related " + "cube(s) first",
- reasons.size(), tableName, buf.toString()));
+ reasons.size(), tableName, buf.toString()),
+ impactedCubes);
}
}
- TableSchemaUpdateChecker(TableMetadataManager metadataManager, CubeManager
cubeManager, DataModelManager dataModelManager) {
+ TableSchemaUpdateChecker(TableMetadataManager metadataManager, CubeManager
cubeManager,
+ DataModelManager dataModelManager) {
this.metadataManager = checkNotNull(metadataManager, "metadataManager
is null");
this.cubeManager = checkNotNull(cubeManager, "cubeManager is null");
this.dataModelManager = checkNotNull(dataModelManager,
"dataModelManager is null");
@@ -195,32 +230,36 @@ public class TableSchemaUpdateChecker {
}
List<String> issues = Lists.newArrayList();
- for (DataModelDesc usedModel : findModelByTable(newTableDesc, prj)){
+ for (DataModelDesc usedModel : findModelByTable(newTableDesc, prj)) {
checkValidationInModel(newTableDesc, issues, usedModel);
}
+ Set<CubeInstance> impactedCubes = Sets.newHashSet();
for (CubeInstance cube : findCubeByTable(newTableDesc)) {
- checkValidationInCube(newTableDesc, issues, cube);
+ boolean ifImpacted = checkValidationInCube(newTableDesc, issues,
cube);
+ if (ifImpacted) {
+ impactedCubes.add(cube);
+ }
}
if (issues.isEmpty()) {
return CheckResult.validOnCompatibleSchema(fullTableName);
}
- return CheckResult.invalidOnIncompatibleSchema(fullTableName, issues);
+ return CheckResult.invalidOnIncompatibleSchema(fullTableName, issues,
impactedCubes);
}
private Iterable<? extends DataModelDesc> findModelByTable(TableDesc
newTableDesc, String prj) {
List<DataModelDesc> usedModels = Lists.newArrayList();
List<String> modelNames =
dataModelManager.getModelsUsingTable(newTableDesc, prj);
- modelNames.stream()
- .map(mn -> dataModelManager.getDataModelDesc(mn))
- .filter(m -> null != m)
+ modelNames.stream().map(mn ->
dataModelManager.getDataModelDesc(mn)).filter(m -> null != m)
.forEach(m -> usedModels.add(m));
return usedModels;
}
- private void checkValidationInCube(TableDesc newTableDesc, List<String>
issues, CubeInstance cube) {
+ private boolean checkValidationInCube(TableDesc newTableDesc, List<String>
issues, CubeInstance cube) {
+ boolean ifImpacted = false;
+
final String fullTableName = newTableDesc.getIdentity();
String modelName = cube.getModel().getName();
// if user reloads a fact table used by cube, then all used columns
must match current schema
@@ -230,6 +269,7 @@ public class TableSchemaUpdateChecker {
if (!violateColumns.isEmpty()) {
issues.add(format(Locale.ROOT, "Column %s used in cube[%s] and
model[%s], but changed " + "in hive",
violateColumns, cube.getName(), modelName));
+ ifImpacted = true;
}
}
@@ -238,11 +278,15 @@ public class TableSchemaUpdateChecker {
if (cube.getModel().isLookupTable(fullTableName)) {
TableDesc lookupTable =
cube.getModel().findFirstTable(fullTableName).getTableDesc();
if (!checkAllColumnsInTableDesc(lookupTable, newTableDesc)) {
- issues.add(format(Locale.ROOT, "Table '%s' is used as Lookup
Table in cube[%s] and model[%s], but "
- + "changed in " + "hive, only append operation
are supported on hive table as lookup table",
+ issues.add(format(Locale.ROOT,
+ "Table '%s' is used as Lookup Table in cube[%s] and
model[%s], but " + "changed in "
+ + "hive, only append operation are supported
on hive table as lookup table",
lookupTable.getIdentity(), cube.getName(), modelName));
+ ifImpacted = true;
}
}
+
+ return ifImpacted;
}
private void checkValidationInModel(TableDesc newTableDesc, List<String>
issues, DataModelDesc usedModel) {
@@ -281,7 +325,7 @@ public class TableSchemaUpdateChecker {
return violateColumns;
}
-
+
private List<String> checkAllColumnsInFactTable(DataModelDesc usedModel,
TableDesc factTable,
TableDesc newTableDesc) {
List<String> violateColumns = Lists.newArrayList();
@@ -316,8 +360,8 @@ public class TableSchemaUpdateChecker {
private ColumnDesc mustGetColumnDesc(TableDesc factTable, String
columnName) {
ColumnDesc columnDesc = factTable.findColumnByName(columnName);
- Preconditions.checkNotNull(columnDesc,
- format(Locale.ROOT, "Can't find column %s in current fact
table %s.", columnName, factTable.getIdentity()));
+ Preconditions.checkNotNull(columnDesc, format(Locale.ROOT, "Can't find
column %s in current fact table %s.",
+ columnName, factTable.getIdentity()));
return columnDesc;
}
@@ -348,10 +392,10 @@ public class TableSchemaUpdateChecker {
List<String> issues = Lists.newArrayList();
checkAllColumnsInHiveTableDesc(hiveTableDesc, newTableDesc, issues);
if (issues.isEmpty()) {
- return new CheckResult(true,
+ return new CheckResult(fullTableName, true,
format(Locale.ROOT, "Table '%s' is compatible with
existing hive table", fullTableName));
} else {
- return new CheckResult(false, format(Locale.ROOT,
+ return new CheckResult(fullTableName, false, format(Locale.ROOT,
"Table '%s' is incompatible with existing hive table due
to '%s'", fullTableName, issues));
}
}
@@ -390,4 +434,49 @@ public class TableSchemaUpdateChecker {
issues.add(format(Locale.ROOT, "Columns %s are incompatible " +
"in hive", violateColumns));
}
}
+
+ private static void notifyInvalid(String tableName, String reason,
Set<CubeInstance> impactedCubes) {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+
+ Set<String> users = Sets.newHashSet();
+ for (CubeInstance cube : impactedCubes) {
+ users.addAll(cube.getDescriptor().getNotifyList());
+ }
+ final String[] adminDls = kylinConfig.getAdminDls();
+ if (adminDls != null) {
+ users.addAll(Lists.newArrayList(adminDls));
+ }
+
+ logger.debug("send notification to owner of {} impacted cubes for
table {}", impactedCubes.size(), tableName);
+ Pair<String, String> mail = formatNotifications(tableName,
impactedCubes, reason);
+ new MailService(kylinConfig).sendMail(Lists.newArrayList(users),
mail.getFirst(), mail.getSecond());
+ }
+
+ private static Pair<String, String> formatNotifications(String tableName,
Set<CubeInstance> cubes,
+ String issueDetails) {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ ProjectManager projectManager =
ProjectManager.getInstance(kylinConfig);
+
+ Map<String, Object> root = Maps.newHashMap();
+ root.put("job_name", "Source Data Schema Change");
+ root.put("env_name", kylinConfig.getDeployEnv());
+ root.put("change_details",
Matcher.quoteReplacement(StringUtil.noBlank(issueDetails, "no change info")));
+ root.put("source_type", "HIVE");
+ root.put("source_data", tableName);
+
+ List<Map> impactInfoList = Lists.newArrayList();
+ for (CubeInstance cube : cubes) {
+ Map<String, Object> impactInfoMap = Maps.newHashMap();
+ impactInfoMap.put("project_name",
projectManager.getProjectOfModel(cube.getModel().getName()));
+ impactInfoMap.put("cube_name", cube.getName());
+ impactInfoMap.put("cube_owner", cube.getOwner());
+ impactInfoList.add(impactInfoMap);
+ }
+
+ root.put("impacted_info_List", impactInfoList);
+
+ String content =
MailTemplateProvider.getInstance().buildMailContent("HIVE_SCHEMA_CHANGED",
root);
+ String title = MailNotificationUtil.getMailTitle("TABLE SCHEMA",
"CHANGED", kylinConfig.getDeployEnv());
+ return Pair.newPair(title, content);
+ }
}
diff --git a/tool/src/main/resources/mail_templates/HIVE_SCHEMA_CHANGED.ftl
b/tool/src/main/resources/mail_templates/HIVE_SCHEMA_CHANGED.ftl
new file mode 100644
index 0000000..6e0929a
--- /dev/null
+++ b/tool/src/main/resources/mail_templates/HIVE_SCHEMA_CHANGED.ftl
@@ -0,0 +1,264 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd";>
+<html xmlns="http://www.w3.org/1999/xhtml";>
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative;
charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+<span style="line-height: 1;font-size: 16px;">
+<p style="text-align:left;">Dear Kylin user,</p>
+<p>We have detected that the schema of source data has the following
incompatible changes, which affects the usability
+of the cube. Please upgrade your cube ASAP. Thank you! </p>
+</span>
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+
+ <span style="display: inline;
+background-color: #f0ad4e;
+color: #fff;
+line-height: 1;
+font-weight: 700;
+font-size:36px;
+text-align: center;"> Warn </span>
+
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
style="border-collapse: collapse;border:1px solid #faebcc;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+background-color: #fcf8e3;
+border:1px solid #faebcc;">
+ <h4 style="margin-top: 0;
+margin-bottom: 0;
+font-size: 14px;
+color: #8a6d3b;
+font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${job_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+background-color: #fcf8e3;
+border:1px solid #faebcc;">
+ <h4 style="margin-top: 0;
+margin-bottom: 0;
+font-size: 14px;
+color: #8a6d3b;
+font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${env_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid
#ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica,
sans-serif;">
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+padding: 8px;">
+ <h4 style="
+margin-top: 0;
+margin-bottom: 0;
+line-height: 1.5;
+text-align: left;
+font-size: 14px;
+font-style: normal;">Source Type</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+padding: 8px;">
+ <h4 style="margin-top: 0;
+margin-bottom: 0;
+line-height: 1.5;
+text-align: left;
+font-size: 14px;
+font-style: normal;
+font-weight: 300;">
+ ${source_type}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+padding: 8px;">
+ <h4 style="
+margin-top: 0;
+margin-bottom: 0;
+line-height: 1.5;
+text-align: left;
+font-size: 14px;
+font-style: normal;">Source Data</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+padding: 8px;">
+ <h4 style="margin-top: 0;
+margin-bottom: 0;
+line-height: 1.5;
+text-align: left;
+font-size: 14px;
+font-style: normal;
+font-weight: 300;">
+ ${source_data}</h4>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+background-color: #fcf8e3;
+border:1px solid #faebcc;">
+ <h4 style="margin-top: 0;
+margin-bottom: 0;
+font-size: 14px;
+color: #8a6d3b;
+font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Change Details
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid
#ddd;border-collapse: collapse;table-layout: fixed;font-family: 'Trebuchet MS
', Arial, Helvetica, sans-serif;">
+ <tr>
+ <td style="border: 1px solid #ddd;
+padding: 8px;">
+ <h4 style="margin-top: 0;
+margin-bottom: 0;
+line-height: 1.5;
+text-align: left;
+font-size: 14px;
+font-style: normal;
+font-weight: 300;">
+ <pre style="white-space:
pre-wrap;">${change_details}</pre>
+ </h4>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+background-color: #fcf8e3;
+border:1px solid #faebcc;">
+ <h4 style="margin-top: 0;
+margin-bottom: 0;
+font-size: 14px;
+color: #8a6d3b;
+font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Impacts
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid
#ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica,
sans-serif;">
+ <#list impacted_info_List as impacted_info>
+ <tr>
+ <td style="border: 1px solid #ddd;
+padding: 8px;">
+ <h4 style="margin-top: 0;
+margin-bottom: 0;
+line-height: 1.5;
+text-align: left;
+font-size: 14px;
+font-style: normal;
+font-weight: 300;">
+ ${impacted_info.project_name}</h4>
+ </td>
+ <td style="border: 1px solid #ddd;
+padding: 8px;">
+ <h4 style="margin-top: 0;
+margin-bottom: 0;
+line-height: 1.5;
+text-align: left;
+font-size: 14px;
+font-style: normal;
+font-weight: 300;">
+ ${impacted_info.cube_name}</h4>
+ </td>
+ <td style="border: 1px solid #ddd;
+padding: 8px;">
+ <h4 style="margin-top: 0;
+margin-bottom: 0;
+line-height: 1.5;
+text-align: left;
+font-size: 14px;
+font-style: normal;
+font-weight: 300;">
+ ${impacted_info.cube_owner}</h4>
+ </td>
+ </tr>
+ </#list>
+ </table>
+ </td>
+ </tr>
+ </table>
+ <hr style="margin-top: 20px;
+margin-bottom: 20px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <h4 style="font-weight: 500;
+line-height: 1;font-size:16px;">
+ <p>Best Wishes!</p>
+ <p style="margin: 0 0 10px;">
+ <a href="mailto:dl-ebay-kylin-c...@ebay.com " style="color:
#337ab7;text-decoration: none;">
+ eBay ADI Kylin Team
+ </a>
+ </p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
