[
https://issues.apache.org/jira/browse/GOBBLIN-2211?focusedWorklogId=975345&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-975345
]
ASF GitHub Bot logged work on GOBBLIN-2211:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 20/Jul/25 14:07
Start Date: 20/Jul/25 14:07
Worklog Time Spent: 10m
Work Description: abhishekmjain commented on code in PR #4121:
URL: https://github.com/apache/gobblin/pull/4121#discussion_r2217796831
##########
gobblin-metastore/src/main/java/org/apache/gobblin/metastore/ErrorPatternStore.java:
##########
@@ -0,0 +1,46 @@
+package org.apache.gobblin.metastore;
+
+import org.apache.gobblin.configuration.ErrorPatternProfile;
+import org.apache.gobblin.configuration.Category;
+
+import java.io.IOException;
+import java.util.List;
+
+
+/**
+ * Interface for a store that persists Errors and Categories.
+ */
+public interface ErrorPatternStore {
+ void addErrorPattern(ErrorPatternProfile issue)
+ throws IOException;
+
+ boolean deleteErrorPattern(String descriptionRegex)
+ throws IOException;
+
+ ErrorPatternProfile getErrorPattern(String descriptionRegex)
+ throws IOException;
+
+ List<ErrorPatternProfile> getAllErrorPatterns()
+ throws IOException;
+
+ List<ErrorPatternProfile> getErrorPatternsByCategory(String categoryName)
+ throws IOException;
+
+ void addErrorCategory(Category category)
+ throws IOException;
+
+ Category getErrorCategory(String categoryName)
+ throws IOException;
+
+ int getErrorCategoryPriority(String categoryName)
+ throws IOException;
+
+ List<Category> getAllErrorCategories()
+ throws IOException;
+
+ List<ErrorPatternProfile> getAllErrorIssuesOrderedByCategoryPriority()
Review Comment:
we can call this `getAllErrorPatternsOrderedByCategoryPriority`
##########
gobblin-api/src/main/java/org/apache/gobblin/configuration/Category.java:
##########
@@ -0,0 +1,22 @@
+package org.apache.gobblin.configuration;
+
+import java.io.Serializable;
+
+/**
+ * POJO representing a Category, similar to Spec or State.
Review Comment:
We can elaborate on the usage of this class, a reference to other classes
may not be needed.
##########
gobblin-api/src/main/java/org/apache/gobblin/configuration/ErrorPatternProfile.java:
##########
@@ -0,0 +1,34 @@
+package org.apache.gobblin.configuration;
+
+import java.io.Serializable;
+
+
+/**
+ * POJO representing an ErrorPatternProfile, similar to Spec or State.
+ */
+public class ErrorPatternProfile implements Serializable {
Review Comment:
Similar to Category, we can use Lombok and update the comment
##########
gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlErrorPatternStore.java:
##########
@@ -0,0 +1,364 @@
+package org.apache.gobblin.metastore;
+
+import org.apache.gobblin.broker.SharedResourcesBrokerFactory;
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.configuration.ErrorPatternProfile;
+import org.apache.gobblin.configuration.Category;
+
+import com.typesafe.config.Config;
+
+import org.apache.gobblin.util.ConfigUtils;
+
+import javax.sql.DataSource;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * MySQL-backed implementation of IssueStore.
+ *
+ * Expected table schemas:
+ *
+ * 1. error_summary_regex_store
+ * - description_regex: VARCHAR(255) NOT NULL UNIQUE
+ * - error_category_name: VARCHAR(255) NOT NULL
+ *
+ * 2. error_categories
+ * - error_category_name: VARCHAR(255) PRIMARY KEY
+ * - priority: INT UNIQUE NOT NULL
+ * - is_default: BOOLEAN (optional, not compulsory; used if present to
indicate the default category)
+ *
+ * This class provides methods to primarily retrieve error regex patterns and
error categories.
+ * There are also methods to add and delete, which should be used with
caution, and retrieve error patterns and categories.
+ */
+@Slf4j
+public class MysqlErrorPatternStore implements ErrorPatternStore {
+ private final DataSource dataSource;
+ private final String errorRegexSummaryStoreTable;
+ private final String errorCategoriesTable;
+ public static final String CONFIG_PREFIX = "MysqlErrorPatternStore";
+
+ private static final int DEFAULT_MAX_CHARACTERS_IN_SQL_DESCRIPTION_REGEX =
2000;
+ private static final int DEFAULT_MAX_CHARACTERS_IN_SQL_CATEGORY_NAME = 255;
+ private final int maxCharactersInSqlDescriptionRegex;
+ private final int maxCharactersInSqlCategoryName;
+
+ private static final String CREATE_ERROR_REGEX_SUMMARY_STORE_TABLE_STATEMENT
=
+ "CREATE TABLE IF NOT EXISTS %s (" + " description_regex VARCHAR(%d) NOT
NULL UNIQUE, "
+ + " error_category_name VARCHAR(%d) NOT NULL" + ")";
+
+ private static final String CREATE_ERROR_CATEGORIES_TABLE_NAME =
+ "CREATE TABLE IF NOT EXISTS %s (" + " error_category_name VARCHAR(%d)
PRIMARY KEY, priority INT UNIQUE NOT NULL" + " )";
+
+ private static final String INSERT_ERROR_CATEGORY_STATEMENT = "INSERT INTO
%s (error_category_name, priority) "
+ + "VALUES (?, ?) ON DUPLICATE KEY UPDATE priority=VALUES(priority)";
+
+ private static final String GET_ERROR_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s WHERE error_category_name
= ?";
+
+ private static final String GET_ALL_ERROR_CATEGORIES_STATEMENT =
+ "SELECT error_category_name, priority FROM %s ORDER BY priority ASC";
+
+ private static final String INSERT_ERROR_REGEX_SUMMARY_STATEMENT =
+ "INSERT INTO %s (description_regex, error_category_name) "
+ + "VALUES (?, ?) ON DUPLICATE KEY UPDATE
error_category_name=VALUES(error_category_name)";
+
+ private static final String DELETE_ERROR_REGEX_SUMMARY_STATEMENT = "DELETE
FROM %s WHERE description_regex = ?";
+
+ private static final String GET_ERROR_REGEX_SUMMARY_STATEMENT =
+ "SELECT description_regex, error_category_name FROM %s WHERE
description_regex = ?";
+
+ private static final String GET_ALL_ERROR_REGEX_SUMMARIES_STATEMENT =
+ "SELECT description_regex, error_category_name FROM %s";
+
+ private static final String GET_DEFAULT_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s WHERE is_default = TRUE
ORDER BY priority DESC";
+
+ private static final String GET_HIGHEST_ERROR_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s ORDER BY priority ASC
LIMIT 1";
+
+ private static final String
GET_ALL_ERROR_ISSUES_ORDERED_BY_CATEGORY_PRIORITY_STATEMENT =
+ "SELECT e.description_regex, e.error_category_name FROM %s e "
+ + "JOIN %s c ON e.error_category_name = c.error_category_name "
+ + "ORDER BY c.priority ASC, e.description_regex ASC";
+
+ @Inject
+ public MysqlErrorPatternStore(Config config)
+ throws IOException {
+ log.info("Inside MysqlErrorPatternStore constructor");
+ if (config.hasPath(CONFIG_PREFIX)) {
+ config = config.getConfig(CONFIG_PREFIX).withFallback(config);
+ } else {
+ throw new IOException("Please specify the config for
MysqlErrorPatternStore");
+ }
+ this.errorRegexSummaryStoreTable =
+ ConfigUtils.getString(config,
ConfigurationKeys.ERROR_REGEX_DB_TABLE_KEY, "error_summary_regex_store");
+ this.errorCategoriesTable =
+ ConfigUtils.getString(config,
ConfigurationKeys.ERROR_CATEGORIES_DB_TABLE_KEY, "error_categories");
+ this.dataSource = MysqlDataSourceFactory.get(config,
SharedResourcesBrokerFactory.getImplicitBroker());
+
+ this.maxCharactersInSqlDescriptionRegex =
+ ConfigUtils.getInt(config,
ConfigurationKeys.ERROR_REGEX_MAX_VARCHAR_SIZE_KEY,
+ DEFAULT_MAX_CHARACTERS_IN_SQL_DESCRIPTION_REGEX);
+
+ this.maxCharactersInSqlCategoryName =
+ ConfigUtils.getInt(config,
ConfigurationKeys.ERROR_CATEGORY_MAX_VARCHAR_SIZE_KEY,
+ DEFAULT_MAX_CHARACTERS_IN_SQL_CATEGORY_NAME);
+
+ log.info("MysqlErrorPatternStore almost initialized");
+ createTablesIfNotExist();
+ log.info("MysqlErrorPatternStore initialized");
+ }
+
+ private void createTablesIfNotExist()
+ throws IOException {
+
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement createStatement = connection.prepareStatement(
+ String.format(CREATE_ERROR_REGEX_SUMMARY_STORE_TABLE_STATEMENT,
errorRegexSummaryStoreTable,
+ maxCharactersInSqlDescriptionRegex,
maxCharactersInSqlCategoryName))) {
+ createStatement.executeUpdate();
+ connection.commit();
+ } catch (SQLException e) {
+ throw new IOException("Failure creation: error_regex_summary.", e);
+ }
+
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement createStatement = connection.prepareStatement(
+ String.format(CREATE_ERROR_CATEGORIES_TABLE_NAME,
errorCategoriesTable, maxCharactersInSqlCategoryName))) {
+ createStatement.executeUpdate();
+ connection.commit();
+ } catch (SQLException e) {
+ throw new IOException("Failure creation: error_categories.", e);
+ }
+ }
Review Comment:
We can use the same connection object for executing both the queries
##########
gobblin-api/src/main/java/org/apache/gobblin/configuration/Category.java:
##########
@@ -0,0 +1,22 @@
+package org.apache.gobblin.configuration;
+
+import java.io.Serializable;
+
+/**
+ * POJO representing a Category, similar to Spec or State.
+ */
+public class Category implements Serializable {
Review Comment:
We can call this ErrorCategory as I don't think it could be used elsewhere.
##########
gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlErrorPatternStore.java:
##########
@@ -0,0 +1,364 @@
+package org.apache.gobblin.metastore;
+
+import org.apache.gobblin.broker.SharedResourcesBrokerFactory;
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.configuration.ErrorPatternProfile;
+import org.apache.gobblin.configuration.Category;
+
+import com.typesafe.config.Config;
+
+import org.apache.gobblin.util.ConfigUtils;
+
+import javax.sql.DataSource;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * MySQL-backed implementation of IssueStore.
+ *
+ * Expected table schemas:
+ *
+ * 1. error_summary_regex_store
+ * - description_regex: VARCHAR(255) NOT NULL UNIQUE
+ * - error_category_name: VARCHAR(255) NOT NULL
+ *
+ * 2. error_categories
+ * - error_category_name: VARCHAR(255) PRIMARY KEY
+ * - priority: INT UNIQUE NOT NULL
+ * - is_default: BOOLEAN (optional, not compulsory; used if present to
indicate the default category)
+ *
+ * This class provides methods to primarily retrieve error regex patterns and
error categories.
+ * There are also methods to add and delete, which should be used with
caution, and retrieve error patterns and categories.
+ */
+@Slf4j
+public class MysqlErrorPatternStore implements ErrorPatternStore {
+ private final DataSource dataSource;
+ private final String errorRegexSummaryStoreTable;
+ private final String errorCategoriesTable;
+ public static final String CONFIG_PREFIX = "MysqlErrorPatternStore";
+
+ private static final int DEFAULT_MAX_CHARACTERS_IN_SQL_DESCRIPTION_REGEX =
2000;
+ private static final int DEFAULT_MAX_CHARACTERS_IN_SQL_CATEGORY_NAME = 255;
+ private final int maxCharactersInSqlDescriptionRegex;
+ private final int maxCharactersInSqlCategoryName;
+
+ private static final String CREATE_ERROR_REGEX_SUMMARY_STORE_TABLE_STATEMENT
=
+ "CREATE TABLE IF NOT EXISTS %s (" + " description_regex VARCHAR(%d) NOT
NULL UNIQUE, "
+ + " error_category_name VARCHAR(%d) NOT NULL" + ")";
+
+ private static final String CREATE_ERROR_CATEGORIES_TABLE_NAME =
+ "CREATE TABLE IF NOT EXISTS %s (" + " error_category_name VARCHAR(%d)
PRIMARY KEY, priority INT UNIQUE NOT NULL" + " )";
+
+ private static final String INSERT_ERROR_CATEGORY_STATEMENT = "INSERT INTO
%s (error_category_name, priority) "
+ + "VALUES (?, ?) ON DUPLICATE KEY UPDATE priority=VALUES(priority)";
+
+ private static final String GET_ERROR_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s WHERE error_category_name
= ?";
+
+ private static final String GET_ALL_ERROR_CATEGORIES_STATEMENT =
+ "SELECT error_category_name, priority FROM %s ORDER BY priority ASC";
+
+ private static final String INSERT_ERROR_REGEX_SUMMARY_STATEMENT =
+ "INSERT INTO %s (description_regex, error_category_name) "
+ + "VALUES (?, ?) ON DUPLICATE KEY UPDATE
error_category_name=VALUES(error_category_name)";
+
+ private static final String DELETE_ERROR_REGEX_SUMMARY_STATEMENT = "DELETE
FROM %s WHERE description_regex = ?";
+
+ private static final String GET_ERROR_REGEX_SUMMARY_STATEMENT =
+ "SELECT description_regex, error_category_name FROM %s WHERE
description_regex = ?";
+
+ private static final String GET_ALL_ERROR_REGEX_SUMMARIES_STATEMENT =
+ "SELECT description_regex, error_category_name FROM %s";
+
+ private static final String GET_DEFAULT_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s WHERE is_default = TRUE
ORDER BY priority DESC";
+
+ private static final String GET_HIGHEST_ERROR_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s ORDER BY priority ASC
LIMIT 1";
+
+ private static final String
GET_ALL_ERROR_ISSUES_ORDERED_BY_CATEGORY_PRIORITY_STATEMENT =
+ "SELECT e.description_regex, e.error_category_name FROM %s e "
+ + "JOIN %s c ON e.error_category_name = c.error_category_name "
+ + "ORDER BY c.priority ASC, e.description_regex ASC";
+
+ @Inject
+ public MysqlErrorPatternStore(Config config)
+ throws IOException {
+ log.info("Inside MysqlErrorPatternStore constructor");
Review Comment:
Remove debugging statements
##########
gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlErrorPatternStore.java:
##########
@@ -0,0 +1,364 @@
+package org.apache.gobblin.metastore;
+
+import org.apache.gobblin.broker.SharedResourcesBrokerFactory;
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.configuration.ErrorPatternProfile;
+import org.apache.gobblin.configuration.Category;
+
+import com.typesafe.config.Config;
+
+import org.apache.gobblin.util.ConfigUtils;
+
+import javax.sql.DataSource;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * MySQL-backed implementation of IssueStore.
+ *
+ * Expected table schemas:
+ *
+ * 1. error_summary_regex_store
+ * - description_regex: VARCHAR(255) NOT NULL UNIQUE
+ * - error_category_name: VARCHAR(255) NOT NULL
+ *
+ * 2. error_categories
+ * - error_category_name: VARCHAR(255) PRIMARY KEY
+ * - priority: INT UNIQUE NOT NULL
+ * - is_default: BOOLEAN (optional, not compulsory; used if present to
indicate the default category)
+ *
+ * This class provides methods to primarily retrieve error regex patterns and
error categories.
+ * There are also methods to add and delete, which should be used with
caution, and retrieve error patterns and categories.
+ */
+@Slf4j
+public class MysqlErrorPatternStore implements ErrorPatternStore {
+ private final DataSource dataSource;
+ private final String errorRegexSummaryStoreTable;
+ private final String errorCategoriesTable;
+ public static final String CONFIG_PREFIX = "MysqlErrorPatternStore";
+
+ private static final int DEFAULT_MAX_CHARACTERS_IN_SQL_DESCRIPTION_REGEX =
2000;
+ private static final int DEFAULT_MAX_CHARACTERS_IN_SQL_CATEGORY_NAME = 255;
+ private final int maxCharactersInSqlDescriptionRegex;
+ private final int maxCharactersInSqlCategoryName;
+
+ private static final String CREATE_ERROR_REGEX_SUMMARY_STORE_TABLE_STATEMENT
=
+ "CREATE TABLE IF NOT EXISTS %s (" + " description_regex VARCHAR(%d) NOT
NULL UNIQUE, "
Review Comment:
nit: string join is not required here, can be a continuous string
##########
gobblin-api/src/main/java/org/apache/gobblin/configuration/Category.java:
##########
@@ -0,0 +1,22 @@
+package org.apache.gobblin.configuration;
+
+import java.io.Serializable;
+
+/**
+ * POJO representing a Category, similar to Spec or State.
+ */
+public class Category implements Serializable {
+ private String categoryName;
+ private int priority;
+
+ public Category(String categoryName, int priority) {
+ this.categoryName = categoryName;
+ this.priority = priority;
+ }
+
+ public String getCategoryName() { return categoryName; }
Review Comment:
We can use lombok for getters and setters
##########
gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlErrorPatternStore.java:
##########
@@ -0,0 +1,364 @@
+package org.apache.gobblin.metastore;
+
+import org.apache.gobblin.broker.SharedResourcesBrokerFactory;
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.configuration.ErrorPatternProfile;
+import org.apache.gobblin.configuration.Category;
+
+import com.typesafe.config.Config;
+
+import org.apache.gobblin.util.ConfigUtils;
+
+import javax.sql.DataSource;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * MySQL-backed implementation of IssueStore.
Review Comment:
`ErrorPatternStore`
##########
gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java:
##########
@@ -85,6 +85,14 @@ public class ConfigurationKeys {
public static final String DATASETURN_STATESTORE_NAME_PARSER =
"state.store.datasetUrnStateStoreNameParser";
+ /**
+ * Error Pattern Store related to configuration properties.
+ */
+ public static final String ERROR_REGEX_DB_TABLE_KEY =
"state.store.db.regexTableName";
+ public static final String ERROR_CATEGORIES_DB_TABLE_KEY =
"state.store.db.categoriesTableName";
+ public static final String ERROR_REGEX_MAX_VARCHAR_SIZE_KEY =
"error.regex.max.varchar.size";
+ public static final String ERROR_CATEGORY_MAX_VARCHAR_SIZE_KEY =
"error.category.max.varchar.size";
Review Comment:
Since it represents the size of type varchar column, we can drop max from
the names.
```
public static final String ERROR_REGEX_VARCHAR_SIZE_KEY =
"error.regex.varchar.size";
public static final String ERROR_CATEGORY_VARCHAR_SIZE_KEY =
"error.category.varchar.size";
```
##########
gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlErrorPatternStore.java:
##########
@@ -0,0 +1,364 @@
+package org.apache.gobblin.metastore;
+
+import org.apache.gobblin.broker.SharedResourcesBrokerFactory;
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.configuration.ErrorPatternProfile;
+import org.apache.gobblin.configuration.Category;
+
+import com.typesafe.config.Config;
+
+import org.apache.gobblin.util.ConfigUtils;
+
+import javax.sql.DataSource;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * MySQL-backed implementation of IssueStore.
+ *
+ * Expected table schemas:
+ *
+ * 1. error_summary_regex_store
+ * - description_regex: VARCHAR(255) NOT NULL UNIQUE
+ * - error_category_name: VARCHAR(255) NOT NULL
+ *
+ * 2. error_categories
+ * - error_category_name: VARCHAR(255) PRIMARY KEY
+ * - priority: INT UNIQUE NOT NULL
+ * - is_default: BOOLEAN (optional, not compulsory; used if present to
indicate the default category)
+ *
+ * This class provides methods to primarily retrieve error regex patterns and
error categories.
+ * There are also methods to add and delete, which should be used with
caution, and retrieve error patterns and categories.
+ */
+@Slf4j
+public class MysqlErrorPatternStore implements ErrorPatternStore {
+ private final DataSource dataSource;
+ private final String errorRegexSummaryStoreTable;
+ private final String errorCategoriesTable;
+ public static final String CONFIG_PREFIX = "MysqlErrorPatternStore";
+
+ private static final int DEFAULT_MAX_CHARACTERS_IN_SQL_DESCRIPTION_REGEX =
2000;
+ private static final int DEFAULT_MAX_CHARACTERS_IN_SQL_CATEGORY_NAME = 255;
+ private final int maxCharactersInSqlDescriptionRegex;
+ private final int maxCharactersInSqlCategoryName;
+
+ private static final String CREATE_ERROR_REGEX_SUMMARY_STORE_TABLE_STATEMENT
=
+ "CREATE TABLE IF NOT EXISTS %s (" + " description_regex VARCHAR(%d) NOT
NULL UNIQUE, "
+ + " error_category_name VARCHAR(%d) NOT NULL" + ")";
+
+ private static final String CREATE_ERROR_CATEGORIES_TABLE_NAME =
+ "CREATE TABLE IF NOT EXISTS %s (" + " error_category_name VARCHAR(%d)
PRIMARY KEY, priority INT UNIQUE NOT NULL" + " )";
+
+ private static final String INSERT_ERROR_CATEGORY_STATEMENT = "INSERT INTO
%s (error_category_name, priority) "
+ + "VALUES (?, ?) ON DUPLICATE KEY UPDATE priority=VALUES(priority)";
+
+ private static final String GET_ERROR_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s WHERE error_category_name
= ?";
+
+ private static final String GET_ALL_ERROR_CATEGORIES_STATEMENT =
+ "SELECT error_category_name, priority FROM %s ORDER BY priority ASC";
+
+ private static final String INSERT_ERROR_REGEX_SUMMARY_STATEMENT =
+ "INSERT INTO %s (description_regex, error_category_name) "
+ + "VALUES (?, ?) ON DUPLICATE KEY UPDATE
error_category_name=VALUES(error_category_name)";
+
+ private static final String DELETE_ERROR_REGEX_SUMMARY_STATEMENT = "DELETE
FROM %s WHERE description_regex = ?";
+
+ private static final String GET_ERROR_REGEX_SUMMARY_STATEMENT =
+ "SELECT description_regex, error_category_name FROM %s WHERE
description_regex = ?";
+
+ private static final String GET_ALL_ERROR_REGEX_SUMMARIES_STATEMENT =
+ "SELECT description_regex, error_category_name FROM %s";
+
+ private static final String GET_DEFAULT_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s WHERE is_default = TRUE
ORDER BY priority DESC";
+
+ private static final String GET_HIGHEST_ERROR_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s ORDER BY priority ASC
LIMIT 1";
+
+ private static final String
GET_ALL_ERROR_ISSUES_ORDERED_BY_CATEGORY_PRIORITY_STATEMENT =
+ "SELECT e.description_regex, e.error_category_name FROM %s e "
+ + "JOIN %s c ON e.error_category_name = c.error_category_name "
+ + "ORDER BY c.priority ASC, e.description_regex ASC";
Review Comment:
if this ordering is not required, we can skip it
##########
gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlErrorPatternStore.java:
##########
@@ -0,0 +1,364 @@
+package org.apache.gobblin.metastore;
+
+import org.apache.gobblin.broker.SharedResourcesBrokerFactory;
+import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.configuration.ErrorPatternProfile;
+import org.apache.gobblin.configuration.Category;
+
+import com.typesafe.config.Config;
+
+import org.apache.gobblin.util.ConfigUtils;
+
+import javax.sql.DataSource;
+
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * MySQL-backed implementation of IssueStore.
+ *
+ * Expected table schemas:
+ *
+ * 1. error_summary_regex_store
+ * - description_regex: VARCHAR(255) NOT NULL UNIQUE
+ * - error_category_name: VARCHAR(255) NOT NULL
+ *
+ * 2. error_categories
+ * - error_category_name: VARCHAR(255) PRIMARY KEY
+ * - priority: INT UNIQUE NOT NULL
+ * - is_default: BOOLEAN (optional, not compulsory; used if present to
indicate the default category)
+ *
+ * This class provides methods to primarily retrieve error regex patterns and
error categories.
+ * There are also methods to add and delete, which should be used with
caution, and retrieve error patterns and categories.
+ */
+@Slf4j
+public class MysqlErrorPatternStore implements ErrorPatternStore {
+ private final DataSource dataSource;
+ private final String errorRegexSummaryStoreTable;
+ private final String errorCategoriesTable;
+ public static final String CONFIG_PREFIX = "MysqlErrorPatternStore";
+
+ private static final int DEFAULT_MAX_CHARACTERS_IN_SQL_DESCRIPTION_REGEX =
2000;
+ private static final int DEFAULT_MAX_CHARACTERS_IN_SQL_CATEGORY_NAME = 255;
+ private final int maxCharactersInSqlDescriptionRegex;
+ private final int maxCharactersInSqlCategoryName;
+
+ private static final String CREATE_ERROR_REGEX_SUMMARY_STORE_TABLE_STATEMENT
=
+ "CREATE TABLE IF NOT EXISTS %s (" + " description_regex VARCHAR(%d) NOT
NULL UNIQUE, "
+ + " error_category_name VARCHAR(%d) NOT NULL" + ")";
+
+ private static final String CREATE_ERROR_CATEGORIES_TABLE_NAME =
+ "CREATE TABLE IF NOT EXISTS %s (" + " error_category_name VARCHAR(%d)
PRIMARY KEY, priority INT UNIQUE NOT NULL" + " )";
+
+ private static final String INSERT_ERROR_CATEGORY_STATEMENT = "INSERT INTO
%s (error_category_name, priority) "
+ + "VALUES (?, ?) ON DUPLICATE KEY UPDATE priority=VALUES(priority)";
+
+ private static final String GET_ERROR_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s WHERE error_category_name
= ?";
+
+ private static final String GET_ALL_ERROR_CATEGORIES_STATEMENT =
+ "SELECT error_category_name, priority FROM %s ORDER BY priority ASC";
+
+ private static final String INSERT_ERROR_REGEX_SUMMARY_STATEMENT =
+ "INSERT INTO %s (description_regex, error_category_name) "
+ + "VALUES (?, ?) ON DUPLICATE KEY UPDATE
error_category_name=VALUES(error_category_name)";
+
+ private static final String DELETE_ERROR_REGEX_SUMMARY_STATEMENT = "DELETE
FROM %s WHERE description_regex = ?";
+
+ private static final String GET_ERROR_REGEX_SUMMARY_STATEMENT =
+ "SELECT description_regex, error_category_name FROM %s WHERE
description_regex = ?";
+
+ private static final String GET_ALL_ERROR_REGEX_SUMMARIES_STATEMENT =
+ "SELECT description_regex, error_category_name FROM %s";
+
+ private static final String GET_DEFAULT_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s WHERE is_default = TRUE
ORDER BY priority DESC";
+
+ private static final String GET_HIGHEST_ERROR_CATEGORY_STATEMENT =
+ "SELECT error_category_name, priority FROM %s ORDER BY priority ASC
LIMIT 1";
Review Comment:
This should be DESC right? `UNKNOWN` will have the largest `priority` value,
hence the least preference.
Issue Time Tracking
-------------------
Worklog Id: (was: 975345)
Time Spent: 0.5h (was: 20m)
> Implement Error Classification based on execution issues
> --------------------------------------------------------
>
> Key: GOBBLIN-2211
> URL: https://issues.apache.org/jira/browse/GOBBLIN-2211
> Project: Apache Gobblin
> Issue Type: Bug
> Components: gobblin-service
> Reporter: Abhishek Jain
> Assignee: Abhishek Tiwari
> Priority: Major
> Time Spent: 0.5h
> Remaining Estimate: 0h
>
> Implement Error Classification to categorize the failure reason based on
> issues encountered.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
