Repository: oozie Updated Branches: refs/heads/master e08d696e4 -> eeb4529df
OOZIE-2684 Bad database schema error for WF_ACTIONS table (abhishekbafna, kmarton via andras.piros) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/eeb4529d Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/eeb4529d Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/eeb4529d Branch: refs/heads/master Commit: eeb4529df5b3138596e6e02b9b81ead24494a0f0 Parents: e08d696 Author: Andras Piros <andras.pi...@cloudera.com> Authored: Wed Aug 29 16:22:42 2018 +0200 Committer: Andras Piros <andras.pi...@cloudera.com> Committed: Wed Aug 29 16:22:42 2018 +0200 ---------------------------------------------------------------------- .../oozie/command/SchemaCheckXCommand.java | 9 ++- .../apache/oozie/util/db/CompositeIndex.java | 65 ++++++++++++++++++++ .../oozie/util/db/TestCompositeIndex.java | 53 ++++++++++++++++ release-log.txt | 1 + .../java/org/apache/oozie/tools/OozieDBCLI.java | 21 ++----- .../org/apache/oozie/tools/TestOozieDBCLI.java | 5 +- 6 files changed, 130 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/eeb4529d/core/src/main/java/org/apache/oozie/command/SchemaCheckXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/SchemaCheckXCommand.java b/core/src/main/java/org/apache/oozie/command/SchemaCheckXCommand.java index 521408e..0e69eb2 100644 --- a/core/src/main/java/org/apache/oozie/command/SchemaCheckXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/SchemaCheckXCommand.java @@ -36,6 +36,7 @@ import org.apache.oozie.sla.SLARegistrationBean; import org.apache.oozie.sla.SLASummaryBean; import org.apache.oozie.util.Pair; import org.apache.oozie.util.XLog; +import org.apache.oozie.util.db.CompositeIndex; import org.apache.openjpa.persistence.jdbc.Index; import javax.persistence.Column; @@ -251,7 +252,9 @@ public class SchemaCheckXCommand extends XCommand<Void> { ResultSet rs = metaData.getIndexInfo(catalog, null, table, false, true); while (rs.next()) { String colName = rs.getString("COLUMN_NAME"); - if (colName != null) { + String indexName = rs.getString("INDEX_NAME"); + final boolean isExtraIndexedColumn = !CompositeIndex.find(indexName) && colName != null; + if (isExtraIndexedColumn) { foundIndexedColumns.add(colName); } } @@ -432,10 +435,6 @@ public class SchemaCheckXCommand extends XCommand<Void> { columnTypes.put(name, type); indexedColumns.add(name); } - // For some reason, MySQL doesn't end up having this index... - if (dbType.equals("mysql") && clazz.equals(WorkflowActionBean.class)) { - indexedColumns.remove("wf_id"); - } } private static Integer getSQLType(Class<?> clazz, boolean isLob, String dbType) { http://git-wip-us.apache.org/repos/asf/oozie/blob/eeb4529d/core/src/main/java/org/apache/oozie/util/db/CompositeIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/db/CompositeIndex.java b/core/src/main/java/org/apache/oozie/util/db/CompositeIndex.java new file mode 100644 index 0000000..88b9ca3 --- /dev/null +++ b/core/src/main/java/org/apache/oozie/util/db/CompositeIndex.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.util.db; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; + +import java.util.Arrays; +import java.util.List; + +public enum CompositeIndex { + I_WF_JOBS_STATUS_CREATED_TIME ("WF_JOBS", "status", "created_time"), + I_COORD_ACTIONS_JOB_ID_STATUS ("COORD_ACTIONS", "job_id", "status"), + I_COORD_JOBS_STATUS_CREATED_TIME ("COORD_JOBS", "status", "created_time"), + I_COORD_JOBS_STATUS_LAST_MODIFIED_TIME ("COORD_JOBS", "status", "last_modified_time"), + I_COORD_JOBS_PENDING_DONE_MATERIALIZATION_LAST_MODIFIED_TIME + ("COORD_JOBS", "pending", "done_materialization", "last_modified_time"), + I_COORD_JOBS_PENDING_LAST_MODIFIED_TIME ("COORD_JOBS", "pending", "last_modified_time"), + I_BUNLDE_JOBS_STATUS_CREATED_TIME ("BUNDLE_JOBS", "status", "created_time"), + I_BUNLDE_JOBS_STATUS_LAST_MODIFIED_TIME ("BUNDLE_JOBS", "status", "last_modified_time"), + I_BUNLDE_ACTIONS_PENDING_LAST_MODIFIED_TIME ("BUNDLE_ACTIONS", "pending", "last_modified_time"); + + private final String createStatement; + + CompositeIndex(String tableName, String ... columnNames) { + final String columns = Joiner.on(", ").join(columnNames); + this.createStatement = String.format("CREATE INDEX %s ON %s (%s)", + name(), tableName.toUpperCase(), columns); + } + + public static List<String> getIndexStatements() { + Function<CompositeIndex, String> compositeIndexToString = new Function<CompositeIndex, String>() { + public String apply(CompositeIndex i) { return i != null ? i.createStatement : null; } + }; + + List<CompositeIndex> indexList = Arrays.asList(values()); + return Lists.transform(indexList, compositeIndexToString); + } + + public static boolean find(String indexName) { + try { + valueOf(indexName.toUpperCase()); + } catch (IllegalArgumentException | NullPointerException ex) { + return false; + } + return true; + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/eeb4529d/core/src/test/java/org/apache/oozie/util/db/TestCompositeIndex.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/db/TestCompositeIndex.java b/core/src/test/java/org/apache/oozie/util/db/TestCompositeIndex.java new file mode 100644 index 0000000..4a5c6e4 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/util/db/TestCompositeIndex.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.util.db; + +import org.junit.Assert; +import org.junit.Test; + +public class TestCompositeIndex { + + @Test + public void testisCompositIndexLowerCase() { + Assert.assertTrue("This should be a valid composite index", + CompositeIndex.find("i_coord_actions_job_id_status")); + } + + @Test + public void testisCompositIndexUpperCase() { + Assert.assertTrue("This should be a valid composite index", + CompositeIndex.find("I_COORD_ACTIONS_JOB_ID_STATUS")); + } + + @Test + public void testisCompositIndexNull() { + Assert.assertFalse("This is not a valid composite index", + CompositeIndex.find(null)); + } + + @Test + public void testisCompositIndexEmptyString() { + Assert.assertFalse("This is not a valid composite index", CompositeIndex.find("")); + } + + @Test + public void testIsCompositIndexNegative() { + Assert.assertFalse("This is not a valid composite index", CompositeIndex.find("NotAnIndex")); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/eeb4529d/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 3c569d0..ad517e3 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.1.0 release (trunk - unreleased) +OOZIE-2684 Bad database schema error for WF_ACTIONS table (abhishekbafna, kmarton via andras.piros) OOZIE-3332 [examples] Spark examples should feature yarn client and cluster modes (daniel.becker via andras.piros) OOZIE-3318 [build] Fix Javadoc check in the pre-commit (kmarton via andras.piros) OOZIE-3317 [build] Fix false positive precommit reports (kmarton via andras.piros) http://git-wip-us.apache.org/repos/asf/oozie/blob/eeb4529d/tools/src/main/java/org/apache/oozie/tools/OozieDBCLI.java ---------------------------------------------------------------------- diff --git a/tools/src/main/java/org/apache/oozie/tools/OozieDBCLI.java b/tools/src/main/java/org/apache/oozie/tools/OozieDBCLI.java index eeaaa60..b421aa6 100644 --- a/tools/src/main/java/org/apache/oozie/tools/OozieDBCLI.java +++ b/tools/src/main/java/org/apache/oozie/tools/OozieDBCLI.java @@ -18,6 +18,7 @@ package org.apache.oozie.tools; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; @@ -29,6 +30,7 @@ import org.apache.oozie.cli.CLIParser; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.JPAService; import org.apache.oozie.service.Services; +import org.apache.oozie.util.db.CompositeIndex; import java.io.ByteArrayInputStream; import java.io.File; @@ -747,23 +749,8 @@ public class OozieDBCLI { } } - static String[] getIndexStatementsFor50() { - return new String[]{"CREATE INDEX I_WF_JOBS_STATUS_CREATED_TIME ON WF_JOBS (status, created_time)", - - "CREATE INDEX I_COORD_ACTIONS_JOB_ID_STATUS ON COORD_ACTIONS (job_id, status)", - - "CREATE INDEX I_COORD_JOBS_STATUS_CREATED_TIME ON COORD_JOBS (status, created_time)", - "CREATE INDEX I_COORD_JOBS_STATUS_LAST_MODIFIED_TIME ON COORD_JOBS (status, last_modified_time)", - "CREATE INDEX I_COORD_JOBS_PENDING_DONE_MATERIALIZATION_LAST_MODIFIED_TIME ON COORD_JOBS " + - "(pending, done_materialization, last_modified_time)", - "CREATE INDEX I_COORD_JOBS_PENDING_LAST_MODIFIED_TIME ON COORD_JOBS (pending, last_modified_time)", - - "CREATE INDEX I_BUNLDE_JOBS_STATUS_CREATED_TIME ON BUNDLE_JOBS (status, created_time)", - "CREATE INDEX I_BUNLDE_JOBS_STATUS_LAST_MODIFIED_TIME ON BUNDLE_JOBS (status, last_modified_time)", - - "CREATE INDEX I_BUNLDE_ACTIONS_PENDING_LAST_MODIFIED_TIME ON BUNDLE_ACTIONS (pending, last_modified_time)"}; - } + @SuppressFBWarnings(value = {"SQL_INJECTION_JDBC"}, justification = "Final values are used") private void ddlTweaksFor50(final String sqlFile, final boolean run) throws Exception { System.out.println("Creating composite indexes"); try (final Connection conn = createConnection(); @@ -771,7 +758,7 @@ public class OozieDBCLI { final Statement stmt = conn.createStatement()) { writer.println(); - final String[] createCoveringIndexStatements = getIndexStatementsFor50(); + final List<String> createCoveringIndexStatements = CompositeIndex.getIndexStatements(); for (final String query : createCoveringIndexStatements) { writer.println(query + ";"); http://git-wip-us.apache.org/repos/asf/oozie/blob/eeb4529d/tools/src/test/java/org/apache/oozie/tools/TestOozieDBCLI.java ---------------------------------------------------------------------- diff --git a/tools/src/test/java/org/apache/oozie/tools/TestOozieDBCLI.java b/tools/src/test/java/org/apache/oozie/tools/TestOozieDBCLI.java index cf3427f..3489f20 100644 --- a/tools/src/test/java/org/apache/oozie/tools/TestOozieDBCLI.java +++ b/tools/src/test/java/org/apache/oozie/tools/TestOozieDBCLI.java @@ -23,6 +23,7 @@ import org.apache.oozie.action.hadoop.security.LauncherSecurityManager; import org.apache.oozie.service.Services; import org.apache.hadoop.fs.FileUtil; import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.db.CompositeIndex; import org.junit.Assert; import java.io.ByteArrayOutputStream; @@ -177,7 +178,7 @@ public class TestOozieDBCLI extends XTestCase { final Charset charset = Charset.defaultCharset(); final List<String> stringList = Files.readAllLines(upgrade.toPath(), charset); final List<String> actualIndexStatements = Arrays.asList(stringList.toArray(new String[]{})); - final String[] expectedIndexStatements = OozieDBCLI.getIndexStatementsFor50(); + final List<String> expectedIndexStatements = CompositeIndex.getIndexStatements(); for (final String indexStmt : expectedIndexStatements) { Assert.assertTrue(actualIndexStatements.contains(indexStmt + ";")); @@ -186,7 +187,7 @@ public class TestOozieDBCLI extends XTestCase { private void verifyIndexesCreated() throws SQLException { final List<String> indexes = getIndexes(); - for (final String indexStmt : OozieDBCLI.getIndexStatementsFor50()) { + for (final String indexStmt : CompositeIndex.getIndexStatements()) { final String index = indexStmt.split(" ")[2]; Assert.assertTrue(indexes.contains(index)); }