Merge remote-tracking branch 'upstream/master' into CAY-1977_1 Conflicts: cayenne-server/src/main/java/org/apache/cayenne/access/DbLoader.java modeler/cayenne-modeler/src/main/java/org/apache/cayenne/modeler/dialog/db/DbLoaderHelper.java
Project: http://git-wip-us.apache.org/repos/asf/cayenne/repo Commit: http://git-wip-us.apache.org/repos/asf/cayenne/commit/2a6ef007 Tree: http://git-wip-us.apache.org/repos/asf/cayenne/tree/2a6ef007 Diff: http://git-wip-us.apache.org/repos/asf/cayenne/diff/2a6ef007 Branch: refs/heads/master Commit: 2a6ef007b32c250d825725214073f4ab7c854502 Parents: 0b2acf6 2894d5f Author: Savva Kolbachev <s.kolbac...@gmail.com> Authored: Wed Dec 10 14:11:03 2014 +0300 Committer: Savva Kolbachev <s.kolbac...@gmail.com> Committed: Wed Dec 10 14:11:03 2014 +0300 ---------------------------------------------------------------------- .../org/apache/cayenne/access/DbLoader.java | 606 +++++---- .../loader/ManyToManyCandidateEntity.java | 3 +- .../cayenne/access/loader/filters/DbPath.java | 6 + .../access/loader/filters/EntityFilters.java | 4 + .../translator/select/QualifierTranslator.java | 52 +- .../cayenne/exp/LikeExpressionHelper.java | 1 - .../java/org/apache/cayenne/map/DataMap.java | 34 +- .../apache/cayenne/map/naming/ExportedKey.java | 111 +- .../java/org/apache/cayenne/merge/DbMerger.java | 48 +- .../org/apache/cayenne/access/DbLoaderIT.java | 26 +- .../access/loader/filters/DbPathTest.java | 18 + .../cayenne/exp/LikeExpressionHelperTest.java | 83 ++ .../map/naming/LegacyNameGeneratorTest.java | 4 +- .../merge/DropRelationshipToModelIT.java | 30 +- .../org/apache/cayenne/merge/MergeCase.java | 24 +- .../apache/cayenne/merge/MergerFactoryIT.java | 47 +- .../org/apache/cayenne/query/SelectQueryIT.java | 1243 +++++++++--------- .../cayenne/tools/dbimport/DbImportAction.java | 3 +- .../map/naming/DefaultNameGeneratorTest.java | 6 +- .../tools/dbimport/DbImportActionTest.java | 6 +- docs/doc/src/main/resources/RELEASE-NOTES.txt | 1 + .../InferRelationshipsControllerBase.java | 2 +- .../modeler/dialog/db/DbLoaderHelper.java | 40 +- .../dialog/db/DbLoaderOptionsDialog.java | 72 +- .../dialog/objentity/ObjRelationshipInfo.java | 8 +- 25 files changed, 1400 insertions(+), 1078 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cayenne/blob/2a6ef007/cayenne-server/src/main/java/org/apache/cayenne/access/DbLoader.java ---------------------------------------------------------------------- diff --cc cayenne-server/src/main/java/org/apache/cayenne/access/DbLoader.java index dec9201,5537ac9..3b098c8 --- a/cayenne-server/src/main/java/org/apache/cayenne/access/DbLoader.java +++ b/cayenne-server/src/main/java/org/apache/cayenne/access/DbLoader.java @@@ -18,27 -18,26 +18,12 @@@ ****************************************************************/ package org.apache.cayenne.access; --import java.sql.Connection; --import java.sql.DatabaseMetaData; --import java.sql.ResultSet; --import java.sql.SQLException; --import java.util.ArrayList; --import java.util.Arrays; --import java.util.Collection; --import java.util.HashMap; - import java.util.HashSet; --import java.util.LinkedList; --import java.util.List; --import java.util.Map; --import java.util.Set; - - import org.apache.cayenne.CayenneException; -import java.util.TreeSet; - import org.apache.cayenne.access.loader.DbLoaderConfiguration; import org.apache.cayenne.access.loader.ManyToManyCandidateEntity; ++import org.apache.cayenne.access.loader.filters.DbPath; import org.apache.cayenne.access.loader.filters.EntityFilters; import org.apache.cayenne.access.loader.filters.Filter; import org.apache.cayenne.access.loader.filters.FiltersConfig; --import org.apache.cayenne.access.loader.filters.DbPath; import org.apache.cayenne.dba.DbAdapter; import org.apache.cayenne.dba.TypesMapping; import org.apache.cayenne.map.DataMap; @@@ -61,7 -59,7 +45,23 @@@ import org.apache.cayenne.util.EntityMe import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; --import static org.apache.cayenne.access.loader.filters.FilterFactory.*; ++import java.sql.Connection; ++import java.sql.DatabaseMetaData; ++import java.sql.ResultSet; ++import java.sql.SQLException; ++import java.util.ArrayList; ++import java.util.Arrays; ++import java.util.Collection; ++import java.util.HashMap; ++import java.util.LinkedList; ++import java.util.List; ++import java.util.Map; ++import java.util.Set; ++import java.util.TreeSet; ++ ++import static org.apache.cayenne.access.loader.filters.FilterFactory.NULL; ++import static org.apache.cayenne.access.loader.filters.FilterFactory.TRUE; ++import static org.apache.cayenne.access.loader.filters.FilterFactory.include; /** * Utility class that does reverse engineering of the database. It can create @@@ -97,7 -95,7 +97,7 @@@ public class DbLoader /** * Creates new DbLoader with specified naming strategy. -- * ++ * * @since 3.0 */ public DbLoader(Connection connection, DbAdapter adapter, DbLoaderDelegate delegate, ObjectNameGenerator strategy) { @@@ -137,7 -135,7 +137,7 @@@ /** * Returns database connection used by this DbLoader. -- * ++ * * @since 3.0 */ public Connection getConnection() { @@@ -146,7 -144,7 +146,7 @@@ /** * Returns DbAdapter associated with this DbLoader. -- * ++ * * @since 1.1 */ public DbAdapter getAdapter() { @@@ -155,7 -153,7 +155,7 @@@ /** * Retrieves catalogs for the database associated with this DbLoader. -- * ++ * * @return List with the catalog names, empty Array if none found. */ public List<String> getCatalogs() throws SQLException { @@@ -164,7 -162,7 +164,7 @@@ /** * Retrieves the schemas for the database. -- * ++ * * @return List with the schema names, empty Array if none found. */ public List<String> getSchemas() throws SQLException { @@@ -186,9 -184,9 +186,9 @@@ /** * Returns all the table types for the given database. Types may be such as * Typical types are "TABLE", -- * "VIEW", "SYSTEM TABLE", "GLOBAL TEMPORARY", -- * "LOCAL TEMPORARY", "ALIAS", "SYNONYM"., etc. -- * ++ * "VIEW", "SYSTEM TABLE", "GLOBAL TEMPORARY", ++ * "LOCAL TEMPORARY", "ALIAS", "SYNONYM"., etc. ++ * * @return List of Strings, empty array if nothing found. */ public List<String> getTableTypes() throws SQLException { @@@ -208,12 -206,12 +208,9 @@@ /** * Returns all tables for given combination of the criteria. Tables returned * as DbEntities without any attributes or relationships. -- * * * @param config -- * -- * @param types -- * The types of table names to retrieve, null returns all types. ++ * @param types The types of table names to retrieve, null returns all types. * @return * @since 3.2 */ @@@ -229,7 -230,16 +229,14 @@@ return tables; } - private List<DbEntity> getDbEntities(FiltersConfig filters, DbPath dbPath, String[] types) throws SQLException { + /** - * + * @param filters + * @param dbPath + * @param types + * @return Map<TableName, DbEntity> - * + * @throws SQLException + */ + private Map<String, DbEntity> getDbEntities(FiltersConfig filters, DbPath dbPath, String[] types) throws SQLException { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Read tables: catalog=" + dbPath.catalog + ", schema=" + dbPath.schema + ", types=" + Arrays.toString(types)); @@@ -270,73 -280,52 +277,62 @@@ /** * Loads dbEntities for the specified tables. -- * -- * @param map -- * DataMap to be populated with DbEntities. ++ * ++ * @param map DataMap to be populated with DbEntities. * @param config -- * @param tables -- * The list of org.apache.cayenne.ashwood.dbutil.Table objects -- * for which DbEntities must be created. @return false if loading must be immediately aborted. ++ * @param tables The list of org.apache.cayenne.ashwood.dbutil.Table objects ++ * for which DbEntities must be created. @return false if loading must be immediately aborted. */ - public List<DbEntity> loadDbEntities(DataMap map, DbLoaderConfiguration config, Collection<? extends DbEntity> tables) throws SQLException { + public List<DbEntity> loadDbEntities(DataMap map, DbLoaderConfiguration config, Map<DbPath, Map<String, DbEntity>> tables) throws SQLException { /** List of db entities to process. */ - List<DbEntity> dbEntityList = new ArrayList<DbEntity>(); - for (DbEntity dbEntity : tables) { - - // Check if there already is a DbEntity under such name - // if so, consult the delegate what to do - DbEntity oldEnt = map.getDbEntity(dbEntity.getName()); - if (oldEnt != null) { - if (delegate == null) { - break; // no delegate, don't know what to do, cancel import - // TODO continue? - } - Collection<ObjEntity> oldObjEnt = map.getMappedEntities(oldEnt); - if (!oldObjEnt.isEmpty()) { - for (ObjEntity objEntity : oldObjEnt) { - LOGGER.debug("Delete ObjEntity: " + objEntity.getName()); - map.removeObjEntity(objEntity.getName(), true); - delegate.objEntityRemoved(objEntity); + List<DbEntity> dbEntityList = new ArrayList<DbEntity>(); + for (Map.Entry<DbPath, Map<String, DbEntity>> tablesMap : tables.entrySet()) { + for (DbEntity dbEntity : tablesMap.getValue().values()) { + + // Check if there already is a DbEntity under such name + // if so, consult the delegate what to do + DbEntity oldEnt = map.getDbEntity(dbEntity.getName()); + if (oldEnt != null) { ++ Collection<ObjEntity> oldObjEnt = map.getMappedEntities(oldEnt); ++ if (!oldObjEnt.isEmpty()) { ++ for (ObjEntity objEntity : oldObjEnt) { ++ LOGGER.debug("Delete ObjEntity: " + objEntity.getName()); ++ map.removeObjEntity(objEntity.getName(), true); ++ delegate.objEntityRemoved(objEntity); ++ } + } - } + - LOGGER.debug("Overwrite DbEntity: " + oldEnt.getName()); - map.removeDbEntity(oldEnt.getName(), true); - delegate.dbEntityRemoved(oldEnt); ++ LOGGER.debug("Overwrite DbEntity: " + oldEnt.getName()); + map.removeDbEntity(oldEnt.getName(), true); ++ delegate.dbEntityRemoved(oldEnt); + } + - } + map.addDbEntity(dbEntity); - map.addDbEntity(dbEntity); + // notify delegate + if (delegate != null) { + delegate.dbEntityAdded(dbEntity); + } - // notify delegate - if (delegate != null) { - delegate.dbEntityAdded(dbEntity); + // delegate might have thrown this entity out... so check if it is still + // around before continuing processing + if (map.getDbEntity(dbEntity.getName()) == dbEntity) { + dbEntityList.add(dbEntity); + } } - loadDbAttributes(config.getFiltersConfig(), dbEntity); - // delegate might have thrown this entity out... so check if it is still - // around before continuing processing - if (map.getDbEntity(dbEntity.getName()) == dbEntity) { - dbEntityList.add(dbEntity); - } - } + loadDbAttributes(config.getFiltersConfig(), tablesMap.getKey(), tablesMap.getValue()); - // get primary keys for each table and store it in dbEntity - getPrimaryKeysForEachTableAndStoreItInDbEntity(map, tables); + // get primary keys for each table and store it in dbEntity + getPrimaryKeyForTable(tablesMap.getValue()); + } return dbEntityList; - } - private void getPrimaryKeysForEachTableAndStoreItInDbEntity(DataMap map, Collection<? extends DbEntity> tables) - throws SQLException { - - for (DbEntity dbEntity : map.getDbEntities()) { - if (!tables.contains(dbEntity)) { // TODO is it ok? equals is not overridden - continue; - } - + private void getPrimaryKeyForTable(Map<String, DbEntity> tables) throws SQLException { + for (DbEntity dbEntity : tables.values()) { ResultSet rs = getMetaData().getPrimaryKeys(dbEntity.getCatalog(), dbEntity.getSchema(), dbEntity.getName()); try { while (rs.next()) { @@@ -448,14 -443,14 +450,14 @@@ for (DbEntity dbEntity : entities) { // check if there are existing entities -- -- // TODO: performance. This is an O(n^2) search and it shows on -- // YourKit profiles. Pre-cache mapped entities perhaps (?) -- Collection<ObjEntity> existing = map.getMappedEntities(dbEntity); -- if (!existing.isEmpty()) { -- loadedEntities.addAll(existing); -- continue; -- } ++ ++ // TODO: performance. This is an O(n^2) search and it shows on ++ // YourKit profiles. Pre-cache mapped entities perhaps (?) ++ Collection<ObjEntity> existing = map.getMappedEntities(dbEntity); ++ if (!existing.isEmpty()) { ++ loadedEntities.addAll(existing); ++ continue; ++ } String objEntityName = DefaultUniqueNameGenerator.generate(NameCheckers.objEntity, map, nameGenerator.createObjEntityName(dbEntity)); @@@ -678,31 -678,31 +688,31 @@@ * Performs database reverse engineering and generates DataMap that contains * default mapping of the tables and views. By default will include regular * tables and views. -- * ++ * * @since 1.0.7 * @deprecated since 4.0 use -- * {@link #load(org.apache.cayenne.map.DataMap, DbLoaderConfiguration, String...)} -- * method that supports catalogs. ++ * {@link #load(org.apache.cayenne.map.DataMap, DbLoaderConfiguration, String...)} ++ * method that supports catalogs. */ @Deprecated -- public DataMap loadDataMapFromDB(String schemaPattern, String tablePattern, DataMap dataMap) throws SQLException { ++ public DataMap loadDataMapFromDB(String schemaPattern, String tablePattern, DataMap dataMap) throws SQLException { -- DbLoaderConfiguration configuration = new DbLoaderConfiguration(); -- configuration.setFiltersConfig(new FiltersConfig(new EntityFilters(new DbPath(null, schemaPattern), -- include(tablePattern), TRUE, NULL))); ++ DbLoaderConfiguration configuration = new DbLoaderConfiguration(); ++ configuration.setFiltersConfig(new FiltersConfig(new EntityFilters(new DbPath(null, schemaPattern), ++ include(tablePattern), TRUE, NULL))); -- load(dataMap, configuration); -- return dataMap; -- } ++ load(dataMap, configuration); ++ return dataMap; ++ } /** * Performs database reverse engineering and generates DataMap object that * contains default mapping of the tables and views. Allows to limit types * of tables to read. -- * ++ * * @deprecated since 4.0 use -- * {@link #load(org.apache.cayenne.map.DataMap, DbLoaderConfiguration, String...)} -- * method that supports catalogs. ++ * {@link #load(org.apache.cayenne.map.DataMap, DbLoaderConfiguration, String...)} ++ * method that supports catalogs. */ @Deprecated public DataMap loadDataMapFromDB(String schemaPattern, String tablePattern, String[] tableTypes, DataMap dataMap) @@@ -729,27 -729,41 +739,41 @@@ } /** -- * Performs database reverse engineering based on the specified config ++ * Performs database reverse engineering based on the specified config * and fills the specified * DataMap object with DB and object mapping info. * * @since 4.0 */ - public void load(DataMap dataMap, DbLoaderConfiguration config) throws SQLException { + public void load(DataMap dataMap, DbLoaderConfiguration config, String... tableTypes) throws SQLException { + + Map<DbPath, Map<String, DbEntity>> tables = getTables(config, tableTypes); + List<DbEntity> entities = loadDbEntities(dataMap, config, tables); - String[] tableTypes = config.getTableTypes() == null ? this.getDefaultTableTypes() : config.getTableTypes(); - List<DbEntity> entities = loadDbEntities(dataMap, config, getTables(config, tableTypes)); + if (entities != null) { + loadDbRelationships(config, tables); ++ Collection<ObjEntity> loadedObjEntities = loadObjEntities(dataMap, config, entities); - if (entities != null) { - loadDbRelationships(dataMap, config, entities); - Collection<ObjEntity> loadedObjEntities = loadObjEntities(dataMap, config, entities); - loadObjEntities(dataMap, config, entities); - flattenManyToManyRelationships(dataMap); - fireObjEntitiesAddedEvents(dataMap); ++ flattenManyToManyRelationships(dataMap, loadedObjEntities); ++ fireObjEntitiesAddedEvents(loadedObjEntities); + } + } - flattenManyToManyRelationships(dataMap, loadedObjEntities); - fireObjEntitiesAddedEvents(loadedObjEntities); - } + /** + * Performs database reverse engineering to match the specified catalog, + * schema, table name and table type patterns and fills the specified + * DataMap object with DB and object mapping info. + * + * @since 3.2 + */ + public DataMap load(DbLoaderConfiguration config) throws SQLException { - loadProcedures(dataMap, config); - } + DataMap dataMap = new DataMap(); + load(dataMap, config, config.getTableTypes()); + loadProcedures(dataMap, config); + + return dataMap; + } /** * Loads database stored procedures into the DataMap. @@@ -759,7 -773,7 +783,7 @@@ * currently this method is NOT CALLED from "loadDataMapFromDB" and should * be invoked explicitly by the user. </i> * </p> -- * ++ * * @since 1.1 * @deprecated since 4.0 use loadProcedures(DataMap, String, String, String) that supports "catalog" pattern. */ @@@ -780,7 -794,7 +804,7 @@@ * currently this method is NOT CALLED from "loadDataMapFromDB" and should * be invoked explicitly by the user. </i> * </p> -- * ++ * * @since 4.0 */ public Map<String, Procedure> loadProcedures(DataMap dataMap, DbLoaderConfiguration config) @@@ -800,65 -814,75 +824,75 @@@ return procedures; } - private void loadProceduresColumns(Map<String, Procedure> procedures) throws SQLException { - ResultSet columnsRS = getMetaData().getProcedureColumns(null, null, null, null); // TODO catalog, schema - try { - while (columnsRS.next()) { - - String schema = columnsRS.getString("PROCEDURE_SCHEM"); - String name = columnsRS.getString("PROCEDURE_NAME"); - String key = (schema == null ? "" : schema + '.') + name ; - Procedure procedure = procedures.get(key); - if (procedure == null) { - continue; - } - - String columnName = columnsRS.getString("COLUMN_NAME"); + private void loadProceduresColumns(DbLoaderConfiguration config, Map<String, Procedure> procedures) throws SQLException { + for (DbPath dbPath : config.getFiltersConfig().pathsForQueries()) { + ResultSet columnsRS = getMetaData().getProcedureColumns(dbPath.catalog, dbPath.schema, null, null); + try { + while (columnsRS.next()) { - // skip ResultSet columns, as they are not described in Cayenne procedures yet... - short type = columnsRS.getShort("COLUMN_TYPE"); - if (type == DatabaseMetaData.procedureColumnResult) { - LOGGER.debug("skipping ResultSet column: " + key + "." + columnName); - } + String schema = columnsRS.getString("PROCEDURE_SCHEM"); + String name = columnsRS.getString("PROCEDURE_NAME"); - String key = (schema == null ? "" : schema + '.') + name ; ++ String key = (schema == null ? "" : schema + '.') + name; + Procedure procedure = procedures.get(key); + if (procedure == null) { + continue; + } - if (columnName == null) { - if (type == DatabaseMetaData.procedureColumnReturn) { - LOGGER.debug("null column name, assuming result column: " + key); - columnName = "_return_value"; - procedure.setReturningValue(true); - } else { - LOGGER.info("invalid null column name, skipping column : " + key); + ProcedureParameter column = loadProcedureParams(columnsRS, key, procedure); + if (column == null) { continue; } + procedure.addCallParameter(column); } + } finally { + columnsRS.close(); + } + } + } - int columnType = columnsRS.getInt("DATA_TYPE"); + private ProcedureParameter loadProcedureParams(ResultSet columnsRS, String key, Procedure procedure) throws SQLException { + String columnName = columnsRS.getString("COLUMN_NAME"); - // ignore precision of non-decimal columns - int decimalDigits = -1; - if (TypesMapping.isDecimal(columnType)) { - decimalDigits = columnsRS.getShort("SCALE"); - if (columnsRS.wasNull()) { - decimalDigits = -1; - } - } + // skip ResultSet columns, as they are not described in Cayenne procedures yet... + short type = columnsRS.getShort("COLUMN_TYPE"); + if (type == DatabaseMetaData.procedureColumnResult) { + LOGGER.debug("skipping ResultSet column: " + key + "." + columnName); + } - ProcedureParameter column = new ProcedureParameter(columnName); - int direction = getDirection(type); - if (direction != -1) { - column.setDirection(direction); - } + if (columnName == null) { + if (type == DatabaseMetaData.procedureColumnReturn) { + LOGGER.debug("null column name, assuming result column: " + key); + columnName = "_return_value"; + procedure.setReturningValue(true); + } else { + LOGGER.info("invalid null column name, skipping column : " + key); + return null; + } + } - column.setType(columnType); - column.setMaxLength(columnsRS.getInt("LENGTH")); - column.setPrecision(decimalDigits); + int columnType = columnsRS.getInt("DATA_TYPE"); - column.setProcedure(procedure); - procedure.addCallParameter(column); + // ignore precision of non-decimal columns + int decimalDigits = -1; + if (TypesMapping.isDecimal(columnType)) { + decimalDigits = columnsRS.getShort("SCALE"); + if (columnsRS.wasNull()) { + decimalDigits = -1; } - } finally { - columnsRS.close(); } + + ProcedureParameter column = new ProcedureParameter(columnName); + int direction = getDirection(type); + if (direction != -1) { + column.setDirection(direction); + } + + column.setType(columnType); + column.setMaxLength(columnsRS.getInt("LENGTH")); + column.setPrecision(decimalDigits); + + column.setProcedure(procedure); + return column; } private static int getDirection(short type) { @@@ -902,7 -926,7 +936,7 @@@ procedure.setSchema(rs.getString("PROCEDURE_SCHEM")); if (filters.filter(new DbPath(procedure.getCatalog(), procedure.getSchema())) -- .procedureFilter().isInclude(procedure)) { ++ .procedureFilter().isInclude(procedure)) { LOGGER.info("skipping Cayenne PK procedure: " + name); continue; } @@@ -927,7 -951,7 +961,7 @@@ /** * Sets new naming strategy for reverse engineering -- * ++ * * @since 3.0 */ public void setNameGenerator(ObjectNameGenerator strategy) { http://git-wip-us.apache.org/repos/asf/cayenne/blob/2a6ef007/cayenne-server/src/main/java/org/apache/cayenne/access/loader/ManyToManyCandidateEntity.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cayenne/blob/2a6ef007/modeler/cayenne-modeler/src/main/java/org/apache/cayenne/modeler/dialog/db/DbLoaderHelper.java ----------------------------------------------------------------------