http://git-wip-us.apache.org/repos/asf/hive/blob/12041d39/standalone-metastore/src/main/thrift/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift index 65e2f78..b816eb6 100644 --- a/standalone-metastore/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -183,6 +183,39 @@ enum EventRequestType { DELETE = 3, } +enum SerdeType { + HIVE = 1, + SCHEMA_REGISTRY = 2, +} + +enum SchemaType { + HIVE = 1, + AVRO = 2, +} + +enum SchemaCompatibility { + NONE = 1, + BACKWARD = 2, + FORWARD = 3, + BOTH = 4 +} + +enum SchemaValidation { + LATEST = 1, + ALL = 2 +} + +enum SchemaVersionState { + INITIATED = 1, + START_REVIEW = 2, + CHANGES_REQUIRED = 3, + REVIEWED = 4, + ENABLED = 5, + DISABLED = 6, + ARCHIVED = 7, + DELETED = 8 +} + struct HiveObjectRef{ 1: HiveObjectType objectType, 2: string dbName, @@ -289,7 +322,11 @@ struct Database { struct SerDeInfo { 1: string name, // name of the serde, table name by default 2: string serializationLib, // usually the class that implements the extractor & loader - 3: map<string, string> parameters // initialization parameters + 3: map<string, string> parameters, // initialization parameters + 4: optional string description, + 5: optional string serializerClass, + 6: optional string deserializerClass, + 7: optional SerdeType serdeType } // sort order of a column (column name along with asc(1)/desc(0)) @@ -1318,6 +1355,71 @@ struct WMCreateOrDropTriggerToPoolMappingRequest { struct WMCreateOrDropTriggerToPoolMappingResponse { } +// Schema objects +// Schema is already taken, so for the moment I'm calling it an ISchema for Independent Schema +struct ISchema { + 1: SchemaType schemaType, + 2: string name, + 3: string dbName, + 4: SchemaCompatibility compatibility, + 5: SchemaValidation validationLevel, + 6: bool canEvolve, + 7: optional string schemaGroup, + 8: optional string description +} + +struct ISchemaName { + 1: string dbName, + 2: string schemaName +} + +struct AlterISchemaRequest { + 1: ISchemaName name, + 3: ISchema newSchema +} + +struct SchemaVersion { + 1: ISchemaName schema, + 2: i32 version, + 3: i64 createdAt, + 4: list<FieldSchema> cols, + 5: optional SchemaVersionState state, + 6: optional string description, + 7: optional string schemaText, + 8: optional string fingerprint, + 9: optional string name, + 10: optional SerDeInfo serDe +} + +struct SchemaVersionDescriptor { + 1: ISchemaName schema, + 2: i32 version +} + +struct FindSchemasByColsRqst { + 1: optional string colName, + 2: optional string colNamespace, + 3: optional string type +} + +struct FindSchemasByColsResp { + 1: list<SchemaVersionDescriptor> schemaVersions +} + +struct MapSchemaVersionToSerdeRequest { + 1: SchemaVersionDescriptor schemaVersion, + 2: string serdeName +} + +struct SetSchemaVersionStateRequest { + 1: SchemaVersionDescriptor schemaVersion, + 2: SchemaVersionState state +} + +struct GetSerdeRequest { + 1: string serdeName +} + // Exceptions. exception MetaException { @@ -1929,6 +2031,38 @@ service ThriftHiveMetastore extends fb303.FacebookService WMCreateOrDropTriggerToPoolMappingResponse create_or_drop_wm_trigger_to_pool_mapping(1:WMCreateOrDropTriggerToPoolMappingRequest request) throws(1:AlreadyExistsException o1, 2:NoSuchObjectException o2, 3:InvalidObjectException o3, 4:MetaException o4) + + // Schema calls + void create_ischema(1:ISchema schema) throws(1:AlreadyExistsException o1, + NoSuchObjectException o2, 3:MetaException o3) + void alter_ischema(1:AlterISchemaRequest rqst) + throws(1:NoSuchObjectException o1, 2:MetaException o2) + ISchema get_ischema(1:ISchemaName name) throws (1:NoSuchObjectException o1, 2:MetaException o2) + void drop_ischema(1:ISchemaName name) + throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3) + + void add_schema_version(1:SchemaVersion schemaVersion) + throws(1:AlreadyExistsException o1, 2:NoSuchObjectException o2, 3:MetaException o3) + SchemaVersion get_schema_version(1: SchemaVersionDescriptor schemaVersion) + throws (1:NoSuchObjectException o1, 2:MetaException o2) + SchemaVersion get_schema_latest_version(1: ISchemaName schemaName) + throws (1:NoSuchObjectException o1, 2:MetaException o2) + list<SchemaVersion> get_schema_all_versions(1: ISchemaName schemaName) + throws (1:NoSuchObjectException o1, 2:MetaException o2) + void drop_schema_version(1: SchemaVersionDescriptor schemaVersion) + throws(1:NoSuchObjectException o1, 2:MetaException o2) + FindSchemasByColsResp get_schemas_by_cols(1: FindSchemasByColsRqst rqst) + throws(1:MetaException o1) + // There is no blanket update of SchemaVersion since it is (mostly) immutable. The only + // updates are the specific ones to associate a version with a serde and to change its state + void map_schema_version_to_serde(1: MapSchemaVersionToSerdeRequest rqst) + throws(1:NoSuchObjectException o1, 2:MetaException o2) + void set_schema_version_state(1: SetSchemaVersionStateRequest rqst) + throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3) + + void add_serde(1: SerDeInfo serde) throws(1:AlreadyExistsException o1, 2:MetaException o2) + SerDeInfo get_serde(1: GetSerdeRequest rqst) throws(1:NoSuchObjectException o1, 2:MetaException o2) + } // * Note about the DDL_TIME: When creating or altering a table or a partition,
http://git-wip-us.apache.org/repos/asf/hive/blob/12041d39/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 1c0eafd..5edf8b3 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.hive.metastore.api.CreationMetadata; +import org.apache.hadoop.hive.metastore.api.ISchemaName; +import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; import java.nio.ByteBuffer; @@ -38,6 +40,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; +import org.apache.hadoop.hive.metastore.api.ISchema; import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; @@ -67,6 +70,8 @@ import org.apache.hadoop.hive.metastore.api.SQLForeignKey; import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import org.apache.hadoop.hive.metastore.api.SchemaVersion; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableMeta; import org.apache.hadoop.hive.metastore.api.Type; @@ -1047,4 +1052,74 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { // TODO Auto-generated method stub return null; } + + public void createISchema(ISchema schema) throws AlreadyExistsException, MetaException, + NoSuchObjectException { + objectStore.createISchema(schema); + } + + @Override + public void alterISchema(ISchemaName schemaName, ISchema newSchema) throws NoSuchObjectException, + MetaException { + objectStore.alterISchema(schemaName, newSchema); + } + + @Override + public ISchema getISchema(ISchemaName schemaName) throws MetaException { + return objectStore.getISchema(schemaName); + } + + @Override + public void dropISchema(ISchemaName schemaName) throws NoSuchObjectException, MetaException { + objectStore.dropISchema(schemaName); + } + + @Override + public void addSchemaVersion(SchemaVersion schemaVersion) throws + AlreadyExistsException, InvalidObjectException, NoSuchObjectException, MetaException { + objectStore.addSchemaVersion(schemaVersion); + } + + @Override + public void alterSchemaVersion(SchemaVersionDescriptor version, SchemaVersion newVersion) throws + NoSuchObjectException, MetaException { + objectStore.alterSchemaVersion(version, newVersion); + } + + @Override + public SchemaVersion getSchemaVersion(SchemaVersionDescriptor version) throws MetaException { + return objectStore.getSchemaVersion(version); + } + + @Override + public SchemaVersion getLatestSchemaVersion(ISchemaName schemaName) throws MetaException { + return objectStore.getLatestSchemaVersion(schemaName); + } + + @Override + public List<SchemaVersion> getAllSchemaVersion(ISchemaName schemaName) throws MetaException { + return objectStore.getAllSchemaVersion(schemaName); + } + + @Override + public List<SchemaVersion> getSchemaVersionsByColumns(String colName, String colNamespace, + String type) throws MetaException { + return objectStore.getSchemaVersionsByColumns(colName, colNamespace, type); + } + + @Override + public void dropSchemaVersion(SchemaVersionDescriptor version) throws NoSuchObjectException, + MetaException { + objectStore.dropSchemaVersion(version); + } + + @Override + public SerDeInfo getSerDeInfo(String serDeName) throws NoSuchObjectException, MetaException { + return objectStore.getSerDeInfo(serDeName); + } + + @Override + public void addSerde(SerDeInfo serde) throws AlreadyExistsException, MetaException { + objectStore.addSerde(serde); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/12041d39/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index cbd8678..132cdc3 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.hive.metastore.api.CreationMetadata; +import org.apache.hadoop.hive.metastore.api.ISchemaName; +import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; import java.nio.ByteBuffer; @@ -36,6 +38,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; +import org.apache.hadoop.hive.metastore.api.ISchema; import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; @@ -65,6 +68,8 @@ import org.apache.hadoop.hive.metastore.api.SQLForeignKey; import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import org.apache.hadoop.hive.metastore.api.SchemaVersion; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableMeta; import org.apache.hadoop.hive.metastore.api.Type; @@ -1034,4 +1039,73 @@ public class DummyRawStoreForJdoConnection implements RawStore { // TODO Auto-generated method stub return null; } + + public void createISchema(ISchema schema) throws AlreadyExistsException, MetaException { + + } + + @Override + public void alterISchema(ISchemaName schemaName, ISchema newSchema) throws NoSuchObjectException, + MetaException { + + } + + @Override + public ISchema getISchema(ISchemaName schemaName) throws MetaException { + return null; + } + + @Override + public void dropISchema(ISchemaName schemaName) throws NoSuchObjectException, MetaException { + + } + + @Override + public void addSchemaVersion(SchemaVersion schemaVersion) throws + AlreadyExistsException, InvalidObjectException, NoSuchObjectException, MetaException { + + } + + @Override + public void alterSchemaVersion(SchemaVersionDescriptor version, SchemaVersion newVersion) throws + NoSuchObjectException, MetaException { + + } + + @Override + public SchemaVersion getSchemaVersion(SchemaVersionDescriptor version) throws MetaException { + return null; + } + + @Override + public SchemaVersion getLatestSchemaVersion(ISchemaName schemaName) throws MetaException { + return null; + } + + @Override + public List<SchemaVersion> getAllSchemaVersion(ISchemaName schemaName) throws MetaException { + return null; + } + + @Override + public List<SchemaVersion> getSchemaVersionsByColumns(String colName, String colNamespace, + String type) throws MetaException { + return null; + } + + @Override + public void dropSchemaVersion(SchemaVersionDescriptor version) throws NoSuchObjectException, + MetaException { + + } + + @Override + public SerDeInfo getSerDeInfo(String serDeName) throws MetaException { + return null; + } + + @Override + public void addSerde(SerDeInfo serde) throws AlreadyExistsException, MetaException { + + } } http://git-wip-us.apache.org/repos/asf/hive/blob/12041d39/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreSchemaMethods.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreSchemaMethods.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreSchemaMethods.java new file mode 100644 index 0000000..1b30090 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreSchemaMethods.java @@ -0,0 +1,1180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.FindSchemasByColsResp; +import org.apache.hadoop.hive.metastore.api.FindSchemasByColsRqst; +import org.apache.hadoop.hive.metastore.api.ISchema; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.SchemaCompatibility; +import org.apache.hadoop.hive.metastore.api.SchemaType; +import org.apache.hadoop.hive.metastore.api.SchemaValidation; +import org.apache.hadoop.hive.metastore.api.SchemaVersion; +import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; +import org.apache.hadoop.hive.metastore.api.SchemaVersionState; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.SerdeType; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.ISchemaBuilder; +import org.apache.hadoop.hive.metastore.client.builder.SchemaVersionBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.events.AddSchemaVersionEvent; +import org.apache.hadoop.hive.metastore.events.AlterISchemaEvent; +import org.apache.hadoop.hive.metastore.events.AlterSchemaVersionEvent; +import org.apache.hadoop.hive.metastore.events.CreateISchemaEvent; +import org.apache.hadoop.hive.metastore.events.DropISchemaEvent; +import org.apache.hadoop.hive.metastore.events.DropSchemaVersionEvent; +import org.apache.hadoop.hive.metastore.events.PreEventContext; +import org.apache.hadoop.hive.metastore.messaging.EventMessage; +import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.apache.thrift.TException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; + +// This does the testing using a remote metastore, as that finds more issues in thrift +@Category(MetastoreCheckinTest.class) +public class TestHiveMetaStoreSchemaMethods { + private static Map<EventMessage.EventType, Integer> events; + private static Map<EventMessage.EventType, Integer> transactionalEvents; + private static Map<PreEventContext.PreEventType, Integer> preEvents; + + private static IMetaStoreClient client; + + + @BeforeClass + public static void startMetastore() throws Exception { + Configuration conf = MetastoreConf.newMetastoreConf(); + MetaStoreTestUtils.setConfForStandloneMode(conf); + MetastoreConf.setClass(conf, ConfVars.EVENT_LISTENERS, SchemaEventListener.class, + MetaStoreEventListener.class); + MetastoreConf.setClass(conf, ConfVars.TRANSACTIONAL_EVENT_LISTENERS, + TransactionalSchemaEventListener.class, MetaStoreEventListener.class); + MetastoreConf.setClass(conf, ConfVars.PRE_EVENT_LISTENERS, SchemaPreEventListener.class, + MetaStorePreEventListener.class); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), + conf); + MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); + client = new HiveMetaStoreClient(conf); + } + + @Before + public void newMaps() { + events = new HashMap<>(); + transactionalEvents = new HashMap<>(); + preEvents = new HashMap<>(); + } + + @Test(expected = NoSuchObjectException.class) + public void getNonExistentSchema() throws TException { + client.getISchema(DEFAULT_DATABASE_NAME, "no.such.schema"); + } + + @Test + public void iSchema() throws TException { + String schemaName = uniqueSchemaName(); + String schemaGroup = "group1"; + String description = "This is a description"; + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .setCompatibility(SchemaCompatibility.FORWARD) + .setValidationLevel(SchemaValidation.LATEST) + .setCanEvolve(false) + .setSchemaGroup(schemaGroup) + .setDescription(description) + .build(); + client.createISchema(schema); + + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.CREATE_ISCHEMA)); + Assert.assertEquals(1, (int)events.get(EventMessage.EventType.CREATE_ISCHEMA)); + Assert.assertEquals(1, (int)transactionalEvents.get(EventMessage.EventType.CREATE_ISCHEMA)); + + schema = client.getISchema(DEFAULT_DATABASE_NAME, schemaName); + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.READ_ISCHEMA)); + + Assert.assertEquals(SchemaType.AVRO, schema.getSchemaType()); + Assert.assertEquals(schemaName, schema.getName()); + Assert.assertEquals(SchemaCompatibility.FORWARD, schema.getCompatibility()); + Assert.assertEquals(SchemaValidation.LATEST, schema.getValidationLevel()); + Assert.assertFalse(schema.isCanEvolve()); + Assert.assertEquals(schemaGroup, schema.getSchemaGroup()); + Assert.assertEquals(description, schema.getDescription()); + + schemaGroup = "new group"; + description = "new description"; + schema.setCompatibility(SchemaCompatibility.BOTH); + schema.setValidationLevel(SchemaValidation.ALL); + schema.setCanEvolve(true); + schema.setSchemaGroup(schemaGroup); + schema.setDescription(description); + client.alterISchema(DEFAULT_DATABASE_NAME, schemaName, schema); + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.ALTER_ISCHEMA)); + Assert.assertEquals(1, (int)events.get(EventMessage.EventType.ALTER_ISCHEMA)); + Assert.assertEquals(1, (int)transactionalEvents.get(EventMessage.EventType.ALTER_ISCHEMA)); + + schema = client.getISchema(DEFAULT_DATABASE_NAME, schemaName); + Assert.assertEquals(2, (int)preEvents.get(PreEventContext.PreEventType.READ_ISCHEMA)); + + Assert.assertEquals(SchemaType.AVRO, schema.getSchemaType()); + Assert.assertEquals(schemaName, schema.getName()); + Assert.assertEquals(SchemaCompatibility.BOTH, schema.getCompatibility()); + Assert.assertEquals(SchemaValidation.ALL, schema.getValidationLevel()); + Assert.assertTrue(schema.isCanEvolve()); + Assert.assertEquals(schemaGroup, schema.getSchemaGroup()); + Assert.assertEquals(description, schema.getDescription()); + + client.dropISchema(DEFAULT_DATABASE_NAME, schemaName); + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.DROP_ISCHEMA)); + Assert.assertEquals(1, (int)events.get(EventMessage.EventType.DROP_ISCHEMA)); + Assert.assertEquals(1, (int)transactionalEvents.get(EventMessage.EventType.DROP_ISCHEMA)); + try { + client.getISchema(DEFAULT_DATABASE_NAME, schemaName); + Assert.fail(); + } catch (NoSuchObjectException e) { + // all good + } + } + + @Test + public void iSchemaOtherDatabase() throws TException { + String dbName = "other_db"; + Database db = new DatabaseBuilder() + .setName(dbName) + .build(); + client.createDatabase(db); + + String schemaName = uniqueSchemaName(); + String schemaGroup = "group1"; + String description = "This is a description"; + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .setDbName(dbName) + .setCompatibility(SchemaCompatibility.FORWARD) + .setValidationLevel(SchemaValidation.LATEST) + .setCanEvolve(false) + .setSchemaGroup(schemaGroup) + .setDescription(description) + .build(); + client.createISchema(schema); + + schema = client.getISchema(dbName, schemaName); + + Assert.assertEquals(SchemaType.AVRO, schema.getSchemaType()); + Assert.assertEquals(schemaName, schema.getName()); + Assert.assertEquals(dbName, schema.getDbName()); + Assert.assertEquals(SchemaCompatibility.FORWARD, schema.getCompatibility()); + Assert.assertEquals(SchemaValidation.LATEST, schema.getValidationLevel()); + Assert.assertFalse(schema.isCanEvolve()); + Assert.assertEquals(schemaGroup, schema.getSchemaGroup()); + Assert.assertEquals(description, schema.getDescription()); + + schemaGroup = "new group"; + description = "new description"; + schema.setCompatibility(SchemaCompatibility.BOTH); + schema.setValidationLevel(SchemaValidation.ALL); + schema.setCanEvolve(true); + schema.setSchemaGroup(schemaGroup); + schema.setDescription(description); + client.alterISchema(dbName, schemaName, schema); + + schema = client.getISchema(dbName, schemaName); + + Assert.assertEquals(SchemaType.AVRO, schema.getSchemaType()); + Assert.assertEquals(schemaName, schema.getName()); + Assert.assertEquals(dbName, schema.getDbName()); + Assert.assertEquals(SchemaCompatibility.BOTH, schema.getCompatibility()); + Assert.assertEquals(SchemaValidation.ALL, schema.getValidationLevel()); + Assert.assertTrue(schema.isCanEvolve()); + Assert.assertEquals(schemaGroup, schema.getSchemaGroup()); + Assert.assertEquals(description, schema.getDescription()); + + client.dropISchema(dbName, schemaName); + try { + client.getISchema(dbName, schemaName); + Assert.fail(); + } catch (NoSuchObjectException e) { + // all good + } + } + + @Test(expected = NoSuchObjectException.class) + public void schemaWithInvalidDatabase() throws TException { + ISchema schema = new ISchemaBuilder() + .setName("thisSchemaDoesntHaveADb") + .setDbName("no.such.database") + .setSchemaType(SchemaType.AVRO) + .build(); + client.createISchema(schema); + } + + @Test(expected = AlreadyExistsException.class) + public void schemaAlreadyExists() throws TException { + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.HIVE) + .setName(schemaName) + .build(); + client.createISchema(schema); + + schema = client.getISchema(DEFAULT_DATABASE_NAME, schemaName); + Assert.assertNotNull(schema); + + Assert.assertEquals(SchemaType.HIVE, schema.getSchemaType()); + Assert.assertEquals(schemaName, schema.getName()); + Assert.assertEquals(SchemaCompatibility.BACKWARD, schema.getCompatibility()); + Assert.assertEquals(SchemaValidation.ALL, schema.getValidationLevel()); + Assert.assertTrue(schema.isCanEvolve()); + + // This second attempt to create it should throw + client.createISchema(schema); + } + + @Test(expected = NoSuchObjectException.class) + public void alterNonExistentSchema() throws TException { + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.HIVE) + .setName(schemaName) + .setDescription("a new description") + .build(); + client.alterISchema(DEFAULT_DATABASE_NAME, schemaName, schema); + } + + @Test(expected = NoSuchObjectException.class) + public void dropNonExistentSchema() throws TException { + client.dropISchema(DEFAULT_DATABASE_NAME, "no_such_schema"); + } + + @Test(expected = NoSuchObjectException.class) + public void createVersionOfNonExistentSchema() throws TException { + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .setSchemaName("noSchemaOfThisNameExists") + .setDbName(DEFAULT_DATABASE_NAME) + .setVersion(1) + .addCol("a", ColumnType.STRING_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + } + + @Test + public void addSchemaVersion() throws TException { + String schemaName = uniqueSchemaName(); + int version = 1; + + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + + String description = "very descriptive"; + String schemaText = "this should look like json, but oh well"; + String fingerprint = "this should be an md5 string"; + String versionName = "why would I name a version?"; + long creationTime = 10; + String serdeName = "serde_for_schema37"; + String serializer = "org.apache.hadoop.hive.metastore.test.Serializer"; + String deserializer = "org.apache.hadoop.hive.metastore.test.Deserializer"; + String serdeDescription = "how do you describe a serde?"; + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(version) + .addCol("a", ColumnType.INT_TYPE_NAME) + .addCol("b", ColumnType.FLOAT_TYPE_NAME) + .setCreatedAt(creationTime) + .setState(SchemaVersionState.INITIATED) + .setDescription(description) + .setSchemaText(schemaText) + .setFingerprint(fingerprint) + .setName(versionName) + .setSerdeName(serdeName) + .setSerdeSerializerClass(serializer) + .setSerdeDeserializerClass(deserializer) + .setSerdeDescription(serdeDescription) + .build(); + client.addSchemaVersion(schemaVersion); + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.ADD_SCHEMA_VERSION)); + Assert.assertEquals(1, (int)events.get(EventMessage.EventType.ADD_SCHEMA_VERSION)); + Assert.assertEquals(1, (int)transactionalEvents.get(EventMessage.EventType.ADD_SCHEMA_VERSION)); + + schemaVersion = client.getSchemaVersion(DEFAULT_DATABASE_NAME, schemaName, version); + Assert.assertNotNull(schemaVersion); + Assert.assertEquals(schemaName, schemaVersion.getSchema().getSchemaName()); + Assert.assertEquals(DEFAULT_DATABASE_NAME, schemaVersion.getSchema().getDbName()); + Assert.assertEquals(version, schemaVersion.getVersion()); + Assert.assertEquals(creationTime, schemaVersion.getCreatedAt()); + Assert.assertEquals(SchemaVersionState.INITIATED, schemaVersion.getState()); + Assert.assertEquals(description, schemaVersion.getDescription()); + Assert.assertEquals(schemaText, schemaVersion.getSchemaText()); + Assert.assertEquals(fingerprint, schemaVersion.getFingerprint()); + Assert.assertEquals(versionName, schemaVersion.getName()); + Assert.assertEquals(serdeName, schemaVersion.getSerDe().getName()); + Assert.assertEquals(serializer, schemaVersion.getSerDe().getSerializerClass()); + Assert.assertEquals(deserializer, schemaVersion.getSerDe().getDeserializerClass()); + Assert.assertEquals(serdeDescription, schemaVersion.getSerDe().getDescription()); + Assert.assertEquals(2, schemaVersion.getColsSize()); + List<FieldSchema> cols = schemaVersion.getCols(); + Collections.sort(cols); + Assert.assertEquals("a", cols.get(0).getName()); + Assert.assertEquals(ColumnType.INT_TYPE_NAME, cols.get(0).getType()); + Assert.assertEquals("b", cols.get(1).getName()); + Assert.assertEquals(ColumnType.FLOAT_TYPE_NAME, cols.get(1).getType()); + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.READ_SCHEMA_VERSION)); + + client.dropSchemaVersion(DEFAULT_DATABASE_NAME, schemaName, version); + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.DROP_SCHEMA_VERSION)); + Assert.assertEquals(1, (int)events.get(EventMessage.EventType.DROP_SCHEMA_VERSION)); + Assert.assertEquals(1, (int)transactionalEvents.get(EventMessage.EventType.DROP_SCHEMA_VERSION)); + try { + client.getSchemaVersion(DEFAULT_DATABASE_NAME, schemaName, version); + Assert.fail(); + } catch (NoSuchObjectException e) { + // all good + } + } + + @Test + public void addSchemaVersionOtherDb() throws TException { + String dbName = "other_db_for_schema_version"; + Database db = new DatabaseBuilder() + .setName(dbName) + .build(); + client.createDatabase(db); + + String schemaName = uniqueSchemaName(); + int version = 1; + + ISchema schema = new ISchemaBuilder() + .setDbName(dbName) + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + + String description = "very descriptive"; + String schemaText = "this should look like json, but oh well"; + String fingerprint = "this should be an md5 string"; + String versionName = "why would I name a version?"; + long creationTime = 10; + String serdeName = "serde_for_schema37"; + String serializer = "org.apache.hadoop.hive.metastore.test.Serializer"; + String deserializer = "org.apache.hadoop.hive.metastore.test.Deserializer"; + String serdeDescription = "how do you describe a serde?"; + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(version) + .addCol("a", ColumnType.INT_TYPE_NAME) + .addCol("b", ColumnType.FLOAT_TYPE_NAME) + .setCreatedAt(creationTime) + .setState(SchemaVersionState.INITIATED) + .setDescription(description) + .setSchemaText(schemaText) + .setFingerprint(fingerprint) + .setName(versionName) + .setSerdeName(serdeName) + .setSerdeSerializerClass(serializer) + .setSerdeDeserializerClass(deserializer) + .setSerdeDescription(serdeDescription) + .build(); + client.addSchemaVersion(schemaVersion); + + schemaVersion = client.getSchemaVersion(dbName, schemaName, version); + Assert.assertNotNull(schemaVersion); + Assert.assertEquals(schemaName, schemaVersion.getSchema().getSchemaName()); + Assert.assertEquals(dbName, schemaVersion.getSchema().getDbName()); + Assert.assertEquals(version, schemaVersion.getVersion()); + Assert.assertEquals(creationTime, schemaVersion.getCreatedAt()); + Assert.assertEquals(SchemaVersionState.INITIATED, schemaVersion.getState()); + Assert.assertEquals(description, schemaVersion.getDescription()); + Assert.assertEquals(schemaText, schemaVersion.getSchemaText()); + Assert.assertEquals(fingerprint, schemaVersion.getFingerprint()); + Assert.assertEquals(versionName, schemaVersion.getName()); + Assert.assertEquals(serdeName, schemaVersion.getSerDe().getName()); + Assert.assertEquals(serializer, schemaVersion.getSerDe().getSerializerClass()); + Assert.assertEquals(deserializer, schemaVersion.getSerDe().getDeserializerClass()); + Assert.assertEquals(serdeDescription, schemaVersion.getSerDe().getDescription()); + Assert.assertEquals(2, schemaVersion.getColsSize()); + List<FieldSchema> cols = schemaVersion.getCols(); + Collections.sort(cols); + Assert.assertEquals("a", cols.get(0).getName()); + Assert.assertEquals(ColumnType.INT_TYPE_NAME, cols.get(0).getType()); + Assert.assertEquals("b", cols.get(1).getName()); + Assert.assertEquals(ColumnType.FLOAT_TYPE_NAME, cols.get(1).getType()); + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.READ_SCHEMA_VERSION)); + + client.dropSchemaVersion(dbName, schemaName, version); + try { + client.getSchemaVersion(dbName, schemaName, version); + Assert.fail(); + } catch (NoSuchObjectException e) { + // all good + } + } + + // Test that adding multiple versions of the same schema + @Test + public void multipleSchemaVersions() throws TException { + String schemaName = uniqueSchemaName(); + + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(1) + .addCol("a", ColumnType.BIGINT_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + + schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(2) + .addCol("a", ColumnType.BIGINT_TYPE_NAME) + .addCol("b", ColumnType.DATE_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + + schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(3) + .addCol("a", ColumnType.BIGINT_TYPE_NAME) + .addCol("b", ColumnType.DATE_TYPE_NAME) + .addCol("c", ColumnType.TIMESTAMP_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + Assert.assertEquals(3, (int)preEvents.get(PreEventContext.PreEventType.ADD_SCHEMA_VERSION)); + Assert.assertEquals(3, (int)events.get(EventMessage.EventType.ADD_SCHEMA_VERSION)); + Assert.assertEquals(3, (int)transactionalEvents.get(EventMessage.EventType.ADD_SCHEMA_VERSION)); + + schemaVersion = client.getSchemaLatestVersion(DEFAULT_DATABASE_NAME, schemaName); + Assert.assertEquals(3, schemaVersion.getVersion()); + Assert.assertEquals(3, schemaVersion.getColsSize()); + List<FieldSchema> cols = schemaVersion.getCols(); + Collections.sort(cols); + Assert.assertEquals("a", cols.get(0).getName()); + Assert.assertEquals("b", cols.get(1).getName()); + Assert.assertEquals("c", cols.get(2).getName()); + Assert.assertEquals(ColumnType.BIGINT_TYPE_NAME, cols.get(0).getType()); + Assert.assertEquals(ColumnType.DATE_TYPE_NAME, cols.get(1).getType()); + Assert.assertEquals(ColumnType.TIMESTAMP_TYPE_NAME, cols.get(2).getType()); + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.READ_SCHEMA_VERSION)); + + List<SchemaVersion> versions = client.getSchemaAllVersions(DEFAULT_DATABASE_NAME, schemaName); + Assert.assertEquals(2, (int)preEvents.get(PreEventContext.PreEventType.READ_SCHEMA_VERSION)); + Assert.assertEquals(3, versions.size()); + versions.sort(Comparator.comparingInt(SchemaVersion::getVersion)); + Assert.assertEquals(1, versions.get(0).getVersion()); + Assert.assertEquals(1, versions.get(0).getColsSize()); + Assert.assertEquals(ColumnType.BIGINT_TYPE_NAME, versions.get(0).getCols().get(0).getType()); + + Assert.assertEquals(2, versions.get(1).getVersion()); + Assert.assertEquals(2, versions.get(1).getColsSize()); + cols = versions.get(1).getCols(); + Collections.sort(cols); + Assert.assertEquals("a", cols.get(0).getName()); + Assert.assertEquals("b", cols.get(1).getName()); + Assert.assertEquals(ColumnType.BIGINT_TYPE_NAME, cols.get(0).getType()); + Assert.assertEquals(ColumnType.DATE_TYPE_NAME, cols.get(1).getType()); + + Assert.assertEquals(3, versions.get(2).getVersion()); + Assert.assertEquals(3, versions.get(2).getColsSize()); + cols = versions.get(2).getCols(); + Collections.sort(cols); + Assert.assertEquals("a", cols.get(0).getName()); + Assert.assertEquals("b", cols.get(1).getName()); + Assert.assertEquals("c", cols.get(2).getName()); + Assert.assertEquals(ColumnType.BIGINT_TYPE_NAME, cols.get(0).getType()); + Assert.assertEquals(ColumnType.DATE_TYPE_NAME, cols.get(1).getType()); + Assert.assertEquals(ColumnType.TIMESTAMP_TYPE_NAME, cols.get(2).getType()); + } + + @Test(expected = NoSuchObjectException.class) + public void nonExistentSchemaVersion() throws TException { + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + client.getSchemaVersion(DEFAULT_DATABASE_NAME, schemaName, 1); + } + + @Test(expected = NoSuchObjectException.class) + public void schemaVersionBogusDb() throws TException { + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + client.getSchemaVersion("bogus", schemaName, 1); + } + + @Test(expected = NoSuchObjectException.class) + public void nonExistentSchemaVersionButOtherVersionsExist() throws TException { + String schemaName = uniqueSchemaName(); + + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(1) + .addCol("a", ColumnType.INT_TYPE_NAME) + .addCol("b", ColumnType.FLOAT_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + + client.getSchemaVersion(DEFAULT_DATABASE_NAME, schemaName, 2); + } + + @Test(expected = NoSuchObjectException.class) + public void getLatestSchemaButNoVersions() throws TException { + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + client.getSchemaLatestVersion(DEFAULT_DATABASE_NAME, schemaName); + } + + @Test(expected = NoSuchObjectException.class) + public void getLatestSchemaNoSuchSchema() throws TException { + client.getSchemaLatestVersion(DEFAULT_DATABASE_NAME, "no.such.schema.with.this.name"); + } + + @Test(expected = NoSuchObjectException.class) + public void latestSchemaVersionBogusDb() throws TException { + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + client.getSchemaLatestVersion("bogus", schemaName); + } + + @Test(expected = NoSuchObjectException.class) + public void getAllSchemaButNoVersions() throws TException { + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + client.getSchemaAllVersions(DEFAULT_DATABASE_NAME, schemaName); + } + + @Test(expected = NoSuchObjectException.class) + public void getAllSchemaNoSuchSchema() throws TException { + client.getSchemaAllVersions(DEFAULT_DATABASE_NAME, "no.such.schema.with.this.name"); + } + + @Test(expected = NoSuchObjectException.class) + public void allSchemaVersionBogusDb() throws TException { + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + client.getSchemaAllVersions("bogus", schemaName); + } + + @Test(expected = AlreadyExistsException.class) + public void addDuplicateSchemaVersion() throws TException { + String schemaName = uniqueSchemaName(); + int version = 1; + + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(version) + .addCol("a", ColumnType.INT_TYPE_NAME) + .addCol("b", ColumnType.FLOAT_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + + client.addSchemaVersion(schemaVersion); + } + + @Test(expected = NoSuchObjectException.class) + public void mapSerDeNoSuchSchema() throws TException { + SerDeInfo serDeInfo = new SerDeInfo(uniqueSerdeName(), "lib", Collections.emptyMap()); + client.mapSchemaVersionToSerde(DEFAULT_DATABASE_NAME, uniqueSchemaName(), 1, serDeInfo.getName()); + } + + @Test(expected = NoSuchObjectException.class) + public void mapSerDeNoSuchSchemaVersion() throws TException { + SerDeInfo serDeInfo = new SerDeInfo(uniqueSerdeName(), "lib", Collections.emptyMap()); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(uniqueSchemaName()) + .build(); + client.createISchema(schema); + client.mapSchemaVersionToSerde(DEFAULT_DATABASE_NAME, schema.getName(), 3, serDeInfo.getName()); + } + + @Test(expected = NoSuchObjectException.class) + public void mapNonExistentSerdeToSchemaVersion() throws TException { + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(uniqueSchemaName()) + .build(); + client.createISchema(schema); + + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(1) + .addCol("x", ColumnType.BOOLEAN_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + client.mapSchemaVersionToSerde(DEFAULT_DATABASE_NAME, schema.getName(), schemaVersion.getVersion(), uniqueSerdeName()); + } + + @Test + public void mapSerdeToSchemaVersion() throws TException { + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(uniqueSchemaName()) + .build(); + client.createISchema(schema); + + // Create schema with no serde, then map it + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(1) + .addCol("x", ColumnType.BOOLEAN_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + + SerDeInfo serDeInfo = new SerDeInfo(uniqueSerdeName(), "lib", Collections.emptyMap()); + client.addSerDe(serDeInfo); + + client.mapSchemaVersionToSerde(DEFAULT_DATABASE_NAME, schema.getName(), schemaVersion.getVersion(), serDeInfo.getName()); + schemaVersion = client.getSchemaVersion(DEFAULT_DATABASE_NAME, schema.getName(), schemaVersion.getVersion()); + Assert.assertEquals(serDeInfo.getName(), schemaVersion.getSerDe().getName()); + + // Create schema with a serde, then remap it + String serDeName = uniqueSerdeName(); + schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(2) + .addCol("x", ColumnType.BOOLEAN_TYPE_NAME) + .setSerdeName(serDeName) + .setSerdeLib("x") + .build(); + client.addSchemaVersion(schemaVersion); + + schemaVersion = client.getSchemaVersion(DEFAULT_DATABASE_NAME, schema.getName(), 2); + Assert.assertEquals(serDeName, schemaVersion.getSerDe().getName()); + + serDeInfo = new SerDeInfo(uniqueSerdeName(), "y", Collections.emptyMap()); + client.addSerDe(serDeInfo); + client.mapSchemaVersionToSerde(DEFAULT_DATABASE_NAME, schema.getName(), 2, serDeInfo.getName()); + schemaVersion = client.getSchemaVersion(DEFAULT_DATABASE_NAME, schema.getName(), 2); + Assert.assertEquals(serDeInfo.getName(), schemaVersion.getSerDe().getName()); + } + + @Test + public void mapSerdeToSchemaVersionOtherDb() throws TException { + String dbName = "map_other_db"; + Database db = new DatabaseBuilder() + .setName(dbName) + .build(); + client.createDatabase(db); + + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setDbName(dbName) + .setName(uniqueSchemaName()) + .build(); + client.createISchema(schema); + + // Create schema with no serde, then map it + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(1) + .addCol("x", ColumnType.BOOLEAN_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + + SerDeInfo serDeInfo = new SerDeInfo(uniqueSerdeName(), "lib", Collections.emptyMap()); + client.addSerDe(serDeInfo); + + client.mapSchemaVersionToSerde(dbName, schema.getName(), schemaVersion.getVersion(), serDeInfo.getName()); + schemaVersion = client.getSchemaVersion(dbName, schema.getName(), schemaVersion.getVersion()); + Assert.assertEquals(serDeInfo.getName(), schemaVersion.getSerDe().getName()); + + // Create schema with a serde, then remap it + String serDeName = uniqueSerdeName(); + schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(2) + .addCol("x", ColumnType.BOOLEAN_TYPE_NAME) + .setSerdeName(serDeName) + .setSerdeLib("x") + .build(); + client.addSchemaVersion(schemaVersion); + + schemaVersion = client.getSchemaVersion(dbName, schema.getName(), 2); + Assert.assertEquals(serDeName, schemaVersion.getSerDe().getName()); + + serDeInfo = new SerDeInfo(uniqueSerdeName(), "y", Collections.emptyMap()); + client.addSerDe(serDeInfo); + client.mapSchemaVersionToSerde(dbName, schema.getName(), 2, serDeInfo.getName()); + schemaVersion = client.getSchemaVersion(dbName, schema.getName(), 2); + Assert.assertEquals(serDeInfo.getName(), schemaVersion.getSerDe().getName()); + + } + + @Test + public void addSerde() throws TException { + String serdeName = uniqueSerdeName(); + SerDeInfo serDeInfo = new SerDeInfo(serdeName, "serdeLib", Collections.singletonMap("a", "b")); + serDeInfo.setSerializerClass("serializer"); + serDeInfo.setDeserializerClass("deserializer"); + serDeInfo.setDescription("description"); + serDeInfo.setSerdeType(SerdeType.SCHEMA_REGISTRY); + client.addSerDe(serDeInfo); + + serDeInfo = client.getSerDe(serdeName); + Assert.assertEquals(serdeName, serDeInfo.getName()); + Assert.assertEquals("serdeLib", serDeInfo.getSerializationLib()); + Assert.assertEquals(1, serDeInfo.getParametersSize()); + Assert.assertEquals("b", serDeInfo.getParameters().get("a")); + Assert.assertEquals("serializer", serDeInfo.getSerializerClass()); + Assert.assertEquals("deserializer", serDeInfo.getDeserializerClass()); + Assert.assertEquals("description", serDeInfo.getDescription()); + Assert.assertEquals(SerdeType.SCHEMA_REGISTRY, serDeInfo.getSerdeType()); + } + + @Test(expected = AlreadyExistsException.class) + public void duplicateSerde() throws TException { + String serdeName = uniqueSerdeName(); + SerDeInfo serDeInfo = new SerDeInfo(serdeName, "x", Collections.emptyMap()); + client.addSerDe(serDeInfo); + client.addSerDe(serDeInfo); + } + + @Test(expected = NoSuchObjectException.class) + public void noSuchSerDe() throws TException { + client.getSerDe(uniqueSerdeName()); + } + + @Test(expected = NoSuchObjectException.class) + public void setVersionStateNoSuchSchema() throws TException { + client.setSchemaVersionState(DEFAULT_DATABASE_NAME, "no.such.schema", 1, SchemaVersionState.INITIATED); + } + + @Test(expected = NoSuchObjectException.class) + public void setVersionStateNoSuchVersion() throws TException { + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + client.setSchemaVersionState(DEFAULT_DATABASE_NAME, schemaName, 1, SchemaVersionState.INITIATED); + } + + @Test + public void setVersionState() throws TException { + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .build(); + client.createISchema(schema); + + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(1) + .addCol("a", ColumnType.BINARY_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + + schemaVersion = client.getSchemaVersion(DEFAULT_DATABASE_NAME, schemaName, 1); + Assert.assertNull(schemaVersion.getState()); + + client.setSchemaVersionState(DEFAULT_DATABASE_NAME, schemaName, 1, SchemaVersionState.INITIATED); + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.ALTER_SCHEMA_VERSION)); + Assert.assertEquals(1, (int)events.get(EventMessage.EventType.ALTER_SCHEMA_VERSION)); + Assert.assertEquals(1, (int)transactionalEvents.get(EventMessage.EventType.ALTER_SCHEMA_VERSION)); + schemaVersion = client.getSchemaVersion(DEFAULT_DATABASE_NAME, schemaName, 1); + Assert.assertEquals(SchemaVersionState.INITIATED, schemaVersion.getState()); + + client.setSchemaVersionState(DEFAULT_DATABASE_NAME, schemaName, 1, SchemaVersionState.REVIEWED); + Assert.assertEquals(2, (int)preEvents.get(PreEventContext.PreEventType.ALTER_SCHEMA_VERSION)); + Assert.assertEquals(2, (int)events.get(EventMessage.EventType.ALTER_SCHEMA_VERSION)); + Assert.assertEquals(2, (int)transactionalEvents.get(EventMessage.EventType.ALTER_SCHEMA_VERSION)); + schemaVersion = client.getSchemaVersion(DEFAULT_DATABASE_NAME, schemaName, 1); + Assert.assertEquals(SchemaVersionState.REVIEWED, schemaVersion.getState()); + } + + @Test + public void setVersionStateOtherDb() throws TException { + String dbName = "other_db_set_state"; + Database db = new DatabaseBuilder() + .setName(dbName) + .build(); + client.createDatabase(db); + + String schemaName = uniqueSchemaName(); + ISchema schema = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setName(schemaName) + .setDbName(dbName) + .build(); + client.createISchema(schema); + + SchemaVersion schemaVersion = new SchemaVersionBuilder() + .versionOf(schema) + .setVersion(1) + .addCol("a", ColumnType.BINARY_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion); + + schemaVersion = client.getSchemaVersion(dbName, schemaName, 1); + Assert.assertNull(schemaVersion.getState()); + + client.setSchemaVersionState(dbName, schemaName, 1, SchemaVersionState.INITIATED); + Assert.assertEquals(1, (int)preEvents.get(PreEventContext.PreEventType.ALTER_SCHEMA_VERSION)); + Assert.assertEquals(1, (int)events.get(EventMessage.EventType.ALTER_SCHEMA_VERSION)); + Assert.assertEquals(1, (int)transactionalEvents.get(EventMessage.EventType.ALTER_SCHEMA_VERSION)); + schemaVersion = client.getSchemaVersion(dbName, schemaName, 1); + Assert.assertEquals(SchemaVersionState.INITIATED, schemaVersion.getState()); + + client.setSchemaVersionState(dbName, schemaName, 1, SchemaVersionState.REVIEWED); + Assert.assertEquals(2, (int)preEvents.get(PreEventContext.PreEventType.ALTER_SCHEMA_VERSION)); + Assert.assertEquals(2, (int)events.get(EventMessage.EventType.ALTER_SCHEMA_VERSION)); + Assert.assertEquals(2, (int)transactionalEvents.get(EventMessage.EventType.ALTER_SCHEMA_VERSION)); + schemaVersion = client.getSchemaVersion(dbName, schemaName, 1); + Assert.assertEquals(SchemaVersionState.REVIEWED, schemaVersion.getState()); + } + + @Test(expected = NoSuchObjectException.class) + public void dropNonExistentSchemaVersion() throws TException { + client.dropSchemaVersion(DEFAULT_DATABASE_NAME, "ther is no schema named this", 23); + } + + @Test + public void schemaQuery() throws TException { + String dbName = "schema_query_db"; + Database db = new DatabaseBuilder() + .setName(dbName) + .build(); + client.createDatabase(db); + + String schemaName1 = uniqueSchemaName(); + ISchema schema1 = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setDbName(dbName) + .setName(schemaName1) + .build(); + client.createISchema(schema1); + + String schemaName2 = uniqueSchemaName(); + ISchema schema2 = new ISchemaBuilder() + .setSchemaType(SchemaType.AVRO) + .setDbName(dbName) + .setName(schemaName2) + .build(); + client.createISchema(schema2); + + SchemaVersion schemaVersion1_1 = new SchemaVersionBuilder() + .versionOf(schema1) + .setVersion(1) + .addCol("alpha", ColumnType.BIGINT_TYPE_NAME) + .addCol("beta", ColumnType.DATE_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion1_1); + + SchemaVersion schemaVersion1_2 = new SchemaVersionBuilder() + .versionOf(schema1) + .setVersion(2) + .addCol("alpha", ColumnType.BIGINT_TYPE_NAME) + .addCol("beta", ColumnType.DATE_TYPE_NAME) + .addCol("gamma", ColumnType.BIGINT_TYPE_NAME, "namespace=x") + .build(); + client.addSchemaVersion(schemaVersion1_2); + + SchemaVersion schemaVersion2_1 = new SchemaVersionBuilder() + .versionOf(schema2) + .setVersion(1) + .addCol("ALPHA", ColumnType.SMALLINT_TYPE_NAME) + .addCol("delta", ColumnType.DOUBLE_TYPE_NAME) + .build(); + client.addSchemaVersion(schemaVersion2_1); + + SchemaVersion schemaVersion2_2 = new SchemaVersionBuilder() + .versionOf(schema2) + .setVersion(2) + .addCol("ALPHA", ColumnType.SMALLINT_TYPE_NAME) + .addCol("delta", ColumnType.DOUBLE_TYPE_NAME) + .addCol("epsilon", ColumnType.STRING_TYPE_NAME, "namespace=x") + .build(); + client.addSchemaVersion(schemaVersion2_2); + + // Query that should return nothing + FindSchemasByColsRqst rqst = new FindSchemasByColsRqst(); + rqst.setColName("x"); + rqst.setColNamespace("y"); + rqst.setType("z"); + FindSchemasByColsResp rsp = client.getSchemaByCols(rqst); + Assert.assertEquals(0, rsp.getSchemaVersionsSize()); + + // Query that should fetch one column + rqst = new FindSchemasByColsRqst(); + rqst.setColName("gamma"); + rsp = client.getSchemaByCols(rqst); + Assert.assertEquals(1, rsp.getSchemaVersionsSize()); + Assert.assertEquals(schemaName1, rsp.getSchemaVersions().get(0).getSchema().getSchemaName()); + Assert.assertEquals(dbName, rsp.getSchemaVersions().get(0).getSchema().getDbName()); + Assert.assertEquals(2, rsp.getSchemaVersions().get(0).getVersion()); + + // fetch 2 in same schema + rqst = new FindSchemasByColsRqst(); + rqst.setColName("beta"); + rsp = client.getSchemaByCols(rqst); + Assert.assertEquals(2, rsp.getSchemaVersionsSize()); + List<SchemaVersionDescriptor> results = new ArrayList<>(rsp.getSchemaVersions()); + Collections.sort(results); + Assert.assertEquals(schemaName1, results.get(0).getSchema().getSchemaName()); + Assert.assertEquals(dbName, results.get(0).getSchema().getDbName()); + Assert.assertEquals(1, results.get(0).getVersion()); + Assert.assertEquals(schemaName1, results.get(1).getSchema().getSchemaName()); + Assert.assertEquals(dbName, results.get(1).getSchema().getDbName()); + Assert.assertEquals(2, results.get(1).getVersion()); + + // fetch across schemas + rqst = new FindSchemasByColsRqst(); + rqst.setColName("alpha"); + rsp = client.getSchemaByCols(rqst); + Assert.assertEquals(4, rsp.getSchemaVersionsSize()); + results = new ArrayList<>(rsp.getSchemaVersions()); + Collections.sort(results); + Assert.assertEquals(schemaName1, results.get(0).getSchema().getSchemaName()); + Assert.assertEquals(dbName, results.get(0).getSchema().getDbName()); + Assert.assertEquals(1, results.get(0).getVersion()); + Assert.assertEquals(schemaName1, results.get(1).getSchema().getSchemaName()); + Assert.assertEquals(dbName, results.get(1).getSchema().getDbName()); + Assert.assertEquals(2, results.get(1).getVersion()); + Assert.assertEquals(schemaName2, results.get(2).getSchema().getSchemaName()); + Assert.assertEquals(dbName, results.get(2).getSchema().getDbName()); + Assert.assertEquals(1, results.get(2).getVersion()); + Assert.assertEquals(schemaName2, results.get(3).getSchema().getSchemaName()); + Assert.assertEquals(dbName, results.get(3).getSchema().getDbName()); + Assert.assertEquals(2, results.get(3).getVersion()); + + // fetch by namespace + rqst = new FindSchemasByColsRqst(); + rqst.setColNamespace("namespace=x"); + rsp = client.getSchemaByCols(rqst); + Assert.assertEquals(2, rsp.getSchemaVersionsSize()); + results = new ArrayList<>(rsp.getSchemaVersions()); + Collections.sort(results); + Assert.assertEquals(schemaName1, results.get(0).getSchema().getSchemaName()); + Assert.assertEquals(dbName, results.get(0).getSchema().getDbName()); + Assert.assertEquals(2, results.get(0).getVersion()); + Assert.assertEquals(schemaName2, results.get(1).getSchema().getSchemaName()); + Assert.assertEquals(dbName, results.get(1).getSchema().getDbName()); + Assert.assertEquals(2, results.get(1).getVersion()); + + // fetch by name and type + rqst = new FindSchemasByColsRqst(); + rqst.setColName("alpha"); + rqst.setType(ColumnType.SMALLINT_TYPE_NAME); + rsp = client.getSchemaByCols(rqst); + Assert.assertEquals(2, rsp.getSchemaVersionsSize()); + results = new ArrayList<>(rsp.getSchemaVersions()); + Collections.sort(results); + Assert.assertEquals(schemaName2, results.get(0).getSchema().getSchemaName()); + Assert.assertEquals(dbName, results.get(0).getSchema().getDbName()); + Assert.assertEquals(1, results.get(0).getVersion()); + Assert.assertEquals(schemaName2, results.get(1).getSchema().getSchemaName()); + Assert.assertEquals(dbName, results.get(1).getSchema().getDbName()); + Assert.assertEquals(2, results.get(1).getVersion()); + + // Make sure matching name but wrong type doesn't return + rqst = new FindSchemasByColsRqst(); + rqst.setColName("alpha"); + rqst.setType(ColumnType.STRING_TYPE_NAME); + rsp = client.getSchemaByCols(rqst); + Assert.assertEquals(0, rsp.getSchemaVersionsSize()); + } + + @Test(expected = MetaException.class) + public void schemaVersionQueryNoNameOrNamespace() throws TException { + FindSchemasByColsRqst rqst = new FindSchemasByColsRqst(); + rqst.setType(ColumnType.STRING_TYPE_NAME); + client.getSchemaByCols(rqst); + } + + private static int nextSchemaNum = 1; + + private String uniqueSchemaName() { + return "uniqueschema" + nextSchemaNum++; + + } + + private String uniqueSerdeName() { + return "uniqueSerde" + nextSchemaNum++; + } + + public static class SchemaEventListener extends MetaStoreEventListener { + + public SchemaEventListener(Configuration config) { + super(config); + } + + @Override + public void onCreateISchema(CreateISchemaEvent createISchemaEvent) throws MetaException { + Integer cnt = events.get(EventMessage.EventType.CREATE_ISCHEMA); + events.put(EventMessage.EventType.CREATE_ISCHEMA, cnt == null ? 1 : cnt + 1); + } + + @Override + public void onAlterISchema(AlterISchemaEvent alterISchemaEvent) throws MetaException { + Integer cnt = events.get(EventMessage.EventType.ALTER_ISCHEMA); + events.put(EventMessage.EventType.ALTER_ISCHEMA, cnt == null ? 1 : cnt + 1); + } + + @Override + public void onDropISchema(DropISchemaEvent dropISchemaEvent) throws MetaException { + Integer cnt = events.get(EventMessage.EventType.DROP_ISCHEMA); + events.put(EventMessage.EventType.DROP_ISCHEMA, cnt == null ? 1 : cnt + 1); + } + + @Override + public void onAddSchemaVersion(AddSchemaVersionEvent addSchemaVersionEvent) throws + MetaException { + Integer cnt = events.get(EventMessage.EventType.ADD_SCHEMA_VERSION); + events.put(EventMessage.EventType.ADD_SCHEMA_VERSION, cnt == null ? 1 : cnt + 1); + } + + @Override + public void onAlterSchemaVersion(AlterSchemaVersionEvent alterSchemaVersionEvent) throws + MetaException { + Integer cnt = events.get(EventMessage.EventType.ALTER_SCHEMA_VERSION); + events.put(EventMessage.EventType.ALTER_SCHEMA_VERSION, cnt == null ? 1 : cnt + 1); + } + + @Override + public void onDropSchemaVersion(DropSchemaVersionEvent dropSchemaVersionEvent) throws + MetaException { + Integer cnt = events.get(EventMessage.EventType.DROP_SCHEMA_VERSION); + events.put(EventMessage.EventType.DROP_SCHEMA_VERSION, cnt == null ? 1 : cnt + 1); + } + } + + public static class TransactionalSchemaEventListener extends MetaStoreEventListener { + + public TransactionalSchemaEventListener(Configuration config) { + super(config); + } + + @Override + public void onCreateISchema(CreateISchemaEvent createISchemaEvent) throws MetaException { + Integer cnt = transactionalEvents.get(EventMessage.EventType.CREATE_ISCHEMA); + transactionalEvents.put(EventMessage.EventType.CREATE_ISCHEMA, cnt == null ? 1 : cnt + 1); + } + + @Override + public void onAlterISchema(AlterISchemaEvent alterISchemaEvent) throws MetaException { + Integer cnt = transactionalEvents.get(EventMessage.EventType.ALTER_ISCHEMA); + transactionalEvents.put(EventMessage.EventType.ALTER_ISCHEMA, cnt == null ? 1 : cnt + 1); + } + + @Override + public void onDropISchema(DropISchemaEvent dropISchemaEvent) throws MetaException { + Integer cnt = transactionalEvents.get(EventMessage.EventType.DROP_ISCHEMA); + transactionalEvents.put(EventMessage.EventType.DROP_ISCHEMA, cnt == null ? 1 : cnt + 1); + } + + @Override + public void onAddSchemaVersion(AddSchemaVersionEvent addSchemaVersionEvent) throws + MetaException { + Integer cnt = transactionalEvents.get(EventMessage.EventType.ADD_SCHEMA_VERSION); + transactionalEvents.put(EventMessage.EventType.ADD_SCHEMA_VERSION, cnt == null ? 1 : cnt + 1); + } + + @Override + public void onAlterSchemaVersion(AlterSchemaVersionEvent alterSchemaVersionEvent) throws + MetaException { + Integer cnt = transactionalEvents.get(EventMessage.EventType.ALTER_SCHEMA_VERSION); + transactionalEvents.put(EventMessage.EventType.ALTER_SCHEMA_VERSION, cnt == null ? 1 : cnt + 1); + } + + @Override + public void onDropSchemaVersion(DropSchemaVersionEvent dropSchemaVersionEvent) throws + MetaException { + Integer cnt = transactionalEvents.get(EventMessage.EventType.DROP_SCHEMA_VERSION); + transactionalEvents.put(EventMessage.EventType.DROP_SCHEMA_VERSION, cnt == null ? 1 : cnt + 1); + } + } + + public static class SchemaPreEventListener extends MetaStorePreEventListener { + + public SchemaPreEventListener(Configuration config) { + super(config); + } + + @Override + public void onEvent(PreEventContext context) throws MetaException, NoSuchObjectException, + InvalidOperationException { + Integer cnt = preEvents.get(context.getEventType()); + preEvents.put(context.getEventType(), cnt == null ? 1 : cnt + 1); + + } + } + + +} http://git-wip-us.apache.org/repos/asf/hive/blob/12041d39/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index b7f7be92..ca33b7d 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -27,7 +27,6 @@ import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; @@ -50,7 +49,6 @@ import org.apache.hadoop.hive.metastore.metrics.Metrics; import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; import org.apache.hadoop.hive.metastore.model.MNotificationLog; import org.apache.hadoop.hive.metastore.model.MNotificationNextId; -import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.junit.Assert; import org.junit.Assume; import org.junit.Before; @@ -98,36 +96,6 @@ public class TestObjectStore { } } - public static class MockPartitionExpressionProxy implements PartitionExpressionProxy { - @Override - public String convertExprToFilter(byte[] expr) throws MetaException { - return null; - } - - @Override - public boolean filterPartitionsByExpr(List<FieldSchema> partColumns, - byte[] expr, String defaultPartitionName, - List<String> partitionNames) - throws MetaException { - return false; - } - - @Override - public FileMetadataExprType getMetadataType(String inputFormat) { - return null; - } - - @Override - public SearchArgument createSarg(byte[] expr) { - return null; - } - - @Override - public FileFormatProxy getFileFormatProxy(FileMetadataExprType type) { - return null; - } - } - @Before public void setUp() throws Exception { Configuration conf = MetastoreConf.newMetastoreConf();