This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new 7232d2461f4 [HUDI-8005] Add a new DynamoDB based lock provider with auto-generated partition key (#11667) 7232d2461f4 is described below commit 7232d2461f4295c80001df73da2ddc150f84c8cd Author: Davis-Zhang-Onehouse <169106455+davis-zhang-oneho...@users.noreply.github.com> AuthorDate: Thu Jul 25 08:43:50 2024 -0700 [HUDI-8005] Add a new DynamoDB based lock provider with auto-generated partition key (#11667) --- ...amoDBBasedImplicitPartitionKeyLockProvider.java | 67 +++++++ .../lock/DynamoDBBasedLockProvider.java | 187 ++---------------- ...der.java => DynamoDBBasedLockProviderBase.java} | 91 +++++---- .../hudi/config/DynamoDbBasedLockConfig.java | 15 +- .../integ/ITTestDynamoDBBasedLockProvider.java | 211 ++++++++++++++++----- .../org/apache/hudi/common/util/hash/HashID.java | 22 ++- .../apache/hudi/common/util/hash/TestHashID.java | 40 +++- .../src/test/resources/hash/magic_input.txt | 10 + .../hash/xxhash_BITS_128_for_magic_input.txt | 10 + .../hash/xxhash_BITS_32_for_magic_input.txt | 10 + .../hash/xxhash_BITS_64_for_magic_input.txt | 10 + .../org/apache/hudi/common/util/StringUtils.java | 54 ++++++ .../apache/hudi/common/util/TestStringUtils.java | 92 ++++++++- scripts/release/validate_source_copyright.sh | 4 +- 14 files changed, 533 insertions(+), 290 deletions(-) diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java new file mode 100644 index 00000000000..96f8f7f30e4 --- /dev/null +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.aws.transaction.lock; + +import org.apache.hudi.common.config.HoodieCommonConfig; +import org.apache.hudi.common.config.LockConfiguration; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.util.hash.HashID; +import org.apache.hudi.storage.StorageConfiguration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +import javax.annotation.concurrent.NotThreadSafe; + +import static org.apache.hudi.common.util.StringUtils.concatenateWithThreshold; +import static org.apache.hudi.config.DynamoDbBasedLockConfig.MAX_PARTITION_KEY_SIZE_BYTE; + +/** + * A DynamoDB based lock. + * It implicitly derives the partition key from the hudi table name and hudi table base path + * available in the lock configuration. + */ +@NotThreadSafe +public class DynamoDBBasedImplicitPartitionKeyLockProvider extends DynamoDBBasedLockProviderBase { + protected static final Logger LOG = LoggerFactory.getLogger(DynamoDBBasedImplicitPartitionKeyLockProvider.class); + + public DynamoDBBasedImplicitPartitionKeyLockProvider(final LockConfiguration lockConfiguration, final StorageConfiguration<?> conf) { + this(lockConfiguration, conf, null); + } + + public DynamoDBBasedImplicitPartitionKeyLockProvider( + final LockConfiguration lockConfiguration, final StorageConfiguration<?> conf, DynamoDbClient dynamoDB) { + super(lockConfiguration, conf, dynamoDB); + } + + public static String generatePartitionKey(String basePath, String tableName) { + String hashPart = '-' + HashID.generateXXHashAsString(basePath, HashID.Size.BITS_64); + String partitionKey = concatenateWithThreshold(tableName, hashPart, MAX_PARTITION_KEY_SIZE_BYTE); + LOG.info(String.format("Partition key for base path %s is %s", basePath, partitionKey)); + return partitionKey; + } + + @Override + public String getDynamoDBPartitionKey(LockConfiguration lockConfiguration) { + String hudiTableBasePath = lockConfiguration.getConfig().getString(HoodieCommonConfig.BASE_PATH.key()); + String hudiTableName = lockConfiguration.getConfig().getString(HoodieTableConfig.HOODIE_TABLE_NAME_KEY); + return generatePartitionKey(hudiTableBasePath, hudiTableName); + } +} diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java index bbdcb4e7003..2876513b11e 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java @@ -18,200 +18,39 @@ package org.apache.hudi.aws.transaction.lock; -import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory; -import org.apache.hudi.aws.utils.DynamoTableUtils; import org.apache.hudi.common.config.LockConfiguration; -import org.apache.hudi.common.lock.LockProvider; -import org.apache.hudi.common.lock.LockState; -import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.config.DynamoDbBasedLockConfig; -import org.apache.hudi.exception.HoodieLockException; import org.apache.hudi.storage.StorageConfiguration; -import com.amazonaws.services.dynamodbv2.AcquireLockOptions; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBLockClient; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBLockClientOptions; -import com.amazonaws.services.dynamodbv2.LockItem; -import com.amazonaws.services.dynamodbv2.model.LockNotGrantedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; -import software.amazon.awssdk.services.dynamodb.model.BillingMode; -import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest; -import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; -import software.amazon.awssdk.services.dynamodb.model.KeyType; -import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput; -import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; import javax.annotation.concurrent.NotThreadSafe; -import java.io.Serializable; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.TimeUnit; +import static org.apache.hudi.config.DynamoDbBasedLockConfig.DYNAMODB_LOCK_PARTITION_KEY; /** - * A DynamoDB based lock. This {@link LockProvider} implementation allows to lock table operations - * using DynamoDB. Users need to have access to AWS DynamoDB to be able to use this lock. + * A DynamoDB based lock. + * It expects partition key is explicitly available in DynamoDbBasedLockConfig. */ @NotThreadSafe -public class DynamoDBBasedLockProvider implements LockProvider<LockItem>, Serializable { - - private static final Logger LOG = LoggerFactory.getLogger(DynamoDBBasedLockProvider.class); - - private static final String DYNAMODB_ATTRIBUTE_NAME = "key"; - - private final transient AmazonDynamoDBLockClient client; - private final String tableName; - private final String dynamoDBPartitionKey; - protected final DynamoDbBasedLockConfig dynamoDBLockConfiguration; - private volatile LockItem lock; +public class DynamoDBBasedLockProvider extends DynamoDBBasedLockProviderBase { public DynamoDBBasedLockProvider(final LockConfiguration lockConfiguration, final StorageConfiguration<?> conf) { this(lockConfiguration, conf, null); } public DynamoDBBasedLockProvider(final LockConfiguration lockConfiguration, final StorageConfiguration<?> conf, DynamoDbClient dynamoDB) { - this.dynamoDBLockConfiguration = DynamoDbBasedLockConfig.newBuilder() - .fromProperties(lockConfiguration.getConfig()) - .build(); - this.tableName = dynamoDBLockConfiguration.getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME); - this.dynamoDBPartitionKey = dynamoDBLockConfiguration.getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_PARTITION_KEY); - long leaseDuration = dynamoDBLockConfiguration.getInt(DynamoDbBasedLockConfig.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY); - if (dynamoDB == null) { - dynamoDB = getDynamoDBClient(); - } - // build the dynamoDb lock client - this.client = new AmazonDynamoDBLockClient( - AmazonDynamoDBLockClientOptions.builder(dynamoDB, tableName) - .withTimeUnit(TimeUnit.MILLISECONDS) - .withLeaseDuration(leaseDuration) - .withHeartbeatPeriod(leaseDuration / 3) - .withCreateHeartbeatBackgroundThread(true) - .build()); - - if (!this.client.lockTableExists()) { - createLockTableInDynamoDB(dynamoDB, tableName); - } - } - - @Override - public boolean tryLock(long time, TimeUnit unit) { - LOG.info(generateLogStatement(LockState.ACQUIRING, generateLogSuffixString())); - try { - lock = client.acquireLock(AcquireLockOptions.builder(dynamoDBPartitionKey) - .withAdditionalTimeToWaitForLock(time) - .withTimeUnit(TimeUnit.MILLISECONDS) - .build()); - LOG.info(generateLogStatement(LockState.ACQUIRED, generateLogSuffixString())); - } catch (InterruptedException e) { - throw new HoodieLockException(generateLogStatement(LockState.FAILED_TO_ACQUIRE, generateLogSuffixString()), e); - } catch (LockNotGrantedException e) { - return false; - } - return lock != null && !lock.isExpired(); + super(lockConfiguration, conf, dynamoDB); } @Override - public void unlock() { - try { - LOG.info(generateLogStatement(LockState.RELEASING, generateLogSuffixString())); - if (lock == null) { - return; - } - if (!client.releaseLock(lock)) { - LOG.warn("The lock has already been stolen"); - } - lock = null; - LOG.info(generateLogStatement(LockState.RELEASED, generateLogSuffixString())); - } catch (Exception e) { - throw new HoodieLockException(generateLogStatement(LockState.FAILED_TO_RELEASE, generateLogSuffixString()), e); - } - } - - @Override - public void close() { - try { - if (lock != null) { - if (!client.releaseLock(lock)) { - LOG.warn("The lock has already been stolen"); - } - lock = null; - } - this.client.close(); - } catch (Exception e) { - LOG.error(generateLogStatement(LockState.FAILED_TO_RELEASE, generateLogSuffixString())); - } - } - - @Override - public LockItem getLock() { - return lock; - } - - private DynamoDbClient getDynamoDBClient() { - String region = this.dynamoDBLockConfiguration.getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_REGION); - String endpointURL = this.dynamoDBLockConfiguration.contains(DynamoDbBasedLockConfig.DYNAMODB_ENDPOINT_URL.key()) - ? this.dynamoDBLockConfiguration.getString(DynamoDbBasedLockConfig.DYNAMODB_ENDPOINT_URL) - : DynamoDbClient.serviceMetadata().endpointFor(Region.of(region)).toString(); - - if (!endpointURL.startsWith("https://") && !endpointURL.startsWith("http://")) { - endpointURL = "https://" + endpointURL; - } - - return DynamoDbClient.builder() - .endpointOverride(URI.create(endpointURL)) - .credentialsProvider(HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(dynamoDBLockConfiguration.getProps())) - .build(); - } - - private void createLockTableInDynamoDB(DynamoDbClient dynamoDB, String tableName) { - String billingMode = dynamoDBLockConfiguration.getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_BILLING_MODE); - KeySchemaElement partitionKeyElement = KeySchemaElement - .builder() - .attributeName(DYNAMODB_ATTRIBUTE_NAME) - .keyType(KeyType.HASH) - .build(); - - List<KeySchemaElement> keySchema = new ArrayList<>(); - keySchema.add(partitionKeyElement); - - Collection<AttributeDefinition> attributeDefinitions = new ArrayList<>(); - attributeDefinitions.add(AttributeDefinition.builder().attributeName(DYNAMODB_ATTRIBUTE_NAME).attributeType(ScalarAttributeType.S).build()); - CreateTableRequest.Builder createTableRequestBuilder = CreateTableRequest.builder(); - if (billingMode.equals(BillingMode.PROVISIONED.name())) { - createTableRequestBuilder.provisionedThroughput(ProvisionedThroughput.builder() - .readCapacityUnits(dynamoDBLockConfiguration.getLong(DynamoDbBasedLockConfig.DYNAMODB_LOCK_READ_CAPACITY)) - .writeCapacityUnits(dynamoDBLockConfiguration.getLong(DynamoDbBasedLockConfig.DYNAMODB_LOCK_WRITE_CAPACITY)) - .build()); - } - createTableRequestBuilder.tableName(tableName) - .keySchema(keySchema) - .attributeDefinitions(attributeDefinitions) - .billingMode(billingMode); - dynamoDB.createTable(createTableRequestBuilder.build()); - - LOG.info("Creating dynamoDB table " + tableName + ", waiting for table to be active"); - try { - - DynamoTableUtils.waitUntilActive(dynamoDB, tableName, dynamoDBLockConfiguration.getInt(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_CREATION_TIMEOUT), 20 * 1000); - } catch (DynamoTableUtils.TableNeverTransitionedToStateException e) { - throw new HoodieLockException("Created dynamoDB table never transits to active", e); - } catch (InterruptedException e) { - throw new HoodieLockException("Thread interrupted while waiting for dynamoDB table to turn active", e); - } - LOG.info("Created dynamoDB table " + tableName); - } - - private String generateLogSuffixString() { - return StringUtils.join("DynamoDb table = ", tableName, ", partition key = ", dynamoDBPartitionKey); - } - - protected String generateLogStatement(LockState state, String suffix) { - return StringUtils.join(state.name(), " lock at ", suffix); + public String getDynamoDBPartitionKey(LockConfiguration lockConfiguration) { + DynamoDbBasedLockConfig config = new DynamoDbBasedLockConfig.Builder() + .fromProperties(lockConfiguration.getConfig()).build(); + ValidationUtils.checkArgument( + config.contains(DYNAMODB_LOCK_PARTITION_KEY), + "Config key is not found: " + DYNAMODB_LOCK_PARTITION_KEY.key()); + return config.getString(DYNAMODB_LOCK_PARTITION_KEY); } } diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProviderBase.java similarity index 70% copy from hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java copy to hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProviderBase.java index bbdcb4e7003..e9a3228025f 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProviderBase.java @@ -47,44 +47,47 @@ import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; import javax.annotation.concurrent.NotThreadSafe; -import java.io.Serializable; import java.net.URI; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.concurrent.TimeUnit; +import static org.apache.hudi.config.DynamoDbBasedLockConfig.DYNAMODB_ENDPOINT_URL; +import static org.apache.hudi.config.DynamoDbBasedLockConfig.DYNAMODB_LOCK_BILLING_MODE; +import static org.apache.hudi.config.DynamoDbBasedLockConfig.DYNAMODB_LOCK_READ_CAPACITY; +import static org.apache.hudi.config.DynamoDbBasedLockConfig.DYNAMODB_LOCK_REGION; +import static org.apache.hudi.config.DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_CREATION_TIMEOUT; +import static org.apache.hudi.config.DynamoDbBasedLockConfig.DYNAMODB_LOCK_WRITE_CAPACITY; + /** * A DynamoDB based lock. This {@link LockProvider} implementation allows to lock table operations * using DynamoDB. Users need to have access to AWS DynamoDB to be able to use this lock. */ @NotThreadSafe -public class DynamoDBBasedLockProvider implements LockProvider<LockItem>, Serializable { +public abstract class DynamoDBBasedLockProviderBase implements LockProvider<LockItem> { - private static final Logger LOG = LoggerFactory.getLogger(DynamoDBBasedLockProvider.class); + protected static final Logger LOG = LoggerFactory.getLogger(DynamoDBBasedLockProviderBase.class); - private static final String DYNAMODB_ATTRIBUTE_NAME = "key"; + protected static final String DYNAMODB_ATTRIBUTE_NAME = "key"; - private final transient AmazonDynamoDBLockClient client; - private final String tableName; + protected final DynamoDbBasedLockConfig dynamoDbBasedLockConfig; + protected final AmazonDynamoDBLockClient client; + protected final String tableName; private final String dynamoDBPartitionKey; - protected final DynamoDbBasedLockConfig dynamoDBLockConfiguration; - private volatile LockItem lock; - - public DynamoDBBasedLockProvider(final LockConfiguration lockConfiguration, final StorageConfiguration<?> conf) { - this(lockConfiguration, conf, null); - } + protected volatile LockItem lock; - public DynamoDBBasedLockProvider(final LockConfiguration lockConfiguration, final StorageConfiguration<?> conf, DynamoDbClient dynamoDB) { - this.dynamoDBLockConfiguration = DynamoDbBasedLockConfig.newBuilder() - .fromProperties(lockConfiguration.getConfig()) - .build(); - this.tableName = dynamoDBLockConfiguration.getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME); - this.dynamoDBPartitionKey = dynamoDBLockConfiguration.getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_PARTITION_KEY); - long leaseDuration = dynamoDBLockConfiguration.getInt(DynamoDbBasedLockConfig.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY); + protected DynamoDBBasedLockProviderBase(final LockConfiguration lockConfiguration, final StorageConfiguration<?> conf, DynamoDbClient dynamoDB) { if (dynamoDB == null) { dynamoDB = getDynamoDBClient(); } + this.dynamoDbBasedLockConfig = new DynamoDbBasedLockConfig.Builder() + .fromProperties(lockConfiguration.getConfig()) + .build(); + this.tableName = dynamoDbBasedLockConfig.getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME); + long leaseDuration = dynamoDbBasedLockConfig.getInt(DynamoDbBasedLockConfig.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY); + dynamoDBPartitionKey = getDynamoDBPartitionKey(lockConfiguration); + // build the dynamoDb lock client this.client = new AmazonDynamoDBLockClient( AmazonDynamoDBLockClientOptions.builder(dynamoDB, tableName) @@ -99,14 +102,20 @@ public class DynamoDBBasedLockProvider implements LockProvider<LockItem>, Serial } } + public abstract String getDynamoDBPartitionKey(LockConfiguration lockConfiguration); + + public String getPartitionKey() { + return dynamoDBPartitionKey; + } + @Override public boolean tryLock(long time, TimeUnit unit) { LOG.info(generateLogStatement(LockState.ACQUIRING, generateLogSuffixString())); try { lock = client.acquireLock(AcquireLockOptions.builder(dynamoDBPartitionKey) - .withAdditionalTimeToWaitForLock(time) - .withTimeUnit(TimeUnit.MILLISECONDS) - .build()); + .withAdditionalTimeToWaitForLock(time) + .withTimeUnit(TimeUnit.MILLISECONDS) + .build()); LOG.info(generateLogStatement(LockState.ACQUIRED, generateLogSuffixString())); } catch (InterruptedException e) { throw new HoodieLockException(generateLogStatement(LockState.FAILED_TO_ACQUIRE, generateLogSuffixString()), e); @@ -154,28 +163,28 @@ public class DynamoDBBasedLockProvider implements LockProvider<LockItem>, Serial } private DynamoDbClient getDynamoDBClient() { - String region = this.dynamoDBLockConfiguration.getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_REGION); - String endpointURL = this.dynamoDBLockConfiguration.contains(DynamoDbBasedLockConfig.DYNAMODB_ENDPOINT_URL.key()) - ? this.dynamoDBLockConfiguration.getString(DynamoDbBasedLockConfig.DYNAMODB_ENDPOINT_URL) - : DynamoDbClient.serviceMetadata().endpointFor(Region.of(region)).toString(); + String region = dynamoDbBasedLockConfig.getString(DYNAMODB_LOCK_REGION); + String endpointURL = dynamoDbBasedLockConfig.contains(DYNAMODB_ENDPOINT_URL.key()) + ? dynamoDbBasedLockConfig.getString(DYNAMODB_ENDPOINT_URL) + : DynamoDbClient.serviceMetadata().endpointFor(Region.of(region)).toString(); if (!endpointURL.startsWith("https://") && !endpointURL.startsWith("http://")) { endpointURL = "https://" + endpointURL; } return DynamoDbClient.builder() - .endpointOverride(URI.create(endpointURL)) - .credentialsProvider(HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(dynamoDBLockConfiguration.getProps())) - .build(); + .endpointOverride(URI.create(endpointURL)) + .credentialsProvider(HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(dynamoDbBasedLockConfig.getProps())) + .build(); } private void createLockTableInDynamoDB(DynamoDbClient dynamoDB, String tableName) { - String billingMode = dynamoDBLockConfiguration.getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_BILLING_MODE); + String billingMode = dynamoDbBasedLockConfig.getString(DYNAMODB_LOCK_BILLING_MODE); KeySchemaElement partitionKeyElement = KeySchemaElement - .builder() - .attributeName(DYNAMODB_ATTRIBUTE_NAME) - .keyType(KeyType.HASH) - .build(); + .builder() + .attributeName(DYNAMODB_ATTRIBUTE_NAME) + .keyType(KeyType.HASH) + .build(); List<KeySchemaElement> keySchema = new ArrayList<>(); keySchema.add(partitionKeyElement); @@ -185,20 +194,20 @@ public class DynamoDBBasedLockProvider implements LockProvider<LockItem>, Serial CreateTableRequest.Builder createTableRequestBuilder = CreateTableRequest.builder(); if (billingMode.equals(BillingMode.PROVISIONED.name())) { createTableRequestBuilder.provisionedThroughput(ProvisionedThroughput.builder() - .readCapacityUnits(dynamoDBLockConfiguration.getLong(DynamoDbBasedLockConfig.DYNAMODB_LOCK_READ_CAPACITY)) - .writeCapacityUnits(dynamoDBLockConfiguration.getLong(DynamoDbBasedLockConfig.DYNAMODB_LOCK_WRITE_CAPACITY)) - .build()); + .readCapacityUnits(dynamoDbBasedLockConfig.getLong(DYNAMODB_LOCK_READ_CAPACITY)) + .writeCapacityUnits(dynamoDbBasedLockConfig.getLong(DYNAMODB_LOCK_WRITE_CAPACITY)) + .build()); } createTableRequestBuilder.tableName(tableName) - .keySchema(keySchema) - .attributeDefinitions(attributeDefinitions) - .billingMode(billingMode); + .keySchema(keySchema) + .attributeDefinitions(attributeDefinitions) + .billingMode(billingMode); dynamoDB.createTable(createTableRequestBuilder.build()); LOG.info("Creating dynamoDB table " + tableName + ", waiting for table to be active"); try { - DynamoTableUtils.waitUntilActive(dynamoDB, tableName, dynamoDBLockConfiguration.getInt(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_CREATION_TIMEOUT), 20 * 1000); + DynamoTableUtils.waitUntilActive(dynamoDB, tableName, dynamoDbBasedLockConfig.getInt(DYNAMODB_LOCK_TABLE_CREATION_TIMEOUT), 20 * 1000); } catch (DynamoTableUtils.TableNeverTransitionedToStateException e) { throw new HoodieLockException("Created dynamoDB table never transits to active", e); } catch (InterruptedException e) { diff --git a/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java b/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java index 0e884a6797f..207c7aba9da 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java +++ b/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java @@ -47,6 +47,8 @@ public class DynamoDbBasedLockConfig extends HoodieConfig { return new DynamoDbBasedLockConfig.Builder(); } + public static final int MAX_PARTITION_KEY_SIZE_BYTE = 2048; + // configs for DynamoDb based locks public static final String DYNAMODB_BASED_LOCK_PROPERTY_PREFIX = LockConfiguration.LOCK_PREFIX + "dynamodb."; @@ -140,7 +142,7 @@ public class DynamoDbBasedLockConfig extends HoodieConfig { public DynamoDbBasedLockConfig build() { lockConfig.setDefaults(DynamoDbBasedLockConfig.class.getName()); - checkRequiredProps(lockConfig); + checkRequiredProps(); return lockConfig; } @@ -149,17 +151,14 @@ public class DynamoDbBasedLockConfig extends HoodieConfig { return this; } - private void checkRequiredProps(final DynamoDbBasedLockConfig config) { + private void checkRequiredProps() { String errorMsg = "Config key is not found: "; ValidationUtils.checkArgument( - config.contains(DYNAMODB_LOCK_TABLE_NAME.key()), + lockConfig.contains(DYNAMODB_LOCK_TABLE_NAME.key()), errorMsg + DYNAMODB_LOCK_TABLE_NAME.key()); ValidationUtils.checkArgument( - config.contains(DYNAMODB_LOCK_REGION.key()), + lockConfig.contains(DYNAMODB_LOCK_REGION.key()), errorMsg + DYNAMODB_LOCK_REGION.key()); - ValidationUtils.checkArgument( - config.contains(DYNAMODB_LOCK_PARTITION_KEY.key()), - errorMsg + DYNAMODB_LOCK_PARTITION_KEY.key()); } } -} +} \ No newline at end of file diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/transaction/integ/ITTestDynamoDBBasedLockProvider.java b/hudi-aws/src/test/java/org/apache/hudi/aws/transaction/integ/ITTestDynamoDBBasedLockProvider.java index 10d8b068279..e28c2cf0b32 100644 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/transaction/integ/ITTestDynamoDBBasedLockProvider.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/transaction/integ/ITTestDynamoDBBasedLockProvider.java @@ -18,14 +18,24 @@ package org.apache.hudi.aws.transaction.integ; +import org.apache.hudi.aws.transaction.lock.DynamoDBBasedImplicitPartitionKeyLockProvider; import org.apache.hudi.aws.transaction.lock.DynamoDBBasedLockProvider; +import org.apache.hudi.aws.transaction.lock.DynamoDBBasedLockProviderBase; +import org.apache.hudi.common.config.HoodieCommonConfig; import org.apache.hudi.common.config.LockConfiguration; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hudi.common.util.hash.HashID; import org.apache.hudi.config.DynamoDbBasedLockConfig; +import org.apache.hudi.storage.StorageConfiguration; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @@ -37,6 +47,7 @@ import java.net.URI; import java.util.Properties; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY; @@ -47,62 +58,78 @@ import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_WAIT_ @Disabled("HUDI-7475 The tests do not work. Disabling them to unblock Azure CI") public class ITTestDynamoDBBasedLockProvider { - private static LockConfiguration lockConfiguration; - private static DynamoDbClient dynamoDb; - + private static final LockConfiguration LOCK_CONFIGURATION; + private static final LockConfiguration IMPLICIT_PART_KEY_LOCK_CONFIG_NO_BASE_PATH; + private static final LockConfiguration IMPLICIT_PART_KEY_LOCK_CONFIG_WITH_PART_KEY; + private static final LockConfiguration IMPLICIT_PART_KEY_LOCK_CONFIG_NO_TBL_NAME; + private static final LockConfiguration IMPLICIT_PART_KEY_LOCK_CONFIG; private static final String TABLE_NAME_PREFIX = "testDDBTable-"; private static final String REGION = "us-east-2"; + private static DynamoDbClient dynamoDb; - @BeforeAll - public static void setup() throws InterruptedException { - Properties properties = new Properties(); - properties.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_BILLING_MODE.key(), BillingMode.PAY_PER_REQUEST.name()); - // properties.setProperty(AWSLockConfig.DYNAMODB_LOCK_TABLE_NAME.key(), TABLE_NAME_PREFIX); - properties.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_PARTITION_KEY.key(), "testKey"); - properties.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_REGION.key(), REGION); - properties.setProperty(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "1000"); - properties.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_READ_CAPACITY.key(), "0"); - properties.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_WRITE_CAPACITY.key(), "0"); - lockConfiguration = new LockConfiguration(properties); - dynamoDb = getDynamoClientWithLocalEndpoint(); - } + static { + Properties dynamoDblpProps = new Properties(); + Properties implicitPartKeyLpProps; + // Common properties shared by both lock providers. + dynamoDblpProps.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_BILLING_MODE.key(), BillingMode.PAY_PER_REQUEST.name()); + dynamoDblpProps.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_CREATION_TIMEOUT.key(), Integer.toString(20 * 1000 * 5)); + dynamoDblpProps.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_REGION.key(), REGION); + dynamoDblpProps.setProperty(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "1000"); + dynamoDblpProps.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_READ_CAPACITY.key(), "0"); + dynamoDblpProps.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_WRITE_CAPACITY.key(), "0"); - @Test - public void testAcquireLock() { - lockConfiguration.getConfig().setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME.key(), TABLE_NAME_PREFIX + UUID.randomUUID()); - DynamoDBBasedLockProvider dynamoDbBasedLockProvider = new DynamoDBBasedLockProvider(lockConfiguration, null, dynamoDb); - Assertions.assertTrue(dynamoDbBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); - dynamoDbBasedLockProvider.unlock(); + implicitPartKeyLpProps = new TypedProperties(dynamoDblpProps); + dynamoDblpProps = new TypedProperties(dynamoDblpProps); + + // For the day-1 DynamoDbBasedLockProvider, it requires an optional partition key + dynamoDblpProps.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_PARTITION_KEY.key(), "testKey"); + LOCK_CONFIGURATION = new LockConfiguration(dynamoDblpProps); + + // For the newly added implicit partition key DDB lock provider, it can derive the partition key from + // hudi table base path and hudi table name. These properties are available in lockConfig as of today. + implicitPartKeyLpProps.setProperty( + HoodieCommonConfig.BASE_PATH.key(), + "gs://my-bucket-8b2a4b30/1718662238400/be715573/my_lake/my_table"); + implicitPartKeyLpProps.setProperty( + HoodieTableConfig.HOODIE_TABLE_NAME_KEY, "ma_po_tofu_is_awesome"); + IMPLICIT_PART_KEY_LOCK_CONFIG = new LockConfiguration(implicitPartKeyLpProps); + // With partition key nothing should break, the field should be simply ignored. + TypedProperties withPartKey = new TypedProperties(implicitPartKeyLpProps); + withPartKey.setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_PARTITION_KEY.key(), "testKey"); + IMPLICIT_PART_KEY_LOCK_CONFIG_WITH_PART_KEY = new LockConfiguration(withPartKey); + + // Missing either base path or hoodie table name is a bad config for implicit partition key lock provider. + TypedProperties missingBasePath = new TypedProperties(implicitPartKeyLpProps); + missingBasePath.remove(HoodieCommonConfig.BASE_PATH.key()); + IMPLICIT_PART_KEY_LOCK_CONFIG_NO_BASE_PATH = new LockConfiguration(missingBasePath); + + TypedProperties missingTableName = new TypedProperties(implicitPartKeyLpProps); + missingTableName.remove(HoodieTableConfig.HOODIE_TABLE_NAME_KEY); + IMPLICIT_PART_KEY_LOCK_CONFIG_NO_TBL_NAME = new LockConfiguration(missingTableName); } - @Test - public void testUnlock() { - lockConfiguration.getConfig().setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME.key(), TABLE_NAME_PREFIX + UUID.randomUUID()); - DynamoDBBasedLockProvider dynamoDbBasedLockProvider = new DynamoDBBasedLockProvider(lockConfiguration, null, dynamoDb); - Assertions.assertTrue(dynamoDbBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); - dynamoDbBasedLockProvider.unlock(); - Assertions.assertTrue(dynamoDbBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); + @BeforeAll + public static void setup() throws InterruptedException { + dynamoDb = getDynamoClientWithLocalEndpoint(); } - @Test - public void testReentrantLock() { - lockConfiguration.getConfig().setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME.key(), TABLE_NAME_PREFIX + UUID.randomUUID()); - DynamoDBBasedLockProvider dynamoDbBasedLockProvider = new DynamoDBBasedLockProvider(lockConfiguration, null, dynamoDb); - Assertions.assertTrue(dynamoDbBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); - Assertions.assertFalse(dynamoDbBasedLockProvider.tryLock(lockConfiguration.getConfig() - .getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); - dynamoDbBasedLockProvider.unlock(); + public static Stream<Object> testDimensions() { + return Stream.of( + // Without parititon key, only table name is used. + Arguments.of(IMPLICIT_PART_KEY_LOCK_CONFIG, DynamoDBBasedLockProvider.class), + Arguments.of(LOCK_CONFIGURATION, DynamoDBBasedLockProvider.class), + // Even if we have partition key set, nothing would break. + Arguments.of(IMPLICIT_PART_KEY_LOCK_CONFIG_WITH_PART_KEY, DynamoDBBasedImplicitPartitionKeyLockProvider.class), + Arguments.of(IMPLICIT_PART_KEY_LOCK_CONFIG, DynamoDBBasedImplicitPartitionKeyLockProvider.class) + ); } - @Test - public void testUnlockWithoutLock() { - lockConfiguration.getConfig().setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME.key(), TABLE_NAME_PREFIX + UUID.randomUUID()); - DynamoDBBasedLockProvider dynamoDbBasedLockProvider = new DynamoDBBasedLockProvider(lockConfiguration, null, dynamoDb); - dynamoDbBasedLockProvider.unlock(); + public static Stream<Object> badTestDimensions() { + return Stream.of( + Arguments.of(IMPLICIT_PART_KEY_LOCK_CONFIG_NO_TBL_NAME, DynamoDBBasedLockProvider.class), + Arguments.of(IMPLICIT_PART_KEY_LOCK_CONFIG_NO_TBL_NAME, DynamoDBBasedImplicitPartitionKeyLockProvider.class), + Arguments.of(IMPLICIT_PART_KEY_LOCK_CONFIG_NO_BASE_PATH, DynamoDBBasedImplicitPartitionKeyLockProvider.class) + ); } private static DynamoDbClient getDynamoClientWithLocalEndpoint() { @@ -111,13 +138,95 @@ public class ITTestDynamoDBBasedLockProvider { throw new IllegalStateException("dynamodb-local.endpoint system property not set"); } return DynamoDbClient.builder() - .region(Region.of(REGION)) - .endpointOverride(URI.create(endpoint)) - .credentialsProvider(getCredentials()) - .build(); + .region(Region.of(REGION)) + .endpointOverride(URI.create(endpoint)) + .credentialsProvider(getCredentials()) + .build(); } private static AwsCredentialsProvider getCredentials() { return StaticCredentialsProvider.create(AwsBasicCredentials.create("random-access-key", "random-secret-key")); } + + @ParameterizedTest + @MethodSource("badTestDimensions") + void testBadConfig(LockConfiguration lockConfig, Class<?> lockProviderClass) { + lockConfig.getConfig().setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME.key(), TABLE_NAME_PREFIX + UUID.randomUUID()); + Exception e = new Exception(); + try { + ReflectionUtils.loadClass( + lockProviderClass.getName(), + new Class<?>[] {LockConfiguration.class, StorageConfiguration.class, DynamoDbClient.class}, + lockConfig, null, dynamoDb); + } catch (Exception ex) { + e = ex; + } + Assertions.assertEquals(IllegalArgumentException.class, e.getCause().getCause().getClass()); + } + + @ParameterizedTest + @MethodSource("testDimensions") + void testAcquireLock(LockConfiguration lockConfig, Class<?> lockProviderClass) { + lockConfig.getConfig().setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME.key(), TABLE_NAME_PREFIX + UUID.randomUUID()); + DynamoDBBasedLockProviderBase dynamoDbBasedLockProvider = (DynamoDBBasedLockProviderBase) ReflectionUtils.loadClass( + lockProviderClass.getName(), + new Class<?>[] {LockConfiguration.class, StorageConfiguration.class, DynamoDbClient.class}, + new Object[] {lockConfig, null, dynamoDb}); + + // Also validate the partition key is properly constructed. + if (lockProviderClass.equals(DynamoDBBasedLockProvider.class)) { + String tableName = (String) lockConfig.getConfig().get(HoodieTableConfig.HOODIE_TABLE_NAME_KEY); + String partitionKey = (String) lockConfig.getConfig().get(DynamoDbBasedLockConfig.DYNAMODB_LOCK_PARTITION_KEY.key()); + if (partitionKey != null) { + Assertions.assertEquals(partitionKey, dynamoDbBasedLockProvider.getPartitionKey()); + } else { + Assertions.assertEquals(tableName, dynamoDbBasedLockProvider.getPartitionKey()); + } + } else if (lockProviderClass.equals(DynamoDBBasedImplicitPartitionKeyLockProvider.class)) { + String tableName = (String) lockConfig.getConfig().get(HoodieTableConfig.HOODIE_TABLE_NAME_KEY); + String basePath = (String) lockConfig.getConfig().get(HoodieCommonConfig.BASE_PATH.key()); + Assertions.assertEquals( + tableName + '-' + HashID.generateXXHashAsString(basePath, HashID.Size.BITS_64), + dynamoDbBasedLockProvider.getPartitionKey()); + } + Assertions.assertTrue(dynamoDbBasedLockProvider.tryLock(lockConfig.getConfig().getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); + dynamoDbBasedLockProvider.unlock(); + } + + @ParameterizedTest + @MethodSource("testDimensions") + void testUnlock(LockConfiguration lockConfig, Class<?> lockProviderClass) { + lockConfig.getConfig().setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME.key(), TABLE_NAME_PREFIX + UUID.randomUUID()); + DynamoDBBasedLockProviderBase dynamoDbBasedLockProvider = (DynamoDBBasedLockProviderBase) ReflectionUtils.loadClass( + lockProviderClass.getName(), + new Class<?>[] {LockConfiguration.class, StorageConfiguration.class, DynamoDbClient.class}, + new Object[] {lockConfig, null, dynamoDb}); + Assertions.assertTrue(dynamoDbBasedLockProvider.tryLock(lockConfig.getConfig().getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); + dynamoDbBasedLockProvider.unlock(); + Assertions.assertTrue(dynamoDbBasedLockProvider.tryLock(lockConfig.getConfig().getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); + } + + @ParameterizedTest + @MethodSource("testDimensions") + void testReentrantLock(LockConfiguration lockConfig, Class<?> lockProviderClass) { + lockConfig.getConfig().setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME.key(), TABLE_NAME_PREFIX + UUID.randomUUID()); + DynamoDBBasedLockProviderBase dynamoDbBasedLockProvider = (DynamoDBBasedLockProviderBase) ReflectionUtils.loadClass( + lockProviderClass.getName(), + new Class<?>[] {LockConfiguration.class, StorageConfiguration.class, DynamoDbClient.class}, + new Object[] {lockConfig, null, dynamoDb}); + Assertions.assertTrue(dynamoDbBasedLockProvider.tryLock(lockConfig.getConfig().getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); + Assertions.assertFalse(dynamoDbBasedLockProvider.tryLock(lockConfig.getConfig().getLong(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY), TimeUnit.MILLISECONDS)); + dynamoDbBasedLockProvider.unlock(); + } + + @ParameterizedTest + @MethodSource("testDimensions") + void testUnlockWithoutLock(LockConfiguration lockConfig, Class<?> lockProviderClass) { + lockConfig.getConfig().setProperty(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME.key(), TABLE_NAME_PREFIX + UUID.randomUUID()); + DynamoDBBasedLockProviderBase dynamoDbBasedLockProvider = (DynamoDBBasedLockProviderBase) ReflectionUtils.loadClass( + lockProviderClass.getName(), + new Class<?>[] {LockConfiguration.class, StorageConfiguration.class, DynamoDbClient.class}, + new Object[] {lockConfig, null, dynamoDb}); + dynamoDbBasedLockProvider.unlock(); + } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/hash/HashID.java b/hudi-common/src/main/java/org/apache/hudi/common/util/hash/HashID.java index 4df8c385289..e76c5a8d073 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/hash/HashID.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/hash/HashID.java @@ -79,10 +79,10 @@ public class HashID implements Serializable { } /** - * Get the hash value for a string message and for the desired @{@link Size}. + * Get the hash value for a string message and for the desired @{@link HashID.Size}. * * @param message - String message to get the hash value for - * @param bits - @{@link Size} of the hash value + * @param bits - @{@link HashID.Size} of the hash value * @return Hash value for the message as byte array */ public static byte[] hash(final String message, final Size bits) { @@ -108,6 +108,24 @@ public class HashID implements Serializable { } } + /** + * Get the hash value as string for a given string and for the desired @{@link Size}. + * + * @param input - String message to get the hash value for. + * @param size - @{@link Size} of the hash value + * @return Hash value for the message as string. + */ + public static String generateXXHashAsString(String input, Size size) { + // Compute the hash + byte[] hashBytes = hash(input, size); + // Convert the hash value to a hexadecimal string + StringBuilder hexString = new StringBuilder(); + for (byte hashByte : hashBytes) { + hexString.append(String.format("%02X", hashByte)); + } + return hexString.toString(); + } + public static int getXXHash32(final String message, int hashSeed) { return getXXHash32(getUTF8Bytes(message), hashSeed); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/hash/TestHashID.java b/hudi-common/src/test/java/org/apache/hudi/common/util/hash/TestHashID.java index 1ab9d82b2b9..319e216ff4f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/hash/TestHashID.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/hash/TestHashID.java @@ -21,24 +21,40 @@ package org.apache.hudi.common.util.hash; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import javax.xml.bind.DatatypeConverter; -import java.util.Arrays; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Random; +import static java.util.Arrays.asList; import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests {@link HashID}. */ public class TestHashID { + public static List<Arguments> hashIdSizes() { + return asList( + Arguments.of(HashID.Size.BITS_32), + Arguments.of(HashID.Size.BITS_64), + Arguments.of(HashID.Size.BITS_128) + ); + } + /** * Test HashID of all sizes for ByteArray type input message. */ @@ -124,8 +140,26 @@ public class TestHashID { for (Map.Entry<String, String> sizeEntry : allSizeEntries.getValue().entrySet()) { final byte[] actualHashBytes = HashID.hash(sizeEntry.getKey(), allSizeEntries.getKey()); final byte[] expectedHashBytes = DatatypeConverter.parseHexBinary(sizeEntry.getValue()); - assertTrue(Arrays.equals(expectedHashBytes, actualHashBytes)); + assertArrayEquals(expectedHashBytes, actualHashBytes); } } } + + @ParameterizedTest + @MethodSource("hashIdSizes") + void testGenerateXXHashMagicNumber(HashID.Size size) throws IOException, URISyntaxException { + // We need to make sure we always generate the same hash value for the same input. This test + // guards against unexpected change of hash value due to accidents like library version upgrade. + // Load inputs and expected hash values from files. + List<String> inputs = Files.readAllLines(Paths.get(Objects.requireNonNull( + getClass().getClassLoader().getResource("hash/magic_input.txt")).toURI())); + List<String> expectedHash = Files.readAllLines(Paths.get(Objects.requireNonNull( + getClass().getClassLoader().getResource(String.format("hash/xxhash_%s_for_magic_input.txt", size.name()))).toURI())); + + for (int i = 0; i < expectedHash.size(); ++i) { + String hash = HashID.generateXXHashAsString(inputs.get(i), size); + // Magic number test to guard against accidental upgrade that changes the hash value + assertEquals(expectedHash.get(i), hash); + } + } } diff --git a/hudi-common/src/test/resources/hash/magic_input.txt b/hudi-common/src/test/resources/hash/magic_input.txt new file mode 100644 index 00000000000..ef6468b087a --- /dev/null +++ b/hudi-common/src/test/resources/hash/magic_input.txt @@ -0,0 +1,10 @@ +vB1xYjlEfL8tN9Ap6NMCwraFQlpQ6QU2z9XCPCv9lfIxzRfs6GfxKir2FuU:Xbl9uMZqjNRhKLqZY2MIH5vPCKZbV1zjoHVaVBnyRfzJLZ0jwWZQjJqjpdg/PsrIq3YdyAsDHcb__oXfbZUpMFX00g6ALt/uRZ2LOFFdQA9Ofuu_BLhhPl4gIfGNJIt1ATuCGIFXFG4zg1qn96MsLy8Nr2v/6pxzBXM5Rhzn6o_HhiFo1RzooJcm1tJlObZroxBNKSDbDcdYkm4KnAVUAVcyVfO2DRx_KQfphLYbG1rNRTV03RNN7jQakMf8Vq38Q1Joz:STilSqalcvrDmdPTY1RETh8eVP0ELqsKevQ22OKKl5ypy4I2Y6nqRfTfoepo_l0zc_QEZNawdif5l:SpvaJUaWU9rpUpqxiz2T_40ekLVXDM_B9oGgS4e9AMT50OASHp/cVue2qFpgrYSptp4EZZ9yfEqPXX3Ij7hSUGL29zar [...] +EXDDWHY_7:5oYXsoCWM1LVD2mxXAilEXwsA8J_btuAAPUm6ODVlM/njpnAyULYvBKrAiwjY4xMWkJLCGS09qqmvu8ea5:ib/WYg1nHAELdy8fiREkQ/QXIByXL1xJ40ZfPYiMzUlNoUMXJNVqK59bipG7NSOhEZ0KSAW9uxv77S4G:8B1ybZNgWbYx_f1/DqYgPD7PcHfr3c_NMYvHVOCEOB3oYZwU2p_/BRWfaY/aJfydpG_VivzSqYM7qCghR8LXEL8955iXjPVMpXHu03BdAqE/uXsrlvTm8sdzum/PNsrwHYH:2YCHRjJ7B9yawP2mYZmmCTA9FSAqhizLrs7JvqUQNfF0DW8Imy_ld__eTq0hM55H3QEIzDs:u45zd6Y/1hEeBMFEGmgXHJDOGfSt1boX_lO6q/M33U5n0LEv1o_4_tQeqnsqODq509y7rbgfOVg7whAtZ81QlXNQ3VITG0kadegE3EFcD6OwC1I3BE [...] +qbl3zptxyhBdGZzXYSuz/dc_W79I3nnZIg41:_ujQnA8NnzrsENtSvxUX5Hj5nHyjmlzS9vH2zawvXaoeTtwXPqEVp0Vc2p7Ho93iRUq:Zj/bjAxIuxBXEJMPYTE5PBVPSR2yPP4ci0ChrVtIRCXTD7v9gluqs4ouz_G8sNuXaT8uCP7ypOMRUG:VQuQ69yY/KIS1_AffBHL8n7jgT7nGfzra5dZZUBe4oqm9BI3nyHpCa5ookke:703QP:8sDMg3UdCg8uw6ccqAcco7xBbOgdvS6qpz1LZdku0:As2LOmX42B0aEReh52BdtZtBFnk5t_cm0m5ZAuEH_Bii8:reXocImMCSsnL2hVpbs8gn8jI4QAGW2rDOvj1N/byJbjrJxhlsZ6Y9dDZc7Z8B8aCx8kRoRQVkskAO4lULO1pNaIINreWaij:tJhHeF_kkhSevooTRkGXoWouI/1a4ADi5ijTESgsrBleqpU2e4yGmBKw [...] +kha:j60RU2k4jsMcyoK/gJ7F4IEkoWzrtVN3kUEPvP/9M4NfaoPtruIe0reUxlMhoTKJbLTLFJ0pIVfqGrx/VAcW1ZWdjret19cb7TAP08wpOP_t5ljP/pUYR5KsEn5zUNtKWiNHJxPsT7u9FYNupiIRIe_VGG_J:1wPQJ2xBtWCbiCOGk15zGoOHhAjTE8V5JwyyJHrxSfZQpHXMqOYICII9/rXoKpTsV/uUASF58e/OuQ9pg0IA4KENs3_XnptctzloUYr0AI4zTLIRL5N:CvFRujqaOmKgiMGYmC1V9DdYkUTAiv_3NYvUnJuV54mXMEQlAoyrILPimonjdpjLx7fjep9vPjImfsyIXK3EoStTNJoWcAWsOzSn:wb84sBDg4mfDYvqz3pL13SFbIxtlsAl66/11xBVIUI0TRK6KWX5yX806JnbFSG:oi6X4_OdwucUDSx:ac_RqzZiA9Ig:wFg2dg6JYrVt3iLj/KDaS0 [...] +XViqCa5XfqAB0kski4A4N82FE8i0U/WjPvcgLbi:X9LhXRV0qfusVp19Xr_5pCQUDFRr34z_foGnpbWC5:pOn:jQHwVJIer:hfh5uErwLimX6OC1nLPXdPVrhKJ4/XN2YBfF5luB_7:dEGJ1E22iCTgto6sQW63f/c:wUtZdRioJ3JiA14KMVRKI8KD79zrCF1p61XSjQUvzrBh4VOjxGervgu2kZarv4HvvzvTuzThk4yrE1YvQse5iKeZWHbjCYBS7Tr0eTAjZLIqZRZ4/t/zuI9ZgnIerWJ8z3kfYv:maAiYP:Fjc4k32CXdXVhfiP9OLMqu4XWA2U1i7htnAotjF3G1ECwGLo9faRUxp5XwK85DhoQqrld/9e4BX4FxI61omgU_sa9SlSI:oU4/GrmsejIJn8adPcxwUyYf5uetKOVbcuBesp9l7pVmc4O97sojQ6:ohnhcvGssks0UpzdUideM8Hfo0yIWF1XPSJjJV [...] +uYeJoV7jMpUbSfo0PZfVqxapg8VlFDxp7t6x3M:r/hMj3EdSrXNmtnAJOg/DubMz6uGvkZM3equxs8LflyyjZYFypahe90QXYM6mngjaMpXGl/ngy6_wQ512E7tqMnaVaz52xbKAdCCgastCJ5xM2jikxmIOK/28J77VXwkL6T43JGKXrFzoqm/eelXf:G5oihZt/jBKTMg/wXgC0:BvTVM9A1e4nmXw_ANjnBxCWe7QD7O/A/k4hrVt0MKpH2glsJKAnz_ZAY_qZp33vN5v975ZFwhAJtkYwtsXl4UFT_06Uy9BHyWnMcvHaPgEgslv/_P2FLzlJWfsSI8UjWUdMVhYAff4NE7UQF2Oye6ciXsW1ga_TS0VN8tugmqE:xs:5/SlyG3puDh:mzA60V7EpM38_rsKO/v9l1ZG9ICucglBv0hzHuVz/_vUR_5B_scT4nEIFmLOB67RsoGXMDVI7oJzETHzA3DLk:nX1yF4g_VT [...] +1/dFjdWmwjcDBD6QBMjRqWtTJEdCZ2:2X2:NiPutlDhlxkp7k2MBqiAMDffvA5bbe0Yxel0IF94is_h0cMtgXKwYgCPKgFQgH0O2hnFrWp2fmETy4DCHnUV54mNUF4_xMESiZwJaUQFCPUxK8j28qlPppbt2M:zsJ64LUVIoCZpJsc1GRVY9lImikNRoEiOXKOQIMLOHlDfVEYyJubQy14zWQsqSoHK6Du_S76_MdZi3DEHJmInbDHvykJRoKke2QQL7wae/7D4FVg/zKbfnOSrgDTWbdDewy2lJKMti5MnrMnevTCQSSFhueoekPYsLiyoDIDWDoqAzdiWP2vkzisR1CbZ74DQeOyyk4rvF85TRTfjPWjdv6PZX40W0K2VOPWUiys5224nSnfT886LVBkuKwjImo8NqZskfp1LYJtMRYlad9/fRLVwo65h:l4FFLEuPH1OzlFaz6wsMqqh4GGtQeScwfzTWA78A9iosPNyn [...] +twVaFB_AiBxWsfHRYHdhnwMtrtp/PzyU6lJivvSGGRfGmLUu0EcaW5dzO0vx/Hy/E33JY:2oeb1p:Hl35zPFj:98DQRRyodl5JJrBiNeyavBKlQPoYglYIpWK3uy47lsTHXe2SYrEEscKtwH/:BJiQn4XPQejfVBPNPI35Othp64j0tf2GBiqrAETQT0xnSrN6vnQnNWAj:RDR9IbUSZISaJ/GrQoT651W7iwQQvBo5L/XcdCrvWGbmw/3Yp5S2HeoA/:qJ:6gmrOQ9eoFGCe8X9yERKKbPzcmSB7SXKuP3UUbT1L0_vvFOP2mIsr46k:77G31gPJv6zzdDZLR1Ewxaz38vR6MeSjTtKR53yw44cWgWPD2wS/GHJDI4PhHxnsgUamNi8_PVmdFceTXouehmnrdrM_byV_iQOF:3Jq7czDhcAwMF2eYqZkdlPdG:kr:WQoIBu2UTgZB/uKI8kf7l4qosy:aTz_QLmouh5ilZ1 [...] +wM51f5wQY1p4N_J4hC/oN9BryO/ZUwUisXuYaGHIJGDZmTE52hPf7vJdPTR:NiJiPCNMG:cnWGoJwNRiKIQY3u/Yppx6Ig6wdvScLqPhXI5Fxq/W4XUgBkCwEVkhfd3FCv0/LmaNvg6RJelbz:/ympzvRLOpAgsYmxtnqh17rEqCkxKp4nC4o_j7VLJgyHqXYP9RnKw2sTpMP7seq1zHe00drKb_k:_iS59NZ2NnAJBjjKCo2kbNmT4LvAVa9COquUyCF4Q:Qp39Rxs30mNtTW30x_0LuL8TH4pg65Z0OjBoQtwo4:GBF_2rE2WbkRDeNeGGB:DUU4p0MqkkYTT7jraK10pmKi2C:wH6HXN8SX6UwCJoY2m24sIaU3f:k2vBzqYu4Mq5E9slvsx:vNjlA4KtSiy5XLt01s0bRuXhXXrjQuSsFXvGUdlppWtQcR8Xbg6W0Bk5Qdx267BBbsQ5bk6JOg8xHfGqcy2Kde_u_ZFz [...] +AXipEHIS1WnAkrLS_TZT8AEBe2KPR:0tgOLRhUIEiN33Ml4_pOtN6TuDNvfSa4MSPexRluckSpCnl6le__MucXEilapxfqXY/MW3ebuICROpXQr9rn4fvJxxR25eeyDDOJfVkXu9gTwPJD:CAqb:byah1PXndGFR3L4NNgzofPKVvyh/liGhvcU54qF3:U8B4Ed4K5VVZKlDC_LIjyPpO:fzwhCY:a7plQHp3G8rwqZlVWll8gZt1BRTJS3nvxCLNe2IPjq5aj5QD/P3O4QkLPDW/bdXTFxfILaQ3xrBO:kqPMyqowiLcm6gAwY5fdkPgvv6xh7yC5t8d98io1UxEOoVdWdGUXgztIKs2gxjzVUqj6qsbLaIg2ft/j4txWJMQ3z3anRSmCZ1Th6V7LFEKaX2PBlK//eC0Sm1f45XPf5MWpHuRMxBzxFqY60IXZVJ1tx/Daa86UC5A0jyyJyPxmdUlt1FTqDqZ_TQK4qTWGrz [...] diff --git a/hudi-common/src/test/resources/hash/xxhash_BITS_128_for_magic_input.txt b/hudi-common/src/test/resources/hash/xxhash_BITS_128_for_magic_input.txt new file mode 100644 index 00000000000..ee2a4f5fbea --- /dev/null +++ b/hudi-common/src/test/resources/hash/xxhash_BITS_128_for_magic_input.txt @@ -0,0 +1,10 @@ +AF26454CFF3D72BBDB7A6CD6BAF23770 +080D42690A45825782F4CD4DB31AA449 +E14D364EF059C966A7F50708E9CF2E7B +68F4BCB6F281DD101D5B7375652E574E +1F2A77EB57C4091E9F89887DA88169B2 +1A57459BE2F7ECD655C090BADAA3ED1A +E79D2F90D440296F94419C7093518DE4 +86C4CFD114B59C0CB29AF95425767B63 +EE70F0B8607E339D92E6823DA2538113 +0E6F75EC00B024201B91FBCEBDB9F971 \ No newline at end of file diff --git a/hudi-common/src/test/resources/hash/xxhash_BITS_32_for_magic_input.txt b/hudi-common/src/test/resources/hash/xxhash_BITS_32_for_magic_input.txt new file mode 100644 index 00000000000..4cba48b9f4e --- /dev/null +++ b/hudi-common/src/test/resources/hash/xxhash_BITS_32_for_magic_input.txt @@ -0,0 +1,10 @@ +0304CA31 +71EC1ABE +4F003C35 +79A69D14 +62CAF229 +0136BCE2 +1D68C842 +0B416E0E +CF7373EB +F5933F12 \ No newline at end of file diff --git a/hudi-common/src/test/resources/hash/xxhash_BITS_64_for_magic_input.txt b/hudi-common/src/test/resources/hash/xxhash_BITS_64_for_magic_input.txt new file mode 100644 index 00000000000..bc06cc5614e --- /dev/null +++ b/hudi-common/src/test/resources/hash/xxhash_BITS_64_for_magic_input.txt @@ -0,0 +1,10 @@ +F0E17DF624BF7B6A +07EC6A7186973385 +24A8A6F1FEA11BA2 +BFE00C8A367BAF03 +C323770B790400C8 +9F96033C96E80D39 +5B99015437BBEA94 +C6C54FCA5141979D +3580238C2761459E +D9DBB1D9BF7174BC \ No newline at end of file diff --git a/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java b/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java index ded0d876eef..01e7426a6a6 100644 --- a/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java +++ b/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java @@ -327,4 +327,58 @@ public class StringUtils { } return str.substring(0, end); } + + /** + * Concatenates two strings such that the total byte length does not exceed the threshold. + * If the total byte length exceeds the threshold, the function will find the maximum length of the first string + * that fits within the threshold and concatenate that with the second string. + * + * @param a The first string + * @param b The second string + * @param threshold The maximum byte length + */ + public static String concatenateWithThreshold(String a, String b, int threshold) { + // Convert both strings to byte arrays in UTF-8 encoding + byte[] bytesA = getUTF8Bytes(a); + byte[] bytesB = getUTF8Bytes(b); + if (bytesB.length > threshold) { + throw new IllegalArgumentException(String.format( + "Length of the Second string to concatenate exceeds the threshold (%d > %d)", + bytesB.length, threshold)); + } + + // Calculate total bytes + int totalBytes = bytesA.length + bytesB.length; + + // If total bytes is within the threshold, return concatenated string + if (totalBytes <= threshold) { + return a + b; + } + + // Calculate the maximum bytes 'a' can take + int bestLength = getBestLength(a, threshold - bytesB.length); + + // Concatenate the valid substring of 'a' with 'b' + return a.substring(0, bestLength) + b; + } + + private static int getBestLength(String a, int threshold) { + // Binary search to find the maximum length of substring of 'a' that fits within maxBytesForA + int low = 0; + int high = Math.min(a.length(), threshold); + int bestLength = 0; + + while (low <= high) { + int mid = (low + high) / 2; + byte[] subABytes = getUTF8Bytes(a.substring(0, mid)); + + if (subABytes.length <= threshold) { + bestLength = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + return bestLength; + } } diff --git a/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java b/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java index 7cbf065b880..049601e3517 100644 --- a/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java +++ b/hudi-io/src/test/java/org/apache/hudi/common/util/TestStringUtils.java @@ -22,14 +22,19 @@ package org.apache.hudi.common.util; import org.junit.jupiter.api.Test; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Random; +import static org.apache.hudi.common.util.StringUtils.concatenateWithThreshold; import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -39,6 +44,29 @@ public class TestStringUtils { private static final String[] STRINGS = {"This", "is", "a", "test"}; + private static final String CHARACTERS_FOR_RANDOM_GEN = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_/:"; + private static final Random RANDOM = new SecureRandom(); + + private static String toHexString(byte[] bytes) { + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } + + private static String generateRandomString(int length) { + if (length < 1) { + throw new IllegalArgumentException("Length must be greater than 0"); + } + StringBuilder builder = new StringBuilder(length); + for (int i = 0; i < length; i++) { + int randomIndex = RANDOM.nextInt(CHARACTERS_FOR_RANDOM_GEN.length()); + builder.append(CHARACTERS_FOR_RANDOM_GEN.charAt(randomIndex)); + } + return new String(getUTF8Bytes(builder.toString()), StandardCharsets.UTF_8); + } + @Test public void testStringJoinWithDelim() { String joinedString = StringUtils.joinUsingDelim("-", STRINGS); @@ -108,14 +136,6 @@ public class TestStringUtils { assertEquals(StringUtils.toHexString(getUTF8Bytes(str)), toHexString(getUTF8Bytes(str))); } - private static String toHexString(byte[] bytes) { - StringBuilder sb = new StringBuilder(bytes.length * 2); - for (byte b : bytes) { - sb.append(String.format("%02x", b)); - } - return sb.toString(); - } - @Test public void testTruncate() { assertNull(StringUtils.truncate(null, 10, 10)); @@ -129,7 +149,61 @@ public class TestStringUtils { assertTrue(StringUtils.compareVersions("1.9", "1.10") < 0); assertTrue(StringUtils.compareVersions("1.100.1", "1.10") > 0); assertTrue(StringUtils.compareVersions("1.10.1", "1.10") > 0); - assertTrue(StringUtils.compareVersions("1.10", "1.10") == 0); + assertEquals(0, StringUtils.compareVersions("1.10", "1.10")); + } + + @Test + void testConcatenateWithinThreshold() { + String a = generateRandomString(1000); // 1000 bytes in UTF-8 + String b = generateRandomString(1048); // 1048 bytes in UTF-8 + int threshold = 2048; + + // The total length of bytes of `a` + `b` exceeds the threshold + String result = StringUtils.concatenateWithThreshold(a, b, threshold); + + // The resulting string should be exactly `threshold` bytes long + assertEquals(threshold, getUTF8Bytes(result).length); + assertEquals(a + b, result); + + // Test case when a + b is within the threshold + String a2 = generateRandomString(900); + String b2 = generateRandomString(1000); + String result2 = concatenateWithThreshold(a2, b2, threshold); + + // The resulting string should be `a2 + b2` + assertEquals(a2 + b2, result2); + } + + @Test + void testConcatenateInvalidInput() { + // Test case when b alone exceeds the threshold + String a = generateRandomString(900); + String b = generateRandomString(3000); // 3000 bytes in UTF-8 + Exception exception = assertThrows(IllegalArgumentException.class, () -> { + concatenateWithThreshold(a, b, 2048); + }); + + String expectedMessage = "Length of the Second string to concatenate exceeds the threshold (3000 > 2048)"; + String actualMessage = exception.getMessage(); + + assertTrue(actualMessage.contains(expectedMessage)); + } + + @Test + void testConcatenateTruncateCase() { + // 'é' is 2 bytes + assertEquals("ad", concatenateWithThreshold("aé", "d", 3)); + // Chinese chars are 3 bytes + assertEquals("世d", concatenateWithThreshold("世界", "d", 4)); + assertEquals("ad", concatenateWithThreshold("ab", "d", 2)); + } + + @Test + void testGenerateInvalidRandomString() { + assertThrows( + IllegalArgumentException.class, + () -> generateRandomString(-1) + ); } @Test diff --git a/scripts/release/validate_source_copyright.sh b/scripts/release/validate_source_copyright.sh index d44864135be..deb79666269 100755 --- a/scripts/release/validate_source_copyright.sh +++ b/scripts/release/validate_source_copyright.sh @@ -46,10 +46,10 @@ echo -e "\t\tNotice file exists ? [OK]\n" ### Licensing Check echo "Performing custom Licensing Check " -numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.jpg' | grep -v '.json' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v emptyFile | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | grep -v 'banner.txt' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l` +numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.jpg' | grep -v '.json' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v emptyFile | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | grep -v 'banner.txt' | grep -v '.txt' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l` if [ "$numfilesWithNoLicense" -gt "0" ]; then echo "There were some source files that did not have Apache License [ERROR]" - find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.jpg' | grep -v '.json' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v emptyFile | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'banner.txt' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" + find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.jpg' | grep -v '.json' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v emptyFile | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'banner.txt' | grep -v '.txt' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" exit 1 fi echo -e "\t\tLicensing Check Passed [OK]\n"