[
https://issues.apache.org/jira/browse/HADOOP-16118?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16798037#comment-16798037
]
Steve Loughran commented on HADOOP-16118:
-----------------------------------------
the metastore scale tests are failing because throttling isn't detected any
more.
Proposed: Identify when store is PAYG and skip the scale tests. We know it will
scale to the limits of your AWS credit line and overall AZ load.
{code}
[ERROR] Tests run: 9, Failures: 5, Errors: 0, Skipped: 0, Time elapsed: 45.172
s <<< FAILURE! - in
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale
[ERROR]
test_040_get(org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale)
Time elapsed: 7.944 s <<< FAILURE!
java.lang.AssertionError: No throttling detected in Tracker with read throttle
events = 0; write events = 0; batch throttles = 0 against
DynamoDBMetadataStore{region=eu-west-1, tableName=hwdev-steve-ireland-new,
tableArn=arn:aws:dynamodb:eu-west-1:980678866538:table/hwdev-steve-ireland-new}
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale$ThrottleTracker.assertThrottlingDetected(ITestDynamoDBMetadataStoreScale.java:577)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale.execute(ITestDynamoDBMetadataStoreScale.java:464)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale.test_040_get(ITestDynamoDBMetadataStoreScale.java:275)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:745)
[ERROR]
test_050_getVersionMarkerItem(org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale)
Time elapsed: 7.924 s <<< FAILURE!
java.lang.AssertionError: No throttling detected in Tracker with read throttle
events = 0; write events = 0; batch throttles = 0 against
DynamoDBMetadataStore{region=eu-west-1, tableName=hwdev-steve-ireland-new,
tableArn=arn:aws:dynamodb:eu-west-1:980678866538:table/hwdev-steve-ireland-new}
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale$ThrottleTracker.assertThrottlingDetected(ITestDynamoDBMetadataStoreScale.java:577)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale.execute(ITestDynamoDBMetadataStoreScale.java:464)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale.test_050_getVersionMarkerItem(ITestDynamoDBMetadataStoreScale.java:290)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:745)
[ERROR]
test_060_list(org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale)
Time elapsed: 4.417 s <<< FAILURE!
java.lang.AssertionError: No throttling detected in Tracker with read throttle
events = 0; write events = 0; batch throttles = 0 against
DynamoDBMetadataStore{region=eu-west-1, tableName=hwdev-steve-ireland-new,
tableArn=arn:aws:dynamodb:eu-west-1:980678866538:table/hwdev-steve-ireland-new}
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale$ThrottleTracker.assertThrottlingDetected(ITestDynamoDBMetadataStoreScale.java:577)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale.execute(ITestDynamoDBMetadataStoreScale.java:464)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale.test_060_list(ITestDynamoDBMetadataStoreScale.java:320)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:745)
[ERROR]
test_080_fullPathsToPut(org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale)
Time elapsed: 4.451 s <<< FAILURE!
java.lang.AssertionError: No throttling detected in Tracker with read throttle
events = 0; write events = 0; batch throttles = 0 against
DynamoDBMetadataStore{region=eu-west-1, tableName=hwdev-steve-ireland-new,
tableArn=arn:aws:dynamodb:eu-west-1:980678866538:table/hwdev-steve-ireland-new}
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale$ThrottleTracker.assertThrottlingDetected(ITestDynamoDBMetadataStoreScale.java:577)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale.execute(ITestDynamoDBMetadataStoreScale.java:464)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale.test_080_fullPathsToPut(ITestDynamoDBMetadataStoreScale.java:363)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:745)
[ERROR]
test_900_instrumentation(org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale)
Time elapsed: 1.574 s <<< FAILURE!
java.lang.AssertionError: No increment of s3guard_metadatastore_retry in
S3AFileSystem{uri=s3a://hwdev-steve-ireland-new,
workingDir=s3a://hwdev-steve-ireland-new/user/stevel, inputPolicy=normal,
partSize=8388608, enableMultiObjectsDelete=true, maxKeys=5000,
readAhead=524288, blockSize=33554432, multiPartThreshold=2147483647,
serverSideEncryptionAlgorithm='NONE',
blockFactory=org.apache.hadoop.fs.s3a.S3ADataBlocks$DiskBlockFactory@7dfdf165,
metastore=DynamoDBMetadataStore{region=eu-west-1,
tableName=hwdev-steve-ireland-new,
tableArn=arn:aws:dynamodb:eu-west-1:980678866538:table/hwdev-steve-ireland-new},
authoritative=false, useListV1=false, magicCommitter=true,
boundedExecutor=BlockingThreadPoolExecutorService{SemaphoredDelegatingExecutor{permitCount=65,
available=65, waiting=0}, activeCount=0},
unboundedExecutor=java.util.concurrent.ThreadPoolExecutor@3586a1d3[Running,
pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0],
credentials=AWSCredentialProviderList[refcount= 2:
[TemporaryAWSCredentialsProvider, SimpleAWSCredentialsProvider,
EnvironmentVariableCredentialsProvider,
org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider@bab6086] last
provider: SimpleAWSCredentialsProvider, delegation tokens=disabled, statistics
{42095917 bytes read, 52437966 bytes written, 375 read ops, 0 large read ops,
522 write ops}, metrics {{Context=s3aFileSystem}
{s3aFileSystemId=29672135-ff7d-46b4-b277-a0445f30cfc3}
{bucket=hwdev-steve-ireland-new} {stream_opened=0} {stream_close_operations=0}
{stream_closed=0} {stream_aborted=0} {stream_seek_operations=0}
{stream_read_exceptions=0} {stream_forward_seek_operations=0}
{stream_backward_seek_operations=0} {stream_bytes_skipped_on_seek=0}
{stream_bytes_backwards_on_seek=0} {stream_bytes_read=0}
{stream_read_operations=0} {stream_read_fully_operations=0}
{stream_read_operations_incomplete=0} {stream_bytes_read_in_close=0}
{stream_bytes_discarded_in_abort=0} {files_created=0} {files_copied=0}
{files_copied_bytes=0} {files_deleted=0} {fake_directories_deleted=0}
{directories_created=1} {directories_deleted=0} {ignored_errors=0}
{op_copy_from_local_file=0} {op_create=0} {op_create_non_recursive=0}
{op_delete=0} {op_exists=0} {op_get_delegation_token=0}
{op_get_file_checksum=0} {op_get_file_status=2} {op_glob_status=0}
{op_is_directory=0} {op_is_file=0} {op_list_files=0} {op_list_located_status=0}
{op_list_status=0} {op_mkdirs=1} {op_open=0} {op_rename=0}
{object_copy_requests=0} {object_delete_requests=0} {object_list_requests=0}
{object_continue_list_requests=0} {object_metadata_requests=0}
{object_multipart_aborted=0} {object_put_bytes=0} {object_put_requests=1}
{object_put_requests_completed=1} {object_select_requests=0}
{stream_read_version_mismatches=0} {stream_write_failures=0}
{stream_write_block_uploads=0} {stream_write_block_uploads_committed=0}
{stream_write_block_uploads_aborted=0} {stream_write_total_time=0}
{stream_write_total_data=0} {committer_commits_created=0}
{committer_commits_completed=0} {committer_jobs_completed=0}
{committer_jobs_failed=0} {committer_tasks_completed=0}
{committer_tasks_failed=0} {committer_bytes_committed=0}
{committer_bytes_uploaded=0} {committer_commits_failed=0}
{committer_commits_aborted=0} {committer_commits_reverted=0}
{committer_magic_files_created=0} {s3guard_metadatastore_put_path_request=1}
{s3guard_metadatastore_initialization=1} {s3guard_metadatastore_retry=0}
{s3guard_metadatastore_throttled=0} {store_io_throttled=0}
{delegation_tokens_issued=0} {object_put_requests_active=0}
{object_put_bytes_pending=0} {stream_write_block_uploads_active=0}
{stream_write_block_uploads_pending=0}
{stream_write_block_uploads_data_pending=0}
{S3guard_metadatastore_put_path_latencyNumOps=0}
{S3guard_metadatastore_put_path_latency50thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency75thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency90thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency95thPercentileLatency=0}
{S3guard_metadatastore_put_path_latency99thPercentileLatency=0}
{S3guard_metadatastore_throttle_rateNumEvents=0}
{S3guard_metadatastore_throttle_rate50thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate75thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate90thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate95thPercentileFrequency (Hz)=0}
{S3guard_metadatastore_throttle_rate99thPercentileFrequency (Hz)=0} }}
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at
org.apache.hadoop.fs.s3a.s3guard.ITestDynamoDBMetadataStoreScale.test_900_instrumentation(ITestDynamoDBMetadataStoreScale.java:385)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:745)
{code}
> S3Guard to support on-demand DDB tables
> ---------------------------------------
>
> Key: HADOOP-16118
> URL: https://issues.apache.org/jira/browse/HADOOP-16118
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/s3
> Affects Versions: 3.3.0
> Reporter: Steve Loughran
> Priority: Major
>
> AWS now supports [on demand DDB
> capacity|https://aws.amazon.com/blogs/aws/amazon-dynamodb-on-demand-no-capacity-planning-and-pay-per-request-pricing/]
>
> This has lowest cost and best scalability, so could be the default capacity.
> + add a new option to set-capacity.
> Will depend on an SDK update: created HADOOP-16117.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
