fedorzipp opened a new issue, #12114:
URL: https://github.com/apache/iceberg/issues/12114
When I do close in try-with resources way I got
"java.lang.IllegalStateException: Connection pool shut down" on s3's connection
pool.
Using S3FileIO impl to store on S3.
Version: 1.7.1 (latest release).
```
try (FileAppender<Record> appender = createFileAppender(table,
outputFile)) {
records.forEach(appender::add);
} catch (final IOException ex) {
log.error("Cannot append records to table {}: {}", table.name(),
ex.getMessage(), ex);
throw new IllegalStateException("Cannot append records to table "
+ table.name(), ex);
}
FileAppender<Record> createFileAppender( final Table table, final
OutputFile outputFile) throws IOException {
return Parquet.write(outputFile)
.forTable(table)
.schema(table.schema())
.createWriterFunc(GenericParquetWriter::buildWriter)
.build();
}
}
```
```
Exception:
{
"class": "org.apache.http.util.Asserts",
"method": "check",
"line": 34
},
{
"class": "org.apache.http.impl.conn.PoolingHttpClientConnectionManager",
"method": "requestConnection",
"line": 269
},
{
"class":
"software.amazon.awssdk.http.apache.internal.conn.ClientConnectionManagerFactory$DelegatingHttpClientConnectionManager",
"method": "requestConnection",
"line": 75
},
{
"class":
"software.amazon.awssdk.http.apache.internal.conn.ClientConnectionManagerFactory$InstrumentedHttpClientConnectionManager",
"method": "requestConnection",
"line": 57
},
{
"class": "org.apache.http.impl.execchain.MainClientExec",
"method": "execute",
"line": 176
},
{
"class": "org.apache.http.impl.execchain.ProtocolExec",
"method": "execute",
"line": 186
},
{
"class": "org.apache.http.impl.client.InternalHttpClient",
"method": "doExecute",
"line": 185
},
{
"class": "org.apache.http.impl.client.CloseableHttpClient",
"method": "execute",
"line": 83
},
{
"class": "org.apache.http.impl.client.CloseableHttpClient",
"method": "execute",
"line": 56
},
{
"class":
"software.amazon.awssdk.http.apache.internal.impl.ApacheSdkHttpClient",
"method": "execute",
"line": 72
},
{
"class": "software.amazon.awssdk.http.apache.ApacheHttpClient",
"method": "execute",
"line": 254
},
{
"class": "software.amazon.awssdk.http.apache.ApacheHttpClient",
"method": "access$500",
"line": 104
},
{
"class": "software.amazon.awssdk.http.apache.ApacheHttpClient$1",
"method": "call",
"line": 231
},
{
"class": "software.amazon.awssdk.http.apache.ApacheHttpClient$1",
"method": "call",
"line": 228
},
{
"class": "software.amazon.awssdk.core.internal.util.MetricUtils",
"method": "measureDurationUnsafe",
"line": 99
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage",
"method": "executeHttpRequest",
"line": 79
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage",
"method": "execute",
"line": 57
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage",
"method": "execute",
"line": 40
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage",
"method": "execute",
"line": 206
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage",
"method": "execute",
"line": 206
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage",
"method": "execute",
"line": 206
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage",
"method": "execute",
"line": 206
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage",
"method": "execute",
"line": 72
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage",
"method": "execute",
"line": 42
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage",
"method": "execute",
"line": 78
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage",
"method": "execute",
"line": 40
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage",
"method": "execute",
"line": 55
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage",
"method": "execute",
"line": 39
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage",
"method": "execute",
"line": 81
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage",
"method": "execute",
"line": 36
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage",
"method": "execute",
"line": 206
},
{
"class": "software.amazon.awssdk.core.internal.http.StreamManagingStage",
"method": "execute",
"line": 56
},
{
"class": "software.amazon.awssdk.core.internal.http.StreamManagingStage",
"method": "execute",
"line": 36
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage",
"method": "executeWithTimer",
"line": 80
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage",
"method": "execute",
"line": 60
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage",
"method": "execute",
"line": 42
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage",
"method": "execute",
"line": 50
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage",
"method": "execute",
"line": 32
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage",
"method": "execute",
"line": 206
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage",
"method": "execute",
"line": 206
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage",
"method": "execute",
"line": 37
},
{
"class":
"software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage",
"method": "execute",
"line": 26
},
{
"class":
"software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl",
"method": "execute",
"line": 224
},
{
"class":
"software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler",
"method": "invoke",
"line": 103
},
{
"class":
"software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler",
"method": "doExecute",
"line": 173
},
{
"class":
"software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler",
"method": "lambda$execute$1",
"line": 80
},
{
"class":
"software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler",
"method": "measureApiCallSuccess",
"line": 182
},
{
"class":
"software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler",
"method": "execute",
"line": 74
},
{
"class": "software.amazon.awssdk.core.client.handler.SdkSyncClientHandler",
"method": "execute",
"line": 45
},
{
"class":
"software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler",
"method": "execute",
"line": 53
},
{
"class": "software.amazon.awssdk.services.sts.DefaultStsClient",
"method": "assumeRoleWithWebIdentity",
"line": 767
},
{
"class":
"software.amazon.awssdk.services.sts.auth.StsAssumeRoleWithWebIdentityCredentialsProvider",
"method": "getUpdatedCredentials",
"line": 74
},
{
"class": "software.amazon.awssdk.services.sts.auth.StsCredentialsProvider",
"method": "updateSessionCredentials",
"line": 92
},
{
"class": "software.amazon.awssdk.utils.cache.CachedSupplier",
"method": "lambda$jitteredPrefetchValueSupplier$8",
"line": 300
},
{
"class":
"software.amazon.awssdk.utils.cache.CachedSupplier$PrefetchStrategy",
"method": "fetch",
"line": 448
},
{
"class": "software.amazon.awssdk.utils.cache.CachedSupplier",
"method": "refreshCache",
"line": 208
},
{
"class": "software.amazon.awssdk.utils.cache.CachedSupplier",
"method": "get",
"line": 135
},
{
"class": "software.amazon.awssdk.services.sts.auth.StsCredentialsProvider",
"method": "resolveCredentials",
"line": 105
},
{
"class":
"software.amazon.awssdk.services.sts.internal.StsWebIdentityCredentialsProviderFactory$StsWebIdentityCredentialsProvider",
"method": "resolveCredentials",
"line": 109
},
{
"class":
"software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider",
"method": "resolveCredentials",
"line": 143
},
{
"class": "software.amazon.awssdk.auth.credentials.AwsCredentialsProvider",
"method": "resolveIdentity",
"line": 54
},
{
"class": "software.amazon.awssdk.identity.spi.IdentityProvider",
"method": "resolveIdentity",
"line": 60
},
{
"class":
"software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain",
"method": "resolveCredentials",
"line": 103
},
{
"class":
"software.amazon.awssdk.auth.credentials.internal.LazyAwsCredentialsProvider",
"method": "resolveCredentials",
"line": 45
},
{
"class":
"software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider",
"method": "resolveCredentials",
"line": 128
},
{
"class": "software.amazon.awssdk.auth.credentials.AwsCredentialsProvider",
"method": "resolveIdentity",
"line": 54
},
{
"class":
"software.amazon.awssdk.services.s3.auth.scheme.internal.S3AuthSchemeInterceptor",
"method": "lambda$trySelectAuthScheme$3",
"line": 152
},
{
"class": "software.amazon.awssdk.core.internal.util.MetricUtils",
"method": "reportDuration",
"line": 77
},
{
"class":
"software.amazon.awssdk.services.s3.auth.scheme.internal.S3AuthSchemeInterceptor",
"method": "trySelectAuthScheme",
"line": 152
},
{
"class":
"software.amazon.awssdk.services.s3.auth.scheme.internal.S3AuthSchemeInterceptor",
"method": "selectAuthScheme",
"line": 83
},
{
"class":
"software.amazon.awssdk.services.s3.auth.scheme.internal.S3AuthSchemeInterceptor",
"method": "beforeExecution",
"line": 63
},
{
"class":
"software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain",
"method": "lambda$beforeExecution$1",
"line": 59
},
{
"class": "java.util.ArrayList",
"method": "forEach",
"line": 1596
},
{
"class":
"software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain",
"method": "beforeExecution",
"line": 59
},
{
"class":
"software.amazon.awssdk.awscore.internal.AwsExecutionContextBuilder",
"method": "runInitialInterceptors",
"line": 239
},
{
"class":
"software.amazon.awssdk.awscore.internal.AwsExecutionContextBuilder",
"method": "invokeInterceptorsAndCreateExecutionContext",
"line": 130
},
{
"class":
"software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler",
"method": "invokeInterceptorsAndCreateExecutionContext",
"line": 67
},
{
"class":
"software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler",
"method": "lambda$execute$1",
"line": 76
},
{
"class":
"software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler",
"method": "measureApiCallSuccess",
"line": 182
},
{
"class":
"software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler",
"method": "execute",
"line": 74
},
{
"class": "software.amazon.awssdk.core.client.handler.SdkSyncClientHandler",
"method": "execute",
"line": 45
},
{
"class":
"software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler",
"method": "execute",
"line": 53
},
{
"class": "software.amazon.awssdk.services.s3.DefaultS3Client",
"method": "headObject",
"line": 6319
},
{
"class": "org.apache.iceberg.aws.s3.BaseS3File",
"method": "getObjectMetadata",
"line": 85
},
{
"class": "org.apache.iceberg.aws.s3.BaseS3File",
"method": "exists",
"line": 70
},
{
"class": "org.apache.iceberg.aws.s3.S3OutputFile",
"method": "exists",
"line": 32
},
{
"class": "org.apache.iceberg.aws.s3.S3OutputFile",
"method": "create",
"line": 60
},
{
"class": "org.apache.iceberg.parquet.ParquetIO$ParquetOutputFile",
"method": "create",
"line": 146
},
{
"class": "org.apache.parquet.hadoop.ParquetFileWriter",
"method": "<init>",
"line": 473
},
{
"class": "org.apache.parquet.hadoop.ParquetFileWriter",
"method": "<init>",
"line": 431
},
{
"class": "org.apache.iceberg.parquet.ParquetWriter",
"method": "ensureWriterInitialized",
"line": 111
},
{
"class": "org.apache.iceberg.parquet.ParquetWriter",
"method": "flushRowGroup",
"line": 210
},
{
"class": "org.apache.iceberg.parquet.ParquetWriter",
"method": "close",
"line": 254
},
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
