This is an automated email from the ASF dual-hosted git repository.
emaynard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new ec97c1b7e Upgrade Iceberg REST Spec to match Iceberg 1.8 (#1283)
ec97c1b7e is described below
commit ec97c1b7ecdce7a34b1a8bc109bcdfe1d3ce339f
Author: Eric Maynard <[email protected]>
AuthorDate: Wed Apr 30 12:10:47 2025 -0700
Upgrade Iceberg REST Spec to match Iceberg 1.8 (#1283)
* prep for review
* reset
* more changes
* fixes
* github action change
* another build change
* try api revert
* re-all
* custom type mappings, rebuild
* autolint
* polish
* yank custom types
* update
* autolint
* wip
* Revert build changes
* example
* autolint
---
api/iceberg-service/build.gradle.kts | 2 +
.../catalog/iceberg/IcebergCatalogAdapter.java | 5 +-
spec/iceberg-rest-catalog-open-api.yaml | 214 +++++++++++++--------
3 files changed, 134 insertions(+), 87 deletions(-)
diff --git a/api/iceberg-service/build.gradle.kts
b/api/iceberg-service/build.gradle.kts
index a940a1b5e..d1fdf0cc7 100644
--- a/api/iceberg-service/build.gradle.kts
+++ b/api/iceberg-service/build.gradle.kts
@@ -120,3 +120,5 @@ sourceSets {
tasks.named("javadoc") { dependsOn("jandex") }
tasks.named("processResources") { dependsOn("openApiGenerate") }
+
+tasks.named("openApiGenerate") { outputs.cacheIf { false } }
diff --git
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
index d1c930bf4..5baffa395 100644
---
a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
+++
b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java
@@ -365,6 +365,7 @@ public class IcebergCatalogAdapter
String namespace,
String table,
String accessDelegationMode,
+ String ifNoneMatchString,
String snapshots,
RealmContext realmContext,
SecurityContext securityContext) {
@@ -373,9 +374,7 @@ public class IcebergCatalogAdapter
Namespace ns = decodeNamespace(namespace);
TableIdentifier tableIdentifier = TableIdentifier.of(ns,
RESTUtil.decodeString(table));
- // TODO: Populate with header value from parameter once the generated
interface
- // contains the if-none-match header
- IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader(null);
+ IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader(ifNoneMatchString);
if (ifNoneMatch.isWildcard()) {
throw new BadRequestException("If-None-Match may not take the value of
'*'");
diff --git a/spec/iceberg-rest-catalog-open-api.yaml
b/spec/iceberg-rest-catalog-open-api.yaml
index b4adb6be9..728d0cb53 100644
--- a/spec/iceberg-rest-catalog-open-api.yaml
+++ b/spec/iceberg-rest-catalog-open-api.yaml
@@ -19,10 +19,10 @@
# CODE_COPIED_TO_POLARIS
-# Apache Iceberg Version: 1.7.1
+# Apache Iceberg Version: 1.8.1
---
-openapi: 3.0.3
+openapi: 3.1.1
info:
title: Apache Iceberg REST Catalog API
license:
@@ -934,6 +934,15 @@ paths:
key. For example, "urn:ietf:params:oauth:token-type:jwt=<JWT-token>".
parameters:
- $ref: '#/components/parameters/data-access'
+ - name: If-None-Match
+ in: header
+ description:
+ An optional header that allows the server to return 304 (Not
Modified) if the metadata
+ is current. The content is the value of the ETag received in a
CreateTableResponse or
+ LoadTableResponse.
+ required: false
+ schema:
+ type: string
- in: query
name: snapshots
description:
@@ -949,6 +958,10 @@ paths:
responses:
200:
$ref: '#/components/responses/LoadTableResponse'
+ 304:
+ description:
+ Not Modified - Based on the content of the 'If-None-Match' header
the table metadata has
+ not changed since.
400:
$ref: '#/components/responses/BadRequestErrorResponse'
401:
@@ -1877,6 +1890,14 @@ components:
type: integer
minimum: 1
+ etag:
+ name: ETag
+ in: header
+ description: Identifies a unique version of the table metadata.
+ required: false
+ schema:
+ type: string
+
##############################
# Application Schema Objects #
##############################
@@ -2052,6 +2073,10 @@ components:
type: boolean
doc:
type: string
+ initial-default:
+ $ref: "#/components/schemas/PrimitiveTypeValue"
+ write-default:
+ $ref: "#/components/schemas/PrimitiveTypeValue"
StructType:
type: object
@@ -2061,7 +2086,7 @@ components:
properties:
type:
type: string
- enum: ["struct"]
+ const: "struct"
fields:
type: array
items:
@@ -2077,7 +2102,7 @@ components:
properties:
type:
type: string
- enum: ["list"]
+ const: "list"
element-id:
type: integer
element:
@@ -2097,7 +2122,7 @@ components:
properties:
type:
type: string
- enum: ["map"]
+ const: "map"
key-id:
type: integer
key:
@@ -2169,7 +2194,7 @@ components:
properties:
type:
$ref: '#/components/schemas/ExpressionType'
- enum: ["true"]
+ const: "true"
FalseExpression:
type: object
@@ -2178,7 +2203,7 @@ components:
properties:
type:
$ref: '#/components/schemas/ExpressionType'
- enum: ["false"]
+ const: "false"
AndOrExpression:
type: object
@@ -2203,7 +2228,7 @@ components:
properties:
type:
$ref: '#/components/schemas/ExpressionType'
- enum: ["not"]
+ const: "not"
child:
$ref: '#/components/schemas/Expression'
@@ -2273,7 +2298,7 @@ components:
properties:
type:
type: string
- enum: ["transform"]
+ const: "transform"
transform:
$ref: '#/components/schemas/Transform'
term:
@@ -2648,6 +2673,7 @@ components:
set-partition-statistics:
'#/components/schemas/SetPartitionStatisticsUpdate'
remove-partition-statistics:
'#/components/schemas/RemovePartitionStatisticsUpdate'
remove-partition-specs:
'#/components/schemas/RemovePartitionSpecsUpdate'
+ enable-row-lineage: '#/components/schemas/EnableRowLineageUpdate'
type: object
required:
- action
@@ -2660,12 +2686,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- uuid
properties:
action:
type: string
- enum: ["assign-uuid"]
+ const: "assign-uuid"
uuid:
type: string
@@ -2673,12 +2698,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- format-version
properties:
action:
type: string
- enum: ["upgrade-format-version"]
+ const: "upgrade-format-version"
format-version:
type: integer
@@ -2686,28 +2710,32 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- schema
properties:
action:
type: string
- enum: ["add-schema"]
+ const: "add-schema"
schema:
$ref: '#/components/schemas/Schema'
last-column-id:
type: integer
- description: The highest assigned column ID for the table. This is
used to ensure columns are always assigned an unused ID when evolving schemas.
When omitted, it will be computed on the server side.
+ deprecated: true
+ description:
+ This optional field is **DEPRECATED for REMOVAL** since it more
safe to handle this internally,
+ and shouldn't be exposed to the clients.
+
+ The highest assigned column ID for the table. This is used to
ensure columns are always
+ assigned an unused ID when evolving schemas. When omitted, it will
be computed on the server side.
SetCurrentSchemaUpdate:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- schema-id
properties:
action:
type: string
- enum: ["set-current-schema"]
+ const: "set-current-schema"
schema-id:
type: integer
description: Schema ID to set as current, or -1 to set last added
schema
@@ -2716,12 +2744,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- spec
properties:
action:
type: string
- enum: ["add-spec"]
+ const: "add-spec"
spec:
$ref: '#/components/schemas/PartitionSpec'
@@ -2729,12 +2756,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- spec-id
properties:
action:
type: string
- enum: [ "set-default-spec" ]
+ const: "set-default-spec"
spec-id:
type: integer
description: Partition spec ID to set as the default, or -1 to set
last added spec
@@ -2743,12 +2769,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- sort-order
properties:
action:
type: string
- enum: [ "add-sort-order" ]
+ const: "add-sort-order"
sort-order:
$ref: '#/components/schemas/SortOrder'
@@ -2756,12 +2781,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- sort-order-id
properties:
action:
type: string
- enum: [ "set-default-sort-order" ]
+ const: "set-default-sort-order"
sort-order-id:
type: integer
description: Sort order ID to set as the default, or -1 to set last
added sort order
@@ -2770,12 +2794,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- snapshot
properties:
action:
type: string
- enum: [ "add-snapshot" ]
+ const: "add-snapshot"
snapshot:
$ref: '#/components/schemas/Snapshot'
@@ -2784,12 +2807,11 @@ components:
- $ref: '#/components/schemas/BaseUpdate'
- $ref: '#/components/schemas/SnapshotReference'
required:
- - action
- ref-name
properties:
action:
type: string
- enum: [ "set-snapshot-ref" ]
+ const: "set-snapshot-ref"
ref-name:
type: string
@@ -2797,12 +2819,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- snapshot-ids
properties:
action:
type: string
- enum: [ "remove-snapshots" ]
+ const: "remove-snapshots"
snapshot-ids:
type: array
items:
@@ -2813,12 +2834,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- ref-name
properties:
action:
type: string
- enum: [ "remove-snapshot-ref" ]
+ const: "remove-snapshot-ref"
ref-name:
type: string
@@ -2826,12 +2846,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- location
properties:
action:
type: string
- enum: [ "set-location" ]
+ const: "set-location"
location:
type: string
@@ -2839,12 +2858,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- updates
properties:
action:
type: string
- enum: [ "set-properties" ]
+ const: "set-properties"
updates:
type: object
additionalProperties:
@@ -2854,12 +2872,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- removals
properties:
action:
type: string
- enum: [ "remove-properties" ]
+ const: "remove-properties"
removals:
type: array
items:
@@ -2869,12 +2886,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- view-version
properties:
action:
type: string
- enum: [ "add-view-version" ]
+ const: "add-view-version"
view-version:
$ref: '#/components/schemas/ViewVersion'
@@ -2882,12 +2898,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- view-version-id
properties:
action:
type: string
- enum: [ "set-current-view-version" ]
+ const: "set-current-view-version"
view-version-id:
type: integer
description: The view version id to set as current, or -1 to set
last added view version id
@@ -2896,16 +2911,18 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- - snapshot-id
- statistics
properties:
action:
type: string
- enum: [ "set-statistics" ]
+ const: "set-statistics"
snapshot-id:
type: integer
format: int64
+ deprecated: true
+ description:
+ This optional field is **DEPRECATED for REMOVAL** since it
contains redundant information.
+ Clients should use the `statistics.snapshot-id` field instead.
statistics:
$ref: '#/components/schemas/StatisticsFile'
@@ -2913,12 +2930,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- snapshot-id
properties:
action:
type: string
- enum: [ "remove-statistics" ]
+ const: "remove-statistics"
snapshot-id:
type: integer
format: int64
@@ -2927,12 +2943,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- partition-statistics
properties:
action:
type: string
- enum: [ "set-partition-statistics" ]
+ const: "set-partition-statistics"
partition-statistics:
$ref: '#/components/schemas/PartitionStatisticsFile'
@@ -2940,12 +2955,11 @@ components:
allOf:
- $ref: '#/components/schemas/BaseUpdate'
required:
- - action
- snapshot-id
properties:
action:
type: string
- enum: [ "remove-partition-statistics" ]
+ const: "remove-partition-statistics"
snapshot-id:
type: integer
format: int64
@@ -2958,12 +2972,21 @@ components:
properties:
action:
type: string
- enum: [ "remove-partition-specs" ]
+ const: "remove-partition-specs"
spec-ids:
type: array
items:
type: integer
+ # Disabling Row Lineage is Forbidden
+ EnableRowLineageUpdate:
+ allOf:
+ - $ref: '#/components/schemas/BaseUpdate'
+ properties:
+ action:
+ type: string
+ const: "enable-row-lineage"
+
TableUpdate:
anyOf:
- $ref: '#/components/schemas/AssignUUIDUpdate'
@@ -2984,6 +3007,7 @@ components:
- $ref: '#/components/schemas/SetStatisticsUpdate'
- $ref: '#/components/schemas/RemoveStatisticsUpdate'
- $ref: '#/components/schemas/RemovePartitionSpecsUpdate'
+ - $ref: '#/components/schemas/EnableRowLineageUpdate'
ViewUpdate:
anyOf:
@@ -3009,35 +3033,35 @@ components:
assert-last-assigned-partition-id:
'#/components/schemas/AssertLastAssignedPartitionId'
assert-default-spec-id: '#/components/schemas/AssertDefaultSpecId'
assert-default-sort-order-id:
'#/components/schemas/AssertDefaultSortOrderId'
- oneOf:
- - $ref: '#/components/schemas/AssertCreate'
- - $ref: '#/components/schemas/AssertTableUUID'
- - $ref: '#/components/schemas/AssertRefSnapshotId'
- - $ref: '#/components/schemas/AssertLastAssignedFieldId'
- - $ref: '#/components/schemas/AssertCurrentSchemaId'
- - $ref: '#/components/schemas/AssertLastAssignedPartitionId'
- - $ref: '#/components/schemas/AssertDefaultSpecId'
- - $ref: '#/components/schemas/AssertDefaultSortOrderId'
+ properties:
+ type:
+ type: string
+ required:
+ - type
AssertCreate:
type: object
description: The table must not already exist; used for create
transactions
+ allOf:
+ - $ref: '#/components/schemas/TableRequirement'
required:
- type
properties:
type:
type: string
- enum: ["assert-create"]
+ const: "assert-create"
AssertTableUUID:
description: The table UUID must match the requirement's `uuid`
+ allOf:
+ - $ref: '#/components/schemas/TableRequirement'
required:
- type
- uuid
properties:
type:
type: string
- enum: ["assert-table-uuid"]
+ const: "assert-table-uuid"
uuid:
type: string
@@ -3045,14 +3069,15 @@ components:
description:
The table branch or tag identified by the requirement's `ref` must
reference the requirement's `snapshot-id`;
if `snapshot-id` is `null` or missing, the ref must not already exist
+ allOf:
+ - $ref: '#/components/schemas/TableRequirement'
required:
- - type
- ref
- snapshot-id
properties:
type:
type: string
- enum: [ "assert-ref-snapshot-id" ]
+ const: "assert-ref-snapshot-id"
ref:
type: string
snapshot-id:
@@ -3062,65 +3087,70 @@ components:
AssertLastAssignedFieldId:
description:
The table's last assigned column id must match the requirement's
`last-assigned-field-id`
+ allOf:
+ - $ref: '#/components/schemas/TableRequirement'
required:
- - type
- last-assigned-field-id
properties:
type:
type: string
- enum: [ "assert-last-assigned-field-id" ]
+ const: "assert-last-assigned-field-id"
last-assigned-field-id:
type: integer
AssertCurrentSchemaId:
description:
The table's current schema id must match the requirement's
`current-schema-id`
+ allOf:
+ - $ref: '#/components/schemas/TableRequirement'
required:
- - type
- current-schema-id
properties:
type:
type: string
- enum: [ "assert-current-schema-id" ]
+ const: "assert-current-schema-id"
current-schema-id:
type: integer
AssertLastAssignedPartitionId:
description:
The table's last assigned partition id must match the requirement's
`last-assigned-partition-id`
+ allOf:
+ - $ref: '#/components/schemas/TableRequirement'
required:
- - type
- last-assigned-partition-id
properties:
type:
type: string
- enum: [ "assert-last-assigned-partition-id" ]
+ const: "assert-last-assigned-partition-id"
last-assigned-partition-id:
type: integer
AssertDefaultSpecId:
description:
The table's default spec id must match the requirement's
`default-spec-id`
+ allOf:
+ - $ref: '#/components/schemas/TableRequirement'
required:
- - type
- default-spec-id
properties:
type:
type: string
- enum: [ "assert-default-spec-id" ]
+ const: "assert-default-spec-id"
default-spec-id:
type: integer
AssertDefaultSortOrderId:
description:
The table's default sort order id must match the requirement's
`default-sort-order-id`
+ allOf:
+ - $ref: '#/components/schemas/TableRequirement'
required:
- - type
- default-sort-order-id
properties:
type:
type: string
- enum: [ "assert-default-sort-order-id" ]
+ const: "assert-default-sort-order-id"
default-sort-order-id:
type: integer
@@ -3141,7 +3171,7 @@ components:
properties:
type:
type: string
- enum: [ "assert-view-uuid" ]
+ const: "assert-view-uuid"
uuid:
type: string
@@ -3196,6 +3226,7 @@ components:
- `s3.secret-access-key`: secret for credentials that provide access
to data in S3
- `s3.session-token`: if present, this value should be used for as
the session token
- `s3.remote-signing-enabled`: if `true` remote signing should be
performed as described in the `s3-signer-open-api.yaml` specification
+ - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket
access is enabled
## Storage Credentials
@@ -3264,7 +3295,7 @@ components:
properties:
status:
$ref: '#/components/schemas/PlanStatus'
- enum: ["completed"]
+ const: "completed"
CompletedPlanningWithIDResult:
type: object
@@ -3287,7 +3318,7 @@ components:
properties:
status:
$ref: '#/components/schemas/PlanStatus'
- enum: ["failed"]
+ const: "failed"
AsyncPlanningResult:
type: object
@@ -3296,7 +3327,7 @@ components:
properties:
status:
$ref: '#/components/schemas/PlanStatus'
- enum: ["submitted"]
+ const: "submitted"
plan-id:
description: ID used to track a planning request
type: string
@@ -4136,6 +4167,7 @@ components:
- avro
- orc
- parquet
+ - puffin
ContentFile:
discriminator:
@@ -4200,7 +4232,7 @@ components:
properties:
content:
type: string
- enum: [ "data" ]
+ const: "data"
column-sizes:
allOf:
- $ref: '#/components/schemas/CountMap'
@@ -4244,7 +4276,15 @@ components:
properties:
content:
type: string
- enum: [ "position-deletes" ]
+ const: "position-deletes"
+ content-offset:
+ type: integer
+ format: int64
+ description: Offset within the delete file of delete content
+ content-size-in-bytes:
+ type: integer
+ format: int64
+ description: Length, in bytes, of the delete content; required if
content-offset is present
EqualityDeleteFile:
allOf:
@@ -4254,7 +4294,7 @@ components:
properties:
content:
type: string
- enum: [ "equality-deletes" ]
+ const: "equality-deletes"
equality-ids:
type: array
items:
@@ -4576,6 +4616,9 @@ components:
application/json:
schema:
$ref: '#/components/schemas/LoadTableResult'
+ headers:
+ etag:
+ $ref: '#/components/parameters/etag'
PlanTableScanResponse:
description: Result of submitting a table scan to plan
@@ -4604,6 +4647,9 @@ components:
application/json:
schema:
$ref: '#/components/schemas/LoadTableResult'
+ headers:
+ etag:
+ $ref: '#/components/parameters/etag'
LoadViewResponse:
description: View metadata result when loading a view
