This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 8cefa386f8 Add Fast Mode for Documentation Builds (#14267)
8cefa386f8 is described below
commit 8cefa386f8ed53585f024e729e33fd3d194712f3
Author: Talat UYARER <[email protected]>
AuthorDate: Thu Nov 6 07:54:49 2025 -0800
Add Fast Mode for Documentation Builds (#14267)
---------
Co-authored-by: Kevin Liu <[email protected]>
---
site/Makefile | 4 ++
site/README.md | 20 +++++++
site/dev/common.sh | 49 +++++++++++------
site/{Makefile => dev/serve-dev.sh} | 37 ++++++-------
site/mkdocs-dev.yml | 107 ++++++++++++++++++++++++++++++++++++
5 files changed, 178 insertions(+), 39 deletions(-)
diff --git a/site/Makefile b/site/Makefile
index 14c73134f1..5dbb8b6f24 100755
--- a/site/Makefile
+++ b/site/Makefile
@@ -21,6 +21,10 @@ help: # Show help for each of the Makefile recipes.
serve: # Clean, build, and run the docs site locally.
dev/serve.sh
+.PHONY: serve-dev
+serve-dev:
+ dev/serve-dev.sh
+
.PHONY: build
build: # Clean and build the docs site locally.
dev/build.sh
diff --git a/site/README.md b/site/README.md
index aae719fcbf..c0fd351adc 100644
--- a/site/README.md
+++ b/site/README.md
@@ -75,6 +75,7 @@ The docs are built, run, and released using
[make](https://www.gnu.org/software/
> [deploy](dev/deploy.sh): Clean, build, and deploy the Iceberg docs site.
> help: Show help for each of the Makefile recipes.
> [serve](dev/serve.sh): Clean, build, and run the site locally.
+> [serve-dev](dev/serve-dev.sh): Fast iterative development mode - only builds
nightly and latest.
> [lint](dev/lint.sh): Scan markdown files for style issues.
> [lint-fix](dev/lint.sh): Run linting with auto-fix on the markdown files.
@@ -129,6 +130,25 @@ To clear all build files, run `clean`.
make clean
```
+#### Fast iterative development mode
+
+When working on the documentation, building all historical versions
significantly slows down the build process. For faster iteration during
development, use the `serve-dev` recipe:
+
+```sh
+make serve-dev
+```
+
+This development mode:
+- **Only builds `nightly` and `latest` versions** - Skips all historical
versions
+- **Significantly reduces build time** - Typically 5-10x faster than building
all versions
+- **Uses the `--dirty` flag** - Only rebuilds changed files for even faster
iteration
+- **Perfect for iterative development** - Great for working on documentation
content
+
+The development mode sets the `ICEBERG_DEV_MODE=true` environment variable and
uses a simplified mkdocs configuration (`mkdocs-dev.yml`) that only includes
the most recent versions.
+
+> [!NOTE]
+> Development mode is only for local iteration. Always use `make serve` or
`make build` before creating a pull request to ensure all versioned docs build
correctly.
+
#### Testing local changes on versioned docs
When you build the docs as described above, by default the versioned docs are
mounted from the upstream remote repository called `iceberg_docs`. One
exception is the `nightly` version that is a soft link to the local `docs/`
folder.
diff --git a/site/dev/common.sh b/site/dev/common.sh
index 2788ce1e86..147c17d225 100755
--- a/site/dev/common.sh
+++ b/site/dev/common.sh
@@ -92,14 +92,11 @@ create_nightly () {
cd -
}
-# Finds and retrieves the latest version of the documentation based on the
directory structure.
-# Assumes the documentation versions are numeric folders within 'docs/docs/'.
+# Finds and retrieves the latest version of the documentation from mkdocs.yml.
+# Reads the icebergVersion from the extra section of mkdocs.yml.
get_latest_version () {
- # Find the latest numeric folder within 'docs/docs/' structure
- local latest=$(ls -d docs/docs/[0-9]* | sort -V | tail -1)
-
- # Extract the version number from the latest directory path
- local latest_version=$(basename "${latest}")
+ # Extract the icebergVersion from mkdocs.yml in the site directory
+ local latest_version=$(grep "icebergVersion:" mkdocs.yml | sed -E
"s/.*icebergVersion:[[:space:]]*['\"]?([^'\"]+)['\"]?.*/\1/")
# Output the latest version number
echo "${latest_version}"
@@ -168,29 +165,45 @@ update_version () {
# Sets up local worktrees for the documentation and performs operations
related to different versions.
pull_versioned_docs () {
+ # Retrieve the latest version of documentation for processing
+ local latest_version=$(get_latest_version)
+
+ # Output the latest version for debugging purposes
+ echo "Latest version is: ${latest_version}"
+
echo " --> pull versioned docs"
-
- # Ensure the remote repository for documentation exists and is up-to-date
- create_or_update_docs_remote
+ # Ensure the remote repository for documentation exists and is up-to-date
+ create_or_update_docs_remote
+
# Add local worktrees for documentation and javadoc either from the remote
repository
# or from a local branch.
local docs_branch="${ICEBERG_VERSIONED_DOCS_BRANCH:-${REMOTE}/docs}"
local javadoc_branch="${ICEBERG_VERSIONED_JAVADOC_BRANCH:-${REMOTE}/javadoc}"
- git worktree add -f docs/docs "${docs_branch}"
- git worktree add -f docs/javadoc "${javadoc_branch}"
-
- # Retrieve the latest version of documentation for processing
- local latest_version=$(get_latest_version)
- # Output the latest version for debugging purposes
- echo "Latest version is: ${latest_version}"
+ # Check if running in dev mode (only build nightly and latest for faster
iteration)
+ if [ "${ICEBERG_DEV_MODE:-false}" = "true" ]; then
+ echo " --> running in DEV MODE - only building nightly and latest"
+ echo " --> This significantly reduces build time by skipping historical
versions"
+
+ # Create docs worktree with sparse checkout for latest version only
+ git worktree add --no-checkout -f docs/docs "${docs_branch}"
+ (cd docs/docs && git sparse-checkout init --cone && git sparse-checkout
set "${latest_version}" && git checkout)
+
+ # Create javadoc worktree with sparse checkout for latest version only
+ git worktree add --no-checkout -f docs/javadoc "${javadoc_branch}"
+ (cd docs/javadoc && git sparse-checkout init --cone && git sparse-checkout
set "${latest_version}" && git checkout)
+ else
+ # Full checkout of all versions
+ git worktree add -f docs/docs "${docs_branch}"
+ git worktree add -f docs/javadoc "${javadoc_branch}"
+ fi
# Create the 'latest' version of documentation
create_latest "${latest_version}"
# Create the 'nightly' version of documentation
- create_nightly
+ create_nightly
}
check_markdown_files () {
diff --git a/site/Makefile b/site/dev/serve-dev.sh
similarity index 52%
copy from site/Makefile
copy to site/dev/serve-dev.sh
index 14c73134f1..8e3ef3d1ac 100755
--- a/site/Makefile
+++ b/site/dev/serve-dev.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -12,31 +14,24 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#
-.PHONY: help
-help: # Show help for each of the Makefile recipes.
- @grep -E '^[a-zA-Z0-9 -]+:.*#' Makefile | sort | while read -r l; do
printf "\033[1;32m$$(echo $$l | cut -f 1 -d':')\033[00m:$$(echo $$l | cut -f 2-
-d'#')\n"; done
+# Development mode serve script - only builds nightly and latest for fast
iteration
-.PHONY: serve
-serve: # Clean, build, and run the docs site locally.
- dev/serve.sh
+source dev/common.sh
-.PHONY: build
-build: # Clean and build the docs site locally.
- dev/build.sh
+set -e
-.PHONY: deploy
-deploy: # Clean, build, and deploy the Iceberg docs site.
- dev/deploy.sh $(remote_name)
+export ICEBERG_DEV_MODE=true
-.PHONY: lint
-lint: # Check linting on the docs.
- dev/lint.sh
+echo "=========================================="
+echo "RUNNING IN DEVELOPMENT MODE"
+echo "Only building nightly and latest versions"
+echo "=========================================="
+echo ""
-.PHONY: lint-fix
-lint-fix: # Run linting with auto-fix on the docs.
- dev/lint.sh --fix
+./dev/setup_env.sh
-.PHONY: clean
-clean: # Clean the local docs site.
- dev/clean.sh
+# Using mkdocs serve with --dirty flag for even faster rebuilds
+# The --dirty flag means only changed files are rebuilt
+"${VENV_DIR}/bin/python3" -m mkdocs serve --dirty --watch . -f mkdocs-dev.yml
diff --git a/site/mkdocs-dev.yml b/site/mkdocs-dev.yml
new file mode 100644
index 0000000000..5466094de4
--- /dev/null
+++ b/site/mkdocs-dev.yml
@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This is a development navigation configuration that only includes
+# nightly and latest documentation for faster build times during
+# iterative development. Use this with ICEBERG_DEV_MODE=true.
+
+INHERIT: ./mkdocs.yml
+
+nav:
+ - Home: index.md
+ - Quickstart:
+ - Spark: spark-quickstart.md
+ - Hive: hive-quickstart.md
+ - Docs:
+ - Java:
+ - Nightly: '!include docs/docs/nightly/mkdocs.yml'
+ - Latest (1.10.0): '!include docs/docs/latest/mkdocs.yml'
+ - Other Implementations:
+ - Python: https://py.iceberg.apache.org/
+ - Rust: https://rust.iceberg.apache.org/
+ - Go: https://go.iceberg.apache.org/
+ - C++: https://github.com/apache/iceberg-cpp/
+ - Third-party:
+ - Catalogs:
+ - Apache Gravitino: https://gravitino.apache.org/
+ - Apache Polaris: https://polaris.apache.org/
+ - Boring Catalog: https://github.com/boringdata/boring-catalog
+ - DataHub: https://docs.datahub.com/docs/iceberg-catalog
+ - Google BigLake metastore:
https://cloud.google.com/bigquery/docs/blms-manage-resources
+ - Lakekeeper: https://github.com/lakekeeper/lakekeeper
+ - Integrations:
+ - Amazon Athena:
https://docs.aws.amazon.com/athena/latest/ug/querying-iceberg.html
+ - Amazon Data Firehose:
https://docs.aws.amazon.com/firehose/latest/dev/apache-iceberg-destination.html
+ - Amazon EMR:
https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-iceberg-use-cluster.html
+ - Amazon Redshift:
https://docs.aws.amazon.com/redshift/latest/dg/querying-iceberg.html
+ - Apache Amoro: integrations/amoro.md
+ - Apache Doris:
https://doris.apache.org/docs/dev/lakehouse/catalogs/iceberg-catalog
+ - Apache Druid:
https://druid.apache.org/docs/latest/development/extensions-contrib/iceberg/
+ - BladePipe:
https://www.bladepipe.com/docs/dataMigrationAndSync/datasource_func/Iceberg/props_for_iceberg_ds
+ - ClickHouse:
https://clickhouse.com/docs/en/engines/table-engines/integrations/iceberg
+ - Daft: integrations/daft.md
+ - Databend: https://docs.databend.com/guides/access-data-lake/iceberg
+ - Dremio: https://docs.dremio.com/data-formats/apache-iceberg/
+ - DuckDB:
https://duckdb.org/docs/preview/core_extensions/iceberg/overview
+ - Estuary:
https://docs.estuary.dev/reference/Connectors/materialization-connectors/apache-iceberg/
+ - Firebolt:
https://docs.firebolt.io/reference-sql/functions-reference/table-valued/read_iceberg
+ - Google BigQuery:
https://cloud.google.com/bigquery/docs/iceberg-tables
+ - Impala:
https://impala.apache.org/docs/build/html/topics/impala_iceberg.html
+ - Memiiso Debezium: https://memiiso.github.io/debezium-server-iceberg/
+ - Nimtable: https://github.com/nimtable/nimtable
+ - OLake: https://olake.io/docs
+ - Presto: https://prestodb.io/docs/current/connector/iceberg.html
+ - Redpanda:
https://docs.redpanda.com/current/manage/iceberg/about-iceberg-topics
+ - RisingWave: integrations/risingwave.md
+ - Snowflake: https://docs.snowflake.com/en/user-guide/tables-iceberg
+ - Starburst: https://docs.starburst.io/latest/connector/iceberg.html
+ - Starrocks:
https://docs.starrocks.io/en-us/latest/data_source/catalog/iceberg_catalog
+ - Tinybird:
https://www.tinybird.co/docs/forward/get-data-in/table-functions/iceberg
+ - Trino: https://trino.io/docs/current/connector/iceberg.html
+ - Releases: releases.md
+ - Project:
+ - Contributing: contribute.md
+ - Multi-engine support: multi-engine-support.md
+ - How to release: how-to-release.md
+ - ASF:
+ - Sponsorship: https://www.apache.org/foundation/sponsorship.html
+ - Events: https://www.apache.org/events/current-event.html
+ - Privacy: https://privacy.apache.org/policies/privacy-policy-public.html
+ - License: https://www.apache.org/licenses/
+ - Security: https://www.apache.org/security/
+ - Sponsors: https://www.apache.org/foundation/sponsors.html
+ - Community:
+ - Community: community.md
+ - Talks: talks.md
+ - Vendors: vendors.md
+ - Specification:
+ - Terms: terms.md
+ - REST Catalog Spec: rest-catalog-spec.md
+ - Table Spec: spec.md
+ - View spec: view-spec.md
+ - Puffin spec: puffin-spec.md
+ - AES GCM Stream spec: gcm-stream-spec.md
+ - Implementation status: status.md
+
+exclude_docs: |
+ !.asf.yaml
+ docs/
+ javadoc/
+ !docs/nightly/
+ !javadoc/nightly/
+ !docs/latest/
+ !javadoc/latest/
