This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hive-test-datasets.git
The following commit(s) were added to refs/heads/main by this push:
new 983a9bb Add info about dataset location and repo status
983a9bb is described below
commit 983a9bbe4a683dc485112fd91ba1e1605203d7f5
Author: Stamatis Zampetakis <[email protected]>
AuthorDate: Thu Nov 5 09:42:03 2026 +0100
Add info about dataset location and repo status
---
README.md | 65 ++++-----------------------------------------------
metastore/cleanup.sql | 54 ------------------------------------------
2 files changed, 5 insertions(+), 114 deletions(-)
diff --git a/README.md b/README.md
index 08324c5..2262f1e 100644
--- a/README.md
+++ b/README.md
@@ -1,64 +1,9 @@
# Test datasets for Apache Hive
-A repository holding datasets and dumps that are used for testing Apache Hive.
-The data can be found in the GitHub release assets.
-They are not under version control since they are in general large and not
expected to change very often.
-In addition, some parts are in binary format so they cannot be easily
inspected by humans.
+Test datasets are stored and maintained under
[nightlies.apache.org](https://nightlies.apache.org/hive/datasets/).
-The datasets are **not** meant to be used/downloaded by end-users and are only
relevant for internal testing purposes.
-They are **not official** ASF release artifacts and they don't follow the ASF
release life-cycle.
-
-## Metastore database dumps
-
-A collection of database dumps obtained from various versions of Hive.
-
-#### Postgres
-
-The database dumps were obtained from a Hive metastore running Cloudera's Hive
3.1.3000 using the following command:
-
- pg_dump -h localhost -p 5432 -U hive -d metastore -Fc > meta.dump.gz
-
-Some manual post-processing has been performed to remove redundant information
from the dump using the [cleanup.sql](metastore/cleanup.sql) file.
-Nevertheless, the dumps may still contain additional databases and tables but
these are not actively used.
-
-##### TPC-DS
-
-Metastore dumps from various [TPC-DS](http://www.tpc.org/tpcds/) scale factors.
-*
[metastore_tpcds10tb_3_1_3000](https://github.com/apache/hive-test-datasets/releases/download/1.0/metastore_tpcds10tb_3_1_3000.dump.gz)
-*
[metastore_tpcds30tb_3_1_3000](https://github.com/apache/hive-test-datasets/releases/download/1.0/metastore_tpcds30tb_3_1_3000.dump.gz)
-
-## Iceberg
-
-### Metadata
-
-A collection of Iceberg table directories holding only metadata and statistics
(no data files) meant to be used mainly in planner tests.
-
-#### TPC-DS
-
-Metadata dumps from various [TPC-DS](http://www.tpc.org/tpcds/) scale factors.
-
-##### 10TB metadata from S3
-
-Metadata and stats obtained by downloading the content of the following S3
bucket (excluding data files and unrelated content):
-```
-export AWS_ACCESS_KEY_ID=****************
-export AWS_SECRET_ACCESS_KEY=************
-
-aws s3 sync
s3://dw-team-bucket/data/warehouse/tablespace/external/hive/tpcds_partitioned_iceberg_parquet_10000.db/
iceberg_s3_tpcds10tb --exclude "*" --include "*metadata*"
-cd iceberg_s3_tpcds10tb
-zip -r ../iceberg_s3_tpcds10tb.zip .
-```
-
-The DDL statements for querying the tables through Hive can be obtained by
running the following script inside the downloaded directory.
-
-```bash
-for f in `ls`; do \
- metafile=`ls $f/metadata/*metadata.json | sort | tail -1 | sed
's|.*metadata/||'`; \
- echo "CREATE EXTERNAL TABLE $f STORED BY
'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
TBLPROPERTIES('metadata_location'='s3://dw-team-bucket/data/warehouse/tablespace/external/hive/tpcds_partitioned_iceberg_parquet_10000.db/$f/metadata/$metafile');";
\
-done
-```
-
-WARNING: The script arbitrarily picks the *current* metadata file based on
alphanumeric ordering. Ideally, this information should be obtained from the
respective catalog.
-
-Download:
[iceberg_s3_tpcds10tb](https://github.com/apache/hive-test-datasets/releases/download/1.0/iceberg_s3_tpcds10tb.zip)
+This repository *is not used at the moment*.
+Historically it was meant to hold datasets under the release assets but this
approach raised legal concerns and was abandoned.
+For more context see
[LEGAL-738](https://issues.apache.org/jira/browse/LEGAL-738) and the discussion
in the [dev
list](https://lists.apache.org/thread.html/24j774wyz51sc83zr6b6o0mjv7ym01rq).
+The repository may be permanently removed in the near future.
diff --git a/metastore/cleanup.sql b/metastore/cleanup.sql
deleted file mode 100644
index 311a937..0000000
--- a/metastore/cleanup.sql
+++ /dev/null
@@ -1,54 +0,0 @@
--- Script for partially anonymizing and cleaning up the dump.
--- WARNING: The script is not guaranteed to cover any kind of metastore
--- (not even close). It is mostly suitable for cases encountered so far.
-
--- Statements to remove potentially sensitive URIs and usernames.
-UPDATE "CTLGS" SET "LOCATION_URI" =
REGEXP_REPLACE("LOCATION_URI",'(s3a|hdfs)://[^/]+','hdfs://localhost:40889');
-UPDATE "DBS" SET "DB_LOCATION_URI" =
REGEXP_REPLACE("DB_LOCATION_URI",'(s3a|hdfs)://[^/]+','hdfs://localhost:40889');
-UPDATE "DBS" SET "OWNER_NAME" = 'hive' WHERE "OWNER_NAME" IS NOT NULL AND
"OWNER_NAME" <> 'public';
-UPDATE "SDS" SET "LOCATION" =
REGEXP_REPLACE("LOCATION",'(s3a|hdfs)://[^/]+','hdfs://localhost:40889') WHERE
"LOCATION" IS NOT NULL;
-UPDATE "TBLS" SET "OWNER" = 'hive' WHERE "OWNER" IS NOT NULL AND "OWNER" <>
'public';
--- Statements to cleanup transactions related state in metastore
-TRUNCATE TABLE "HIVE_LOCKS" CASCADE ;
-TRUNCATE TABLE "NOTIFICATION_LOG";
-TRUNCATE TABLE "TXNS" CASCADE;
-TRUNCATE TABLE "TXN_TO_WRITE_ID"CASCADE;
-TRUNCATE TABLE "TXN_COMPONENTS" CASCADE;
-TRUNCATE TABLE "COMPLETED_TXN_COMPONENTS" CASCADE;
-TRUNCATE TABLE "NEXT_WRITE_ID" CASCADE;
-TRUNCATE TABLE "NEXT_LOCK_ID" CASCADE;
-TRUNCATE TABLE "COMPACTION_QUEUE" CASCADE;
-TRUNCATE TABLE "NEXT_COMPACTION_QUEUE_ID" CASCADE;
-TRUNCATE TABLE "COMPLETED_COMPACTIONS" CASCADE;
-TRUNCATE TABLE "AUX_TABLE" CASCADE;
-TRUNCATE TABLE "WRITE_SET" CASCADE;
-TRUNCATE TABLE "REPL_TXN_MAP" CASCADE;
-TRUNCATE TABLE "MATERIALIZATION_REBUILD_LOCKS" CASCADE;
-TRUNCATE TABLE "MIN_HISTORY_LEVEL" CASCADE;
-
-INSERT INTO "NEXT_LOCK_ID" VALUES(1);
-INSERT INTO "NEXT_COMPACTION_QUEUE_ID" VALUES(1);
-
-
--- Statements to replace Hive default database with tpcds
-DELETE FROM "TABLE_PARAMS" WHERE "TBL_ID" IN (SELECT "TBLS"."TBL_ID" FROM
"TBLS" WHERE EXISTS (SELECT 1 FROM "DBS" WHERE "DBS"."DB_ID"="TBLS"."DB_ID" AND
"NAME" = 'default'));
-DELETE FROM "TAB_COL_STATS" WHERE "TBL_ID" IN (SELECT "TBLS"."TBL_ID" FROM
"TBLS" WHERE EXISTS (SELECT 1 FROM "DBS" WHERE "DBS"."DB_ID"="TBLS"."DB_ID" AND
"NAME" = 'default'));
-DELETE FROM "MV_TABLES_USED" WHERE "TBL_ID" IN (SELECT "TBLS"."TBL_ID" FROM
"TBLS" WHERE EXISTS (SELECT 1 FROM "DBS" WHERE "DBS"."DB_ID"="TBLS"."DB_ID" AND
"NAME" = 'default'));
-DELETE FROM "TBL_PRIVS" WHERE "TBL_ID" IN (SELECT "TBLS"."TBL_ID" FROM "TBLS"
WHERE EXISTS (SELECT 1 FROM "DBS" WHERE "DBS"."DB_ID"="TBLS"."DB_ID" AND "NAME"
= 'default'));
-DELETE FROM "PARTITION_KEYS" WHERE "TBL_ID" IN (SELECT "TBLS"."TBL_ID" FROM
"TBLS" WHERE EXISTS (SELECT 1 FROM "DBS" WHERE "DBS"."DB_ID"="TBLS"."DB_ID" AND
"NAME" = 'default'));
-DELETE FROM "PARTITION_KEY_VALS" WHERE "PART_ID" IN (SELECT "PART_ID" FROM
"PARTITIONS" WHERE "TBL_ID" IN (SELECT "TBLS"."TBL_ID" FROM "TBLS" WHERE EXISTS
(SELECT 1 FROM "DBS" WHERE "DBS"."DB_ID"="TBLS"."DB_ID" AND "NAME" =
'default')));
-DELETE FROM "PARTITION_PARAMS" WHERE "PART_ID" IN (SELECT "PART_ID" FROM
"PARTITIONS" WHERE "TBL_ID" IN (SELECT "TBLS"."TBL_ID" FROM "TBLS" WHERE EXISTS
(SELECT 1 FROM "DBS" WHERE "DBS"."DB_ID"="TBLS"."DB_ID" AND "NAME" =
'default')));
-DELETE FROM "PART_COL_STATS" WHERE "PART_ID" IN (SELECT "PART_ID" FROM
"PARTITIONS" WHERE "TBL_ID" IN (SELECT "TBLS"."TBL_ID" FROM "TBLS" WHERE EXISTS
(SELECT 1 FROM "DBS" WHERE "DBS"."DB_ID"="TBLS"."DB_ID" AND "NAME" =
'default')));
-DELETE FROM "PARTITIONS" WHERE "TBL_ID" IN (SELECT "TBLS"."TBL_ID" FROM "TBLS"
WHERE EXISTS (SELECT 1 FROM "DBS" WHERE "DBS"."DB_ID"="TBLS"."DB_ID" AND "NAME"
= 'default'));
-DELETE FROM "KEY_CONSTRAINTS" WHERE "PARENT_TBL_ID" IN (SELECT "TBLS"."TBL_ID"
FROM "TBLS" WHERE EXISTS (SELECT 1 FROM "DBS" WHERE
"DBS"."DB_ID"="TBLS"."DB_ID" AND "NAME" = 'default'));
-DELETE FROM "KEY_CONSTRAINTS" WHERE "CHILD_TBL_ID" IN (SELECT "TBLS"."TBL_ID"
FROM "TBLS" WHERE EXISTS (SELECT 1 FROM "DBS" WHERE
"DBS"."DB_ID"="TBLS"."DB_ID" AND "NAME" = 'default'));
-
-DELETE FROM "TBLS" WHERE "DB_ID" IN (SELECT "DB_ID" FROM "DBS" WHERE "NAME" =
'default');
-
-UPDATE "TBLS" SET "DB_ID" = (SELECT "DB_ID" FROM "DBS" WHERE "NAME" =
'default') WHERE EXISTS (SELECT 1 FROM "DBS" WHERE "DBS"."DB_ID"="TBLS"."DB_ID"
AND "DBS"."NAME" LIKE 'tpcds%');
-
-TRUNCATE "DATABASE_PARAMS";
-DELETE FROM "DBS" WHERE "NAME" LIKE 'tpcds%';
-
-UPDATE "TAB_COL_STATS" SET "DB_NAME" = 'default' WHERE "DB_NAME" LIKE 'tpcds%';
-UPDATE "PART_COL_STATS" SET "DB_NAME" = 'default' WHERE "DB_NAME" LIKE
'tpcds%';