From 53c1799cc731b1c254cd3fe185c12532367e61c1 Mon Sep 17 00:00:00 2001 From: "Andrey M. Borodin" Date: Sun, 20 Aug 2023 23:55:31 +0300 Subject: [PATCH v19] Implement UUID v7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds function for UUID generation. Most important function here is uuidv7() which generates new UUID according to the new standard. For code readability this commit adds alias uuidv4() to function gen_random_uuid(). Also we add a function to extract timestamp from UUID v1, v6 and v7. To allow user to distinguish various UUID versions and variants we add functions uuid_extract_version() and uuid_extract_variant(). Author: Andrey Borodin Reviewers: Sergey Prokhorenko, Kirk Wolak, Przemysław Sztoch Reviewers: Nikolay Samokhvalov, Jelte Fennema-Nio, Aleksander Alekseev Reviewers: Peter Eisentraut, Chris Travers, Lukas Fittl Discussion: https://postgr.es/m/CAAhFRxitJv%3DyoGnXUgeLB_O%2BM7J2BJAmb5jqAT9gZ3bij3uLDA%40mail.gmail.com --- doc/src/sgml/func.sgml | 54 ++++- src/backend/utils/adt/uuid.c | 278 +++++++++++++++++++++++ src/include/catalog/pg_proc.dat | 15 ++ src/include/datatype/timestamp.h | 3 +- src/test/regress/expected/opr_sanity.out | 5 + src/test/regress/expected/uuid.out | 71 ++++++ src/test/regress/sql/uuid.sql | 26 +++ 7 files changed, 448 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 5030a1045f..34977163cf 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -14130,13 +14130,61 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple gen_random_uuid + + uuidv4 + + + + uuidv7 + + + + uuid_extract_timestamp + + + + uuid_extract_version + + + + uuid_extract_variant + + - PostgreSQL includes one function to generate a UUID: + PostgreSQL includes several functions to generate a UUID. gen_random_uuid () uuid +uuidv4 () uuid + + These functions return a version 4 (random) UUID. + +uuidv7 () uuid + + This function returns a version 7 UUID (UNIX timestamp with 1ms precision + + randomly seeded counter + random). + + + + +uuid_extract_timestamp (uuid) timestamptz + + This function extracts a timestamp with time zone from UUID versions 1, 6 and 7. + For other versions and variants this function returns NULL. The extracted timestamp + does not necessarily equate to the time of UUID generation. How close it is + to the actual time depends on the implementation that generated to UUID. + The uuidv7() function provided by PostgreSQL will normally store the actual time, + with some exceptions: prevention of time leaps backwards and counter overflow + being carried to time step. + +uuid_extract_version (uuid) smallint + + This function extracts a version bits from UUID of variant described by + IETF standard draft + (b10xx variant). For other variants this function returns NULL. + +uuid_extract_variant (uuid) smallint - This function returns a version 4 (random) UUID. This is the most commonly - used type of UUID and is appropriate for most applications. + This function extracts a vartiant bits from UUID. diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index 73dfd711c7..72faec3e66 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -13,13 +13,18 @@ #include "postgres.h" +#include + +#include "access/xlog.h" #include "common/hashfn.h" #include "lib/hyperloglog.h" #include "libpq/pqformat.h" #include "port/pg_bswap.h" #include "utils/builtins.h" +#include "utils/datetime.h" #include "utils/guc.h" #include "utils/sortsupport.h" +#include "utils/timestamp.h" #include "utils/uuid.h" /* sortsupport for uuid */ @@ -402,6 +407,11 @@ uuid_hash_extended(PG_FUNCTION_ARGS) return hash_any_extended(key->data, UUID_LEN, PG_GETARG_INT64(1)); } +/* + * Routine to generate UUID version 4. + * All UUID bytes are filled with strong random numbers except version and + * variant 0b10 bits. + */ Datum gen_random_uuid(PG_FUNCTION_ARGS) { @@ -421,3 +431,271 @@ gen_random_uuid(PG_FUNCTION_ARGS) PG_RETURN_UUID_P(uuid); } + +static uint32_t sequence_counter; +static uint64_t previous_timestamp = 0; + +/* + * Routine to generate UUID version 7. + * Following description is taken from RFC draft and slightly extended to + * reflect implementation specific choices. + * + * UUIDv7 Field and Bit Layout: + * ---------- + * 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | unix_ts_ms | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | unix_ts_ms | ver | rand_a | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |var| rand_b | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | rand_b | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * unix_ts_ms: + * 48 bit big-endian unsigned number of Unix epoch timestamp in milliseconds + * as per Section 6.1. Occupies bits 0 through 47 (octets 0-5). + * + * ver: + * The 4 bit version field as defined by Section 4.2, set to 0b0111 (7). + * Occupies bits 48 through 51 of octet 6. + * + * rand_a: + * Most significant 12 bits of 18-bit counter. This counter is designed to + * guarantee additional monotonicity as per Section 6.2 (Method 2). rand_a + * occupies bits 52 through 63 (octets 6-7). + * + * var: + * The 2 bit variant field as defined by Section 4.1, set to 0b10. Occupies + * bits 64 and 65 of octet 8. + * + * rand_b: + * Starting 6 bits are least significant 6 bits of a counter. The final 56 + * bits filled with pseudo-random data to provide uniqueness as per + * Section 6.9. rand_b Occupies bits 66 through 127 (octets 8-15). + * ---------- + * + * Monotonic Random (Method 2) can be implemented with arbitrary size of a + * counter. We choose size 18 to reuse all space of bytes that are touched by + * ver and var fields + rand_a bytes between them. + * Whenever timestamp unix_ts_ms is moving forward, this counter bits are + * reinitialized. Rinilialization always sets most significant bit to 0, other + * bits are initialized with random numbers. This gives as approximately 192K + * UUIDs within one millisecond without overflow. Outh to be enough. + * Whenever counter overflow happens, this overflow is translated to increment + * of unix_ts_ms. So generation of UUIDs ate rate higher than 128MHz might lead + * to using timestamps ahead of time. + * + * All UUID generator state is backend-local. For UUIDs generated in one + * backend we guarantee monotonicity. UUIDs generated on different backends + * will be mostly monotonic if they are generated at frequences less than 1KHz, + * but this monotonicity is not strictly guaranteed. UUIDs generated on + * different nodes are mostly monotonic with regards to possible clock drift. + */ +Datum +uuidv7(PG_FUNCTION_ARGS) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + uint64_t tms; + struct timeval tp; + bool increment_counter; + + gettimeofday(&tp, NULL); + tms = ((uint64_t)tp.tv_sec) * 1000 + (tp.tv_usec) / 1000; + /* time from clock is protected from backward leaps */ + increment_counter = (tms <= previous_timestamp); + + if (increment_counter) + { + /* Time did not advance from the previous generation, we must increment counter */ + ++sequence_counter; + if (sequence_counter > 0x3ffff) + { + /* We only have 18-bit counter */ + sequence_counter = 0; + previous_timestamp++; + } + + /* protection from leap backward */ + tms = previous_timestamp; + + /* fill everything after the timestamp and counter with random bytes */ + if (!pg_strong_random(&uuid->data[8], UUID_LEN - 8)) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate random values"))); + + /* most significant 4 bits of 18-bit counter */ + uuid->data[6] = (unsigned char)(sequence_counter >> 14); + /* next 8 bits */ + uuid->data[7] = (unsigned char)(sequence_counter >> 6); + /* least significant 6 bits */ + uuid->data[8] = (unsigned char)(sequence_counter); + } + else + { + /* fill everything after the timestamp with random bytes */ + if (!pg_strong_random(&uuid->data[6], UUID_LEN - 6)) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate random values"))); + + /* + * Left-most counter bits are initialized as zero for the sole purpose + * of guarding against counter rollovers. + * See section "Fixed-Length Dedicated Counter Seeding" + * https://datatracker.ietf.org/doc/html/draft-ietf-uuidrev-rfc4122bis-09#monotonicity_counters + */ + uuid->data[6] = (uuid->data[6] & 0xf7); + + /* read randomly initialized bits of counter */ + sequence_counter = ((uint32_t)uuid->data[8] & 0x3f) + + (((uint32_t)uuid->data[7]) << 6) + + (((uint32_t)uuid->data[6] & 0x0f) << 14); + + previous_timestamp = tms; + } + + /* Fill in time part */ + uuid->data[0] = (unsigned char)(tms >> 40); + uuid->data[1] = (unsigned char)(tms >> 32); + uuid->data[2] = (unsigned char)(tms >> 24); + uuid->data[3] = (unsigned char)(tms >> 16); + uuid->data[4] = (unsigned char)(tms >> 8); + uuid->data[5] = (unsigned char)tms; + + /* + * Set magic numbers for a "version 7" (pseudorandom) UUID, see + * https://datatracker.ietf.org/doc/html/draft-ietf-uuidrev-rfc4122bis + */ + /* set version field, top four bits are 0, 1, 1, 1 */ + uuid->data[6] = (uuid->data[6] & 0x0f) | 0x70; + /* set variant field, top two bits are 1, 0 */ + uuid->data[8] = (uuid->data[8] & 0x3f) | 0x80; + + PG_RETURN_UUID_P(uuid); +} + +/* + * Routine to extract UUID version from variant 0b10 + * Returns NULL if UUID is not 0b10 or version is not 1,6 or7. + */ +Datum +uuid_extract_timestamp(PG_FUNCTION_ARGS) +{ + pg_uuid_t *uuid = PG_GETARG_UUID_P(0); + TimestampTz ts; + uint64_t tms; + + if ((uuid->data[8] & 0xc0) != 0x80) + PG_RETURN_NULL(); + + if ((uuid->data[6] & 0xf0) == 0x70) + { + tms = uuid->data[5]; + tms += ((uint64_t)uuid->data[4]) << 8; + tms += ((uint64_t)uuid->data[3]) << 16; + tms += ((uint64_t)uuid->data[2]) << 24; + tms += ((uint64_t)uuid->data[1]) << 32; + tms += ((uint64_t)uuid->data[0]) << 40; + + ts = (TimestampTz) (tms * 1000) - /* convert ms to us, than adjust */ + (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY * USECS_PER_SEC; + + PG_RETURN_TIMESTAMPTZ(ts); + } + + if ((uuid->data[6] & 0xf0) == 0x10) + { + tms = ((uint64_t)uuid->data[0]) << 24; + tms += ((uint64_t)uuid->data[1]) << 16; + tms += ((uint64_t)uuid->data[2]) << 8; + tms += ((uint64_t)uuid->data[3]); + tms += ((uint64_t)uuid->data[4]) << 40; + tms += ((uint64_t)uuid->data[5]) << 32; + tms += (((uint64_t)uuid->data[6])&0xf) << 56; + tms += ((uint64_t)uuid->data[7]) << 48; + + ts = (TimestampTz) (tms / 10) - /* convert 100-ns intervals to us, than adjust */ + ((uint64_t)POSTGRES_EPOCH_JDATE - GREGORIAN_EPOCH_JDATE) * SECS_PER_DAY * USECS_PER_SEC; + + PG_RETURN_TIMESTAMPTZ(ts); + } + + if ((uuid->data[6] & 0xf0) == 0x60) + { + tms = ((uint64_t)uuid->data[0]) << 52; + tms += ((uint64_t)uuid->data[1]) << 44; + tms += ((uint64_t)uuid->data[2]) << 36; + tms += ((uint64_t)uuid->data[3]) << 28; + tms += ((uint64_t)uuid->data[4]) << 20; + tms += ((uint64_t)uuid->data[5]) << 12; + tms += (((uint64_t)uuid->data[6])&0xf) << 8; + tms += ((uint64_t)uuid->data[7]); + + ts = (TimestampTz) (tms / 10) - /* convert 100-ns intervals to us, than adjust */ + ((uint64_t)POSTGRES_EPOCH_JDATE - GREGORIAN_EPOCH_JDATE) * SECS_PER_DAY * USECS_PER_SEC; + + PG_RETURN_TIMESTAMPTZ(ts); + } + + PG_RETURN_NULL(); +} + +/* + * Routine to extract UUID version from variant 0b10 + * Returns NULL if UUID is not 0b10 + */ +Datum +uuid_extract_version(PG_FUNCTION_ARGS) +{ + pg_uuid_t *uuid = PG_GETARG_UUID_P(0); + uint16_t result; + + if ((uuid->data[8] & 0xc0) != 0x80) + PG_RETURN_NULL(); + result = uuid->data[6] >> 4; + + PG_RETURN_UINT16(result); +} + +/* + * Routine to extract UUID variant. Can return only 0, 0b10, 0b110 and 0b111. + */ +Datum +uuid_extract_variant(PG_FUNCTION_ARGS) +{ + pg_uuid_t *uuid = PG_GETARG_UUID_P(0); + uint16_t result; + + /* + * The contents of the variant field, where the letter "x" indicates a + * "don't-care" value. + * ---------- + * Msb0 Msb1 Msb2 Msb3 Variant Description + * 0 x x x 1-7 Reserved, NCS backward + * compatibility and includes Nil + * UUID as per Section 5.9. + * 1 0 x x 8-9,A-B The variant specified in RFC. + * 1 1 0 x C-D Reserved, Microsoft Corporation + * backward compatibility. + * 1 1 1 x E-F Reserved for future definition + * and includes Max UUID as per + * Section 5.10 of RFC. + * ---------- + */ + + uint8_t nibble = uuid->data[8] >> 4; + if (nibble < 8) + result = 0; + else if (nibble < 0xC) + result = 0b10; + else if (nibble < 0xE) + result = 0b110; + else + result = 0b111; + + PG_RETURN_UINT16(result); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index e4115cd084..eb1d075595 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -9174,6 +9174,21 @@ { oid => '3432', descr => 'generate random UUID', proname => 'gen_random_uuid', proleakproof => 't', provolatile => 'v', prorettype => 'uuid', proargtypes => '', prosrc => 'gen_random_uuid' }, +{ oid => '9895', descr => 'generate UUID version 4', + proname => 'uuidv4', proleakproof => 't', provolatile => 'v', + prorettype => 'uuid', proargtypes => '', prosrc => 'gen_random_uuid' }, +{ oid => '9896', descr => 'generate UUID version 7', + proname => 'uuidv7', proleakproof => 't', provolatile => 'v', + prorettype => 'uuid', proargtypes => '', prosrc => 'uuidv7' }, +{ oid => '9897', descr => 'extract timestamp from UUID version 1, 6 or 7', + proname => 'uuid_extract_timestamp', proleakproof => 't', + prorettype => 'timestamptz', proargtypes => 'uuid', prosrc => 'uuid_extract_timestamp' }, +{ oid => '9898', descr => 'extract version from RFC 4122 UUID', + proname => 'uuid_extract_version', proleakproof => 't', + prorettype => 'int2', proargtypes => 'uuid', prosrc => 'uuid_extract_version' }, +{ oid => '9899', descr => 'extract variant from UUID', + proname => 'uuid_extract_variant', proleakproof => 't', + prorettype => 'int2', proargtypes => 'uuid', prosrc => 'uuid_extract_variant' }, # pg_lsn { oid => '3229', descr => 'I/O', diff --git a/src/include/datatype/timestamp.h b/src/include/datatype/timestamp.h index 3a37cb661e..652aeb428e 100644 --- a/src/include/datatype/timestamp.h +++ b/src/include/datatype/timestamp.h @@ -230,9 +230,10 @@ struct pg_itm_in ((y) < JULIAN_MAXYEAR || \ ((y) == JULIAN_MAXYEAR && ((m) < JULIAN_MAXMONTH)))) -/* Julian-date equivalents of Day 0 in Unix and Postgres reckoning */ +/* Julian-date equivalents of Day 0 in Unix, Postgres and Gregorian epochs */ #define UNIX_EPOCH_JDATE 2440588 /* == date2j(1970, 1, 1) */ #define POSTGRES_EPOCH_JDATE 2451545 /* == date2j(2000, 1, 1) */ +#define GREGORIAN_EPOCH_JDATE 2299161 /* == date2j(1582,10,15) */ /* * Range limits for dates and timestamps. diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 7610b011d6..44542c56ef 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -872,6 +872,11 @@ xid8ge(xid8,xid8) xid8eq(xid8,xid8) xid8ne(xid8,xid8) xid8cmp(xid8,xid8) +uuidv4() +uuidv7() +uuid_extract_timestamp(uuid) +uuid_extract_version(uuid) +uuid_extract_variant(uuid) -- restore normal output mode \a\t -- List of functions used by libpq's fe-lobj.c diff --git a/src/test/regress/expected/uuid.out b/src/test/regress/expected/uuid.out index 8e7f21910d..2930bcc7f0 100644 --- a/src/test/regress/expected/uuid.out +++ b/src/test/regress/expected/uuid.out @@ -168,5 +168,76 @@ SELECT count(DISTINCT guid_field) FROM guid1; 2 (1 row) +-- test of uuidv4() alias +TRUNCATE guid1; +INSERT INTO guid1 (guid_field) VALUES (uuidv4()); +INSERT INTO guid1 (guid_field) VALUES (uuidv4()); +SELECT count(DISTINCT guid_field) FROM guid1; + count +------- + 2 +(1 row) + +-- generation test for v7 +TRUNCATE guid1; +INSERT INTO guid1 (guid_field) VALUES (uuidv7()); +INSERT INTO guid1 (guid_field) VALUES (uuidv7()); +SELECT count(DISTINCT guid_field) FROM guid1; + count +------- + 2 +(1 row) + +-- support functions for UUID versions and variants +SELECT uuid_extract_version(uuidv7()); + uuid_extract_version +---------------------- + 7 +(1 row) + +SELECT uuid_extract_version('11111111-1111-1111-1111-111111111111') IS NULL; + ?column? +---------- + t +(1 row) + +SELECT uuid_extract_version('11111111-1111-5111-8111-111111111111'); + uuid_extract_version +---------------------- + 5 +(1 row) + +SELECT uuid_extract_variant(uuidv7()); + uuid_extract_variant +---------------------- + 2 +(1 row) + +-- uuid_extract_timestamp() must refuse to accept non-UUIDv7 +SELECT uuid_extract_timestamp(gen_random_uuid()); + uuid_extract_timestamp +------------------------ + +(1 row) + +-- extract UUID v1, v6 and v7 timestamp +SELECT uuid_extract_timestamp('C232AB00-9414-11EC-B3C8-9F6BDECED846') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; + ?column? +---------- + t +(1 row) + +SELECT uuid_extract_timestamp('1EC9414C-232A-6B00-B3C8-9F6BDECED846') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; + ?column? +---------- + t +(1 row) + +SELECT uuid_extract_timestamp('017F22E2-79B0-7CC3-98C4-DC0C0C07398F') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; + ?column? +---------- + t +(1 row) + -- clean up DROP TABLE guid1, guid2 CASCADE; diff --git a/src/test/regress/sql/uuid.sql b/src/test/regress/sql/uuid.sql index 9a8f437c7d..0f9bd1a661 100644 --- a/src/test/regress/sql/uuid.sql +++ b/src/test/regress/sql/uuid.sql @@ -85,5 +85,31 @@ INSERT INTO guid1 (guid_field) VALUES (gen_random_uuid()); INSERT INTO guid1 (guid_field) VALUES (gen_random_uuid()); SELECT count(DISTINCT guid_field) FROM guid1; +-- test of uuidv4() alias +TRUNCATE guid1; +INSERT INTO guid1 (guid_field) VALUES (uuidv4()); +INSERT INTO guid1 (guid_field) VALUES (uuidv4()); +SELECT count(DISTINCT guid_field) FROM guid1; + +-- generation test for v7 +TRUNCATE guid1; +INSERT INTO guid1 (guid_field) VALUES (uuidv7()); +INSERT INTO guid1 (guid_field) VALUES (uuidv7()); +SELECT count(DISTINCT guid_field) FROM guid1; + +-- support functions for UUID versions and variants +SELECT uuid_extract_version(uuidv7()); +SELECT uuid_extract_version('11111111-1111-1111-1111-111111111111') IS NULL; +SELECT uuid_extract_version('11111111-1111-5111-8111-111111111111'); +SELECT uuid_extract_variant(uuidv7()); + +-- uuid_extract_timestamp() must refuse to accept non-UUIDv7 +SELECT uuid_extract_timestamp(gen_random_uuid()); + +-- extract UUID v1, v6 and v7 timestamp +SELECT uuid_extract_timestamp('C232AB00-9414-11EC-B3C8-9F6BDECED846') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; +SELECT uuid_extract_timestamp('1EC9414C-232A-6B00-B3C8-9F6BDECED846') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; +SELECT uuid_extract_timestamp('017F22E2-79B0-7CC3-98C4-DC0C0C07398F') = 'Tuesday, February 22, 2022 2:22:22.00 PM GMT+05:00'; + -- clean up DROP TABLE guid1, guid2 CASCADE; -- 2.37.1 (Apple Git-137.1)