Hi Hackers, This is the first piece of file level incremental backup support, as described on wiki page https://wiki.postgresql.org/wiki/Incremental_backup
It is not yet complete, but I wish to share it on the list to receive comments and suggestions. The point of the patch is adding an option to pg_basebackup and replication protocol BASE_BACKUP command to generate a backup_profile file. When taking a full backup with pg_basebackup, the user can request Postgres to generate a backup_profile file through the --profile option (-B short option, which I've arbitrarily picked up because both -P and -p was already taken) At the moment the backup profile consists of a file with one line per file detailing modification time, md5, size, tablespace and path relative to tablespace root (PGDATA or the tablespace) To calculate the md5 checksum I've used the md5 code present in pgcrypto contrib as the code in src/include/libpq/md5.h is not suitable for large files. Since a core feature cannot depend on a piece of contrib, I've moved the files contrib/pgcrypto/md5.c contrib/pgcrypto/md5.h to src/backend/utils/hash/md5.c src/include/utils/md5.h changing the pgcrypto extension to use them. There are still some TODOs: * User documentation * Remove the pg_basebackup code duplication I've introduced with the ReceiveAndUnpackTarFileToDir function, which is almost the same of ReceiveAndUnpackTarFile but does not expect to handle a tablespace. It instead simply extract a tar stream in a destination directory. The latter could probably be rewritten using the former as component, but it needs some adjustment to the "progress reporting" part, which is not present at the moment in ReceiveAndUnpackTarFileToDir. * Add header section to backup_profile file which at the moment contains only the body part. I'm thinking to change the original design and put the whole backup label as header, which is IMHO clearer and well known. I would use something like: START WAL LOCATION: 0/E000028 (file 00000001000000000000000E) CHECKPOINT LOCATION: 0/E000060 BACKUP METHOD: streamed BACKUP FROM: master START TIME: 2014-08-14 18:54:01 CEST LABEL: pg_basebackup base backup END LABEL I've attached the current patch based on master. Any comment will be appreciated. Regards, Marco -- Marco Nenciarini - 2ndQuadrant Italy PostgreSQL Training, Services and Support marco.nenciar...@2ndquadrant.it | www.2ndQuadrant.it
diff --git a/contrib/pgcrypto/Makefile b/contrib/pgcrypto/Makefile
index b6c9484..daf2ba3 100644
*** a/contrib/pgcrypto/Makefile
--- b/contrib/pgcrypto/Makefile
***************
*** 1,6 ****
# contrib/pgcrypto/Makefile
! INT_SRCS = md5.c sha1.c sha2.c internal.c internal-sha2.c blf.c rijndael.c \
fortuna.c random.c pgp-mpi-internal.c imath.c
INT_TESTS = sha2
--- 1,6 ----
# contrib/pgcrypto/Makefile
! INT_SRCS = sha1.c sha2.c internal.c internal-sha2.c blf.c rijndael.c \
fortuna.c random.c pgp-mpi-internal.c imath.c
INT_TESTS = sha2
diff --git a/contrib/pgcrypto/internal.c b/contrib/pgcrypto/internal.c
index cb8ba26..0d2db23 100644
*** a/contrib/pgcrypto/internal.c
--- b/contrib/pgcrypto/internal.c
***************
*** 30,40 ****
*/
#include "postgres.h"
#include <time.h>
#include "px.h"
- #include "md5.h"
#include "sha1.h"
#include "blf.h"
#include "rijndael.h"
--- 30,40 ----
*/
#include "postgres.h"
+ #include "utils/md5.h"
#include <time.h>
#include "px.h"
#include "sha1.h"
#include "blf.h"
#include "rijndael.h"
diff --git a/contrib/pgcrypto/md5.c b/contrib/pgcrypto/md5.c
index cac4e40..e69de29 .
*** a/contrib/pgcrypto/md5.c
--- b/contrib/pgcrypto/md5.c
***************
*** 1,397 ****
- /* $KAME: md5.c,v 1.3 2000/02/22 14:01:17 itojun Exp $ */
-
- /*
- * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the project nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * contrib/pgcrypto/md5.c
- */
-
- #include "postgres.h"
-
- #include <sys/param.h>
-
- #include "md5.h"
-
- #define SHIFT(X, s) (((X) << (s)) | ((X) >> (32 - (s))))
-
- #define F(X, Y, Z) (((X) & (Y)) | ((~X) & (Z)))
- #define G(X, Y, Z) (((X) & (Z)) | ((Y) & (~Z)))
- #define H(X, Y, Z) ((X) ^ (Y) ^ (Z))
- #define I(X, Y, Z) ((Y) ^ ((X) | (~Z)))
-
- #define ROUND1(a, b, c, d, k, s, i) \
- do { \
- (a) = (a) + F((b), (c), (d)) + X[(k)] + T[(i)]; \
- (a) = SHIFT((a), (s)); \
- (a) = (b) + (a); \
- } while (0)
-
- #define ROUND2(a, b, c, d, k, s, i) \
- do { \
- (a) = (a) + G((b), (c), (d)) + X[(k)] + T[(i)]; \
- (a) = SHIFT((a), (s)); \
- (a) = (b) + (a); \
- } while (0)
-
- #define ROUND3(a, b, c, d, k, s, i) \
- do { \
- (a) = (a) + H((b), (c), (d)) + X[(k)] + T[(i)]; \
- (a) = SHIFT((a), (s)); \
- (a) = (b) + (a); \
- } while (0)
-
- #define ROUND4(a, b, c, d, k, s, i) \
- do { \
- (a) = (a) + I((b), (c), (d)) + X[(k)] + T[(i)]; \
- (a) = SHIFT((a), (s)); \
- (a) = (b) + (a); \
- } while (0)
-
- #define Sa 7
- #define Sb 12
- #define Sc 17
- #define Sd 22
-
- #define Se 5
- #define Sf 9
- #define Sg 14
- #define Sh 20
-
- #define Si 4
- #define Sj 11
- #define Sk 16
- #define Sl 23
-
- #define Sm 6
- #define Sn 10
- #define So 15
- #define Sp 21
-
- #define MD5_A0 0x67452301
- #define MD5_B0 0xefcdab89
- #define MD5_C0 0x98badcfe
- #define MD5_D0 0x10325476
-
- /* Integer part of 4294967296 times abs(sin(i)), where i is in radians. */
- static const uint32 T[65] = {
- 0,
- 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee,
- 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
- 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
- 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
-
- 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa,
- 0xd62f105d, 0x2441453, 0xd8a1e681, 0xe7d3fbc8,
- 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
- 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a,
-
- 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
- 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70,
- 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x4881d05,
- 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
-
- 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039,
- 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
- 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
- 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391,
- };
-
- static const uint8 md5_paddat[MD5_BUFLEN] = {
- 0x80, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- };
-
- static void md5_calc(uint8 *, md5_ctxt *);
-
- void
- md5_init(md5_ctxt *ctxt)
- {
- ctxt->md5_n = 0;
- ctxt->md5_i = 0;
- ctxt->md5_sta = MD5_A0;
- ctxt->md5_stb = MD5_B0;
- ctxt->md5_stc = MD5_C0;
- ctxt->md5_std = MD5_D0;
- memset(ctxt->md5_buf, 0, sizeof(ctxt->md5_buf));
- }
-
- void
- md5_loop(md5_ctxt *ctxt, const uint8 *input, unsigned len)
- {
- unsigned int gap,
- i;
-
- ctxt->md5_n += len * 8; /* byte to bit */
- gap = MD5_BUFLEN - ctxt->md5_i;
-
- if (len >= gap)
- {
- memmove(ctxt->md5_buf + ctxt->md5_i, input, gap);
- md5_calc(ctxt->md5_buf, ctxt);
-
- for (i = gap; i + MD5_BUFLEN <= len; i += MD5_BUFLEN)
- md5_calc((uint8 *) (input + i), ctxt);
-
- ctxt->md5_i = len - i;
- memmove(ctxt->md5_buf, input + i, ctxt->md5_i);
- }
- else
- {
- memmove(ctxt->md5_buf + ctxt->md5_i, input, len);
- ctxt->md5_i += len;
- }
- }
-
- void
- md5_pad(md5_ctxt *ctxt)
- {
- unsigned int gap;
-
- /* Don't count up padding. Keep md5_n. */
- gap = MD5_BUFLEN - ctxt->md5_i;
- if (gap > 8)
- {
- memmove(ctxt->md5_buf + ctxt->md5_i, md5_paddat,
- gap - sizeof(ctxt->md5_n));
- }
- else
- {
- /* including gap == 8 */
- memmove(ctxt->md5_buf + ctxt->md5_i, md5_paddat, gap);
- md5_calc(ctxt->md5_buf, ctxt);
- memmove(ctxt->md5_buf, md5_paddat + gap,
- MD5_BUFLEN - sizeof(ctxt->md5_n));
- }
-
- /* 8 byte word */
- #ifndef WORDS_BIGENDIAN
- memmove(&ctxt->md5_buf[56], &ctxt->md5_n8[0], 8);
- #else
- ctxt->md5_buf[56] = ctxt->md5_n8[7];
- ctxt->md5_buf[57] = ctxt->md5_n8[6];
- ctxt->md5_buf[58] = ctxt->md5_n8[5];
- ctxt->md5_buf[59] = ctxt->md5_n8[4];
- ctxt->md5_buf[60] = ctxt->md5_n8[3];
- ctxt->md5_buf[61] = ctxt->md5_n8[2];
- ctxt->md5_buf[62] = ctxt->md5_n8[1];
- ctxt->md5_buf[63] = ctxt->md5_n8[0];
- #endif
-
- md5_calc(ctxt->md5_buf, ctxt);
- }
-
- void
- md5_result(uint8 *digest, md5_ctxt *ctxt)
- {
- /* 4 byte words */
- #ifndef WORDS_BIGENDIAN
- memmove(digest, &ctxt->md5_st8[0], 16);
- #else
- digest[0] = ctxt->md5_st8[3];
- digest[1] = ctxt->md5_st8[2];
- digest[2] = ctxt->md5_st8[1];
- digest[3] = ctxt->md5_st8[0];
- digest[4] = ctxt->md5_st8[7];
- digest[5] = ctxt->md5_st8[6];
- digest[6] = ctxt->md5_st8[5];
- digest[7] = ctxt->md5_st8[4];
- digest[8] = ctxt->md5_st8[11];
- digest[9] = ctxt->md5_st8[10];
- digest[10] = ctxt->md5_st8[9];
- digest[11] = ctxt->md5_st8[8];
- digest[12] = ctxt->md5_st8[15];
- digest[13] = ctxt->md5_st8[14];
- digest[14] = ctxt->md5_st8[13];
- digest[15] = ctxt->md5_st8[12];
- #endif
- }
-
- #ifdef WORDS_BIGENDIAN
- static uint32 X[16];
- #endif
-
- static void
- md5_calc(uint8 *b64, md5_ctxt *ctxt)
- {
- uint32 A = ctxt->md5_sta;
- uint32 B = ctxt->md5_stb;
- uint32 C = ctxt->md5_stc;
- uint32 D = ctxt->md5_std;
-
- #ifndef WORDS_BIGENDIAN
- uint32 *X = (uint32 *) b64;
- #else
- /* 4 byte words */
- /* what a brute force but fast! */
- uint8 *y = (uint8 *) X;
-
- y[0] = b64[3];
- y[1] = b64[2];
- y[2] = b64[1];
- y[3] = b64[0];
- y[4] = b64[7];
- y[5] = b64[6];
- y[6] = b64[5];
- y[7] = b64[4];
- y[8] = b64[11];
- y[9] = b64[10];
- y[10] = b64[9];
- y[11] = b64[8];
- y[12] = b64[15];
- y[13] = b64[14];
- y[14] = b64[13];
- y[15] = b64[12];
- y[16] = b64[19];
- y[17] = b64[18];
- y[18] = b64[17];
- y[19] = b64[16];
- y[20] = b64[23];
- y[21] = b64[22];
- y[22] = b64[21];
- y[23] = b64[20];
- y[24] = b64[27];
- y[25] = b64[26];
- y[26] = b64[25];
- y[27] = b64[24];
- y[28] = b64[31];
- y[29] = b64[30];
- y[30] = b64[29];
- y[31] = b64[28];
- y[32] = b64[35];
- y[33] = b64[34];
- y[34] = b64[33];
- y[35] = b64[32];
- y[36] = b64[39];
- y[37] = b64[38];
- y[38] = b64[37];
- y[39] = b64[36];
- y[40] = b64[43];
- y[41] = b64[42];
- y[42] = b64[41];
- y[43] = b64[40];
- y[44] = b64[47];
- y[45] = b64[46];
- y[46] = b64[45];
- y[47] = b64[44];
- y[48] = b64[51];
- y[49] = b64[50];
- y[50] = b64[49];
- y[51] = b64[48];
- y[52] = b64[55];
- y[53] = b64[54];
- y[54] = b64[53];
- y[55] = b64[52];
- y[56] = b64[59];
- y[57] = b64[58];
- y[58] = b64[57];
- y[59] = b64[56];
- y[60] = b64[63];
- y[61] = b64[62];
- y[62] = b64[61];
- y[63] = b64[60];
- #endif
-
- ROUND1(A, B, C, D, 0, Sa, 1);
- ROUND1(D, A, B, C, 1, Sb, 2);
- ROUND1(C, D, A, B, 2, Sc, 3);
- ROUND1(B, C, D, A, 3, Sd, 4);
- ROUND1(A, B, C, D, 4, Sa, 5);
- ROUND1(D, A, B, C, 5, Sb, 6);
- ROUND1(C, D, A, B, 6, Sc, 7);
- ROUND1(B, C, D, A, 7, Sd, 8);
- ROUND1(A, B, C, D, 8, Sa, 9);
- ROUND1(D, A, B, C, 9, Sb, 10);
- ROUND1(C, D, A, B, 10, Sc, 11);
- ROUND1(B, C, D, A, 11, Sd, 12);
- ROUND1(A, B, C, D, 12, Sa, 13);
- ROUND1(D, A, B, C, 13, Sb, 14);
- ROUND1(C, D, A, B, 14, Sc, 15);
- ROUND1(B, C, D, A, 15, Sd, 16);
-
- ROUND2(A, B, C, D, 1, Se, 17);
- ROUND2(D, A, B, C, 6, Sf, 18);
- ROUND2(C, D, A, B, 11, Sg, 19);
- ROUND2(B, C, D, A, 0, Sh, 20);
- ROUND2(A, B, C, D, 5, Se, 21);
- ROUND2(D, A, B, C, 10, Sf, 22);
- ROUND2(C, D, A, B, 15, Sg, 23);
- ROUND2(B, C, D, A, 4, Sh, 24);
- ROUND2(A, B, C, D, 9, Se, 25);
- ROUND2(D, A, B, C, 14, Sf, 26);
- ROUND2(C, D, A, B, 3, Sg, 27);
- ROUND2(B, C, D, A, 8, Sh, 28);
- ROUND2(A, B, C, D, 13, Se, 29);
- ROUND2(D, A, B, C, 2, Sf, 30);
- ROUND2(C, D, A, B, 7, Sg, 31);
- ROUND2(B, C, D, A, 12, Sh, 32);
-
- ROUND3(A, B, C, D, 5, Si, 33);
- ROUND3(D, A, B, C, 8, Sj, 34);
- ROUND3(C, D, A, B, 11, Sk, 35);
- ROUND3(B, C, D, A, 14, Sl, 36);
- ROUND3(A, B, C, D, 1, Si, 37);
- ROUND3(D, A, B, C, 4, Sj, 38);
- ROUND3(C, D, A, B, 7, Sk, 39);
- ROUND3(B, C, D, A, 10, Sl, 40);
- ROUND3(A, B, C, D, 13, Si, 41);
- ROUND3(D, A, B, C, 0, Sj, 42);
- ROUND3(C, D, A, B, 3, Sk, 43);
- ROUND3(B, C, D, A, 6, Sl, 44);
- ROUND3(A, B, C, D, 9, Si, 45);
- ROUND3(D, A, B, C, 12, Sj, 46);
- ROUND3(C, D, A, B, 15, Sk, 47);
- ROUND3(B, C, D, A, 2, Sl, 48);
-
- ROUND4(A, B, C, D, 0, Sm, 49);
- ROUND4(D, A, B, C, 7, Sn, 50);
- ROUND4(C, D, A, B, 14, So, 51);
- ROUND4(B, C, D, A, 5, Sp, 52);
- ROUND4(A, B, C, D, 12, Sm, 53);
- ROUND4(D, A, B, C, 3, Sn, 54);
- ROUND4(C, D, A, B, 10, So, 55);
- ROUND4(B, C, D, A, 1, Sp, 56);
- ROUND4(A, B, C, D, 8, Sm, 57);
- ROUND4(D, A, B, C, 15, Sn, 58);
- ROUND4(C, D, A, B, 6, So, 59);
- ROUND4(B, C, D, A, 13, Sp, 60);
- ROUND4(A, B, C, D, 4, Sm, 61);
- ROUND4(D, A, B, C, 11, Sn, 62);
- ROUND4(C, D, A, B, 2, So, 63);
- ROUND4(B, C, D, A, 9, Sp, 64);
-
- ctxt->md5_sta += A;
- ctxt->md5_stb += B;
- ctxt->md5_stc += C;
- ctxt->md5_std += D;
- }
--- 0 ----
diff --git a/contrib/pgcrypto/md5.h b/contrib/pgcrypto/md5.h
index 07d08c1..e69de29 .
*** a/contrib/pgcrypto/md5.h
--- b/contrib/pgcrypto/md5.h
***************
*** 1,79 ****
- /* contrib/pgcrypto/md5.h */
- /* $KAME: md5.h,v 1.3 2000/02/22 14:01:18 itojun Exp $ */
-
- /*
- * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the project nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
- #ifndef _NETINET6_MD5_H_
- #define _NETINET6_MD5_H_
-
- #define MD5_BUFLEN 64
-
- typedef struct
- {
- union
- {
- uint32 md5_state32[4];
- uint8 md5_state8[16];
- } md5_st;
-
- #define md5_sta md5_st.md5_state32[0]
- #define md5_stb md5_st.md5_state32[1]
- #define md5_stc md5_st.md5_state32[2]
- #define md5_std md5_st.md5_state32[3]
- #define md5_st8 md5_st.md5_state8
-
- union
- {
- uint64 md5_count64;
- uint8 md5_count8[8];
- } md5_count;
- #define md5_n md5_count.md5_count64
- #define md5_n8 md5_count.md5_count8
-
- unsigned int md5_i;
- uint8 md5_buf[MD5_BUFLEN];
- } md5_ctxt;
-
- extern void md5_init(md5_ctxt *);
- extern void md5_loop(md5_ctxt *, const uint8 *, unsigned int);
- extern void md5_pad(md5_ctxt *);
- extern void md5_result(uint8 *, md5_ctxt *);
-
- /* compatibility */
- #define MD5_CTX md5_ctxt
- #define MD5Init(x) md5_init((x))
- #define MD5Update(x, y, z) md5_loop((x), (y), (z))
- #define MD5Final(x, y) \
- do { \
- md5_pad((y)); \
- md5_result((x), (y)); \
- } while (0)
-
- #endif /* ! _NETINET6_MD5_H_ */
--- 0 ----
diff --git a/src/backend/replication/basebackup.c
b/src/backend/replication/basebackup.c
index fbcecbb..0f29f18 100644
*** a/src/backend/replication/basebackup.c
--- b/src/backend/replication/basebackup.c
***************
*** 34,39 ****
--- 34,40 ----
#include "storage/ipc.h"
#include "utils/builtins.h"
#include "utils/elog.h"
+ #include "utils/md5.h"
#include "utils/ps_status.h"
#include "utils/timestamp.h"
*************** typedef struct
*** 46,54 ****
--- 47,58 ----
bool nowait;
bool includewal;
uint32 maxrate;
+ bool profile;
} basebackup_options;
+ static void writeProfileLine(const char *filename, pgoff_t len,
+ struct stat * statbuf,
MD5_CTX *ctx);
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List
*tablespaces);
static int64 sendTablespace(char *path, bool sizeonly);
static bool sendFile(char *readfilename, char *tarfilename,
*************** static bool backup_started_in_recovery =
*** 71,76 ****
--- 75,85 ----
/* Relative path of temporary statistics directory */
static char *statrelpath = NULL;
+ /* Temporary file containing the backup profile */
+ static File backup_profile_fd = 0;
+ /* Tablespace being currently sent. Used in backup profile generation */
+ static char *current_tablespace = NULL;
+
/*
* Size of each block sent into the tar stream for larger files.
*/
*************** perform_base_backup(basebackup_options *
*** 132,137 ****
--- 141,153 ----
backup_started_in_recovery = RecoveryInProgress();
+ /*
+ * If "profile" option is enabled, open a temporary file to hold the
+ * profile content.
+ */
+ if (opt->profile)
+ backup_profile_fd = OpenTemporaryFile(false);
+
startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint,
&starttli,
&labelfile);
/*
*************** perform_base_backup(basebackup_options *
*** 267,272 ****
--- 283,290 ----
pq_sendint(&buf, 0, 2); /* natts */
pq_endmessage(&buf);
+ current_tablespace = ti->oid;
+
if (ti->path == NULL)
{
struct stat statbuf;
*************** perform_base_backup(basebackup_options *
*** 527,532 ****
--- 545,575 ----
pq_putemptymessage('c');
}
SendXlogRecPtrResult(endptr, endtli);
+
+ /* If "profile" option is enabled send the profile file. */
+ if (backup_profile_fd > 0)
+ {
+ StringInfoData buf;
+ struct stat statbuf;
+ char *backup_profile = FilePathName(backup_profile_fd);
+
+ /* Send CopyOutResponse message */
+ pq_beginmessage(&buf, 'H');
+ pq_sendbyte(&buf, 0); /* overall format */
+ pq_sendint(&buf, 0, 2); /* natts */
+ pq_endmessage(&buf);
+
+ /* Send the backup profile content */
+ if (lstat(backup_profile, &statbuf) != 0)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not stat backup_profile
file \"%s\": %m",
+ backup_profile)));
+ FileSeek(backup_profile_fd, 0, SEEK_SET);
+ sendFile(backup_profile, BACKUP_PROFILE_FILE, &statbuf, false);
+
+ pq_putemptymessage('c'); /* CopyDone */
+ }
}
/*
*************** parse_basebackup_options(List *options,
*** 555,560 ****
--- 598,604 ----
bool o_nowait = false;
bool o_wal = false;
bool o_maxrate = false;
+ bool o_profile = false;
MemSet(opt, 0, sizeof(*opt));
foreach(lopt, options)
*************** parse_basebackup_options(List *options,
*** 625,630 ****
--- 669,683 ----
opt->maxrate = (uint32) maxrate;
o_maxrate = true;
}
+ else if (strcmp(defel->defname, "profile") == 0)
+ {
+ if (o_profile)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("duplicate option
\"%s\"", defel->defname)));
+ opt->profile = true;
+ o_profile = true;
+ }
else
elog(ERROR, "option \"%s\" not recognized",
defel->defname);
*************** SendXlogRecPtrResult(XLogRecPtr ptr, Tim
*** 803,808 ****
--- 856,885 ----
pq_puttextmessage('C', "SELECT");
}
+
+ static void
+ writeProfileLine(const char *filename, pgoff_t len, struct stat * statbuf,
+ MD5_CTX *ctx)
+ {
+ char buf[MAXPGPATH + 128];
+ uint8 sum[16];
+ char hexsum[33];
+ int rowlen;
+
+ Assert(backup_profile_fd > 0);
+
+ /* Finalize the checksum */
+ MD5Final(sum, ctx);
+ hex_encode((const char *) sum, sizeof(sum), hexsum);
+ hexsum[sizeof(hexsum)-1] = '\0';
+
+ rowlen = snprintf(buf, sizeof(buf), "%f\t%s\t%lld\t%s\t%s\n",
+ (double) statbuf->st_mtime, hexsum ,
len,
+
current_tablespace?current_tablespace:"\\N",
+ filename);
+ FileWrite(backup_profile_fd, buf, rowlen);
+ }
+
/*
* Inject a file with given name and content in the output tar stream.
*/
*************** sendFileWithContent(const char *filename
*** 844,849 ****
--- 921,936 ----
MemSet(buf, 0, pad);
pq_putmessage('d', buf, pad);
}
+
+ /*
+ * If backup profile is active write an entry for this file.
+ */
+ if (backup_profile_fd > 0) {
+ MD5_CTX ctx;
+ MD5Init(&ctx);
+ MD5Update(&ctx, (unsigned char *) content, len);
+ writeProfileLine(filename, len, &statbuf, &ctx);
+ }
}
/*
*************** sendFile(char *readfilename, char *tarfi
*** 1156,1161 ****
--- 1243,1249 ----
size_t cnt;
pgoff_t len = 0;
size_t pad;
+ MD5_CTX ctx;
fp = AllocateFile(readfilename, "rb");
if (fp == NULL)
*************** sendFile(char *readfilename, char *tarfi
*** 1178,1183 ****
--- 1266,1277 ----
_tarWriteHeader(tarfilename, NULL, statbuf);
+ /*
+ * If backup profile is active initialize the md5 context
+ */
+ if (backup_profile_fd > 0)
+ MD5Init(&ctx);
+
while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len),
fp)) > 0)
{
/* Send the chunk as a CopyData message */
*************** sendFile(char *readfilename, char *tarfi
*** 1185,1190 ****
--- 1279,1290 ----
ereport(ERROR,
(errmsg("base backup could not send data, aborting
backup")));
+ /*
+ * If backup profile is active update the checksum with current
chunk
+ */
+ if (backup_profile_fd > 0)
+ MD5Update(&ctx, (unsigned char *) buf, cnt);
+
len += cnt;
throttle(cnt);
*************** sendFile(char *readfilename, char *tarfi
*** 1225,1230 ****
--- 1325,1336 ----
FreeFile(fp);
+ /*
+ * If backup profile is active write an entry for this file.
+ */
+ if (backup_profile_fd > 0)
+ writeProfileLine(tarfilename, len, statbuf, &ctx);
+
return true;
}
diff --git a/src/backend/replication/repl_gram.y
b/src/backend/replication/repl_gram.y
index 154aaac..0528a78 100644
*** a/src/backend/replication/repl_gram.y
--- b/src/backend/replication/repl_gram.y
*************** Node *replication_parse_result;
*** 75,80 ****
--- 75,81 ----
%token K_PHYSICAL
%token K_LOGICAL
%token K_SLOT
+ %token K_PROFILE
%type <node> command
%type <node> base_backup start_replication start_logical_replication
create_replication_slot drop_replication_slot identify_system timeline_history
*************** base_backup_opt:
*** 168,173 ****
--- 169,179 ----
$$ = makeDefElem("max_rate",
(Node
*)makeInteger($2));
}
+ | K_PROFILE
+ {
+ $$ = makeDefElem("profile",
+ (Node
*)makeInteger(TRUE));
+ }
;
create_replication_slot:
diff --git a/src/backend/replication/repl_scanner.l
b/src/backend/replication/repl_scanner.l
index a257124..a3c2164 100644
*** a/src/backend/replication/repl_scanner.l
--- b/src/backend/replication/repl_scanner.l
*************** TIMELINE_HISTORY { return K_TIMELINE_HIS
*** 96,101 ****
--- 96,102 ----
PHYSICAL { return K_PHYSICAL; }
LOGICAL { return K_LOGICAL; }
SLOT { return K_SLOT; }
+ PROFILE { return K_PROFILE; }
"," { return ','; }
";" { return ';'; }
diff --git a/src/backend/utils/hash/Makefile b/src/backend/utils/hash/Makefile
index 05d347c..71e34a6 100644
*** a/src/backend/utils/hash/Makefile
--- b/src/backend/utils/hash/Makefile
*************** subdir = src/backend/utils/hash
*** 12,17 ****
top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
! OBJS = dynahash.o hashfn.o
include $(top_srcdir)/src/backend/common.mk
--- 12,17 ----
top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
! OBJS = dynahash.o hashfn.o md5.o
include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/utils/hash/md5.c b/src/backend/utils/hash/md5.c
index ...ec9a851 100644
*** a/src/backend/utils/hash/md5.c
--- b/src/backend/utils/hash/md5.c
***************
*** 0 ****
--- 1,397 ----
+ /* $KAME: md5.c,v 1.3 2000/02/22 14:01:17 itojun Exp $ */
+
+ /*
+ * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the project nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * src/backend/utils/hash/md5.c
+ */
+
+ #include "postgres.h"
+
+ #include <sys/param.h>
+
+ #include "utils/md5.h"
+
+ #define SHIFT(X, s) (((X) << (s)) | ((X) >> (32 - (s))))
+
+ #define F(X, Y, Z) (((X) & (Y)) | ((~X) & (Z)))
+ #define G(X, Y, Z) (((X) & (Z)) | ((Y) & (~Z)))
+ #define H(X, Y, Z) ((X) ^ (Y) ^ (Z))
+ #define I(X, Y, Z) ((Y) ^ ((X) | (~Z)))
+
+ #define ROUND1(a, b, c, d, k, s, i) \
+ do { \
+ (a) = (a) + F((b), (c), (d)) + X[(k)] + T[(i)]; \
+ (a) = SHIFT((a), (s)); \
+ (a) = (b) + (a); \
+ } while (0)
+
+ #define ROUND2(a, b, c, d, k, s, i) \
+ do { \
+ (a) = (a) + G((b), (c), (d)) + X[(k)] + T[(i)]; \
+ (a) = SHIFT((a), (s)); \
+ (a) = (b) + (a); \
+ } while (0)
+
+ #define ROUND3(a, b, c, d, k, s, i) \
+ do { \
+ (a) = (a) + H((b), (c), (d)) + X[(k)] + T[(i)]; \
+ (a) = SHIFT((a), (s)); \
+ (a) = (b) + (a); \
+ } while (0)
+
+ #define ROUND4(a, b, c, d, k, s, i) \
+ do { \
+ (a) = (a) + I((b), (c), (d)) + X[(k)] + T[(i)]; \
+ (a) = SHIFT((a), (s)); \
+ (a) = (b) + (a); \
+ } while (0)
+
+ #define Sa 7
+ #define Sb 12
+ #define Sc 17
+ #define Sd 22
+
+ #define Se 5
+ #define Sf 9
+ #define Sg 14
+ #define Sh 20
+
+ #define Si 4
+ #define Sj 11
+ #define Sk 16
+ #define Sl 23
+
+ #define Sm 6
+ #define Sn 10
+ #define So 15
+ #define Sp 21
+
+ #define MD5_A0 0x67452301
+ #define MD5_B0 0xefcdab89
+ #define MD5_C0 0x98badcfe
+ #define MD5_D0 0x10325476
+
+ /* Integer part of 4294967296 times abs(sin(i)), where i is in radians. */
+ static const uint32 T[65] = {
+ 0,
+ 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee,
+ 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
+ 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
+ 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
+
+ 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa,
+ 0xd62f105d, 0x2441453, 0xd8a1e681, 0xe7d3fbc8,
+ 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
+ 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a,
+
+ 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
+ 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70,
+ 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x4881d05,
+ 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
+
+ 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039,
+ 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
+ 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
+ 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391,
+ };
+
+ static const uint8 md5_paddat[MD5_BUFLEN] = {
+ 0x80, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ };
+
+ static void md5_calc(uint8 *, md5_ctxt *);
+
+ void
+ md5_init(md5_ctxt *ctxt)
+ {
+ ctxt->md5_n = 0;
+ ctxt->md5_i = 0;
+ ctxt->md5_sta = MD5_A0;
+ ctxt->md5_stb = MD5_B0;
+ ctxt->md5_stc = MD5_C0;
+ ctxt->md5_std = MD5_D0;
+ memset(ctxt->md5_buf, 0, sizeof(ctxt->md5_buf));
+ }
+
+ void
+ md5_loop(md5_ctxt *ctxt, const uint8 *input, unsigned len)
+ {
+ unsigned int gap,
+ i;
+
+ ctxt->md5_n += len * 8; /* byte to bit */
+ gap = MD5_BUFLEN - ctxt->md5_i;
+
+ if (len >= gap)
+ {
+ memmove(ctxt->md5_buf + ctxt->md5_i, input, gap);
+ md5_calc(ctxt->md5_buf, ctxt);
+
+ for (i = gap; i + MD5_BUFLEN <= len; i += MD5_BUFLEN)
+ md5_calc((uint8 *) (input + i), ctxt);
+
+ ctxt->md5_i = len - i;
+ memmove(ctxt->md5_buf, input + i, ctxt->md5_i);
+ }
+ else
+ {
+ memmove(ctxt->md5_buf + ctxt->md5_i, input, len);
+ ctxt->md5_i += len;
+ }
+ }
+
+ void
+ md5_pad(md5_ctxt *ctxt)
+ {
+ unsigned int gap;
+
+ /* Don't count up padding. Keep md5_n. */
+ gap = MD5_BUFLEN - ctxt->md5_i;
+ if (gap > 8)
+ {
+ memmove(ctxt->md5_buf + ctxt->md5_i, md5_paddat,
+ gap - sizeof(ctxt->md5_n));
+ }
+ else
+ {
+ /* including gap == 8 */
+ memmove(ctxt->md5_buf + ctxt->md5_i, md5_paddat, gap);
+ md5_calc(ctxt->md5_buf, ctxt);
+ memmove(ctxt->md5_buf, md5_paddat + gap,
+ MD5_BUFLEN - sizeof(ctxt->md5_n));
+ }
+
+ /* 8 byte word */
+ #ifndef WORDS_BIGENDIAN
+ memmove(&ctxt->md5_buf[56], &ctxt->md5_n8[0], 8);
+ #else
+ ctxt->md5_buf[56] = ctxt->md5_n8[7];
+ ctxt->md5_buf[57] = ctxt->md5_n8[6];
+ ctxt->md5_buf[58] = ctxt->md5_n8[5];
+ ctxt->md5_buf[59] = ctxt->md5_n8[4];
+ ctxt->md5_buf[60] = ctxt->md5_n8[3];
+ ctxt->md5_buf[61] = ctxt->md5_n8[2];
+ ctxt->md5_buf[62] = ctxt->md5_n8[1];
+ ctxt->md5_buf[63] = ctxt->md5_n8[0];
+ #endif
+
+ md5_calc(ctxt->md5_buf, ctxt);
+ }
+
+ void
+ md5_result(uint8 *digest, md5_ctxt *ctxt)
+ {
+ /* 4 byte words */
+ #ifndef WORDS_BIGENDIAN
+ memmove(digest, &ctxt->md5_st8[0], 16);
+ #else
+ digest[0] = ctxt->md5_st8[3];
+ digest[1] = ctxt->md5_st8[2];
+ digest[2] = ctxt->md5_st8[1];
+ digest[3] = ctxt->md5_st8[0];
+ digest[4] = ctxt->md5_st8[7];
+ digest[5] = ctxt->md5_st8[6];
+ digest[6] = ctxt->md5_st8[5];
+ digest[7] = ctxt->md5_st8[4];
+ digest[8] = ctxt->md5_st8[11];
+ digest[9] = ctxt->md5_st8[10];
+ digest[10] = ctxt->md5_st8[9];
+ digest[11] = ctxt->md5_st8[8];
+ digest[12] = ctxt->md5_st8[15];
+ digest[13] = ctxt->md5_st8[14];
+ digest[14] = ctxt->md5_st8[13];
+ digest[15] = ctxt->md5_st8[12];
+ #endif
+ }
+
+ #ifdef WORDS_BIGENDIAN
+ static uint32 X[16];
+ #endif
+
+ static void
+ md5_calc(uint8 *b64, md5_ctxt *ctxt)
+ {
+ uint32 A = ctxt->md5_sta;
+ uint32 B = ctxt->md5_stb;
+ uint32 C = ctxt->md5_stc;
+ uint32 D = ctxt->md5_std;
+
+ #ifndef WORDS_BIGENDIAN
+ uint32 *X = (uint32 *) b64;
+ #else
+ /* 4 byte words */
+ /* what a brute force but fast! */
+ uint8 *y = (uint8 *) X;
+
+ y[0] = b64[3];
+ y[1] = b64[2];
+ y[2] = b64[1];
+ y[3] = b64[0];
+ y[4] = b64[7];
+ y[5] = b64[6];
+ y[6] = b64[5];
+ y[7] = b64[4];
+ y[8] = b64[11];
+ y[9] = b64[10];
+ y[10] = b64[9];
+ y[11] = b64[8];
+ y[12] = b64[15];
+ y[13] = b64[14];
+ y[14] = b64[13];
+ y[15] = b64[12];
+ y[16] = b64[19];
+ y[17] = b64[18];
+ y[18] = b64[17];
+ y[19] = b64[16];
+ y[20] = b64[23];
+ y[21] = b64[22];
+ y[22] = b64[21];
+ y[23] = b64[20];
+ y[24] = b64[27];
+ y[25] = b64[26];
+ y[26] = b64[25];
+ y[27] = b64[24];
+ y[28] = b64[31];
+ y[29] = b64[30];
+ y[30] = b64[29];
+ y[31] = b64[28];
+ y[32] = b64[35];
+ y[33] = b64[34];
+ y[34] = b64[33];
+ y[35] = b64[32];
+ y[36] = b64[39];
+ y[37] = b64[38];
+ y[38] = b64[37];
+ y[39] = b64[36];
+ y[40] = b64[43];
+ y[41] = b64[42];
+ y[42] = b64[41];
+ y[43] = b64[40];
+ y[44] = b64[47];
+ y[45] = b64[46];
+ y[46] = b64[45];
+ y[47] = b64[44];
+ y[48] = b64[51];
+ y[49] = b64[50];
+ y[50] = b64[49];
+ y[51] = b64[48];
+ y[52] = b64[55];
+ y[53] = b64[54];
+ y[54] = b64[53];
+ y[55] = b64[52];
+ y[56] = b64[59];
+ y[57] = b64[58];
+ y[58] = b64[57];
+ y[59] = b64[56];
+ y[60] = b64[63];
+ y[61] = b64[62];
+ y[62] = b64[61];
+ y[63] = b64[60];
+ #endif
+
+ ROUND1(A, B, C, D, 0, Sa, 1);
+ ROUND1(D, A, B, C, 1, Sb, 2);
+ ROUND1(C, D, A, B, 2, Sc, 3);
+ ROUND1(B, C, D, A, 3, Sd, 4);
+ ROUND1(A, B, C, D, 4, Sa, 5);
+ ROUND1(D, A, B, C, 5, Sb, 6);
+ ROUND1(C, D, A, B, 6, Sc, 7);
+ ROUND1(B, C, D, A, 7, Sd, 8);
+ ROUND1(A, B, C, D, 8, Sa, 9);
+ ROUND1(D, A, B, C, 9, Sb, 10);
+ ROUND1(C, D, A, B, 10, Sc, 11);
+ ROUND1(B, C, D, A, 11, Sd, 12);
+ ROUND1(A, B, C, D, 12, Sa, 13);
+ ROUND1(D, A, B, C, 13, Sb, 14);
+ ROUND1(C, D, A, B, 14, Sc, 15);
+ ROUND1(B, C, D, A, 15, Sd, 16);
+
+ ROUND2(A, B, C, D, 1, Se, 17);
+ ROUND2(D, A, B, C, 6, Sf, 18);
+ ROUND2(C, D, A, B, 11, Sg, 19);
+ ROUND2(B, C, D, A, 0, Sh, 20);
+ ROUND2(A, B, C, D, 5, Se, 21);
+ ROUND2(D, A, B, C, 10, Sf, 22);
+ ROUND2(C, D, A, B, 15, Sg, 23);
+ ROUND2(B, C, D, A, 4, Sh, 24);
+ ROUND2(A, B, C, D, 9, Se, 25);
+ ROUND2(D, A, B, C, 14, Sf, 26);
+ ROUND2(C, D, A, B, 3, Sg, 27);
+ ROUND2(B, C, D, A, 8, Sh, 28);
+ ROUND2(A, B, C, D, 13, Se, 29);
+ ROUND2(D, A, B, C, 2, Sf, 30);
+ ROUND2(C, D, A, B, 7, Sg, 31);
+ ROUND2(B, C, D, A, 12, Sh, 32);
+
+ ROUND3(A, B, C, D, 5, Si, 33);
+ ROUND3(D, A, B, C, 8, Sj, 34);
+ ROUND3(C, D, A, B, 11, Sk, 35);
+ ROUND3(B, C, D, A, 14, Sl, 36);
+ ROUND3(A, B, C, D, 1, Si, 37);
+ ROUND3(D, A, B, C, 4, Sj, 38);
+ ROUND3(C, D, A, B, 7, Sk, 39);
+ ROUND3(B, C, D, A, 10, Sl, 40);
+ ROUND3(A, B, C, D, 13, Si, 41);
+ ROUND3(D, A, B, C, 0, Sj, 42);
+ ROUND3(C, D, A, B, 3, Sk, 43);
+ ROUND3(B, C, D, A, 6, Sl, 44);
+ ROUND3(A, B, C, D, 9, Si, 45);
+ ROUND3(D, A, B, C, 12, Sj, 46);
+ ROUND3(C, D, A, B, 15, Sk, 47);
+ ROUND3(B, C, D, A, 2, Sl, 48);
+
+ ROUND4(A, B, C, D, 0, Sm, 49);
+ ROUND4(D, A, B, C, 7, Sn, 50);
+ ROUND4(C, D, A, B, 14, So, 51);
+ ROUND4(B, C, D, A, 5, Sp, 52);
+ ROUND4(A, B, C, D, 12, Sm, 53);
+ ROUND4(D, A, B, C, 3, Sn, 54);
+ ROUND4(C, D, A, B, 10, So, 55);
+ ROUND4(B, C, D, A, 1, Sp, 56);
+ ROUND4(A, B, C, D, 8, Sm, 57);
+ ROUND4(D, A, B, C, 15, Sn, 58);
+ ROUND4(C, D, A, B, 6, So, 59);
+ ROUND4(B, C, D, A, 13, Sp, 60);
+ ROUND4(A, B, C, D, 4, Sm, 61);
+ ROUND4(D, A, B, C, 11, Sn, 62);
+ ROUND4(C, D, A, B, 2, So, 63);
+ ROUND4(B, C, D, A, 9, Sp, 64);
+
+ ctxt->md5_sta += A;
+ ctxt->md5_stb += B;
+ ctxt->md5_stc += C;
+ ctxt->md5_std += D;
+ }
diff --git a/src/bin/pg_basebackup/pg_basebackup.c
b/src/bin/pg_basebackup/pg_basebackup.c
index 3d26e22..30e02f5 100644
*** a/src/bin/pg_basebackup/pg_basebackup.c
--- b/src/bin/pg_basebackup/pg_basebackup.c
*************** static bool writerecoveryconf = false;
*** 66,71 ****
--- 66,72 ----
static int standby_message_timeout = 10 * 1000; /* 10 sec =
default */
static pg_time_t last_progress_report = 0;
static int32 maxrate = 0; /* no limit by default */
+ static bool profile = false;
/* Progress counters */
*************** static void progress_report(int tablespa
*** 101,106 ****
--- 102,109 ----
static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
+ static void ReceiveAndUnpackTarFileToDir(PGconn *conn,
+
const char *destination_dir);
static void GenerateRecoveryConf(PGconn *conn);
static void WriteRecoveryConf(void);
static void BaseBackup(void);
*************** usage(void)
*** 232,237 ****
--- 235,241 ----
printf(_("\nOptions controlling the output:\n"));
printf(_(" -D, --pgdata=DIRECTORY receive base backup into
directory\n"));
printf(_(" -F, --format=p|t output format (plain (default),
tar)\n"));
+ printf(_(" -B, --profile add a backup_profile file\n"));
printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer
data directory\n"
" (in kB/s, or use suffix
\"k\" or \"M\")\n"));
printf(_(" -R, --write-recovery-conf\n"
*************** ReceiveAndUnpackTarFile(PGconn *conn, PG
*** 1389,1394 ****
--- 1393,1637 ----
}
/*
+ * Receive a tar format stream from the connection to the server, and unpack
+ * the contents of it into a specified directory. Only files, directories and
+ * symlinks are supported, no other kinds of special files.
+ *
+ */
+ static void
+ ReceiveAndUnpackTarFileToDir(PGconn *conn, const char *destination_dir)
+ {
+ char filename[MAXPGPATH];
+ int current_len_left;
+ int current_padding = 0;
+ char *copybuf = NULL;
+ FILE *file = NULL;
+ PGresult *res = NULL;
+
+ /*
+ * Get the COPY data
+ */
+ res = PQgetResult(conn);
+ if (PQresultStatus(res) != PGRES_COPY_OUT)
+ {
+ fprintf(stderr, _("%s: could not get COPY data stream: %s"),
+ progname, PQerrorMessage(conn));
+ disconnect_and_exit(1);
+ }
+
+ while (1)
+ {
+ int r;
+
+ if (copybuf != NULL)
+ {
+ PQfreemem(copybuf);
+ copybuf = NULL;
+ }
+
+ r = PQgetCopyData(conn, ©buf, 0);
+
+ if (r == -1)
+ {
+ /*
+ * End of chunk
+ */
+ if (file)
+ fclose(file);
+
+ break;
+ }
+ else if (r == -2)
+ {
+ fprintf(stderr, _("%s: could not read COPY data: %s"),
+ progname, PQerrorMessage(conn));
+ disconnect_and_exit(1);
+ }
+
+ if (file == NULL)
+ {
+ int filemode;
+
+ /*
+ * No current file, so this must be the header for a
new file
+ */
+ if (r != 512)
+ {
+ fprintf(stderr, _("%s: invalid tar block header
size: %d\n"),
+ progname, r);
+ disconnect_and_exit(1);
+ }
+ totaldone += 512;
+
+ if (sscanf(copybuf + 124, "%11o", ¤t_len_left) !=
1)
+ {
+ fprintf(stderr, _("%s: could not parse file
size\n"),
+ progname);
+ disconnect_and_exit(1);
+ }
+
+ /* Set permissions on the file */
+ if (sscanf(©buf[100], "%07o ", &filemode) != 1)
+ {
+ fprintf(stderr, _("%s: could not parse file
mode\n"),
+ progname);
+ disconnect_and_exit(1);
+ }
+
+ /*
+ * All files are padded up to 512 bytes
+ */
+ current_padding =
+ ((current_len_left + 511) & ~511) -
current_len_left;
+
+ /*
+ * First part of header is zero terminated filename
+ */
+ snprintf(filename, sizeof(filename), "%s/%s",
destination_dir,
+ copybuf);
+ if (filename[strlen(filename) - 1] == '/')
+ {
+ /*
+ * Ends in a slash means directory or symlink
to directory
+ */
+ if (copybuf[156] == '5')
+ {
+ /*
+ * Directory
+ */
+ filename[strlen(filename) - 1] = '\0';
/* Remove trailing slash */
+ if (mkdir(filename, S_IRWXU) != 0)
+ {
+ /*
+ * When streaming WAL, pg_xlog
will have been created
+ * by the wal receiver process.
Also, when transaction
+ * log directory location was
specified, pg_xlog has
+ * already been created as a
symbolic link before
+ * starting the actual backup.
So just ignore failure
+ * on them.
+ */
+ if ((!streamwal &&
(strcmp(xlog_dir, "") == 0))
+ || strcmp(filename +
strlen(filename) - 8, "/pg_xlog") != 0)
+ {
+ fprintf(stderr,
+ _("%s: could not create
directory \"%s\": %s\n"),
+
progname, filename, strerror(errno));
+ disconnect_and_exit(1);
+ }
+ }
+ #ifndef WIN32
+ if (chmod(filename, (mode_t) filemode))
+ fprintf(stderr,
+ _("%s: could
not set permissions on directory \"%s\": %s\n"),
+ progname,
filename, strerror(errno));
+ #endif
+ }
+ else if (copybuf[156] == '2')
+ {
+ /*
+ * Symbolic link
+ */
+ filename[strlen(filename) - 1] = '\0';
/* Remove trailing slash */
+ if (symlink(©buf[157], filename) !=
0)
+ {
+ fprintf(stderr,
+ _("%s: could
not create symbolic link from \"%s\" to \"%s\": %s\n"),
+ progname, filename,
©buf[157], strerror(errno));
+ disconnect_and_exit(1);
+ }
+ }
+ else
+ {
+ fprintf(stderr,
+ _("%s: unrecognized
link indicator \"%c\"\n"),
+ progname, copybuf[156]);
+ disconnect_and_exit(1);
+ }
+ continue; /* directory or link
handled */
+ }
+
+ /*
+ * regular file
+ */
+ file = fopen(filename, "wb");
+ if (!file)
+ {
+ fprintf(stderr, _("%s: could not create file
\"%s\": %s\n"),
+ progname, filename,
strerror(errno));
+ disconnect_and_exit(1);
+ }
+
+ #ifndef WIN32
+ if (chmod(filename, (mode_t) filemode))
+ fprintf(stderr, _("%s: could not set
permissions on file \"%s\": %s\n"),
+ progname, filename,
strerror(errno));
+ #endif
+
+ if (current_len_left == 0)
+ {
+ /*
+ * Done with this file, next one will be a new
tar header
+ */
+ fclose(file);
+ file = NULL;
+ continue;
+ }
+ } /* new file */
+ else
+ {
+ /*
+ * Continuing blocks in existing file
+ */
+ if (current_len_left == 0 && r == current_padding)
+ {
+ /*
+ * Received the padding block for this file,
ignore it and
+ * close the file, then move on to the next tar
header.
+ */
+ fclose(file);
+ file = NULL;
+ totaldone += r;
+ continue;
+ }
+
+ if (fwrite(copybuf, r, 1, file) != 1)
+ {
+ fprintf(stderr, _("%s: could not write to file
\"%s\": %s\n"),
+ progname, filename,
strerror(errno));
+ disconnect_and_exit(1);
+ }
+ totaldone += r;
+
+ current_len_left -= r;
+ if (current_len_left == 0 && current_padding == 0)
+ {
+ /*
+ * Received the last block, and there is no
padding to be
+ * expected. Close the file and move on to the
next tar
+ * header.
+ */
+ fclose(file);
+ file = NULL;
+ continue;
+ }
+ } /* continuing
data in existing file */
+ } /* loop over
all data blocks */
+
+ if (file != NULL)
+ {
+ fprintf(stderr,
+ _("%s: COPY stream ended before last file was
finished\n"),
+ progname);
+ disconnect_and_exit(1);
+ }
+
+ if (copybuf != NULL)
+ PQfreemem(copybuf);
+
+ }
+
+
+ /*
* Escape a parameter value so that it can be used as part of a libpq
* connection string, e.g. in:
*
*************** BaseBackup(void)
*** 1664,1676 ****
maxrate_clause = psprintf("MAX_RATE %u", maxrate);
basebkp =
! psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s",
escaped_label,
showprogress ? "PROGRESS" : "",
includewal && !streamwal ? "WAL" : "",
fastcheckpoint ? "FAST" : "",
includewal ? "NOWAIT" : "",
! maxrate_clause ? maxrate_clause : "");
if (PQsendQuery(conn, basebkp) == 0)
{
--- 1907,1920 ----
maxrate_clause = psprintf("MAX_RATE %u", maxrate);
basebkp =
! psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s",
escaped_label,
showprogress ? "PROGRESS" : "",
includewal && !streamwal ? "WAL" : "",
fastcheckpoint ? "FAST" : "",
includewal ? "NOWAIT" : "",
! maxrate_clause ? maxrate_clause : "",
! profile ? "PROFILE" : "");
if (PQsendQuery(conn, basebkp) == 0)
{
*************** BaseBackup(void)
*** 1829,1834 ****
--- 2073,2086 ----
fprintf(stderr, "transaction log end point: %s\n", xlogend);
PQclear(res);
+ /*
+ * Get the backup profile
+ */
+ if (profile)
+ {
+ ReceiveAndUnpackTarFileToDir(conn, basedir);
+ }
+
res = PQgetResult(conn);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
*************** main(int argc, char **argv)
*** 1968,1973 ****
--- 2220,2226 ----
{"username", required_argument, NULL, 'U'},
{"no-password", no_argument, NULL, 'w'},
{"password", no_argument, NULL, 'W'},
+ {"profile", no_argument, NULL, 'B'},
{"status-interval", required_argument, NULL, 's'},
{"verbose", no_argument, NULL, 'v'},
{"progress", no_argument, NULL, 'P'},
*************** main(int argc, char **argv)
*** 1996,2002 ****
}
}
! while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:zZ:d:c:h:p:U:s:wWvP",
long_options,
&option_index)) != -1)
{
switch (c)
--- 2249,2255 ----
}
}
! while ((c = getopt_long(argc, argv,
"D:F:r:RT:xX:l:zZ:d:c:h:p:U:s:wWBvP",
long_options,
&option_index)) != -1)
{
switch (c)
*************** main(int argc, char **argv)
*** 2114,2119 ****
--- 2367,2375 ----
case 'W':
dbgetpassword = 1;
break;
+ case 'B':
+ profile = 1;
+ break;
case 's':
standby_message_timeout = atoi(optarg) * 1000;
if (standby_message_timeout < 0)
diff --git a/src/include/replication/basebackup.h
b/src/include/replication/basebackup.h
index 988bce7..394ec45 100644
*** a/src/include/replication/basebackup.h
--- b/src/include/replication/basebackup.h
***************
*** 20,25 ****
--- 20,27 ----
#define MAX_RATE_LOWER 32
#define MAX_RATE_UPPER 1048576
+ /* Profile metadata file names */
+ #define BACKUP_PROFILE_FILE "backup_profile"
extern void SendBaseBackup(BaseBackupCmd *cmd);
diff --git a/src/include/utils/md5.h b/src/include/utils/md5.h
index ...cd856bf 100644
*** a/src/include/utils/md5.h
--- b/src/include/utils/md5.h
***************
*** 0 ****
--- 1,79 ----
+ /* src/include/utils/md5.h */
+ /* $KAME: md5.h,v 1.3 2000/02/22 14:01:18 itojun Exp $ */
+
+ /*
+ * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the project nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+ #ifndef _NETINET6_MD5_H_
+ #define _NETINET6_MD5_H_
+
+ #define MD5_BUFLEN 64
+
+ typedef struct
+ {
+ union
+ {
+ uint32 md5_state32[4];
+ uint8 md5_state8[16];
+ } md5_st;
+
+ #define md5_sta md5_st.md5_state32[0]
+ #define md5_stb md5_st.md5_state32[1]
+ #define md5_stc md5_st.md5_state32[2]
+ #define md5_std md5_st.md5_state32[3]
+ #define md5_st8 md5_st.md5_state8
+
+ union
+ {
+ uint64 md5_count64;
+ uint8 md5_count8[8];
+ } md5_count;
+ #define md5_n md5_count.md5_count64
+ #define md5_n8 md5_count.md5_count8
+
+ unsigned int md5_i;
+ uint8 md5_buf[MD5_BUFLEN];
+ } md5_ctxt;
+
+ extern void md5_init(md5_ctxt *);
+ extern void md5_loop(md5_ctxt *, const uint8 *, unsigned int);
+ extern void md5_pad(md5_ctxt *);
+ extern void md5_result(uint8 *, md5_ctxt *);
+
+ /* compatibility */
+ #define MD5_CTX md5_ctxt
+ #define MD5Init(x) md5_init((x))
+ #define MD5Update(x, y, z) md5_loop((x), (y), (z))
+ #define MD5Final(x, y) \
+ do { \
+ md5_pad((y)); \
+ md5_result((x), (y)); \
+ } while (0)
+
+ #endif /* ! _NETINET6_MD5_H_ */
signature.asc
Description: OpenPGP digital signature
