Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On Tue, Jun 17, 2014 at 08:46:00PM -0400, Bruce Momjian wrote:
On Tue, Jun 17, 2014 at 07:12:16PM -0400, Tom Lane wrote:
Bruce Momjian br...@momjian.us writes:
On Tue, Jun 17, 2014 at 03:55:02PM -0400, Alvaro Herrera wrote:
Can't you compare it to the historic default value? I mean, add an
assumption that people thus far has never tweaked it.
Well, if they did tweak it, then they would be unable to use pg_upgrade
because it would complain about a mismatch if they actually matched the
old and new servers.
What about comparing to the symbolic value LOBLKSIZE? This would make
pg_upgrade assume that the old installation had been tweaked the same
as in its own build. This ends up being the same as what you said,
ie, effectively no comparison ... but it might be less complicated to
code/understand.
OK, assume the compiled-in default is the value for an old cluster that
has no value --- yeah, I could do that.
Turns out I already had values that could be missing in the old cluster,
so I just used the same format for this, rather than testing for
LOBLKSIZE.
Attached patch applied.
--
Bruce Momjian br...@momjian.ushttp://momjian.us
EnterpriseDB http://enterprisedb.com
+ Everyone has their own god. +
diff --git a/contrib/pg_upgrade/controldata.c b/contrib/pg_upgrade/controldata.c
new file mode 100644
index 9282b8e..d105a59
*** a/contrib/pg_upgrade/controldata.c
--- b/contrib/pg_upgrade/controldata.c
*** get_control_data(ClusterInfo *cluster, b
*** 54,59
--- 54,60
bool got_ident = false;
bool got_index = false;
bool got_toast = false;
+ bool got_large_object = false;
bool got_date_is_int = false;
bool got_float8_pass_by_value = false;
bool got_data_checksum_version = false;
*** get_control_data(ClusterInfo *cluster, b
*** 357,362
--- 358,374
cluster-controldata.toast = str2uint(p);
got_toast = true;
}
+ else if ((p = strstr(bufin, Size of a large-object chunk:)) != NULL)
+ {
+ p = strchr(p, ':');
+
+ if (p == NULL || strlen(p) = 1)
+ pg_fatal(%d: controldata retrieval problem\n, __LINE__);
+
+ p++;/* removing ':' char */
+ cluster-controldata.large_object = str2uint(p);
+ got_large_object = true;
+ }
else if ((p = strstr(bufin, Date/time type storage:)) != NULL)
{
p = strchr(p, ':');
*** get_control_data(ClusterInfo *cluster, b
*** 475,480
--- 487,494
!got_tli ||
!got_align || !got_blocksz || !got_largesz || !got_walsz ||
!got_walseg || !got_ident || !got_index || !got_toast ||
+ (!got_large_object
+ cluster-controldata.cat_ver = LARGE_OBJECT_SIZE_PG_CONTROL_VER) ||
!got_date_is_int || !got_float8_pass_by_value || !got_data_checksum_version)
{
pg_log(PG_REPORT,
*** get_control_data(ClusterInfo *cluster, b
*** 527,532
--- 541,550
if (!got_toast)
pg_log(PG_REPORT, maximum TOAST chunk size\n);
+ if (!got_large_object
+ cluster-controldata.cat_ver = LARGE_OBJECT_SIZE_PG_CONTROL_VER)
+ pg_log(PG_REPORT, large-object chunk size\n);
+
if (!got_date_is_int)
pg_log(PG_REPORT, dates/times are integers?\n);
*** check_control_data(ControlData *oldctrl,
*** 576,581
--- 594,602
if (oldctrl-toast == 0 || oldctrl-toast != newctrl-toast)
pg_fatal(old and new pg_controldata maximum TOAST chunk sizes are invalid or do not match\n);
+ if (oldctrl-large_object == 0 || oldctrl-large_object != newctrl-large_object)
+ pg_fatal(old and new pg_controldata large-object chunk sizes are invalid or do not match\n);
+
if (oldctrl-date_is_int != newctrl-date_is_int)
pg_fatal(old and new pg_controldata date/time storage types do not match\n);
diff --git a/contrib/pg_upgrade/pg_upgrade.h b/contrib/pg_upgrade/pg_upgrade.h
new file mode 100644
index 1ac3394..0207391
*** a/contrib/pg_upgrade/pg_upgrade.h
--- b/contrib/pg_upgrade/pg_upgrade.h
*** extern char *output_files[];
*** 116,121
--- 116,127
#define MULTIXACT_FORMATCHANGE_CAT_VER 201301231
/*
+ * large object chunk size added to pg_controldata,
+ * commit 5f93c37805e7485488480916b4585e098d3cc883
+ */
+ #define LARGE_OBJECT_SIZE_PG_CONTROL_VER 942
+
+ /*
* Each relation is represented by a relinfo structure.
*/
typedef struct
*** typedef struct
*** 203,208
--- 209,215
uint32 ident;
uint32 index;
uint32 toast;
+ uint32 large_object;
bool date_is_int;
bool float8_pass_by_value;
bool data_checksum_version;
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On Tue, Jun 17, 2014 at 8:46 PM, Bruce Momjian br...@momjian.us wrote: On Tue, Jun 17, 2014 at 07:12:16PM -0400, Tom Lane wrote: Bruce Momjian br...@momjian.us writes: On Tue, Jun 17, 2014 at 03:55:02PM -0400, Alvaro Herrera wrote: Can't you compare it to the historic default value? I mean, add an assumption that people thus far has never tweaked it. Well, if they did tweak it, then they would be unable to use pg_upgrade because it would complain about a mismatch if they actually matched the old and new servers. What about comparing to the symbolic value LOBLKSIZE? This would make pg_upgrade assume that the old installation had been tweaked the same as in its own build. This ends up being the same as what you said, ie, effectively no comparison ... but it might be less complicated to code/understand. OK, assume the compiled-in default is the value for an old cluster that has no value --- yeah, I could do that. I'm not really sure why this is better than Bruce's original proposal, though. -- Robert Haas EnterpriseDB: http://www.enterprisedb.com The Enterprise PostgreSQL Company -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
Robert Haas robertmh...@gmail.com writes: On Tue, Jun 17, 2014 at 8:46 PM, Bruce Momjian br...@momjian.us wrote: On Tue, Jun 17, 2014 at 07:12:16PM -0400, Tom Lane wrote: What about comparing to the symbolic value LOBLKSIZE? This would make pg_upgrade assume that the old installation had been tweaked the same as in its own build. This ends up being the same as what you said, ie, effectively no comparison ... but it might be less complicated to code/understand. OK, assume the compiled-in default is the value for an old cluster that has no value --- yeah, I could do that. I'm not really sure why this is better than Bruce's original proposal, though. The net behavior would be the same, but I thought it might be easier to code by thinking of it this way. Or maybe it wouldn't --- it's just a suggestion. regards, tom lane -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On Wed, Jun 18, 2014 at 2:55 PM, Tom Lane t...@sss.pgh.pa.us wrote: Robert Haas robertmh...@gmail.com writes: On Tue, Jun 17, 2014 at 8:46 PM, Bruce Momjian br...@momjian.us wrote: On Tue, Jun 17, 2014 at 07:12:16PM -0400, Tom Lane wrote: What about comparing to the symbolic value LOBLKSIZE? This would make pg_upgrade assume that the old installation had been tweaked the same as in its own build. This ends up being the same as what you said, ie, effectively no comparison ... but it might be less complicated to code/understand. OK, assume the compiled-in default is the value for an old cluster that has no value --- yeah, I could do that. I'm not really sure why this is better than Bruce's original proposal, though. The net behavior would be the same, but I thought it might be easier to code by thinking of it this way. Or maybe it wouldn't --- it's just a suggestion. Well, the difference is that if we just don't check it, there can never be an error. Basically, it's the user's job to DTRT. If we check it against some semi-arbitrary value, we'll catch the case where the old cluster was modified with a custom setting and the new one was not - but couldn't we also get false positives under obscure circumstances? -- Robert Haas EnterpriseDB: http://www.enterprisedb.com The Enterprise PostgreSQL Company -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
Robert Haas robertmh...@gmail.com writes: On Wed, Jun 18, 2014 at 2:55 PM, Tom Lane t...@sss.pgh.pa.us wrote: The net behavior would be the same, but I thought it might be easier to code by thinking of it this way. Or maybe it wouldn't --- it's just a suggestion. Well, the difference is that if we just don't check it, there can never be an error. Basically, it's the user's job to DTRT. If we check it against some semi-arbitrary value, we'll catch the case where the old cluster was modified with a custom setting and the new one was not - but couldn't we also get false positives under obscure circumstances? Huh? What we'd be checking is the LOBLKSIZE compiled into pg_upgrade versus that stored into pg_control by the new postmaster. If those are different, then pg_upgrade didn't come from the same build as the new postmaster, which is already a pretty hazardous situation (especially if the user is fooling with low-level stuff like this). regards, tom lane -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On Wed, Jun 18, 2014 at 3:28 PM, Tom Lane t...@sss.pgh.pa.us wrote: Robert Haas robertmh...@gmail.com writes: On Wed, Jun 18, 2014 at 2:55 PM, Tom Lane t...@sss.pgh.pa.us wrote: The net behavior would be the same, but I thought it might be easier to code by thinking of it this way. Or maybe it wouldn't --- it's just a suggestion. Well, the difference is that if we just don't check it, there can never be an error. Basically, it's the user's job to DTRT. If we check it against some semi-arbitrary value, we'll catch the case where the old cluster was modified with a custom setting and the new one was not - but couldn't we also get false positives under obscure circumstances? Huh? What we'd be checking is the LOBLKSIZE compiled into pg_upgrade versus that stored into pg_control by the new postmaster. If those are different, then pg_upgrade didn't come from the same build as the new postmaster, which is already a pretty hazardous situation (especially if the user is fooling with low-level stuff like this). OK, I agree that checking that wouldn't hurt anything. -- Robert Haas EnterpriseDB: http://www.enterprisedb.com The Enterprise PostgreSQL Company -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On Wed, Jun 4, 2014 at 06:57:31PM -0400, Tom Lane wrote: Stephen Frost sfr...@snowman.net writes: * Tom Lane (t...@sss.pgh.pa.us) wrote: There are at least two places in inv_api.c where we have Assert(pagelen = LOBLKSIZE) that is protecting a subsequent memcpy into a local variable of size LOBLKSIZE, so that the only thing standing between us and a stack-smash security issue that's trivially exploitable in production builds is that on-disk data conforms to our expectation about LOBLKSIZE. I think it's definitely worth promoting these checks to regular runtime-if-test-and-elog. Agreed. Promoting that to a run-time check seems well worth it to me. Here's a draft patch for this. Barring objections I'll commit the whole thing to HEAD, and the inv_api.c changes to the back branches as well. Uh, I think pg_upgrade needs to check that they match too. -- Bruce Momjian br...@momjian.ushttp://momjian.us EnterpriseDB http://enterprisedb.com + Everyone has their own god. + -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
Bruce Momjian br...@momjian.us writes: Uh, I think pg_upgrade needs to check that they match too. Possibly. What do you think it should do when examining a pg_control version that lacks the field? regards, tom lane -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On Tue, Jun 17, 2014 at 12:28:46PM -0400, Tom Lane wrote: Bruce Momjian br...@momjian.us writes: Uh, I think pg_upgrade needs to check that they match too. Possibly. What do you think it should do when examining a pg_control version that lacks the field? Good question. I have existing cases where fields were removed, but not ones that were added. As we have no way to query the old cluster's value for LOBLKSIZE, I think I will just add code to compare them if they _both_ exist. -- Bruce Momjian br...@momjian.ushttp://momjian.us EnterpriseDB http://enterprisedb.com + Everyone has their own god. + -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
Bruce Momjian wrote: On Tue, Jun 17, 2014 at 12:28:46PM -0400, Tom Lane wrote: Bruce Momjian br...@momjian.us writes: Uh, I think pg_upgrade needs to check that they match too. Possibly. What do you think it should do when examining a pg_control version that lacks the field? Good question. I have existing cases where fields were removed, but not ones that were added. As we have no way to query the old cluster's value for LOBLKSIZE, I think I will just add code to compare them if they _both_ exist. Can't you compare it to the historic default value? I mean, add an assumption that people thus far has never tweaked it. -- Álvaro Herrerahttp://www.2ndQuadrant.com/ PostgreSQL Development, 24x7 Support, Training Services -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On Tue, Jun 17, 2014 at 03:55:02PM -0400, Alvaro Herrera wrote: Bruce Momjian wrote: On Tue, Jun 17, 2014 at 12:28:46PM -0400, Tom Lane wrote: Bruce Momjian br...@momjian.us writes: Uh, I think pg_upgrade needs to check that they match too. Possibly. What do you think it should do when examining a pg_control version that lacks the field? Good question. I have existing cases where fields were removed, but not ones that were added. As we have no way to query the old cluster's value for LOBLKSIZE, I think I will just add code to compare them if they _both_ exist. Can't you compare it to the historic default value? I mean, add an assumption that people thus far has never tweaked it. Well, if they did tweak it, then they would be unable to use pg_upgrade because it would complain about a mismatch if they actually matched the old and new servers. -- Bruce Momjian br...@momjian.ushttp://momjian.us EnterpriseDB http://enterprisedb.com + Everyone has their own god. + -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
Bruce Momjian br...@momjian.us writes: On Tue, Jun 17, 2014 at 03:55:02PM -0400, Alvaro Herrera wrote: Can't you compare it to the historic default value? I mean, add an assumption that people thus far has never tweaked it. Well, if they did tweak it, then they would be unable to use pg_upgrade because it would complain about a mismatch if they actually matched the old and new servers. What about comparing to the symbolic value LOBLKSIZE? This would make pg_upgrade assume that the old installation had been tweaked the same as in its own build. This ends up being the same as what you said, ie, effectively no comparison ... but it might be less complicated to code/understand. regards, tom lane -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On Tue, Jun 17, 2014 at 07:12:16PM -0400, Tom Lane wrote: Bruce Momjian br...@momjian.us writes: On Tue, Jun 17, 2014 at 03:55:02PM -0400, Alvaro Herrera wrote: Can't you compare it to the historic default value? I mean, add an assumption that people thus far has never tweaked it. Well, if they did tweak it, then they would be unable to use pg_upgrade because it would complain about a mismatch if they actually matched the old and new servers. What about comparing to the symbolic value LOBLKSIZE? This would make pg_upgrade assume that the old installation had been tweaked the same as in its own build. This ends up being the same as what you said, ie, effectively no comparison ... but it might be less complicated to code/understand. OK, assume the compiled-in default is the value for an old cluster that has no value --- yeah, I could do that. -- Bruce Momjian br...@momjian.ushttp://momjian.us EnterpriseDB http://enterprisedb.com + Everyone has their own god. + -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
[HACKERS] pg_control is missing a field for LOBLKSIZE
I just chanced to notice that if someone were to change the value for LOBLKSIZE and recompile, there'd be nothing to stop him from starting that postmaster against an existing database, even though it would completely misinterpret and mangle any data in pg_largeobject. I think there ought to be a guard for that, for exactly the same reasons that we check TOAST_MAX_CHUNK_SIZE: correct interpretation of on-disk data requires that this value match the original database configuration. Obviously it's too late to do anything about this in existing branches, but I propose to add a field to pg_control after we branch off 9.4. regards, tom lane -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On 06/04/2014 10:03 AM, Tom Lane wrote: I just chanced to notice that if someone were to change the value for LOBLKSIZE and recompile, there'd be nothing to stop him from starting that postmaster against an existing database, even though it would completely misinterpret and mangle any data in pg_largeobject. I think there ought to be a guard for that, for exactly the same reasons that we check TOAST_MAX_CHUNK_SIZE: correct interpretation of on-disk data requires that this value match the original database configuration. Obviously it's too late to do anything about this in existing branches, but I propose to add a field to pg_control after we branch off 9.4. If we did an initdb-requiring change for 9.4 could we piggy-back this onto it? cheers andrew -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On 2014-06-04 10:25:07 -0400, Andrew Dunstan wrote: If we did an initdb-requiring change for 9.4 could we piggy-back this onto it? Do you know of a problem requiring that? Greetings, Andres Freund -- Andres Freund http://www.2ndQuadrant.com/ PostgreSQL Development, 24x7 Support, Training Services -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
* Andrew Dunstan (and...@dunslane.net) wrote: On 06/04/2014 10:03 AM, Tom Lane wrote: I just chanced to notice that if someone were to change the value for LOBLKSIZE and recompile, there'd be nothing to stop him from starting that postmaster against an existing database, even though it would completely misinterpret and mangle any data in pg_largeobject. I think there ought to be a guard for that, for exactly the same reasons that we check TOAST_MAX_CHUNK_SIZE: correct interpretation of on-disk data requires that this value match the original database configuration. Obviously it's too late to do anything about this in existing branches, but I propose to add a field to pg_control after we branch off 9.4. If we did an initdb-requiring change for 9.4 could we piggy-back this onto it? I was thinking more-or-less the same thing... Then again, I've never heard of a field complaint regarding this, so pehraps it's not worth it. Thanks, Stephen signature.asc Description: Digital signature
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On 06/04/2014 10:27 AM, Andres Freund wrote: On 2014-06-04 10:25:07 -0400, Andrew Dunstan wrote: If we did an initdb-requiring change for 9.4 could we piggy-back this onto it? Do you know of a problem requiring that? No, just thinking ahead. cheers andrew -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
* Tom Lane (t...@sss.pgh.pa.us) wrote: Stephen Frost sfr...@snowman.net writes: * Andrew Dunstan (and...@dunslane.net) wrote: On 06/04/2014 10:03 AM, Tom Lane wrote: I just chanced to notice that if someone were to change the value for LOBLKSIZE and recompile, there'd be nothing to stop him from starting that postmaster against an existing database, even though it would completely misinterpret and mangle any data in pg_largeobject. Then again, I've never heard of a field complaint regarding this, so pehraps it's not worth it. I've not heard one either, but there was just somebody asking in pgsql-general about changing LOBLKSIZE, so he's going to be at risk. That's not a big enough sample size to make me panic about getting a hasty fix into 9.4, but I do think we should fix this going forward. Agreed. Thanks, Stephen signature.asc Description: Digital signature
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
Stephen Frost sfr...@snowman.net writes: * Andrew Dunstan (and...@dunslane.net) wrote: On 06/04/2014 10:03 AM, Tom Lane wrote: I just chanced to notice that if someone were to change the value for LOBLKSIZE and recompile, there'd be nothing to stop him from starting that postmaster against an existing database, even though it would completely misinterpret and mangle any data in pg_largeobject. Then again, I've never heard of a field complaint regarding this, so pehraps it's not worth it. I've not heard one either, but there was just somebody asking in pgsql-general about changing LOBLKSIZE, so he's going to be at risk. That's not a big enough sample size to make me panic about getting a hasty fix into 9.4, but I do think we should fix this going forward. regards, tom lane -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On 2014-06-04 10:03:09 -0400, Tom Lane wrote: I just chanced to notice that if someone were to change the value for LOBLKSIZE and recompile, there'd be nothing to stop him from starting that postmaster against an existing database, even though it would completely misinterpret and mangle any data in pg_largeobject. I think there ought to be a guard for that, for exactly the same reasons that we check TOAST_MAX_CHUNK_SIZE: correct interpretation of on-disk data requires that this value match the original database configuration. Obviously it's too late to do anything about this in existing branches, but I propose to add a field to pg_control after we branch off 9.4. Btw, I had wondered before if we shouldn't also add sizeof(long) to pg_control to catch cases where a database is copied between a LLP64 (64bit windows) and an LP64 (nearly every other 64bit system) system. I have my doubts that we're completely clean about the size difference. Not to speak of extension datatypes. Greetings, Andres Freund -- Andres Freund http://www.2ndQuadrant.com/ PostgreSQL Development, 24x7 Support, Training Services -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
Andres Freund and...@2ndquadrant.com writes: Btw, I had wondered before if we shouldn't also add sizeof(long) to pg_control to catch cases where a database is copied between a LLP64 (64bit windows) and an LP64 (nearly every other 64bit system) system. I have my doubts that we're completely clean about the size difference. Not to speak of extension datatypes. I don't believe that this is necessary. It's certainly true that some in-memory structures will be laid out differently, but not on-disk. regards, tom lane -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
Stephen Frost sfr...@snowman.net writes: * Tom Lane (t...@sss.pgh.pa.us) wrote: I've not heard one either, but there was just somebody asking in pgsql-general about changing LOBLKSIZE, so he's going to be at risk. That's not a big enough sample size to make me panic about getting a hasty fix into 9.4, but I do think we should fix this going forward. Agreed. BTW, just comparing the handling of TOAST_MAX_CHUNK_SIZE and LOBLKSIZE, I noticed that the tuptoaster.c functions are reasonably paranoid about checking that toast chunks are the expected size, but the large object functions are not: the latter have either no check at all, or just an Assert that the size is not more than expected. So we could provide at least a partial guard against a wrong LOBLKSIZE configuration by making all the large-object functions throw elog(ERROR) if the length of a LO chunk is more than LOBLKSIZE. Unfortunately, length *less* than LOBLKSIZE is an expected case, so this would only help in one direction. Still, it'd be an easy and back-patchable change that would provide at least some defense, so I'm thinking of doing it. regards, tom lane -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
On Wed, Jun 4, 2014 at 1:21 PM, Tom Lane t...@sss.pgh.pa.us wrote: Stephen Frost sfr...@snowman.net writes: * Tom Lane (t...@sss.pgh.pa.us) wrote: I've not heard one either, but there was just somebody asking in pgsql-general about changing LOBLKSIZE, so he's going to be at risk. That's not a big enough sample size to make me panic about getting a hasty fix into 9.4, but I do think we should fix this going forward. Agreed. BTW, just comparing the handling of TOAST_MAX_CHUNK_SIZE and LOBLKSIZE, I noticed that the tuptoaster.c functions are reasonably paranoid about checking that toast chunks are the expected size, but the large object functions are not: the latter have either no check at all, or just an Assert that the size is not more than expected. So we could provide at least a partial guard against a wrong LOBLKSIZE configuration by making all the large-object functions throw elog(ERROR) if the length of a LO chunk is more than LOBLKSIZE. Unfortunately, length *less* than LOBLKSIZE is an expected case, so this would only help in one direction. Still, it'd be an easy and back-patchable change that would provide at least some defense, so I'm thinking of doing it. This seems like a pretty weak argument for adding run-time overhead. Maybe it can be justified on the grounds that it would catch corrupted TOAST data, but I've never heard of anyone changing LOBLKSIZE and see no reason to get agitated about it. -- Robert Haas EnterpriseDB: http://www.enterprisedb.com The Enterprise PostgreSQL Company -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
Robert Haas robertmh...@gmail.com writes: On Wed, Jun 4, 2014 at 1:21 PM, Tom Lane t...@sss.pgh.pa.us wrote: BTW, just comparing the handling of TOAST_MAX_CHUNK_SIZE and LOBLKSIZE, I noticed that the tuptoaster.c functions are reasonably paranoid about checking that toast chunks are the expected size, but the large object functions are not: the latter have either no check at all, or just an Assert that the size is not more than expected. So we could provide at least a partial guard against a wrong LOBLKSIZE configuration by making all the large-object functions throw elog(ERROR) if the length of a LO chunk is more than LOBLKSIZE. Unfortunately, length *less* than LOBLKSIZE is an expected case, so this would only help in one direction. Still, it'd be an easy and back-patchable change that would provide at least some defense, so I'm thinking of doing it. This seems like a pretty weak argument for adding run-time overhead. Maybe it can be justified on the grounds that it would catch corrupted TOAST data, but I've never heard of anyone changing LOBLKSIZE and see no reason to get agitated about it. One if-test per fetched tuple hardly seems likely to add measurable overhead. As for never heard of, see today's thread in pgsql-general: http://www.postgresql.org/message-id/flat/CAGou9Mg=9qpytdh18ndo3ltjtwqn8urdtwabfkcymrut6d_...@mail.gmail.com There was a similar gripe a few months ago: http://www.postgresql.org/message-id/cacg6vwxy_84shccxzncsz9xlfwnx5szvqru6ancrr0c3xw1...@mail.gmail.com There are at least two places in inv_api.c where we have Assert(pagelen = LOBLKSIZE) that is protecting a subsequent memcpy into a local variable of size LOBLKSIZE, so that the only thing standing between us and a stack-smash security issue that's trivially exploitable in production builds is that on-disk data conforms to our expectation about LOBLKSIZE. I think it's definitely worth promoting these checks to regular runtime-if-test-and-elog. regards, tom lane -- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
* Tom Lane (t...@sss.pgh.pa.us) wrote: There are at least two places in inv_api.c where we have Assert(pagelen = LOBLKSIZE) that is protecting a subsequent memcpy into a local variable of size LOBLKSIZE, so that the only thing standing between us and a stack-smash security issue that's trivially exploitable in production builds is that on-disk data conforms to our expectation about LOBLKSIZE. I think it's definitely worth promoting these checks to regular runtime-if-test-and-elog. Agreed. Promoting that to a run-time check seems well worth it to me. Thanks, Stephen signature.asc Description: Digital signature
Re: [HACKERS] pg_control is missing a field for LOBLKSIZE
Stephen Frost sfr...@snowman.net writes:
* Tom Lane (t...@sss.pgh.pa.us) wrote:
There are at least two places in inv_api.c where we have
Assert(pagelen = LOBLKSIZE) that is protecting a subsequent memcpy
into a local variable of size LOBLKSIZE, so that the only thing standing
between us and a stack-smash security issue that's trivially exploitable
in production builds is that on-disk data conforms to our expectation
about LOBLKSIZE. I think it's definitely worth promoting these checks
to regular runtime-if-test-and-elog.
Agreed. Promoting that to a run-time check seems well worth it to me.
Here's a draft patch for this. Barring objections I'll commit the whole
thing to HEAD, and the inv_api.c changes to the back branches as well.
regards, tom lane
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index d675560..a61878e 100644
*** a/src/backend/access/transam/xlog.c
--- b/src/backend/access/transam/xlog.c
***
*** 49,54
--- 49,55
#include storage/bufmgr.h
#include storage/fd.h
#include storage/ipc.h
+ #include storage/large_object.h
#include storage/latch.h
#include storage/pmsignal.h
#include storage/predicate.h
*** WriteControlFile(void)
*** 4352,4357
--- 4353,4359
ControlFile-indexMaxKeys = INDEX_MAX_KEYS;
ControlFile-toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE;
+ ControlFile-loblksize = LOBLKSIZE;
#ifdef HAVE_INT64_TIMESTAMP
ControlFile-enableIntTimes = true;
*** ReadControlFile(void)
*** 4545,4550
--- 4547,4559
but the server was compiled with TOAST_MAX_CHUNK_SIZE %d.,
ControlFile-toast_max_chunk_size, (int) TOAST_MAX_CHUNK_SIZE),
errhint(It looks like you need to recompile or initdb.)));
+ if (ControlFile-loblksize != LOBLKSIZE)
+ ereport(FATAL,
+ (errmsg(database files are incompatible with server),
+ errdetail(The database cluster was initialized with LOBLKSIZE %d,
+ but the server was compiled with LOBLKSIZE %d.,
+ ControlFile-loblksize, (int) LOBLKSIZE),
+ errhint(It looks like you need to recompile or initdb.)));
#ifdef HAVE_INT64_TIMESTAMP
if (ControlFile-enableIntTimes != true)
diff --git a/src/backend/storage/large_object/inv_api.c b/src/backend/storage/large_object/inv_api.c
index 57ec1c2..0918142 100644
*** a/src/backend/storage/large_object/inv_api.c
--- b/src/backend/storage/large_object/inv_api.c
*** myLargeObjectExists(Oid loid, Snapshot s
*** 173,185
}
! static int32
! getbytealen(bytea *data)
{
! Assert(!VARATT_IS_EXTENDED(data));
! if (VARSIZE(data) VARHDRSZ)
! elog(ERROR, invalid VARSIZE(data));
! return (VARSIZE(data) - VARHDRSZ);
}
--- 173,210
}
! /*
! * Extract data field from a pg_largeobject tuple, detoasting if needed
! * and verifying that the length is sane. Returns data pointer (a bytea *),
! * data length, and an indication of whether to pfree the data pointer.
! */
! static void
! getdatafield(Form_pg_largeobject tuple,
! bytea **pdatafield,
! int *plen,
! bool *pfreeit)
{
! bytea *datafield;
! int len;
! bool freeit;
!
! datafield = (tuple-data); /* see note at top of file */
! freeit = false;
! if (VARATT_IS_EXTENDED(datafield))
! {
! datafield = (bytea *)
! heap_tuple_untoast_attr((struct varlena *) datafield);
! freeit = true;
! }
! len = VARSIZE(datafield) - VARHDRSZ;
! if (len 0 || len LOBLKSIZE)
! ereport(ERROR,
! (errcode(ERRCODE_DATA_CORRUPTED),
! errmsg(pg_largeobject entry for OID %u, page %d has invalid data field size %d,
! tuple-loid, tuple-pageno, len)));
! *pdatafield = datafield;
! *plen = len;
! *pfreeit = freeit;
}
*** inv_getsize(LargeObjectDesc *obj_desc)
*** 366,385
{
Form_pg_largeobject data;
bytea *datafield;
bool pfreeit;
if (HeapTupleHasNulls(tuple)) /* paranoia */
elog(ERROR, null field found in pg_largeobject);
data = (Form_pg_largeobject) GETSTRUCT(tuple);
! datafield = (data-data); /* see note at top of file */
! pfreeit = false;
! if (VARATT_IS_EXTENDED(datafield))
! {
! datafield = (bytea *)
! heap_tuple_untoast_attr((struct varlena *) datafield);
! pfreeit = true;
! }
! lastbyte = (uint64) data-pageno * LOBLKSIZE + getbytealen(datafield);
if (pfreeit)
pfree(datafield);
}
--- 391,404
{
Form_pg_largeobject data;
bytea *datafield;
+ int len;
bool pfreeit;
if (HeapTupleHasNulls(tuple)) /* paranoia */
elog(ERROR, null field found in pg_largeobject);
data = (Form_pg_largeobject) GETSTRUCT(tuple);
! getdatafield(data, datafield, len, pfreeit);
! lastbyte = (uint64) data-pageno * LOBLKSIZE + len;
if (pfreeit)
pfree(datafield);
}
*** inv_read(LargeObjectDesc *obj_desc, char
*** 506,520
off = (int) (obj_desc-offset - pageoff);
