[Cluster-devel] [PATCH v3 08/20] gfs2: Get rid of on-stack transactions
On-stack transactions were introduced to work around a transaction glock deadlock in gfs2_trans_begin in commit d8348de06f70 ("GFS2: Fix deadlock on journal flush"). Subsequently, transaction glocks were eliminated in favor of the more efficient freeze glocks in commit 24972557b12c ("GFS2: remove transaction glock") without also removing the on-stack transactions. It has now turned out that committing on-stack transactions significantly complicates journal free space accounting when no system transaction (sdp->sd_log_tr) is active at the time. It doesn't seem that on-stack transactions provide a significant benefit beyond their original purpose (as an optimization), so remove them to allow fixing the journal free space accounting in a reasonable way in a subsequent patch. FIXME: Can we better handle a gfs2_trans_begin failure in gfs2_ail_empty_gl? If we skip the __gfs2_ail_flush, we'll just end up with leftover items on gl_ail_list. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/glops.c | 29 +++-- fs/gfs2/incore.h | 1 - fs/gfs2/log.c| 1 - fs/gfs2/trans.c | 25 + fs/gfs2/trans.h | 2 ++ 5 files changed, 22 insertions(+), 36 deletions(-) diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c index 3faa421568b0..853e590ccc15 100644 --- a/fs/gfs2/glops.c +++ b/fs/gfs2/glops.c @@ -84,18 +84,11 @@ static void __gfs2_ail_flush(struct gfs2_glock *gl, bool fsync, static int gfs2_ail_empty_gl(struct gfs2_glock *gl) { + unsigned int revokes = atomic_read(&gl->gl_ail_count); struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; - struct gfs2_trans tr; int ret; - memset(&tr, 0, sizeof(tr)); - INIT_LIST_HEAD(&tr.tr_buf); - INIT_LIST_HEAD(&tr.tr_databuf); - INIT_LIST_HEAD(&tr.tr_ail1_list); - INIT_LIST_HEAD(&tr.tr_ail2_list); - tr.tr_revokes = atomic_read(&gl->gl_ail_count); - - if (!tr.tr_revokes) { + if (!revokes) { bool have_revokes; bool log_in_flight; @@ -122,20 +115,12 @@ static int gfs2_ail_empty_gl(struct gfs2_glock *gl) return 0; } - /* A shortened, inline version of gfs2_trans_begin() - * tr->alloced is not set since the transaction structure is - * on the stack */ - tr.tr_reserved = 1 + gfs2_struct2blk(sdp, tr.tr_revokes); - tr.tr_ip = _RET_IP_; - ret = gfs2_log_reserve(sdp, tr.tr_reserved); - if (ret < 0) - return ret; - WARN_ON_ONCE(current->journal_info); - current->journal_info = &tr; - - __gfs2_ail_flush(gl, 0, tr.tr_revokes); - + ret = __gfs2_trans_begin(sdp, 0, revokes, GFP_NOFS | __GFP_NOFAIL); + if (ret) + goto flush; + __gfs2_ail_flush(gl, 0, revokes); gfs2_trans_end(sdp); + flush: gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_AIL_EMPTY_GL); diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index 8e1ab8ed4abc..958810e533ad 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -490,7 +490,6 @@ struct gfs2_quota_data { enum { TR_TOUCHED = 1, TR_ATTACHED = 2, - TR_ALLOCED = 3, }; struct gfs2_trans { diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index e4dc23a24569..721d2d7f0efd 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -1114,7 +1114,6 @@ static void log_refund(struct gfs2_sbd *sdp, struct gfs2_trans *tr) if (sdp->sd_log_tr) { gfs2_merge_trans(sdp, tr); } else if (tr->tr_num_buf_new || tr->tr_num_databuf_new) { - gfs2_assert_withdraw(sdp, test_bit(TR_ALLOCED, &tr->tr_flags)); sdp->sd_log_tr = tr; set_bit(TR_ATTACHED, &tr->tr_flags); } diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index 7705f04621f4..4f461ab37ced 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -37,8 +37,8 @@ static void gfs2_print_trans(struct gfs2_sbd *sdp, const struct gfs2_trans *tr) tr->tr_num_revoke, tr->tr_num_revoke_rm); } -int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, -unsigned int revokes) +int __gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, + unsigned int revokes, gfp_t gfp_mask) { struct gfs2_trans *tr; int error; @@ -52,7 +52,7 @@ int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, if (!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) return -EROFS; - tr = kmem_cache_zalloc(gfs2_trans_cachep, GFP_NOFS); + tr = kmem_cache_zalloc(gfs2_trans_cachep, gfp_mask); if (!tr) return -ENOMEM; @@ -60,7 +60,6 @@ int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, tr->tr_blocks = blocks; tr->tr_revokes = revokes; tr->tr_reserved = 1; - set_bit(TR_ALLOCED, &tr->tr_flags); if (blocks) tr->tr_reserved += 6 + blocks;
Re: [Cluster-devel] [PATCH v3 08/20] gfs2: Get rid of on-stack transactions
Hi, On 27/01/2021 21:07, Andreas Gruenbacher wrote: On-stack transactions were introduced to work around a transaction glock deadlock in gfs2_trans_begin in commit d8348de06f70 ("GFS2: Fix deadlock on journal flush"). Subsequently, transaction glocks were eliminated in favor of the more efficient freeze glocks in commit 24972557b12c ("GFS2: remove transaction glock") without also removing the on-stack transactions. It has now turned out that committing on-stack transactions significantly complicates journal free space accounting when no system transaction (sdp->sd_log_tr) is active at the time. It doesn't seem that on-stack transactions provide a significant benefit beyond their original purpose (as an optimization), so remove them to allow fixing the journal free space accounting in a reasonable way in a subsequent patch. FIXME: Can we better handle a gfs2_trans_begin failure in gfs2_ail_empty_gl? If we skip the __gfs2_ail_flush, we'll just end up with leftover items on gl_ail_list. The reason for the on-stack allocation is to avoid the GFP_NOFAIL allocation here. Please don't add it back, we have gradually been working to eliminate those. Thoes allocations may not fail, but they might also take a long enough time that it would make little difference if they did. So perhaps we need to look at another solution? Steve. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/glops.c | 29 +++-- fs/gfs2/incore.h | 1 - fs/gfs2/log.c| 1 - fs/gfs2/trans.c | 25 + fs/gfs2/trans.h | 2 ++ 5 files changed, 22 insertions(+), 36 deletions(-) diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c index 3faa421568b0..853e590ccc15 100644 --- a/fs/gfs2/glops.c +++ b/fs/gfs2/glops.c @@ -84,18 +84,11 @@ static void __gfs2_ail_flush(struct gfs2_glock *gl, bool fsync, static int gfs2_ail_empty_gl(struct gfs2_glock *gl) { + unsigned int revokes = atomic_read(&gl->gl_ail_count); struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; - struct gfs2_trans tr; int ret; - memset(&tr, 0, sizeof(tr)); - INIT_LIST_HEAD(&tr.tr_buf); - INIT_LIST_HEAD(&tr.tr_databuf); - INIT_LIST_HEAD(&tr.tr_ail1_list); - INIT_LIST_HEAD(&tr.tr_ail2_list); - tr.tr_revokes = atomic_read(&gl->gl_ail_count); - - if (!tr.tr_revokes) { + if (!revokes) { bool have_revokes; bool log_in_flight; @@ -122,20 +115,12 @@ static int gfs2_ail_empty_gl(struct gfs2_glock *gl) return 0; } - /* A shortened, inline version of gfs2_trans_begin() - * tr->alloced is not set since the transaction structure is - * on the stack */ - tr.tr_reserved = 1 + gfs2_struct2blk(sdp, tr.tr_revokes); - tr.tr_ip = _RET_IP_; - ret = gfs2_log_reserve(sdp, tr.tr_reserved); - if (ret < 0) - return ret; - WARN_ON_ONCE(current->journal_info); - current->journal_info = &tr; - - __gfs2_ail_flush(gl, 0, tr.tr_revokes); - + ret = __gfs2_trans_begin(sdp, 0, revokes, GFP_NOFS | __GFP_NOFAIL); + if (ret) + goto flush; + __gfs2_ail_flush(gl, 0, revokes); gfs2_trans_end(sdp); + flush: gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_AIL_EMPTY_GL); diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index 8e1ab8ed4abc..958810e533ad 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -490,7 +490,6 @@ struct gfs2_quota_data { enum { TR_TOUCHED = 1, TR_ATTACHED = 2, - TR_ALLOCED = 3, }; struct gfs2_trans { diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index e4dc23a24569..721d2d7f0efd 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -1114,7 +1114,6 @@ static void log_refund(struct gfs2_sbd *sdp, struct gfs2_trans *tr) if (sdp->sd_log_tr) { gfs2_merge_trans(sdp, tr); } else if (tr->tr_num_buf_new || tr->tr_num_databuf_new) { - gfs2_assert_withdraw(sdp, test_bit(TR_ALLOCED, &tr->tr_flags)); sdp->sd_log_tr = tr; set_bit(TR_ATTACHED, &tr->tr_flags); } diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index 7705f04621f4..4f461ab37ced 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -37,8 +37,8 @@ static void gfs2_print_trans(struct gfs2_sbd *sdp, const struct gfs2_trans *tr) tr->tr_num_revoke, tr->tr_num_revoke_rm); } -int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, -unsigned int revokes) +int __gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, + unsigned int revokes, gfp_t gfp_mask) { struct gfs2_trans *tr; int error; @@ -52,7 +52,7 @@ int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, if (!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) return -EROFS; - tr = kmem_cache_zalloc(gfs2_trans_cachep, G