Re: [Ocfs2-devel] [PATCH v2 3/3] ocfs2/dlm: continue to purge recovery lockres when recovery master goes down

2017-01-05 Thread Gechangwei 
On 2017/1/6 10:08, piaojun wrote:
>
> On 2017/1/5 15:44, Gechangwei wrote:
>> On 2017/1/5 15:28, gechangwei 12382 (Cloud) wrote:
>>
>> Hi Jun,
>> I suppose that a defect hid your patch.
>>
>>
>>> We found a dlm-blocked situation caused by continuous breakdown of recovery 
>>> masters described below. To solve this problem, we should purge recovery 
>>> lock once detecting recovery master goes down.
>>>
>>> N3  N2   N1(reco master)
>>> go down
>>>  pick up recovery lock and
>>>  begin recoverying for N2
>>>
>>>  go down
>>>
>>> pick up recovery
>>> lock failed, then
>>> purge it:
>>> dlm_purge_lockres
>>>   ->DROPPING_REF is set
>>>
>>> send deref to N1 failed,
>>> recovery lock is not purged
>>>
>>> find N1 go down, begin
>>> recoverying for N1, but
>>> blocked in dlm_do_recovery
>>> as DROPPING_REF is set:
>>> dlm_do_recovery
>>>   ->dlm_pick_recovery_master
>>> ->dlmlock
>>>   ->dlm_get_lock_resource
>>> ->__dlm_wait_on_lockres_flags(tmpres,
>>>   DLM_LOCK_RES_DROPPING_REF);
>>>
>>> Fixes: 8c0343968163 ("ocfs2/dlm: clear DROPPING_REF flag when the master 
>>> goes down")
>>>
>>> Signed-off-by: Jun Piao 
>>> Reviewed-by: Joseph Qi 
>>> Reviewed-by: Jiufei Xue 
>>> ---
>>>  fs/ocfs2/dlm/dlmcommon.h   |  2 ++
>>>  fs/ocfs2/dlm/dlmmaster.c   | 38 +++--
>>>  fs/ocfs2/dlm/dlmrecovery.c | 29 +++---
>>>  fs/ocfs2/dlm/dlmthread.c   | 52 
>>> ++
>>>  4 files changed, 74 insertions(+), 47 deletions(-)
>>>
>>> diff --git a/fs/ocfs2/dlm/dlmcommon.h b/fs/ocfs2/dlm/dlmcommon.h index 
>>> 004f2cb..3e3e9ba8 100644
>>> --- a/fs/ocfs2/dlm/dlmcommon.h
>>> +++ b/fs/ocfs2/dlm/dlmcommon.h
>>> @@ -1004,6 +1004,8 @@ int dlm_finalize_reco_handler(struct o2net_msg *msg, 
>>> u32 len, void *data,  int dlm_do_master_requery(struct dlm_ctxt *dlm, 
>>> struct dlm_lock_resource *res,
>>> u8 nodenum, u8 *real_master);
>>>
>>> +void __dlm_do_purge_lockres(struct dlm_ctxt *dlm,
>>> + struct dlm_lock_resource *res);
>>>
>>>  int dlm_dispatch_assert_master(struct dlm_ctxt *dlm,
>>>  struct dlm_lock_resource *res, diff --git 
>>> a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c index 
>>> 311404f..1d87e0f 100644
>>> --- a/fs/ocfs2/dlm/dlmmaster.c
>>> +++ b/fs/ocfs2/dlm/dlmmaster.c
>>> @@ -2425,52 +2425,20 @@ int dlm_deref_lockres_done_handler(struct o2net_msg 
>>> *msg, u32 len, void *data,
>>>   mlog(ML_NOTICE, "%s:%.*s: node %u sends deref done "
>>>   "but it is already derefed!\n", dlm->name,
>>>   res->lockname.len, res->lockname.name, node);
>>> - dlm_lockres_put(res);
>>>   ret = 0;
>>>   goto done;
>>>   }
>>> -
>>> - if (!list_empty(&res->purge)) {
>>> - mlog(0, "%s: Removing res %.*s from purgelist\n",
>>> - dlm->name, res->lockname.len, res->lockname.name);
>>> - list_del_init(&res->purge);
>>> - dlm_lockres_put(res);
>>> - dlm->purge_count--;
>>> - }
>>> -
>>> - if (!__dlm_lockres_unused(res)) {
>>> - mlog(ML_ERROR, "%s: res %.*s in use after deref\n",
>>> - dlm->name, res->lockname.len, res->lockname.name);
>>> - __dlm_print_one_lock_resource(res);
>>> - BUG();
>>> - }
>>> -
>>> - __dlm_unhash_lockres(dlm, res);
>>> -
>>> - spin_lock(&dlm->track_lock);
>>> - if (!list_empty(&res->tracking))
>>> - list_del_init(&res->tracking);
>>> - else {
>>> - mlog(ML_ERROR, "%s: Resource %.*s not on the Tracking list\n",
>>> -  dlm->name, res->lockname.len, res->lockname.name);
>>> - __dlm_print_one_lock_resource(res);
>>> - }
>>> - spin_unlock(&dlm->track_lock);
>>> -
>>> - /* lockres is not in the hash now. drop the flag and wake up
>>> -  * any processes waiting in dlm_get_lock_resource.
>>> -  */
>>> - res->state &= ~DLM_LOCK_RES_DROPPING_REF;
>>> + __dlm_do_purge_lockres(dlm, res);
>>>   spin_unlock(&res->spinlock);
>>>   wake_up(&res->wq);
>>>
>>> - dlm_lockres_put(res);
>>> -
>>>   spin_unlock(&dlm->spinlock);
>>>
>>>   ret = 0;
>>>
>>>  done:
>>> + if (res)
>>> + dlm_lockres_put(res);
>>>   dlm_put(dlm);
>>>   return ret;
>>>  }
>>> diff --git a/fs/ocfs2/dlm/dlmrecovery.c b/fs/ocfs2/dlm/dlmrecovery.c index 
>>> f6b3138..dd5cb8b 100644
>>> --- a/fs/ocfs2/dlm/dlmrecovery.c
>>> +++ b/fs/ocfs2/dlm/dlmrecovery.c
>>> @@ -2343,6 +2343,7 @@ static void dlm_do_local_recovery_cleanup(struct 
>>> dlm_ctxt *dlm, u8 dead_node)
>>>   struct dlm_lock_resource *res;
>>>   int i;
>>>   struct hlist_he

Re: [Ocfs2-devel] [PATCH v2 3/3] ocfs2/dlm: continue to purge recovery lockres when recovery master goes down

2017-01-05 Thread piaojun 


On 2017/1/5 15:44, Gechangwei wrote:
> On 2017/1/5 15:28, gechangwei 12382 (Cloud) wrote:
> 
> Hi Jun,
> I suppose that a defect hid your patch.
> 
> 
>> We found a dlm-blocked situation caused by continuous breakdown of recovery 
>> masters described below. To solve this problem, we should purge recovery 
>> lock once detecting recovery master goes down.
>>
>> N3  N2   N1(reco master)
>> go down
>>  pick up recovery lock and
>>  begin recoverying for N2
>>
>>  go down
>>
>> pick up recovery
>> lock failed, then
>> purge it:
>> dlm_purge_lockres
>>   ->DROPPING_REF is set
>>
>> send deref to N1 failed,
>> recovery lock is not purged
>>
>> find N1 go down, begin
>> recoverying for N1, but
>> blocked in dlm_do_recovery
>> as DROPPING_REF is set:
>> dlm_do_recovery
>>   ->dlm_pick_recovery_master
>> ->dlmlock
>>   ->dlm_get_lock_resource
>> ->__dlm_wait_on_lockres_flags(tmpres,
>>   DLM_LOCK_RES_DROPPING_REF);
>>
>> Fixes: 8c0343968163 ("ocfs2/dlm: clear DROPPING_REF flag when the master 
>> goes down")
>>
>> Signed-off-by: Jun Piao 
>> Reviewed-by: Joseph Qi 
>> Reviewed-by: Jiufei Xue 
>> ---
>>  fs/ocfs2/dlm/dlmcommon.h   |  2 ++
>>  fs/ocfs2/dlm/dlmmaster.c   | 38 +++--
>>  fs/ocfs2/dlm/dlmrecovery.c | 29 +++---
>>  fs/ocfs2/dlm/dlmthread.c   | 52 
>> ++
>>  4 files changed, 74 insertions(+), 47 deletions(-)
>>
>> diff --git a/fs/ocfs2/dlm/dlmcommon.h b/fs/ocfs2/dlm/dlmcommon.h index 
>> 004f2cb..3e3e9ba8 100644
>> --- a/fs/ocfs2/dlm/dlmcommon.h
>> +++ b/fs/ocfs2/dlm/dlmcommon.h
>> @@ -1004,6 +1004,8 @@ int dlm_finalize_reco_handler(struct o2net_msg *msg, 
>> u32 len, void *data,  int dlm_do_master_requery(struct dlm_ctxt *dlm, struct 
>> dlm_lock_resource *res,
>> u8 nodenum, u8 *real_master);
>>
>> +void __dlm_do_purge_lockres(struct dlm_ctxt *dlm,
>> + struct dlm_lock_resource *res);
>>
>>  int dlm_dispatch_assert_master(struct dlm_ctxt *dlm,
>>  struct dlm_lock_resource *res, diff --git 
>> a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c index 311404f..1d87e0f 
>> 100644
>> --- a/fs/ocfs2/dlm/dlmmaster.c
>> +++ b/fs/ocfs2/dlm/dlmmaster.c
>> @@ -2425,52 +2425,20 @@ int dlm_deref_lockres_done_handler(struct o2net_msg 
>> *msg, u32 len, void *data,
>>   mlog(ML_NOTICE, "%s:%.*s: node %u sends deref done "
>>   "but it is already derefed!\n", dlm->name,
>>   res->lockname.len, res->lockname.name, node);
>> - dlm_lockres_put(res);
>>   ret = 0;
>>   goto done;
>>   }
>> -
>> - if (!list_empty(&res->purge)) {
>> - mlog(0, "%s: Removing res %.*s from purgelist\n",
>> - dlm->name, res->lockname.len, res->lockname.name);
>> - list_del_init(&res->purge);
>> - dlm_lockres_put(res);
>> - dlm->purge_count--;
>> - }
>> -
>> - if (!__dlm_lockres_unused(res)) {
>> - mlog(ML_ERROR, "%s: res %.*s in use after deref\n",
>> - dlm->name, res->lockname.len, res->lockname.name);
>> - __dlm_print_one_lock_resource(res);
>> - BUG();
>> - }
>> -
>> - __dlm_unhash_lockres(dlm, res);
>> -
>> - spin_lock(&dlm->track_lock);
>> - if (!list_empty(&res->tracking))
>> - list_del_init(&res->tracking);
>> - else {
>> - mlog(ML_ERROR, "%s: Resource %.*s not on the Tracking list\n",
>> -  dlm->name, res->lockname.len, res->lockname.name);
>> - __dlm_print_one_lock_resource(res);
>> - }
>> - spin_unlock(&dlm->track_lock);
>> -
>> - /* lockres is not in the hash now. drop the flag and wake up
>> -  * any processes waiting in dlm_get_lock_resource.
>> -  */
>> - res->state &= ~DLM_LOCK_RES_DROPPING_REF;
>> + __dlm_do_purge_lockres(dlm, res);
>>   spin_unlock(&res->spinlock);
>>   wake_up(&res->wq);
>>
>> - dlm_lockres_put(res);
>> -
>>   spin_unlock(&dlm->spinlock);
>>
>>   ret = 0;
>>
>>  done:
>> + if (res)
>> + dlm_lockres_put(res);
>>   dlm_put(dlm);
>>   return ret;
>>  }
>> diff --git a/fs/ocfs2/dlm/dlmrecovery.c b/fs/ocfs2/dlm/dlmrecovery.c index 
>> f6b3138..dd5cb8b 100644
>> --- a/fs/ocfs2/dlm/dlmrecovery.c
>> +++ b/fs/ocfs2/dlm/dlmrecovery.c
>> @@ -2343,6 +2343,7 @@ static void dlm_do_local_recovery_cleanup(struct 
>> dlm_ctxt *dlm, u8 dead_node)
>>   struct dlm_lock_resource *res;
>>   int i;
>>   struct hlist_head *bucket;
>> + struct hlist_node *tmp;
>>   struct dlm_lock *lock;
>>
>>
>> @@ -2365,7 +2366,7 @@ static void dlm_do_local_recovery_cleanup(struct 
>> dlm_ctx

Re: [Ocfs2-devel] [PATCH v2 3/3] ocfs2/dlm: continue to purge recovery lockres when recovery master goes down

2017-01-04 Thread Gechangwei 
On 2017/1/5 15:28, gechangwei 12382 (Cloud) wrote:

Hi Jun,
I suppose that a defect hid your patch.


> We found a dlm-blocked situation caused by continuous breakdown of recovery 
> masters described below. To solve this problem, we should purge recovery lock 
> once detecting recovery master goes down.
>
> N3  N2   N1(reco master)
> go down
>  pick up recovery lock and
>  begin recoverying for N2
>
>  go down
>
> pick up recovery
> lock failed, then
> purge it:
> dlm_purge_lockres
>   ->DROPPING_REF is set
>
> send deref to N1 failed,
> recovery lock is not purged
>
> find N1 go down, begin
> recoverying for N1, but
> blocked in dlm_do_recovery
> as DROPPING_REF is set:
> dlm_do_recovery
>   ->dlm_pick_recovery_master
> ->dlmlock
>   ->dlm_get_lock_resource
> ->__dlm_wait_on_lockres_flags(tmpres,
>   DLM_LOCK_RES_DROPPING_REF);
>
> Fixes: 8c0343968163 ("ocfs2/dlm: clear DROPPING_REF flag when the master goes 
> down")
>
> Signed-off-by: Jun Piao 
> Reviewed-by: Joseph Qi 
> Reviewed-by: Jiufei Xue 
> ---
>  fs/ocfs2/dlm/dlmcommon.h   |  2 ++
>  fs/ocfs2/dlm/dlmmaster.c   | 38 +++--
>  fs/ocfs2/dlm/dlmrecovery.c | 29 +++---
>  fs/ocfs2/dlm/dlmthread.c   | 52 
> ++
>  4 files changed, 74 insertions(+), 47 deletions(-)
>
> diff --git a/fs/ocfs2/dlm/dlmcommon.h b/fs/ocfs2/dlm/dlmcommon.h index 
> 004f2cb..3e3e9ba8 100644
> --- a/fs/ocfs2/dlm/dlmcommon.h
> +++ b/fs/ocfs2/dlm/dlmcommon.h
> @@ -1004,6 +1004,8 @@ int dlm_finalize_reco_handler(struct o2net_msg *msg, 
> u32 len, void *data,  int dlm_do_master_requery(struct dlm_ctxt *dlm, struct 
> dlm_lock_resource *res,
> u8 nodenum, u8 *real_master);
>
> +void __dlm_do_purge_lockres(struct dlm_ctxt *dlm,
> + struct dlm_lock_resource *res);
>
>  int dlm_dispatch_assert_master(struct dlm_ctxt *dlm,
>  struct dlm_lock_resource *res, diff --git 
> a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c index 311404f..1d87e0f 
> 100644
> --- a/fs/ocfs2/dlm/dlmmaster.c
> +++ b/fs/ocfs2/dlm/dlmmaster.c
> @@ -2425,52 +2425,20 @@ int dlm_deref_lockres_done_handler(struct o2net_msg 
> *msg, u32 len, void *data,
>   mlog(ML_NOTICE, "%s:%.*s: node %u sends deref done "
>   "but it is already derefed!\n", dlm->name,
>   res->lockname.len, res->lockname.name, node);
> - dlm_lockres_put(res);
>   ret = 0;
>   goto done;
>   }
> -
> - if (!list_empty(&res->purge)) {
> - mlog(0, "%s: Removing res %.*s from purgelist\n",
> - dlm->name, res->lockname.len, res->lockname.name);
> - list_del_init(&res->purge);
> - dlm_lockres_put(res);
> - dlm->purge_count--;
> - }
> -
> - if (!__dlm_lockres_unused(res)) {
> - mlog(ML_ERROR, "%s: res %.*s in use after deref\n",
> - dlm->name, res->lockname.len, res->lockname.name);
> - __dlm_print_one_lock_resource(res);
> - BUG();
> - }
> -
> - __dlm_unhash_lockres(dlm, res);
> -
> - spin_lock(&dlm->track_lock);
> - if (!list_empty(&res->tracking))
> - list_del_init(&res->tracking);
> - else {
> - mlog(ML_ERROR, "%s: Resource %.*s not on the Tracking list\n",
> -  dlm->name, res->lockname.len, res->lockname.name);
> - __dlm_print_one_lock_resource(res);
> - }
> - spin_unlock(&dlm->track_lock);
> -
> - /* lockres is not in the hash now. drop the flag and wake up
> -  * any processes waiting in dlm_get_lock_resource.
> -  */
> - res->state &= ~DLM_LOCK_RES_DROPPING_REF;
> + __dlm_do_purge_lockres(dlm, res);
>   spin_unlock(&res->spinlock);
>   wake_up(&res->wq);
>
> - dlm_lockres_put(res);
> -
>   spin_unlock(&dlm->spinlock);
>
>   ret = 0;
>
>  done:
> + if (res)
> + dlm_lockres_put(res);
>   dlm_put(dlm);
>   return ret;
>  }
> diff --git a/fs/ocfs2/dlm/dlmrecovery.c b/fs/ocfs2/dlm/dlmrecovery.c index 
> f6b3138..dd5cb8b 100644
> --- a/fs/ocfs2/dlm/dlmrecovery.c
> +++ b/fs/ocfs2/dlm/dlmrecovery.c
> @@ -2343,6 +2343,7 @@ static void dlm_do_local_recovery_cleanup(struct 
> dlm_ctxt *dlm, u8 dead_node)
>   struct dlm_lock_resource *res;
>   int i;
>   struct hlist_head *bucket;
> + struct hlist_node *tmp;
>   struct dlm_lock *lock;
>
>
> @@ -2365,7 +2366,7 @@ static void dlm_do_local_recovery_cleanup(struct 
> dlm_ctxt *dlm, u8 dead_node)
>*/
>   for (i = 0; i < DLM_HASH_BUCKETS; i++) {
>   bucket = dlm_lockres_hash(dlm, i);
> - hlist_for_each_entry(res, bucket, ha

[Ocfs2-devel] [PATCH v2 3/3] ocfs2/dlm: continue to purge recovery lockres when recovery master goes down

2016-07-11 Thread piaojun 
We found a dlm-blocked situation caused by continuous breakdown of
recovery masters described below. To solve this problem, we should purge
recovery lock once detecting recovery master goes down.

N3  N2   N1(reco master)
go down
 pick up recovery lock and
 begin recoverying for N2

 go down

pick up recovery
lock failed, then
purge it:
dlm_purge_lockres
  ->DROPPING_REF is set

send deref to N1 failed,
recovery lock is not purged

find N1 go down, begin
recoverying for N1, but
blocked in dlm_do_recovery
as DROPPING_REF is set:
dlm_do_recovery
  ->dlm_pick_recovery_master
->dlmlock
  ->dlm_get_lock_resource
->__dlm_wait_on_lockres_flags(tmpres,
DLM_LOCK_RES_DROPPING_REF);

Fixes: 8c0343968163 ("ocfs2/dlm: clear DROPPING_REF flag when the master goes 
down")

Signed-off-by: Jun Piao 
Reviewed-by: Joseph Qi 
Reviewed-by: Jiufei Xue 
---
 fs/ocfs2/dlm/dlmcommon.h   |  2 ++
 fs/ocfs2/dlm/dlmmaster.c   | 38 +++--
 fs/ocfs2/dlm/dlmrecovery.c | 29 +++---
 fs/ocfs2/dlm/dlmthread.c   | 52 ++
 4 files changed, 74 insertions(+), 47 deletions(-)

diff --git a/fs/ocfs2/dlm/dlmcommon.h b/fs/ocfs2/dlm/dlmcommon.h
index 004f2cb..3e3e9ba8 100644
--- a/fs/ocfs2/dlm/dlmcommon.h
+++ b/fs/ocfs2/dlm/dlmcommon.h
@@ -1004,6 +1004,8 @@ int dlm_finalize_reco_handler(struct o2net_msg *msg, u32 
len, void *data,
 int dlm_do_master_requery(struct dlm_ctxt *dlm, struct dlm_lock_resource *res,
  u8 nodenum, u8 *real_master);
 
+void __dlm_do_purge_lockres(struct dlm_ctxt *dlm,
+   struct dlm_lock_resource *res);
 
 int dlm_dispatch_assert_master(struct dlm_ctxt *dlm,
   struct dlm_lock_resource *res,
diff --git a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c
index 311404f..1d87e0f 100644
--- a/fs/ocfs2/dlm/dlmmaster.c
+++ b/fs/ocfs2/dlm/dlmmaster.c
@@ -2425,52 +2425,20 @@ int dlm_deref_lockres_done_handler(struct o2net_msg 
*msg, u32 len, void *data,
mlog(ML_NOTICE, "%s:%.*s: node %u sends deref done "
"but it is already derefed!\n", dlm->name,
res->lockname.len, res->lockname.name, node);
-   dlm_lockres_put(res);
ret = 0;
goto done;
}
-
-   if (!list_empty(&res->purge)) {
-   mlog(0, "%s: Removing res %.*s from purgelist\n",
-   dlm->name, res->lockname.len, res->lockname.name);
-   list_del_init(&res->purge);
-   dlm_lockres_put(res);
-   dlm->purge_count--;
-   }
-
-   if (!__dlm_lockres_unused(res)) {
-   mlog(ML_ERROR, "%s: res %.*s in use after deref\n",
-   dlm->name, res->lockname.len, res->lockname.name);
-   __dlm_print_one_lock_resource(res);
-   BUG();
-   }
-
-   __dlm_unhash_lockres(dlm, res);
-
-   spin_lock(&dlm->track_lock);
-   if (!list_empty(&res->tracking))
-   list_del_init(&res->tracking);
-   else {
-   mlog(ML_ERROR, "%s: Resource %.*s not on the Tracking list\n",
-dlm->name, res->lockname.len, res->lockname.name);
-   __dlm_print_one_lock_resource(res);
-   }
-   spin_unlock(&dlm->track_lock);
-
-   /* lockres is not in the hash now. drop the flag and wake up
-* any processes waiting in dlm_get_lock_resource.
-*/
-   res->state &= ~DLM_LOCK_RES_DROPPING_REF;
+   __dlm_do_purge_lockres(dlm, res);
spin_unlock(&res->spinlock);
wake_up(&res->wq);
 
-   dlm_lockres_put(res);
-
spin_unlock(&dlm->spinlock);
 
ret = 0;
 
 done:
+   if (res)
+   dlm_lockres_put(res);
dlm_put(dlm);
return ret;
 }
diff --git a/fs/ocfs2/dlm/dlmrecovery.c b/fs/ocfs2/dlm/dlmrecovery.c
index f6b3138..dd5cb8b 100644
--- a/fs/ocfs2/dlm/dlmrecovery.c
+++ b/fs/ocfs2/dlm/dlmrecovery.c
@@ -2343,6 +2343,7 @@ static void dlm_do_local_recovery_cleanup(struct dlm_ctxt 
*dlm, u8 dead_node)
struct dlm_lock_resource *res;
int i;
struct hlist_head *bucket;
+   struct hlist_node *tmp;
struct dlm_lock *lock;
 
 
@@ -2365,7 +2366,7 @@ static void dlm_do_local_recovery_cleanup(struct dlm_ctxt 
*dlm, u8 dead_node)
 */
for (i = 0; i < DLM_HASH_BUCKETS; i++) {
bucket = dlm_lockres_hash(dlm, i);
-   hlist_for_each_entry(res, bucket, hash_node) {
+   hlist_for_each_entry_safe(res, tmp, bucket, hash_node) {
/* always prune any $RECOVERY entries for dead nodes,
 * otherwise hangs can occur during later recovery */