Hi Orit, It appears it must have been one of the known bugs in 10.2.2. I just upgraded to 10.2.3 and bi-directional syncing now works.
I am still seeing some errors when I run synch-related commands but they don't seem to be affecting operations as of yet: radosgw-admin sync status 2016-09-27 16:17:15.270331 7fe5e83ad9c0 0 error in read_id for id : (2) No such file or directory 2016-09-27 16:17:15.270883 7fe5e83ad9c0 0 error in read_id for id : (2) No such file or directory realm 3af93a86-916a-490f-b38f-17922b472b19 (my_realm) zonegroup 235b010c-22e2-4b43-8fcc-8ae01939273e (us) zone 58aa3eef-fc1f-492c-a08e-9c6019e7c266 (us-phx) metadata sync preparing for full sync full sync: 0/64 shards metadata is caught up with master incremental sync: 64/64 shards data sync source: 6c830b44-4e39-4e19-9bd8-03c37c2021f2 (us-dfw) preparing for full sync full sync: 18/128 shards full sync: 0 buckets to sync incremental sync: 110/128 shards data is behind on 20 shards oldest incremental change not applied: 2016-09-27 16:17:08.0.922757s I've also verified all of the existing data within each bucket has sync'd over as well On Tue, Sep 27, 2016 at 9:21 AM John Rowe <john.r...@salesforce.com> wrote: > Hi Orit, > > That was my failed attempt at sanitizing :) > > They are actually all identical: > > Periods: > MD5 (cephrgw1-1-dfw-period.json) = 12ed481381c1f2937a27b57db0473d6d > MD5 (cephrgw1-1-phx-period.json) = 12ed481381c1f2937a27b57db0473d6d > MD5 (cephrgw1-2-dfw-period.json) = 12ed481381c1f2937a27b57db0473d6d > MD5 (cephrgw1-2-phx-period.json) = 12ed481381c1f2937a27b57db0473d6d > MD5 (cephrgw1-3-dfw-period.json) = 12ed481381c1f2937a27b57db0473d6d > MD5 (cephrgw1-3-phx-period.json) = 12ed481381c1f2937a27b57db0473d6d > > Realms: > MD5 (cephrgw1-1-dfw-realm.json) = 39a4e63bab64ed756961117d3629b109 > MD5 (cephrgw1-1-phx-realm.json) = 39a4e63bab64ed756961117d3629b109 > MD5 (cephrgw1-2-dfw-realm.json) = 39a4e63bab64ed756961117d3629b109 > MD5 (cephrgw1-2-phx-realm.json) = 39a4e63bab64ed756961117d3629b109 > MD5 (cephrgw1-3-dfw-realm.json) = 39a4e63bab64ed756961117d3629b109 > MD5 (cephrgw1-3-phx-realm.json) = 39a4e63bab64ed756961117d3629b109 > > > On Tue, Sep 27, 2016 at 5:32 AM Orit Wasserman <owass...@redhat.com> > wrote: > >> see comment below >> >> On Mon, Sep 26, 2016 at 10:00 PM, John Rowe <john.r...@salesforce.com> >> wrote: >> > Hi Orit, >> > >> > Sure thing, please see below. >> > >> > Thanks! >> > >> > >> > DFW (Primary) >> > radosgw-admin zonegroupmap get >> > { >> > "zonegroups": [ >> > { >> > "key": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "val": { >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "name": "us", >> > "api_name": "us", >> > "is_master": "true", >> > "endpoints": [ >> > "http:\/\/ELB_FQDN:80" >> > ], >> > "hostnames": [], >> > "hostnames_s3website": [], >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "zones": [ >> > { >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "name": "us-phx", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1:80" >> > ], >> > "log_meta": "false", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > }, >> > { >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "name": "us-dfw", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-dfw:80" >> > ], >> > "log_meta": "true", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > } >> > ], >> > "placement_targets": [ >> > { >> > "name": "default-placement", >> > "tags": [] >> > } >> > ], >> > "default_placement": "default-placement", >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> > } >> > } >> > ], >> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "bucket_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > }, >> > "user_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > } >> > } >> > >> > PHX (Secondary) >> > { >> > "zonegroups": [ >> > { >> > "key": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "val": { >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "name": "us", >> > "api_name": "us", >> > "is_master": "true", >> > "endpoints": [ >> > "http:\/\/ELB_FQDN:80" >> > ], >> > "hostnames": [], >> > "hostnames_s3website": [], >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "zones": [ >> > { >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "name": "us-phx", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-phx:80" >> > ], >> > "log_meta": "false", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > }, >> > { >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "name": "us-dfw", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-dfw:80" >> > ], >> > "log_meta": "true", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > } >> > ], >> > "placement_targets": [ >> > { >> > "name": "default-placement", >> > "tags": [] >> > } >> > ], >> > "default_placement": "default-placement", >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> > } >> > } >> > ], >> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "bucket_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > }, >> > "user_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > } >> > } >> > >> > >> > Realms and Periods: >> > DFW (Primary) >> > rgw1-1-dfw >> > radosgw-admin realm get >> > { >> > "id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "name": "my_realm", >> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 2 >> > } >> > radosgw-admin period get >> > { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 4, >> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697", >> > "sync_status": [ >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "" >> > ], >> > "period_map": { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "zonegroups": [ >> > { >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "name": "us", >> > "api_name": "us", >> > "is_master": "true", >> > "endpoints": [ >> > "http:\/\/ELB_FQDN:80" >> > ], >> > "hostnames": [], >> > "hostnames_s3website": [], >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "zones": [ >> > { >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "name": "us-phx", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-phx:80" >> > ], >> > "log_meta": "false", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > }, >> > { >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "name": "us-dfw", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-dfw:80" >> > ], >> > "log_meta": "true", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > } >> > ], >> > "placement_targets": [ >> > { >> > "name": "default-placement", >> > "tags": [] >> > } >> > ], >> > "default_placement": "default-placement", >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> > } >> > ], >> > "short_zone_ids": [ >> > { >> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "val": 1694436416 >> > }, >> > { >> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "val": 2500379964 >> > } >> > ] >> > }, >> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "period_config": { >> > "bucket_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > }, >> > "user_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > } >> > }, >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "realm_name": "my_realm", >> > "realm_epoch": 2 >> > } >> > >> > rgw1-2-dfw >> > radosgw-admin realm get >> > { >> > "id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "name": "my_realm", >> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 2 >> > } >> > >> > radosgw-admin period get >> > { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 4, >> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697", >> > "sync_status": [ >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "" >> > ], >> > "period_map": { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "zonegroups": [ >> > { >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "name": "us", >> > "api_name": "us", >> > "is_master": "true", >> > "endpoints": [ >> > "http:\/\/ELB_FQDN:80" >> > ], >> > "hostnames": [], >> > "hostnames_s3website": [], >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "zones": [ >> > { >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "name": "us-phx", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-phx:80" >> > ], >> > "log_meta": "false", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > }, >> > { >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "name": "us-dfw", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-dfw:80" >> > ], >> > "log_meta": "true", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > } >> > ], >> > "placement_targets": [ >> > { >> > "name": "default-placement", >> > "tags": [] >> > } >> > ], >> > "default_placement": "default-placement", >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> > } >> > ], >> > "short_zone_ids": [ >> > { >> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "val": 1694436416 >> > }, >> > { >> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "val": 2500379964 >> > } >> > ] >> > }, >> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "period_config": { >> > "bucket_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > }, >> > "user_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > } >> > }, >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "realm_name": "my_realm", >> > "realm_epoch": 2 >> > } >> > >> > rgw1-3-dfw >> > radosgw-admin realm get >> > radosgw-admin realm get >> > { >> > "id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "name": "my_realm", >> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 2 >> > } >> > >> > radosgw-admin period get >> > { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 4, >> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697", >> > "sync_status": [ >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "" >> > ], >> > "period_map": { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "zonegroups": [ >> > { >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "name": "us", >> > "api_name": "us", >> > "is_master": "true", >> > "endpoints": [ >> > "http:\/\/ELB_FQDN:80" >> > ], >> > "hostnames": [], >> > "hostnames_s3website": [], >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "zones": [ >> > { >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "name": "us-phx", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-phx:80" >> > ], >> > "log_meta": "false", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > }, >> > { >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "name": "us-dfw", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-dfw:80" >> > ], >> > "log_meta": "true", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > } >> > ], >> > "placement_targets": [ >> > { >> > "name": "default-placement", >> > "tags": [] >> > } >> > ], >> > "default_placement": "default-placement", >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> > } >> > ], >> > "short_zone_ids": [ >> > { >> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "val": 1694436416 >> > }, >> > { >> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "val": 2500379964 >> > } >> > ] >> > }, >> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "period_config": { >> > "bucket_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > }, >> > "user_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > } >> > }, >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "realm_name": "my_realm", >> > "realm_epoch": 2 >> > } >> > >> > >> > PHX (secondary) >> > radosgw-admin realm get >> > { >> > "id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "name": "my_realm", >> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 2 >> > } >> > radosgw-admin period get >> > { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 4, >> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697", >> > "sync_status": [ >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "" >> > ], >> > "period_map": { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "zonegroups": [ >> > { >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "name": "us", >> > "api_name": "us", >> > "is_master": "true", >> > "endpoints": [ >> > "http:\/\/ELB_FQDN:80" >> > ], >> > "hostnames": [], >> > "hostnames_s3website": [], >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "zones": [ >> > { >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "name": "us-phx", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-phx:80" >> > ], >> > "log_meta": "false", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > }, >> > { >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "name": "us-dfw", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-dfw:80" >> > ], >> > "log_meta": "true", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > } >> > ], >> > "placement_targets": [ >> > { >> > "name": "default-placement", >> > "tags": [] >> > } >> > ], >> > "default_placement": "default-placement", >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> > } >> > ], >> > "short_zone_ids": [ >> > { >> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "val": 1694436416 >> > }, >> > { >> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "val": 2500379964 >> > } >> > ] >> > }, >> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "period_config": { >> > "bucket_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > }, >> > "user_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > } >> > }, >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "realm_name": "my_realm", >> > "realm_epoch": 2 >> > } >> > >> > cephrgw1-3-phx >> > radosgw-admin realm get >> > { >> > "id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "name": "pardot", >> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 2 >> > } >> > radosgw-admin period get >> > { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 4, >> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697", >> > "sync_status": [ >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "" >> > ], >> > "period_map": { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "zonegroups": [ >> > { >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "name": "us", >> > "api_name": "us", >> > "is_master": "true", >> > "endpoints": [ >> > "http:\/\/ELB_FQDN:80" >> > ], >> > "hostnames": [], >> > "hostnames_s3website": [], >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "zones": [ >> > { >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "name": "us-phx", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-phx:80" >> > ], >> > "log_meta": "false", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > }, >> > { >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "name": "us-dfw", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-dfw:80" >> > ], >> > "log_meta": "true", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > } >> > ], >> > "placement_targets": [ >> > { >> > "name": "default-placement", >> > "tags": [] >> > } >> > ], >> > "default_placement": "default-placement", >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> > } >> > ], >> > "short_zone_ids": [ >> > { >> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "val": 1694436416 >> > }, >> > { >> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "val": 2500379964 >> > } >> > ] >> > }, >> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "period_config": { >> > "bucket_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > }, >> > "user_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > } >> > }, >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "realm_name": "my_realm", >> > "realm_epoch": 2 >> > } >> > >> > cephrgw-1-3-phx >> > radosgw-admin realm get >> > { >> > "id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "name": "pardot", >> >> In other nodes the name is "my_realm" , maybe that is the problem. >> You can try fixing it by renaming it using realm rename or using realm >> set command >> >> Orit >> > "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 2 >> > } >> > >> > radsogw-admin period get >> > { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "epoch": 4, >> > "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697", >> > "sync_status": [ >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "", >> > "" >> > ], >> > "period_map": { >> > "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1", >> > "zonegroups": [ >> > { >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "name": "us", >> > "api_name": "us", >> > "is_master": "true", >> > "endpoints": [ >> > "http:\/\/ELB_FQDN:80" >> > ], >> > "hostnames": [], >> > "hostnames_s3website": [], >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "zones": [ >> > { >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "name": "us-phx", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-phx:80" >> > ], >> > "log_meta": "false", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > }, >> > { >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "name": "us-dfw", >> > "endpoints": [ >> > "http:\/\/cephrgw1-1-dfw:80" >> > ], >> > "log_meta": "true", >> > "log_data": "true", >> > "bucket_index_max_shards": 0, >> > "read_only": "false" >> > } >> > ], >> > "placement_targets": [ >> > { >> > "name": "default-placement", >> > "tags": [] >> > } >> > ], >> > "default_placement": "default-placement", >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> > } >> > ], >> > "short_zone_ids": [ >> > { >> > "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> > "val": 1694436416 >> > }, >> > { >> > "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "val": 2500379964 >> > } >> > ] >> > }, >> > "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> > "period_config": { >> > "bucket_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > }, >> > "user_quota": { >> > "enabled": false, >> > "max_size_kb": -1, >> > "max_objects": -1 >> > } >> > }, >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19", >> > "realm_name": "my_realm", >> > "realm_epoch": 2 >> > } >> > >> > >> > On Mon, Sep 26, 2016 at 5:27 AM Orit Wasserman <owass...@redhat.com> >> wrote: >> >> >> >> Hi John, >> >> Can you provide: >> >> radosgw-admin zonegroupmap get on both us-dfw and us-phx? >> >> radosgw-admin realm get and radosgw-admin period get on all the >> gateways? >> >> >> >> Orit >> >> >> >> On Thu, Sep 22, 2016 at 4:37 PM, John Rowe <john.r...@salesforce.com> >> >> wrote: >> >> > Hello Orit, thanks. >> >> > >> >> > I will do all 6 just in case. Also as an FYI I originally had all 6 >> as >> >> > endpoint (3 in each zone) but have it down to just the two "1" >> servers >> >> > talking to each other until I can get it working. Eventually I would >> >> > like to >> >> > have all 6 cross connecting again. >> >> > >> >> > rgw-primary-1: >> >> > radosgw-admin zonegroup get >> >> > { >> >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> >> > "name": "us", >> >> > "api_name": "us", >> >> > "is_master": "true", >> >> > "endpoints": [ >> >> > "http:\/\/LB_FQDN:80" >> >> > ], >> >> > "hostnames": [], >> >> > "hostnames_s3website": [], >> >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> >> > "zones": [ >> >> > { >> >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> >> > "name": "us-phx", >> >> > "endpoints": [ >> >> > "http:\/\/PHX-RGW-1:80" >> >> > ], >> >> > "log_meta": "false", >> >> > "log_data": "true", >> >> > "bucket_index_max_shards": 0, >> >> > "read_only": "false" >> >> > }, >> >> > { >> >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> >> > "name": "us-dfw", >> >> > "endpoints": [ >> >> > "http:\/\/DFW-RGW-1:80" >> >> > ], >> >> > "log_meta": "true", >> >> > "log_data": "true", >> >> > "bucket_index_max_shards": 0, >> >> > "read_only": "false" >> >> > } >> >> > ], >> >> > "placement_targets": [ >> >> > { >> >> > "name": "default-placement", >> >> > "tags": [] >> >> > } >> >> > ], >> >> > "default_placement": "default-placement", >> >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> >> > } >> >> > radosgw-admin zone get >> >> > { >> >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> >> > "name": "us-dfw", >> >> > "domain_root": "us-dfw.rgw.data.root", >> >> > "control_pool": "us-dfw.rgw.control", >> >> > "gc_pool": "us-dfw.rgw.gc", >> >> > "log_pool": "us-dfw.rgw.log", >> >> > "intent_log_pool": "us-dfw.rgw.intent-log", >> >> > "usage_log_pool": "us-dfw.rgw.usage", >> >> > "user_keys_pool": "us-dfw.rgw.users.keys", >> >> > "user_email_pool": "us-dfw.rgw.users.email", >> >> > "user_swift_pool": "us-dfw.rgw.users.swift", >> >> > "user_uid_pool": "us-dfw.rgw.users.uid", >> >> > "system_key": { >> >> > "access_key": "SYSTEM_ACCESS_KEY", >> >> > "secret_key": "SYSTEM_SECRET_KEY" >> >> > }, >> >> > "placement_pools": [ >> >> > { >> >> > "key": "default-placement", >> >> > "val": { >> >> > "index_pool": "us-dfw.rgw.buckets.index", >> >> > "data_pool": "us-dfw.rgw.buckets.data", >> >> > "data_extra_pool": "us-dfw.rgw.buckets.non-ec", >> >> > "index_type": 0 >> >> > } >> >> > } >> >> > ], >> >> > "metadata_heap": "us-dfw.rgw.meta", >> >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> >> > } >> >> > >> >> > ---- >> >> > rgw-primary-2: >> >> > radosgw-admin zonegroup get >> >> > { >> >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> >> > "name": "us", >> >> > "api_name": "us", >> >> > "is_master": "true", >> >> > "endpoints": [ >> >> > "http:\/\/LB_FQDN:80" >> >> > ], >> >> > "hostnames": [], >> >> > "hostnames_s3website": [], >> >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> >> > "zones": [ >> >> > { >> >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> >> > "name": "us-phx", >> >> > "endpoints": [ >> >> > "http:\/\/PHX-RGW-1:80" >> >> > ], >> >> > "log_meta": "false", >> >> > "log_data": "true", >> >> > "bucket_index_max_shards": 0, >> >> > "read_only": "false" >> >> > }, >> >> > { >> >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> >> > "name": "us-dfw", >> >> > "endpoints": [ >> >> > "http:\/\/DFW-RGW-1:80" >> >> > ], >> >> > "log_meta": "true", >> >> > "log_data": "true", >> >> > "bucket_index_max_shards": 0, >> >> > "read_only": "false" >> >> > } >> >> > ], >> >> > "placement_targets": [ >> >> > { >> >> > "name": "default-placement", >> >> > "tags": [] >> >> > } >> >> > ], >> >> > "default_placement": "default-placement", >> >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> >> > } >> >> > >> >> > radosgw-admin zone get >> >> > { >> >> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> >> > "name": "us-dfw", >> >> > "domain_root": "us-dfw.rgw.data.root", >> >> > "control_pool": "us-dfw.rgw.control", >> >> > "gc_pool": "us-dfw.rgw.gc", >> >> > "log_pool": "us-dfw.rgw.log", >> >> > "intent_log_pool": "us-dfw.rgw.intent-log", >> >> > "usage_log_pool": "us-dfw.rgw.usage", >> >> > "user_keys_pool": "us-dfw.rgw.users.keys", >> >> > "user_email_pool": "us-dfw.rgw.users.email", >> >> > "user_swift_pool": "us-dfw.rgw.users.swift", >> >> > "user_uid_pool": "us-dfw.rgw.users.uid", >> >> > "system_key": { >> >> > "access_key": "SYSTEM_ACCESS_KEY", >> >> > "secret_key": "SYSTEM_SECRET_KEY" >> >> > }, >> >> > "placement_pools": [ >> >> > { >> >> > "key": "default-placement", >> >> > "val": { >> >> > "index_pool": "us-dfw.rgw.buckets.index", >> >> > "data_pool": "us-dfw.rgw.buckets.data", >> >> > "data_extra_pool": "us-dfw.rgw.buckets.non-ec", >> >> > "index_type": 0 >> >> > } >> >> > } >> >> > ], >> >> > "metadata_heap": "us-dfw.rgw.meta", >> >> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19" >> >> > } >> >> > >> >> > ---- >> >> > rgw-primary-3: >> >> > radosgw-admin zonegroup get >> >> > { >> >> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e", >> >> > "name": "us", >> >> > "api_name": "us", >> >> > "is_master": "true", >> >> > "endpoints": [ >> >> > "http:\/\/LB_FQDN:80" >> >> > ], >> >> > "hostnames": [], >> >> > "hostnames_s3website": [], >> >> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2", >> >> > "zones": [ >> >> > { >> >> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266", >> >> > "name": "us-phx", >> >
_______________________________________________ ceph-users mailing list ceph-users@lists.ceph.com http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com