diff --git a/src/control/server/mgmt_check.go b/src/control/server/mgmt_check.go index 913a0d2575a..513cb929593 100644 --- a/src/control/server/mgmt_check.go +++ b/src/control/server/mgmt_check.go @@ -33,6 +33,7 @@ import ( const ( checkerEnabledKey = "checker_enabled" + checkerEnabledValue = "true" checkerPoliciesKey = "checker_policies" checkerLatestPolicyKey = "checker_latest_policy" ) @@ -40,29 +41,29 @@ const ( var errNoSavedPolicies = errors.New("no previous policies have been saved") func (svc *mgmtSvc) enableChecker() error { - if err := system.SetMgmtProperty(svc.sysdb, checkerEnabledKey, "true"); err != nil { + if err := system.SetMgmtProperty(svc.sysdb, checkerEnabledKey, checkerEnabledValue); err != nil { return errors.Wrap(err, "failed to enable checker") } return nil } func (svc *mgmtSvc) disableChecker() error { - if err := system.SetMgmtProperty(svc.sysdb, checkerEnabledKey, "false"); err != nil { - return errors.Wrap(err, "failed to disable checker") + if err := system.DelMgmtProperty(svc.sysdb, checkerEnabledKey); err != nil { + return errors.Wrap(err, "failed to delete checker policies") } return nil } -func (svc *mgmtSvc) checkerIsEnabled() bool { +func (svc *mgmtSvc) checkerIsEnabled() (bool, error) { value, err := system.GetMgmtProperty(svc.sysdb, checkerEnabledKey) if err != nil { - if !system.IsNotLeader(err) && !system.IsErrSystemAttrNotFound(err) && - !system.IsNotReplica(err) && !errors.Is(err, system.ErrUninitialized) { - svc.log.Errorf("failed to get checker enabled value: %s", err) + // Never-written property means the checker was never enabled. + if system.IsErrSystemAttrNotFound(err) { + return false, nil } - return false + return false, errors.Wrap(err, "failed to get checker enabled value") } - return value == "true" + return value == checkerEnabledValue, nil } // checkerRequest is a wrapper around a request that is made on behalf of @@ -84,7 +85,11 @@ func (svc *mgmtSvc) unwrapCheckerReq(req proto.Message) (proto.Message, error) { return cr.Message, nil } - if svc.checkerIsEnabled() { + enabled, err := svc.checkerIsEnabled() + if err != nil { + return nil, err + } + if enabled { return nil, checker.FaultCheckerEnabled } @@ -133,7 +138,11 @@ func (svc *mgmtSvc) makeCheckerCall(ctx context.Context, method drpc.Method, req } func (svc *mgmtSvc) verifyCheckerReady() error { - if !svc.checkerIsEnabled() { + enabled, err := svc.checkerIsEnabled() + if err != nil { + return err + } + if !enabled { return checker.FaultCheckerNotEnabled } @@ -194,16 +203,31 @@ func (svc *mgmtSvc) startSystemRanks(ctx context.Context, sys string) error { availRanks.Add(rank) } + checkMode, err := svc.checkerIsEnabled() + if err != nil { + return err + } + // Finally, restart all of the ranks so that they join in // checker mode. startReq := &mgmtpb.SystemStartReq{ Sys: sys, - CheckMode: svc.checkerIsEnabled(), + CheckMode: checkMode, Ranks: availRanks.String(), } - if _, err := svc.SystemStart(ctx, startReq); err != nil { + startResp, err := svc.SystemStart(ctx, startReq) + if err != nil { return errors.Wrap(err, "failed to start all ranks") } + failed := ranklist.NewRankSet() + for _, result := range startResp.Results { + if result.Errored { + failed.Add(ranklist.Rank(result.Rank)) + } + } + if failed.Count() > 0 { + return errors.Errorf("failed to start rank(s) %s", failed.String()) + } return nil } @@ -214,16 +238,26 @@ func (svc *mgmtSvc) SystemCheckEnable(ctx context.Context, req *mgmtpb.CheckEnab return nil, err } - if svc.checkerIsEnabled() { - return &mgmtpb.DaosResp{Status: int32(daos.Already)}, nil - } - - if err := svc.checkMemberStates( - system.MemberStateAdminExcluded, - system.MemberStateStopped, - ); err != nil { + enabled, err := svc.checkerIsEnabled() + if err != nil { return nil, err } + if enabled { + // Double-check that all of the ranks are in a known state. + if err := svc.checkMemberStates( + system.MemberStateAdminExcluded, + system.MemberStateCheckerStarted, + ); err == nil { + return &mgmtpb.DaosResp{Status: int32(daos.Already)}, nil + } + } else { + if err := svc.checkMemberStates( + system.MemberStateAdminExcluded, + system.MemberStateStopped, + ); err != nil { + return nil, err + } + } if err := svc.enableChecker(); err != nil { return nil, err @@ -242,27 +276,48 @@ func (svc *mgmtSvc) SystemCheckDisable(ctx context.Context, req *mgmtpb.CheckDis return nil, err } - if !svc.checkerIsEnabled() { - return &mgmtpb.DaosResp{Status: int32(daos.Already)}, nil - } - - if err := svc.disableChecker(); err != nil { + enabled, err := svc.checkerIsEnabled() + if err != nil { return nil, err } - // Stop all of the ranks that are currently running in checker mode. checkRanks, err := svc.sysdb.MemberRanks(system.MemberStateCheckerStarted) if err != nil { return nil, err } + + // If the system checker mode is disabled, then we should confirm that all of + // the ranks were taken out of checker mode before we declare that the operation + // was already performed. + if !enabled && len(checkRanks) == 0 { + return &mgmtpb.DaosResp{Status: int32(daos.Already)}, nil + } + + // Stop all ranks in the system so that they can be restarted in normal mode. stopReq := &mgmtpb.SystemStopReq{ Sys: req.Sys, - // Do not force stop system, it may cause resource leak and fail next system start. - Force: false, - Ranks: ranklist.RankSetFromRanks(checkRanks).String(), + // Force-stop to match current system default (DAOS-16312); avoids + // a hang when some ranks get stuck on SWIM RPCs to dead ranks. + Force: true, + } + stopResp, err := svc.SystemStop(ctx, stopReq) + if err != nil { + return nil, errors.Wrap(err, "failed to stop checker ranks") + } + failed := ranklist.NewRankSet() + for _, result := range stopResp.Results { + if result.Errored { + failed.Add(ranklist.Rank(result.Rank)) + } + } + if failed.Count() > 0 { + return nil, errors.Errorf("failed to stop rank(s) %s", failed.String()) } - if _, err := svc.SystemStop(ctx, stopReq); err != nil { - return nil, errors.Wrap(err, "failed to stop all checker ranks") + + // Persist the mode change only after every rank is stopped; while the + // flag is set, (re-)joining ranks are quarantined into checker mode. + if err := svc.disableChecker(); err != nil { + return nil, err } return &mgmtpb.DaosResp{}, nil diff --git a/src/control/server/mgmt_check_test.go b/src/control/server/mgmt_check_test.go index 83e26ca7000..0e9fc4aac0e 100644 --- a/src/control/server/mgmt_check_test.go +++ b/src/control/server/mgmt_check_test.go @@ -1349,3 +1349,274 @@ func TestServer_mgmtSvc_SystemCheckRepair(t *testing.T) { }) } } + +func TestServer_mgmtSvc_checkerIsEnabled(t *testing.T) { + t.Run("property never written means disabled", func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + svc := newTestMgmtSvc(t, log) + enabled, err := svc.checkerIsEnabled() + if err != nil { + t.Fatal(err) + } + test.AssertEqual(t, false, enabled, "checker enabled state") + }) + + t.Run("enabled then disabled", func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + svc := newTestMgmtSvc(t, log) + if err := svc.enableChecker(); err != nil { + t.Fatal(err) + } + enabled, err := svc.checkerIsEnabled() + if err != nil { + t.Fatal(err) + } + test.AssertEqual(t, true, enabled, "checker enabled state") + + if err := svc.disableChecker(); err != nil { + t.Fatal(err) + } + enabled, err = svc.checkerIsEnabled() + if err != nil { + t.Fatal(err) + } + test.AssertEqual(t, false, enabled, "checker enabled state") + }) + + t.Run("read failure propagates instead of reading as disabled", func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + svc := newTestMgmtSvc(t, log) + svc.sysdb = raft.MockDatabaseWithCfg(t, log, &raft.DatabaseConfig{}) + if _, err := svc.checkerIsEnabled(); err == nil { + t.Fatal("expected error from non-replica db, got nil") + } + }) +} + +func TestServer_mgmtSvc_SystemCheckEnable(t *testing.T) { + hr := func(a int32, rrs ...*sharedpb.RankResult) *control.HostResponse { + return &control.HostResponse{ + Addr: test.MockHostAddr(a).String(), + Message: &mgmtpb.SystemStartResp{Results: rrs}, + } + } + + for name, tc := range map[string]struct { + enabled bool + members system.Members + mResps [][]*control.HostResponse + expResp *mgmtpb.DaosResp + expErr error + expInvokeCount int + expEnabled bool + }{ + "already enabled, all ranks in checker mode": { + enabled: true, + members: system.Members{ + mockMember(t, 0, 1, "checkerstarted"), + mockMember(t, 1, 1, "checkerstarted"), + }, + expResp: &mgmtpb.DaosResp{Status: int32(daos.Already)}, + expEnabled: true, + }, + "retry after half-failed enable starts remaining ranks": { + // Flag persisted but ranks never started; a retry must not + // short-circuit as Already. + enabled: true, + members: system.Members{ + mockMember(t, 0, 1, "stopped"), + mockMember(t, 1, 1, "stopped"), + }, + mResps: [][]*control.HostResponse{{ + hr(1, mockRankSuccess("start", 0), mockRankSuccess("start", 1)), + }}, + expResp: &mgmtpb.DaosResp{}, + expInvokeCount: 1, + expEnabled: true, + }, + "rank fails to start, flag stays set for retry": { + members: system.Members{ + mockMember(t, 0, 1, "stopped"), + mockMember(t, 1, 1, "stopped"), + }, + mResps: [][]*control.HostResponse{{ + hr(1, mockRankSuccess("start", 0), mockRankFail("start", 1)), + }}, + expErr: errors.New("failed to start rank(s) 1"), + expEnabled: true, + }, + "bad member states": { + members: system.Members{ + mockMember(t, 0, 1, "joined"), + mockMember(t, 1, 1, "stopped"), + }, + expErr: errors.New("expected states"), + }, + "enables and starts ranks in check mode": { + members: system.Members{ + mockMember(t, 0, 1, "stopped"), + mockMember(t, 1, 1, "stopped"), + }, + mResps: [][]*control.HostResponse{{ + hr(1, mockRankSuccess("start", 0), mockRankSuccess("start", 1)), + }}, + expResp: &mgmtpb.DaosResp{}, + expInvokeCount: 1, + expEnabled: true, + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + if tc.mResps == nil { + tc.mResps = [][]*control.HostResponse{{}} + } + svc := mgmtSystemTestSetup(t, log, tc.members, tc.mResps...) + if tc.enabled { + if err := svc.enableChecker(); err != nil { + t.Fatal(err) + } + } + + req := &mgmtpb.CheckEnableReq{Sys: build.DefaultSystemName} + gotResp, gotErr := svc.SystemCheckEnable(test.Context(t), req) + test.CmpErr(t, tc.expErr, gotErr) + + enabled, err := svc.checkerIsEnabled() + if err != nil { + t.Fatal(err) + } + test.AssertEqual(t, tc.expEnabled, enabled, "checker enabled state") + if tc.expErr != nil { + return + } + + test.CmpAny(t, "response", tc.expResp, gotResp, cmpopts.IgnoreUnexported(mgmtpb.DaosResp{})) + + mi := svc.rpcClient.(*control.MockInvoker) + test.AssertEqual(t, tc.expInvokeCount, mi.GetInvokeCount(), "rpc client invoke count") + if tc.expInvokeCount > 0 { + startReqSent := mi.SentReqs[0].(*control.RanksReq) + test.AssertEqual(t, true, startReqSent.CheckMode, "start request check mode") + } + }) + } +} + +func TestServer_mgmtSvc_SystemCheckDisable(t *testing.T) { + hr := func(a int32, rrs ...*sharedpb.RankResult) *control.HostResponse { + return &control.HostResponse{ + Addr: test.MockHostAddr(a).String(), + Message: &mgmtpb.SystemStopResp{Results: rrs}, + } + } + + for name, tc := range map[string]struct { + enabled bool + members system.Members + mResps [][]*control.HostResponse + expResp *mgmtpb.DaosResp + expErr error + expInvokeCount int + expFanoutRanks string + expEnabled bool + }{ + "not enabled, no checker ranks": { + members: system.Members{ + mockMember(t, 0, 1, "stopped"), + mockMember(t, 1, 1, "stopped"), + }, + expResp: &mgmtpb.DaosResp{Status: int32(daos.Already)}, + }, + "rank fails to stop, flag stays set": { + // The mode change must not be persisted over a partial stop; + // while the flag is set, joins stay quarantined to checker mode. + enabled: true, + members: system.Members{ + mockMember(t, 0, 1, "checkerstarted"), + mockMember(t, 1, 1, "checkerstarted"), + }, + mResps: [][]*control.HostResponse{{ + hr(1, mockRankSuccess("stop", 0), mockRankFail("stop", 1)), + }}, + expErr: errors.New("failed to stop rank(s) 1"), + expEnabled: true, + }, + "disables and force-stops the whole system": { + // The stop must be unscoped: rank 2 started in checker mode + // but has not joined, so it is not in CheckerStarted. + enabled: true, + members: system.Members{ + mockMember(t, 0, 1, "checkerstarted"), + mockMember(t, 1, 1, "checkerstarted"), + mockMember(t, 2, 1, "starting"), + }, + mResps: [][]*control.HostResponse{{ + hr(1, mockRankSuccess("stop", 0), mockRankSuccess("stop", 1), + mockRankSuccess("stop", 2)), + }}, + expResp: &mgmtpb.DaosResp{}, + expInvokeCount: 1, + expFanoutRanks: "0-2", + }, + "flag already cleared but checker ranks still running": { + // A prior disable that failed after clearing the flag must be + // retryable, not short-circuited as Already. + members: system.Members{ + mockMember(t, 0, 1, "checkerstarted"), + mockMember(t, 1, 1, "checkerstarted"), + }, + mResps: [][]*control.HostResponse{{ + hr(1, mockRankSuccess("stop", 0), mockRankSuccess("stop", 1)), + }}, + expResp: &mgmtpb.DaosResp{}, + expInvokeCount: 1, + expFanoutRanks: "0-1", + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + if tc.mResps == nil { + tc.mResps = [][]*control.HostResponse{{}} + } + svc := mgmtSystemTestSetup(t, log, tc.members, tc.mResps...) + if tc.enabled { + if err := svc.enableChecker(); err != nil { + t.Fatal(err) + } + } + + req := &mgmtpb.CheckDisableReq{Sys: build.DefaultSystemName} + gotResp, gotErr := svc.SystemCheckDisable(test.Context(t), req) + test.CmpErr(t, tc.expErr, gotErr) + + enabled, err := svc.checkerIsEnabled() + if err != nil { + t.Fatal(err) + } + test.AssertEqual(t, tc.expEnabled, enabled, "checker enabled state") + if tc.expErr != nil { + return + } + + test.CmpAny(t, "response", tc.expResp, gotResp, cmpopts.IgnoreUnexported(mgmtpb.DaosResp{})) + + mi := svc.rpcClient.(*control.MockInvoker) + test.AssertEqual(t, tc.expInvokeCount, mi.GetInvokeCount(), "rpc client invoke count") + if tc.expInvokeCount > 0 { + stopReqSent := mi.SentReqs[0].(*control.RanksReq) + test.AssertEqual(t, true, stopReqSent.Force, "stop request force flag") + test.AssertEqual(t, tc.expFanoutRanks, stopReqSent.Ranks, "stop request ranks") + } + }) + } +} diff --git a/src/control/server/mgmt_system.go b/src/control/server/mgmt_system.go index 7eb43f6232e..e3aa6acb1d8 100644 --- a/src/control/server/mgmt_system.go +++ b/src/control/server/mgmt_system.go @@ -263,8 +263,12 @@ func (svc *mgmtSvc) join(ctx context.Context, req *mgmtpb.JoinReq, peerAddr *net member.Rank, member.PrimaryFabricURI, member.SecondaryFabricURIs, joinResponse.PrevState, member.State) } + checkMode, err := svc.checkerIsEnabled() + if err != nil { + return nil, err + } joinState := mgmtpb.JoinResp_IN - if svc.checkerIsEnabled() { + if checkMode { joinState = mgmtpb.JoinResp_CHECK } resp := &mgmtpb.JoinResp{ diff --git a/src/tests/ftest/recovery/check_start_corner_case.py b/src/tests/ftest/recovery/check_start_corner_case.py index 20363eaea58..a9bb4c39d6a 100644 --- a/src/tests/ftest/recovery/check_start_corner_case.py +++ b/src/tests/ftest/recovery/check_start_corner_case.py @@ -359,7 +359,9 @@ def test_two_pools_corrupted(self): self.log_step(msg) query_reports = None for _ in range(8): - check_query_out = dmg_command.check_query() + # Scope by pool UUID: resolved findings from prior checker instances survive + # disable/enable and reuse pool labels (DAOS-18773). + check_query_out = dmg_command.check_query(pool=pool_1.uuid) # Status becomes RUNNING immediately, but it may take a while to detect the # inconsistency. If detected, "reports" field is filled. if check_query_out["response"]["status"] == "RUNNING": @@ -392,26 +394,22 @@ def test_two_pools_corrupted(self): dmg_command.check_start(pool=corrupted_diff) self.log_step("Wait for checker to detect inconsistent container label for pool_2 pool_3.") - query_reports = None - for _ in range(8): - check_query_out = dmg_command.check_query() - # Status becomes RUNNING immediately, but it may take a while to detect the - # inconsistency. If detected, "reports" field is filled. - if check_query_out["response"]["status"] == "RUNNING": - query_reports = check_query_out["response"]["reports"] - # We have three corrupted pools, so wait for three reports. - if query_reports and len(query_reports) == 3: - break - time.sleep(5) - if not query_reports: - self.fail("Checker didn't detect any inconsistency!") - if len(query_reports) < 3: - self.fail(f"Checker only detected {len(query_reports)}/3 consistencies!") - # Obtain the seq nums (ID) to repair. seq_nums = [] - for query_report in query_reports: - if query_report["pool_label"] in (pool_2.label.value, pool_3.label.value): - seq_nums.append(str(query_report["seq"])) + for pool in (pool_2, pool_3): + query_reports = None + for _ in range(8): + check_query_out = dmg_command.check_query(pool=pool.uuid) + # Status becomes RUNNING immediately, but it may take a while to detect the + # inconsistency. If detected, "reports" field is filled. + if check_query_out["response"]["status"] == "RUNNING": + query_reports = check_query_out["response"]["reports"] + if query_reports: + break + time.sleep(5) + if not query_reports: + self.fail(f"Checker didn't detect any inconsistency in {pool.identifier}!") + # Obtain the seq num (ID) to repair. + seq_nums.append(str(query_reports[0]["seq"])) self.log_step("Repair with option 2 for pool_2 pool_3.") for seq_num in seq_nums: diff --git a/src/tests/suite/daos_cr.c b/src/tests/suite/daos_cr.c index d8be2771abb..79725669bdd 100644 --- a/src/tests/suite/daos_cr.c +++ b/src/tests/suite/daos_cr.c @@ -26,8 +26,14 @@ * #define CR_ACCURATE_QUERY_RESULT 1 */ -/* Start pool service may take sometime, let's wait for at most CR_WAIT_MAX * 2 seconds. */ -#define CR_WAIT_MAX (45) +/* + * Wall-clock budget for checker-progress waits. The pre-bindings dmg helpers + * fork/exec'd dmg for every query (~1-2s each), so the historical + * 45-iteration polls implied roughly three minutes of wall time; the + * bindings answer in milliseconds, which silently halved the wait. Budget + * time, not iterations. + */ +#define CR_WAIT_SECS 300 /* 256MB for CR pool size. */ #define CR_POOL_SIZE (1 << 28) @@ -233,8 +239,15 @@ cr_fault_inject(uuid_t uuid, bool mgmt, const char *fault) static inline int cr_mode_switch(bool enable) { + time_t deadline = time(NULL) + CR_WAIT_SECS; + int rc; + print_message("CR: %s check mode\n", enable ? "enable" : "disable"); - return dmg_check_switch(dmg_config_file, enable); + /* member states may not have settled yet after a system stop/start */ + while ((rc = dmg_check_switch(dmg_config_file, enable)) == -DER_AGAIN && + time(NULL) < deadline) + sleep(1); + return rc; } static inline int @@ -356,11 +369,11 @@ cr_check_query(uint32_t pool_nr, uuid_t uuids[], struct daos_check_info *dci) } static inline int -cr_check_repair(uint64_t seq, uint32_t opt) +cr_check_repair(uint64_t seq, uint32_t action) { - print_message("CR: handle check interaction for seq %lu, option %u ...\n", - (unsigned long)seq, opt); - return dmg_check_repair(dmg_config_file, seq, opt); + print_message("CR: handle check interaction for seq %lu, action %u ...\n", + (unsigned long)seq, action); + return dmg_check_repair(dmg_config_file, seq, action); } static inline int @@ -374,10 +387,13 @@ cr_check_set_policy(uint32_t flags, const char *policies) static struct daos_check_report_info * cr_locate_dcri(struct daos_check_info *dci, struct daos_check_report_info *base, uuid_t uuid) { - struct daos_check_report_info *last = &dci->dci_reports[dci->dci_report_nr - 1]; + struct daos_check_report_info *last; struct daos_check_report_info *dcri = NULL; bool found = false; + D_ASSERTF(dci->dci_report_nr > 0, "no check reports for pool " DF_UUID "\n", DP_UUID(uuid)); + last = &dci->dci_reports[dci->dci_report_nr - 1]; + if (base != NULL) dcri = base + 1; else @@ -442,7 +458,7 @@ cr_cleanup(test_arg_t *arg, struct test_pool *pools, uint32_t nr) } rc = dmg_pool_destroy(dmg_config_file, pools[i].pool_uuid, arg->group, 1); - if (rc != 0 && rc != -DER_NONEXIST && rc != -DER_MISC) + if (rc != 0 && rc != -DER_NONEXIST) print_message("CR: dmg_pool_destroy failed: "DF_RC"\n", DP_RC(rc)); } } @@ -451,18 +467,26 @@ static void cr_ins_wait(uint32_t pool_nr, uuid_t uuids[], struct daos_check_info *dci) { int rc; - int i; + + time_t deadline = time(NULL) + CR_WAIT_SECS; print_message("CR: waiting check instance ...\n"); - for (i = 0; i < CR_WAIT_MAX; i++) { + for (;;) { cr_dci_fini(dci); rc = dmg_check_query(dmg_config_file, pool_nr, uuids, dci); - assert_rc_equal(rc, 0); + if (rc != 0) + print_message("CR: check query failed (" DF_RC "), retrying\n", DP_RC(rc)); + else if (!cr_ins_status_init(dci->dci_status) && + !cr_ins_status_running(dci->dci_status)) + break; - if (!cr_ins_status_init(dci->dci_status) && !cr_ins_status_running(dci->dci_status)) + if (time(NULL) >= deadline) { + /* tolerate transient query failures, not persistent ones */ + assert_rc_equal(rc, 0); break; + } sleep(2); } @@ -471,21 +495,27 @@ cr_ins_wait(uint32_t pool_nr, uuid_t uuids[], struct daos_check_info *dci) static void cr_pool_wait(uint32_t pool_nr, uuid_t uuids[], struct daos_check_info *dci) { - int rc; - int i; + time_t deadline = time(NULL) + CR_WAIT_SECS; + int rc; print_message("CR: waiting check pool ...\n"); cr_dump_pools(pool_nr, uuids); - for (i = 0; i < CR_WAIT_MAX; i++) { + for (;;) { cr_dci_fini(dci); rc = dmg_check_query(dmg_config_file, pool_nr, uuids, dci); - assert_rc_equal(rc, 0); + if (rc != 0) + print_message("CR: check query failed (" DF_RC "), retrying\n", DP_RC(rc)); + else if (!cr_ins_status_init(dci->dci_status) && dci->dci_pools != NULL && + !cr_pool_status_checking(dci->dci_pools[0].dcpi_status)) + break; - if (!cr_ins_status_init(dci->dci_status) && dci->dci_pools != NULL && - !cr_pool_status_checking(dci->dci_pools[0].dcpi_status)) + if (time(NULL) >= deadline) { + /* tolerate transient query failures, not persistent ones */ + assert_rc_equal(rc, 0); break; + } sleep(2); } @@ -1244,7 +1274,7 @@ cr_leader_interaction(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == action) { - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } } @@ -1335,7 +1365,7 @@ cr_engine_interaction(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == action) { - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } } @@ -1433,7 +1463,7 @@ cr_repair_forall_leader(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == action) { - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } } @@ -1550,7 +1580,7 @@ cr_repair_forall_engine(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == action) { - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } } @@ -1837,7 +1867,7 @@ cr_stop_specified(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == action) { - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } } @@ -1950,7 +1980,7 @@ cr_auto_reset(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == action) { - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } } @@ -2003,8 +2033,8 @@ cr_pause(void **state, bool force) struct daos_check_info dci = { 0 }; uint32_t class = TCC_POOL_BAD_LABEL; uint32_t action = TCA_INTERACT; - int rc; - int i; + time_t deadline; + int rc; rc = cr_pool_create(state, &pool, false, class); assert_rc_equal(rc, 0); @@ -2032,7 +2062,8 @@ cr_pause(void **state, bool force) rc = cr_system_start(); assert_rc_equal(rc, 0); - for (i = 0; i < CR_WAIT_MAX; i++) { + deadline = time(NULL) + CR_WAIT_SECS; + do { /* Sleep for a while after system re-started under check mode. */ sleep(2); @@ -2042,7 +2073,7 @@ cr_pause(void **state, bool force) break; assert_rc_equal(rc, -DER_INVAL); - } + } while (time(NULL) < deadline); rc = cr_ins_verify(&dci, TCIS_PAUSED); assert_rc_equal(rc, 0); @@ -2925,7 +2956,7 @@ cr_engine_death(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == action) { /* Repair the pool label with the lost rank. */ - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } } @@ -3041,7 +3072,7 @@ cr_engine_rejoin_succ(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == action) { - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } } @@ -3146,7 +3177,7 @@ cr_engine_rejoin_fail(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == action) { /* Repair the inconsistency with the lost rank. */ - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } } @@ -3291,7 +3322,7 @@ cr_multiple_pools(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == actions[1]) { - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } } @@ -3348,7 +3379,7 @@ cr_multiple_pools(void **state) dcri = cr_locate_dcri(&dci, dcri, uuids[i]); for (j = 0; j < dcri->dcri_option_nr; j++) { if (dcri->dcri_options[j] == actions[0]) { - rc = cr_check_repair(dcri->dcri_seq, j); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[j]); break; } } @@ -3389,7 +3420,7 @@ cr_multiple_pools(void **state) for (i = 0; i < dcri->dcri_option_nr; i++) { if (dcri->dcri_options[i] == actions[1]) { - rc = cr_check_repair(dcri->dcri_seq, i); + rc = cr_check_repair(dcri->dcri_seq, dcri->dcri_options[i]); break; } }