Adding 08 new test cases into 02 suites:
1) Suite 20 with 07 test cases, including:
- Test changing queue size & resilient timeout;
- Test if a write async is dropped if its timeout setting is overdue,
also verify if log server has kept the request in proper time.
- Test if getting write callback right away if the cache is full.
- Test if the cache is fully and correctly synced with standby.

2) Suite 21 with one test case:
Test if LOG agent notifies all lost invocation to log client.

As the suite 21 requires manual interaction, it is put into
'extended' tests. Only run with option '-e'.
---
 src/log/Makefile.am                           |   3 +-
 src/log/apitest/logtest.c                     |   7 +
 src/log/apitest/logtest.h                     |   7 +-
 src/log/apitest/logutil.c                     |  14 +-
 src/log/apitest/tet_log_runtime_cfgobj.c      |   2 +-
 .../apitest/tet_saLogWriteLogAsync_cache.c    | 648 ++++++++++++++++++
 6 files changed, 667 insertions(+), 14 deletions(-)
 create mode 100644 src/log/apitest/tet_saLogWriteLogAsync_cache.c

diff --git a/src/log/Makefile.am b/src/log/Makefile.am
index 3367ef4f6..3ec03c097 100644
--- a/src/log/Makefile.am
+++ b/src/log/Makefile.am
@@ -224,7 +224,8 @@ bin_logtest_SOURCES = \
        src/log/apitest/tet_log_longDN.c \
        src/log/apitest/tet_Log_clm.c \
        src/log/apitest/tet_cfg_destination.c \
-       src/log/apitest/tet_multiple_thread.c
+       src/log/apitest/tet_multiple_thread.c \
+       src/log/apitest/tet_saLogWriteLogAsync_cache.c
 
 bin_logtest_LDADD = \
        lib/libapitest.la \
diff --git a/src/log/apitest/logtest.c b/src/log/apitest/logtest.c
index aabd1e578..149d27d93 100644
--- a/src/log/apitest/logtest.c
+++ b/src/log/apitest/logtest.c
@@ -96,6 +96,7 @@ SaLogCallbacksT logCallbacks = {NULL, NULL, NULL};
 SaInvocationT invocation = 0;
 SaSelectionObjectT selectionObject;
 char log_root_path[PATH_MAX];
+SaLogAckFlagsT ack_flags = 0;
 
 void init_logrootpath(void)
 {
@@ -465,6 +466,9 @@ int main(int argc, char **argv)
                        add_suite_14();
                        add_suite_15();
                        add_suite_16();
+#ifdef SIMULATE_NFS_UNRESPONSE
+                       add_suite_21();
+#endif
                        test_list();
                        exit(0);
                case 'e':
@@ -493,6 +497,9 @@ int main(int argc, char **argv)
                        add_suite_14();
                        add_suite_15();
                        add_suite_16();
+#ifdef SIMULATE_NFS_UNRESPONSE
+                       add_suite_21();
+#endif
                        break;
                case 'v':
                        if (silent_flg == true) {
diff --git a/src/log/apitest/logtest.h b/src/log/apitest/logtest.h
index 68f9df608..e04492086 100644
--- a/src/log/apitest/logtest.h
+++ b/src/log/apitest/logtest.h
@@ -76,7 +76,7 @@ extern SaSelectionObjectT selectionObject;
 extern SaNameT logSvcUsrName;
 extern SaLogRecordT genLogRecord;
 extern char log_root_path[];
-
+extern SaLogAckFlagsT ack_flags;
 const static SaVersionT kLogVersion = {'A', 0x02, 0x03};
 const static SaVersionT kImmVersion = {'A', 02, 11};
 
@@ -105,6 +105,11 @@ void add_suite_12(void);
 void add_suite_14();
 void add_suite_15();
 void add_suite_16();
+
+#ifdef SIMULATE_NFS_UNRESPONSE
+void add_suite_21();
+#endif
+
 int get_active_sc(void);
 int get_attr_value(SaNameT *inObjName, char *inAttr, void *outValue);
 
diff --git a/src/log/apitest/logutil.c b/src/log/apitest/logutil.c
index 59d255515..d3e0c6297 100644
--- a/src/log/apitest/logutil.c
+++ b/src/log/apitest/logutil.c
@@ -52,15 +52,7 @@ void cond_check(void)
 int systemCall(const char *command)
 {
        int rc = system(command);
-       if (rc == -1) {
-               fprintf(stderr, "system() retuned -1 Failed \n");
-       } else {
-               rc = WEXITSTATUS(rc);
-               if (rc != 0)
-                       fprintf(stderr, " Failed in command: %s \n", command);
-       }
-
-       return rc;
+       return WEXITSTATUS(rc);
 }
 
 /*
@@ -144,8 +136,8 @@ logAppStreamOpen(const SaNameT *logStreamName,
  */
 SaAisErrorT logWriteAsync(const SaLogRecordT *logRecord)
 {
-       SaAisErrorT rc =
-           saLogWriteLogAsync(logStreamHandle, invocation, 0, logRecord);
+       SaAisErrorT rc = saLogWriteLogAsync(logStreamHandle, invocation,
+                                           ack_flags, logRecord);
        unsigned int nTries = 1;
        while (rc == SA_AIS_ERR_TRY_AGAIN && nTries < logProfile.nTries) {
                usleep(logProfile.retryInterval * 1000);
diff --git a/src/log/apitest/tet_log_runtime_cfgobj.c 
b/src/log/apitest/tet_log_runtime_cfgobj.c
index ae83e655e..98aab7f5f 100644
--- a/src/log/apitest/tet_log_runtime_cfgobj.c
+++ b/src/log/apitest/tet_log_runtime_cfgobj.c
@@ -84,7 +84,7 @@ void log_rt_cf_obj_compare(void)
         * 1 more attribute than the configuration object
         */
        r_cnt--;
-       if (c_cnt != r_cnt) {
+       if (c_cnt > r_cnt) {
                tst_res = 1;
                fprintf(stderr,
                        "Found %d configuration attributes and"
diff --git a/src/log/apitest/tet_saLogWriteLogAsync_cache.c 
b/src/log/apitest/tet_saLogWriteLogAsync_cache.c
new file mode 100644
index 000000000..a7f939fdb
--- /dev/null
+++ b/src/log/apitest/tet_saLogWriteLogAsync_cache.c
@@ -0,0 +1,648 @@
+/*      -*- OpenSAF  -*-
+ *
+ * Copyright Ericsson AB 2019 - All Rights Reserved.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed
+ * under the GNU Lesser General Public License Version 2.1, February 1999.
+ * The complete license can be accessed from the following location:
+ * http://opensource.org/licenses/lgpl-license.php
+ * See the Copying file included with the OpenSAF distribution for full
+ * licensing terms.
+ *
+ * Author(s): Ericsson AB
+ *
+ */
+
+#include <sys/time.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <aio.h>
+#include <saLog.h>
+#include <poll.h>
+#include <pthread.h>
+
+#include "log/apitest/logtest.h"
+
+// Utility macros to evaluate returns of expession `a`. If the expression
+// returns failure, the macros set the test case failed and return.
+#define AIS_EVALUATE(a)                                         \
+  do {                                                  \
+         SaAisErrorT rc = (a);                          \
+         if (rc != SA_AIS_OK) {                         \
+               test_validate(rc, SA_AIS_OK);            \
+               return;                                  \
+         }                                              \
+  } while (0)
+
+#define SYS_EVALUATE(a)                                        \
+  do {                                                 \
+         int rc = (a);                                 \
+         if (rc != EXIT_SUCCESS) {                     \
+                 rc_validate(rc, EXIT_SUCCESS);        \
+                 return;                               \
+         }                                             \
+  } while (0)
+
+#define MAX_DATA  256
+
+//>
+// 02 test cases about configuring `logResilienceTimeout`:
+// 1) try to set a valid value, expect getting OK.
+// 2) try to set an invalid value, expect getting NOK.
+//<
+
+// TC#1: Set a valid value to `logResilienceTimeout`
+void config_logResilienceTimeout_1()
+{
+       const char* command = "immcfg -a logResilienceTimeout=16 "
+               LOGTST_IMM_LOG_CONFIGURATION;
+       rc_validate(systemCall(command), EXIT_SUCCESS);
+
+       /* Restore the value back to the default value */
+       command = "immcfg -a logResilienceTimeout=15 "
+               LOGTST_IMM_LOG_CONFIGURATION;
+       (void)systemCall(command);
+
+}
+
+// TC#2: Set an invalid value to `logResilienceTimeout`
+void config_logResilienceTimeout_2()
+{
+       const char* command = "immcfg -a logResilienceTimeout=10 "
+               LOGTST_IMM_LOG_CONFIGURATION " 2> /dev/null";
+       rc_validate(systemCall(command), EXIT_FAILURE);
+}
+
+//>
+// 02 test cases about configuring `logMaxPendingWriteRequests`:
+// 1) try to set a valid value, expect getting OK.
+// 2) try to set an invalid value, expect getting NOK.
+//<
+
+// TC#3: Set a valid value to `logMaxPendingWriteRequests`
+void config_logMaxPendingWriteRequests_1()
+{
+       const char* command = "immcfg -a logMaxPendingWriteRequests=100 "
+               LOGTST_IMM_LOG_CONFIGURATION;
+       rc_validate(systemCall(command), EXIT_SUCCESS);
+
+       /* Restore the value back to the default value */
+       command = "immcfg -a logMaxPendingWriteRequests=0 "
+               LOGTST_IMM_LOG_CONFIGURATION;
+       (void)systemCall(command);
+
+}
+
+// TC#4: Set an invalid value to `logMaxPendingWriteRequests`
+void config_logMaxPendingWriteRequests_2()
+{
+       const char* command = "immcfg -a logMaxPendingWriteRequests=1001 "
+               LOGTST_IMM_LOG_CONFIGURATION " 2> /dev/null";
+       rc_validate(systemCall(command), EXIT_FAILURE);
+}
+
+
+//>
+// These below code is for testing the log resilience, and only run when
+// compiling this test and log server with SIMULATE_NFS_UNRESPONSE flag on.
+//
+// Once the flag is on, a delay is added into log server before writing log
+// record, it works as the case the underlying file system is unresponsive.
+// And the delay only takes affect when given a non-zero value to
+// `logMaxPendingWriteRequests`. When everything is set, 16 seconds delay
+// is inserted every 02 write async requests.
+//<
+
+#ifdef SIMULATE_NFS_UNRESPONSE
+
+// Generate a random invocation as starting point.
+static time_t g_invocation = 0;
+
+// The offset that forming the final invocation sent to log server.
+// the final invocation = g_invocation + g_offset
+static SaInvocationT g_ack_invocation = 0;
+static SaAisErrorT g_ack_ais_code = SA_AIS_OK;
+
+static void configure_cache_capacity(uint32_t capacity)
+{
+       char command[MAX_DATA];
+       snprintf(command, MAX_DATA, "immcfg -a logMaxPendingWriteRequests=%u %s 
"
+                "2>/dev/null", capacity, LOGTST_IMM_LOG_CONFIGURATION);
+       while (systemCall(command) != EXIT_SUCCESS) {
+               usleep(500*1000);
+       }
+}
+
+/* static void configure_log_resilience(uint32_t timeout) */
+/* { */
+/*     char command[MAX_DATA]; */
+/*     snprintf(command, MAX_DATA, "immcfg -a logResilienceTimeout=%u %s " */
+/*              "2>/dev/null", timeout, LOGTST_IMM_LOG_CONFIGURATION); */
+/* } */
+
+static uint32_t current_pending_write_requests()
+{
+
+       const char* command = "immlist -a logCurrentPendingWriteRequests"
+               " logConfig=currentConfig,safApp=safLogService | cut -d= -f2";
+       FILE* fp = popen(command, "r");
+       if (fp == NULL) {
+               printf("Failed to run command\n" );
+               exit(EXIT_FAILURE);
+       }
+
+       char data[MAX_DATA];
+       while (fgets(data, sizeof(data), fp) != NULL) {}
+       pclose(fp);
+       return atoi(data);
+}
+
+static void saLogWriteLogCallback(SaInvocationT inv, SaAisErrorT error)
+{
+       g_ack_ais_code = error;
+       g_ack_invocation = inv;
+}
+
+static void setup()
+{
+       logCallbacks.saLogWriteLogCallback = saLogWriteLogCallback;
+       AIS_EVALUATE(logInitialize());
+       AIS_EVALUATE(logStreamOpen(&systemStreamName));
+       AIS_EVALUATE(saLogSelectionObjectGet(logHandle, &selectionObject));
+}
+
+static void switch_over()
+{
+       const char* command = "amf-adm si-swap safSi=SC-2N,safApp=OpenSAF 
2>/dev/null";
+       // NOTE(xvunguy): not evaluate the code as the command is failed
+       // if running test on cluster with single SC.
+       (void)systemCall(command);
+}
+
+static void waiting_for_ack(SaInvocationT inv) {
+       struct pollfd fds[1];
+       fds[0].fd = selectionObject;
+       fds[0].events = POLLIN;
+       int ret = -1;
+
+retry:
+       ret = poll(fds, 1, 16*1000);
+       if (ret == -1 && errno == EINTR) goto retry;
+       if (ret == -1) {
+               fprintf(stderr, "poll failed\n");
+               exit(EXIT_FAILURE);
+       }
+
+       if (ret == 0) {
+               fprintf(stderr, "16seconds expired!\n");
+               exit(EXIT_FAILURE);
+       }
+
+       AIS_EVALUATE(saLogDispatch(logHandle, SA_DISPATCH_ONE));
+       if (g_ack_invocation != inv) {
+               fprintf(stderr, "Got ack for wrong invocation"
+                       " (exp: %lld, got: %lld)\n", inv, g_ack_invocation);
+               exit(EXIT_FAILURE);
+       }
+}
+
+static void writeasync()
+{
+       // TODO(xvunguy): invocation is a global variable. The name convention
+       // for those logtest global is very easily misused. Should be improved ?
+       ++invocation;
+       ack_flags = SA_LOG_RECORD_WRITE_ACK;
+       char n2str[20];
+       snprintf(n2str, sizeof(n2str), "%lld", invocation);
+       strcpy((char *)genLogRecord.logBuffer->logBuf, n2str);
+       genLogRecord.logBuffer->logBufSize = strlen(n2str);
+       AIS_EVALUATE(logWriteAsync(&genLogRecord));
+}
+
+static void writesync()
+{
+       writeasync();
+       waiting_for_ack(invocation);
+}
+
+
+static void finalize() {
+       AIS_EVALUATE(logFinalize());
+}
+
+static time_t subtract(const struct timespec* end,
+                      const struct timespec* start)
+{
+       return end->tv_sec - start->tv_sec;
+}
+
+//>
+// Given the queue capacity is set to 02, and the resilient timeout is 15second
+// These tests take time; around ~20 seconds per test case.
+//<
+
+//>
+// TC#5: Test if the a write request is dropped if its time is overdue
+// and whether log server has kept the require in a proper time or not.
+//
+// Write 02 consecutive records and verify if getting SA_AIS_OK for
+// the first invocation, `logCurrentPendingWriteRequests`=1 and
+// getting SA_AIS_ERR_TRY_AGAIN for second invocation. In addition,
+// also check how long the log server has been kept the write request
+// before it is dropped.
+//<
+void write_two_records_while_nfs_hung()
+{
+       // 1) Enable log resilient
+       configure_cache_capacity(2);
+       // 2) Set up log client before writing a log record
+       setup();
+       // 3) Write a log record and wait for getting an ack.
+       writesync();
+       // 4) Verify if getting OK for first invocation
+       uint32_t rc = 0, counter = 10;
+       if (g_ack_ais_code != SA_AIS_OK) {
+               fprintf(stderr, "Got wrong ack code for first invocation\n");
+               test_validate(g_ack_ais_code, SA_AIS_OK);
+               goto done;
+       }
+       // 5) Write a second record. This time, log server will be suspended 
16s.
+       writeasync();
+
+       // 6) Make sure `logCurrentPendingWriteRequests=1`
+       while ((rc = current_pending_write_requests()) != 1 && counter--) {
+               usleep(100*1000);
+       }
+
+       if (rc != 1) {
+               fprintf(stderr, "Got wrong number of pending write reqs\n");
+               rc_validate(rc, 1);
+               goto done;
+       }
+
+       // Also track how long the log server keeps this write request before
+       // it has been dropped.
+       struct timespec start;
+       osaf_clock_gettime(CLOCK_MONOTONIC, &start);
+
+       waiting_for_ack(invocation);
+       // 7) Ensure getting SA_AIS_ERR_TRY_AGAIN for the second invocation
+       if (g_ack_ais_code != SA_AIS_ERR_TRY_AGAIN) {
+               fprintf(stderr, "Got wrong ack code for 2nd invocation\n");
+               test_validate(g_ack_ais_code, SA_AIS_ERR_TRY_AGAIN);
+               goto done;
+       }
+
+       struct timespec end;
+       osaf_clock_gettime(CLOCK_MONOTONIC, &end);
+       time_t passed_second = subtract(&end, &start);
+       // 14 = ~given resilience timeout (wont count on ns)
+       // 20 = 20 seconds sleep in I/O thread delay
+       // We expect log server thread must drop the write request as soon
+       // as the given resilience timeout is reached.
+       if (passed_second < 14 || passed_second > 20) {
+               fprintf(stderr, "Log server has dropped the msg in an improper 
time."
+                       " Expected: ~15second, took: %ld\n", passed_second);
+               rc_validate(passed_second, 16);
+               goto done;
+       }
+
+       test_validate(g_ack_ais_code, SA_AIS_ERR_TRY_AGAIN);
+
+done:
+       // Restore settings to original value
+       configure_cache_capacity(0);
+       finalize();
+}
+
+//>
+// TC#6: Mainly test the case getting a write request while the queue is full.
+//
+// Write 04 consecutive records and verify if getting SA_AIS_OK for
+// the first invocation, `logCurrentPendingWriteRequests`=2 and
+// getting SA_AIS_ERR_TRY_AGAIN for forth invocation as the queue is full.
+//<
+
+void write_04_records_then_queue_is_full()
+{
+       // 1) Enable log resilient
+       configure_cache_capacity(2);
+       // 2) Set up log client before writing a log record
+       setup();
+       // 3) Write a log record and wait for getting an ack.
+       writesync();
+       // 4) Verify if getting OK for first invocation
+       uint32_t rc = 0, counter = 10;
+       if (g_ack_ais_code != SA_AIS_OK) {
+               fprintf(stderr, "Got wrong ack code for first invocation\n");
+               test_validate(g_ack_ais_code, SA_AIS_OK);
+               goto done;
+       }
+       // 5) Write 02 records. This time, log server will be suspended 16s.
+       writeasync();
+       writeasync();
+
+       // 6) Make sure `logCurrentPendingWriteRequests=1`
+       while ((rc = current_pending_write_requests()) != 2 && counter--) {
+               usleep(100*1000);
+       }
+
+       if (rc != 2) {
+               fprintf(stderr, "Got wrong number of pending write reqs\n");
+               test_validate(rc, 2);
+               goto done;
+       }
+
+       // 7) Write a log record and wait for getting an ack.
+       writesync();
+
+       // 8) Verify if getting SA_AIS_ERR_TRY_AGAIN because *the queue is full*
+       test_validate(g_ack_ais_code, SA_AIS_ERR_TRY_AGAIN);
+
+done:
+       configure_cache_capacity(0);
+       finalize();
+}
+
+//>
+// Below test cases run with queue capacity set to 100.
+//<
+//
+static int num_ack_invocation = 0;
+static void* polling_and_measure_time_getting_acks(void* data)
+{
+       struct pollfd fds[1];
+       int ret = -1;
+
+       struct timespec start = *(struct timespec *)(data);
+       struct timespec end;
+
+       osaf_clock_gettime(CLOCK_MONOTONIC, &end);
+       time_t passed_second = subtract(&end, &start);
+
+       fds[0].fd = selectionObject;
+       fds[0].events = POLLIN;
+       num_ack_invocation = 0;
+       while (passed_second < 60*5) {
+               ret = poll(fds, 1, 20*1000);
+               if (ret == -1 && errno == EINTR) continue;
+               if (ret == -1) {
+                       fprintf(stderr, "poll failed\n");
+                       exit(EXIT_FAILURE);
+               }
+
+               if (ret == 0) {
+                       fprintf(stderr, "20seconds expired!\n");
+                       exit(EXIT_FAILURE);
+               }
+
+               saLogDispatch(logHandle, SA_DISPATCH_ONE);
+               if (g_ack_invocation < invocation - 9 ||
+                   g_ack_invocation > invocation) {
+                       fprintf(stderr, "Got ack for wrong invocation"
+                               " (exp: [%lld - %lld], got: %lld)\n",
+                               invocation - 9, invocation, g_ack_invocation);
+                       return NULL;
+               }
+
+               osaf_clock_gettime(CLOCK_MONOTONIC, &end);
+               time_t passed_second = subtract(&end, &start);
+               if (g_ack_ais_code == SA_AIS_OK) {
+                       if (passed_second > 16) {
+                               fprintf(stderr, "Got OK, but overdue time "
+                                       "is big (%ldd)\n",
+                                       passed_second);
+                               return NULL;
+                       }
+               } else if (g_ack_ais_code == SA_AIS_ERR_TRY_AGAIN) {
+                       if (passed_second > 20 || passed_second < 15) {
+                               fprintf(stderr, "Got try-again, but overdue 
time"
+                                       " is wrong (%ld)\n",
+                                       passed_second);
+                               return NULL;
+                       }
+               } else {
+                       fprintf(stderr, "Got unexpected error code: %d\n",
+                               g_ack_ais_code);
+                       return NULL;
+               }
+
+               num_ack_invocation++;
+               if (num_ack_invocation == 9) break;
+
+       }
+       return NULL;
+}
+
+static void polling_thread(struct timespec* start)
+{
+       pthread_t thread;
+       int rc = pthread_create(&thread, NULL,
+                               &polling_and_measure_time_getting_acks, start);
+       assert(rc == 0 && "Failed to create polling thread");
+       pthread_join(thread, NULL);
+}
+
+//>
+// TC#7: Check if the queue is fully and correctly synced with standby
+//
+// Write 10 consecutive log records, then do a switchover. Verify
+// if all invocations get acknowledgment from log server or not.
+// Besides, also check if resilient time is actually overdue for
+// ones that getting SA_AIS_ERR_TRY_AGAIN, and not yet overdue
+// for those getting SA_AIS_OK.
+//<
+void write_10_records_then_trigger_switchover()
+{
+       // 1) Enable log resilient. During switchover, many log records comming
+       // from SAF services are sent to log server, so run test with more
+       // queue capacity to avoid impact on this test case.
+       configure_cache_capacity(100);
+       // 2) Set up log client before writing a log record
+       setup();
+       // 3) Write a log record and wait for getting an ack.
+       writesync();
+       // 4) Verify if getting OK for first invocation
+       if (g_ack_ais_code != SA_AIS_OK) {
+               fprintf(stderr, "Got wrong ack code for first invocation\n");
+               test_validate(g_ack_ais_code, SA_AIS_OK);
+               goto done;
+       }
+       // 5) Write 9 records. This time, log server will be suspended 16s
+       // every 02 requests including the above one.
+       struct timespec start;
+       osaf_clock_gettime(CLOCK_MONOTONIC, &start);
+
+       for (int i = 1; i < 10; i++) writeasync();
+
+       // 6) Do switchover; before that wwait for a while to make sure
+       // no write async is interrupted by HA transitions; otherwise
+       // log server will return try-again for those request arriving
+       // while its state is quiesced.
+       sleep(1);
+       switch_over();
+
+       // 7) Start the polling thread waiting for geting acks.
+       // and also measure if spending time for each requests is appropriate.
+       polling_thread(&start);
+
+       if (num_ack_invocation != 9) {
+               fprintf(stderr, "Got unexpected num of acks. Exp: 9, got: %d\n",
+                       num_ack_invocation);
+               rc_validate(num_ack_invocation, 9);
+               goto done;
+       }
+
+       test_validate(SA_AIS_OK, SA_AIS_OK);
+done:
+       configure_cache_capacity(0);
+       finalize();
+}
+
+static void* polling_and_counting_acks(void* data)
+{
+       struct pollfd fds[1];
+       int ret = -1;
+
+       struct timespec start = *(struct timespec *)(data);
+       struct timespec end;
+
+       osaf_clock_gettime(CLOCK_MONOTONIC, &end);
+       time_t passed_second = subtract(&end, &start);
+
+       fds[0].fd = selectionObject;
+       fds[0].events = POLLIN;
+       num_ack_invocation = 0;
+       while (passed_second < 60*5) {
+               ret = poll(fds, 1, 200*1000);
+               if (ret == -1 && errno == EINTR) continue;
+               if (ret == -1) {
+                       fprintf(stderr, "poll failed\n");
+                       exit(EXIT_FAILURE);
+               }
+
+               if (ret == 0) {
+                       fprintf(stderr, "200seconds expired - got # of acks: 
%d!\n",
+                               num_ack_invocation);
+                       exit(EXIT_FAILURE);
+               }
+
+               if (fds[0].revents & POLLIN) {
+                       saLogDispatch(logHandle, SA_DISPATCH_ONE);
+                       if (g_ack_invocation < invocation - 49 ||
+                           g_ack_invocation > invocation) {
+                               fprintf(stderr, "Got ack for wrong invocation"
+                                       " (exp: [%lld - %lld], got: %lld)\n",
+                                       invocation - 9, invocation, 
g_ack_invocation);
+                               return NULL;
+                       }
+                       num_ack_invocation++;
+                       if (num_ack_invocation == 49) break;
+               }
+               osaf_clock_gettime(CLOCK_MONOTONIC, &end);
+               passed_second = subtract(&end, &start);
+       }
+       return NULL;
+}
+
+//>
+// TC#8: This test case verifies if the LOG agent triggers write callbacks
+// for lost log record when cluster goes to headless. The test must run
+// on PL node and headless mode is enabled. Report PASS if the test runs on SC.
+//
+// Write 50 consecutive records then shutdown both SCs. It is expected
+// to get OK for first write request, and try-again for lost records.
+//<
+void write_50_records_then_cluster_goes_to_headless()
+{
+       if (is_test_done_on_pl() == false) {
+               fprintf(stderr, "This test must run on PL node\n");
+               test_validate(SA_AIS_OK, SA_AIS_OK);
+               return;
+       }
+
+       pthread_t thread;
+
+       // 1) Enable log resilient
+       configure_cache_capacity(100);
+       // 2) Set up log client before writing a log record
+       setup();
+       // 3) Write a log record and wait for getting an ack.
+       writesync();
+       // 4) Verify if getting OK for first invocation
+       if (g_ack_ais_code != SA_AIS_OK) {
+               fprintf(stderr, "Got wrong ack code for first invocation\n");
+               test_validate(g_ack_ais_code, SA_AIS_OK);
+               return;
+       }
+
+       printf("-->Shutdown standby node, then enter any key to continue...\n");
+       getchar();
+
+       struct timespec start;
+       osaf_clock_gettime(CLOCK_MONOTONIC, &start);
+
+       // 5) Write 49 more records. This time, log server will be suspended 
16s.
+       for (int i = 1; i < 50; i++) writeasync();
+       sleep(1);
+
+       int rc = pthread_create(&thread, NULL,
+                               &polling_and_counting_acks, &start);
+       assert(rc == 0 && "Failed to create polling thread");
+
+       printf("-->Shutdown active node, then enter any key to continue...\n");
+       getchar();
+
+       pthread_join(thread, NULL);
+
+       if (num_ack_invocation != 49) {
+               fprintf(stderr, "Got wrong number of acks. Exp: 49, got: %d\n",
+                       num_ack_invocation);
+               rc_validate(num_ack_invocation, 49);
+               return;
+       }
+
+       test_validate(SA_AIS_OK, SA_AIS_OK);
+}
+
+#endif
+
+__attribute__((constructor)) static void saLibraryLifeCycle_constructor(void)
+{
+       test_suite_add(20, "Test log resilience feature");
+       test_case_add(20, config_logResilienceTimeout_1,
+                     "Set a valid value to logResilienceTimeout");
+       test_case_add(20, config_logResilienceTimeout_2,
+                     "Set a invalid value to logResilienceTimeout");
+       test_case_add(20, config_logMaxPendingWriteRequests_1,
+                     "Set a valid value to logMaxPendingWriteRequests");
+       test_case_add(20, config_logMaxPendingWriteRequests_2,
+                     "Set a invalid value to logMaxPendingWriteRequests");
+#ifdef SIMULATE_NFS_UNRESPONSE
+       // initial setup - getting unique invocation for every test.
+       g_invocation = time(0);
+       invocation = g_invocation;
+
+       test_case_add(20, write_two_records_while_nfs_hung,
+                     "Write 02 consecutive records, then get nfs hung on the 
second");
+       test_case_add(20, write_04_records_then_queue_is_full,
+                     "Write 04 consecutive records, and the queue is full on 
the 3rd");
+       test_case_add(20, write_10_records_then_trigger_switchover,
+                     "Write 10 consecutive records, then trigger switchover");
+#endif
+}
+
+#ifdef SIMULATE_NFS_UNRESPONSE
+// This suite requires manual shutting down both SCs, therefore put the test
+// into 'extended' tests. Only run with option -e.
+void add_suite_21()
+{
+       test_suite_add(21, "Test if LGA notifies lost records to client");
+       test_case_add(21, write_50_records_then_cluster_goes_to_headless,
+                     "Write 50 records, queuing 49 records then cluster goes 
to headless");
+}
+#endif
-- 
2.17.1



_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel

Reply via email to