[ https://issues.apache.org/jira/browse/KUDU-3397?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
shenxingwuying updated KUDU-3397: --------------------------------- Description: I have added a SchedulerThread function, as Jira https://issues.apache.org/jira/browse/KUDU-3364 said, some periodic tasks can be refactor using SchedulerThread. There are 2 benefits: # Use a thread pool(very few threads 1 or 2 is ok) with SchedulerThread to replace all the independent, isolated Threads. less threads. # The root reason of refactor. By the refactor, some tasks can support manual trigger, more maintainable, This is the root of refactor. ## auto-rebalancer ## leader-auto-rebalancer(doing at: ) ## compaction jobs The latent Thread can be refactor may exist below. They should be analysed more details. {code:java} //代码占位符 ag Thread::Create src/kudu/ | grep -v test | grep -v src/kudu/util/threadpool.cc src/kudu/clock/builtin_ntp.cc:651: Thread::Create("ntp", "ntp client", [this]() { this->PollThread(); }, &thread_), src/kudu/hms/hms_catalog.cc:132: RETURN_NOT_OK(Thread::Create("hms_catalog", "fetch_uuid", src/kudu/master/hms_notification_log_listener.cc:98: return kudu::Thread::Create("catalog manager", "hms-notification-log-listener", src/kudu/master/master.cc:452: return Thread::Create("master", src/kudu/master/catalog_manager.cc:787: RETURN_NOT_OK(kudu::Thread::Create("catalog manager", "bgtasks", src/kudu/rpc/acceptor_pool.cc:96: Status s = kudu::Thread::Create("acceptor pool", "acceptor", src/kudu/rpc/reactor.cc:193: return kudu::Thread::Create("reactor", "rpc reactor", src/kudu/rpc/reactor.cc:660:Status ReactorThread::CreateClientSocket(int family, Socket* sock) { src/kudu/rpc/result_tracker.cc:462: CHECK_OK(Thread::Create("server", "result-tracker", src/kudu/rpc/service_pool.cc:93: CHECK_OK(kudu::Thread::Create( src/kudu/security/init.cc:318: RETURN_NOT_OK(Thread::Create("kerberos", "reacquire thread", src/kudu/server/diagnostics_log.cc:144: Status s = Thread::Create("server", "diag-logger", src/kudu/server/server_base.cc:837: return Thread::Create("server", "excess-log-deleter", src/kudu/server/server_base.cc:885: return Thread::Create("server", "tcmalloc-memory-gc", src/kudu/subprocess/server.cc:130: RETURN_NOT_OK(Thread::Create("subprocess", "start", src/kudu/subprocess/server.cc:153: RETURN_NOT_OK(Thread::Create("subprocess", "responder", src/kudu/subprocess/server.cc:157: RETURN_NOT_OK(Thread::Create("subprocess", "reader", src/kudu/subprocess/server.cc:160: RETURN_NOT_OK(Thread::Create("subprocess", "writer", src/kudu/subprocess/server.cc:163: return Thread::Create("subprocess", "deadline-checker", src/kudu/tools/tool_action_local_replica.cc:265: RETURN_NOT_OK(Thread::Create("tool-tablet-copy", "check-progress", src/kudu/tserver/heartbeater.cc:669: return kudu::Thread::Create("heartbeater", "heartbeat", src/kudu/tserver/scanners.cc:118: RETURN_NOT_OK(Thread::Create("scanners", "removal_thread", src/kudu/tserver/tablet_copy_service.cc:114: CHECK_OK(Thread::Create("tablet-copy", "tc-session-exp", src/kudu/util/cloud/instance_detector.cc:76: RETURN_NOT_OK(Thread::Create( src/kudu/util/debug/trace_event_impl.cc:1364: Status s = Thread::CreateWithFlags( src/kudu/util/file_cache.cc:504: return Thread::Create("cache", Substitute("$0-evict", cache_name_), src/kudu/util/kernel_stack_watchdog.cc:71: CHECK_OK(Thread::CreateWithFlags( src/kudu/util/maintenance_manager.cc:209: return Thread::Create("maintenance", "maintenance_scheduler", src/kudu/util/minidump.cc:288: return Thread::Create("minidump", "sigusr1-handler", src/kudu/util/pstack_watcher.cc:53: CHECK_OK(Thread::Create("pstack_watcher", "pstack_watcher", src/kudu/util/thread.h:118: // Flags passed to Thread::CreateWithFlags(). src/kudu/util/ttl_cache.h:139: CHECK_OK(Thread::Create( {code} The direct purpose, the threadpool can unified some seperator threads, eg: master's 'auto-rebalancer' (the Thread), 'expired-reserved-tables-deleter', 'hms-notification-log-listener' ("catalog manager", "bgtasks"), and later's leader-rebalancer's Thread, and some other threads. As tserver has some others thread The first one : [https://gerrit.cloudera.org/c/18679/] was: I have added a SchedulerThread function, as Jira https://issues.apache.org/jira/browse/KUDU-3364 said, some periodic tasks can be refactor using SchedulerThread. There are 2 benefits: # Use a thread pool(very few threads 1 or 2 is ok) with SchedulerThread to replace all the independent, isolated Threads. less threads. # By the refactor, some tasks can support manual trigger, more maintainable. ## auto-rebalancer ## leader-auto-rebalancer(doing at: ) ## compaction jobs The latent Thread can be refactor may exist below. They should be analysed more details. {code:java} //代码占位符 ag Thread::Create src/kudu/ | grep -v test | grep -v src/kudu/util/threadpool.cc src/kudu/clock/builtin_ntp.cc:651: Thread::Create("ntp", "ntp client", [this]() { this->PollThread(); }, &thread_), src/kudu/hms/hms_catalog.cc:132: RETURN_NOT_OK(Thread::Create("hms_catalog", "fetch_uuid", src/kudu/master/hms_notification_log_listener.cc:98: return kudu::Thread::Create("catalog manager", "hms-notification-log-listener", src/kudu/master/master.cc:452: return Thread::Create("master", src/kudu/master/catalog_manager.cc:787: RETURN_NOT_OK(kudu::Thread::Create("catalog manager", "bgtasks", src/kudu/rpc/acceptor_pool.cc:96: Status s = kudu::Thread::Create("acceptor pool", "acceptor", src/kudu/rpc/reactor.cc:193: return kudu::Thread::Create("reactor", "rpc reactor", src/kudu/rpc/reactor.cc:660:Status ReactorThread::CreateClientSocket(int family, Socket* sock) { src/kudu/rpc/result_tracker.cc:462: CHECK_OK(Thread::Create("server", "result-tracker", src/kudu/rpc/service_pool.cc:93: CHECK_OK(kudu::Thread::Create( src/kudu/security/init.cc:318: RETURN_NOT_OK(Thread::Create("kerberos", "reacquire thread", src/kudu/server/diagnostics_log.cc:144: Status s = Thread::Create("server", "diag-logger", src/kudu/server/server_base.cc:837: return Thread::Create("server", "excess-log-deleter", src/kudu/server/server_base.cc:885: return Thread::Create("server", "tcmalloc-memory-gc", src/kudu/subprocess/server.cc:130: RETURN_NOT_OK(Thread::Create("subprocess", "start", src/kudu/subprocess/server.cc:153: RETURN_NOT_OK(Thread::Create("subprocess", "responder", src/kudu/subprocess/server.cc:157: RETURN_NOT_OK(Thread::Create("subprocess", "reader", src/kudu/subprocess/server.cc:160: RETURN_NOT_OK(Thread::Create("subprocess", "writer", src/kudu/subprocess/server.cc:163: return Thread::Create("subprocess", "deadline-checker", src/kudu/tools/tool_action_local_replica.cc:265: RETURN_NOT_OK(Thread::Create("tool-tablet-copy", "check-progress", src/kudu/tserver/heartbeater.cc:669: return kudu::Thread::Create("heartbeater", "heartbeat", src/kudu/tserver/scanners.cc:118: RETURN_NOT_OK(Thread::Create("scanners", "removal_thread", src/kudu/tserver/tablet_copy_service.cc:114: CHECK_OK(Thread::Create("tablet-copy", "tc-session-exp", src/kudu/util/cloud/instance_detector.cc:76: RETURN_NOT_OK(Thread::Create( src/kudu/util/debug/trace_event_impl.cc:1364: Status s = Thread::CreateWithFlags( src/kudu/util/file_cache.cc:504: return Thread::Create("cache", Substitute("$0-evict", cache_name_), src/kudu/util/kernel_stack_watchdog.cc:71: CHECK_OK(Thread::CreateWithFlags( src/kudu/util/maintenance_manager.cc:209: return Thread::Create("maintenance", "maintenance_scheduler", src/kudu/util/minidump.cc:288: return Thread::Create("minidump", "sigusr1-handler", src/kudu/util/pstack_watcher.cc:53: CHECK_OK(Thread::Create("pstack_watcher", "pstack_watcher", src/kudu/util/thread.h:118: // Flags passed to Thread::CreateWithFlags(). src/kudu/util/ttl_cache.h:139: CHECK_OK(Thread::Create( {code} The direct purpose, the threadpool can unified some seperator threads, eg: master's 'auto-rebalancer' (the Thread), 'expired-reserved-tables-deleter', 'hms-notification-log-listener' ("catalog manager", "bgtasks"), and later's leader-rebalancer's Thread, and some other threads. As tserver has some others thread The first one : [https://gerrit.cloudera.org/c/18679/] > do refactor for some periodic tasks using SchedulerThread > --------------------------------------------------------- > > Key: KUDU-3397 > URL: https://issues.apache.org/jira/browse/KUDU-3397 > Project: Kudu > Issue Type: Improvement > Components: master, tserver > Reporter: shenxingwuying > Priority: Minor > > I have added a SchedulerThread function, as Jira > https://issues.apache.org/jira/browse/KUDU-3364 said, > some periodic tasks can be refactor using SchedulerThread. > > There are 2 benefits: > # Use a thread pool(very few threads 1 or 2 is ok) with SchedulerThread to > replace all the independent, isolated Threads. less threads. > # The root reason of refactor. By the refactor, some tasks can support > manual trigger, more maintainable, This is the root of refactor. > ## auto-rebalancer > ## leader-auto-rebalancer(doing at: ) > ## compaction jobs > > The latent Thread can be refactor may exist below. > They should be analysed more details. > {code:java} > //代码占位符 > ag Thread::Create src/kudu/ | grep -v test | grep -v > src/kudu/util/threadpool.cc > src/kudu/clock/builtin_ntp.cc:651: Thread::Create("ntp", "ntp client", > [this]() { this->PollThread(); }, &thread_), > src/kudu/hms/hms_catalog.cc:132: RETURN_NOT_OK(Thread::Create("hms_catalog", > "fetch_uuid", > src/kudu/master/hms_notification_log_listener.cc:98: return > kudu::Thread::Create("catalog manager", "hms-notification-log-listener", > src/kudu/master/master.cc:452: return Thread::Create("master", > src/kudu/master/catalog_manager.cc:787: > RETURN_NOT_OK(kudu::Thread::Create("catalog manager", "bgtasks", > src/kudu/rpc/acceptor_pool.cc:96: Status s = > kudu::Thread::Create("acceptor pool", "acceptor", > src/kudu/rpc/reactor.cc:193: return kudu::Thread::Create("reactor", "rpc > reactor", > src/kudu/rpc/reactor.cc:660:Status ReactorThread::CreateClientSocket(int > family, Socket* sock) { > src/kudu/rpc/result_tracker.cc:462: CHECK_OK(Thread::Create("server", > "result-tracker", > src/kudu/rpc/service_pool.cc:93: CHECK_OK(kudu::Thread::Create( > src/kudu/security/init.cc:318: RETURN_NOT_OK(Thread::Create("kerberos", > "reacquire thread", > src/kudu/server/diagnostics_log.cc:144: Status s = Thread::Create("server", > "diag-logger", > src/kudu/server/server_base.cc:837: return Thread::Create("server", > "excess-log-deleter", > src/kudu/server/server_base.cc:885: return Thread::Create("server", > "tcmalloc-memory-gc", > src/kudu/subprocess/server.cc:130: > RETURN_NOT_OK(Thread::Create("subprocess", "start", > src/kudu/subprocess/server.cc:153: > RETURN_NOT_OK(Thread::Create("subprocess", "responder", > src/kudu/subprocess/server.cc:157: > RETURN_NOT_OK(Thread::Create("subprocess", "reader", > src/kudu/subprocess/server.cc:160: > RETURN_NOT_OK(Thread::Create("subprocess", "writer", > src/kudu/subprocess/server.cc:163: return Thread::Create("subprocess", > "deadline-checker", > src/kudu/tools/tool_action_local_replica.cc:265: > RETURN_NOT_OK(Thread::Create("tool-tablet-copy", "check-progress", > src/kudu/tserver/heartbeater.cc:669: return > kudu::Thread::Create("heartbeater", "heartbeat", > src/kudu/tserver/scanners.cc:118: RETURN_NOT_OK(Thread::Create("scanners", > "removal_thread", > src/kudu/tserver/tablet_copy_service.cc:114: > CHECK_OK(Thread::Create("tablet-copy", "tc-session-exp", > src/kudu/util/cloud/instance_detector.cc:76: RETURN_NOT_OK(Thread::Create( > src/kudu/util/debug/trace_event_impl.cc:1364: Status s = > Thread::CreateWithFlags( > src/kudu/util/file_cache.cc:504: return Thread::Create("cache", > Substitute("$0-evict", cache_name_), > src/kudu/util/kernel_stack_watchdog.cc:71: CHECK_OK(Thread::CreateWithFlags( > src/kudu/util/maintenance_manager.cc:209: return > Thread::Create("maintenance", "maintenance_scheduler", > src/kudu/util/minidump.cc:288: return Thread::Create("minidump", > "sigusr1-handler", > src/kudu/util/pstack_watcher.cc:53: > CHECK_OK(Thread::Create("pstack_watcher", "pstack_watcher", > src/kudu/util/thread.h:118: // Flags passed to Thread::CreateWithFlags(). > src/kudu/util/ttl_cache.h:139: CHECK_OK(Thread::Create( {code} > > The direct purpose, the threadpool can unified some seperator threads, eg: > master's 'auto-rebalancer' (the Thread), 'expired-reserved-tables-deleter', > 'hms-notification-log-listener' ("catalog manager", "bgtasks"), and later's > leader-rebalancer's Thread, and some other threads. As tserver has some > others thread > > > The first one : [https://gerrit.cloudera.org/c/18679/] > -- This message was sent by Atlassian Jira (v8.20.10#820010)