This is an automated email from the ASF dual-hosted git repository. jermy pushed a commit to branch server-info-to-global in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git
commit f4c0f4c160be0fc25c100609b2c034ae0e76d7c2 Author: Jermy Li <[email protected]> AuthorDate: Sun Dec 3 17:01:19 2023 +0800 enhence ServerInfoManager.heartbeat() Change-Id: I6932893c4be8331547f3b721083ca00430f85e58 --- .../apache/hugegraph/auth/HugeGraphAuthProxy.java | 8 +- .../org/apache/hugegraph/core/GraphManager.java | 14 +-- .../main/java/org/apache/hugegraph/HugeGraph.java | 4 +- .../org/apache/hugegraph/StandardHugeGraph.java | 14 +-- .../hugegraph/masterelection/GlobalMasterInfo.java | 42 ++++---- .../apache/hugegraph/task/ServerInfoManager.java | 113 +++++++++++++-------- .../hugegraph/task/StandardTaskScheduler.java | 14 +-- .../org/apache/hugegraph/task/TaskManager.java | 8 +- 8 files changed, 121 insertions(+), 96 deletions(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java index 6fa34ff18..d22b86bab 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java @@ -669,9 +669,9 @@ public final class HugeGraphAuthProxy implements HugeGraph { } @Override - public void serverStarted(GlobalMasterInfo serverInfo) { + public void serverStarted(GlobalMasterInfo nodeInfo) { this.verifyAdminPermission(); - this.hugegraph.serverStarted(serverInfo); + this.hugegraph.serverStarted(nodeInfo); } @Override @@ -776,9 +776,9 @@ public final class HugeGraphAuthProxy implements HugeGraph { } @Override - public void create(String configPath, GlobalMasterInfo serverInfo) { + public void create(String configPath, GlobalMasterInfo nodeInfo) { this.verifyPermission(HugePermission.WRITE, ResourceType.STATUS); - this.hugegraph.create(configPath, serverInfo); + this.hugegraph.create(configPath, nodeInfo); } @Override diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java index 5ca7d0a0f..b8770ca7d 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java @@ -455,23 +455,23 @@ public final class GraphManager { } private void serverStarted(HugeConfig config) { - String serverId = config.get(ServerOptions.SERVER_ID); + String id = config.get(ServerOptions.SERVER_ID); String role = config.get(ServerOptions.SERVER_ROLE); - E.checkArgument(StringUtils.isNotEmpty(serverId), + E.checkArgument(StringUtils.isNotEmpty(id), "The server name can't be null or empty"); E.checkArgument(StringUtils.isNotEmpty(role), "The server role can't be null or empty"); - NodeRole serverRole = NodeRole.valueOf(role.toUpperCase()); - boolean supportRoleElection = !serverRole.computer() && + NodeRole nodeRole = NodeRole.valueOf(role.toUpperCase()); + boolean supportRoleElection = !nodeRole.computer() && this.supportRoleElection(); if (supportRoleElection) { // Init any server as Worker role, then do role election - serverRole = NodeRole.WORKER; + nodeRole = NodeRole.WORKER; } - this.globalNodeRoleInfo.serverId(IdGenerator.of(serverId)); - this.globalNodeRoleInfo.initServerRole(serverRole); + this.globalNodeRoleInfo.initNodeId(IdGenerator.of(id)); + this.globalNodeRoleInfo.initNodeRole(nodeRole); for (String graph : this.graphs()) { HugeGraph hugegraph = this.graph(graph); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraph.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraph.java index 420f8bd4d..85c093e59 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraph.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraph.java @@ -201,7 +201,7 @@ public interface HugeGraph extends Graph { void waitReady(RpcServer rpcServer); - void serverStarted(GlobalMasterInfo serverInfo); + void serverStarted(GlobalMasterInfo nodeInfo); boolean started(); @@ -221,7 +221,7 @@ public interface HugeGraph extends Graph { void resumeSnapshot(); - void create(String configPath, GlobalMasterInfo serverInfo); + void create(String configPath, GlobalMasterInfo nodeInfo); void drop(); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java index df3de8671..c671056e0 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java @@ -267,15 +267,15 @@ public class StandardHugeGraph implements HugeGraph { } @Override - public void serverStarted(GlobalMasterInfo serverInfo) { + public void serverStarted(GlobalMasterInfo nodeInfo) { LOG.info("Init system info for graph '{}'", this.name); this.initSystemInfo(); LOG.info("Init server info [{}-{}] for graph '{}'...", - serverInfo.serverId(), serverInfo.serverRole(), this.name); - this.serverInfoManager().initServerInfo(serverInfo); + nodeInfo.nodeId(), nodeInfo.nodeRole(), this.name); + this.serverInfoManager().initServerInfo(nodeInfo); - this.initRoleStateMachine(serverInfo.serverId()); + this.initRoleStateMachine(nodeInfo.nodeId()); // TODO: check necessary? LOG.info("Check olap property-key tables for graph '{}'", this.name); @@ -401,7 +401,7 @@ public class StandardHugeGraph implements HugeGraph { try { this.storeProvider.truncate(); // TODO: remove this after serverinfo saved in etcd - this.serverStarted(this.serverInfoManager().serverInfo()); + this.serverStarted(this.serverInfoManager().globalNodeRoleInfo()); } finally { LockUtil.unlock(this.name, LockUtil.GRAPH_LOCK); } @@ -975,9 +975,9 @@ public class StandardHugeGraph implements HugeGraph { } @Override - public void create(String configPath, GlobalMasterInfo serverInfo) { + public void create(String configPath, GlobalMasterInfo nodeInfo) { this.initBackend(); - this.serverStarted(serverInfo); + this.serverStarted(nodeInfo); // Write config to disk file String confPath = ConfigUtil.writeToFile(configPath, this.name(), diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/masterelection/GlobalMasterInfo.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/masterelection/GlobalMasterInfo.java index 709b1b75d..9124fcd0b 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/masterelection/GlobalMasterInfo.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/masterelection/GlobalMasterInfo.java @@ -30,8 +30,8 @@ public final class GlobalMasterInfo { private volatile boolean supportElection; private volatile NodeInfo masterNodeInfo; - private volatile Id serverId; - private volatile NodeRole serverRole; + private volatile Id nodeId; + private volatile NodeRole nodeRole; public GlobalMasterInfo() { this(NO_MASTER); @@ -41,8 +41,8 @@ public final class GlobalMasterInfo { this.supportElection = false; this.masterNodeInfo = masterInfo; - this.serverId = null; - this.serverRole = null; + this.nodeId = null; + this.nodeRole = null; } public void supportElection(boolean featureSupport) { @@ -66,36 +66,36 @@ public final class GlobalMasterInfo { return this.masterNodeInfo; } - public Id serverId() { - return this.serverId; + public Id nodeId() { + return this.nodeId; } - public NodeRole serverRole() { - return this.serverRole; + public NodeRole nodeRole() { + return this.nodeRole; } - public void serverId(Id id) { - this.serverId = id; + public void initNodeId(Id id) { + this.nodeId = id; } - public void initServerRole(NodeRole role) { + public void initNodeRole(NodeRole role) { E.checkArgument(role != null, "The server role can't be null"); - E.checkArgument(this.serverRole == null, + E.checkArgument(this.nodeRole == null, "The server role can't be init twice"); - this.serverRole = role; + this.nodeRole = role; } - public void changeServerRole(NodeRole role) { + public void changeNodeRole(NodeRole role) { E.checkArgument(role != null, "The server role can't be null"); - this.serverRole = role; + this.nodeRole = role; } - public static GlobalMasterInfo master(String serverId) { - NodeInfo masterInfo = new NodeInfo(true, serverId); - GlobalMasterInfo serverInfo = new GlobalMasterInfo(masterInfo); - serverInfo.serverId = IdGenerator.of(serverId); - serverInfo.serverRole = NodeRole.MASTER; - return serverInfo; + public static GlobalMasterInfo master(String nodeId) { + NodeInfo masterInfo = new NodeInfo(true, nodeId); + GlobalMasterInfo nodeInfo = new GlobalMasterInfo(masterInfo); + nodeInfo.nodeId = IdGenerator.of(nodeId); + nodeInfo.nodeRole = NodeRole.MASTER; + return nodeInfo; } public static class NodeInfo { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/ServerInfoManager.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/ServerInfoManager.java index edf3e4973..ee14b4ceb 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/ServerInfoManager.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/ServerInfoManager.java @@ -62,7 +62,7 @@ public class ServerInfoManager { private final HugeGraphParams graph; private final ExecutorService dbExecutor; - private GlobalMasterInfo globalServerInfo; + private volatile GlobalMasterInfo globalNodeInfo; private volatile boolean onlySingleNode; private volatile boolean closed; @@ -75,7 +75,7 @@ public class ServerInfoManager { this.graph = graph; this.dbExecutor = dbExecutor; - this.globalServerInfo = null; + this.globalNodeInfo = null; this.onlySingleNode = false; this.closed = false; @@ -85,7 +85,7 @@ public class ServerInfoManager { HugeServerInfo.schema(this.graph).initSchemaIfNeeded(); } - public boolean close() { + public synchronized boolean close() { this.closed = true; if (!this.dbExecutor.isShutdown()) { this.removeSelfServerInfo(); @@ -102,16 +102,16 @@ public class ServerInfoManager { return true; } - public synchronized void initServerInfo(GlobalMasterInfo serverInfo) { - E.checkArgument(serverInfo != null, "The global node info can't be null"); - this.globalServerInfo = serverInfo; - Id serverId = this.globalServerInfo.serverId(); + public synchronized void initServerInfo(GlobalMasterInfo nodeInfo) { + E.checkArgument(nodeInfo != null, "The global node info can't be null"); + Id serverId = nodeInfo.nodeId(); HugeServerInfo existed = this.serverInfo(serverId); E.checkArgument(existed == null || !existed.alive(), "The server with name '%s' already in cluster", serverId); - if (this.globalServerInfo.serverRole().master()) { + + if (nodeInfo.nodeRole().master()) { String page = this.supportsPaging() ? PageInfo.PAGE_NONE : null; do { Iterator<HugeServerInfo> servers = this.serverInfos(PAGE_SIZE, page); @@ -127,55 +127,80 @@ public class ServerInfoManager { } while (page != null); } - // TODO: save ServerInfo at AuthServer - this.saveServerInfo(this.selfServerId(), this.selfServerRole()); + this.globalNodeInfo = nodeInfo; + + // TODO: save ServerInfo to AuthServer + this.saveServerInfo(this.selfNodeId(), this.selfNodeRole()); } - public synchronized void changeServerRole(NodeRole serverRole) { + public synchronized void changeServerRole(NodeRole nodeRole) { if (this.closed) { return; } - this.globalServerInfo.changeServerRole(serverRole); + this.globalNodeInfo.changeNodeRole(nodeRole); - this.saveServerInfo(this.selfServerId(), this.selfServerRole()); + // TODO: save ServerInfo to AuthServer + this.saveServerInfo(this.selfNodeId(), this.selfNodeRole()); } - public GlobalMasterInfo serverInfo() { - return this.globalServerInfo; + public GlobalMasterInfo globalNodeRoleInfo() { + return this.globalNodeInfo; } - public Id selfServerId() { - if (this.globalServerInfo == null) { + public Id selfNodeId() { + if (this.globalNodeInfo == null) { return null; } - return this.globalServerInfo.serverId(); + return this.globalNodeInfo.nodeId(); } - public NodeRole selfServerRole() { - if (this.globalServerInfo == null) { + public NodeRole selfNodeRole() { + if (this.globalNodeInfo == null) { return null; } - return this.globalServerInfo.serverRole(); + return this.globalNodeInfo.nodeRole(); } - public boolean master() { - return this.selfServerRole() != null && this.selfServerRole().master(); + public boolean selfIsMaster() { + return this.selfNodeRole() != null && this.selfNodeRole().master(); } public boolean onlySingleNode() { - // Only has one master node + // Only exists one node in the whole master return this.onlySingleNode; } - public void heartbeat() { + public synchronized void heartbeat() { + assert this.graphIsReady(); + HugeServerInfo serverInfo = this.selfServerInfo(); - if (serverInfo == null && this.selfServerId() != null && - this.selfServerRole() != NodeRole.MASTER) { - serverInfo = this.saveServerInfo(this.selfServerId(), this.selfServerRole()); + if (serverInfo != null) { + // Update heartbeat time for this server + serverInfo.updateTime(DateUtil.now()); + this.save(serverInfo); + return; } - serverInfo.updateTime(DateUtil.now()); - this.save(serverInfo); + + /* ServerInfo is missing */ + if (this.selfNodeId() == null) { + // Ignore if ServerInfo is not initialized + LOG.info("ServerInfo is missing: {}, may not be initialized yet"); + return; + } + if (this.selfIsMaster()) { + // On master node, just wait for ServerInfo re-init + LOG.warn("ServerInfo is missing: {}, may be cleared before", + this.selfNodeId()); + return; + } + /* + * Missing server info on non-master node, may be caused by graph + * truncated on master node then synced by raft. + * TODO: we just patch it here currently, to be improved. + */ + serverInfo = this.saveServerInfo(this.selfNodeId(), this.selfNodeRole()); + assert serverInfo != null; } public synchronized void decreaseLoad(int load) { @@ -190,7 +215,7 @@ public class ServerInfoManager { return 10000; } - protected boolean graphReady() { + protected boolean graphIsReady() { return !this.closed && this.graph.started() && this.graph.initialized(); } @@ -244,8 +269,8 @@ public class ServerInfoManager { return this.graph.systemTransaction(); } - private HugeServerInfo saveServerInfo(Id server, NodeRole role) { - HugeServerInfo serverInfo = new HugeServerInfo(server, role); + private HugeServerInfo saveServerInfo(Id serverId, NodeRole serverRole) { + HugeServerInfo serverInfo = new HugeServerInfo(serverId, serverRole); serverInfo.maxLoad(this.calcMaxLoad()); this.save(serverInfo); @@ -312,16 +337,16 @@ public class ServerInfoManager { } private HugeServerInfo selfServerInfo() { - HugeServerInfo selfServerInfo = this.serverInfo(this.selfServerId()); - if (selfServerInfo == null) { - LOG.warn("ServerInfo is missing: {}", this.selfServerId()); + HugeServerInfo selfServerInfo = this.serverInfo(this.selfNodeId()); + if (selfServerInfo == null && this.selfNodeId() != null) { + LOG.warn("ServerInfo is missing: {}", this.selfNodeId()); } return selfServerInfo; } - private HugeServerInfo serverInfo(Id server) { + private HugeServerInfo serverInfo(Id serverId) { return this.call(() -> { - Iterator<Vertex> vertices = this.tx().queryVertices(server); + Iterator<Vertex> vertices = this.tx().queryVertices(serverId); Vertex vertex = QueryResults.one(vertices); if (vertex == null) { return null; @@ -337,19 +362,19 @@ public class ServerInfoManager { * backend store, initServerInfo() is not called in this case, so * this.selfServerId is null at this time. */ - if (this.selfServerId() != null && this.graph.initialized()) { - return this.removeServerInfo(this.selfServerId()); + if (this.selfNodeId() != null && this.graph.initialized()) { + return this.removeServerInfo(this.selfNodeId()); } return null; } - private HugeServerInfo removeServerInfo(Id server) { - if (server == null) { + private HugeServerInfo removeServerInfo(Id serverId) { + if (serverId == null) { return null; } - LOG.info("Remove server info: {}", server); + LOG.info("Remove server info: {}", serverId); return this.call(() -> { - Iterator<Vertex> vertices = this.tx().queryVertices(server); + Iterator<Vertex> vertices = this.tx().queryVertices(serverId); Vertex vertex = QueryResults.one(vertices); if (vertex == null) { return null; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java index bf0a16314..120aeb0d6 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java @@ -141,7 +141,7 @@ public class StandardTaskScheduler implements TaskScheduler { @Override public <V> void restoreTasks() { - Id selfServer = this.serverManager().selfServerId(); + Id selfServer = this.serverManager().selfNodeId(); // Restore 'RESTORING', 'RUNNING' and 'QUEUED' tasks in order. for (TaskStatus status : TaskStatus.PENDING_STATUSES) { String page = this.supportsPaging() ? PageInfo.PAGE_NONE : null; @@ -205,7 +205,7 @@ public class StandardTaskScheduler implements TaskScheduler { * this code can be removed without affecting code logic */ task.status(TaskStatus.QUEUED); - task.server(this.serverManager().selfServerId()); + task.server(this.serverManager().selfNodeId()); this.save(task); return this.submitTask(task); } else { @@ -278,8 +278,8 @@ public class StandardTaskScheduler implements TaskScheduler { // The task scheduled to workers, let the worker node to cancel this.save(task); assert task.server() != null : task; - assert this.serverManager().master(); - if (!task.server().equals(this.serverManager().selfServerId())) { + assert this.serverManager().selfIsMaster(); + if (!task.server().equals(this.serverManager().selfNodeId())) { /* * Remove task from memory if it's running on worker node, * but keep task in memory if it's running on master node. @@ -318,7 +318,7 @@ public class StandardTaskScheduler implements TaskScheduler { continue; } - if (!this.serverManager.master()) { + if (!this.serverManager.selfIsMaster()) { return; } @@ -423,7 +423,7 @@ public class StandardTaskScheduler implements TaskScheduler { protected void taskDone(HugeTask<?> task) { this.remove(task); - Id selfServerId = this.serverManager().selfServerId(); + Id selfServerId = this.serverManager().selfNodeId(); try { this.serverManager().decreaseLoad(task.load()); } catch (Throwable e) { @@ -719,7 +719,7 @@ public class StandardTaskScheduler implements TaskScheduler { } private void checkOnMasterNode(String op) { - if (!this.serverManager().master()) { + if (!this.serverManager().selfIsMaster()) { throw new HugeException("Can't %s task on non-master server", op); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/TaskManager.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/TaskManager.java index bb3af44f4..b3726d383 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/TaskManager.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/TaskManager.java @@ -324,7 +324,7 @@ public final class TaskManager { * If graph is closed, don't call serverManager.initialized() * due to it will reopen graph tx. */ - if (!serverManager.graphReady()) { + if (!serverManager.graphIsReady()) { return; } @@ -338,7 +338,7 @@ public final class TaskManager { * However, when enableRoleElected=false, a Master is only set by the * config assignment, assigned-Master always stays the same state. */ - if (serverManager.master()) { + if (serverManager.selfIsMaster()) { scheduler.scheduleTasksOnMaster(); if (!this.enableRoleElected && !serverManager.onlySingleNode()) { // assigned-Master + non-single-node don't need to execute tasks @@ -347,10 +347,10 @@ public final class TaskManager { } // Execute queued tasks scheduled to current server - scheduler.executeTasksOnWorker(serverManager.selfServerId()); + scheduler.executeTasksOnWorker(serverManager.selfNodeId()); // Cancel tasks scheduled to current server - scheduler.cancelTasksOnWorker(serverManager.selfServerId()); + scheduler.cancelTasksOnWorker(serverManager.selfNodeId()); } finally { LockUtil.unlock(graph, LockUtil.GRAPH_LOCK); }
