This is an automated email from the ASF dual-hosted git repository. cwylie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push: new cbdac49 Web console - add enable/disable actions for middle manager workers (#7642) cbdac49 is described below commit cbdac49ab393c68f8f17d6cdcc604897d8843f37 Author: Bartosz Ćugowski <blugow...@users.noreply.github.com> AuthorDate: Fri May 24 01:47:23 2019 +0200 Web console - add enable/disable actions for middle manager workers (#7642) * Overlord console - add enable/disable button for remote workers. * Overlord console - add proxy for remote workers API. * WorkerResourceTest - revert newline change. * Remote worker proxy tests - remove empty line. * Refactor remote worker proxy for readability and security * Rename method in remote task runner tests for readability * Remove enable/disable button for remote workers from old web console * Add enable/disable actions for middle manager worker in new web console * Fix variable type * Add worker task runner query adapter * Fix web console tests: segments-view, servers-view * Fix overlord resource tests --- .../druid/indexing/overlord/WorkerTaskRunner.java | 6 + .../overlord/WorkerTaskRunnerQueryAdapter.java | 133 +++++++ .../indexing/overlord/http/OverlordResource.java | 48 ++- .../overlord/WorkerTaskRunnerQueryAdpaterTest.java | 197 ++++++++++ .../overlord/http/OverlordResourceTest.java | 403 +++++++++++++++++---- .../druid/indexing/overlord/http/OverlordTest.java | 5 +- web-console/README.md | 1 + .../__snapshots__/segments-view.spec.tsx.snap | 317 ++++------------ .../src/views/segments-view/segments-view.spec.tsx | 8 +- .../__snapshots__/servers-view.spec.tsx.snap | 41 +++ .../src/views/servers-view/servers-view.tsx | 238 ++++++++---- 11 files changed, 1018 insertions(+), 379 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/WorkerTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/WorkerTaskRunner.java index c0620f8..6520de2 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/WorkerTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/WorkerTaskRunner.java @@ -30,6 +30,12 @@ import java.util.Collection; @PublicApi public interface WorkerTaskRunner extends TaskRunner { + enum ActionType + { + ENABLE, + DISABLE + } + /** * List of known workers who can accept tasks for running */ diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdapter.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdapter.java new file mode 100644 index 0000000..7656761 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdapter.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord; + +import com.google.common.base.Optional; +import com.google.common.base.Throwables; +import com.google.common.collect.Iterables; +import io.netty.handler.timeout.TimeoutException; +import org.apache.druid.guice.annotations.EscalatedGlobal; +import org.apache.druid.indexing.overlord.hrtr.HttpRemoteTaskRunner; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.java.util.http.client.response.StatusResponseHandler; +import org.apache.druid.java.util.http.client.response.StatusResponseHolder; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +import javax.inject.Inject; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutionException; + +public class WorkerTaskRunnerQueryAdapter +{ + private static final EmittingLogger log = new EmittingLogger(HttpRemoteTaskRunner.class); + + private final TaskMaster taskMaster; + private final HttpClient httpClient; + + @Inject + public WorkerTaskRunnerQueryAdapter(TaskMaster taskMaster, @EscalatedGlobal final HttpClient httpClient) + { + this.taskMaster = taskMaster; + this.httpClient = httpClient; + } + + public void enableWorker(String host) + { + sendRequestToWorker(host, WorkerTaskRunner.ActionType.ENABLE); + } + + public void disableWorker(String host) + { + sendRequestToWorker(host, WorkerTaskRunner.ActionType.DISABLE); + } + + private void sendRequestToWorker(String workerHost, WorkerTaskRunner.ActionType action) + { + WorkerTaskRunner workerTaskRunner = getWorkerTaskRunner(); + + if (workerTaskRunner == null) { + throw new RE("Task Runner does not support enable/disable worker actions"); + } + + Optional<ImmutableWorkerInfo> workerInfo = Iterables.tryFind( + workerTaskRunner.getWorkers(), + entry -> entry.getWorker() + .getHost() + .equals(workerHost) + ); + + if (!workerInfo.isPresent()) { + throw new RE( + "Worker on host %s does not exists", + workerHost + ); + } + + String actionName = WorkerTaskRunner.ActionType.ENABLE.equals(action) ? "enable" : "disable"; + final URL workerUrl = TaskRunnerUtils.makeWorkerURL( + workerInfo.get().getWorker(), + "/druid/worker/v1/%s", + actionName + ); + + try { + final StatusResponseHolder response = httpClient.go( + new Request(HttpMethod.POST, workerUrl), + new StatusResponseHandler(StandardCharsets.UTF_8) + ).get(); + + log.info( + "Sent %s action request to worker: %s, status: %s, response: %s", + action, + workerHost, + response.getStatus(), + response.getContent() + ); + + if (!HttpResponseStatus.OK.equals(response.getStatus())) { + throw new RE( + "Action [%s] failed for worker [%s] with status %s(%s)", + action, + workerHost, + response.getStatus().getCode(), + response.getStatus().getReasonPhrase() + ); + } + } + catch (ExecutionException | InterruptedException | TimeoutException e) { + Throwables.propagate(e); + } + } + + private WorkerTaskRunner getWorkerTaskRunner() + { + Optional<TaskRunner> taskRunnerOpt = taskMaster.getTaskRunner(); + if (taskRunnerOpt.isPresent() && taskRunnerOpt.get() instanceof WorkerTaskRunner) { + return (WorkerTaskRunner) taskRunnerOpt.get(); + } else { + return null; + } + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index e5abac4..0efb5a5 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -50,6 +50,7 @@ import org.apache.druid.indexing.overlord.TaskRunner; import org.apache.druid.indexing.overlord.TaskRunnerWorkItem; import org.apache.druid.indexing.overlord.TaskStorageQueryAdapter; import org.apache.druid.indexing.overlord.WorkerTaskRunner; +import org.apache.druid.indexing.overlord.WorkerTaskRunnerQueryAdapter; import org.apache.druid.indexing.overlord.autoscaling.ScalingStats; import org.apache.druid.indexing.overlord.http.security.TaskResourceFilter; import org.apache.druid.indexing.overlord.setup.WorkerBehaviorConfig; @@ -104,6 +105,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; /** + * */ @Path("/druid/indexer/v1") public class OverlordResource @@ -117,6 +119,7 @@ public class OverlordResource private final JacksonConfigManager configManager; private final AuditManager auditManager; private final AuthorizerMapper authorizerMapper; + private final WorkerTaskRunnerQueryAdapter workerTaskRunnerQueryAdapter; private AtomicReference<WorkerBehaviorConfig> workerConfigRef = null; private static final List API_TASK_STATES = ImmutableList.of("pending", "waiting", "running", "complete"); @@ -129,7 +132,8 @@ public class OverlordResource TaskLogStreamer taskLogStreamer, JacksonConfigManager configManager, AuditManager auditManager, - AuthorizerMapper authorizerMapper + AuthorizerMapper authorizerMapper, + WorkerTaskRunnerQueryAdapter workerTaskRunnerQueryAdapter ) { this.taskMaster = taskMaster; @@ -139,6 +143,7 @@ public class OverlordResource this.configManager = configManager; this.auditManager = auditManager; this.authorizerMapper = authorizerMapper; + this.workerTaskRunnerQueryAdapter = workerTaskRunnerQueryAdapter; } @POST @@ -726,6 +731,47 @@ public class OverlordResource ); } + @POST + @Path("/worker/{host}/enable") + @Produces(MediaType.APPLICATION_JSON) + @ResourceFilters(StateResourceFilter.class) + public Response enableWorker(@PathParam("host") final String host) + { + return changeWorkerStatus(host, WorkerTaskRunner.ActionType.ENABLE); + } + + @POST + @Path("/worker/{host}/disable") + @Produces(MediaType.APPLICATION_JSON) + @ResourceFilters(StateResourceFilter.class) + public Response disableWorker(@PathParam("host") final String host) + { + return changeWorkerStatus(host, WorkerTaskRunner.ActionType.DISABLE); + } + + private Response changeWorkerStatus(String host, WorkerTaskRunner.ActionType action) + { + try { + if (WorkerTaskRunner.ActionType.DISABLE.equals(action)) { + workerTaskRunnerQueryAdapter.disableWorker(host); + return Response.ok(ImmutableMap.of(host, "disabled")).build(); + } else if (WorkerTaskRunner.ActionType.ENABLE.equals(action)) { + workerTaskRunnerQueryAdapter.enableWorker(host); + return Response.ok(ImmutableMap.of(host, "enabled")).build(); + } else { + return Response.serverError() + .entity(ImmutableMap.of("error", "Worker does not support " + action + " action!")) + .build(); + } + } + catch (Exception e) { + log.error(e, "Error in posting [%s] action to [%s]", action, host); + return Response.serverError() + .entity(ImmutableMap.of("error", e.getMessage())) + .build(); + } + } + @GET @Path("/scaling") @Produces(MediaType.APPLICATION_JSON) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java new file mode 100644 index 0000000..330ff2c --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.SettableFuture; +import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.java.util.http.client.response.HttpResponseHandler; +import org.apache.druid.java.util.http.client.response.StatusResponseHolder; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.net.URL; + + +public class WorkerTaskRunnerQueryAdpaterTest +{ + private WorkerTaskRunnerQueryAdapter workerTaskRunnerQueryAdapter; + private HttpClient httpClient; + private WorkerTaskRunner workerTaskRunner; + private TaskMaster taskMaster; + + @Before + public void setup() + { + httpClient = EasyMock.createNiceMock(HttpClient.class); + workerTaskRunner = EasyMock.createMock(WorkerTaskRunner.class); + taskMaster = EasyMock.createStrictMock(TaskMaster.class); + + workerTaskRunnerQueryAdapter = new WorkerTaskRunnerQueryAdapter(taskMaster, httpClient); + + EasyMock.expect(taskMaster.getTaskRunner()).andReturn( + Optional.of(workerTaskRunner) + ).once(); + + EasyMock.expect(workerTaskRunner.getWorkers()).andReturn( + ImmutableList.of( + new ImmutableWorkerInfo( + new Worker( + "http", "worker-host1", "192.0.0.1", 10, "v1" + ), + 2, + ImmutableSet.of("grp1", "grp2"), + ImmutableSet.of("task1", "task2"), + DateTimes.of("2015-01-01T01:01:01Z") + ), + new ImmutableWorkerInfo( + new Worker( + "https", "worker-host2", "192.0.0.2", 4, "v1" + ), + 1, + ImmutableSet.of("grp1"), + ImmutableSet.of("task1"), + DateTimes.of("2015-01-01T01:01:01Z") + ) + ) + ).once(); + } + + @After + public void tearDown() + { + EasyMock.verify(workerTaskRunner, taskMaster, httpClient); + } + + @Test + public void testDisableWorker() throws Exception + { + final URL workerUrl = new URL("http://worker-host1/druid/worker/v1/disable"); + final String workerResponse = "{\"worker-host1\":\"disabled\"}"; + Capture<Request> capturedRequest = getHttpClientRequestCapture(HttpResponseStatus.OK, workerResponse); + + EasyMock.replay(workerTaskRunner, taskMaster, httpClient); + + workerTaskRunnerQueryAdapter.disableWorker("worker-host1"); + + Assert.assertEquals(HttpMethod.POST, capturedRequest.getValue().getMethod()); + Assert.assertEquals(workerUrl, capturedRequest.getValue().getUrl()); + } + + @Test + public void testDisableWorkerWhenWorkerRaisesError() throws Exception + { + final URL workerUrl = new URL("http://worker-host1/druid/worker/v1/disable"); + Capture<Request> capturedRequest = getHttpClientRequestCapture(HttpResponseStatus.INTERNAL_SERVER_ERROR, ""); + + EasyMock.replay(workerTaskRunner, taskMaster, httpClient); + + try { + workerTaskRunnerQueryAdapter.disableWorker("worker-host1"); + Assert.fail("Should raise RE exception!"); + } + catch (RE re) { + } + + Assert.assertEquals(HttpMethod.POST, capturedRequest.getValue().getMethod()); + Assert.assertEquals(workerUrl, capturedRequest.getValue().getUrl()); + } + + @Test(expected = RE.class) + public void testDisableWorkerWhenWorkerNotExists() + { + EasyMock.replay(workerTaskRunner, taskMaster, httpClient); + + workerTaskRunnerQueryAdapter.disableWorker("not-existing-worker"); + } + + @Test + public void testEnableWorker() throws Exception + { + final URL workerUrl = new URL("https://worker-host2/druid/worker/v1/enable"); + final String workerResponse = "{\"worker-host2\":\"enabled\"}"; + Capture<Request> capturedRequest = getHttpClientRequestCapture(HttpResponseStatus.OK, workerResponse); + + EasyMock.replay(workerTaskRunner, taskMaster, httpClient); + + workerTaskRunnerQueryAdapter.enableWorker("worker-host2"); + + Assert.assertEquals(HttpMethod.POST, capturedRequest.getValue().getMethod()); + Assert.assertEquals(workerUrl, capturedRequest.getValue().getUrl()); + } + + @Test + public void testEnableWorkerWhenWorkerRaisesError() throws Exception + { + final URL workerUrl = new URL("https://worker-host2/druid/worker/v1/enable"); + Capture<Request> capturedRequest = getHttpClientRequestCapture(HttpResponseStatus.INTERNAL_SERVER_ERROR, ""); + + EasyMock.replay(workerTaskRunner, taskMaster, httpClient); + + try { + workerTaskRunnerQueryAdapter.enableWorker("worker-host2"); + Assert.fail("Should raise RE exception!"); + } + catch (RE re) { + } + + Assert.assertEquals(HttpMethod.POST, capturedRequest.getValue().getMethod()); + Assert.assertEquals(workerUrl, capturedRequest.getValue().getUrl()); + } + + @Test(expected = RE.class) + public void testEnableWorkerWhenWorkerNotExists() + { + EasyMock.replay(workerTaskRunner, taskMaster, httpClient); + + workerTaskRunnerQueryAdapter.enableWorker("not-existing-worker"); + } + + private Capture<Request> getHttpClientRequestCapture(HttpResponseStatus httpStatus, String responseContent) + { + SettableFuture<StatusResponseHolder> futureResult = SettableFuture.create(); + futureResult.set( + new StatusResponseHolder(httpStatus, new StringBuilder(responseContent)) + ); + Capture<Request> capturedRequest = EasyMock.newCapture(); + EasyMock.expect( + httpClient.go( + EasyMock.capture(capturedRequest), + EasyMock.<HttpResponseHandler>anyObject() + ) + ) + .andReturn(futureResult) + .once(); + + return capturedRequest; + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java index 89824b0..3a58c0c 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java @@ -40,7 +40,9 @@ import org.apache.druid.indexing.overlord.TaskQueue; import org.apache.druid.indexing.overlord.TaskRunner; import org.apache.druid.indexing.overlord.TaskRunnerWorkItem; import org.apache.druid.indexing.overlord.TaskStorageQueryAdapter; +import org.apache.druid.indexing.overlord.WorkerTaskRunnerQueryAdapter; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.RE; import org.apache.druid.segment.TestHelper; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; @@ -51,6 +53,7 @@ import org.apache.druid.server.security.AuthorizerMapper; import org.apache.druid.server.security.ForbiddenException; import org.apache.druid.server.security.Resource; import org.easymock.EasyMock; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.Interval; @@ -79,6 +82,7 @@ public class OverlordResourceTest private IndexerMetadataStorageAdapter indexerMetadataStorageAdapter; private HttpServletRequest req; private TaskRunner taskRunner; + private WorkerTaskRunnerQueryAdapter workerTaskRunnerQueryAdapter; @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -91,6 +95,7 @@ public class OverlordResourceTest taskStorageQueryAdapter = EasyMock.createStrictMock(TaskStorageQueryAdapter.class); indexerMetadataStorageAdapter = EasyMock.createStrictMock(IndexerMetadataStorageAdapter.class); req = EasyMock.createStrictMock(HttpServletRequest.class); + workerTaskRunnerQueryAdapter = EasyMock.createStrictMock(WorkerTaskRunnerQueryAdapter.class); EasyMock.expect(taskMaster.getTaskRunner()).andReturn( Optional.of(taskRunner) @@ -124,21 +129,36 @@ public class OverlordResourceTest null, null, null, - authMapper + authMapper, + workerTaskRunnerQueryAdapter ); } @After public void tearDown() { - EasyMock.verify(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.verify( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); } @Test public void testLeader() { EasyMock.expect(taskMaster.getCurrentLeader()).andReturn("boz").once(); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); final Response response = overlordResource.getLeader(); Assert.assertEquals("boz", response.getEntity()); @@ -150,7 +170,14 @@ public class OverlordResourceTest { EasyMock.expect(taskMaster.isLeader()).andReturn(true).once(); EasyMock.expect(taskMaster.isLeader()).andReturn(false).once(); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); // true final Response response1 = overlordResource.isLeader(); @@ -207,7 +234,14 @@ public class OverlordResourceTest ) ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); List<TaskStatusPlus> responseObjects = (List<TaskStatusPlus>) overlordResource.getWaitingTasks(req) .getEntity(); @@ -224,7 +258,8 @@ public class OverlordResourceTest ImmutableList.of( new MockTaskRunnerWorkItem(tasksIds.get(0), null), new MockTaskRunnerWorkItem(tasksIds.get(1), null), - new MockTaskRunnerWorkItem(tasksIds.get(2), null))); + new MockTaskRunnerWorkItem(tasksIds.get(2), null) + )); EasyMock.expect(taskStorageQueryAdapter.getCompletedTaskInfoByCreatedTimeDuration(null, null, null)).andStubReturn( ImmutableList.of( @@ -251,11 +286,18 @@ public class OverlordResourceTest ) ) ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); Assert.assertTrue(taskStorageQueryAdapter.getCompletedTaskInfoByCreatedTimeDuration(null, null, null).size() == 3); Assert.assertTrue(taskRunner.getRunningTasks().size() == 3); List<TaskStatusPlus> responseObjects = (List) overlordResource - .getCompleteTasks(null, req).getEntity(); + .getCompleteTasks(null, req).getEntity(); Assert.assertEquals(2, responseObjects.size()); Assert.assertEquals(tasksIds.get(1), responseObjects.get(0).getId()); @@ -292,7 +334,14 @@ public class OverlordResourceTest ) ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); List<TaskStatusPlus> responseObjects = (List) overlordResource.getRunningTasks(null, req) .getEntity(); @@ -384,7 +433,14 @@ public class OverlordResourceTest ) ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); List<TaskStatusPlus> responseObjects = (List<TaskStatusPlus>) overlordResource .getTasks(null, null, null, null, null, req) .getEntity(); @@ -396,31 +452,32 @@ public class OverlordResourceTest { expectAuthorizationTokenCheck(); //completed tasks - EasyMock.expect(taskStorageQueryAdapter.getCompletedTaskInfoByCreatedTimeDuration(null, null, "allow")).andStubReturn( - ImmutableList.of( - new TaskInfo( - "id_5", - DateTime.now(ISOChronology.getInstanceUTC()), - TaskStatus.success("id_5"), - "allow", - getTaskWithIdAndDatasource("id_5", "allow") - ), - new TaskInfo( - "id_6", - DateTime.now(ISOChronology.getInstanceUTC()), - TaskStatus.success("id_6"), - "allow", - getTaskWithIdAndDatasource("id_6", "allow") - ), - new TaskInfo( - "id_7", - DateTime.now(ISOChronology.getInstanceUTC()), - TaskStatus.success("id_7"), - "allow", - getTaskWithIdAndDatasource("id_7", "allow") + EasyMock.expect(taskStorageQueryAdapter.getCompletedTaskInfoByCreatedTimeDuration(null, null, "allow")) + .andStubReturn( + ImmutableList.of( + new TaskInfo( + "id_5", + DateTime.now(ISOChronology.getInstanceUTC()), + TaskStatus.success("id_5"), + "allow", + getTaskWithIdAndDatasource("id_5", "allow") + ), + new TaskInfo( + "id_6", + DateTime.now(ISOChronology.getInstanceUTC()), + TaskStatus.success("id_6"), + "allow", + getTaskWithIdAndDatasource("id_6", "allow") + ), + new TaskInfo( + "id_7", + DateTime.now(ISOChronology.getInstanceUTC()), + TaskStatus.success("id_7"), + "allow", + getTaskWithIdAndDatasource("id_7", "allow") + ) ) - ) - ); + ); //active tasks EasyMock.expect(taskStorageQueryAdapter.getActiveTaskInfo("allow")).andStubReturn( ImmutableList.of( @@ -471,7 +528,14 @@ public class OverlordResourceTest new MockTaskRunnerWorkItem("id_1", null) ) ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); List<TaskStatusPlus> responseObjects = (List<TaskStatusPlus>) overlordResource .getTasks(null, "allow", null, null, null, req) @@ -526,7 +590,14 @@ public class OverlordResourceTest ) ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); List<TaskStatusPlus> responseObjects = (List<TaskStatusPlus>) overlordResource .getTasks( "waiting", @@ -586,7 +657,14 @@ public class OverlordResourceTest ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); List<TaskStatusPlus> responseObjects = (List) overlordResource .getTasks("running", "allow", null, null, null, req) @@ -644,7 +722,14 @@ public class OverlordResourceTest ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); List<TaskStatusPlus> responseObjects = (List<TaskStatusPlus>) overlordResource .getTasks("pending", null, null, null, null, req) @@ -685,7 +770,14 @@ public class OverlordResourceTest ) ) ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); List<TaskStatusPlus> responseObjects = (List<TaskStatusPlus>) overlordResource .getTasks("complete", null, null, null, null, req) .getEntity(); @@ -700,33 +792,41 @@ public class OverlordResourceTest expectAuthorizationTokenCheck(); List<String> tasksIds = ImmutableList.of("id_1", "id_2", "id_3"); Duration duration = new Period("PT86400S").toStandardDuration(); - EasyMock.expect(taskStorageQueryAdapter.getCompletedTaskInfoByCreatedTimeDuration(null, duration, null)).andStubReturn( - ImmutableList.of( - new TaskInfo( - "id_1", - DateTime.now(ISOChronology.getInstanceUTC()), - TaskStatus.success("id_1"), - "deny", - getTaskWithIdAndDatasource("id_1", "deny") - ), - new TaskInfo( - "id_2", - DateTime.now(ISOChronology.getInstanceUTC()), - TaskStatus.success("id_2"), - "allow", - getTaskWithIdAndDatasource("id_2", "allow") - ), - new TaskInfo( - "id_3", - DateTime.now(ISOChronology.getInstanceUTC()), - TaskStatus.success("id_3"), - "allow", - getTaskWithIdAndDatasource("id_3", "allow") + EasyMock.expect(taskStorageQueryAdapter.getCompletedTaskInfoByCreatedTimeDuration(null, duration, null)) + .andStubReturn( + ImmutableList.of( + new TaskInfo( + "id_1", + DateTime.now(ISOChronology.getInstanceUTC()), + TaskStatus.success("id_1"), + "deny", + getTaskWithIdAndDatasource("id_1", "deny") + ), + new TaskInfo( + "id_2", + DateTime.now(ISOChronology.getInstanceUTC()), + TaskStatus.success("id_2"), + "allow", + getTaskWithIdAndDatasource("id_2", "allow") + ), + new TaskInfo( + "id_3", + DateTime.now(ISOChronology.getInstanceUTC()), + TaskStatus.success("id_3"), + "allow", + getTaskWithIdAndDatasource("id_3", "allow") + ) ) - ) - ); + ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); String interval = "2010-01-01_P1D"; List<TaskStatusPlus> responseObjects = (List<TaskStatusPlus>) overlordResource .getTasks("complete", null, interval, null, null, req) @@ -765,7 +865,14 @@ public class OverlordResourceTest ) ) ); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); List<TaskStatusPlus> responseObjects = (List<TaskStatusPlus>) overlordResource .getTasks("complete", null, null, null, null, req) .getEntity(); @@ -779,7 +886,14 @@ public class OverlordResourceTest @Test public void testGetTasksNegativeState() { - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); Object responseObject = overlordResource .getTasks("blah", "ds_test", null, null, null, req) .getEntity(); @@ -795,7 +909,14 @@ public class OverlordResourceTest expectedException.expect(ForbiddenException.class); expectAuthorizationTokenCheck(); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); Task task = NoopTask.create(); overlordResource.taskPost(task, req); } @@ -815,7 +936,14 @@ public class OverlordResourceTest ) .andReturn(2); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); final Map<String, Integer> response = (Map<String, Integer>) overlordResource .killPendingSegments("allow", new Interval(DateTimes.MIN, DateTimes.nowUtc()).toString(), req) @@ -837,7 +965,14 @@ public class OverlordResourceTest EasyMock.expect(taskStorageQueryAdapter.getTask("othertask")) .andReturn(Optional.absent()); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); final Response response1 = overlordResource.getTaskPayload("mytask"); final TaskPayloadResponse taskPayloadResponse1 = TestHelper.makeJsonMapper().readValue( @@ -873,7 +1008,14 @@ public class OverlordResourceTest EasyMock.<Collection<? extends TaskRunnerWorkItem>>expect(taskRunner.getKnownTasks()) .andReturn(ImmutableList.of()); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); final Response response1 = overlordResource.getTaskStatus("mytask"); final TaskStatusResponse taskStatusResponse1 = TestHelper.makeJsonMapper().readValue( @@ -926,7 +1068,15 @@ public class OverlordResourceTest mockQueue.shutdown("id_1", "Shutdown request from user"); EasyMock.expectLastCall(); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req, mockQueue); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + mockQueue, + workerTaskRunnerQueryAdapter + ); final Map<String, Integer> response = (Map<String, Integer>) overlordResource .doShutdown("id_1") @@ -969,7 +1119,15 @@ public class OverlordResourceTest mockQueue.shutdown("id_2", "Shutdown request from user"); EasyMock.expectLastCall(); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req, mockQueue); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + mockQueue, + workerTaskRunnerQueryAdapter + ); final Map<String, Integer> response = (Map<String, Integer>) overlordResource .shutdownTasksForDataSource("datasource") @@ -984,12 +1142,111 @@ public class OverlordResourceTest EasyMock.expect(taskMaster.isLeader()).andReturn(true).anyTimes(); EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes(); EasyMock.expect(taskStorageQueryAdapter.getActiveTaskInfo(EasyMock.anyString())).andReturn(Collections.emptyList()); - EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req); + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); final Response response = overlordResource.shutdownTasksForDataSource("notExisting"); Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus()); } + @Test + public void testEnableWorker() + { + final String host = "worker-host"; + + workerTaskRunnerQueryAdapter.enableWorker(host); + EasyMock.expectLastCall().once(); + + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); + + final Response response = overlordResource.enableWorker(host); + + Assert.assertEquals(HttpResponseStatus.OK.getCode(), response.getStatus()); + Assert.assertEquals(ImmutableMap.of(host, "enabled"), response.getEntity()); + } + + @Test + public void testDisableWorker() + { + final String host = "worker-host"; + + workerTaskRunnerQueryAdapter.disableWorker(host); + EasyMock.expectLastCall().once(); + + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); + + final Response response = overlordResource.disableWorker(host); + + Assert.assertEquals(HttpResponseStatus.OK.getCode(), response.getStatus()); + Assert.assertEquals(ImmutableMap.of(host, "disabled"), response.getEntity()); + } + + @Test + public void testEnableWorkerWhenWorkerAPIRaisesError() + { + final String host = "worker-host"; + + workerTaskRunnerQueryAdapter.enableWorker(host); + EasyMock.expectLastCall().andThrow(new RE("Worker API returns error!")).once(); + + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); + + final Response response = overlordResource.enableWorker(host); + + Assert.assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR.getCode(), response.getStatus()); + Assert.assertEquals(ImmutableMap.of("error", "Worker API returns error!"), response.getEntity()); + } + + @Test + public void testDisableWorkerWhenWorkerAPIRaisesError() + { + final String host = "worker-host"; + + workerTaskRunnerQueryAdapter.disableWorker(host); + EasyMock.expectLastCall().andThrow(new RE("Worker API returns error!")).once(); + + EasyMock.replay( + taskRunner, + taskMaster, + taskStorageQueryAdapter, + indexerMetadataStorageAdapter, + req, + workerTaskRunnerQueryAdapter + ); + + final Response response = overlordResource.disableWorker(host); + + Assert.assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR.getCode(), response.getStatus()); + Assert.assertEquals(ImmutableMap.of("error", "Worker API returns error!"), response.getEntity()); + } + private void expectAuthorizationTokenCheck() { AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null, null); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java index 6b34ea4..2503014 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordTest.java @@ -52,6 +52,7 @@ import org.apache.druid.indexing.overlord.TaskRunnerListener; import org.apache.druid.indexing.overlord.TaskRunnerWorkItem; import org.apache.druid.indexing.overlord.TaskStorage; import org.apache.druid.indexing.overlord.TaskStorageQueryAdapter; +import org.apache.druid.indexing.overlord.WorkerTaskRunnerQueryAdapter; import org.apache.druid.indexing.overlord.autoscaling.ScalingStats; import org.apache.druid.indexing.overlord.config.TaskQueueConfig; import org.apache.druid.indexing.overlord.helpers.OverlordHelperManager; @@ -212,6 +213,7 @@ public class OverlordTest Assert.assertEquals(taskMaster.getCurrentLeader(), druidNode.getHostAndPort()); final TaskStorageQueryAdapter taskStorageQueryAdapter = new TaskStorageQueryAdapter(taskStorage); + final WorkerTaskRunnerQueryAdapter workerTaskRunnerQueryAdapter = new WorkerTaskRunnerQueryAdapter(taskMaster, null); // Test Overlord resource stuff overlordResource = new OverlordResource( taskMaster, @@ -220,7 +222,8 @@ public class OverlordTest null, null, null, - AuthTestUtils.TEST_AUTHORIZER_MAPPER + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + workerTaskRunnerQueryAdapter ); Response response = overlordResource.getLeader(); Assert.assertEquals(druidNode.getHostAndPort(), response.getEntity()); diff --git a/web-console/README.md b/web-console/README.md index ad02c10..8ea53e8 100644 --- a/web-console/README.md +++ b/web-console/README.md @@ -55,6 +55,7 @@ Generated/copied dynamically ``` GET /status GET /druid/indexer/v1/supervisor?full +POST /druid/indexer/v1/worker GET /druid/indexer/v1/workers GET /druid/coordinator/v1/loadqueue?simple GET /druid/coordinator/v1/config diff --git a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap index 339d639..ee2f486 100644 --- a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap +++ b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap @@ -2,10 +2,10 @@ exports[`describe segments-view segments view snapshot 1`] = ` <div - className="servers-view app-view" + className="segments-view app-view" > <ViewControlBar - label="Historicals" + label="Segments" > <Blueprint3.Button icon="refresh" @@ -13,26 +13,27 @@ exports[`describe segments-view segments view snapshot 1`] = ` text="Refresh" /> <Blueprint3.Button + hidden={false} icon="application" onClick={[Function]} text="Go to SQL" /> - <Blueprint3.Switch - checked={false} - label="Group by tier" - onChange={[Function]} - /> <TableColumnSelection columns={ Array [ - "Server", - "Tier", - "Curr size", - "Max size", - "Usage", - "Load/drop queues", - "Host", - "Port", + "Segment ID", + "Datasource", + "Start", + "End", + "Version", + "Partition", + "Size", + "Num rows", + "Replicas", + "Is published", + "Is realtime", + "Is available", + "Is overshadowed", ] } onChange={[Function]} @@ -49,6 +50,7 @@ exports[`describe segments-view segments view snapshot 1`] = ` PaginationComponent={[Function]} PivotValueComponent={[Function]} ResizerComponent={[Function]} + SubComponent={[Function]} TableComponent={[Function]} TbodyComponent={[Function]} TdComponent={[Function]} @@ -95,267 +97,97 @@ exports[`describe segments-view segments view snapshot 1`] = ` columns={ Array [ Object { - "Aggregated": [Function], - "Header": "Server", - "accessor": "server", + "Header": "Segment ID", + "accessor": "segment_id", "show": true, "width": 300, }, Object { "Cell": [Function], - "Header": "Tier", - "accessor": "tier", + "Header": "Datasource", + "accessor": "datasource", "show": true, }, Object { - "Aggregated": [Function], "Cell": [Function], - "Header": "Curr size", - "accessor": "curr_size", - "filterable": false, - "id": "curr_size", + "Header": "Start", + "accessor": "start", + "defaultSortDesc": true, "show": true, - "width": 100, + "width": 120, }, Object { - "Aggregated": [Function], "Cell": [Function], - "Header": "Max size", - "accessor": "max_size", - "filterable": false, - "id": "max_size", + "Header": "End", + "accessor": "end", + "defaultSortDesc": true, "show": true, - "width": 100, + "width": 120, }, Object { - "Aggregated": [Function], - "Cell": [Function], - "Header": "Usage", - "accessor": [Function], - "filterable": false, - "id": "usage", + "Header": "Version", + "accessor": "version", + "defaultSortDesc": true, "show": true, - "width": 100, + "width": 120, }, Object { - "Aggregated": [Function], - "Cell": [Function], - "Header": "Load/drop queues", - "accessor": [Function], + "Header": "Partition", + "accessor": "partition_num", "filterable": false, - "id": "queue", "show": true, - "width": 400, + "width": 60, }, Object { - "Aggregated": [Function], - "Header": "Host", - "accessor": "host", + "Cell": [Function], + "Header": "Size", + "accessor": "size", + "defaultSortDesc": true, + "filterable": false, "show": true, }, Object { - "Aggregated": [Function], - "Header": "Port", - "accessor": [Function], - "id": "port", + "Cell": [Function], + "Header": "Num rows", + "accessor": "num_rows", + "defaultSortDesc": true, + "filterable": false, "show": true, }, - ] - } - data={Array []} - defaultExpanded={Object {}} - defaultFilterMethod={[Function]} - defaultFiltered={Array []} - defaultPageSize={10} - defaultResized={Array []} - defaultSortDesc={false} - defaultSortMethod={[Function]} - defaultSorted={Array []} - expanderDefaults={ - Object { - "filterable": false, - "resizable": false, - "sortable": false, - "width": 35, - } - } - filterable={true} - filtered={Array []} - freezeWhenExpanded={false} - getLoadingProps={[Function]} - getNoDataProps={[Function]} - getPaginationProps={[Function]} - getProps={[Function]} - getResizerProps={[Function]} - getTableProps={[Function]} - getTbodyProps={[Function]} - getTdProps={[Function]} - getTfootProps={[Function]} - getTfootTdProps={[Function]} - getTfootTrProps={[Function]} - getTheadFilterProps={[Function]} - getTheadFilterThProps={[Function]} - getTheadFilterTrProps={[Function]} - getTheadGroupProps={[Function]} - getTheadGroupThProps={[Function]} - getTheadGroupTrProps={[Function]} - getTheadProps={[Function]} - getTheadThProps={[Function]} - getTheadTrProps={[Function]} - getTrGroupProps={[Function]} - getTrProps={[Function]} - groupedByPivotKey="_groupedByPivot" - indexKey="_index" - loading={true} - loadingText="Loading..." - multiSort={true} - nestingLevelKey="_nestingLevel" - nextText="Next" - noDataText="" - ofText="of" - onFetchData={[Function]} - onFilteredChange={[Function]} - originalKey="_original" - pageSizeOptions={ - Array [ - 5, - 10, - 20, - 25, - 50, - 100, - ] - } - pageText="Page" - pivotBy={Array []} - pivotDefaults={Object {}} - pivotIDKey="_pivotID" - pivotValKey="_pivotVal" - previousText="Previous" - resizable={true} - resolveData={[Function]} - rowsText="rows" - showPageJump={true} - showPageSizeOptions={true} - showPagination={true} - showPaginationBottom={true} - showPaginationTop={false} - sortable={true} - style={Object {}} - subRowsKey="_subRows" - /> - <div - className="control-separator" - /> - <ViewControlBar - label="MiddleManagers" - > - <Blueprint3.Button - icon="refresh" - onClick={[Function]} - text="Refresh" - /> - <TableColumnSelection - columns={ - Array [ - "Host", - "Usage", - "Availability groups", - "Last completed task time", - "Blacklisted until", - ] - } - onChange={[Function]} - tableColumnsHidden={Array []} - /> - </ViewControlBar> - <ReactTable - AggregatedComponent={[Function]} - ExpanderComponent={[Function]} - FilterComponent={[Function]} - LoadingComponent={[Function]} - NoDataComponent={[Function]} - PadRowComponent={[Function]} - PaginationComponent={[Function]} - PivotValueComponent={[Function]} - ResizerComponent={[Function]} - SubComponent={[Function]} - TableComponent={[Function]} - TbodyComponent={[Function]} - TdComponent={[Function]} - TfootComponent={[Function]} - ThComponent={[Function]} - TheadComponent={[Function]} - TrComponent={[Function]} - TrGroupComponent={[Function]} - aggregatedKey="_aggregated" - className="-striped -highlight" - collapseOnDataChange={true} - collapseOnPageChange={true} - collapseOnSortingChange={true} - column={ - Object { - "Aggregated": undefined, - "Cell": undefined, - "Expander": undefined, - "Filter": undefined, - "Footer": undefined, - "Header": undefined, - "Pivot": undefined, - "PivotValue": undefined, - "aggregate": undefined, - "className": "", - "filterAll": false, - "filterMethod": undefined, - "filterable": undefined, - "footerClassName": "", - "footerStyle": Object {}, - "getFooterProps": [Function], - "getHeaderProps": [Function], - "getProps": [Function], - "headerClassName": "", - "headerStyle": Object {}, - "minWidth": 100, - "resizable": undefined, - "show": true, - "sortMethod": undefined, - "sortable": undefined, - "style": Object {}, - } - } - columns={ - Array [ Object { - "Cell": [Function], - "Header": "Host", - "accessor": [Function], - "id": "host", + "Header": "Replicas", + "accessor": "num_replicas", + "defaultSortDesc": true, + "filterable": false, "show": true, + "width": 60, }, Object { - "Header": "Usage", + "Filter": [Function], + "Header": "Is published", "accessor": [Function], - "filterable": false, - "id": "usage", + "id": "is_published", "show": true, - "width": 60, }, Object { - "Header": "Availability groups", + "Filter": [Function], + "Header": "Is realtime", "accessor": [Function], - "filterable": false, - "id": "availabilityGroups", + "id": "is_realtime", "show": true, - "width": 60, }, Object { - "Header": "Last completed task time", - "accessor": "lastCompletedTaskTime", + "Filter": [Function], + "Header": "Is available", + "accessor": [Function], + "id": "is_available", "show": true, }, Object { - "Header": "Blacklisted until", - "accessor": "blacklistedUntil", + "Filter": [Function], + "Header": "Is overshadowed", + "accessor": [Function], + "id": "is_overshadowed", "show": true, }, ] @@ -364,11 +196,18 @@ exports[`describe segments-view segments view snapshot 1`] = ` defaultExpanded={Object {}} defaultFilterMethod={[Function]} defaultFiltered={Array []} - defaultPageSize={10} + defaultPageSize={50} defaultResized={Array []} defaultSortDesc={false} defaultSortMethod={[Function]} - defaultSorted={Array []} + defaultSorted={ + Array [ + Object { + "desc": true, + "id": "start", + }, + ] + } expanderDefaults={ Object { "filterable": false, @@ -381,7 +220,7 @@ exports[`describe segments-view segments view snapshot 1`] = ` filtered={ Array [ Object { - "id": "host", + "id": "datasource", "value": "test", }, ] @@ -413,11 +252,12 @@ exports[`describe segments-view segments view snapshot 1`] = ` indexKey="_index" loading={true} loadingText="Loading..." + manual={true} multiSort={true} nestingLevelKey="_nestingLevel" nextText="Next" noDataText="" - ofText="of" + ofText="" onFetchData={[Function]} onFilteredChange={[Function]} originalKey="_original" @@ -432,6 +272,7 @@ exports[`describe segments-view segments view snapshot 1`] = ` ] } pageText="Page" + pages={10000000} pivotDefaults={Object {}} pivotIDKey="_pivotID" pivotValKey="_pivotVal" @@ -439,7 +280,7 @@ exports[`describe segments-view segments view snapshot 1`] = ` resizable={true} resolveData={[Function]} rowsText="rows" - showPageJump={true} + showPageJump={false} showPageSizeOptions={true} showPagination={true} showPaginationBottom={true} diff --git a/web-console/src/views/segments-view/segments-view.spec.tsx b/web-console/src/views/segments-view/segments-view.spec.tsx index c1dbb74..7b4531f 100644 --- a/web-console/src/views/segments-view/segments-view.spec.tsx +++ b/web-console/src/views/segments-view/segments-view.spec.tsx @@ -22,16 +22,16 @@ import { shallow } from 'enzyme'; import * as enzymeAdapterReact16 from 'enzyme-adapter-react-16'; import * as React from 'react'; -import {ServersView} from '../servers-view/servers-view'; +import {SegmentsView} from '../segments-view/segments-view'; Enzyme.configure({ adapter: new enzymeAdapterReact16() }); describe('describe segments-view', () => { it('segments view snapshot', () => { const segmentsView = shallow( - <ServersView - middleManager={'test'} + <SegmentsView + datasource={'test'} + onlyUnavailable={false} goToSql={(initSql: string) => {}} - goToTask={(taskId: string) => {}} noSqlMode={false} />); expect(segmentsView).toMatchSnapshot(); diff --git a/web-console/src/views/servers-view/__snapshots__/servers-view.spec.tsx.snap b/web-console/src/views/servers-view/__snapshots__/servers-view.spec.tsx.snap index a68a29e..72048f4 100644 --- a/web-console/src/views/servers-view/__snapshots__/servers-view.spec.tsx.snap +++ b/web-console/src/views/servers-view/__snapshots__/servers-view.spec.tsx.snap @@ -263,6 +263,8 @@ exports[`describe servers view action servers view 1`] = ` "Availability groups", "Last completed task time", "Blacklisted until", + "Status", + "Actions", ] } onChange={[Function]} @@ -358,6 +360,21 @@ exports[`describe servers view action servers view 1`] = ` "accessor": "blacklistedUntil", "show": true, }, + Object { + "Header": "Status", + "accessor": [Function], + "id": "status", + "show": true, + }, + Object { + "Cell": [Function], + "Header": "Actions", + "accessor": [Function], + "filterable": false, + "id": "actions", + "show": true, + "width": 70, + }, ] } data={Array []} @@ -448,5 +465,29 @@ exports[`describe servers view action servers view 1`] = ` style={Object {}} subRowsKey="_subRows" /> + <AsyncActionDialog + action={null} + confirmButtonText="Disable worker" + failText="Could not disable worker" + intent="danger" + onClose={[Function]} + successText="Worker has been disabled" + > + <p> + Are you sure you want to disable worker 'null'? + </p> + </AsyncActionDialog> + <AsyncActionDialog + action={null} + confirmButtonText="Enable worker" + failText="Could not enable worker" + intent="primary" + onClose={[Function]} + successText="Worker has been enabled" + > + <p> + Are you sure you want to enable worker 'null'? + </p> + </AsyncActionDialog> </div> `; diff --git a/web-console/src/views/servers-view/servers-view.tsx b/web-console/src/views/servers-view/servers-view.tsx index 7d1021f..e2c2bdb 100644 --- a/web-console/src/views/servers-view/servers-view.tsx +++ b/web-console/src/views/servers-view/servers-view.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import { Button, Switch } from '@blueprintjs/core'; +import { Button, Icon, Intent, Popover, Position, Switch } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import axios from 'axios'; import { sum } from 'd3-array'; @@ -24,7 +24,12 @@ import * as React from 'react'; import ReactTable from 'react-table'; import { Filter } from 'react-table'; -import { TableColumnSelection, ViewControlBar } from '../../components/index'; +import { + ActionCell, + TableColumnSelection, + ViewControlBar +} from '../../components/index'; +import { AsyncActionDialog } from '../../dialogs/index'; import { addFilter, formatBytes, @@ -32,11 +37,12 @@ import { queryDruidSql, QueryManager, TableColumnSelectionHandler } from '../../utils'; +import { BasicAction, basicActionsToMenu } from '../../utils/basic-action'; import './servers-view.scss'; const serverTableColumns: string[] = ['Server', 'Tier', 'Curr size', 'Max size', 'Usage', 'Load/drop queues', 'Host', 'Port']; -const middleManagerTableColumns: string[] = ['Host', 'Usage', 'Availability groups', 'Last completed task time', 'Blacklisted until']; +const middleManagerTableColumns: string[] = ['Host', 'Usage', 'Availability groups', 'Last completed task time', 'Blacklisted until', 'Status', 'Actions']; function formatQueues(segmentsToLoad: number, segmentsToLoadSize: number, segmentsToDrop: number, segmentsToDropSize: number): string { const queueParts: string[] = []; @@ -67,6 +73,9 @@ export interface ServersViewState { middleManagers: any[] | null; middleManagersError: string | null; middleManagerFilter: Filter[]; + + middleManagerDisableWorkerHost: string | null; + middleManagerEnableWorkerHost: string | null; } interface ServerQueryResultRow { @@ -110,7 +119,10 @@ export class ServersView extends React.Component<ServersViewProps, ServersViewSt middleManagersLoading: true, middleManagers: null, middleManagersError: null, - middleManagerFilter: props.middleManager ? [{ id: 'host', value: props.middleManager }] : [] + middleManagerFilter: props.middleManager ? [{ id: 'host', value: props.middleManager }] : [], + + middleManagerDisableWorkerHost: null, + middleManagerEnableWorkerHost: null }; this.serverTableColumnSelectionHandler = new TableColumnSelectionHandler( @@ -345,69 +357,171 @@ WHERE "server_type" = 'historical'`); const { middleManagers, middleManagersLoading, middleManagersError, middleManagerFilter } = this.state; const { middleManagerTableColumnSelectionHandler } = this; - return <ReactTable - data={middleManagers || []} - loading={middleManagersLoading} - noDataText={!middleManagersLoading && middleManagers && !middleManagers.length ? 'No MiddleManagers' : (middleManagersError || '')} - filterable - filtered={middleManagerFilter} - onFilteredChange={(filtered, column) => { - this.setState({ middleManagerFilter: filtered }); - }} - columns={[ - { - Header: 'Host', - id: 'host', - accessor: (row) => row.worker.host, - Cell: row => { - const value = row.value; - return <a onClick={() => { this.setState({ middleManagerFilter: addFilter(middleManagerFilter, 'host', value) }); }}>{value}</a>; + return <> + <ReactTable + data={middleManagers || []} + loading={middleManagersLoading} + noDataText={!middleManagersLoading && middleManagers && !middleManagers.length ? 'No MiddleManagers' : (middleManagersError || '')} + filterable + filtered={middleManagerFilter} + onFilteredChange={(filtered, column) => { + this.setState({ middleManagerFilter: filtered }); + }} + columns={[ + { + Header: 'Host', + id: 'host', + accessor: (row) => row.worker.host, + Cell: row => { + const value = row.value; + return <a onClick={() => { this.setState({ middleManagerFilter: addFilter(middleManagerFilter, 'host', value) }); }}>{value}</a>; + }, + show: middleManagerTableColumnSelectionHandler.showColumn('Host') }, - show: middleManagerTableColumnSelectionHandler.showColumn('Host') - }, - { - Header: 'Usage', - id: 'usage', - width: 60, - accessor: (row) => `${row.currCapacityUsed} / ${row.worker.capacity}`, - filterable: false, - show: middleManagerTableColumnSelectionHandler.showColumn('Usage') - }, - { - Header: 'Availability groups', - id: 'availabilityGroups', - width: 60, - accessor: (row) => row.availabilityGroups.length, - filterable: false, - show: middleManagerTableColumnSelectionHandler.showColumn('Availability groups') - }, + { + Header: 'Usage', + id: 'usage', + width: 60, + accessor: (row) => `${row.currCapacityUsed} / ${row.worker.capacity}`, + filterable: false, + show: middleManagerTableColumnSelectionHandler.showColumn('Usage') + }, + { + Header: 'Availability groups', + id: 'availabilityGroups', + width: 60, + accessor: (row) => row.availabilityGroups.length, + filterable: false, + show: middleManagerTableColumnSelectionHandler.showColumn('Availability groups') + }, + { + Header: 'Last completed task time', + accessor: 'lastCompletedTaskTime', + show: middleManagerTableColumnSelectionHandler.showColumn('Last completed task time') + }, + { + Header: 'Blacklisted until', + accessor: 'blacklistedUntil', + show: middleManagerTableColumnSelectionHandler.showColumn('Blacklisted until') + }, + { + Header: 'Status', + id: 'status', + accessor: (row) => row.worker.version === '' ? 'Disabled' : 'Enabled', + show: middleManagerTableColumnSelectionHandler.showColumn('Status') + }, + { + Header: 'Actions', + id: 'actions', + width: 70, + accessor: (row) => row.worker, + filterable: false, + Cell: row => { + const disabled = row.value.version === ''; + const workerActions = this.getWorkerActions(row.value.host, disabled); + const workerMenu = basicActionsToMenu(workerActions); + + return <ActionCell> + { + workerMenu && + <Popover content={workerMenu} position={Position.BOTTOM_RIGHT}> + <Icon icon={IconNames.WRENCH}/> + </Popover> + } + </ActionCell>; + }, + show: middleManagerTableColumnSelectionHandler.showColumn('Actions') + } + ]} + defaultPageSize={10} + className="-striped -highlight" + SubComponent={rowInfo => { + const runningTasks = rowInfo.original.runningTasks; + return <div style={{ padding: '20px' }}> + { + runningTasks.length ? + <> + <span>Running tasks:</span> + <ul>{runningTasks.map((t: string) => <li key={t}>{t} <a onClick={() => goToTask(t)}>➚</a></li>)}</ul> + </> : + <span>No running tasks</span> + } + </div>; + }} + /> + {this.renderDisableWorkerAction()} + {this.renderEnableWorkerAction()} + </>; + } + + private getWorkerActions(workerHost: string, disabled: boolean): BasicAction[] { + if (disabled) { + return [ { - Header: 'Last completed task time', - accessor: 'lastCompletedTaskTime', - show: middleManagerTableColumnSelectionHandler.showColumn('Last completed task time') - }, + icon: IconNames.TICK, + title: 'Enable', + onAction: () => this.setState({ middleManagerEnableWorkerHost: workerHost }) + } + ]; + } else { + return [ { - Header: 'Blacklisted until', - accessor: 'blacklistedUntil', - show: middleManagerTableColumnSelectionHandler.showColumn('Blacklisted until') + icon: IconNames.DISABLE, + title: 'Disable', + onAction: () => this.setState({ middleManagerDisableWorkerHost: workerHost }) } - ]} - defaultPageSize={10} - className="-striped -highlight" - SubComponent={rowInfo => { - const runningTasks = rowInfo.original.runningTasks; - return <div style={{ padding: '20px' }}> - { - runningTasks.length ? - <> - <span>Running tasks:</span> - <ul>{runningTasks.map((t: string) => <li key={t}>{t} <a onClick={() => goToTask(t)}>➚</a></li>)}</ul> - </> : - <span>No running tasks</span> - } - </div>; + ]; + } + } + + renderDisableWorkerAction() { + const { middleManagerDisableWorkerHost } = this.state; + + return <AsyncActionDialog + action={ + middleManagerDisableWorkerHost ? async () => { + const resp = await axios.post(`/druid/indexer/v1/worker/${middleManagerDisableWorkerHost}/disable`, {}); + return resp.data; + } : null + } + confirmButtonText="Disable worker" + successText="Worker has been disabled" + failText="Could not disable worker" + intent={Intent.DANGER} + onClose={(success) => { + this.setState({ middleManagerDisableWorkerHost: null }); + if (success) this.middleManagerQueryManager.rerunLastQuery(); }} - />; + > + <p> + {`Are you sure you want to disable worker '${middleManagerDisableWorkerHost}'?`} + </p> + </AsyncActionDialog>; + } + + renderEnableWorkerAction() { + const { middleManagerEnableWorkerHost } = this.state; + + return <AsyncActionDialog + action={ + middleManagerEnableWorkerHost ? async () => { + const resp = await axios.post(`/druid/indexer/v1/worker/${middleManagerEnableWorkerHost}/enable`, {}); + return resp.data; + } : null + } + confirmButtonText="Enable worker" + successText="Worker has been enabled" + failText="Could not enable worker" + intent={Intent.PRIMARY} + onClose={(success) => { + this.setState({ middleManagerEnableWorkerHost: null }); + if (success) this.middleManagerQueryManager.rerunLastQuery(); + }} + > + <p> + {`Are you sure you want to enable worker '${middleManagerEnableWorkerHost}'?`} + </p> + </AsyncActionDialog>; } render() { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org