jihoonson commented on a change in pull request #6094: Introduce SystemSchema tables (#5989) URL: https://github.com/apache/incubator-druid/pull/6094#discussion_r223201837
########## File path: sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java ########## @@ -0,0 +1,695 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.druid.sql.calcite.schema; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.inject.Inject; +import org.apache.calcite.DataContext; +import org.apache.calcite.linq4j.DefaultEnumerable; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.ScannableTable; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.druid.client.ImmutableDruidServer; +import org.apache.druid.client.JsonParserIterator; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.client.indexing.IndexingService; +import org.apache.druid.discovery.DruidLeaderClient; +import org.apache.druid.indexer.TaskStatusPlus; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.java.util.http.client.io.AppendableByteArrayInputStream; +import org.apache.druid.java.util.http.client.response.ClientResponse; +import org.apache.druid.java.util.http.client.response.InputStreamResponseHandler; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.Action; +import org.apache.druid.server.security.AuthenticationResult; +import org.apache.druid.server.security.AuthorizationUtils; +import org.apache.druid.server.security.AuthorizerMapper; +import org.apache.druid.server.security.ForbiddenException; +import org.apache.druid.server.security.Resource; +import org.apache.druid.server.security.ResourceAction; +import org.apache.druid.server.security.ResourceType; +import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.table.RowSignature; +import org.apache.druid.timeline.DataSegment; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponse; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ExecutionException; + +public class SystemSchema extends AbstractSchema +{ + private static final Logger log = new Logger(SystemSchema.class); + + public static final String NAME = "sys"; + private static final String SEGMENTS_TABLE = "segments"; + private static final String SERVERS_TABLE = "servers"; + private static final String SERVER_SEGMENTS_TABLE = "server_segments"; + private static final String TASKS_TABLE = "tasks"; + + private static final RowSignature SEGMENTS_SIGNATURE = RowSignature + .builder() + .add("segment_id", ValueType.STRING) + .add("datasource", ValueType.STRING) + .add("start", ValueType.STRING) + .add("end", ValueType.STRING) + .add("size", ValueType.LONG) + .add("version", ValueType.STRING) + .add("partition_num", ValueType.STRING) + .add("num_replicas", ValueType.LONG) + .add("num_rows", ValueType.LONG) + .add("is_published", ValueType.LONG) + .add("is_available", ValueType.LONG) + .add("is_realtime", ValueType.LONG) + .add("payload", ValueType.STRING) + .build(); + + private static final RowSignature SERVERS_SIGNATURE = RowSignature + .builder() + .add("server", ValueType.STRING) + .add("host", ValueType.STRING) + .add("plaintext_port", ValueType.STRING) + .add("tls_port", ValueType.STRING) + .add("server_type", ValueType.STRING) + .add("tier", ValueType.STRING) + .add("curr_size", ValueType.LONG) + .add("max_size", ValueType.LONG) + .build(); + + private static final RowSignature SERVER_SEGMENTS_SIGNATURE = RowSignature + .builder() + .add("server", ValueType.STRING) + .add("segment_id", ValueType.STRING) + .build(); + + private static final RowSignature TASKS_SIGNATURE = RowSignature + .builder() + .add("task_id", ValueType.STRING) + .add("type", ValueType.STRING) + .add("datasource", ValueType.STRING) + .add("created_time", ValueType.STRING) + .add("queue_insertion_time", ValueType.STRING) + .add("status", ValueType.STRING) + .add("runner_status", ValueType.STRING) + .add("duration", ValueType.LONG) + .add("location", ValueType.STRING) + .add("host", ValueType.STRING) + .add("plaintext_port", ValueType.STRING) + .add("tls_port", ValueType.STRING) + .add("error_msg", ValueType.STRING) + .build(); + + private final Map<String, Table> tableMap; + + @Inject + public SystemSchema( + final DruidSchema druidSchema, + final TimelineServerView serverView, + final AuthorizerMapper authorizerMapper, + final @Coordinator DruidLeaderClient coordinatorDruidLeaderClient, + final @IndexingService DruidLeaderClient overlordDruidLeaderClient, + final ObjectMapper jsonMapper + ) + { + Preconditions.checkNotNull(serverView, "serverView"); + BytesAccumulatingResponseHandler responseHandler = new BytesAccumulatingResponseHandler(); + this.tableMap = ImmutableMap.of( + SEGMENTS_TABLE, new SegmentsTable(druidSchema, coordinatorDruidLeaderClient, jsonMapper, responseHandler, authorizerMapper), + SERVERS_TABLE, new ServersTable(serverView, authorizerMapper), + SERVER_SEGMENTS_TABLE, new ServerSegmentsTable(serverView, authorizerMapper), + TASKS_TABLE, new TasksTable(overlordDruidLeaderClient, jsonMapper, responseHandler, authorizerMapper) + ); + } + + @Override + public Map<String, Table> getTableMap() + { + return tableMap; + } + + static class SegmentsTable extends AbstractTable implements ScannableTable + { + private final DruidSchema druidSchema; + private final DruidLeaderClient druidLeaderClient; + private final ObjectMapper jsonMapper; + private final BytesAccumulatingResponseHandler responseHandler; + private final AuthorizerMapper authorizerMapper; + + public SegmentsTable( + DruidSchema druidSchemna, + DruidLeaderClient druidLeaderClient, + ObjectMapper jsonMapper, + BytesAccumulatingResponseHandler responseHandler, + AuthorizerMapper authorizerMapper + ) + { + this.druidSchema = druidSchemna; + this.druidLeaderClient = druidLeaderClient; + this.jsonMapper = jsonMapper; + this.responseHandler = responseHandler; + this.authorizerMapper = authorizerMapper; + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) + { + return SEGMENTS_SIGNATURE.getRelDataType(typeFactory); + } + + @Override + public TableType getJdbcTableType() + { + return TableType.SYSTEM_TABLE; + } + + @Override + public Enumerable<Object[]> scan(DataContext root) + { + //get available segments from druidSchema + final Map<DataSegment, SegmentMetadataHolder> availableSegmentMetadata = druidSchema.getSegmentMetadata(); + final Iterator<Entry<DataSegment, SegmentMetadataHolder>> availableSegmentEntries = availableSegmentMetadata.entrySet() + .iterator(); + + //get published segments from coordinator + final JsonParserIterator<DataSegment> metadataSegments = getMetadataSegments( + druidLeaderClient, + jsonMapper, + responseHandler + ); + + Set<String> availableSegmentIds = new HashSet<>(); + //auth check for available segments + final Iterator<Entry<DataSegment, SegmentMetadataHolder>> authorizedAvailableSegments = getAuthorizedAvailableSegments( + availableSegmentEntries, + root + ); + + final FluentIterable<Object[]> availableSegments = FluentIterable + .from(() -> authorizedAvailableSegments) + .transform(val -> { + try { + if (!availableSegmentIds.contains(val.getKey().getIdentifier())) { + availableSegmentIds.add(val.getKey().getIdentifier()); + } + return new Object[]{ + val.getKey().getIdentifier(), + val.getKey().getDataSource(), + val.getKey().getInterval().getStart(), + val.getKey().getInterval().getEnd(), + val.getKey().getSize(), + val.getKey().getVersion(), + val.getKey().getShardSpec().getPartitionNum(), + val.getValue().getNumReplicas(), + val.getValue().getNumRows(), + val.getValue().isPublished(), + val.getValue().isAvailable(), + val.getValue().isRealtime(), + jsonMapper.writeValueAsString(val.getKey()) + }; + } + catch (JsonProcessingException e) { + log.error(e, "Error getting segment payload for segment %s", val.getKey().getIdentifier()); + throw new RuntimeException(e); + } + }); + + //auth check for published segments + final CloseableIterator<DataSegment> authorizedPublishedSegments = getAuthorizedPublishedSegments( + metadataSegments, + root + ); + final FluentIterable<Object[]> publishedSegments = FluentIterable + .from(() -> authorizedPublishedSegments) + .transform(val -> { + try { + if (availableSegmentIds.contains(val.getIdentifier())) { + return null; + } + return new Object[]{ + val.getIdentifier(), + val.getDataSource(), + val.getInterval().getStart(), + val.getInterval().getEnd(), + val.getSize(), + val.getVersion(), + val.getShardSpec().getPartitionNum(), + 0L, + -1L, + 1L, + 0L, + 0L, + jsonMapper.writeValueAsString(val) + }; + } + catch (JsonProcessingException e) { + log.error(e, "Error getting segment payload for segment %s", val.getIdentifier()); + throw new RuntimeException(e); + } + }); + + final Iterable<Object[]> allSegments = Iterables.unmodifiableIterable( + Iterables.concat(availableSegments, publishedSegments)); + + return Linq4j.asEnumerable(allSegments).where(t -> t != null); + + } + + private Iterator<Entry<DataSegment, SegmentMetadataHolder>> getAuthorizedAvailableSegments( + Iterator<Entry<DataSegment, SegmentMetadataHolder>> availableSegmentEntries, + DataContext root + ) + { + final AuthenticationResult authenticationResult = + (AuthenticationResult) root.get(PlannerContext.DATA_CTX_AUTHENTICATION_RESULT); + + Function<Entry<DataSegment, SegmentMetadataHolder>, Iterable<ResourceAction>> raGenerator = segment -> Collections + .singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getKey().getDataSource())); + + final Iterable<Entry<DataSegment, SegmentMetadataHolder>> authorizedSegments = AuthorizationUtils.filterAuthorizedResources( + authenticationResult, () -> availableSegmentEntries, raGenerator, authorizerMapper); + + return authorizedSegments.iterator(); + } + + private CloseableIterator<DataSegment> getAuthorizedPublishedSegments( + JsonParserIterator<DataSegment> it, + DataContext root + ) + { + final AuthenticationResult authenticationResult = + (AuthenticationResult) root.get(PlannerContext.DATA_CTX_AUTHENTICATION_RESULT); + + Function<DataSegment, Iterable<ResourceAction>> raGenerator = segment -> Collections.singletonList( + AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getDataSource())); + + final Iterable<DataSegment> authorizedSegments = AuthorizationUtils.filterAuthorizedResources( + authenticationResult, () -> it, raGenerator, authorizerMapper); + + return wrap(authorizedSegments.iterator(), it); + } + } + + // Note that coordinator must be up to get segments + private static JsonParserIterator<DataSegment> getMetadataSegments( + DruidLeaderClient coordinatorClient, + ObjectMapper jsonMapper, + BytesAccumulatingResponseHandler responseHandler + ) + { + + Request request; + try { + request = coordinatorClient.makeRequest( + HttpMethod.GET, + StringUtils.format("/druid/coordinator/v1/metadata/segments") + ); + } + catch (IOException e) { + throw new RuntimeException(e); + } + ListenableFuture<InputStream> future = coordinatorClient.goAsync( + request, + responseHandler + ); + try { + future.get(); + } + catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + if (responseHandler.getStatus() != HttpServletResponse.SC_OK) { + throw new ISE( + "Error while fetching metadata segments status[%s] description[%s]", + responseHandler.status, + responseHandler.description + ); + } + final JavaType typeRef = jsonMapper.getTypeFactory().constructType(new TypeReference<DataSegment>() + { + }); + return new JsonParserIterator<>( + typeRef, + future, + request.getUrl().toString(), + null, + request.getUrl().getHost(), + jsonMapper + ); + } + + static class ServersTable extends AbstractTable implements ScannableTable + { + private final TimelineServerView serverView; + private final AuthorizerMapper authorizerMapper; + + public ServersTable(TimelineServerView serverView, AuthorizerMapper authorizerMapper) + { + this.serverView = serverView; + this.authorizerMapper = authorizerMapper; + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) + { + return SERVERS_SIGNATURE.getRelDataType(typeFactory); + } + + @Override + public TableType getJdbcTableType() + { + return TableType.SYSTEM_TABLE; + } + + @Override + public Enumerable<Object[]> scan(DataContext root) + { + final List<ImmutableDruidServer> druidServers = serverView.getDruidServers(); + final AuthenticationResult authenticationResult = + (AuthenticationResult) root.get(PlannerContext.DATA_CTX_AUTHENTICATION_RESULT); + final Access access = AuthorizationUtils.authorizeAllResourceActions( + authenticationResult, + Collections.singletonList(new ResourceAction(new Resource("STATE", ResourceType.STATE), Action.READ)), + authorizerMapper + ); + if (!access.isAllowed()) { + throw new ForbiddenException("Insufficient permission to view servers :" + access.toString()); + } + final FluentIterable<Object[]> results = FluentIterable + .from(druidServers) + .transform(val -> new Object[]{ + val.getHost(), + val.getHost().split(":")[0], + val.getHostAndPort() == null ? -1 : val.getHostAndPort().split(":")[1], + val.getHostAndTlsPort() == null ? -1 : val.getHostAndTlsPort().split(":")[1], + val.getType(), + val.getTier(), + val.getCurrSize(), + val.getMaxSize() + }); + return Linq4j.asEnumerable(results); + } + } + + private static class ServerSegmentsTable extends AbstractTable implements ScannableTable + { + private final TimelineServerView serverView; + final AuthorizerMapper authorizerMapper; + + public ServerSegmentsTable(TimelineServerView serverView, AuthorizerMapper authorizerMapper) + { + this.serverView = serverView; + this.authorizerMapper = authorizerMapper; + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) + { + return SERVER_SEGMENTS_SIGNATURE.getRelDataType(typeFactory); + } + + @Override + public TableType getJdbcTableType() + { + return TableType.SYSTEM_TABLE; + } + + @Override + public Enumerable<Object[]> scan(DataContext root) + { + final List<Object[]> rows = new ArrayList<>(); + final List<ImmutableDruidServer> druidServers = serverView.getDruidServers(); + final int serverSegmentsTableSize = SERVER_SEGMENTS_SIGNATURE.getRowOrder().size(); + for (ImmutableDruidServer druidServer : druidServers) { + final Map<String, DataSegment> segmentMap = druidServer.getSegments(); + for (DataSegment segment : segmentMap.values()) { + Object[] row = new Object[serverSegmentsTableSize]; + row[0] = druidServer.getHost(); + row[1] = segment.getIdentifier(); + rows.add(row); + } + } + return Linq4j.asEnumerable(rows); + } + } + + static class TasksTable extends AbstractTable implements ScannableTable + { + private final DruidLeaderClient druidLeaderClient; + private final ObjectMapper jsonMapper; + private final BytesAccumulatingResponseHandler responseHandler; + private final AuthorizerMapper authorizerMapper; + + public TasksTable( + DruidLeaderClient druidLeaderClient, + ObjectMapper jsonMapper, + BytesAccumulatingResponseHandler responseHandler, + AuthorizerMapper authorizerMapper + ) + { + this.druidLeaderClient = druidLeaderClient; + this.jsonMapper = jsonMapper; + this.responseHandler = responseHandler; + this.authorizerMapper = authorizerMapper; + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) + { + return TASKS_SIGNATURE.getRelDataType(typeFactory); + } + + @Override + public TableType getJdbcTableType() + { + return TableType.SYSTEM_TABLE; + } + + @Override + public Enumerable<Object[]> scan(DataContext root) + { + class TasksEnumerable extends DefaultEnumerable<Object[]> + { + private final CloseableIterator<TaskStatusPlus> it; + + public TasksEnumerable(JsonParserIterator<TaskStatusPlus> tasks) + { + this.it = getAuthorizedTasks(tasks, root); + } + + @Override + public Iterator<Object[]> iterator() + { + throw new UnsupportedOperationException("Do not use iterator(), it cannot be closed."); + } + + @Override + public Enumerator<Object[]> enumerator() + { + return new Enumerator<Object[]>() + { + @Override + public Object[] current() + { + TaskStatusPlus task = it.next(); + return new Object[]{task.getId(), + task.getType(), + task.getDataSource(), + task.getCreatedTime(), + task.getQueueInsertionTime(), + task.getStatusCode(), + task.getRunnerStatusCode(), + task.getDuration(), + task.getLocation().getHost() + ":" + (task.getLocation().getTlsPort() + == -1 + ? task.getLocation() + .getPort() + : task.getLocation().getTlsPort()), + task.getLocation().getHost(), + task.getLocation().getPort(), + task.getLocation().getTlsPort(), + task.getErrorMsg()}; + } + + @Override + public boolean moveNext() + { + return it.hasNext(); + } + + @Override + public void reset() + { + + } + + @Override + public void close() + { + try { + it.close(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + } + } + + return new TasksEnumerable(getTasks(druidLeaderClient, jsonMapper, responseHandler)); + } + + private CloseableIterator<TaskStatusPlus> getAuthorizedTasks(JsonParserIterator<TaskStatusPlus> it, DataContext root) + { + final AuthenticationResult authenticationResult = + (AuthenticationResult) root.get(PlannerContext.DATA_CTX_AUTHENTICATION_RESULT); + + Function<TaskStatusPlus, Iterable<ResourceAction>> raGenerator = task -> Collections.singletonList( + AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(task.getDataSource())); + + final Iterable<TaskStatusPlus> authorizedTasks = AuthorizationUtils.filterAuthorizedResources( + authenticationResult, () -> it, raGenerator, authorizerMapper); + + return wrap(authorizedTasks.iterator(), it); + } + + } + + //Note that overlord must be up to get tasks + private static JsonParserIterator<TaskStatusPlus> getTasks( + DruidLeaderClient indexingServiceClient, + ObjectMapper jsonMapper, + BytesAccumulatingResponseHandler responseHandler + ) + { + + Request request; + try { + request = indexingServiceClient.makeRequest( + HttpMethod.GET, + StringUtils.format("/druid/indexer/v1/tasks") + ); + } + catch (IOException e) { + throw new RuntimeException(e); + } + ListenableFuture<InputStream> future = indexingServiceClient.goAsync( + request, + responseHandler + ); + try { + future.get(); + } + catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + if (responseHandler.getStatus() != HttpServletResponse.SC_OK) { + throw new ISE( + "Error while fetching tasks status[%s] description[%s]", + responseHandler.status, + responseHandler.description + ); + } + final JavaType typeRef = jsonMapper.getTypeFactory().constructType(new TypeReference<TaskStatusPlus>() + { + }); + return new JsonParserIterator<>( + typeRef, + future, + request.getUrl().toString(), + null, + request.getUrl().getHost(), + jsonMapper + ); + } + + private static <T> CloseableIterator<T> wrap(Iterator<T> iterator, JsonParserIterator<T> it) + { + return new CloseableIterator<T>() + { + @Override + public boolean hasNext() + { + return iterator.hasNext(); + } + + @Override + public T next() + { + return iterator.next(); + } + + @Override + public void close() throws IOException + { + it.close(); + } + }; + } + + static class BytesAccumulatingResponseHandler extends InputStreamResponseHandler Review comment: There are two more same classes. Please extract them to the public class and reuse it. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org