This is an automated email from the ASF dual-hosted git repository. gerlowskija pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 25dbd857a5e87f0e6f1e3cb71cc83edc9b64e69b Author: Jason Gerlowski <[email protected]> AuthorDate: Wed Apr 19 10:49:39 2023 -0400 SOLR-16391: Tweak "create coll" API to be more REST-ful (#1572) This commit makes various cosmetic improvements to Solr's v2 create collection API, to bring it more into line with the more REST-ful v2 design. This mostly includes tweaking a few parameter names, and removing the "create" top-level key representing the "command". It also migrates the API definition to JAX-RS. --- solr/CHANGES.txt | 3 + .../solr/cloud/api/collections/AliasCmd.java | 16 +- .../solr/handler/admin/CollectionsHandler.java | 87 +--- .../solr/handler/admin/api/CreateAliasAPI.java | 2 +- .../handler/admin/api/CreateCollectionAPI.java | 482 ++++++++++++++++++--- .../handler/admin/api/RestoreCollectionAPI.java | 2 +- .../SubResponseAccumulatingJerseyResponse.java | 3 + .../apache/solr/cloud/MultiThreadedOCPTest.java | 3 +- .../org/apache/solr/cloud/TestPullReplica.java | 10 +- .../org/apache/solr/cloud/TestTlogReplica.java | 2 +- .../CollectionsAPIAsyncDistributedZkTest.java | 11 +- .../TestRequestStatusCollectionAPI.java | 35 +- .../solr/handler/admin/TestApiFramework.java | 1 - .../solr/handler/admin/TestCollectionAPIs.java | 29 -- .../handler/admin/V2CollectionsAPIMappingTest.java | 49 --- .../handler/admin/api/CreateCollectionAPITest.java | 233 ++++++++++ .../pages/collection-management.adoc | 37 +- .../client/solrj/request/beans/CreatePayload.java | 55 --- .../client/solrj/request/beans/V2ApiConstants.java | 3 + .../apache/solr/common/util/CollectionUtil.java | 11 +- .../client/solrj/impl/CloudSolrClientTest.java | 2 +- .../solr/client/solrj/request/TestV2Request.java | 44 +- .../cloud/PerReplicaStatesIntegrationTest.java | 2 +- 23 files changed, 772 insertions(+), 350 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d0f39ee0d60..ebc2f9c8d24 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -62,6 +62,9 @@ Improvements /api/collections/collName/snapshots/snapshotName`, listed at `GET /api/collections/collName/snapshots`, and deleted at `DELETE /api/collections/collName/snapshots/snapshotName`. (John Durham via Jason Gerlowski) +* SOLR-16391: Solr's v2 "create collection" API has been tweaked slightly to be more intuitive. It remains available under + `POST /api/collections`, but the top level "create" command-name key has been removed. (Jason Gerlowski) + Optimizations --------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java index 5683807419e..92b18b53a74 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java @@ -23,17 +23,15 @@ import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; import static org.apache.solr.common.params.CommonParams.NAME; import java.util.Map; -import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.OverseerSolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.CollectionProperties; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.handler.admin.CollectionsHandler; -import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.handler.admin.api.CreateCollectionAPI; /** * Common superclass for commands that maintain or manipulate aliases. In the routed alias parlance, @@ -75,12 +73,10 @@ abstract class AliasCmd implements CollApiCmds.CollectionApiCommand { // a CollectionOperation reads params and produces a message (Map) that is supposed to be sent // to the Overseer. Although we could create the Map without it, there are a fair amount of // rules we don't want to reproduce. - final Map<String, Object> createMsgMap = - CollectionsHandler.CollectionOperation.CREATE_OP.execute( - new LocalSolrQueryRequest(null, createReqParams), - null, - ccc.getCoreContainer().getCollectionsHandler()); - createMsgMap.put(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower()); + final var createReqBody = + CreateCollectionAPI.CreateCollectionRequestBody.fromV1Params(createReqParams, true); + CreateCollectionAPI.populateDefaultsIfNecessary(ccc.getCoreContainer(), createReqBody); + final ZkNodeProps createMessage = CreateCollectionAPI.createRemoteMessage(createReqBody); NamedList<Object> results = new NamedList<>(); try { @@ -88,7 +84,7 @@ abstract class AliasCmd implements CollApiCmds.CollectionApiCommand { // CreateCollectionCmd. // note: there's doesn't seem to be any point in locking on the collection name, so we don't. // We currently should already have a lock on the alias name which should be sufficient. - new CreateCollectionCmd(ccc).call(clusterState, new ZkNodeProps(createMsgMap), results); + new CreateCollectionCmd(ccc).call(clusterState, createMessage, results); } catch (SolrException e) { // The collection might already exist, and that's okay -- we can adopt it. if (!e.getMessage().contains("collection already exists")) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 965ed4b67de..37171695925 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -29,7 +29,6 @@ import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.NUM_ import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ONLY_ACTIVE_NODES; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ONLY_IF_DOWN; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.REQUESTID; -import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARDS_PROP; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.apache.solr.cloud.api.collections.RoutedAlias.CREATE_COLLECTION_PREFIX; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; @@ -43,13 +42,11 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS; -import static org.apache.solr.common.params.CollectionAdminParams.ALIAS; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP; import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CollectionAdminParams.PER_REPLICA_STATE; import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_NAME; import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX; import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_VALUE; @@ -192,7 +189,6 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Pair; import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CloudConfig; import org.apache.solr.core.CoreContainer; @@ -241,7 +237,6 @@ import org.apache.solr.handler.admin.api.SyncShardAPI; import org.apache.solr.handler.api.V2ApiUtils; import org.apache.solr.jersey.SolrJerseyResponse; import org.apache.solr.logging.MDCLoggingContext; -import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.AuthorizationContext; @@ -383,15 +378,6 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission if (exp != null) { rsp.setException(exp); } - - // Even if Overseer does wait for the collection to be created, it sees a different cluster - // state than this node, so this wait is required to make sure the local node Zookeeper watches - // fired and now see the collection. - if (action.equals(CollectionAction.CREATE) && asyncId == null) { - if (rsp.getException() == null) { - waitForActiveCollection(zkProps.getStr(NAME), cores, overseerResponse); - } - } } static final Set<String> KNOWN_ROLES = Set.of("overseer"); @@ -451,7 +437,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission } } } else { - r.add("error", "Task with the same requestid already exists. (" + asyncId + ")"); + throw new SolrException( + BAD_REQUEST, "Task with the same requestid already exists. (" + asyncId + ")"); } r.add(CoreAdminParams.REQUESTID, m.get(ASYNC)); @@ -600,58 +587,16 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission CREATE_OP( CREATE, (req, rsp, h) -> { - Map<String, Object> props = copy(req.getParams().required(), null, NAME); - props.put("fromApi", "true"); - copy( - req.getParams(), - props, - REPLICATION_FACTOR, - COLL_CONF, - NUM_SLICES, - CREATE_NODE_SET, - CREATE_NODE_SET_SHUFFLE, - SHARDS_PROP, - PULL_REPLICAS, - TLOG_REPLICAS, - NRT_REPLICAS, - WAIT_FOR_FINAL_STATE, - PER_REPLICA_STATE, - ALIAS); - - if (props.get(REPLICATION_FACTOR) != null && props.get(NRT_REPLICAS) != null) { - // TODO: Remove this in 8.0 . Keep this for SolrJ client back-compat. See SOLR-11676 for - // more details - int replicationFactor = Integer.parseInt((String) props.get(REPLICATION_FACTOR)); - int nrtReplicas = Integer.parseInt((String) props.get(NRT_REPLICAS)); - if (replicationFactor != nrtReplicas) { - throw new SolrException( - ErrorCode.BAD_REQUEST, - "Cannot specify both replicationFactor and nrtReplicas as they mean the same thing"); - } + final CreateCollectionAPI.CreateCollectionRequestBody requestBody = + CreateCollectionAPI.CreateCollectionRequestBody.fromV1Params(req.getParams(), true); + final CreateCollectionAPI createApi = new CreateCollectionAPI(h.coreContainer, req, rsp); + final SolrJerseyResponse response = createApi.createCollection(requestBody); + + // 'rsp' may be null, as when overseer commands execute CollectionAction impl's directly. + if (rsp != null) { + V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, response); } - if (props.get(REPLICATION_FACTOR) != null) { - props.put(NRT_REPLICAS, props.get(REPLICATION_FACTOR)); - } else if (props.get(NRT_REPLICAS) != null) { - props.put(REPLICATION_FACTOR, props.get(NRT_REPLICAS)); - } - - final String collectionName = - SolrIdentifierValidator.validateCollectionName((String) props.get(NAME)); - final String shardsParam = (String) props.get(SHARDS_PROP); - if (StrUtils.isNotNullOrEmpty(shardsParam)) { - verifyShardsParam(shardsParam); - } - if (CollectionAdminParams.SYSTEM_COLL.equals(collectionName)) { - // We must always create a .system collection with only a single shard - props.put(NUM_SLICES, 1); - props.remove(SHARDS_PROP); - createSysConfigSet(h.coreContainer); - } - if (shardsParam == null) h.copyFromClusterProp(props, NUM_SLICES); - for (String prop : Set.of(NRT_REPLICAS, PULL_REPLICAS, TLOG_REPLICAS)) - h.copyFromClusterProp(props, prop); - copyPropertiesWithPrefix(req.getParams(), props, PROPERTY_PREFIX); - return copyPropertiesWithPrefix(req.getParams(), props, "router."); + return null; }), COLSTATUS_OP( COLSTATUS, @@ -859,10 +804,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "We require an explicit " + COLL_CONF); } - // note: could insist on a config name here as well.... or wait to throw at overseer - createCollParams.add(NAME, "TMP_name_TMP_name_TMP"); // just to pass validation - CREATE_OP.execute( - new LocalSolrQueryRequest(null, createCollParams), rsp, h); // ignore results + final var createRequestBody = + CreateCollectionAPI.CreateCollectionRequestBody.fromV1Params(createCollParams, false); + createRequestBody.name = "TMP_name_TMP_name_TMP"; // just to pass validation + createRequestBody.validate(); return result; }), @@ -2001,6 +1946,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission public Collection<Class<? extends JerseyResource>> getJerseyResources() { return List.of( AddReplicaPropertyAPI.class, + CreateCollectionAPI.class, CreateCollectionBackupAPI.class, DeleteAliasAPI.class, DeleteCollectionAPI.class, @@ -2020,7 +1966,6 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission @Override public Collection<Api> getApis() { final List<Api> apis = new ArrayList<>(); - apis.addAll(AnnotatedApi.getApis(new CreateCollectionAPI(this))); apis.addAll(AnnotatedApi.getApis(new CreateAliasAPI(this))); apis.addAll(AnnotatedApi.getApis(new RestoreCollectionAPI(this))); apis.addAll(AnnotatedApi.getApis(new SplitShardAPI(this))); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAliasAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAliasAPI.java index ba7e2e5393a..41ebdbcefda 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAliasAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAliasAPI.java @@ -22,7 +22,7 @@ import static org.apache.solr.cloud.api.collections.RoutedAlias.CREATE_COLLECTIO import static org.apache.solr.common.params.CollectionAdminParams.ROUTER_PREFIX; import static org.apache.solr.common.params.CommonParams.ACTION; import static org.apache.solr.handler.ClusterAPI.wrapParams; -import static org.apache.solr.handler.admin.api.CreateCollectionAPI.convertV2CreateCollectionMapToV1ParamMap; +import static org.apache.solr.handler.admin.api.CreateCollectionAPI.CreateCollectionRequestBody.convertV2CreateCollectionMapToV1ParamMap; import static org.apache.solr.handler.api.V2ApiUtils.flattenMapWithPrefix; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionAPI.java index 3ba329a47c0..1bfb176fd46 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionAPI.java @@ -17,90 +17,454 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST; +import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.ROUTER_KEY; +import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.SHARD_NAMES; +import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; +import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; +import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET_SHUFFLE; +import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.NUM_SLICES; +import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARDS_PROP; +import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; +import static org.apache.solr.common.params.CollectionAdminParams.ALIAS; +import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; +import static org.apache.solr.common.params.CollectionAdminParams.NRT_REPLICAS; +import static org.apache.solr.common.params.CollectionAdminParams.PER_REPLICA_STATE; import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX; -import static org.apache.solr.common.params.CommonParams.ACTION; -import static org.apache.solr.handler.ClusterAPI.wrapParams; +import static org.apache.solr.common.params.CollectionAdminParams.PULL_REPLICAS; +import static org.apache.solr.common.params.CollectionAdminParams.REPLICATION_FACTOR; +import static org.apache.solr.common.params.CollectionAdminParams.TLOG_REPLICAS; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; +import static org.apache.solr.common.params.CoreAdminParams.CONFIG; +import static org.apache.solr.common.params.CoreAdminParams.NAME; +import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; +import static org.apache.solr.handler.admin.CollectionsHandler.waitForActiveCollection; import static org.apache.solr.handler.api.V2ApiUtils.flattenMapWithPrefix; +import static org.apache.solr.schema.IndexSchema.FIELD; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.solr.api.Command; -import org.apache.solr.api.EndPoint; -import org.apache.solr.api.PayloadObj; -import org.apache.solr.client.solrj.request.beans.CreatePayload; +import javax.inject.Inject; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.request.beans.V2ApiConstants; +import org.apache.solr.client.solrj.util.SolrIdentifierValidator; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterProperties; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkCmdExecutor; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CollectionParams; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.CollectionUtil; +import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.admin.CollectionsHandler; +import org.apache.solr.jersey.JacksonReflectMapWriter; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.jersey.SubResponseAccumulatingJerseyResponse; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; -@EndPoint( - path = {"/collections"}, - method = POST, - permission = COLL_EDIT_PERM) -public class CreateCollectionAPI { +/** + * V2 API for creating a SolrCLoud collection + * + * <p>This API is analogous to the v1 /admin/collections?action=CREATE command. + */ +@Path("/collections") +public class CreateCollectionAPI extends AdminAPIBase { + + @Inject + public CreateCollectionAPI( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + super(coreContainer, solrQueryRequest, solrQueryResponse); + } + + @POST + @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, BINARY_CONTENT_TYPE_V2}) + @PermissionName(COLL_EDIT_PERM) + public SubResponseAccumulatingJerseyResponse createCollection( + CreateCollectionRequestBody requestBody) throws Exception { + + if (requestBody == null) { + throw new SolrException(BAD_REQUEST, "Request body is missing but required"); + } + + final SubResponseAccumulatingJerseyResponse response = + instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + recordCollectionForLogAndTracing(requestBody.name, solrQueryRequest); + + // We must always create a .system collection with only a single shard + if (CollectionAdminParams.SYSTEM_COLL.equals(requestBody.name)) { + requestBody.numShards = 1; + requestBody.shardNames = null; + createSysConfigSet(coreContainer); + } + + requestBody.validate(); + + // Populate any 'null' creation parameters that support COLLECTIONPROP defaults. + populateDefaultsIfNecessary(coreContainer, requestBody); + + final ZkNodeProps remoteMessage = createRemoteMessage(requestBody); + final SolrResponse remoteResponse = + CollectionsHandler.submitCollectionApiCommand( + coreContainer, + coreContainer.getDistributedCollectionCommandRunner(), + remoteMessage, + CollectionParams.CollectionAction.CREATE, + DEFAULT_COLLECTION_OP_TIMEOUT); + if (remoteResponse.getException() != null) { + throw remoteResponse.getException(); + } + + if (requestBody.async != null) { + response.requestId = requestBody.async; + return response; + } + + // Values fetched from remoteResponse may be null + response.successfulSubResponsesByNodeName = remoteResponse.getResponse().get("success"); + response.failedSubResponsesByNodeName = remoteResponse.getResponse().get("failure"); + response.warning = (String) remoteResponse.getResponse().get("warning"); + + // Even if Overseer does wait for the collection to be created, it sees a different cluster + // state than this node, so this wait is required to make sure the local node Zookeeper watches + // fired and now see the collection. + if (requestBody.async == null) { + waitForActiveCollection(requestBody.name, coreContainer, remoteResponse); + } - public static final String V2_CREATE_COLLECTION_CMD = "create"; + return response; + } + + public static void populateDefaultsIfNecessary( + CoreContainer coreContainer, CreateCollectionRequestBody requestBody) throws IOException { + if (CollectionUtil.isEmpty(requestBody.shardNames) && requestBody.numShards == null) { + requestBody.numShards = readIntegerDefaultFromClusterProp(coreContainer, NUM_SLICES); + } + if (requestBody.nrtReplicas == null) + requestBody.nrtReplicas = readIntegerDefaultFromClusterProp(coreContainer, NRT_REPLICAS); + if (requestBody.tlogReplicas == null) + requestBody.tlogReplicas = readIntegerDefaultFromClusterProp(coreContainer, TLOG_REPLICAS); + if (requestBody.pullReplicas == null) + requestBody.pullReplicas = readIntegerDefaultFromClusterProp(coreContainer, PULL_REPLICAS); + } + + private static void verifyShardsParam(List<String> shardNames) { + for (String shard : shardNames) { + SolrIdentifierValidator.validateShardName(shard); + } + } + + public static ZkNodeProps createRemoteMessage(CreateCollectionRequestBody reqBody) { + final Map<String, Object> rawProperties = new HashMap<>(); + rawProperties.put("fromApi", "true"); + + rawProperties.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower()); + rawProperties.put(NAME, reqBody.name); + rawProperties.put(COLL_CONF, reqBody.config); + rawProperties.put(NUM_SLICES, reqBody.numShards); + rawProperties.put(CREATE_NODE_SET_SHUFFLE, reqBody.shuffleNodes); + if (CollectionUtil.isNotEmpty(reqBody.shardNames)) + rawProperties.put(SHARDS_PROP, String.join(",", reqBody.shardNames)); + rawProperties.put(PULL_REPLICAS, reqBody.pullReplicas); + rawProperties.put(TLOG_REPLICAS, reqBody.tlogReplicas); + rawProperties.put(WAIT_FOR_FINAL_STATE, reqBody.waitForFinalState); + rawProperties.put(PER_REPLICA_STATE, reqBody.perReplicaState); + rawProperties.put(ALIAS, reqBody.alias); + rawProperties.put(ASYNC, reqBody.async); + if (reqBody.createReplicas == null || reqBody.createReplicas) { + // The remote message expects a single comma-delimited string, so nodeSet requires flattening + if (reqBody.nodeSet != null) { + rawProperties.put(CREATE_NODE_SET, String.join(",", reqBody.nodeSet)); + } + } else { + rawProperties.put(CREATE_NODE_SET, "EMPTY"); + } + // 'nrtReplicas' and 'replicationFactor' are both set on the remote message, despite being + // functionally equivalent. + if (reqBody.replicationFactor != null) { + rawProperties.put(REPLICATION_FACTOR, reqBody.replicationFactor); + if (reqBody.nrtReplicas == null) rawProperties.put(NRT_REPLICAS, reqBody.replicationFactor); + } + if (reqBody.nrtReplicas != null) { + rawProperties.put(NRT_REPLICAS, reqBody.nrtReplicas); + if (reqBody.replicationFactor == null) + rawProperties.put(REPLICATION_FACTOR, reqBody.nrtReplicas); + } + + if (reqBody.properties != null) { + for (Map.Entry<String, String> entry : reqBody.properties.entrySet()) { + rawProperties.put(PROPERTY_PREFIX + entry.getKey(), entry.getValue()); + } + } + + if (reqBody.router != null) { + final RouterProperties routerProps = reqBody.router; + rawProperties.put("router.name", routerProps.name); + rawProperties.put("router.field", routerProps.field); + } - private final CollectionsHandler collectionsHandler; + return new ZkNodeProps(rawProperties); + } - public CreateCollectionAPI(CollectionsHandler collectionsHandler) { - this.collectionsHandler = collectionsHandler; + public static Map<String, String> copyPrefixedPropertiesWithoutPrefix( + SolrParams params, Map<String, String> props, String prefix) { + Iterator<String> iter = params.getParameterNamesIterator(); + while (iter.hasNext()) { + String param = iter.next(); + if (param.startsWith(prefix)) { + final String[] values = params.getParams(param); + if (values.length != 1) { + throw new SolrException( + BAD_REQUEST, "Only one value can be present for parameter " + param); + } + final String modifiedKey = param.replaceFirst(prefix, ""); + props.put(modifiedKey, values[0]); + } + } + return props; } - @Command(name = V2_CREATE_COLLECTION_CMD) - public void create(PayloadObj<CreatePayload> obj) throws Exception { - final CreatePayload v2Body = obj.get(); - final Map<String, Object> v1Params = v2Body.toMap(new HashMap<>()); + private static Integer readIntegerDefaultFromClusterProp( + CoreContainer coreContainer, String propName) throws IOException { + final Object defaultValue = + new ClusterProperties(coreContainer.getZkController().getZkStateReader().getZkClient()) + .getClusterProperty( + List.of(CollectionAdminParams.DEFAULTS, CollectionAdminParams.COLLECTION, propName), + null); + if (defaultValue == null) return null; + + return Integer.valueOf(String.valueOf(defaultValue)); + } - v1Params.put(ACTION, CollectionParams.CollectionAction.CREATE.toLower()); - convertV2CreateCollectionMapToV1ParamMap(v1Params); + private static void createSysConfigSet(CoreContainer coreContainer) + throws KeeperException, InterruptedException { + SolrZkClient zk = coreContainer.getZkController().getZkStateReader().getZkClient(); + ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zk.getZkClientTimeout()); + cmdExecutor.ensureExists(ZkStateReader.CONFIGS_ZKNODE, zk); + cmdExecutor.ensureExists( + ZkStateReader.CONFIGS_ZKNODE + "/" + CollectionAdminParams.SYSTEM_COLL, zk); - collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), v1Params), obj.getResponse()); + try { + String path = + ZkStateReader.CONFIGS_ZKNODE + "/" + CollectionAdminParams.SYSTEM_COLL + "/schema.xml"; + byte[] data; + try (InputStream inputStream = + CollectionsHandler.class.getResourceAsStream("/SystemCollectionSchema.xml")) { + assert inputStream != null; + data = inputStream.readAllBytes(); + } + assert data != null && data.length > 0; + cmdExecutor.ensureExists(path, data, CreateMode.PERSISTENT, zk); + path = + ZkStateReader.CONFIGS_ZKNODE + + "/" + + CollectionAdminParams.SYSTEM_COLL + + "/solrconfig.xml"; + try (InputStream inputStream = + CollectionsHandler.class.getResourceAsStream("/SystemCollectionSolrConfig.xml")) { + assert inputStream != null; + data = inputStream.readAllBytes(); + } + assert data != null && data.length > 0; + cmdExecutor.ensureExists(path, data, CreateMode.PERSISTENT, zk); + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } } - @SuppressWarnings("unchecked") - public static void convertV2CreateCollectionMapToV1ParamMap(Map<String, Object> v2MapVals) { - // Keys are copied so that map can be modified as keys are looped through. - final Set<String> v2Keys = v2MapVals.keySet().stream().collect(Collectors.toSet()); - for (String key : v2Keys) { - switch (key) { - case V2ApiConstants.PROPERTIES_KEY: - final Map<String, Object> propertiesMap = - (Map<String, Object>) v2MapVals.remove(V2ApiConstants.PROPERTIES_KEY); - flattenMapWithPrefix(propertiesMap, v2MapVals, PROPERTY_PREFIX); - break; - case ROUTER_KEY: - final Map<String, Object> routerProperties = - (Map<String, Object>) v2MapVals.remove(V2ApiConstants.ROUTER_KEY); - flattenMapWithPrefix(routerProperties, v2MapVals, CollectionAdminParams.ROUTER_PREFIX); - break; - case V2ApiConstants.CONFIG: - v2MapVals.put(CollectionAdminParams.COLL_CONF, v2MapVals.remove(V2ApiConstants.CONFIG)); - break; - case V2ApiConstants.SHUFFLE_NODES: - v2MapVals.put( - CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM, - v2MapVals.remove(V2ApiConstants.SHUFFLE_NODES)); - break; - case V2ApiConstants.NODE_SET: - final Object nodeSetValUncast = v2MapVals.remove(V2ApiConstants.NODE_SET); - if (nodeSetValUncast instanceof String) { - v2MapVals.put(CollectionAdminParams.CREATE_NODE_SET_PARAM, nodeSetValUncast); - } else { - final List<String> nodeSetList = (List<String>) nodeSetValUncast; - final String nodeSetStr = String.join(",", nodeSetList); - v2MapVals.put(CollectionAdminParams.CREATE_NODE_SET_PARAM, nodeSetStr); - } - break; - default: - break; + /** Request body for v2 "create collection" requests */ + public static class CreateCollectionRequestBody implements JacksonReflectMapWriter { + @JsonProperty(NAME) + public String name; + + @JsonProperty(REPLICATION_FACTOR) + public Integer replicationFactor; + + @JsonProperty(CONFIG) + public String config; + + @JsonProperty(NUM_SLICES) + public Integer numShards; + + @JsonProperty(SHARD_NAMES) + public List<String> shardNames; + + @JsonProperty(PULL_REPLICAS) + public Integer pullReplicas; + + @JsonProperty(TLOG_REPLICAS) + public Integer tlogReplicas; + + @JsonProperty(NRT_REPLICAS) + public Integer nrtReplicas; + + @JsonProperty(WAIT_FOR_FINAL_STATE) + public Boolean waitForFinalState; + + @JsonProperty(PER_REPLICA_STATE) + public Boolean perReplicaState; + + @JsonProperty(ALIAS) + public String alias; + + @JsonProperty("properties") + public Map<String, String> properties; + + @JsonProperty(ASYNC) + public String async; + + @JsonProperty("router") + public RouterProperties router; + + // Parameters below differ from v1 API + // V1 API uses createNodeSet + @JsonProperty("nodeSet") + public List<String> nodeSet; + // v1 API uses createNodeSet=EMPTY + @JsonProperty("createReplicas") + public Boolean createReplicas; + // V1 API uses 'createNodeSet.shuffle' + @JsonProperty("shuffleNodes") + public Boolean shuffleNodes; + + public static CreateCollectionRequestBody fromV1Params( + SolrParams params, boolean nameRequired) { + final var requestBody = new CreateCollectionRequestBody(); + requestBody.name = + nameRequired ? params.required().get(CommonParams.NAME) : params.get(CommonParams.NAME); + requestBody.replicationFactor = params.getInt(ZkStateReader.REPLICATION_FACTOR); + requestBody.config = params.get(COLL_CONF); + requestBody.numShards = params.getInt(NUM_SLICES); + if (params.get(CREATE_NODE_SET) != null) { + final String commaDelimNodeSet = params.get(CREATE_NODE_SET); + if ("EMPTY".equals(commaDelimNodeSet)) { + requestBody.createReplicas = false; + } else { + requestBody.nodeSet = Arrays.asList(params.get(CREATE_NODE_SET).split(",")); + } + } + requestBody.shuffleNodes = params.getBool(CREATE_NODE_SET_SHUFFLE); + requestBody.shardNames = + params.get(SHARDS_PROP) != null + ? Arrays.stream(params.get(SHARDS_PROP).split(",")).collect(Collectors.toList()) + : new ArrayList<>(); + requestBody.tlogReplicas = params.getInt(ZkStateReader.TLOG_REPLICAS); + requestBody.pullReplicas = params.getInt(ZkStateReader.PULL_REPLICAS); + requestBody.nrtReplicas = params.getInt(ZkStateReader.NRT_REPLICAS); + requestBody.waitForFinalState = params.getBool(WAIT_FOR_FINAL_STATE); + requestBody.perReplicaState = params.getBool(PER_REPLICA_STATE); + requestBody.alias = params.get(ALIAS); + requestBody.async = params.get(ASYNC); + requestBody.properties = + copyPrefixedPropertiesWithoutPrefix(params, new HashMap<>(), PROPERTY_PREFIX); + if (params.get("router.name") != null || params.get("router.field") != null) { + final RouterProperties routerProperties = new RouterProperties(); + routerProperties.name = params.get("router.name"); + routerProperties.field = params.get("router.field"); + requestBody.router = routerProperties; + } + + return requestBody; + } + + public void validate() { + if (replicationFactor != null + && nrtReplicas != null + && (!replicationFactor.equals(nrtReplicas))) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Cannot specify both replicationFactor and nrtReplicas as they mean the same thing"); + } + + SolrIdentifierValidator.validateCollectionName(name); + + if (shardNames != null && !shardNames.isEmpty()) { + verifyShardsParam(shardNames); } } + + /** + * Convert a map representing the v2 request body into v1-appropriate query-parameters. + * + * <p>Most v2 APIs using the legacy (i.e. non-JAX-RS) framework implement the v2 API by + * restructuring the provided parameters so that the v1 codepath can be called. This utility + * method is provided in pursuit of that usecase. It's not used directly CreateCollectionAPI, + * which uses the JAX-RS framework, but it's kept here so that logic surrounding + * collection-creation parameters can be kept in a single place. + */ + @SuppressWarnings("unchecked") + public static void convertV2CreateCollectionMapToV1ParamMap(Map<String, Object> v2MapVals) { + // Keys are copied so that map can be modified as keys are looped through. + final Set<String> v2Keys = v2MapVals.keySet().stream().collect(Collectors.toSet()); + for (String key : v2Keys) { + switch (key) { + case V2ApiConstants.PROPERTIES_KEY: + final Map<String, Object> propertiesMap = + (Map<String, Object>) v2MapVals.remove(V2ApiConstants.PROPERTIES_KEY); + flattenMapWithPrefix(propertiesMap, v2MapVals, CollectionAdminParams.PROPERTY_PREFIX); + break; + case ROUTER_KEY: + final Map<String, Object> routerProperties = + (Map<String, Object>) v2MapVals.remove(V2ApiConstants.ROUTER_KEY); + flattenMapWithPrefix(routerProperties, v2MapVals, CollectionAdminParams.ROUTER_PREFIX); + break; + case V2ApiConstants.CONFIG: + v2MapVals.put(CollectionAdminParams.COLL_CONF, v2MapVals.remove(V2ApiConstants.CONFIG)); + break; + case SHARD_NAMES: + v2MapVals.put(SHARDS_PROP, v2MapVals.remove(SHARD_NAMES)); + break; + case V2ApiConstants.SHUFFLE_NODES: + v2MapVals.put( + CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM, + v2MapVals.remove(V2ApiConstants.SHUFFLE_NODES)); + break; + case V2ApiConstants.NODE_SET: + final Object nodeSetValUncast = v2MapVals.remove(V2ApiConstants.NODE_SET); + if (nodeSetValUncast instanceof String) { + v2MapVals.put(CollectionAdminParams.CREATE_NODE_SET_PARAM, nodeSetValUncast); + } else { + final List<String> nodeSetList = (List<String>) nodeSetValUncast; + final String nodeSetStr = String.join(",", nodeSetList); + v2MapVals.put(CollectionAdminParams.CREATE_NODE_SET_PARAM, nodeSetStr); + } + break; + default: + break; + } + } + } + } + + public static class RouterProperties implements JacksonReflectMapWriter { + @JsonProperty(NAME) + public String name; + + @JsonProperty(FIELD) + public String field; } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java index bbbd0304b12..cdefb7b251f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java @@ -21,7 +21,7 @@ import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST; import static org.apache.solr.common.params.CommonParams.ACTION; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.handler.ClusterAPI.wrapParams; -import static org.apache.solr.handler.admin.api.CreateCollectionAPI.convertV2CreateCollectionMapToV1ParamMap; +import static org.apache.solr.handler.admin.api.CreateCollectionAPI.CreateCollectionRequestBody.convertV2CreateCollectionMapToV1ParamMap; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import java.util.HashMap; diff --git a/solr/core/src/java/org/apache/solr/jersey/SubResponseAccumulatingJerseyResponse.java b/solr/core/src/java/org/apache/solr/jersey/SubResponseAccumulatingJerseyResponse.java index 6f02e9b6b1c..a0f0dbfb0e2 100644 --- a/solr/core/src/java/org/apache/solr/jersey/SubResponseAccumulatingJerseyResponse.java +++ b/solr/core/src/java/org/apache/solr/jersey/SubResponseAccumulatingJerseyResponse.java @@ -49,4 +49,7 @@ public class SubResponseAccumulatingJerseyResponse extends AsyncJerseyResponse { @JsonProperty("failure") public Object failedSubResponsesByNodeName; + + @JsonProperty("warning") + public String warning; } diff --git a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java index 3f02637ff36..a3c5b152cb6 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java @@ -23,6 +23,7 @@ import java.lang.invoke.MethodHandles; import java.util.Random; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest.Create; import org.apache.solr.client.solrj.request.CollectionAdminRequest.SplitShard; @@ -260,7 +261,7 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase { // Now submit another task with the same id. At this time, hopefully the previous 3002 should // still be in the queue. expectThrows( - SolrServerException.class, + BaseHttpSolrClient.RemoteSolrException.class, () -> { CollectionAdminRequest.splitShard("ocptest_shardsplit2") .setShardName(SHARD1) diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java index edbbf941057..baaab3d516f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java @@ -125,10 +125,10 @@ public class TestPullReplica extends SolrCloudTestCase { } // 2 times to make sure cleanup is complete, and we can create the same collection - @Repeat(iterations = 2) + @Repeat(iterations = 30) public void testCreateDelete() throws Exception { try { - switch (random().nextInt(3)) { + switch (2 /*random().nextInt(3)*/) { case 0: // Sometimes use SolrJ CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1, 0, 3) @@ -156,15 +156,15 @@ public class TestPullReplica extends SolrCloudTestCase { String requestBody = String.format( Locale.ROOT, - "{create:{name:%s, config:%s, numShards:%s, pullReplicas:%s, %s}}", + "{\"name\": \"%s\", \"config\": \"%s\", \"numShards\": %s, \"pullReplicas\": %s %s}", collectionName, "conf", 2, // numShards 3, // pullReplicas pickRandom( "", - ", nrtReplicas:1", - ", replicationFactor:1")); // These options should all mean the same + ", \"nrtReplicas\": 1", + ", \"replicationFactor\": 1")); // These options should all mean the same HttpPost createCollectionPost = new HttpPost(url); createCollectionPost.setHeader("Content-type", "application/json"); createCollectionPost.setEntity(new StringEntity(requestBody)); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java index a51b0182cb8..ff391813670 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java @@ -182,7 +182,7 @@ public class TestTlogReplica extends SolrCloudTestCase { String requestBody = String.format( Locale.ROOT, - "{create:{name:%s, config:%s, numShards:%s, tlogReplicas:%s}}", + "{\"name\": \"%s\", \"config\": \"%s\", \"numShards\": %s, \"tlogReplicas\": %s}", collectionName, "conf", 2, // numShards diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java index 690642804ee..232475ab6ef 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java @@ -16,6 +16,8 @@ */ package org.apache.solr.cloud.api.collections; +import static org.hamcrest.Matchers.containsString; + import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -28,6 +30,7 @@ import org.apache.lucene.tests.util.TestUtil; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -40,6 +43,7 @@ import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.embedded.JettySolrRunner; +import org.hamcrest.MatcherAssert; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -274,12 +278,13 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase { reloadCollectionRequest.processAsync( "repeatedId", clients[random().nextInt(clients.length)]); numSuccess.incrementAndGet(); - } catch (SolrServerException e) { + } catch (SolrServerException | BaseHttpSolrClient.RemoteSolrException e) { if (log.isInfoEnabled()) { log.info("Exception during collection reloading, we were waiting for one: ", e); } - assertEquals( - "Task with the same requestid already exists. (repeatedId)", e.getMessage()); + MatcherAssert.assertThat( + e.getMessage(), + containsString("Task with the same requestid already exists. (repeatedId)")); numFailure.incrementAndGet(); } catch (IOException e) { throw new RuntimeException(); diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java index 57a952f5f7a..bf4be82d6b7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java @@ -16,6 +16,8 @@ */ package org.apache.solr.cloud.api.collections; +import static org.hamcrest.Matchers.containsString; + import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.Arrays; @@ -23,6 +25,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.RequestStatusState; @@ -31,6 +34,7 @@ import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CommonAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; +import org.hamcrest.MatcherAssert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -159,20 +163,23 @@ public class TestRequestStatusCollectionAPI extends BasicDistributedZkTest { assertEquals("found [1002] in failed tasks", message); - params = new ModifiableSolrParams(); - params.set(CollectionParams.ACTION, CollectionParams.CollectionAction.CREATE.toString()); - params.set("name", "collection3"); - params.set("numShards", 1); - params.set("replicationFactor", 1); - params.set("collection.configName", "conf1"); - params.set(CommonAdminParams.ASYNC, "1002"); - try { - r = sendRequest(params); - } catch (SolrServerException | IOException e) { - log.error("error sending request", e); - } - - assertEquals("Task with the same requestid already exists. (1002)", r.get("error")); + final var duplicateRequestIdParams = new ModifiableSolrParams(); + duplicateRequestIdParams.set( + CollectionParams.ACTION, CollectionParams.CollectionAction.CREATE.toString()); + duplicateRequestIdParams.set("name", "collection3"); + duplicateRequestIdParams.set("numShards", 1); + duplicateRequestIdParams.set("replicationFactor", 1); + duplicateRequestIdParams.set("collection.configName", "conf1"); + duplicateRequestIdParams.set(CommonAdminParams.ASYNC, "1002"); + + final BaseHttpSolrClient.RemoteSolrException thrown = + expectThrows( + BaseHttpSolrClient.RemoteSolrException.class, + () -> { + sendRequest(duplicateRequestIdParams); + }); + MatcherAssert.assertThat( + thrown.getMessage(), containsString("Task with the same requestid already exists. (1002)")); } @SuppressWarnings("unchecked") diff --git a/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java b/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java index 984346553e7..02dc1cf7df8 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java @@ -95,7 +95,6 @@ public class TestApiFramework extends SolrTestCaseJ4 { String fullPath = "/collections/hello/shards"; Api api = V2HttpCall.getApiInfo(containerHandlers, fullPath, "POST", fullPath, parts); assertNotNull(api); - assertConditions(api.getSpec(), Map.of("/methods[0]", "POST", "/commands/create", NOT_NULL)); assertEquals("hello", parts.get("collection")); parts = new HashMap<>(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java b/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java index d00e8e69bd1..3dd7182fb7a 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java @@ -88,35 +88,6 @@ public class TestCollectionAPIs extends SolrTestCaseJ4 { apiBag.registerObject(clusterAPI); apiBag.registerObject(clusterAPI.commands); } - // test a simple create collection call - compareOutput( - apiBag, - "/collections", - POST, - "{create:{name:'newcoll', config:'schemaless', numShards:2, replicationFactor:2 }}", - "{name:newcoll, fromApi:'true', replicationFactor:'2', nrtReplicas:'2', collection.configName:schemaless, numShards:'2', operation:create}"); - - compareOutput( - apiBag, - "/collections", - POST, - "{create:{name:'newcoll', config:'schemaless', numShards:2, nrtReplicas:2 }}", - "{name:newcoll, fromApi:'true', nrtReplicas:'2', replicationFactor:'2', collection.configName:schemaless, numShards:'2', operation:create}"); - - compareOutput( - apiBag, - "/collections", - POST, - "{create:{name:'newcoll', config:'schemaless', numShards:2, nrtReplicas:2, tlogReplicas:2, pullReplicas:2 }}", - "{name:newcoll, fromApi:'true', nrtReplicas:'2', replicationFactor:'2', tlogReplicas:'2', pullReplicas:'2', collection.configName:schemaless, numShards:'2', operation:create}"); - - // test a create collection operation with custom properties - compareOutput( - apiBag, - "/collections", - POST, - "{create:{name:'newcoll', config:'schemaless', numShards:2, replicationFactor:2, properties:{prop1:'prop1val', prop2: prop2val} }}", - "{name:newcoll, fromApi:'true', replicationFactor:'2', nrtReplicas:'2', collection.configName:schemaless, numShards:'2', operation:create, property.prop1:prop1val, property.prop2:prop2val}"); compareOutput( apiBag, diff --git a/solr/core/src/test/org/apache/solr/handler/admin/V2CollectionsAPIMappingTest.java b/solr/core/src/test/org/apache/solr/handler/admin/V2CollectionsAPIMappingTest.java index 68ca362ee05..a9bb20b999f 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/V2CollectionsAPIMappingTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/V2CollectionsAPIMappingTest.java @@ -28,11 +28,9 @@ import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CommonAdminParams; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CoreAdminParams; -import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.backup.BackupManager; import org.apache.solr.handler.admin.api.CreateAliasAPI; -import org.apache.solr.handler.admin.api.CreateCollectionAPI; import org.apache.solr.handler.admin.api.RestoreCollectionAPI; import org.junit.Test; @@ -54,7 +52,6 @@ public class V2CollectionsAPIMappingTest extends V2ApiMappingTest<CollectionsHan @Override public void populateApiBag() { - apiBag.registerObject(new CreateCollectionAPI(getRequestHandler())); apiBag.registerObject(new CreateAliasAPI(getRequestHandler())); apiBag.registerObject(new RestoreCollectionAPI(getRequestHandler())); } @@ -69,52 +66,6 @@ public class V2CollectionsAPIMappingTest extends V2ApiMappingTest<CollectionsHan return false; } - @Test - public void testCreateCollectionAllProperties() throws Exception { - final SolrParams v1Params = - captureConvertedV1Params( - "/collections", - "POST", - "{'create': {" - + "'name': 'techproducts', " - + "'config':'_default', " - + "'router': {'name': 'composite', 'field': 'routeField', 'foo': 'bar'}, " - + "'shards': 'customShardName,anotherCustomShardName', " - + "'replicationFactor': 3," - + "'nrtReplicas': 1, " - + "'tlogReplicas': 1, " - + "'pullReplicas': 1, " - + "'nodeSet': ['localhost:8983_solr', 'localhost:7574_solr']," - + "'shuffleNodes': true," - + "'properties': {'foo': 'bar', 'foo2': 'bar2'}, " - + "'async': 'requestTrackingId', " - + "'waitForFinalState': false, " - + "'perReplicaState': false," - + "'numShards': 1}}"); - - assertEquals(CollectionParams.CollectionAction.CREATE.lowerName, v1Params.get(ACTION)); - assertEquals("techproducts", v1Params.get(CommonParams.NAME)); - assertEquals("_default", v1Params.get(CollectionAdminParams.COLL_CONF)); - assertEquals("composite", v1Params.get("router.name")); - assertEquals("routeField", v1Params.get("router.field")); - assertEquals("bar", v1Params.get("router.foo")); - assertEquals("customShardName,anotherCustomShardName", v1Params.get(ShardParams.SHARDS)); - assertEquals(3, v1Params.getPrimitiveInt(ZkStateReader.REPLICATION_FACTOR)); - assertEquals(1, v1Params.getPrimitiveInt(ZkStateReader.NRT_REPLICAS)); - assertEquals(1, v1Params.getPrimitiveInt(ZkStateReader.TLOG_REPLICAS)); - assertEquals(1, v1Params.getPrimitiveInt(ZkStateReader.PULL_REPLICAS)); - assertEquals( - "localhost:8983_solr,localhost:7574_solr", - v1Params.get(CollectionAdminParams.CREATE_NODE_SET_PARAM)); - assertTrue(v1Params.getPrimitiveBool(CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM)); - assertEquals("bar", v1Params.get("property.foo")); - assertEquals("bar2", v1Params.get("property.foo2")); - assertEquals("requestTrackingId", v1Params.get(CommonAdminParams.ASYNC)); - assertFalse(v1Params.getPrimitiveBool(CommonAdminParams.WAIT_FOR_FINAL_STATE)); - assertFalse(v1Params.getPrimitiveBool(CollectionAdminParams.PER_REPLICA_STATE)); - assertEquals(1, v1Params.getPrimitiveInt(CollectionAdminParams.NUM_SHARDS)); - } - @Test public void testCreateAliasAllProperties() throws Exception { final SolrParams v1Params = diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java new file mode 100644 index 00000000000..c5eb27f0835 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; +import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; +import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.NUM_SLICES; +import static org.apache.solr.common.cloud.DocCollection.CollectionStateProps.SHARDS; +import static org.apache.solr.common.params.CollectionAdminParams.ALIAS; +import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; +import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM; +import static org.apache.solr.common.params.CollectionAdminParams.NRT_REPLICAS; +import static org.apache.solr.common.params.CollectionAdminParams.NUM_SHARDS; +import static org.apache.solr.common.params.CollectionAdminParams.PER_REPLICA_STATE; +import static org.apache.solr.common.params.CollectionAdminParams.PULL_REPLICAS; +import static org.apache.solr.common.params.CollectionAdminParams.REPLICATION_FACTOR; +import static org.apache.solr.common.params.CollectionAdminParams.TLOG_REPLICAS; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; +import static org.apache.solr.common.params.CoreAdminParams.NAME; + +import java.util.List; +import java.util.Map; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.Test; + +/** Unit tests for {@link CreateCollectionAPI}. */ +public class CreateCollectionAPITest extends SolrTestCaseJ4 { + + @Test + public void testReportsErrorIfRequestBodyMissing() { + final SolrException thrown = + expectThrows( + SolrException.class, + () -> { + final var api = new CreateCollectionAPI(null, null, null); + api.createCollection(null); + }); + + assertEquals(400, thrown.code()); + assertEquals("Request body is missing but required", thrown.getMessage()); + } + + @Test + public void testReportsErrorIfReplicationFactorAndNrtReplicasConflict() { + // Valid request body... + final var requestBody = new CreateCollectionAPI.CreateCollectionRequestBody(); + requestBody.name = "someName"; + requestBody.config = "someConfig"; + // ...except for a replicationFactor and nrtReplicas conflicting + requestBody.replicationFactor = 123; + requestBody.nrtReplicas = 321; + + final SolrException thrown = + expectThrows( + SolrException.class, + () -> { + requestBody.validate(); + }); + + assertEquals(400, thrown.code()); + assertEquals( + "Cannot specify both replicationFactor and nrtReplicas as they mean the same thing", + thrown.getMessage()); + } + + @Test + public void testReportsErrorIfCollectionNameInvalid() { + final var requestBody = new CreateCollectionAPI.CreateCollectionRequestBody(); + requestBody.name = "$invalid@collection+name"; + requestBody.config = "someConfig"; + + final SolrException thrown = + expectThrows( + SolrException.class, + () -> { + requestBody.validate(); + }); + + assertEquals(400, thrown.code()); + assertTrue( + "Expected invalid collection name to be rejected", + thrown.getMessage().contains("Invalid collection: [$invalid@collection+name]")); + } + + @Test + public void testReportsErrorIfShardNamesInvalid() { + // Valid request body... + final var requestBody = new CreateCollectionAPI.CreateCollectionRequestBody(); + requestBody.name = "someName"; + requestBody.config = "someConfig"; + // ...except for a bad shard name + requestBody.shardNames = List.of("good-name", "bad;name"); + + final SolrException thrown = + expectThrows( + SolrException.class, + () -> { + requestBody.validate(); + }); + + assertEquals(400, thrown.code()); + assertTrue( + "Expected invalid shard name to be rejected", + thrown.getMessage().contains("Invalid shard: [bad;name]")); + } + + @Test + public void testCreateRemoteMessageAllProperties() { + final var requestBody = new CreateCollectionAPI.CreateCollectionRequestBody(); + requestBody.name = "someName"; + requestBody.replicationFactor = 123; + requestBody.config = "someConfig"; + requestBody.numShards = 456; + requestBody.shardNames = List.of("shard1", "shard2"); + requestBody.pullReplicas = 789; + requestBody.tlogReplicas = 987; + requestBody.waitForFinalState = false; + requestBody.perReplicaState = true; + requestBody.alias = "someAliasName"; + requestBody.properties = Map.of("propName", "propValue"); + requestBody.async = "someAsyncId"; + requestBody.router = new CreateCollectionAPI.RouterProperties(); + requestBody.router.name = "someRouterName"; + requestBody.router.field = "someField"; + requestBody.nodeSet = List.of("node1", "node2"); + requestBody.shuffleNodes = false; + + final var remoteMessage = CreateCollectionAPI.createRemoteMessage(requestBody).getProperties(); + + assertEquals("create", remoteMessage.get(QUEUE_OPERATION)); + assertEquals("true", remoteMessage.get("fromApi")); + assertEquals("someName", remoteMessage.get(NAME)); + assertEquals(123, remoteMessage.get(REPLICATION_FACTOR)); + assertEquals("someConfig", remoteMessage.get(COLL_CONF)); + assertEquals(456, remoteMessage.get(NUM_SLICES)); + assertEquals("shard1,shard2", remoteMessage.get(SHARDS)); + assertEquals(789, remoteMessage.get(PULL_REPLICAS)); + assertEquals(987, remoteMessage.get(TLOG_REPLICAS)); + assertEquals(123, remoteMessage.get(NRT_REPLICAS)); // replicationFactor value used + assertEquals(false, remoteMessage.get(WAIT_FOR_FINAL_STATE)); + assertEquals(true, remoteMessage.get(PER_REPLICA_STATE)); + assertEquals("someAliasName", remoteMessage.get(ALIAS)); + assertEquals("propValue", remoteMessage.get("property.propName")); + assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + assertEquals("someRouterName", remoteMessage.get("router.name")); + assertEquals("someField", remoteMessage.get("router.field")); + assertEquals("node1,node2", remoteMessage.get(CREATE_NODE_SET)); + assertEquals(false, remoteMessage.get(CREATE_NODE_SET_SHUFFLE_PARAM)); + } + + @Test + public void testNoReplicaCreationMessage() { + final var requestBody = new CreateCollectionAPI.CreateCollectionRequestBody(); + requestBody.name = "someName"; + requestBody.createReplicas = false; + + final var remoteMessage = CreateCollectionAPI.createRemoteMessage(requestBody).getProperties(); + + assertEquals("create", remoteMessage.get(QUEUE_OPERATION)); + assertEquals("someName", remoteMessage.get(NAME)); + assertEquals("EMPTY", remoteMessage.get(CREATE_NODE_SET)); + } + + @Test + public void testV1ParamsCanBeConvertedIntoV2RequestBody() { + final ModifiableSolrParams solrParams = new ModifiableSolrParams(); + solrParams.set(NAME, "someName"); + solrParams.set(NUM_SHARDS, 123); + solrParams.set(SHARDS, "shard1,shard2"); + solrParams.set(REPLICATION_FACTOR, 123); + solrParams.set(TLOG_REPLICAS, 456); + solrParams.set(PULL_REPLICAS, 789); + solrParams.set(CREATE_NODE_SET, "node1,node2"); + solrParams.set(CREATE_NODE_SET_SHUFFLE_PARAM, true); + solrParams.set(COLL_CONF, "someConfig"); + solrParams.set(PER_REPLICA_STATE, true); + solrParams.set("router.name", "someRouterName"); + solrParams.set("router.field", "someField"); + solrParams.set("property.somePropName", "somePropValue"); + + final var v2RequestBody = + CreateCollectionAPI.CreateCollectionRequestBody.fromV1Params(solrParams, true); + + assertEquals("someName", v2RequestBody.name); + assertEquals(Integer.valueOf(123), v2RequestBody.numShards); + assertEquals(List.of("shard1", "shard2"), v2RequestBody.shardNames); + assertEquals(Integer.valueOf(123), v2RequestBody.replicationFactor); + assertEquals(Integer.valueOf(456), v2RequestBody.tlogReplicas); + assertEquals(Integer.valueOf(789), v2RequestBody.pullReplicas); + assertEquals(List.of("node1", "node2"), v2RequestBody.nodeSet); + assertNull(v2RequestBody.createReplicas); + assertEquals(Boolean.TRUE, v2RequestBody.shuffleNodes); + assertEquals("someConfig", v2RequestBody.config); + assertEquals(Boolean.TRUE, v2RequestBody.perReplicaState); + assertNotNull(v2RequestBody.router); + assertEquals("someRouterName", v2RequestBody.router.name); + assertEquals("someField", v2RequestBody.router.field); + assertNotNull(v2RequestBody.properties); + assertEquals("somePropValue", v2RequestBody.properties.get("somePropName")); + } + + @Test + public void testV1ParamsWithEmptyNodeSetCanBeConvertedIntoV2RequestBody() { + final ModifiableSolrParams solrParams = new ModifiableSolrParams(); + solrParams.set(NAME, "someName"); + solrParams.set(CREATE_NODE_SET, "EMPTY"); + + final var v2RequestBody = + CreateCollectionAPI.CreateCollectionRequestBody.fromV1Params(solrParams, true); + + assertEquals("someName", v2RequestBody.name); + assertEquals(Boolean.FALSE, v2RequestBody.createReplicas); + assertNull(v2RequestBody.nodeSet); + } +} diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc index 60ba5fc2e59..2110a8fc2c4 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc @@ -54,11 +54,9 @@ With the v2 API, the `create` command is provided as part of the JSON data that ---- curl -X POST http://localhost:8983/api/collections -H 'Content-Type: application/json' -d ' { - "create": { - "name": "techproducts_v2", - "config": "techproducts", - "numShards": 1 - } + "name": "techproducts_v2", + "config": "techproducts", + "numShards": 1 } ' ---- @@ -109,15 +107,17 @@ For more information, see also the section xref:solrcloud-shards-indexing.adoc#d The number of shards to be created as part of the collection. This is a required parameter when the `router.name` is `compositeId`. -`shards`:: +`shards` (v1), `shardNames` (v2):: + [%autowidth,frame=none] |=== |Optional |Default: none |=== + -A comma separated list of shard names, e.g., `shard-x,shard-y,shard-z`. -This is a required parameter when the `router.name` is `implicit`. +The shard names use when creating this collection. +This is a required parameter when `router.name` is `implicit`. +For v1 requests, these are provided as a single comma-delimited query parameter, e.g., `shard-x,shard-y,shard-z`. +For v2 requests shard names are provided as a list of values in the request body, e.g., `["shard-x", "shard-y", "shard-z"]` `replicationFactor`:: + @@ -174,11 +174,26 @@ See the section xref:solrcloud-shards-indexing.adoc#types-of-replicas[Types of R |=== + Allows defining the nodes to spread the new collection across. -The format is a comma-separated list of node_names, such as `localhost:8983_solr,localhost:8984_solr,localhost:8985_solr`. +For v1 requests, node names are provided as a single comma-separated list, such as `localhost:8983_solr,localhost:8984_solr,localhost:8985_solr`. +For v2 requests, node names are provided as a list of individual values, such as `["localhost:8983_solr", "localhost:7574_solr"]`. ++ +If not provided, the CREATE operation will use all live nodes in the cluster as its node set. ++ +Alternatively, v1 requests allow the special value `EMPTY` to initially create no shard-replica within the new collection and then later use the xref:replica-management.adoc#addreplica[ADDREPLICA] operation to add shard-replicas when and where required. +v2 requests may use this same functionality via the `createReplicas` boolean parameter. + +`createReplicas` (v2):: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: true +|=== + -If not provided, the CREATE operation will create shard-replicas spread across all live Solr nodes. +Controls whether Solr creates replicas for each shard as a part of the collection-creation operation. +(Replicas can always be created later using the xref:replica-management.adoc#addreplica[ADDREPLICA] operation.) + -Alternatively, use the special value of `EMPTY` to initially create no shard-replica within the new collection and then later use the xref:replica-management.adoc#addreplica[ADDREPLICA] operation to add shard-replicas when and where required. +Only available on v2 requests. +v1 requests wishing to defer replica-creation may do so my providing the `EMPTY` flag value to the `createNodeSet` parameter. `createNodeSet.shuffle` (v1), `shuffleNodes` (v2):: + diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/CreatePayload.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/CreatePayload.java deleted file mode 100644 index f19a7862fed..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/CreatePayload.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.request.beans; - -import java.util.List; -import java.util.Map; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.util.ReflectMapWriter; - -public class CreatePayload implements ReflectMapWriter { - @JsonProperty(required = true) - public String name; - - @JsonProperty public String config; - - @JsonProperty public Map<String, Object> router; - - @JsonProperty public Integer numShards; - - @JsonProperty public String shards; - - @JsonProperty public Integer replicationFactor; - - @JsonProperty public Integer nrtReplicas; - - @JsonProperty public Integer tlogReplicas; - - @JsonProperty public Integer pullReplicas; - - @JsonProperty public List<String> nodeSet; - - @JsonProperty public Boolean shuffleNodes; - - @JsonProperty public Map<String, Object> properties; - - @JsonProperty public String async; - - @JsonProperty public Boolean waitForFinalState; - - @JsonProperty public Boolean perReplicaState; -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/V2ApiConstants.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/V2ApiConstants.java index 6a8580b1a0f..d499d75c890 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/V2ApiConstants.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/V2ApiConstants.java @@ -32,6 +32,9 @@ public class V2ApiConstants { /** Parameter name for the configset used by a collection */ public static final String CONFIG = "config"; + /** Parameter for explicitly providing a name for each shard during collection creation. */ + public static final String SHARD_NAMES = "shardNames"; + /** Property controlling whether 'nodeSet' should be shuffled before use. */ public static final String SHUFFLE_NODES = "shuffleNodes"; diff --git a/solr/solrj/src/java/org/apache/solr/common/util/CollectionUtil.java b/solr/solrj/src/java/org/apache/solr/common/util/CollectionUtil.java index 8d9a96c2574..92282e1ccfc 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/CollectionUtil.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/CollectionUtil.java @@ -16,12 +16,13 @@ */ package org.apache.solr.common.util; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; /** - * Methods for creating collections with exact sizes. + * Methods for creating collections with exact sizes, and other convenience methods * * @lucene.internal */ @@ -60,4 +61,12 @@ public final class CollectionUtil { // Replace with HashSet.newHashSet when Solr moves to minimum jdk19 return new HashSet<>((int) (size / 0.75f) + 1); } + + public static boolean isEmpty(Collection<?> collection) { + return collection == null || collection.isEmpty(); + } + + public static boolean isNotEmpty(Collection<?> collection) { + return !isEmpty(collection); + } } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java index 4ddeb81059b..d29d6b23153 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java @@ -1205,7 +1205,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase { new V2Request.Builder("/collections") .withMethod(POST) .withPayload( - "{create: {name: perReplicaState_testv2, config : conf, numShards : 2, nrtReplicas : 2, perReplicaState : true, maxShardsPerNode : 5}}") + "{\"name\": \"perReplicaState_testv2\", \"config\" : \"conf\", \"numShards\" : 2, \"nrtReplicas\" : 2, \"perReplicaState\" : true}") .build() .process(cluster.getSolrClient()); cluster.waitForActiveCollection(testCollection, 2, 4); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java index ab8c81c7f24..bec4705eb1e 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java @@ -24,7 +24,6 @@ import java.util.List; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.SolrCloudTestCase; @@ -103,12 +102,10 @@ public class TestV2Request extends SolrCloudTestCase { .withMethod(SolrRequest.METHOD.POST) .withPayload( "{" - + " 'create' : {" - + " 'name' : 'test'," - + " 'numShards' : 2," - + " 'replicationFactor' : 2," - + " 'config' : 'config'" - + " }" + + " \"name\" : \"test\"," + + " \"numShards\" : 2," + + " \"replicationFactor\" : 2," + + " \"config\" : \"config\"" + "}" + "/* ignore comment*/") .build()); @@ -128,29 +125,6 @@ public class TestV2Request extends SolrCloudTestCase { client, new V2Request.Builder("/collections/test").withMethod(SolrRequest.METHOD.DELETE).build()); NamedList<Object> res = client.request(new V2Request.Builder("/collections").build()); - - // TODO: this is not guaranteed now - beast test if you try to fix - // List collections = (List) res.get("collections"); - // assertFalse( collections.contains("test")); - try { - NamedList<Object> res1 = - client.request( - new V2Request.Builder("/collections") - .withMethod(SolrRequest.METHOD.POST) - .withPayload( - "{" - + " 'create' : {" - + " 'name' : 'jsontailtest'," - + " 'numShards' : 2," - + " 'replicationFactor' : 2," - + " 'config' : 'config'" - + " }" - + "}" - + ", 'something':'bogus'") - .build()); - assertFalse("The request failed", res1.get("responseHeader").toString().contains("status=0")); - } catch (BaseHttpSolrClient.RemoteExecutionException itsOk) { - } } public void testV2Forwarding() throws Exception { @@ -161,12 +135,10 @@ public class TestV2Request extends SolrCloudTestCase { .withMethod(SolrRequest.METHOD.POST) .withPayload( "{" - + " 'create' : {" - + " 'name' : 'v2forward'," - + " 'numShards' : 1," - + " 'replicationFactor' : 1," - + " 'config' : 'config'" - + " }" + + " \"name\" : \"v2forward\"," + + " \"numShards\" : 1," + + " \"replicationFactor\" : 1," + + " \"config\" : \"config\"" + "}") .build()); diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/PerReplicaStatesIntegrationTest.java b/solr/solrj/src/test/org/apache/solr/common/cloud/PerReplicaStatesIntegrationTest.java index dac1de68a6d..2edfe3b7d54 100644 --- a/solr/solrj/src/test/org/apache/solr/common/cloud/PerReplicaStatesIntegrationTest.java +++ b/solr/solrj/src/test/org/apache/solr/common/cloud/PerReplicaStatesIntegrationTest.java @@ -105,7 +105,7 @@ public class PerReplicaStatesIntegrationTest extends SolrCloudTestCase { new V2Request.Builder("/collections") .withMethod(POST) .withPayload( - "{create: {name: perReplicaState_testv2, config : conf, numShards : 2, nrtReplicas : 2, perReplicaState : true, maxShardsPerNode : 5}}") + "{\"name\": \"perReplicaState_testv2\", \"config\" : \"conf\", \"numShards\" : 2, \"nrtReplicas\" : 2, \"perReplicaState\" : true}") .build() .process(cluster.getSolrClient()); cluster.waitForActiveCollection(testCollection, 2, 4);
