This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 96552ae6a NUTCH-2932 Create OpenAPI specification for Nutch 1.x REST
API (#896)
96552ae6a is described below
commit 96552ae6a473fa2736fc280710fec676a4d78c80
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Sun Apr 12 15:21:33 2026 -0700
NUTCH-2932 Create OpenAPI specification for Nutch 1.x REST API (#896)
---
.github/workflows/master-build.yml | 23 +
.spectral.yaml | 29 +
openapi.yaml | 1326 ++++++++++++++++++++++++++++++++++++
3 files changed, 1378 insertions(+)
diff --git a/.github/workflows/master-build.yml
b/.github/workflows/master-build.yml
index 76d98db4e..31b06236a 100644
--- a/.github/workflows/master-build.yml
+++ b/.github/workflows/master-build.yml
@@ -85,6 +85,29 @@ jobs:
if: ${{ env.UNKNOWN_LICENSES != '0 Unknown Licenses' }}
run: exit 1
+ openapi-lint:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v5
+ - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
+ id: filter
+ with:
+ filters: |
+ openapi:
+ - 'openapi.yaml'
+ - '.spectral.yaml'
+ - name: Set up Node.js
+ if: steps.filter.outputs.openapi == 'true'
+ uses: actions/setup-node@v4
+ with:
+ node-version: '18'
+ - name: Install IBM OpenAPI Validator
+ if: steps.filter.outputs.openapi == 'true'
+ run: npm install --no-save ibm-openapi-validator
+ - name: Validate OpenAPI specification
+ if: steps.filter.outputs.openapi == 'true'
+ run: ./node_modules/.bin/lint-openapi openapi.yaml
+
# Build verification with Java bytecode target matrix
# Verifies bytecode compatibility for both Java 11 and Java 17 targets
build:
diff --git a/.spectral.yaml b/.spectral.yaml
new file mode 100644
index 000000000..6e42544a1
--- /dev/null
+++ b/.spectral.yaml
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Custom Spectral ruleset for the Apache Nutch REST API.
+# Extends the IBM Cloud Validation Ruleset and disables rules that conflict
+# with the existing Nutch Java wire format (camelCase properties, UPPER_CASE
+# enums, camelCase parameters, bare-array responses, map-typed responses,
+# and consecutive path parameter segments).
+
+extends: "@ibm-cloud/openapi-ruleset"
+rules:
+ ibm-property-casing-convention: off
+ ibm-enum-casing-convention: off
+ ibm-parameter-casing-convention: off
+ ibm-no-array-responses: off
+ ibm-accept-and-return-models: off
+ ibm-no-consecutive-path-parameter-segments: off
diff --git a/openapi.yaml b/openapi.yaml
new file mode 100644
index 000000000..8e3566b0f
--- /dev/null
+++ b/openapi.yaml
@@ -0,0 +1,1326 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+openapi: "3.1.2"
+
+info:
+ title: Apache Nutch REST API
+ description: >-
+ REST API for managing Apache Nutch crawl jobs, configurations, seed lists,
+ database queries, and data readers.
+ version: "1.0.0"
+ license:
+ name: Apache 2.0
+ identifier: Apache-2.0
+ contact:
+ name: Apache Nutch
+ url: https://nutch.apache.org
+
+servers:
+ - url: "{protocol}://localhost:{port}"
+ description: Nutch REST server
+ variables:
+ protocol:
+ default: http
+ enum:
+ - http
+ - https
+ description: The protocol used to access the Nutch server.
+ port:
+ default: "8081"
+ description: >-
+ The port the Nutch server listens on. Configurable via the --port
+ command-line argument.
+
+security:
+ - basicAuth: []
+
+tags:
+ - name: Admin
+ description: Server administration operations
+ - name: Configuration
+ description: Manage Nutch configurations
+ - name: Job
+ description: Manage crawl jobs
+ - name: Database
+ description: Query the CrawlDB and FetchDB
+ - name: Seed
+ description: Manage seed URL lists
+ - name: Reader
+ description: Read sequence files and webgraph data
+ - name: Services
+ description: Auxiliary service operations such as CommonCrawl data dumps
+
+paths:
+ # ---------------------------------------------------------------------------
+ # Admin
+ # ---------------------------------------------------------------------------
+ /admin/:
+ get:
+ tags:
+ - Admin
+ summary: Get server status
+ description: >-
+ Returns the current status of the Nutch server including start date,
+ known configurations, all jobs, and currently running jobs.
+ operationId: getServerStatus
+ responses:
+ "200":
+ description: Server status retrieved successfully.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/NutchServerInfo"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /admin/stop:
+ get:
+ tags:
+ - Admin
+ summary: Stop the Nutch server
+ description: >-
+ Initiates a graceful shutdown of the Nutch server. If jobs are still
+ running and force is not set, the server will refuse to stop.
+ operationId: stopServer
+ parameters:
+ - name: force
+ in: query
+ required: false
+ description: >-
+ If true, kills any running jobs before stopping the server.
+ schema:
+ type: boolean
+ default: false
+ responses:
+ "200":
+ description: Shutdown status message.
+ content:
+ application/json:
+ schema:
+ type: string
+ examples:
+ stopping:
+ value: "Stopping in server on port 8081"
+ busy:
+ value: "Jobs still running -- Cannot stop server now"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ # ---------------------------------------------------------------------------
+ # Configuration
+ # ---------------------------------------------------------------------------
+ /config/:
+ get:
+ tags:
+ - Configuration
+ summary: List all configuration IDs
+ description: Returns the set of all known configuration identifiers.
+ operationId: getConfigs
+ responses:
+ "200":
+ description: A JSON array of configuration ID strings.
+ content:
+ application/json:
+ schema:
+ type: array
+ items:
+ type: string
+ uniqueItems: true
+ example:
+ - default
+ - my-custom-config
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /config/{configId}:
+ get:
+ tags:
+ - Configuration
+ summary: Get configuration properties
+ description: Returns all key-value properties for the specified
configuration.
+ operationId: getConfig
+ parameters:
+ - $ref: "#/components/parameters/configId"
+ responses:
+ "200":
+ description: A JSON object of configuration property key-value pairs.
+ content:
+ application/json:
+ schema:
+ type: object
+ additionalProperties:
+ type: string
+ example:
+ http.agent.name: "NutchBot"
+ http.robots.agents: "NutchBot,*"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "404":
+ $ref: "#/components/responses/NotFound"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+ delete:
+ tags:
+ - Configuration
+ summary: Delete a configuration
+ description: >-
+ Removes the specified configuration from the list of known
+ configurations.
+ operationId: deleteConfig
+ parameters:
+ - $ref: "#/components/parameters/configId"
+ responses:
+ "204":
+ description: Configuration deleted successfully.
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "404":
+ $ref: "#/components/responses/NotFound"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /config/{configId}/{propertyId}:
+ get:
+ tags:
+ - Configuration
+ summary: Get a single configuration property
+ description: >-
+ Returns the value of a specific property within the given
+ configuration.
+ operationId: getProperty
+ parameters:
+ - $ref: "#/components/parameters/configId"
+ - $ref: "#/components/parameters/propertyId"
+ responses:
+ "200":
+ description: The property value as plain text.
+ content:
+ text/plain:
+ schema:
+ type: string
+ example: "NutchBot"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "404":
+ $ref: "#/components/responses/NotFound"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+ put:
+ tags:
+ - Configuration
+ summary: Update a configuration property
+ description: >-
+ Adds or updates the value of a property in the specified
+ configuration.
+ operationId: updateProperty
+ parameters:
+ - $ref: "#/components/parameters/configId"
+ - $ref: "#/components/parameters/propertyId"
+ requestBody:
+ required: true
+ description: The new property value as plain text.
+ content:
+ text/plain:
+ schema:
+ type: string
+ example: "MyNewBot"
+ responses:
+ "200":
+ description: Property updated successfully.
+ "400":
+ $ref: "#/components/responses/BadRequest"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /config/create:
+ post:
+ tags:
+ - Configuration
+ summary: Create a new configuration
+ description: >-
+ Creates a new Nutch configuration with the specified parameters.
+ Returns the configuration ID on success.
+ operationId: createConfig
+ requestBody:
+ required: true
+ description: The configuration to create.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/NutchConfig"
+ responses:
+ "200":
+ description: Configuration created. Returns the configuration ID.
+ content:
+ text/plain:
+ schema:
+ type: string
+ example: "my-custom-config"
+ "400":
+ $ref: "#/components/responses/BadRequest"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ # ---------------------------------------------------------------------------
+ # Job
+ # ---------------------------------------------------------------------------
+ /job/:
+ get:
+ tags:
+ - Job
+ summary: List all jobs
+ description: >-
+ Returns job history for all jobs or filtered by crawl ID, regardless
+ of job state.
+ operationId: getJobs
+ parameters:
+ - name: crawlId
+ in: query
+ required: false
+ description: Optional crawl ID to filter jobs by.
+ schema:
+ type: string
+ responses:
+ "200":
+ description: A JSON array of job information objects.
+ content:
+ application/json:
+ schema:
+ type: array
+ items:
+ $ref: "#/components/schemas/JobInfo"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /job/{id}:
+ get:
+ tags:
+ - Job
+ summary: Get job info
+ description: Returns detailed information for a specific job.
+ operationId: getJobInfo
+ parameters:
+ - $ref: "#/components/parameters/jobId"
+ - name: crawlId
+ in: query
+ required: false
+ description: The crawl ID associated with the job.
+ schema:
+ type: string
+ responses:
+ "200":
+ description: Job details.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/JobInfo"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "404":
+ $ref: "#/components/responses/NotFound"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /job/{id}/stop:
+ get:
+ tags:
+ - Job
+ summary: Stop a running job
+ description: Attempts to gracefully stop a running job.
+ operationId: stopJob
+ parameters:
+ - $ref: "#/components/parameters/jobId"
+ - name: crawlId
+ in: query
+ required: false
+ description: The crawl ID associated with the job.
+ schema:
+ type: string
+ responses:
+ "200":
+ description: Whether the job was successfully stopped.
+ content:
+ application/json:
+ schema:
+ type: boolean
+ example: true
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "404":
+ $ref: "#/components/responses/NotFound"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /job/{id}/abort:
+ get:
+ tags:
+ - Job
+ summary: Abort a job
+ description: >-
+ Forcefully aborts a job. Unlike stop, this kills the job immediately.
+ operationId: abortJob
+ parameters:
+ - $ref: "#/components/parameters/jobId"
+ - name: crawlId
+ in: query
+ required: false
+ description: The crawl ID associated with the job.
+ schema:
+ type: string
+ responses:
+ "200":
+ description: Whether the job was successfully aborted.
+ content:
+ application/json:
+ schema:
+ type: boolean
+ example: true
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "404":
+ $ref: "#/components/responses/NotFound"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /job/create:
+ post:
+ tags:
+ - Job
+ summary: Create a new job
+ description: >-
+ Creates and enqueues a new Nutch job (e.g., inject, generate, fetch,
+ parse, updatedb, index).
+ operationId: createJob
+ requestBody:
+ required: true
+ description: The job configuration specifying type, crawl ID, and
arguments.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/JobConfig"
+ responses:
+ "200":
+ description: Job created. Returns the job information.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/JobInfo"
+ "400":
+ $ref: "#/components/responses/BadRequest"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ # ---------------------------------------------------------------------------
+ # Database
+ # ---------------------------------------------------------------------------
+ /db/crawldb:
+ post:
+ tags:
+ - Database
+ summary: Query the CrawlDB
+ description: >-
+ Executes a query against the Nutch CrawlDB. The type field in the
+ request body determines the operation: stats, dump, topN, or url.
+ The stats and url types return JSON; dump and topN return binary
+ octet-stream data.
+ operationId: readCrawlDb
+ requestBody:
+ required: true
+ description: The database query parameters.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/DbQuery"
+ responses:
+ "200":
+ description: >-
+ Query results. Content type varies by query type: application/json
+ for stats and url queries; application/octet-stream for dump and
+ topN queries.
+ content:
+ application/json:
+ schema:
+ type: object
+ description: >-
+ CrawlDB query result (returned for stats and url query
+ types).
+ application/octet-stream:
+ schema:
+ type: string
+ format: binary
+ description: >-
+ Binary data stream (returned for dump and topN query types).
+ "400":
+ $ref: "#/components/responses/BadRequest"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /db/fetchdb:
+ get:
+ tags:
+ - Database
+ summary: Get FetchDB node information
+ description: >-
+ Returns fetch node database entries for the specified index range.
+ Both from and to default to 0; if to is 0 or exceeds the total
+ number of entries, all entries from the starting index are returned.
+ operationId: fetchDb
+ parameters:
+ - name: from
+ in: query
+ required: false
+ description: Starting index (inclusive). Defaults to 0.
+ schema:
+ type: integer
+ format: int32
+ minimum: 0
+ maximum: 2147483647
+ default: 0
+ - name: to
+ in: query
+ required: false
+ description: Ending index (inclusive). Defaults to 0 (returns all).
+ schema:
+ type: integer
+ format: int32
+ minimum: 0
+ maximum: 2147483647
+ default: 0
+ responses:
+ "200":
+ description: A JSON array of fetch node information objects.
+ content:
+ application/json:
+ schema:
+ type: array
+ items:
+ $ref: "#/components/schemas/FetchNodeDbInfo"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ # ---------------------------------------------------------------------------
+ # Seed
+ # ---------------------------------------------------------------------------
+ /seed/:
+ get:
+ tags:
+ - Seed
+ summary: List all seed lists
+ description: Returns a map of all created seed files keyed by name.
+ operationId: getSeedLists
+ responses:
+ "200":
+ description: A JSON object mapping seed list names to SeedList
objects.
+ content:
+ application/json:
+ schema:
+ type: object
+ additionalProperties:
+ $ref: "#/components/schemas/SeedList"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /seed/create:
+ post:
+ tags:
+ - Seed
+ summary: Create a seed list file
+ description: >-
+ Creates a seed list file from the provided URLs and writes it to
+ HDFS. Returns the path to the created seed file directory.
+ operationId: createSeedFile
+ requestBody:
+ required: true
+ description: The seed list containing URLs to write.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/SeedList"
+ responses:
+ "200":
+ description: Path to the created seed file directory.
+ content:
+ text/plain:
+ schema:
+ type: string
+ example: "seedFiles/seed-1700000000000"
+ "400":
+ $ref: "#/components/responses/BadRequest"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ # ---------------------------------------------------------------------------
+ # Reader
+ # ---------------------------------------------------------------------------
+ /reader/sequence/read:
+ post:
+ tags:
+ - Reader
+ summary: Read a sequence file
+ description: >-
+ Reads key-value pairs from a Hadoop sequence file. Supports reading
+ all rows, a limited number of rows, a row range, or counting the
+ total number of rows.
+ operationId: seqRead
+ parameters:
+ - $ref: "#/components/parameters/nrows"
+ - $ref: "#/components/parameters/start"
+ - $ref: "#/components/parameters/end"
+ - $ref: "#/components/parameters/count"
+ requestBody:
+ required: true
+ description: Reader configuration specifying the file path.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/ReaderConfig"
+ responses:
+ "200":
+ description: >-
+ Sequence file data. Returns application/json when reading rows,
+ or text/plain when count=true.
+ content:
+ application/json:
+ schema:
+ type: array
+ items:
+ type: object
+ text/plain:
+ schema:
+ type: integer
+ format: int32
+ minimum: 0
+ maximum: 2147483647
+ description: Number of rows in the sequence file.
+ "400":
+ $ref: "#/components/responses/BadRequest"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /reader/link:
+ get:
+ tags:
+ - Reader
+ summary: Get link reader schema
+ description: >-
+ Returns the schema describing the fields in link reader responses.
+ operationId: getLinkSchema
+ responses:
+ "200":
+ description: Link reader response schema.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/LinkSchema"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /reader/link/read:
+ post:
+ tags:
+ - Reader
+ summary: Read link objects
+ description: >-
+ Reads link data (LinkDatum) from the Nutch webgraph. Supports
+ reading all rows, a limited number of rows, a row range, or
+ counting the total number of rows.
+ operationId: linkRead
+ parameters:
+ - $ref: "#/components/parameters/nrows"
+ - $ref: "#/components/parameters/start"
+ - $ref: "#/components/parameters/end"
+ - $ref: "#/components/parameters/count"
+ requestBody:
+ required: true
+ description: Reader configuration specifying the file path.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/ReaderConfig"
+ responses:
+ "200":
+ description: >-
+ Link data. Returns application/json when reading rows, or
+ text/plain when count=true.
+ content:
+ application/json:
+ schema:
+ type: array
+ items:
+ type: object
+ text/plain:
+ schema:
+ type: integer
+ format: int32
+ minimum: 0
+ maximum: 2147483647
+ description: Number of link entries.
+ "400":
+ $ref: "#/components/responses/BadRequest"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /reader/node:
+ get:
+ tags:
+ - Reader
+ summary: Get node reader schema
+ description: >-
+ Returns the schema describing the fields in node reader responses.
+ operationId: getNodeSchema
+ responses:
+ "200":
+ description: Node reader response schema.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/NodeSchema"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /reader/node/read:
+ post:
+ tags:
+ - Reader
+ summary: Read node objects
+ description: >-
+ Reads Node objects from the Nutch webgraph. Supports reading all
+ rows, a limited number of rows, a row range, or counting the total
+ number of rows.
+ operationId: nodeRead
+ parameters:
+ - $ref: "#/components/parameters/nrows"
+ - $ref: "#/components/parameters/start"
+ - $ref: "#/components/parameters/end"
+ - $ref: "#/components/parameters/count"
+ requestBody:
+ required: true
+ description: Reader configuration specifying the file path.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/ReaderConfig"
+ responses:
+ "200":
+ description: >-
+ Node data. Returns application/json when reading rows, or
+ text/plain when count=true.
+ content:
+ application/json:
+ schema:
+ type: array
+ items:
+ type: object
+ text/plain:
+ schema:
+ type: integer
+ format: int32
+ minimum: 0
+ maximum: 2147483647
+ description: Number of node entries.
+ "400":
+ $ref: "#/components/responses/BadRequest"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ # ---------------------------------------------------------------------------
+ # Services
+ # ---------------------------------------------------------------------------
+ /services/commoncrawldump/{crawlId}:
+ get:
+ tags:
+ - Services
+ summary: List CommonCrawl dump paths
+ description: >-
+ Lists the dump file paths for a given crawl ID.
+ operationId: listDumpPaths
+ parameters:
+ - name: crawlId
+ in: path
+ required: true
+ description: The crawl ID whose dump paths to list.
+ schema:
+ type: string
+ responses:
+ "200":
+ description: Service information containing the list of dump paths.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/ServiceInfo"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+ /services/commoncrawldump:
+ post:
+ tags:
+ - Services
+ summary: Create a CommonCrawl data dump
+ description: >-
+ Executes a CommonCrawl data dump job for the specified crawl and
+ returns the output directory path.
+ operationId: commoncrawlDump
+ requestBody:
+ required: true
+ description: Service configuration specifying crawl ID and arguments.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/ServiceConfig"
+ responses:
+ "200":
+ description: The output directory path for the dump.
+ content:
+ text/plain:
+ schema:
+ type: string
+ example: "myCrawl/dump/commoncrawl-20260213120000"
+ "401":
+ $ref: "#/components/responses/Unauthorized"
+ "500":
+ $ref: "#/components/responses/InternalServerError"
+
+# =============================================================================
+# Components
+# =============================================================================
+components:
+
+ # ---------------------------------------------------------------------------
+ # Security Schemes
+ # ---------------------------------------------------------------------------
+ securitySchemes:
+ basicAuth:
+ type: http
+ scheme: basic
+ description: HTTP Basic Authentication.
+
+ # ---------------------------------------------------------------------------
+ # Reusable Parameters
+ # ---------------------------------------------------------------------------
+ parameters:
+ configId:
+ name: configId
+ in: path
+ required: true
+ description: The unique identifier for the configuration.
+ schema:
+ type: string
+ propertyId:
+ name: propertyId
+ in: path
+ required: true
+ description: The name (key) of the configuration property.
+ schema:
+ type: string
+ jobId:
+ name: id
+ in: path
+ required: true
+ description: The unique identifier for the job.
+ schema:
+ type: string
+ nrows:
+ name: nrows
+ in: query
+ required: false
+ description: >-
+ Number of rows to read. If not specified (or -1), all rows are
+ returned.
+ schema:
+ type: integer
+ format: int32
+ minimum: -1
+ maximum: 2147483647
+ default: -1
+ start:
+ name: start
+ in: query
+ required: false
+ description: Starting line number for a range read.
+ schema:
+ type: integer
+ format: int32
+ minimum: -1
+ maximum: 2147483647
+ default: -1
+ end:
+ name: end
+ in: query
+ required: false
+ description: Ending line number for a range read.
+ schema:
+ type: integer
+ format: int32
+ minimum: 0
+ maximum: 2147483647
+ count:
+ name: count
+ in: query
+ required: false
+ description: >-
+ If true, returns the number of lines instead of the data itself.
+ When set, the response content type is text/plain.
+ schema:
+ type: boolean
+ default: false
+
+ # ---------------------------------------------------------------------------
+ # Schemas
+ # ---------------------------------------------------------------------------
+ schemas:
+ # -- Request Models -------------------------------------------------------
+ NutchConfig:
+ type: object
+ description: Configuration for creating a new Nutch configuration.
+ properties:
+ configId:
+ type: string
+ description: The identifier for this configuration.
+ force:
+ type: boolean
+ description: >-
+ If true, overwrites an existing configuration with the same ID.
+ default: false
+ params:
+ type: object
+ additionalProperties:
+ type: string
+ description: Key-value pairs of Nutch configuration properties.
+ example:
+ configId: "my-config"
+ force: false
+ params:
+ http.agent.name: "MyBot"
+ http.robots.agents: "MyBot,*"
+
+ JobConfig:
+ type: object
+ description: Configuration for creating a new crawl job.
+ required:
+ - type
+ properties:
+ crawlId:
+ type: string
+ description: The crawl identifier.
+ type:
+ $ref: "#/components/schemas/JobType"
+ confId:
+ type: string
+ description: >-
+ The configuration ID to use for this job. Defaults to "default"
+ if not specified.
+ jobClassName:
+ type: string
+ description: >-
+ Fully qualified class name when type is CLASS.
+ args:
+ type: object
+ additionalProperties: true
+ description: Additional arguments for the job.
+ example:
+ crawlId: "crawl-01"
+ type: "INJECT"
+ confId: "default"
+ args:
+ seedDir: "seedFiles/seed-1700000000000"
+
+ DbQuery:
+ type: object
+ description: Parameters for a CrawlDB query.
+ required:
+ - crawlId
+ - type
+ properties:
+ confId:
+ type: string
+ description: >-
+ Configuration ID. Falls back to "default" if not provided.
+ type:
+ type: string
+ description: The type of CrawlDB query to execute.
+ enum:
+ - stats
+ - dump
+ - topN
+ - url
+ args:
+ type: object
+ additionalProperties:
+ type: string
+ description: Additional arguments for the query.
+ crawlId:
+ type: string
+ description: The crawl identifier.
+ example:
+ confId: "default"
+ type: "stats"
+ crawlId: "crawl-01"
+ args: {}
+
+ ReaderConfig:
+ type: object
+ description: Configuration specifying a file path for reader operations.
+ required:
+ - path
+ properties:
+ path:
+ type: string
+ description: >-
+ The path to the sequence file, link data, or node data to read.
+ example:
+ path: "crawl-01/crawldb/current/part-00000/data"
+
+ SeedList:
+ type: object
+ description: A named list of seed URLs.
+ required:
+ - seedUrls
+ properties:
+ id:
+ type: integer
+ format: int64
+ minimum: 0
+ maximum: 9007199254740991
+ description: The seed list identifier.
+ readOnly: true
+ name:
+ type: string
+ description: A human-readable name for this seed list.
+ seedFilePath:
+ type: string
+ description: >-
+ The HDFS path where the seed file is stored. Populated after
+ creation.
+ readOnly: true
+ seedUrls:
+ type: array
+ items:
+ $ref: "#/components/schemas/SeedUrl"
+ description: The collection of seed URLs in this list.
+ example:
+ name: "my-seeds"
+ seedUrls:
+ - url: "https://example.com"
+ - url: "https://nutch.apache.org"
+
+ SeedUrl:
+ type: object
+ description: A single seed URL entry.
+ properties:
+ id:
+ type: integer
+ format: int64
+ minimum: 0
+ maximum: 9007199254740991
+ description: The seed URL identifier.
+ readOnly: true
+ url:
+ type: string
+ description: The seed URL.
+ example:
+ url: "https://example.com"
+
+ ServiceConfig:
+ type: object
+ description: >-
+ Configuration for service operations such as CommonCrawl data dumps.
+ required:
+ - crawlId
+ properties:
+ crawlId:
+ type: string
+ description: The crawl identifier.
+ confId:
+ type: string
+ description: The configuration ID.
+ args:
+ type: object
+ additionalProperties: true
+ description: Additional arguments for the service operation.
+ example:
+ crawlId: "crawl-01"
+ confId: "default"
+ args: {}
+
+ # -- Response Models ------------------------------------------------------
+ NutchServerInfo:
+ type: object
+ description: Status information about the running Nutch server.
+ required:
+ - configuration
+ - jobs
+ - runningJobs
+ properties:
+ startDate:
+ type: string
+ format: date-time
+ description: The date and time the server was started.
+ configuration:
+ type: array
+ items:
+ type: string
+ uniqueItems: true
+ description: Set of known configuration IDs.
+ jobs:
+ type: array
+ items:
+ $ref: "#/components/schemas/JobInfo"
+ description: All jobs (any state).
+ runningJobs:
+ type: array
+ items:
+ $ref: "#/components/schemas/JobInfo"
+ description: Currently running jobs.
+
+ JobInfo:
+ type: object
+ description: Information about a crawl job.
+ required:
+ - type
+ - state
+ properties:
+ id:
+ type: string
+ description: The unique job identifier.
+ type:
+ $ref: "#/components/schemas/JobType"
+ confId:
+ type: string
+ description: The configuration ID used for this job.
+ args:
+ type: object
+ additionalProperties: true
+ description: Arguments passed to the job.
+ result:
+ type: object
+ additionalProperties: true
+ description: Result data returned after job completion.
+ state:
+ $ref: "#/components/schemas/State"
+ msg:
+ type: string
+ description: A human-readable status or error message.
+ crawlId:
+ type: string
+ description: The crawl identifier associated with this job.
+
+ FetchNodeDbInfo:
+ type: object
+ description: Information about a fetched node in the FetchDB.
+ required:
+ - children
+ properties:
+ url:
+ type: string
+ description: The URL of the fetched node.
+ status:
+ type: integer
+ format: int32
+ minimum: 0
+ maximum: 2147483647
+ description: The HTTP status code of the fetch.
+ numOfOutlinks:
+ type: integer
+ format: int32
+ minimum: 0
+ maximum: 2147483647
+ description: The number of outgoing links discovered.
+ children:
+ type: array
+ items:
+ $ref: "#/components/schemas/ChildNode"
+ description: The outgoing links from this node.
+
+ ChildNode:
+ type: object
+ description: A child (outlink) of a fetched node.
+ properties:
+ childUrl:
+ type: string
+ description: The URL of the child node.
+ anchorText:
+ type: string
+ description: The anchor text of the link.
+
+ ServiceInfo:
+ type: object
+ description: Information returned by service operations.
+ required:
+ - dumpPaths
+ properties:
+ dumpPaths:
+ type: array
+ items:
+ type: string
+ description: List of file paths for the dump output.
+
+ # -- Schema Objects (Reader) ----------------------------------------------
+ LinkSchema:
+ type: object
+ description: Schema describing the fields in a link reader response.
+ properties:
+ key_url:
+ type: string
+ example: "string"
+ timestamp:
+ type: string
+ example: "int"
+ score:
+ type: string
+ example: "float"
+ anchor:
+ type: string
+ example: "string"
+ linktype:
+ type: string
+ example: "string"
+ url:
+ type: string
+ example: "string"
+
+ NodeSchema:
+ type: object
+ description: Schema describing the fields in a node reader response.
+ properties:
+ key_url:
+ type: string
+ example: "string"
+ num_inlinks:
+ type: string
+ example: "int"
+ num_outlinks:
+ type: string
+ example: "int"
+ inlink_score:
+ type: string
+ example: "float"
+ outlink_score:
+ type: string
+ example: "float"
+ metadata:
+ type: string
+ example: "string"
+
+ # -- Enums ----------------------------------------------------------------
+ JobType:
+ type: string
+ description: The type of Nutch crawl job.
+ enum:
+ - INJECT
+ - GENERATE
+ - FETCH
+ - PARSE
+ - UPDATEDB
+ - INDEX
+ - READDB
+ - CLASS
+ - INVERTLINKS
+ - DEDUP
+
+ State:
+ type: string
+ description: The current state of a job.
+ enum:
+ - IDLE
+ - RUNNING
+ - FINISHED
+ - FAILED
+ - KILLED
+ - STOPPING
+ - KILLING
+ - ANY
+
+ # ---------------------------------------------------------------------------
+ # Reusable Responses
+ # ---------------------------------------------------------------------------
+ responses:
+ BadRequest:
+ description: >-
+ Bad request. The request body is missing, malformed, or contains
+ invalid parameters.
+ content:
+ text/plain:
+ schema:
+ type: string
+ example: "Nutch configuration cannot be empty!"
+
+ Unauthorized:
+ description: >-
+ Unauthorized. Basic authentication credentials are missing or
+ invalid.
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ message:
+ type: string
+ example:
+ message: "Authentication required."
+
+ NotFound:
+ description: The requested resource was not found.
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ message:
+ type: string
+ example:
+ message: "Resource not found."
+
+ InternalServerError:
+ description: An unexpected server error occurred.
+ content:
+ text/plain:
+ schema:
+ type: string
+ example: "Internal server error."