This is an automated email from the ASF dual-hosted git repository.

lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git


The following commit(s) were added to refs/heads/master by this push:
     new 96552ae6a NUTCH-2932 Create OpenAPI specification for Nutch 1.x REST 
API (#896)
96552ae6a is described below

commit 96552ae6a473fa2736fc280710fec676a4d78c80
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Sun Apr 12 15:21:33 2026 -0700

    NUTCH-2932 Create OpenAPI specification for Nutch 1.x REST API (#896)
---
 .github/workflows/master-build.yml |   23 +
 .spectral.yaml                     |   29 +
 openapi.yaml                       | 1326 ++++++++++++++++++++++++++++++++++++
 3 files changed, 1378 insertions(+)

diff --git a/.github/workflows/master-build.yml 
b/.github/workflows/master-build.yml
index 76d98db4e..31b06236a 100644
--- a/.github/workflows/master-build.yml
+++ b/.github/workflows/master-build.yml
@@ -85,6 +85,29 @@ jobs:
         if: ${{ env.UNKNOWN_LICENSES != '0 Unknown Licenses' }}
         run: exit 1
 
+  openapi-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
+        id: filter
+        with:
+          filters: |
+            openapi:
+              - 'openapi.yaml'
+              - '.spectral.yaml'
+      - name: Set up Node.js
+        if: steps.filter.outputs.openapi == 'true'
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+      - name: Install IBM OpenAPI Validator
+        if: steps.filter.outputs.openapi == 'true'
+        run: npm install --no-save ibm-openapi-validator
+      - name: Validate OpenAPI specification
+        if: steps.filter.outputs.openapi == 'true'
+        run: ./node_modules/.bin/lint-openapi openapi.yaml
+
   # Build verification with Java bytecode target matrix
   # Verifies bytecode compatibility for both Java 11 and Java 17 targets
   build:
diff --git a/.spectral.yaml b/.spectral.yaml
new file mode 100644
index 000000000..6e42544a1
--- /dev/null
+++ b/.spectral.yaml
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Custom Spectral ruleset for the Apache Nutch REST API.
+# Extends the IBM Cloud Validation Ruleset and disables rules that conflict
+# with the existing Nutch Java wire format (camelCase properties, UPPER_CASE
+# enums, camelCase parameters, bare-array responses, map-typed responses,
+# and consecutive path parameter segments).
+
+extends: "@ibm-cloud/openapi-ruleset"
+rules:
+  ibm-property-casing-convention: off
+  ibm-enum-casing-convention: off
+  ibm-parameter-casing-convention: off
+  ibm-no-array-responses: off
+  ibm-accept-and-return-models: off
+  ibm-no-consecutive-path-parameter-segments: off
diff --git a/openapi.yaml b/openapi.yaml
new file mode 100644
index 000000000..8e3566b0f
--- /dev/null
+++ b/openapi.yaml
@@ -0,0 +1,1326 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+openapi: "3.1.2"
+
+info:
+  title: Apache Nutch REST API
+  description: >-
+    REST API for managing Apache Nutch crawl jobs, configurations, seed lists,
+    database queries, and data readers.
+  version: "1.0.0"
+  license:
+    name: Apache 2.0
+    identifier: Apache-2.0
+  contact:
+    name: Apache Nutch
+    url: https://nutch.apache.org
+
+servers:
+  - url: "{protocol}://localhost:{port}"
+    description: Nutch REST server
+    variables:
+      protocol:
+        default: http
+        enum:
+          - http
+          - https
+        description: The protocol used to access the Nutch server.
+      port:
+        default: "8081"
+        description: >-
+          The port the Nutch server listens on. Configurable via the --port
+          command-line argument.
+
+security:
+  - basicAuth: []
+
+tags:
+  - name: Admin
+    description: Server administration operations
+  - name: Configuration
+    description: Manage Nutch configurations
+  - name: Job
+    description: Manage crawl jobs
+  - name: Database
+    description: Query the CrawlDB and FetchDB
+  - name: Seed
+    description: Manage seed URL lists
+  - name: Reader
+    description: Read sequence files and webgraph data
+  - name: Services
+    description: Auxiliary service operations such as CommonCrawl data dumps
+
+paths:
+  # ---------------------------------------------------------------------------
+  # Admin
+  # ---------------------------------------------------------------------------
+  /admin/:
+    get:
+      tags:
+        - Admin
+      summary: Get server status
+      description: >-
+        Returns the current status of the Nutch server including start date,
+        known configurations, all jobs, and currently running jobs.
+      operationId: getServerStatus
+      responses:
+        "200":
+          description: Server status retrieved successfully.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/NutchServerInfo"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /admin/stop:
+    get:
+      tags:
+        - Admin
+      summary: Stop the Nutch server
+      description: >-
+        Initiates a graceful shutdown of the Nutch server. If jobs are still
+        running and force is not set, the server will refuse to stop.
+      operationId: stopServer
+      parameters:
+        - name: force
+          in: query
+          required: false
+          description: >-
+            If true, kills any running jobs before stopping the server.
+          schema:
+            type: boolean
+            default: false
+      responses:
+        "200":
+          description: Shutdown status message.
+          content:
+            application/json:
+              schema:
+                type: string
+              examples:
+                stopping:
+                  value: "Stopping in server on port 8081"
+                busy:
+                  value: "Jobs still running -- Cannot stop server now"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  # ---------------------------------------------------------------------------
+  # Configuration
+  # ---------------------------------------------------------------------------
+  /config/:
+    get:
+      tags:
+        - Configuration
+      summary: List all configuration IDs
+      description: Returns the set of all known configuration identifiers.
+      operationId: getConfigs
+      responses:
+        "200":
+          description: A JSON array of configuration ID strings.
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+                uniqueItems: true
+              example:
+                - default
+                - my-custom-config
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /config/{configId}:
+    get:
+      tags:
+        - Configuration
+      summary: Get configuration properties
+      description: Returns all key-value properties for the specified 
configuration.
+      operationId: getConfig
+      parameters:
+        - $ref: "#/components/parameters/configId"
+      responses:
+        "200":
+          description: A JSON object of configuration property key-value pairs.
+          content:
+            application/json:
+              schema:
+                type: object
+                additionalProperties:
+                  type: string
+              example:
+                http.agent.name: "NutchBot"
+                http.robots.agents: "NutchBot,*"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "404":
+          $ref: "#/components/responses/NotFound"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+    delete:
+      tags:
+        - Configuration
+      summary: Delete a configuration
+      description: >-
+        Removes the specified configuration from the list of known
+        configurations.
+      operationId: deleteConfig
+      parameters:
+        - $ref: "#/components/parameters/configId"
+      responses:
+        "204":
+          description: Configuration deleted successfully.
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "404":
+          $ref: "#/components/responses/NotFound"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /config/{configId}/{propertyId}:
+    get:
+      tags:
+        - Configuration
+      summary: Get a single configuration property
+      description: >-
+        Returns the value of a specific property within the given
+        configuration.
+      operationId: getProperty
+      parameters:
+        - $ref: "#/components/parameters/configId"
+        - $ref: "#/components/parameters/propertyId"
+      responses:
+        "200":
+          description: The property value as plain text.
+          content:
+            text/plain:
+              schema:
+                type: string
+              example: "NutchBot"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "404":
+          $ref: "#/components/responses/NotFound"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+    put:
+      tags:
+        - Configuration
+      summary: Update a configuration property
+      description: >-
+        Adds or updates the value of a property in the specified
+        configuration.
+      operationId: updateProperty
+      parameters:
+        - $ref: "#/components/parameters/configId"
+        - $ref: "#/components/parameters/propertyId"
+      requestBody:
+        required: true
+        description: The new property value as plain text.
+        content:
+          text/plain:
+            schema:
+              type: string
+            example: "MyNewBot"
+      responses:
+        "200":
+          description: Property updated successfully.
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /config/create:
+    post:
+      tags:
+        - Configuration
+      summary: Create a new configuration
+      description: >-
+        Creates a new Nutch configuration with the specified parameters.
+        Returns the configuration ID on success.
+      operationId: createConfig
+      requestBody:
+        required: true
+        description: The configuration to create.
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/NutchConfig"
+      responses:
+        "200":
+          description: Configuration created. Returns the configuration ID.
+          content:
+            text/plain:
+              schema:
+                type: string
+              example: "my-custom-config"
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  # ---------------------------------------------------------------------------
+  # Job
+  # ---------------------------------------------------------------------------
+  /job/:
+    get:
+      tags:
+        - Job
+      summary: List all jobs
+      description: >-
+        Returns job history for all jobs or filtered by crawl ID, regardless
+        of job state.
+      operationId: getJobs
+      parameters:
+        - name: crawlId
+          in: query
+          required: false
+          description: Optional crawl ID to filter jobs by.
+          schema:
+            type: string
+      responses:
+        "200":
+          description: A JSON array of job information objects.
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/JobInfo"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /job/{id}:
+    get:
+      tags:
+        - Job
+      summary: Get job info
+      description: Returns detailed information for a specific job.
+      operationId: getJobInfo
+      parameters:
+        - $ref: "#/components/parameters/jobId"
+        - name: crawlId
+          in: query
+          required: false
+          description: The crawl ID associated with the job.
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Job details.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JobInfo"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "404":
+          $ref: "#/components/responses/NotFound"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /job/{id}/stop:
+    get:
+      tags:
+        - Job
+      summary: Stop a running job
+      description: Attempts to gracefully stop a running job.
+      operationId: stopJob
+      parameters:
+        - $ref: "#/components/parameters/jobId"
+        - name: crawlId
+          in: query
+          required: false
+          description: The crawl ID associated with the job.
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Whether the job was successfully stopped.
+          content:
+            application/json:
+              schema:
+                type: boolean
+              example: true
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "404":
+          $ref: "#/components/responses/NotFound"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /job/{id}/abort:
+    get:
+      tags:
+        - Job
+      summary: Abort a job
+      description: >-
+        Forcefully aborts a job. Unlike stop, this kills the job immediately.
+      operationId: abortJob
+      parameters:
+        - $ref: "#/components/parameters/jobId"
+        - name: crawlId
+          in: query
+          required: false
+          description: The crawl ID associated with the job.
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Whether the job was successfully aborted.
+          content:
+            application/json:
+              schema:
+                type: boolean
+              example: true
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "404":
+          $ref: "#/components/responses/NotFound"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /job/create:
+    post:
+      tags:
+        - Job
+      summary: Create a new job
+      description: >-
+        Creates and enqueues a new Nutch job (e.g., inject, generate, fetch,
+        parse, updatedb, index).
+      operationId: createJob
+      requestBody:
+        required: true
+        description: The job configuration specifying type, crawl ID, and 
arguments.
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/JobConfig"
+      responses:
+        "200":
+          description: Job created. Returns the job information.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JobInfo"
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  # ---------------------------------------------------------------------------
+  # Database
+  # ---------------------------------------------------------------------------
+  /db/crawldb:
+    post:
+      tags:
+        - Database
+      summary: Query the CrawlDB
+      description: >-
+        Executes a query against the Nutch CrawlDB. The type field in the
+        request body determines the operation: stats, dump, topN, or url.
+        The stats and url types return JSON; dump and topN return binary
+        octet-stream data.
+      operationId: readCrawlDb
+      requestBody:
+        required: true
+        description: The database query parameters.
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/DbQuery"
+      responses:
+        "200":
+          description: >-
+            Query results. Content type varies by query type: application/json
+            for stats and url queries; application/octet-stream for dump and
+            topN queries.
+          content:
+            application/json:
+              schema:
+                type: object
+                description: >-
+                  CrawlDB query result (returned for stats and url query
+                  types).
+            application/octet-stream:
+              schema:
+                type: string
+                format: binary
+                description: >-
+                  Binary data stream (returned for dump and topN query types).
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /db/fetchdb:
+    get:
+      tags:
+        - Database
+      summary: Get FetchDB node information
+      description: >-
+        Returns fetch node database entries for the specified index range.
+        Both from and to default to 0; if to is 0 or exceeds the total
+        number of entries, all entries from the starting index are returned.
+      operationId: fetchDb
+      parameters:
+        - name: from
+          in: query
+          required: false
+          description: Starting index (inclusive). Defaults to 0.
+          schema:
+            type: integer
+            format: int32
+            minimum: 0
+            maximum: 2147483647
+            default: 0
+        - name: to
+          in: query
+          required: false
+          description: Ending index (inclusive). Defaults to 0 (returns all).
+          schema:
+            type: integer
+            format: int32
+            minimum: 0
+            maximum: 2147483647
+            default: 0
+      responses:
+        "200":
+          description: A JSON array of fetch node information objects.
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/FetchNodeDbInfo"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  # ---------------------------------------------------------------------------
+  # Seed
+  # ---------------------------------------------------------------------------
+  /seed/:
+    get:
+      tags:
+        - Seed
+      summary: List all seed lists
+      description: Returns a map of all created seed files keyed by name.
+      operationId: getSeedLists
+      responses:
+        "200":
+          description: A JSON object mapping seed list names to SeedList 
objects.
+          content:
+            application/json:
+              schema:
+                type: object
+                additionalProperties:
+                  $ref: "#/components/schemas/SeedList"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /seed/create:
+    post:
+      tags:
+        - Seed
+      summary: Create a seed list file
+      description: >-
+        Creates a seed list file from the provided URLs and writes it to
+        HDFS. Returns the path to the created seed file directory.
+      operationId: createSeedFile
+      requestBody:
+        required: true
+        description: The seed list containing URLs to write.
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SeedList"
+      responses:
+        "200":
+          description: Path to the created seed file directory.
+          content:
+            text/plain:
+              schema:
+                type: string
+              example: "seedFiles/seed-1700000000000"
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  # ---------------------------------------------------------------------------
+  # Reader
+  # ---------------------------------------------------------------------------
+  /reader/sequence/read:
+    post:
+      tags:
+        - Reader
+      summary: Read a sequence file
+      description: >-
+        Reads key-value pairs from a Hadoop sequence file. Supports reading
+        all rows, a limited number of rows, a row range, or counting the
+        total number of rows.
+      operationId: seqRead
+      parameters:
+        - $ref: "#/components/parameters/nrows"
+        - $ref: "#/components/parameters/start"
+        - $ref: "#/components/parameters/end"
+        - $ref: "#/components/parameters/count"
+      requestBody:
+        required: true
+        description: Reader configuration specifying the file path.
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ReaderConfig"
+      responses:
+        "200":
+          description: >-
+            Sequence file data. Returns application/json when reading rows,
+            or text/plain when count=true.
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+            text/plain:
+              schema:
+                type: integer
+                format: int32
+                minimum: 0
+                maximum: 2147483647
+                description: Number of rows in the sequence file.
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /reader/link:
+    get:
+      tags:
+        - Reader
+      summary: Get link reader schema
+      description: >-
+        Returns the schema describing the fields in link reader responses.
+      operationId: getLinkSchema
+      responses:
+        "200":
+          description: Link reader response schema.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/LinkSchema"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /reader/link/read:
+    post:
+      tags:
+        - Reader
+      summary: Read link objects
+      description: >-
+        Reads link data (LinkDatum) from the Nutch webgraph. Supports
+        reading all rows, a limited number of rows, a row range, or
+        counting the total number of rows.
+      operationId: linkRead
+      parameters:
+        - $ref: "#/components/parameters/nrows"
+        - $ref: "#/components/parameters/start"
+        - $ref: "#/components/parameters/end"
+        - $ref: "#/components/parameters/count"
+      requestBody:
+        required: true
+        description: Reader configuration specifying the file path.
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ReaderConfig"
+      responses:
+        "200":
+          description: >-
+            Link data. Returns application/json when reading rows, or
+            text/plain when count=true.
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+            text/plain:
+              schema:
+                type: integer
+                format: int32
+                minimum: 0
+                maximum: 2147483647
+                description: Number of link entries.
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /reader/node:
+    get:
+      tags:
+        - Reader
+      summary: Get node reader schema
+      description: >-
+        Returns the schema describing the fields in node reader responses.
+      operationId: getNodeSchema
+      responses:
+        "200":
+          description: Node reader response schema.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/NodeSchema"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /reader/node/read:
+    post:
+      tags:
+        - Reader
+      summary: Read node objects
+      description: >-
+        Reads Node objects from the Nutch webgraph. Supports reading all
+        rows, a limited number of rows, a row range, or counting the total
+        number of rows.
+      operationId: nodeRead
+      parameters:
+        - $ref: "#/components/parameters/nrows"
+        - $ref: "#/components/parameters/start"
+        - $ref: "#/components/parameters/end"
+        - $ref: "#/components/parameters/count"
+      requestBody:
+        required: true
+        description: Reader configuration specifying the file path.
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ReaderConfig"
+      responses:
+        "200":
+          description: >-
+            Node data. Returns application/json when reading rows, or
+            text/plain when count=true.
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+            text/plain:
+              schema:
+                type: integer
+                format: int32
+                minimum: 0
+                maximum: 2147483647
+                description: Number of node entries.
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  # ---------------------------------------------------------------------------
+  # Services
+  # ---------------------------------------------------------------------------
+  /services/commoncrawldump/{crawlId}:
+    get:
+      tags:
+        - Services
+      summary: List CommonCrawl dump paths
+      description: >-
+        Lists the dump file paths for a given crawl ID.
+      operationId: listDumpPaths
+      parameters:
+        - name: crawlId
+          in: path
+          required: true
+          description: The crawl ID whose dump paths to list.
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Service information containing the list of dump paths.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ServiceInfo"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+  /services/commoncrawldump:
+    post:
+      tags:
+        - Services
+      summary: Create a CommonCrawl data dump
+      description: >-
+        Executes a CommonCrawl data dump job for the specified crawl and
+        returns the output directory path.
+      operationId: commoncrawlDump
+      requestBody:
+        required: true
+        description: Service configuration specifying crawl ID and arguments.
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ServiceConfig"
+      responses:
+        "200":
+          description: The output directory path for the dump.
+          content:
+            text/plain:
+              schema:
+                type: string
+              example: "myCrawl/dump/commoncrawl-20260213120000"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
+
+# =============================================================================
+# Components
+# =============================================================================
+components:
+
+  # ---------------------------------------------------------------------------
+  # Security Schemes
+  # ---------------------------------------------------------------------------
+  securitySchemes:
+    basicAuth:
+      type: http
+      scheme: basic
+      description: HTTP Basic Authentication.
+
+  # ---------------------------------------------------------------------------
+  # Reusable Parameters
+  # ---------------------------------------------------------------------------
+  parameters:
+    configId:
+      name: configId
+      in: path
+      required: true
+      description: The unique identifier for the configuration.
+      schema:
+        type: string
+    propertyId:
+      name: propertyId
+      in: path
+      required: true
+      description: The name (key) of the configuration property.
+      schema:
+        type: string
+    jobId:
+      name: id
+      in: path
+      required: true
+      description: The unique identifier for the job.
+      schema:
+        type: string
+    nrows:
+      name: nrows
+      in: query
+      required: false
+      description: >-
+        Number of rows to read. If not specified (or -1), all rows are
+        returned.
+      schema:
+        type: integer
+        format: int32
+        minimum: -1
+        maximum: 2147483647
+        default: -1
+    start:
+      name: start
+      in: query
+      required: false
+      description: Starting line number for a range read.
+      schema:
+        type: integer
+        format: int32
+        minimum: -1
+        maximum: 2147483647
+        default: -1
+    end:
+      name: end
+      in: query
+      required: false
+      description: Ending line number for a range read.
+      schema:
+        type: integer
+        format: int32
+        minimum: 0
+        maximum: 2147483647
+    count:
+      name: count
+      in: query
+      required: false
+      description: >-
+        If true, returns the number of lines instead of the data itself.
+        When set, the response content type is text/plain.
+      schema:
+        type: boolean
+        default: false
+
+  # ---------------------------------------------------------------------------
+  # Schemas
+  # ---------------------------------------------------------------------------
+  schemas:
+    # -- Request Models -------------------------------------------------------
+    NutchConfig:
+      type: object
+      description: Configuration for creating a new Nutch configuration.
+      properties:
+        configId:
+          type: string
+          description: The identifier for this configuration.
+        force:
+          type: boolean
+          description: >-
+            If true, overwrites an existing configuration with the same ID.
+          default: false
+        params:
+          type: object
+          additionalProperties:
+            type: string
+          description: Key-value pairs of Nutch configuration properties.
+      example:
+        configId: "my-config"
+        force: false
+        params:
+          http.agent.name: "MyBot"
+          http.robots.agents: "MyBot,*"
+
+    JobConfig:
+      type: object
+      description: Configuration for creating a new crawl job.
+      required:
+        - type
+      properties:
+        crawlId:
+          type: string
+          description: The crawl identifier.
+        type:
+          $ref: "#/components/schemas/JobType"
+        confId:
+          type: string
+          description: >-
+            The configuration ID to use for this job. Defaults to "default"
+            if not specified.
+        jobClassName:
+          type: string
+          description: >-
+            Fully qualified class name when type is CLASS.
+        args:
+          type: object
+          additionalProperties: true
+          description: Additional arguments for the job.
+      example:
+        crawlId: "crawl-01"
+        type: "INJECT"
+        confId: "default"
+        args:
+          seedDir: "seedFiles/seed-1700000000000"
+
+    DbQuery:
+      type: object
+      description: Parameters for a CrawlDB query.
+      required:
+        - crawlId
+        - type
+      properties:
+        confId:
+          type: string
+          description: >-
+            Configuration ID. Falls back to "default" if not provided.
+        type:
+          type: string
+          description: The type of CrawlDB query to execute.
+          enum:
+            - stats
+            - dump
+            - topN
+            - url
+        args:
+          type: object
+          additionalProperties:
+            type: string
+          description: Additional arguments for the query.
+        crawlId:
+          type: string
+          description: The crawl identifier.
+      example:
+        confId: "default"
+        type: "stats"
+        crawlId: "crawl-01"
+        args: {}
+
+    ReaderConfig:
+      type: object
+      description: Configuration specifying a file path for reader operations.
+      required:
+        - path
+      properties:
+        path:
+          type: string
+          description: >-
+            The path to the sequence file, link data, or node data to read.
+      example:
+        path: "crawl-01/crawldb/current/part-00000/data"
+
+    SeedList:
+      type: object
+      description: A named list of seed URLs.
+      required:
+        - seedUrls
+      properties:
+        id:
+          type: integer
+          format: int64
+          minimum: 0
+          maximum: 9007199254740991
+          description: The seed list identifier.
+          readOnly: true
+        name:
+          type: string
+          description: A human-readable name for this seed list.
+        seedFilePath:
+          type: string
+          description: >-
+            The HDFS path where the seed file is stored. Populated after
+            creation.
+          readOnly: true
+        seedUrls:
+          type: array
+          items:
+            $ref: "#/components/schemas/SeedUrl"
+          description: The collection of seed URLs in this list.
+      example:
+        name: "my-seeds"
+        seedUrls:
+          - url: "https://example.com";
+          - url: "https://nutch.apache.org";
+
+    SeedUrl:
+      type: object
+      description: A single seed URL entry.
+      properties:
+        id:
+          type: integer
+          format: int64
+          minimum: 0
+          maximum: 9007199254740991
+          description: The seed URL identifier.
+          readOnly: true
+        url:
+          type: string
+          description: The seed URL.
+      example:
+        url: "https://example.com";
+
+    ServiceConfig:
+      type: object
+      description: >-
+        Configuration for service operations such as CommonCrawl data dumps.
+      required:
+        - crawlId
+      properties:
+        crawlId:
+          type: string
+          description: The crawl identifier.
+        confId:
+          type: string
+          description: The configuration ID.
+        args:
+          type: object
+          additionalProperties: true
+          description: Additional arguments for the service operation.
+      example:
+        crawlId: "crawl-01"
+        confId: "default"
+        args: {}
+
+    # -- Response Models ------------------------------------------------------
+    NutchServerInfo:
+      type: object
+      description: Status information about the running Nutch server.
+      required:
+        - configuration
+        - jobs
+        - runningJobs
+      properties:
+        startDate:
+          type: string
+          format: date-time
+          description: The date and time the server was started.
+        configuration:
+          type: array
+          items:
+            type: string
+          uniqueItems: true
+          description: Set of known configuration IDs.
+        jobs:
+          type: array
+          items:
+            $ref: "#/components/schemas/JobInfo"
+          description: All jobs (any state).
+        runningJobs:
+          type: array
+          items:
+            $ref: "#/components/schemas/JobInfo"
+          description: Currently running jobs.
+
+    JobInfo:
+      type: object
+      description: Information about a crawl job.
+      required:
+        - type
+        - state
+      properties:
+        id:
+          type: string
+          description: The unique job identifier.
+        type:
+          $ref: "#/components/schemas/JobType"
+        confId:
+          type: string
+          description: The configuration ID used for this job.
+        args:
+          type: object
+          additionalProperties: true
+          description: Arguments passed to the job.
+        result:
+          type: object
+          additionalProperties: true
+          description: Result data returned after job completion.
+        state:
+          $ref: "#/components/schemas/State"
+        msg:
+          type: string
+          description: A human-readable status or error message.
+        crawlId:
+          type: string
+          description: The crawl identifier associated with this job.
+
+    FetchNodeDbInfo:
+      type: object
+      description: Information about a fetched node in the FetchDB.
+      required:
+        - children
+      properties:
+        url:
+          type: string
+          description: The URL of the fetched node.
+        status:
+          type: integer
+          format: int32
+          minimum: 0
+          maximum: 2147483647
+          description: The HTTP status code of the fetch.
+        numOfOutlinks:
+          type: integer
+          format: int32
+          minimum: 0
+          maximum: 2147483647
+          description: The number of outgoing links discovered.
+        children:
+          type: array
+          items:
+            $ref: "#/components/schemas/ChildNode"
+          description: The outgoing links from this node.
+
+    ChildNode:
+      type: object
+      description: A child (outlink) of a fetched node.
+      properties:
+        childUrl:
+          type: string
+          description: The URL of the child node.
+        anchorText:
+          type: string
+          description: The anchor text of the link.
+
+    ServiceInfo:
+      type: object
+      description: Information returned by service operations.
+      required:
+        - dumpPaths
+      properties:
+        dumpPaths:
+          type: array
+          items:
+            type: string
+          description: List of file paths for the dump output.
+
+    # -- Schema Objects (Reader) ----------------------------------------------
+    LinkSchema:
+      type: object
+      description: Schema describing the fields in a link reader response.
+      properties:
+        key_url:
+          type: string
+          example: "string"
+        timestamp:
+          type: string
+          example: "int"
+        score:
+          type: string
+          example: "float"
+        anchor:
+          type: string
+          example: "string"
+        linktype:
+          type: string
+          example: "string"
+        url:
+          type: string
+          example: "string"
+
+    NodeSchema:
+      type: object
+      description: Schema describing the fields in a node reader response.
+      properties:
+        key_url:
+          type: string
+          example: "string"
+        num_inlinks:
+          type: string
+          example: "int"
+        num_outlinks:
+          type: string
+          example: "int"
+        inlink_score:
+          type: string
+          example: "float"
+        outlink_score:
+          type: string
+          example: "float"
+        metadata:
+          type: string
+          example: "string"
+
+    # -- Enums ----------------------------------------------------------------
+    JobType:
+      type: string
+      description: The type of Nutch crawl job.
+      enum:
+        - INJECT
+        - GENERATE
+        - FETCH
+        - PARSE
+        - UPDATEDB
+        - INDEX
+        - READDB
+        - CLASS
+        - INVERTLINKS
+        - DEDUP
+
+    State:
+      type: string
+      description: The current state of a job.
+      enum:
+        - IDLE
+        - RUNNING
+        - FINISHED
+        - FAILED
+        - KILLED
+        - STOPPING
+        - KILLING
+        - ANY
+
+  # ---------------------------------------------------------------------------
+  # Reusable Responses
+  # ---------------------------------------------------------------------------
+  responses:
+    BadRequest:
+      description: >-
+        Bad request. The request body is missing, malformed, or contains
+        invalid parameters.
+      content:
+        text/plain:
+          schema:
+            type: string
+          example: "Nutch configuration cannot be empty!"
+
+    Unauthorized:
+      description: >-
+        Unauthorized. Basic authentication credentials are missing or
+        invalid.
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              message:
+                type: string
+            example:
+              message: "Authentication required."
+
+    NotFound:
+      description: The requested resource was not found.
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              message:
+                type: string
+            example:
+              message: "Resource not found."
+
+    InternalServerError:
+      description: An unexpected server error occurred.
+      content:
+        text/plain:
+          schema:
+            type: string
+          example: "Internal server error."


Reply via email to