{ "openapi": "3.1.0", "info": { "title": "Scrape.it API", "version": "2.0.0", "description": "Free scraping compute. Bring your own authenticated proxy. A free API key is required for every compute request.", "license": { "name": "Scrape.it terms", "identifier": "LicenseRef-ScrapeIt-Terms" } }, "servers": [ { "url": "https://api.scrape.it" } ], "paths": { "/v1/auth/magic-link": { "post": { "operationId": "requestMagicLink", "summary": "Request a scanner-safe sign-in link", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MagicLinkRequest" }, "examples": { "example": { "value": { "email": "user@gmail.com", "turnstileToken": "single-use-token" } } } } } }, "responses": { "202": { "description": "Generic response whether or not an account exists", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MagicLinkAccepted" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "security": [] } }, "/auth/confirm": { "get": { "operationId": "showMagicLinkConfirmation", "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Non-consuming confirmation form", "content": { "text/html": { "schema": { "type": "string" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Show Magic Link Confirmation", "security": [] }, "post": { "operationId": "confirmMagicLink", "requestBody": { "required": true, "content": { "application/x-www-form-urlencoded": { "schema": { "type": "object", "required": [ "token", "csrfToken" ], "properties": { "token": { "type": "string", "writeOnly": true }, "csrfToken": { "type": "string", "writeOnly": true } }, "additionalProperties": false } } } }, "responses": { "303": { "description": "Token atomically consumed and session created" }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Confirm Magic Link", "security": [] } }, "/v1/auth/proxy-verification": { "post": { "operationId": "verifyOnboardingProxy", "security": [ { "onboardingSession": [] } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": [ "proxyUrl" ], "properties": { "proxyUrl": { "type": "string", "format": "uri", "writeOnly": true } }, "additionalProperties": false } } } }, "responses": { "200": { "description": "Authenticated proxy verified for first-key issuance", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectorVerification" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Verify Onboarding Proxy" } }, "/v1/me": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [], "operationId": "getAccount", "responses": { "200": { "description": "Account data", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Account" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Account" }, "patch": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "profile:write" ], "operationId": "updateAccount", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "properties": { "displayName": { "type": [ "string", "null" ], "maxLength": 80 } }, "additionalProperties": false, "minProperties": 1 } } } }, "responses": { "200": { "description": "Updated account", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Account" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Update Account" }, "delete": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [], "operationId": "deleteAccount", "responses": { "204": { "description": "Completed" }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Delete Account" } }, "/v1/me/usage": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [], "operationId": "getUsage", "responses": { "200": { "description": "Account data", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Usage" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Usage" } }, "/v1/me/value": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [], "operationId": "getValue", "responses": { "200": { "description": "Account data", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Entitlement" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "deprecated": true, "description": "Compatibility alias for the privacy-safe entitlement summary. Detailed contribution mechanics are private.", "summary": "Get Value" } }, "/v1/me/entitlement": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [], "operationId": "getEntitlement", "responses": { "200": { "description": "Account data", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Entitlement" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Entitlement" } }, "/v1/me/api-keys": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [], "operationId": "listApiKeys", "responses": { "200": { "description": "API keys without raw secrets", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/ApiKey" } } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "List Api Keys" }, "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [], "operationId": "createApiKey", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiKeyCreateRequest" } } } }, "responses": { "201": { "description": "One-time raw API key display", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiKeyIssued" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Create Api Key" } }, "/v1/me/api-keys/{keyId}": { "delete": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [], "operationId": "revokeApiKey", "parameters": [ { "name": "keyId", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "204": { "description": "Completed" }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Revoke Api Key" } }, "/v1/connectors": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "connectors:read" ], "operationId": "listConnectors", "responses": { "200": { "description": "Connector metadata", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Connector" } } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "List Connectors" } }, "/v1/connectors/proxy": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "connectors:write" ], "operationId": "createProxyConnector", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ProxyConnectorCreate" }, "examples": { "example": { "value": { "provider": "bright_data", "name": "my residential account", "proxyUrl": "http://username:password@proxy.example:8000" } } } } } }, "responses": { "201": { "description": "Encrypted proxy connector metadata", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Connector" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Create Proxy Connector" } }, "/v1/connectors/storage": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "connectors:write" ], "operationId": "createStorageConnector", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StorageConnectorCreate" } } } }, "responses": { "201": { "description": "Encrypted storage connector metadata", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Connector" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Create Storage Connector" } }, "/v1/connectors/captcha": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "connectors:write" ], "operationId": "createCaptchaConnector", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CaptchaConnectorCreate" } } } }, "responses": { "201": { "description": "Encrypted CAPTCHA connector metadata", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Connector" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Create Captcha Connector" } }, "/v1/connectors/llm": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "connectors:write" ], "operationId": "createLlmConnector", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/LlmConnectorCreate" } } } }, "responses": { "201": { "description": "Encrypted LLM connector metadata", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Connector" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Create Llm Connector" } }, "/v1/connectors/{id}": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "connectors:read" ], "operationId": "getConnector", "parameters": [ { "name": "id", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Connector metadata", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Connector" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Connector" }, "patch": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "connectors:write" ], "operationId": "updateConnector", "parameters": [ { "name": "id", "in": "path", "required": true, "schema": { "type": "string" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectorPatch" } } } }, "responses": { "200": { "description": "Updated connector metadata", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Connector" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Update Connector" }, "delete": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "connectors:write" ], "operationId": "deleteConnector", "parameters": [ { "name": "id", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "204": { "description": "Completed" }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Delete Connector" } }, "/v1/connectors/{id}/verify": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "connectors:write" ], "operationId": "verifyConnector", "parameters": [ { "name": "id", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Provider-specific verification result", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConnectorVerification" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Verify Connector" } }, "/v1/scrape": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "scrape:http" ], "operationId": "scrape", "summary": "Fetch one public URL through exactly one caller-owned proxy source", "parameters": [ { "name": "Idempotency-Key", "in": "header", "required": false, "schema": { "type": "string" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeRequest" }, "examples": { "example": { "value": { "url": "https://example.com", "proxyConnectorId": "pxy_example", "format": "markdown", "render": false } } } } } }, "responses": { "200": { "description": "Scrape result", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ScrapeResponse" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } } } }, "/v1/extract": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "llm:use" ], "operationId": "extract", "summary": "Fetch one source and perform optional caller-funded schema extraction", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ExtractRequest" }, "examples": { "example": { "value": { "url": "https://example.com", "proxyConnectorId": "pxy_example", "schema": { "type": "object", "properties": { "title": { "type": "string" } }, "required": [ "title" ], "additionalProperties": false }, "llm": { "connectorId": "llm_example", "model": "provider/model", "maxInputBytes": 100000, "maxOutputTokens": 2000, "consent": true } } } } } } }, "responses": { "200": { "description": "Source evidence and advisory structured extraction", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ExtractResponse" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } } } }, "/v1/map": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "map" ], "operationId": "createMapJob", "parameters": [ { "name": "Idempotency-Key", "in": "header", "required": true, "schema": { "type": "string" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MapRequest" } } } }, "responses": { "202": { "description": "Accepted durable job", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Job" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Create Map Job" } }, "/v1/batch/scrape": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "batch" ], "operationId": "createBatchScrapeJob", "parameters": [ { "name": "Idempotency-Key", "in": "header", "required": true, "schema": { "type": "string" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BatchScrapeRequest" } } } }, "responses": { "202": { "description": "Accepted durable job", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Job" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Create Batch Scrape Job" } }, "/v1/batch/scrape/{jobId}": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:read" ], "operationId": "getBatchScrapeJob", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Durable job state", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Job" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Batch Scrape Job" }, "delete": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:cancel" ], "operationId": "cancelBatchScrapeJob", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "204": { "description": "Completed" }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Cancel Batch Scrape Job" } }, "/v1/crawl": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "crawl" ], "operationId": "createCrawlJob", "parameters": [ { "name": "Idempotency-Key", "in": "header", "required": true, "schema": { "type": "string" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CrawlRequest" } } } }, "responses": { "202": { "description": "Accepted durable job", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Job" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Create Crawl Job" } }, "/v1/crawl/{jobId}": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:read" ], "operationId": "getCrawlJob", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Durable job state", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Job" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Crawl Job" }, "delete": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:cancel" ], "operationId": "cancelCrawlJob", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "204": { "description": "Completed" }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Cancel Crawl Job" } }, "/v1/jobs/{jobId}": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:read" ], "operationId": "getJob", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Durable job state", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Job" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Job" }, "delete": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:cancel" ], "operationId": "cancelJob", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "204": { "description": "Completed" }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Cancel Job" } }, "/v1/jobs/{jobId}/pause": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:cancel" ], "operationId": "pauseJob", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "204": { "description": "Completed" }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Pause Job" } }, "/v1/jobs/{jobId}/resume": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:cancel" ], "operationId": "resumeJob", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "204": { "description": "Completed" }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Resume Job" } }, "/v1/jobs/{jobId}/events": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:read" ], "operationId": "streamJobEvents", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } }, { "name": "Last-Event-ID", "in": "header", "required": false, "schema": { "type": "integer", "minimum": 1 } } ], "responses": { "200": { "description": "Replayable compact job events", "headers": { "Cache-Control": { "schema": { "const": "no-store" } }, "X-Accel-Buffering": { "schema": { "const": "no" } } }, "content": { "text/event-stream": { "schema": { "$ref": "#/components/schemas/JobEvent" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Stream Job Events" } }, "/v1/jobs/{jobId}/destination/shards": { "post": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:cancel" ], "operationId": "attachDestinationShards", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } }, { "name": "Idempotency-Key", "in": "header", "required": true, "schema": { "type": "string", "minLength": 8, "maxLength": 200 } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DestinationShards" } } } }, "responses": { "202": { "description": "Secret-free updated output status", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OutputDestination" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Attach Destination Shards" } }, "/v1/jobs/{jobId}/destination": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:read" ], "operationId": "getOutputDestination", "parameters": [ { "name": "jobId", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Secret-free durable output status", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OutputDestination" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Output Destination" } }, "/v1/me/notification-preferences": { "get": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:read" ], "operationId": "getNotificationPreferences", "responses": { "200": { "description": "Operational notification preferences", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/NotificationPreferences" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Notification Preferences" }, "patch": { "security": [ { "apiKeyAuth": [] } ], "x-required-scopes": [ "jobs:read" ], "operationId": "updateNotificationPreferences", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/NotificationPreferencesPatch" } } } }, "responses": { "200": { "description": "Updated operational notification preferences", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/NotificationPreferences" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Update Notification Preferences" } }, "/v1/capabilities": { "get": { "operationId": "getCapabilities", "responses": { "200": { "description": "Current runtime capabilities and degradation", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Capabilities" } } } } }, "summary": "Get Capabilities", "security": [] } }, "/v1/proxy/providers": { "get": { "operationId": "listProxyProviders", "responses": { "200": { "description": "Supported proxy providers" } }, "summary": "List Proxy Providers", "security": [] } }, "/v1/storage/providers": { "get": { "operationId": "listStorageProviders", "responses": { "200": { "description": "Supported storage providers" } }, "summary": "List Storage Providers", "security": [] } }, "/v1/captcha/providers": { "get": { "operationId": "listCaptchaProviders", "responses": { "200": { "description": "Supported CAPTCHA providers" } }, "summary": "List Captcha Providers", "security": [] } }, "/v1/llm/providers": { "get": { "operationId": "listLlmProviders", "responses": { "200": { "description": "Supported LLM providers" } }, "summary": "List Llm Providers", "security": [] } }, "/internal/intel/ip/{ip}": { "get": { "operationId": "getInternalIpIntelligence", "security": [ { "internalAuth": [] } ], "parameters": [ { "name": "ip", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Versioned first-party IP intelligence", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IntelligenceEntity" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Internal Ip Intelligence" } }, "/internal/intel/prefix/{prefix}": { "get": { "operationId": "getInternalPrefixIntelligence", "security": [ { "internalAuth": [] } ], "parameters": [ { "name": "prefix", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Versioned prefix intelligence", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IntelligenceEntity" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Internal Prefix Intelligence" } }, "/internal/intel/asn/{asn}": { "get": { "operationId": "getInternalAsnIntelligence", "security": [ { "internalAuth": [] } ], "parameters": [ { "name": "asn", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Versioned ASN intelligence", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IntelligenceEntity" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Internal Asn Intelligence" } }, "/internal/intel/provider/{provider}": { "get": { "operationId": "getInternalProviderIntelligence", "security": [ { "internalAuth": [] } ], "parameters": [ { "name": "provider", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Versioned provider intelligence", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IntelligenceEntity" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Internal Provider Intelligence" } }, "/internal/intel/provider/{a}/compare/{b}": { "get": { "operationId": "compareInternalProviders", "security": [ { "internalAuth": [] } ], "parameters": [ { "name": "a", "in": "path", "required": true, "schema": { "type": "string" } }, { "name": "b", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Evidence-backed provider comparison", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IntelligenceEntity" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Compare Internal Providers" } }, "/internal/intel/user/{id}/contribution": { "get": { "operationId": "getInternalUserContribution", "security": [ { "internalAuth": [] } ], "parameters": [ { "name": "id", "in": "path", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "description": "Private contribution explanation", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IntelligenceEntity" } } } }, "4XX": { "description": "Request, authentication, entitlement, connector, capacity, or target error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } }, "5XX": { "description": "Runtime error", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Problem" } } } } }, "summary": "Get Internal User Contribution" } } }, "components": { "securitySchemes": { "apiKeyAuth": { "type": "http", "scheme": "bearer", "bearerFormat": "sit_live__", "description": "User-owned API key. Query-string authentication is forbidden." }, "onboardingSession": { "type": "apiKey", "in": "cookie", "name": "onboarding_session" }, "internalAuth": { "type": "http", "scheme": "bearer", "bearerFormat": "opaque-internal-service-token", "description": "Private service authorization; never accepted on the public edge." } }, "schemas": { "MagicLinkRequest": { "additionalProperties": false, "type": "object", "required": [ "email", "turnstileToken" ], "properties": { "email": { "format": "email", "maxLength": 320, "type": "string" }, "turnstileToken": { "minLength": 1, "writeOnly": true, "type": "string" } } }, "MagicLinkAccepted": { "additionalProperties": false, "type": "object", "required": [ "accepted" ], "properties": { "accepted": { "const": true, "type": "boolean" } } }, "ApiKeyCreateRequest": { "additionalProperties": false, "type": "object", "required": [ "scopes" ], "properties": { "name": { "minLength": 1, "maxLength": 80, "type": "string" }, "expiresAt": { "format": "date-time", "type": "string" }, "scopes": { "minItems": 1, "uniqueItems": true, "type": "array", "items": { "anyOf": [ { "const": "scrape:http", "type": "string" }, { "const": "scrape:render", "type": "string" }, { "const": "map", "type": "string" }, { "const": "batch", "type": "string" }, { "const": "crawl", "type": "string" }, { "const": "jobs:read", "type": "string" }, { "const": "jobs:cancel", "type": "string" }, { "const": "connectors:read", "type": "string" }, { "const": "connectors:write", "type": "string" }, { "const": "captcha:use", "type": "string" }, { "const": "llm:use", "type": "string" }, { "const": "profile:write", "type": "string" } ] } } } }, "ApiKey": { "additionalProperties": false, "type": "object", "required": [ "id", "name", "prefix", "scopes", "state", "createdAt", "expiresAt", "lastUsedAt" ], "properties": { "id": { "pattern": "^key_[A-Za-z0-9_-]+$", "type": "string" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "prefix": { "pattern": "^sit_live_", "type": "string" }, "scopes": { "uniqueItems": true, "type": "array", "items": { "anyOf": [ { "const": "scrape:http", "type": "string" }, { "const": "scrape:render", "type": "string" }, { "const": "map", "type": "string" }, { "const": "batch", "type": "string" }, { "const": "crawl", "type": "string" }, { "const": "jobs:read", "type": "string" }, { "const": "jobs:cancel", "type": "string" }, { "const": "connectors:read", "type": "string" }, { "const": "connectors:write", "type": "string" }, { "const": "captcha:use", "type": "string" }, { "const": "llm:use", "type": "string" }, { "const": "profile:write", "type": "string" } ] } }, "state": { "anyOf": [ { "const": "active", "type": "string" }, { "const": "revoked", "type": "string" }, { "const": "expired", "type": "string" } ] }, "createdAt": { "format": "date-time", "type": "string" }, "expiresAt": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "lastUsedAt": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] } } }, "ApiKeyIssued": { "description": "The raw API key is returned once and is never persisted.", "type": "object", "allOf": [ { "additionalProperties": false, "type": "object", "required": [ "id", "name", "prefix", "scopes", "state", "createdAt", "expiresAt", "lastUsedAt" ], "properties": { "id": { "pattern": "^key_[A-Za-z0-9_-]+$", "type": "string" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "prefix": { "pattern": "^sit_live_", "type": "string" }, "scopes": { "uniqueItems": true, "type": "array", "items": { "anyOf": [ { "const": "scrape:http", "type": "string" }, { "const": "scrape:render", "type": "string" }, { "const": "map", "type": "string" }, { "const": "batch", "type": "string" }, { "const": "crawl", "type": "string" }, { "const": "jobs:read", "type": "string" }, { "const": "jobs:cancel", "type": "string" }, { "const": "connectors:read", "type": "string" }, { "const": "connectors:write", "type": "string" }, { "const": "captcha:use", "type": "string" }, { "const": "llm:use", "type": "string" }, { "const": "profile:write", "type": "string" } ] } }, "state": { "anyOf": [ { "const": "active", "type": "string" }, { "const": "revoked", "type": "string" }, { "const": "expired", "type": "string" } ] }, "createdAt": { "format": "date-time", "type": "string" }, "expiresAt": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "lastUsedAt": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] } } }, { "type": "object", "required": [ "apiKey" ], "properties": { "apiKey": { "pattern": "^sit_live_[A-Za-z0-9]+_[A-Za-z0-9_-]+$", "writeOnly": true, "type": "string" } } } ] }, "Account": { "additionalProperties": false, "type": "object", "required": [ "id", "email", "displayName", "publicUserId", "state", "createdAt" ], "properties": { "id": { "pattern": "^usr_[A-Za-z0-9_-]+$", "type": "string" }, "email": { "format": "email", "type": "string" }, "displayName": { "anyOf": [ { "maxLength": 80, "type": "string" }, { "type": "null" } ] }, "publicUserId": { "pattern": "^usr_[A-Za-z0-9_-]+$", "type": "string" }, "state": { "anyOf": [ { "const": "onboarding", "type": "string" }, { "const": "active", "type": "string" }, { "const": "suspended", "type": "string" }, { "const": "deleting", "type": "string" } ] }, "createdAt": { "format": "date-time", "type": "string" } } }, "Usage": { "additionalProperties": false, "type": "object", "required": [ "windowStartedAt", "requests", "resourceUnits", "activeJobs", "bytesReceived" ], "properties": { "windowStartedAt": { "format": "date-time", "type": "string" }, "requests": { "minimum": 0, "type": "integer" }, "resourceUnits": { "minimum": 0, "type": "number" }, "activeJobs": { "minimum": 0, "type": "integer" }, "bytesReceived": { "minimum": 0, "type": "integer" } } }, "Entitlement": { "additionalProperties": false, "type": "object", "required": [ "plan", "baselineUnits", "communityBonusUnits", "availableUnits", "currentConcurrency", "activeJobs", "queuePriority" ], "properties": { "plan": { "const": "free", "type": "string" }, "baselineUnits": { "minimum": 0, "type": "integer" }, "communityBonusUnits": { "minimum": 0, "type": "integer" }, "availableUnits": { "minimum": 0, "type": "integer" }, "currentConcurrency": { "minimum": 0, "type": "integer" }, "activeJobs": { "minimum": 0, "type": "integer" }, "queuePriority": { "const": "standard", "type": "string" } } }, "Connector": { "additionalProperties": false, "description": "Connector metadata never includes encrypted or plaintext secret material.", "type": "object", "required": [ "id", "type", "provider", "name", "publicConfig", "capabilities", "status", "verificationErrorCode", "createdAt", "updatedAt", "lastVerifiedAt", "lastUsedAt" ], "properties": { "id": { "pattern": "^(pxy|sto|cap|llm)_[A-Za-z0-9_-]+$", "type": "string" }, "type": { "anyOf": [ { "const": "proxy", "type": "string" }, { "const": "storage", "type": "string" }, { "const": "captcha", "type": "string" }, { "const": "llm", "type": "string" } ] }, "provider": { "type": "string" }, "name": { "type": "string" }, "publicConfig": { "type": "object", "patternProperties": { "^(.*)$": {} } }, "capabilities": { "type": "object", "patternProperties": { "^(.*)$": {} } }, "status": { "anyOf": [ { "const": "pending", "type": "string" }, { "const": "verifying", "type": "string" }, { "const": "verified", "type": "string" }, { "const": "degraded", "type": "string" }, { "const": "invalid", "type": "string" }, { "const": "disabled", "type": "string" }, { "const": "deleted", "type": "string" } ] }, "verificationErrorCode": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "createdAt": { "format": "date-time", "type": "string" }, "updatedAt": { "format": "date-time", "type": "string" }, "lastVerifiedAt": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "lastUsedAt": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] } } }, "ProxyConnectorCreate": { "additionalProperties": false, "type": "object", "required": [ "provider", "name", "proxyUrl" ], "properties": { "provider": { "anyOf": [ { "const": "bright_data", "type": "string" }, { "const": "oxylabs", "type": "string" }, { "const": "decodo", "type": "string" }, { "const": "soax", "type": "string" }, { "const": "iproyal", "type": "string" }, { "const": "webshare", "type": "string" }, { "const": "generic_authenticated", "type": "string" }, { "const": "diy_mobile", "type": "string" } ] }, "name": { "minLength": 1, "maxLength": 80, "type": "string" }, "proxyUrl": { "format": "uri", "writeOnly": true, "type": "string" } } }, "StorageConnectorCreate": { "additionalProperties": false, "type": "object", "required": [ "provider", "name", "bucket", "accessKeyId", "secretAccessKey" ], "properties": { "provider": { "anyOf": [ { "const": "aws_s3", "type": "string" }, { "const": "cloudflare_r2", "type": "string" }, { "const": "backblaze_b2", "type": "string" } ] }, "name": { "minLength": 1, "maxLength": 80, "type": "string" }, "bucket": { "minLength": 1, "type": "string" }, "region": { "minLength": 1, "type": "string" }, "accountId": { "pattern": "^[a-f0-9]{16,64}$", "type": "string" }, "prefix": { "type": "string" }, "accessKeyId": { "minLength": 1, "writeOnly": true, "type": "string" }, "secretAccessKey": { "minLength": 1, "writeOnly": true, "type": "string" } } }, "CaptchaConnectorCreate": { "additionalProperties": false, "type": "object", "required": [ "provider", "name", "apiKey" ], "properties": { "provider": { "const": "2captcha", "type": "string" }, "name": { "minLength": 1, "maxLength": 80, "type": "string" }, "apiKey": { "minLength": 1, "writeOnly": true, "type": "string" } } }, "LlmConnectorCreate": { "additionalProperties": false, "type": "object", "required": [ "provider", "name", "apiKey" ], "properties": { "provider": { "anyOf": [ { "const": "openai", "type": "string" }, { "const": "anthropic", "type": "string" }, { "const": "gemini", "type": "string" }, { "const": "openrouter", "type": "string" } ] }, "name": { "minLength": 1, "maxLength": 80, "type": "string" }, "apiKey": { "minLength": 1, "writeOnly": true, "type": "string" } } }, "ConnectorPatch": { "additionalProperties": false, "minProperties": 1, "type": "object", "properties": { "name": { "minLength": 1, "maxLength": 80, "type": "string" }, "disabled": { "type": "boolean" }, "credentials": { "writeOnly": true, "type": "object", "patternProperties": { "^(.*)$": { "type": "string" } } } } }, "ConnectorVerification": { "additionalProperties": false, "type": "object", "required": [ "verified", "provider", "capabilities", "latencyMs", "fingerprint", "lastVerifiedAt", "errorCode" ], "properties": { "verified": { "type": "boolean" }, "connectorId": { "pattern": "^pxy_[A-Za-z0-9_-]+$", "type": "string" }, "provider": { "type": "string" }, "capabilities": { "type": "object", "patternProperties": { "^(.*)$": {} } }, "latencyMs": { "minimum": 0, "type": "integer" }, "fingerprint": { "description": "Safe non-secret connector fingerprint.", "type": "string" }, "lastVerifiedAt": { "format": "date-time", "type": "string" }, "errorCode": { "anyOf": [ { "type": "string" }, { "type": "null" } ] } } }, "ScrapeRequest": { "description": "Exactly one caller-owned proxy URL or verified proxy connector is required.", "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "url", "proxy" ], "properties": { "url": { "format": "uri", "type": "string" }, "render": { "default": false, "type": "boolean" }, "format": { "default": "raw", "anyOf": [ { "const": "raw", "type": "string" }, { "const": "html", "type": "string" }, { "const": "text", "type": "string" }, { "const": "markdown", "type": "string" } ] }, "timeoutMs": { "minimum": 1000, "maximum": 90000, "default": 30000, "type": "integer" }, "maxBytes": { "minimum": 1, "maximum": 10485760, "default": 2097152, "type": "integer" }, "headers": { "default": {}, "type": "object", "patternProperties": { "^(.*)$": { "type": "string" } } }, "waitFor": { "additionalProperties": false, "type": "object", "required": [ "selector" ], "properties": { "selector": { "minLength": 1, "type": "string" }, "state": { "default": "attached", "anyOf": [ { "const": "attached", "type": "string" }, { "const": "detached", "type": "string" }, { "const": "visible", "type": "string" }, { "const": "hidden", "type": "string" } ] }, "timeoutMs": { "minimum": 1, "maximum": 90000, "default": 10000, "type": "integer" } } }, "captcha": { "description": "Exactly one saved connector or request-scoped key is required.", "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "provider", "mode", "connectorId" ], "properties": { "provider": { "const": "2captcha", "type": "string" }, "mode": { "const": "on_challenge", "type": "string" }, "maxTasks": { "minimum": 1, "maximum": 10, "default": 1, "type": "integer" }, "connectorId": { "pattern": "^cap_[A-Za-z0-9_-]+$", "type": "string" }, "key": { "not": {} } } }, { "additionalProperties": false, "type": "object", "required": [ "provider", "mode", "key" ], "properties": { "provider": { "const": "2captcha", "type": "string" }, "mode": { "const": "on_challenge", "type": "string" }, "maxTasks": { "minimum": 1, "maximum": 10, "default": 1, "type": "integer" }, "key": { "minLength": 1, "writeOnly": true, "type": "string" }, "connectorId": { "not": {} } } } ] }, "proxy": { "format": "uri", "writeOnly": true, "type": "string" }, "proxyConnectorId": { "not": {} } } }, { "additionalProperties": false, "type": "object", "required": [ "url", "proxyConnectorId" ], "properties": { "url": { "format": "uri", "type": "string" }, "render": { "default": false, "type": "boolean" }, "format": { "default": "raw", "anyOf": [ { "const": "raw", "type": "string" }, { "const": "html", "type": "string" }, { "const": "text", "type": "string" }, { "const": "markdown", "type": "string" } ] }, "timeoutMs": { "minimum": 1000, "maximum": 90000, "default": 30000, "type": "integer" }, "maxBytes": { "minimum": 1, "maximum": 10485760, "default": 2097152, "type": "integer" }, "headers": { "default": {}, "type": "object", "patternProperties": { "^(.*)$": { "type": "string" } } }, "waitFor": { "additionalProperties": false, "type": "object", "required": [ "selector" ], "properties": { "selector": { "minLength": 1, "type": "string" }, "state": { "default": "attached", "anyOf": [ { "const": "attached", "type": "string" }, { "const": "detached", "type": "string" }, { "const": "visible", "type": "string" }, { "const": "hidden", "type": "string" } ] }, "timeoutMs": { "minimum": 1, "maximum": 90000, "default": 10000, "type": "integer" } } }, "captcha": { "description": "Exactly one saved connector or request-scoped key is required.", "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "provider", "mode", "connectorId" ], "properties": { "provider": { "const": "2captcha", "type": "string" }, "mode": { "const": "on_challenge", "type": "string" }, "maxTasks": { "minimum": 1, "maximum": 10, "default": 1, "type": "integer" }, "connectorId": { "pattern": "^cap_[A-Za-z0-9_-]+$", "type": "string" }, "key": { "not": {} } } }, { "additionalProperties": false, "type": "object", "required": [ "provider", "mode", "key" ], "properties": { "provider": { "const": "2captcha", "type": "string" }, "mode": { "const": "on_challenge", "type": "string" }, "maxTasks": { "minimum": 1, "maximum": 10, "default": 1, "type": "integer" }, "key": { "minLength": 1, "writeOnly": true, "type": "string" }, "connectorId": { "not": {} } } } ] }, "proxyConnectorId": { "pattern": "^pxy_[A-Za-z0-9_-]+$", "type": "string" }, "proxy": { "not": {} } } } ] }, "ScrapeResponse": { "additionalProperties": false, "type": "object", "required": [ "success", "requestId", "url", "finalUrl", "status", "format", "data", "execution", "proxy", "challenge" ], "properties": { "success": { "const": true, "type": "boolean" }, "requestId": { "pattern": "^req_[A-Za-z0-9_-]+$", "type": "string" }, "url": { "format": "uri", "type": "string" }, "finalUrl": { "format": "uri", "type": "string" }, "status": { "minimum": 100, "maximum": 599, "type": "integer" }, "format": { "default": "raw", "anyOf": [ { "const": "raw", "type": "string" }, { "const": "html", "type": "string" }, { "const": "text", "type": "string" }, { "const": "markdown", "type": "string" } ] }, "data": { "type": "string" }, "execution": { "additionalProperties": false, "type": "object", "required": [ "engine", "rendered", "recovery", "durationMs", "bytesReceived" ], "properties": { "engine": { "anyOf": [ { "const": "crw_http", "type": "string" }, { "const": "draco_static", "type": "string" }, { "const": "draco_v8", "type": "string" }, { "const": "obscura", "type": "string" }, { "const": "browseroxide", "type": "string" } ] }, "rendered": { "type": "boolean" }, "recovery": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "durationMs": { "minimum": 0, "type": "integer" }, "bytesReceived": { "minimum": 0, "type": "integer" } } }, "proxy": { "additionalProperties": false, "type": "object", "required": [ "provider", "product", "class", "observedExitIp", "exitConfidence" ], "properties": { "provider": { "type": "string" }, "product": { "type": "string" }, "class": { "type": "string" }, "observedExitIp": { "type": "string" }, "exitConfidence": { "type": "string" } } }, "challenge": { "anyOf": [ { "type": "object", "patternProperties": { "^(.*)$": {} } }, { "type": "null" } ] }, "proxyIntelligence": { "additionalProperties": false, "description": "Pool-only responses never claim that sampledExitIp handled the target request.", "type": "object", "required": [ "associationScope", "exitIp", "sampledExitIp", "utilityScore", "utilityConfidence", "providerProductUtility", "providerProductConfidence", "networkOrigin", "proxyDelivery", "firstSeen", "lastSeen", "freshness", "evidenceCount", "classifierVersion", "scorerVersion", "creditsEarned" ], "properties": { "associationScope": { "anyOf": [ { "const": "exact_exit", "type": "string" }, { "const": "verified_sticky_session", "type": "string" }, { "const": "probable_session", "type": "string" }, { "const": "pool_only", "type": "string" }, { "const": "unknown", "type": "string" } ] }, "exitIp": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "sampledExitIp": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "utilityScore": { "anyOf": [ { "minimum": 0, "maximum": 100, "type": "integer" }, { "type": "null" } ] }, "utilityConfidence": { "minimum": 0, "maximum": 1, "type": "number" }, "providerProductUtility": { "anyOf": [ { "minimum": 0, "maximum": 100, "type": "integer" }, { "type": "null" } ] }, "providerProductConfidence": { "minimum": 0, "maximum": 1, "type": "number" }, "networkOrigin": { "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "classification", "confidence" ], "properties": { "classification": { "type": "string" }, "confidence": { "minimum": 0, "maximum": 1, "type": "number" } } }, { "type": "null" } ] }, "proxyDelivery": { "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "classification", "confidence" ], "properties": { "classification": { "type": "string" }, "confidence": { "minimum": 0, "maximum": 1, "type": "number" } } }, { "type": "null" } ] }, "firstSeen": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "lastSeen": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "freshness": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "evidenceCount": { "minimum": 0, "type": "integer" }, "classifierVersion": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "scorerVersion": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "creditsEarned": { "type": "number" } } } } }, "ProxyIntelligence": { "additionalProperties": false, "description": "Pool-only responses never claim that sampledExitIp handled the target request.", "type": "object", "required": [ "associationScope", "exitIp", "sampledExitIp", "utilityScore", "utilityConfidence", "providerProductUtility", "providerProductConfidence", "networkOrigin", "proxyDelivery", "firstSeen", "lastSeen", "freshness", "evidenceCount", "classifierVersion", "scorerVersion", "creditsEarned" ], "properties": { "associationScope": { "anyOf": [ { "const": "exact_exit", "type": "string" }, { "const": "verified_sticky_session", "type": "string" }, { "const": "probable_session", "type": "string" }, { "const": "pool_only", "type": "string" }, { "const": "unknown", "type": "string" } ] }, "exitIp": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "sampledExitIp": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "utilityScore": { "anyOf": [ { "minimum": 0, "maximum": 100, "type": "integer" }, { "type": "null" } ] }, "utilityConfidence": { "minimum": 0, "maximum": 1, "type": "number" }, "providerProductUtility": { "anyOf": [ { "minimum": 0, "maximum": 100, "type": "integer" }, { "type": "null" } ] }, "providerProductConfidence": { "minimum": 0, "maximum": 1, "type": "number" }, "networkOrigin": { "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "classification", "confidence" ], "properties": { "classification": { "type": "string" }, "confidence": { "minimum": 0, "maximum": 1, "type": "number" } } }, { "type": "null" } ] }, "proxyDelivery": { "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "classification", "confidence" ], "properties": { "classification": { "type": "string" }, "confidence": { "minimum": 0, "maximum": 1, "type": "number" } } }, { "type": "null" } ] }, "firstSeen": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "lastSeen": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "freshness": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "evidenceCount": { "minimum": 0, "type": "integer" }, "classifierVersion": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "scorerVersion": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "creditsEarned": { "type": "number" } } }, "IntelligenceEntity": { "additionalProperties": false, "type": "object", "required": [ "entityType", "entityId", "score", "confidence", "evidenceCount", "firstSeen", "lastSeen", "freshness", "associationScope", "classifierVersion", "scorerVersion", "evidenceSummary" ], "properties": { "entityType": { "anyOf": [ { "const": "ip", "type": "string" }, { "const": "prefix", "type": "string" }, { "const": "asn", "type": "string" }, { "const": "provider", "type": "string" }, { "const": "provider_product", "type": "string" }, { "const": "pool", "type": "string" }, { "const": "user_contribution", "type": "string" } ] }, "entityId": { "type": "string" }, "score": { "anyOf": [ { "minimum": 0, "maximum": 100, "type": "number" }, { "type": "null" } ] }, "confidence": { "minimum": 0, "maximum": 1, "type": "number" }, "evidenceCount": { "minimum": 0, "type": "integer" }, "firstSeen": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "lastSeen": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "freshness": { "anyOf": [ { "format": "date-time", "type": "string" }, { "type": "null" } ] }, "associationScope": { "anyOf": [ { "anyOf": [ { "const": "exact_exit", "type": "string" }, { "const": "verified_sticky_session", "type": "string" }, { "const": "probable_session", "type": "string" }, { "const": "pool_only", "type": "string" }, { "const": "unknown", "type": "string" } ] }, { "type": "null" } ] }, "classifierVersion": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "scorerVersion": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "evidenceSummary": { "type": "object", "patternProperties": { "^(.*)$": {} } } } }, "ExtractRequest": { "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "url", "schema", "llm", "proxy" ], "properties": { "url": { "format": "uri", "type": "string" }, "headers": { "type": "object", "patternProperties": { "^(.*)$": { "type": "string" } } }, "schema": { "description": "Bounded JSON Schema validated by the executable request parser.", "type": "object", "patternProperties": { "^(.*)$": {} } }, "llm": { "additionalProperties": false, "description": "Exactly one saved connector or provider/apiKey pair is required.", "type": "object", "required": [ "model", "consent" ], "properties": { "connectorId": { "pattern": "^llm_[A-Za-z0-9_-]+$", "type": "string" }, "provider": { "anyOf": [ { "const": "openai", "type": "string" }, { "const": "anthropic", "type": "string" }, { "const": "gemini", "type": "string" }, { "const": "openrouter", "type": "string" } ] }, "apiKey": { "minLength": 1, "writeOnly": true, "type": "string" }, "model": { "minLength": 1, "maxLength": 160, "pattern": "^[A-Za-z0-9._:/-]+$", "type": "string" }, "maxInputBytes": { "minimum": 1, "maximum": 100000, "default": 100000, "type": "integer" }, "maxOutputTokens": { "minimum": 1, "maximum": 2000, "default": 2000, "type": "integer" }, "consent": { "const": true, "type": "boolean" } } }, "proxy": { "format": "uri", "writeOnly": true, "type": "string" }, "proxyConnectorId": { "not": {} } } }, { "additionalProperties": false, "type": "object", "required": [ "url", "schema", "llm", "proxyConnectorId" ], "properties": { "url": { "format": "uri", "type": "string" }, "headers": { "type": "object", "patternProperties": { "^(.*)$": { "type": "string" } } }, "schema": { "description": "Bounded JSON Schema validated by the executable request parser.", "type": "object", "patternProperties": { "^(.*)$": {} } }, "llm": { "additionalProperties": false, "description": "Exactly one saved connector or provider/apiKey pair is required.", "type": "object", "required": [ "model", "consent" ], "properties": { "connectorId": { "pattern": "^llm_[A-Za-z0-9_-]+$", "type": "string" }, "provider": { "anyOf": [ { "const": "openai", "type": "string" }, { "const": "anthropic", "type": "string" }, { "const": "gemini", "type": "string" }, { "const": "openrouter", "type": "string" } ] }, "apiKey": { "minLength": 1, "writeOnly": true, "type": "string" }, "model": { "minLength": 1, "maxLength": 160, "pattern": "^[A-Za-z0-9._:/-]+$", "type": "string" }, "maxInputBytes": { "minimum": 1, "maximum": 100000, "default": 100000, "type": "integer" }, "maxOutputTokens": { "minimum": 1, "maximum": 2000, "default": 2000, "type": "integer" }, "consent": { "const": true, "type": "boolean" } } }, "proxyConnectorId": { "pattern": "^pxy_[A-Za-z0-9_-]+$", "type": "string" }, "proxy": { "not": {} } } } ] }, "ExtractResponse": { "additionalProperties": false, "type": "object", "required": [ "success", "source", "extraction", "llm" ], "properties": { "success": { "const": true, "type": "boolean" }, "source": { "additionalProperties": false, "type": "object", "required": [ "url", "finalUrl", "sha256", "bytes" ], "properties": { "url": { "format": "uri", "type": "string" }, "finalUrl": { "format": "uri", "type": "string" }, "sha256": { "pattern": "^[a-f0-9]{64}$", "type": "string" }, "bytes": { "minimum": 0, "type": "integer" } } }, "extraction": {}, "llm": { "additionalProperties": false, "type": "object", "required": [ "provider", "model", "inputTokens", "outputTokens", "advisory" ], "properties": { "provider": { "anyOf": [ { "const": "openai", "type": "string" }, { "const": "anthropic", "type": "string" }, { "const": "gemini", "type": "string" }, { "const": "openrouter", "type": "string" } ] }, "model": { "type": "string" }, "inputTokens": { "minimum": 0, "type": "integer" }, "outputTokens": { "minimum": 0, "type": "integer" }, "advisory": { "const": true, "type": "boolean" } } } } }, "MapRequest": { "additionalProperties": false, "type": "object", "required": [ "proxyConnectorId", "destination", "url" ], "properties": { "proxyConnectorId": { "pattern": "^pxy_[A-Za-z0-9_-]+$", "type": "string" }, "destination": { "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "storageConnectorId" ], "properties": { "storageConnectorId": { "pattern": "^sto_[A-Za-z0-9_-]+$", "type": "string" } } }, { "additionalProperties": false, "type": "object", "required": [ "mode", "provider", "shardPutUrls", "format", "compression", "expiresAt" ], "properties": { "mode": { "const": "presigned_shards", "type": "string" }, "provider": { "anyOf": [ { "const": "aws_s3", "type": "string" }, { "const": "cloudflare_r2", "type": "string" }, { "const": "backblaze_b2", "type": "string" } ] }, "manifestPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "summaryPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "successPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "shardPutUrls": { "minItems": 1, "maxItems": 1000, "type": "array", "items": { "additionalProperties": false, "type": "object", "required": [ "index", "url" ], "properties": { "index": { "minimum": 1, "type": "integer" }, "url": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" } } } }, "format": { "const": "ndjson", "type": "string" }, "compression": { "const": "gzip", "type": "string" }, "expiresAt": { "format": "date-time", "type": "string" } } } ] }, "url": { "format": "uri", "type": "string" }, "maxUrls": { "minimum": 1, "type": "integer" } } }, "BatchScrapeRequest": { "additionalProperties": false, "type": "object", "required": [ "proxyConnectorId", "destination", "urls" ], "properties": { "proxyConnectorId": { "pattern": "^pxy_[A-Za-z0-9_-]+$", "type": "string" }, "destination": { "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "storageConnectorId" ], "properties": { "storageConnectorId": { "pattern": "^sto_[A-Za-z0-9_-]+$", "type": "string" } } }, { "additionalProperties": false, "type": "object", "required": [ "mode", "provider", "shardPutUrls", "format", "compression", "expiresAt" ], "properties": { "mode": { "const": "presigned_shards", "type": "string" }, "provider": { "anyOf": [ { "const": "aws_s3", "type": "string" }, { "const": "cloudflare_r2", "type": "string" }, { "const": "backblaze_b2", "type": "string" } ] }, "manifestPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "summaryPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "successPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "shardPutUrls": { "minItems": 1, "maxItems": 1000, "type": "array", "items": { "additionalProperties": false, "type": "object", "required": [ "index", "url" ], "properties": { "index": { "minimum": 1, "type": "integer" }, "url": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" } } } }, "format": { "const": "ndjson", "type": "string" }, "compression": { "const": "gzip", "type": "string" }, "expiresAt": { "format": "date-time", "type": "string" } } } ] }, "urls": { "minItems": 1, "type": "array", "items": { "format": "uri", "type": "string" } }, "format": { "default": "raw", "anyOf": [ { "const": "raw", "type": "string" }, { "const": "html", "type": "string" }, { "const": "text", "type": "string" }, { "const": "markdown", "type": "string" } ] }, "render": { "default": false, "type": "boolean" } } }, "CrawlRequest": { "additionalProperties": false, "type": "object", "required": [ "proxyConnectorId", "destination", "url", "maxDepth", "maxUrls" ], "properties": { "proxyConnectorId": { "pattern": "^pxy_[A-Za-z0-9_-]+$", "type": "string" }, "destination": { "anyOf": [ { "additionalProperties": false, "type": "object", "required": [ "storageConnectorId" ], "properties": { "storageConnectorId": { "pattern": "^sto_[A-Za-z0-9_-]+$", "type": "string" } } }, { "additionalProperties": false, "type": "object", "required": [ "mode", "provider", "shardPutUrls", "format", "compression", "expiresAt" ], "properties": { "mode": { "const": "presigned_shards", "type": "string" }, "provider": { "anyOf": [ { "const": "aws_s3", "type": "string" }, { "const": "cloudflare_r2", "type": "string" }, { "const": "backblaze_b2", "type": "string" } ] }, "manifestPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "summaryPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "successPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "shardPutUrls": { "minItems": 1, "maxItems": 1000, "type": "array", "items": { "additionalProperties": false, "type": "object", "required": [ "index", "url" ], "properties": { "index": { "minimum": 1, "type": "integer" }, "url": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" } } } }, "format": { "const": "ndjson", "type": "string" }, "compression": { "const": "gzip", "type": "string" }, "expiresAt": { "format": "date-time", "type": "string" } } } ] }, "url": { "format": "uri", "type": "string" }, "maxDepth": { "minimum": 0, "type": "integer" }, "maxUrls": { "minimum": 1, "type": "integer" }, "render": { "default": false, "type": "boolean" } } }, "Job": { "additionalProperties": false, "type": "object", "required": [ "id", "type", "status", "discovered", "queued", "inFlight", "completed", "succeeded", "blocked", "failed", "uniqueExitIps", "novelExitIps", "shardsCommitted", "uploadBacklogBytes", "destinationStatus", "currentResourceUnits", "createdAt", "updatedAt" ], "properties": { "id": { "pattern": "^job_[A-Za-z0-9_-]+$", "type": "string" }, "type": { "anyOf": [ { "const": "scrape", "type": "string" }, { "const": "map", "type": "string" }, { "const": "batch_scrape", "type": "string" }, { "const": "crawl", "type": "string" }, { "const": "proxy_discovery", "type": "string" }, { "const": "proxy_qualification", "type": "string" }, { "const": "provider_compare", "type": "string" }, { "const": "extract", "type": "string" } ] }, "status": { "anyOf": [ { "const": "QUEUED", "type": "string" }, { "const": "PREFLIGHT", "type": "string" }, { "const": "RUNNING", "type": "string" }, { "const": "WAITING_FOR_PROXY_CONNECTOR", "type": "string" }, { "const": "WAITING_FOR_STORAGE", "type": "string" }, { "const": "WAITING_FOR_QUOTA", "type": "string" }, { "const": "PAUSED_BY_USER", "type": "string" }, { "const": "PAUSED_BY_OPERATOR", "type": "string" }, { "const": "CANCELLING", "type": "string" }, { "const": "SUCCEEDED", "type": "string" }, { "const": "FAILED", "type": "string" }, { "const": "CANCELLED", "type": "string" }, { "const": "EXPIRED", "type": "string" } ] }, "discovered": { "minimum": 0, "type": "integer" }, "queued": { "minimum": 0, "type": "integer" }, "inFlight": { "minimum": 0, "type": "integer" }, "completed": { "minimum": 0, "type": "integer" }, "succeeded": { "minimum": 0, "type": "integer" }, "blocked": { "minimum": 0, "type": "integer" }, "failed": { "minimum": 0, "type": "integer" }, "uniqueExitIps": { "minimum": 0, "type": "integer" }, "novelExitIps": { "minimum": 0, "type": "integer" }, "shardsCommitted": { "minimum": 0, "type": "integer" }, "uploadBacklogBytes": { "minimum": 0, "type": "integer" }, "destinationStatus": { "anyOf": [ { "const": "healthy", "type": "string" }, { "const": "capacity_low", "type": "string" }, { "const": "failed", "type": "string" }, { "const": "waiting", "type": "string" } ] }, "currentResourceUnits": { "minimum": 0, "type": "number" }, "createdAt": { "format": "date-time", "type": "string" }, "updatedAt": { "format": "date-time", "type": "string" } } }, "JobEvent": { "additionalProperties": false, "description": "Compact progress event; full page bodies are forbidden.", "type": "object", "required": [ "id", "type", "occurredAt", "data" ], "properties": { "id": { "minimum": 1, "type": "integer" }, "type": { "anyOf": [ { "const": "job.started", "type": "string" }, { "const": "job.queued", "type": "string" }, { "const": "job.running", "type": "string" }, { "const": "discovery.progress", "type": "string" }, { "const": "task.completed", "type": "string" }, { "const": "exit.observed", "type": "string" }, { "const": "value.provisional", "type": "string" }, { "const": "value.settled", "type": "string" }, { "const": "shard.uploaded", "type": "string" }, { "const": "manifest.updated", "type": "string" }, { "const": "destination.capacity_low", "type": "string" }, { "const": "destination.failed", "type": "string" }, { "const": "connector.degraded", "type": "string" }, { "const": "job.waiting", "type": "string" }, { "const": "job.resumed", "type": "string" }, { "const": "job.completed", "type": "string" }, { "const": "job.failed", "type": "string" }, { "const": "job.cancelled", "type": "string" } ] }, "occurredAt": { "format": "date-time", "type": "string" }, "data": { "type": "object", "patternProperties": { "^(.*)$": {} } } } }, "NotificationPreferences": { "additionalProperties": false, "type": "object", "required": [ "connectorNeedsAttention", "jobFailed", "jobWaitingForStorage", "jobWaitingForProxy", "quotaExhausted", "jobCompleted", "tierChanged", "weeklyDigest" ], "properties": { "connectorNeedsAttention": { "type": "boolean" }, "jobFailed": { "type": "boolean" }, "jobWaitingForStorage": { "type": "boolean" }, "jobWaitingForProxy": { "type": "boolean" }, "quotaExhausted": { "type": "boolean" }, "jobCompleted": { "type": "boolean" }, "tierChanged": { "type": "boolean" }, "weeklyDigest": { "type": "boolean" } } }, "NotificationPreferencesPatch": { "additionalProperties": false, "type": "object", "properties": { "connectorNeedsAttention": { "type": "boolean" }, "jobFailed": { "type": "boolean" }, "jobWaitingForStorage": { "type": "boolean" }, "jobWaitingForProxy": { "type": "boolean" }, "quotaExhausted": { "type": "boolean" }, "jobCompleted": { "type": "boolean" }, "tierChanged": { "type": "boolean" }, "weeklyDigest": { "type": "boolean" } }, "minProperties": 1 }, "DestinationShards": { "additionalProperties": false, "type": "object", "required": [ "mode", "provider", "shardPutUrls", "format", "compression", "expiresAt" ], "properties": { "mode": { "const": "presigned_shards", "type": "string" }, "provider": { "anyOf": [ { "const": "aws_s3", "type": "string" }, { "const": "cloudflare_r2", "type": "string" }, { "const": "backblaze_b2", "type": "string" } ] }, "manifestPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "summaryPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "successPutUrl": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" }, "shardPutUrls": { "minItems": 1, "maxItems": 1000, "type": "array", "items": { "additionalProperties": false, "type": "object", "required": [ "index", "url" ], "properties": { "index": { "minimum": 1, "type": "integer" }, "url": { "format": "uri", "maxLength": 4096, "writeOnly": true, "type": "string" } } } }, "format": { "const": "ndjson", "type": "string" }, "compression": { "const": "gzip", "type": "string" }, "expiresAt": { "format": "date-time", "type": "string" } } }, "OutputDestination": { "additionalProperties": false, "description": "Secret-free output metadata. Presigned capability URLs are never returned.", "type": "object", "required": [ "mode", "provider", "status", "format", "compression", "sequenceHighWatermark", "recordsCommitted", "shardsCommitted", "manifestStatus", "updatedAt" ], "properties": { "mode": { "anyOf": [ { "const": "saved_connector", "type": "string" }, { "const": "presigned_shards", "type": "string" } ] }, "provider": { "anyOf": [ { "const": "aws_s3", "type": "string" }, { "const": "cloudflare_r2", "type": "string" }, { "const": "backblaze_b2", "type": "string" } ] }, "status": { "anyOf": [ { "const": "healthy", "type": "string" }, { "const": "capacity_low", "type": "string" }, { "const": "waiting", "type": "string" }, { "const": "failed", "type": "string" }, { "const": "complete", "type": "string" } ] }, "format": { "const": "ndjson", "type": "string" }, "compression": { "const": "gzip", "type": "string" }, "sequenceHighWatermark": { "minimum": 0, "type": "integer" }, "recordsCommitted": { "minimum": 0, "type": "integer" }, "shardsCommitted": { "minimum": 0, "type": "integer" }, "manifestStatus": { "anyOf": [ { "const": "running", "type": "string" }, { "const": "terminal_pending", "type": "string" }, { "const": "complete", "type": "string" }, { "const": "failed", "type": "string" }, { "const": "cancelled", "type": "string" } ] }, "updatedAt": { "format": "date-time", "type": "string" } } }, "Capabilities": { "additionalProperties": false, "type": "object", "required": [ "authenticatedCompute", "anonymousCompute", "engines", "jobs", "providers", "limits", "degraded" ], "properties": { "authenticatedCompute": { "const": true, "type": "boolean" }, "anonymousCompute": { "const": false, "type": "boolean" }, "engines": { "additionalProperties": false, "type": "object", "required": [ "http", "staticRecovery", "v8", "render" ], "properties": { "http": { "type": "boolean" }, "staticRecovery": { "type": "boolean" }, "v8": { "type": "boolean" }, "render": { "type": "boolean" } } }, "jobs": { "additionalProperties": false, "type": "object", "required": [ "map", "batchScrape", "crawl", "sse" ], "properties": { "map": { "type": "boolean" }, "batchScrape": { "type": "boolean" }, "crawl": { "type": "boolean" }, "sse": { "type": "boolean" } } }, "providers": { "additionalProperties": false, "type": "object", "required": [ "proxy", "storage", "captcha", "llm" ], "properties": { "proxy": { "type": "array", "items": { "type": "string" } }, "storage": { "type": "array", "items": { "type": "string" } }, "captcha": { "type": "array", "items": { "type": "string" } }, "llm": { "type": "array", "items": { "type": "string" } } } }, "limits": { "additionalProperties": false, "type": "object", "required": [ "maxSyncBytes", "maxSyncTimeoutMs" ], "properties": { "maxSyncBytes": { "minimum": 1, "type": "integer" }, "maxSyncTimeoutMs": { "minimum": 1, "type": "integer" } } }, "degraded": { "type": "array", "items": { "type": "string" } } } }, "Problem": { "additionalProperties": false, "type": "object", "required": [ "type", "title", "status", "detail", "requestId" ], "properties": { "type": { "format": "uri", "type": "string" }, "title": { "type": "string" }, "status": { "minimum": 400, "maximum": 599, "type": "integer" }, "detail": { "type": "string" }, "requestId": { "type": "string" } } } } } }