Forest Web
Interpret the robots.txt crawling rules that apply to a specific public URL and user-agent before an automated fetch or crawl. Returns allow/disallow decision evidence, matched rules, crawl delay and sitemap hints from one bounded robots.txt read. This is protocol guidance, not authorization or an anti-bot bypass. Base mainnet USDC
Offering
- Provider
- http--forest-gas-station-mainnet--lcjl27p8lmjs.code.run
- Method
- POST
- Offering type
- Api Endpoint
- Transport type
- http
- Category
- Web Search Research
- Delivery
- Not provided
- Pricing model
- Not provided
- x402 version
- 2
Endpoint context
- URL
https://http--forest-gas-station-mainnet--lcjl27p8lmjs.code.run/robots/context- MIME type
- Not provided
- First seen
- 2026-10-16T01:50:45.949Z
- Last seen
- Tags
- Web, robots, robots-txt, crawler, crawl-policy
Payment pricing
x402 payment terms are the authoritative price for calling this endpoint.
- Payment option 1
- 0.001 USDC
Atomic price: 1000 · Network: eip155:8453 · Asset: USDC · Scheme: exact
Raw x402 terms
[
{
"scheme": "exact",
"network": "eip155:8453",
"asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
"amount": "1000",
"payTo": "0xb48D918151311F982d542eF5C84cc1cd54CFf7a9"
}
]Observed health
- Status
- Reachable
- Latest latency
- 262 ms
- Average latency
- 578 ms
- HTTP status
- 405
- Last checked
- Reachability
- 100% over 12 checks
Latency is measured to the first response headers. A reachable result does not verify response accuracy or merchant identity.
Input schema
{
"additionalProperties": false,
"properties": {
"body": {
"additionalProperties": false,
"properties": {
"url": {
"description": "Public HTTP(S) URL to inspect, resolve, or compare.",
"maxLength": 2048,
"minLength": 1,
"pattern": "^https?://",
"type": "string"
},
"user_agent": {
"description": "Public user agent value used to scope this lookup.",
"maxLength": 128,
"minLength": 1,
"type": "string"
}
},
"required": [
"url"
],
"type": "object"
},
"bodyType": {
"enum": [
"json",
"form-data",
"text"
],
"type": "string"
},
"method": {
"enum": [
"POST"
],
"type": "string"
},
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type",
"method",
"bodyType",
"body"
],
"type": "object"
}Output schema
{
"properties": {
"example": {
"additionalProperties": false,
"properties": {
"capability": {
"const": "robots.context",
"description": "Public capability value used to scope this lookup."
},
"data": {
"additionalProperties": false,
"description": "Public data value used to scope this lookup.",
"properties": {
"crawl_delay": {
"description": "Public crawl delay value used to scope this lookup.",
"minimum": 0,
"type": [
"number",
"null"
]
},
"decision": {
"description": "Public decision value used to scope this lookup.",
"enum": [
"allowed",
"disallowed",
"unknown"
]
},
"matched_rule": {
"anyOf": [
{
"additionalProperties": false,
"properties": {
"directive": {
"description": "Public directive value used to scope this lookup.",
"enum": [
"allow",
"disallow"
]
},
"pattern": {
"description": "Public pattern value used to scope this lookup.",
"type": "string"
},
"specificity": {
"description": "Public specificity value used to scope this lookup.",
"minimum": 0,
"type": "integer"
}
},
"required": [
"directive",
"pattern",
"specificity"
],
"type": "object"
},
{
"type": "null"
}
],
"description": "Public matched rule value used to scope this lookup."
},
"matching_agents": {
"description": "Public matching agents value used to scope this lookup.",
"items": {
"type": "string"
},
"type": "array"
},
"rendered_javascript": {
"const": false,
"description": "Public rendered javascript value used to scope this lookup."
},
"robots_state": {
"description": "Public robots state value used to scope this lookup.",
"enum": [
"available",
"unavailable",
"temporarily_unavailable"
]
},
"robots_url": {
"description": "Public robots url value used to scope this lookup.",
"type": "string"
},
"rules": {
"description": "Public rules value used to scope this lookup.",
"items": {
"additionalProperties": false,
"properties": {
"directive": {
"description": "Public directive value used to scope this lookup.",
"enum": [
"allow",
"disallow"
]
},
"pattern": {
"description": "Public pattern value used to scope this lookup.",
"type": "string"
}
},
"required": [
"directive",
"pattern"
],
"type": "object"
},
"maxItems": 100,
"type": "array"
},
"sitemaps": {
"description": "Public sitemaps value used to scope this lookup.",
"items": {
"type": "string"
},
"maxItems": 50,
"type": "array"
},
"status": {
"description": "Public status value used to scope this lookup.",
"type": [
"integer",
"null"
]
},
"target_url": {
"description": "Public target url value used to scope this lookup.",
"type": "string"
},
"truncated": {
"description": "Public truncated value used to scope this lookup.",
"type": "boolean"
},
"user_agent": {
"description": "Public user agent value used to scope this lookup.",
"type": "string"
}
},
"required": [
"target_url",
"robots_url",
"status",
"robots_state",
"user_agent",
"matching_agents",
"decision",
"matched_rule",
"rules",
"crawl_delay",
"sitemaps",
"truncated",
"rendered_javascript"
],
"type": "object"
},
"meta": {
"additionalProperties": false,
"description": "Public meta value used to scope this lookup.",
"properties": {
"cache": {
"description": "Public cache value used to scope this lookup.",
"enum": [
"hit",
"miss",
"bypass"
]
},
"checked_at": {
"description": "Public checked at value used to scope this lookup.",
"type": "string"
},
"duration_ms": {
"description": "Public duration ms value used to scope this lookup.",
"minimum": 0,
"type": "integer"
},
"provider": {
"description": "Public provider value used to scope this lookup.",
"type": [
"string",
"null"
]
}
},
"required": [
"provider",
"checked_at",
"cache",
"duration_ms"
],
"type": "object"
},
"ok": {
"const": true,
"description": "Public ok value used to scope this lookup."
},
"version": {
"const": "1",
"description": "Package version whose public metadata should be checked."
}
},
"required": [
"ok",
"capability",
"version",
"data",
"meta"
],
"type": "object"
},
"type": {
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}