Index onlineMerchant portal
Api Endpoint

Tanod sitepeek

sitepeek (Tanod) robots.txt check: whether a crawler may fetch a URL under RFC 9309, with the matching rule, crawl delay and sitemaps. Read-only. Use when an agent must know whether a crawler may fetch a URL under the site's robots.txt before scraping it.

Open interactive test console

Offering

Provider
tanod.dev
Method
POST
Offering type
Api Endpoint
Transport type
http
Category
Infrastructure
Delivery
Not provided
Pricing model
Not provided
x402 version
2

Endpoint context

URL
https://tanod.dev/v1/robots
MIME type
application/json
First seen
2026-10-16T01:51:02.144Z
Last seen
Tags
robots-txt, crawler, web-scraping, crawl-permissions, sitepeek, render, markdown, pdf, ocr, robots

Website

Payment pricing

x402 payment terms are the authoritative price for calling this endpoint.

Payment option 1
0.001 USDC
Atomic price: 1000 · Network: eip155:8453 · Asset: USDC · Scheme: exact
Payment option 2
0.001 USDC
Atomic price: 1000 · Network: eip155:137 · Asset: USDC · Scheme: exact
Payment option 3
0.001 USDC
Atomic price: 1000 · Network: solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp · Asset: USDC · Scheme: exact
Payment option 4
0.001 USDC
Atomic price: 1000000000000000 · Network: eip155:56 · Asset: USDC · Scheme: exact
Payment option 5
0.001 USDC
Atomic price: 1000 · Network: eip155:42161 · Asset: USDC · Scheme: exact
Payment option 6
0.001 USDC
Atomic price: 1000 · Network: eip155:8453 · Asset: USDC · Scheme: exact
Payment option 7
0.001 USDC
Atomic price: 1000 · Network: eip155:137 · Asset: USDC · Scheme: exact
Raw x402 terms
[
  {
    "scheme": "exact",
    "network": "eip155:8453",
    "asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
    "amount": "1000",
    "payTo": "0x593857A4a4F619543ea12394137C3004ce841720"
  },
  {
    "scheme": "exact",
    "network": "eip155:137",
    "asset": "0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359",
    "amount": "1000",
    "payTo": "0x593857A4a4F619543ea12394137C3004ce841720"
  },
  {
    "scheme": "exact",
    "network": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp",
    "asset": "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
    "amount": "1000",
    "payTo": "RMk9k3VC8dr4hhjgFh88rGhjAbzfBzHxJJ5n2fTcqus"
  },
  {
    "scheme": "exact",
    "network": "eip155:56",
    "asset": "0x8AC76a51cc950d9822D68b83fE1Ad97B32Cd580d",
    "amount": "1000000000000000",
    "payTo": "0x593857A4a4F619543ea12394137C3004ce841720"
  },
  {
    "scheme": "exact",
    "network": "eip155:42161",
    "asset": "0xaf88d065e77c8cC2239327C5EDb3A432268e5831",
    "amount": "1000",
    "payTo": "0x593857A4a4F619543ea12394137C3004ce841720"
  },
  {
    "scheme": "exact",
    "network": "eip155:8453",
    "asset": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
    "amount": "1000",
    "payTo": "0x593857A4a4F619543ea12394137C3004ce841720"
  },
  {
    "scheme": "exact",
    "network": "eip155:137",
    "asset": "0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359",
    "amount": "1000",
    "payTo": "0x593857A4a4F619543ea12394137C3004ce841720"
  }
]

Observed health

Status
Healthy
Latest latency
346 ms
Average latency
947 ms
HTTP status
204
Last checked
Reachability
100% over 11 checks

Latency is measured to the first response headers. A reachable result does not verify response accuracy or merchant identity.

Input schema

{
  "type": "object",
  "properties": {
    "type": {
      "type": "string",
      "const": "http"
    },
    "method": {
      "type": "string",
      "enum": [
        "POST"
      ]
    },
    "bodyType": {
      "type": "string",
      "enum": [
        "json",
        "form-data",
        "text"
      ]
    },
    "body": {
      "additionalProperties": false,
      "properties": {
        "url": {
          "description": "Public http(s) URL to test; <origin>/robots.txt is fetched and the URL's path is matched against it.",
          "maxLength": 2048,
          "minLength": 1,
          "type": "string"
        },
        "user_agent": {
          "default": "*",
          "description": "Product token (e.g. Googlebot) or a full User-Agent string; '*' (default) = any crawler.",
          "maxLength": 200,
          "minLength": 1,
          "type": "string"
        }
      },
      "required": [
        "url"
      ],
      "type": "object"
    }
  },
  "required": [
    "type",
    "method",
    "bodyType",
    "body"
  ],
  "additionalProperties": false
}

Output schema

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "properties": {
    "input": {
      "additionalProperties": false,
      "properties": {
        "body": {
          "additionalProperties": false,
          "properties": {
            "url": {
              "description": "Public http(s) URL to test; <origin>/robots.txt is fetched and the URL's path is matched against it.",
              "maxLength": 2048,
              "minLength": 1,
              "type": "string"
            },
            "user_agent": {
              "default": "*",
              "description": "Product token (e.g. Googlebot) or a full User-Agent string; '*' (default) = any crawler.",
              "maxLength": 200,
              "minLength": 1,
              "type": "string"
            }
          },
          "required": [
            "url"
          ],
          "type": "object"
        },
        "bodyType": {
          "enum": [
            "json",
            "form-data",
            "text"
          ],
          "type": "string"
        },
        "method": {
          "enum": [
            "POST"
          ],
          "type": "string"
        },
        "type": {
          "const": "http",
          "type": "string"
        }
      },
      "required": [
        "type",
        "method",
        "bodyType",
        "body"
      ],
      "type": "object"
    },
    "output": {
      "properties": {
        "example": {
          "properties": {
            "allowed": {
              "type": "boolean"
            },
            "crawl_delay": {
              "type": [
                "number",
                "string",
                "null"
              ]
            },
            "fetch": {
              "enum": [
                "ok",
                "unavailable",
                "unreachable"
              ],
              "type": "string"
            },
            "group": {
              "type": [
                "string",
                "null"
              ]
            },
            "http_status": {
              "type": [
                "integer",
                "null"
              ]
            },
            "matched_rule": {
              "properties": {
                "line": {
                  "type": "integer"
                },
                "pattern": {
                  "type": "string"
                },
                "type": {
                  "enum": [
                    "allow",
                    "disallow"
                  ],
                  "type": "string"
                }
              },
              "type": [
                "object",
                "null"
              ]
            },
            "path": {
              "type": "string"
            },
            "product_token": {
              "type": "string"
            },
            "reason": {
              "type": "string"
            },
            "robots_url": {
              "type": [
                "string",
                "null"
              ]
            },
            "sitemaps": {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            "sitemaps_total": {
              "type": "integer"
            },
            "stats": {
              "type": [
                "object",
                "null"
              ]
            },
            "truncated": {
              "type": "boolean"
            },
            "untrusted_content": {
              "type": "boolean"
            },
            "url": {
              "type": "string"
            },
            "user_agent": {
              "type": "string"
            }
          },
          "required": [
            "url",
            "robots_url",
            "user_agent",
            "product_token",
            "path",
            "http_status",
            "fetch",
            "allowed",
            "matched_rule",
            "group",
            "crawl_delay",
            "sitemaps",
            "sitemaps_total",
            "truncated",
            "stats",
            "reason",
            "untrusted_content"
          ],
          "type": "object"
        },
        "type": {
          "type": "string"
        }
      },
      "required": [
        "type"
      ],
      "type": "object"
    }
  },
  "required": [
    "input"
  ],
  "type": "object"
}