{
  "aliases": [],
  "authors": [
    "Divyansh Garg",
    "Shaun VanWeelden",
    "Diego Caples",
    "Andis Draguns",
    "Nikil Ravi",
    "Pranav Putta",
    "Naman Garg",
    "Tomas Abraham",
    "Michael Lara",
    "Federico Lopez",
    "James Liu",
    "Atharva Gundawar",
    "Prannay Hebbar",
    "Youngchul Joo",
    "Jindong Gu",
    "Charles London",
    "Christian Schroeder de Witt",
    "Sumeet Motwani"
  ],
  "details": {
    "arxiv_snapshot": {
      "abstract_hash_method": "sha256-utf8-whitespace-collapsed-v1",
      "abstract_sha256": "d8fe0d8b34d167311d1c37934fdb06e81939652ea579b2578a40b6a709fbfb91",
      "submitted_at": "2025-04-15T18:22:55Z",
      "updated_at": "2025-04-17T16:28:46Z",
      "version": 2,
      "versioned_url": "https://arxiv.org/abs/2504.11543v2"
    }
  },
  "environments": [
    "web"
  ],
  "evidence": [
    {
      "method": "arxiv_api",
      "retrieved_at": "2026-10-17T07:57:47Z",
      "role": "primary",
      "sha256": "27144160e8981bb9f36b60c391689774913ff3f026eeeffc2923154c404a6529",
      "supports": [
        "name",
        "authors",
        "published_at",
        "updated_at",
        "identifiers.arxiv",
        "details.arxiv_snapshot"
      ],
      "url": "https://export.arxiv.org/api/query?id_list=2504.03206%2C2504.03601%2C2504.06821%2C2504.09723%2C2504.11301%2C2504.11543%2C2504.12609%2C2504.14233%2C2504.14348%2C2504.14603%2C2504.16073%2C2504.16419%2C2504.17892%2C2504.18580%2C2504.20965%2C2505.00019%2C2505.00693%2C2505.01578%2C2505.02094%2C2505.03570%2C2505.03574%2C2505.05026%2C2505.05849%2C2505.06507%2C2505.07003%2C2505.07473%2C2505.10468%2C2505.10887%2C2505.12053%2C2505.12490%2C2505.12728%2C2505.13291%2C2505.14030%2C2505.14454%2C2505.14646%2C2505.15801%2C2505.15816%2C2505.17020%2C2505.18746%2C2505.19095%2C2505.19713%2C2505.20023%2C2505.20732%2C2505.20795%2C2505.21055%2C2505.22304%2C2505.22429%2C2505.23450%2C2505.23559%2C2506.00641&max_results=50"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-11-05T23:01:33Z",
      "role": "primary",
      "sha256": "b7001e189c17d6fa072573558661e37a7cb4c7709d33080603d72a969cc4ba39",
      "supports": [
        "name",
        "kind",
        "summary",
        "environments",
        "topics",
        "scope",
        "relations"
      ],
      "url": "https://export.arxiv.org/api/query?id_list=2503.02878%2C2503.06378%2C2503.06580%2C2503.07365%2C2503.07701%2C2503.08026%2C2503.08175%2C2503.10613%2C2503.10809%2C2503.11579%2C2503.12228%2C2503.12461%2C2503.12687%2C2503.13843%2C2503.14495%2C2503.15426%2C2503.16036%2C2503.16248%2C2503.16365%2C2503.16975%2C2503.17671%2C2503.18102%2C2503.20036%2C2503.21889%2C2503.21949%2C2503.23803%2C2504.01328%2C2504.01377%2C2504.03206%2C2504.03601%2C2504.06821%2C2504.07491%2C2504.09723%2C2504.10147%2C2504.11301%2C2504.11543%2C2504.11703%2C2504.12609%2C2504.12682%2C2504.14233%2C2504.14348%2C2504.17892%2C2504.18580%2C2504.20965%2C2505.00019%2C2505.00693%2C2505.01578%2C2505.02094%2C2505.03574%2C2505.03733&max_results=50"
    }
  ],
  "id": "arxiv:2504.11543",
  "identifiers": {
    "arxiv": "2504.11543"
  },
  "kind": "paper",
  "last_verified_at": "2026-11-05T23:01:33Z",
  "licenses": {
    "code": null,
    "data": null,
    "document": null,
    "weights": null
  },
  "links": [
    {
      "rel": "primary",
      "url": "https://arxiv.org/abs/2504.11543"
    }
  ],
  "name": "REAL: Benchmarking Autonomous Agents on Deterministic Simulations of Real Websites",
  "notes": "The annotation and scope were reviewed against the complete retained arXiv abstract (v2). All evaluated interactions occur in controlled simulations of websites. Experiments, reported performance and artifact availability have not been independently reproduced.",
  "published_at": "2025-04-15",
  "relations": [],
  "review_status": "curated",
  "scope": "core",
  "summary": "Introduces REAL, a benchmark of deterministic website replicas with 112 tasks covering information retrieval and state-changing browser actions.",
  "topics": [
    "evaluation",
    "sandbox",
    "web-navigation"
  ],
  "updated_at": "2025-04-17",
  "url": "https://arxiv.org/abs/2504.11543"
}