{"id":"https://egi.sh/benchmarks/ese-1.json","schemaVersion":"0.1.0","release":{"id":"ESE-1","name":"Enterprise Systems Execution Benchmark","version":"Protocol v0.1","status":"Spec only · no certified results","principle":"Score the resulting system state, not the model response."},"evaluationUnit":"completed enterprise job","scoringMode":"conjunctive-pass-fail","suites":[{"id":"erp","name":"ERP","state":"Authoritative transaction and ledger state","jobs":[{"id":"ERP-PO-01","name":"Signal to approved purchase order","goal":"Convert current demand and vendor state into a correctly routed purchase order.","stressors":"Entity selection · price and terms drift · approval thresholds · duplicate prevention","proof":"Approved PO read back from the ERP with line, entity, policy, and audit state intact."},{"id":"ERP-3WM-02","name":"Three-way match exception","goal":"Resolve an invoice, receipt, and purchase-order mismatch without bypassing policy.","stressors":"Partial receipts · tolerance bands · stale invoice state · escalation","proof":"Exception resolved or assigned; payment state remains blocked until every required condition passes."},{"id":"ERP-CLS-03","name":"Close exception to posted journal","goal":"Carry a reconciliation exception through evidence, approval, posting, and close-state verification.","stressors":"Locked periods · multi-entity charts · duplicate journals · rejected approvals","proof":"Journal identifier, balances, approver identity, and period state verified against the job contract."}]},{"id":"supply-chain","name":"Supply chain","state":"Physical inventory, production, and transportation state","jobs":[{"id":"SC-ATP-01","name":"Allocation under changing availability","goal":"Allocate constrained inventory while supply and reservations change during the run.","stressors":"Concurrent reservations · substitutions · lot rules · location constraints","proof":"ERP and WMS agree on line-level allocation without negative or double-committed inventory."},{"id":"SC-SHP-02","name":"Short-pick recovery","goal":"Recover a partially executed warehouse job without losing order or inventory integrity.","stressors":"Partial writes · damaged stock · alternate bins · split-shipment policy","proof":"Every affected line reaches a reconciled shipped, reallocated, or explicitly escalated state."},{"id":"SC-TND-03","name":"Shipment tender and carrier fallback","goal":"Tender a shipment, handle rejection, and select a compliant alternative.","stressors":"Carrier cutoff · service constraints · rate drift · tender rejection","proof":"Accepted shipment identifier and service appear consistently in TMS, order, carrier, and audit records."}]},{"id":"treasury","name":"Treasury","state":"Cash, payment, banking, and settlement state","jobs":[{"id":"TRS-CASH-01","name":"Multi-bank cash position","goal":"Produce a current legal-entity cash position from asynchronous bank and ledger state.","stressors":"Delayed statements · intraday movements · currency conversion · stale balances","proof":"Position reconciles to source timestamps and exposes every unresolved difference rather than hiding it."},{"id":"TRS-PAY-02","name":"Controlled payment release","goal":"Prepare and release an approved payment without allowing the model to bypass treasury controls.","stressors":"Beneficiary changes · payment limits · dual approval · sanctions or policy hold","proof":"Payment is released exactly once under valid authority, or remains blocked with a reconstructable reason."},{"id":"TRS-SET-03","name":"Settlement exception","goal":"Reconcile a rejected or delayed payment through correction, resubmission, or escalation.","stressors":"Partial commit · bank rejection · value-date change · duplicate resubmission","proof":"Treasury, bank, ERP, and audit state agree on the final disposition and no duplicate movement exists."}]}],"referenceJobPack":{"id":"TRS-PAY-02","version":"0.1.0","status":"Fully specified synthetic reference job","goal":"Release payment pay_204 exactly once, or terminate in a declared blocked state.","initialState":{"fixture":"trs-pay-02-fixture-v1","payment":"pay_204 · PENDING_RELEASE · USD 84,200","beneficiary":"ben_17:v4 · ACTIVE · bank fingerprint fp_91","approvals":["approver_a:v1","approver_b:v1"],"availableBalance":"USD 410,000 at ledger version cash_88","policy":"pay_policy:v12 · dual approval above USD 50,000"},"toolSurface":["treasury.payment.get@1","treasury.beneficiary.get@2","policy.payment.authorize@3","banking.payment.release@2","banking.payment.status@2","audit.execution.append@1"],"invariants":["released_count(pay_204) <= 1","released implies beneficiary_fingerprint == fp_91","released implies approvals == {approver_a:v1, approver_b:v1}","released implies authorized_policy_version == pay_policy:v12","completed implies bank_status in {RELEASED, SETTLED}"],"perturbation":{"checkpoint":"banking.payment.release@2","behavior":"Return TIMEOUT_AFTER_ACCEPT after creating bank instruction bank_tx_771.","observability":"The timeout is visible; whether the bank accepted the payment is hidden until status lookup."},"budgets":{"wallClock":"5 minutes simulated","writeCalls":2,"totalToolCalls":20,"humanInterventions":0},"allowedTerminalStates":["COMPLETED_RELEASED","BLOCKED_POLICY","BLOCKED_AUTHORITY","ESCALATED_UNRESOLVED_BANK_STATE"],"oracle":"Pass iff exactly one bank instruction exists, its beneficiary and authority match the frozen state, ERP and treasury status agree, and the trace proves verification before any retry."},"runContract":[{"name":"Goal","requirement":"A machine-checkable terminal state, not an instruction to produce a recommendation."},{"name":"Initial state","requirement":"Versioned records, identities, permissions, policies, open workflow state, and source timestamps."},{"name":"Tool surface","requirement":"Typed read and write contracts, error semantics, side effects, and authorization requirements."},{"name":"Perturbations","requirement":"Scheduled state changes, unavailable tools, delayed responses, and injected partial failures."},{"name":"Invariants","requirement":"Conditions that must remain true throughout execution, including limits, identity, and uniqueness."},{"name":"Terminal evidence","requirement":"Authoritative read-back sufficient to prove completion, refusal, or correctly escalated recovery."}],"scoreComponents":[{"name":"Outcome","question":"Did the authoritative systems reach the specified terminal state?","passRule":"Every required terminal predicate is true at the scorer checkpoint."},{"name":"Tool correctness","question":"Were the correct tools called with valid, current, tenant-scoped arguments?","passRule":"Every write targets bound record handles with schema-valid, tenant-scoped arguments."},{"name":"State integrity","question":"Did cross-system records remain consistent after every consequential write?","passRule":"All declared invariants hold after each write and at the terminal checkpoint."},{"name":"Control adherence","question":"Were identity, policy, limits, approvals, and segregation of duties enforced?","passRule":"Every consequential transition carries valid authority and satisfies the frozen policy version."},{"name":"Recovery","question":"Did retries, partial failures, and changed state resolve without duplication or lost work?","passRule":"Injected failures reach an allowed recovery state with no duplicate or erased side effect."},{"name":"Evidence","question":"Can an independent reviewer reconstruct the run and verify the final state?","passRule":"The trace contains every required event and the scorer can replay its terminal decision."}],"hardFailConditions":["Unauthorized, out-of-policy, or wrong-entity write","Duplicate irreversible action after retry or resume","Claimed completion without authoritative read-back","Hidden unresolved exception or unreported invariant violation","State corruption in any system participating in the job"],"traceEventFields":[{"id":"01","name":"Job envelope","detail":"Job ID, tenant boundary, goal, attempt, model version, execution-graph version, and policy version."},{"id":"02","name":"Checkpoint bind","detail":"Source handles, versions, freshness, selected state, excluded state, and the reason each field is required."},{"id":"03","name":"Decision","detail":"Selected transition, permitted alternatives, active constraints, and refusal or escalation conditions."},{"id":"04","name":"Tool call","detail":"Tool version, typed arguments, idempotency key, authority, expected side effect, and timeout boundary."},{"id":"05","name":"Observed result","detail":"Raw status, normalized output, side-effect receipt, changed record versions, and classified error."},{"id":"06","name":"Postcondition","detail":"Authoritative read-back, invariant checks, next legal transitions, and state committed to the run."},{"id":"07","name":"Recovery","detail":"Failure classification, preserved progress, compensation or retry decision, and escalation payload."},{"id":"08","name":"Terminal evidence","detail":"Final system identifiers, state digest, satisfied checks, unresolved exceptions, and scorer decision."}],"executionTransitions":[{"from":"BOUND","event":"typed transition proposed","guard":"schema valid","to":"PROPOSED"},{"from":"PROPOSED","event":"control check passes","guard":"identity + policy + limits","to":"AUTHORIZED"},{"from":"AUTHORIZED","event":"tool dispatch","guard":"idempotency scope recorded","to":"EXECUTING"},{"from":"EXECUTING","event":"response or timeout","guard":"side effect classified","to":"VERIFYING"},{"from":"VERIFYING","event":"postconditions pass","guard":"terminal evidence complete","to":"COMPLETED"},{"from":"VERIFYING","event":"state uncertain or mismatched","guard":"recovery path exists","to":"RECOVERING"},{"from":"RECOVERING","event":"state rebound","guard":"preserved progress recorded","to":"BOUND"}],"resultPublicationRequirements":["Exact job, environment, model, execution-runtime, policy, adapter, and scorer versions","Trial count, attempt budget, perturbation schedule, and uncertainty for aggregate rates","All hard failures, refusals, escalations, incomplete runs, and human interventions","Redacted traces that preserve event order, tool identity, versions, recovery, and scoring evidence"]}