方法与工具 / INTERACTIVE EDITION · ETCD 3.6.6

真实实验证据

真实保存证据

逐步读懂一次真实运行

逐步回放三次保存运行中的指定观测。

正在加载互动… 交互需要 JavaScript；正文仍可直接阅读。

来自保存 JSON 的有序摘要，不是逐包回放。阅读原理 · 查看真实证据

来自真实进程的保存结果，三次重复全部可检查。

REAL PROCESS EVIDENCE / 2026-11-13T17:39:41.195190+00:00

拓扑与条件

etcd 3.6.6 · Darwin arm64 · Python 3.11.2

同机三个回环投票进程，各有独立 WAL。实际杀死/暂停进程并延迟 HTTP 响应；未做数据包分区、存储毁坏或多机性能测量。耗时使用编排者的单调时钟，包含请求往返。

下载证据 JSON 哈希清单

SHA-256: df927af381c415e574f1a48755345be7a1bcaee243fdbc3f73002a05ab6bbbf6

重复	Leader	1 down	2 down write/read	plain / dedup	Fence tokens
1	n1 → n2	ok	unknown / unknown	2 / 1	9 < 11
2	n1 → n3	ok	unknown / unknown	2 / 1	9 < 11
3	n1 → n3	ok	unknown / unknown	2 / 1	9 < 11

A · Leader 崩溃与多数派丢失

故障前已确认值保留，重启节点完成追赶。一个 Follower 不可响应时仍能写入；暂停两个 Follower 时，写和默认读在截止时间内均未成功。超时写的最终值单独核对。

REAL RUN / 01

n1 → n2; 1 down: ok; 2 down: unknown; reconciled = maybe-committed

检查完整导出记录

{
  "before": {
    "leader": "n1",
    "revision": 2,
    "term": 2
  },
  "new_leader": "n2",
  "committed_write_survived": "acknowledged-before-crash",
  "failover_write": {
    "outcome": "ok",
    "duration_ms": 11.609,
    "response": {
      "header": {
        "cluster_id": "17193519525117484670",
        "member_id": "9419852838338823221",
        "revision": "3",
        "raft_term": "3"
      }
    }
  },
  "wal_restart": {
    "node": "n1",
    "value": "acknowledged-before-crash",
    "after_crash_value": "committed"
  },
  "one_unreachable": {
    "outcome": "ok",
    "duration_ms": 13.268,
    "response": {
      "header": {
        "cluster_id": "17193519525117484670",
        "member_id": "9419852838338823221",
        "revision": "4",
        "raft_term": "3"
      }
    }
  },
  "two_unreachable_write": {
    "outcome": "unknown",
    "duration_ms": 702.506
  },
  "two_unreachable_linearizable_read": {
    "outcome": "unknown",
    "duration_ms": 701.85
  },
  "unknown_write_after_recovery": "maybe-committed",
  "conditions": {
    "nodes": 3,
    "quorum": 2,
    "faults": [
      "leader SIGKILL and same-data-dir restart",
      "one follower SIGSTOP",
      "two followers SIGSTOP"
    ],
    "election_timeout_ms": 1000,
    "client_deadline_ms": 700
  },
  "assertions_passed": true
}

REAL RUN / 02

n1 → n3; 1 down: ok; 2 down: unknown; reconciled = maybe-committed

检查完整导出记录

{
  "before": {
    "leader": "n1",
    "revision": 2,
    "term": 2
  },
  "new_leader": "n3",
  "committed_write_survived": "acknowledged-before-crash",
  "failover_write": {
    "outcome": "ok",
    "duration_ms": 10.696,
    "response": {
      "header": {
        "cluster_id": "4613693674493055459",
        "member_id": "10308100773522122874",
        "revision": "3",
        "raft_term": "3"
      }
    }
  },
  "wal_restart": {
    "node": "n1",
    "value": "acknowledged-before-crash",
    "after_crash_value": "committed"
  },
  "one_unreachable": {
    "outcome": "ok",
    "duration_ms": 16.954,
    "response": {
      "header": {
        "cluster_id": "4613693674493055459",
        "member_id": "10308100773522122874",
        "revision": "4",
        "raft_term": "3"
      }
    }
  },
  "two_unreachable_write": {
    "outcome": "unknown",
    "duration_ms": 702.975
  },
  "two_unreachable_linearizable_read": {
    "outcome": "unknown",
    "duration_ms": 702.011
  },
  "unknown_write_after_recovery": "maybe-committed",
  "conditions": {
    "nodes": 3,
    "quorum": 2,
    "faults": [
      "leader SIGKILL and same-data-dir restart",
      "one follower SIGSTOP",
      "two followers SIGSTOP"
    ],
    "election_timeout_ms": 1000,
    "client_deadline_ms": 700
  },
  "assertions_passed": true
}

REAL RUN / 03

n1 → n3; 1 down: ok; 2 down: unknown; reconciled = maybe-committed

检查完整导出记录

{
  "before": {
    "leader": "n1",
    "revision": 2,
    "term": 2
  },
  "new_leader": "n3",
  "committed_write_survived": "acknowledged-before-crash",
  "failover_write": {
    "outcome": "ok",
    "duration_ms": 12.79,
    "response": {
      "header": {
        "cluster_id": "10587831175002086049",
        "member_id": "15120877559436342554",
        "revision": "3",
        "raft_term": "3"
      }
    }
  },
  "wal_restart": {
    "node": "n1",
    "value": "acknowledged-before-crash",
    "after_crash_value": "committed"
  },
  "one_unreachable": {
    "outcome": "ok",
    "duration_ms": 11.273,
    "response": {
      "header": {
        "cluster_id": "10587831175002086049",
        "member_id": "15120877559436342554",
        "revision": "4",
        "raft_term": "3"
      }
    }
  },
  "two_unreachable_write": {
    "outcome": "unknown",
    "duration_ms": 702.407
  },
  "two_unreachable_linearizable_read": {
    "outcome": "unknown",
    "duration_ms": 702.367
  },
  "unknown_write_after_recovery": "maybe-committed",
  "conditions": {
    "nodes": 3,
    "quorum": 2,
    "faults": [
      "leader SIGKILL and same-data-dir restart",
      "one follower SIGSTOP",
      "two followers SIGSTOP"
    ],
    "election_timeout_ms": 1000,
    "client_deadline_ms": 700
  },
  "assertions_passed": true
}

B · 提交后超时与去重

首个请求先提交再延迟响应，两种模式的调用者都超时。无去重重试产生两次效果，原子操作键模式只产生一次；记录包含并发重放和不同内容的冲突。

REAL RUN / 01

2 → 1 (plain / dedup)

检查完整导出记录

{
  "conditions": {
    "response_delay_after_commit_ms": 800,
    "client_deadline_ms": 150,
    "effect": "etcd CAS counter plus durable operation-result key in one Txn",
    "dedup_retention": "no expiry in this bounded experiment"
  },
  "cases": [
    {
      "dedup": false,
      "first_client": {
        "outcome": "unknown",
        "duration_ms": 151.713
      },
      "commits_before_delayed_response": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 6
        }
      ],
      "retry_response": {
        "amount": 1,
        "value": 2,
        "replayed": false
      },
      "final_counter": 2,
      "commits": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 6
        },
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 2,
          "revision": 7
        }
      ]
    },
    {
      "dedup": true,
      "first_client": {
        "outcome": "unknown",
        "duration_ms": 151.561
      },
      "commits_before_delayed_response": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 8
        }
      ],
      "retry_response": {
        "amount": 1,
        "value": 1,
        "replayed": true
      },
      "final_counter": 1,
      "commits": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 8
        }
      ],
      "concurrent_retries": [
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        },
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        },
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        },
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        }
      ],
      "changed_payload": {
        "error": "id-conflict"
      }
    }
  ],
  "assertions_passed": true
}

REAL RUN / 02

2 → 1 (plain / dedup)

检查完整导出记录

{
  "conditions": {
    "response_delay_after_commit_ms": 800,
    "client_deadline_ms": 150,
    "effect": "etcd CAS counter plus durable operation-result key in one Txn",
    "dedup_retention": "no expiry in this bounded experiment"
  },
  "cases": [
    {
      "dedup": false,
      "first_client": {
        "outcome": "unknown",
        "duration_ms": 151.176
      },
      "commits_before_delayed_response": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 6
        }
      ],
      "retry_response": {
        "amount": 1,
        "value": 2,
        "replayed": false
      },
      "final_counter": 2,
      "commits": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 6
        },
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 2,
          "revision": 7
        }
      ]
    },
    {
      "dedup": true,
      "first_client": {
        "outcome": "unknown",
        "duration_ms": 152.061
      },
      "commits_before_delayed_response": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 8
        }
      ],
      "retry_response": {
        "amount": 1,
        "value": 1,
        "replayed": true
      },
      "final_counter": 1,
      "commits": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 8
        }
      ],
      "concurrent_retries": [
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        },
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        },
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        },
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        }
      ],
      "changed_payload": {
        "error": "id-conflict"
      }
    }
  ],
  "assertions_passed": true
}

REAL RUN / 03

2 → 1 (plain / dedup)

检查完整导出记录

{
  "conditions": {
    "response_delay_after_commit_ms": 800,
    "client_deadline_ms": 150,
    "effect": "etcd CAS counter plus durable operation-result key in one Txn",
    "dedup_retention": "no expiry in this bounded experiment"
  },
  "cases": [
    {
      "dedup": false,
      "first_client": {
        "outcome": "unknown",
        "duration_ms": 151.671
      },
      "commits_before_delayed_response": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 6
        }
      ],
      "retry_response": {
        "amount": 1,
        "value": 2,
        "replayed": false
      },
      "final_counter": 2,
      "commits": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 6
        },
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 2,
          "revision": 7
        }
      ]
    },
    {
      "dedup": true,
      "first_client": {
        "outcome": "unknown",
        "duration_ms": 152.451
      },
      "commits_before_delayed_response": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 8
        }
      ],
      "retry_response": {
        "amount": 1,
        "value": 1,
        "replayed": true
      },
      "final_counter": 1,
      "commits": [
        {
          "kind": "commit",
          "id": "synthetic-op-1",
          "value": 1,
          "revision": 8
        }
      ],
      "concurrent_retries": [
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        },
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        },
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        },
        {
          "amount": 1,
          "value": 1,
          "replayed": true
        }
      ],
      "changed_payload": {
        "error": "id-conflict"
      }
    }
  ],
  "assertions_passed": true
}

C · 租约过期与旧写入者

暂停 A，直到观测租约锁不存在。B 获得更大的 create_revision 并先写入。A 恢复后，无 fence 接受旧值，带 fence 拒绝并保留 B-current。

REAL RUN / 01

tokens 9 < 11; plain = A-late; fenced = B-current

检查完整导出记录

{
  "conditions": {
    "fault": "SIGSTOP owned worker A; no lease keepalive",
    "sink": "HTTP plus SQLite BEGIN IMMEDIATE; token check and write atomic"
  },
  "A": {
    "owner": "A",
    "token": 9,
    "lease": "136130354206351129",
    "requested_ttl_seconds": 2,
    "granted_ttl_seconds": 2
  },
  "B": {
    "owner": "B",
    "token": 11,
    "lease": "136130354206351154",
    "requested_ttl_seconds": 2,
    "granted_ttl_seconds": 2
  },
  "observed_key_absence_after_ms": 2063.627,
  "B_writes": {
    "plain": {
      "accepted": true,
      "stored_token": 11,
      "stored_value": "B-current"
    },
    "fenced": {
      "accepted": true,
      "stored_token": 11,
      "stored_value": "B-current"
    }
  },
  "A_late_writes": {
    "plain": {
      "accepted": true,
      "stored_token": 9,
      "stored_value": "A-late"
    },
    "fenced": {
      "accepted": false,
      "stored_token": 11,
      "stored_value": "B-current"
    }
  },
  "assertions_passed": true
}

REAL RUN / 02

tokens 9 < 11; plain = A-late; fenced = B-current

检查完整导出记录

{
  "conditions": {
    "fault": "SIGSTOP owned worker A; no lease keepalive",
    "sink": "HTTP plus SQLite BEGIN IMMEDIATE; token check and write atomic"
  },
  "A": {
    "owner": "A",
    "token": 9,
    "lease": "4664218304555072282",
    "requested_ttl_seconds": 2,
    "granted_ttl_seconds": 2
  },
  "B": {
    "owner": "B",
    "token": 11,
    "lease": "4664218304555072308",
    "requested_ttl_seconds": 2,
    "granted_ttl_seconds": 2
  },
  "observed_key_absence_after_ms": 2186.343,
  "B_writes": {
    "plain": {
      "accepted": true,
      "stored_token": 11,
      "stored_value": "B-current"
    },
    "fenced": {
      "accepted": true,
      "stored_token": 11,
      "stored_value": "B-current"
    }
  },
  "A_late_writes": {
    "plain": {
      "accepted": true,
      "stored_token": 9,
      "stored_value": "A-late"
    },
    "fenced": {
      "accepted": false,
      "stored_token": 11,
      "stored_value": "B-current"
    }
  },
  "assertions_passed": true
}

REAL RUN / 03

tokens 9 < 11; plain = A-late; fenced = B-current

检查完整导出记录

{
  "conditions": {
    "fault": "SIGSTOP owned worker A; no lease keepalive",
    "sink": "HTTP plus SQLite BEGIN IMMEDIATE; token check and write atomic"
  },
  "A": {
    "owner": "A",
    "token": 9,
    "lease": "7099539803060749087",
    "requested_ttl_seconds": 2,
    "granted_ttl_seconds": 2
  },
  "B": {
    "owner": "B",
    "token": 11,
    "lease": "7099539803060749113",
    "requested_ttl_seconds": 2,
    "granted_ttl_seconds": 2
  },
  "observed_key_absence_after_ms": 2181.863,
  "B_writes": {
    "plain": {
      "accepted": true,
      "stored_token": 11,
      "stored_value": "B-current"
    },
    "fenced": {
      "accepted": true,
      "stored_token": 11,
      "stored_value": "B-current"
    }
  },
  "A_late_writes": {
    "plain": {
      "accepted": true,
      "stored_token": 9,
      "stored_value": "A-late"
    },
    "fenced": {
      "accepted": false,
      "stored_token": 11,
      "stored_value": "B-current"
    }
  },
  "assertions_passed": true
}

尚未测量

这些检查确认有界调度中的断言结果，不证明普遍正确性或生产容量。恢复保留原持久化目录，未执行备份还原。历史检查台使用合成样例，不认证这些记录。原始日志留本地，报告只包含选定合成观测。