feat(etcd): add key TTL synchronization

This commit is contained in:
二丫讲梵 authored and GitHub committed 2026-07-26 11:55:52 +08:00
1 parent 43f73ca4ca
commit f56d3a86c1
32 files changed
+1249 -65

No files matched your search

@@ -14,7 +14,14 @@ import io.etcd.jetcd.KeyValue;
import io.etcd.jetcd.kv.DeleteResponse;
import io.etcd.jetcd.kv.GetResponse;
import io.etcd.jetcd.kv.PutResponse;
import io.etcd.jetcd.kv.TxnResponse;
import io.etcd.jetcd.lease.LeaseGrantResponse;
import io.etcd.jetcd.lease.LeaseTimeToLiveResponse;
import io.etcd.jetcd.op.Cmp;
import io.etcd.jetcd.op.CmpTarget;
import io.etcd.jetcd.op.Op;
import io.etcd.jetcd.options.GetOption;
import io.etcd.jetcd.options.LeaseOption;
import io.etcd.jetcd.options.PutOption;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
@@ -38,10 +45,12 @@ public final class EtcdAgent {
private static final Gson GSON = new Gson();
private static final int DEFAULT_LIMIT = 100;
private static final int RPC_TIMEOUT_SECONDS = 30;
private static final int PRESERVE_LEASE_MAX_ATTEMPTS = 3;
private static final List<String> CAPABILITIES = Collections.unmodifiableList(Arrays.asList(
AgentProtocol.CAPABILITY_CONNECT,
AgentProtocol.CAPABILITY_TEST_CONNECTION,
AgentProtocol.CAPABILITY_KV
AgentProtocol.CAPABILITY_KV,
AgentProtocol.CAPABILITY_KV_TTL
));
private static Client client;
private static KV kv;
@@ -162,7 +171,11 @@ public final class EtcdAgent {
private static Object get(JsonObject params) throws Exception {
KV active = requireKv();
String key = params.get("key").getAsString();
GetResponse response = active.get(byteSequence(key)).get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
boolean metadataOnly = boolOrDefault(params, "metadataOnly", false);
GetOption option = metadataOnly
? GetOption.builder().withKeysOnly(true).build()
: GetOption.DEFAULT;
GetResponse response = active.get(byteSequence(key), option).get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
Map<String, Object> result = new LinkedHashMap<>();
if (response.getKvs().isEmpty()) {
result.put("found", false);
@@ -172,26 +185,101 @@ public final class EtcdAgent {
return result;
}
KeyValue item = response.getKvs().get(0);
Map<String, Object> metadata = metadata(item);
if (item.getLease() > 0) {
LeaseTimeToLiveResponse lease =
requireClient().getLeaseClient().timeToLive(item.getLease(), LeaseOption.DEFAULT)
.get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
metadata.put("ttl", lease.getTTL());
}
result.put("found", true);
result.put("key", displayBytes(item.getKey().getBytes()));
result.put("value", valueObject(item.getValue().getBytes()));
result.put("metadata", metadata(item));
result.put("value", metadataOnly ? null : valueObject(item.getValue().getBytes()));
result.put("metadata", metadata);
return result;
}
private static Object put(JsonObject params) throws Exception {
KV active = requireKv();
Client activeClient = requireClient();
String key = params.get("key").getAsString();
byte[] value = parseValue(params.getAsJsonObject("value"));
ByteSequence keyBytes = byteSequence(key);
ByteSequence valueBytes = ByteSequence.from(value);
JsonElement leaseElement = params.get("lease");
PutResponse response;
if (leaseElement != null && !leaseElement.isJsonNull()) {
PutOption option = PutOption.newBuilder().withLeaseId(leaseElement.getAsLong()).build();
response = active.put(byteSequence(key), ByteSequence.from(value), option).get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} else {
response = active.put(byteSequence(key), ByteSequence.from(value)).get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
JsonElement ttlElement = params.get("ttl");
boolean hasLease = leaseElement != null && !leaseElement.isJsonNull();
boolean hasTtl = ttlElement != null && !ttlElement.isJsonNull();
boolean preserveLease = boolOrDefault(params, "preserveLease", false);
if ((hasLease && hasTtl) || (preserveLease && (hasLease || hasTtl))) {
throw new IllegalArgumentException("lease, ttl, and preserveLease cannot be specified together");
}
return Collections.singletonMap("revision", response.getHeader().getRevision());
if (preserveLease) {
long revision = putPreservingLease(active, keyBytes, valueBytes);
return Collections.singletonMap("revision", revision);
}
Long leaseId = null;
long grantedLeaseId = 0;
if (hasTtl) {
long ttl = ttlElement.getAsLong();
if (ttl <= 0) {
throw new IllegalArgumentException("ttl must be a positive integer");
}
LeaseGrantResponse grant =
activeClient.getLeaseClient().grant(ttl).get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
grantedLeaseId = grant.getID();
leaseId = grantedLeaseId;
} else if (hasLease) {
leaseId = leaseElement.getAsLong();
}
try {
PutResponse response;
if (leaseId != null) {
PutOption option = PutOption.newBuilder().withLeaseId(leaseId).build();
response = active.put(keyBytes, valueBytes, option)
.get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} else {
response = active.put(keyBytes, valueBytes)
.get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
}
return Collections.singletonMap("revision", response.getHeader().getRevision());
} catch (Exception error) {
if (grantedLeaseId != 0) {
try {
activeClient.getLeaseClient().revoke(grantedLeaseId)
.get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} catch (Exception revokeError) {
error.addSuppressed(revokeError);
}
}
throw error;
}
}
static long putPreservingLease(KV active, ByteSequence key, ByteSequence value) throws Exception {
for (int attempt = 0; attempt < PRESERVE_LEASE_MAX_ATTEMPTS; attempt++) {
GetResponse existing =
active.get(key, GetOption.DEFAULT).get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
if (existing.getKvs().isEmpty() || existing.getKvs().get(0).getLease() <= 0) {
throw new IllegalStateException("Cannot preserve lease: key does not exist or has no lease");
}
KeyValue current = existing.getKvs().get(0);
PutOption option = PutOption.builder().withLeaseId(current.getLease()).build();
TxnResponse response = active.txn()
.If(new Cmp(key, Cmp.Op.EQUAL, CmpTarget.modRevision(current.getModRevision())))
.Then(Op.put(key, value, option))
.commit()
.get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
if (response.isSucceeded()) {
return response.getHeader().getRevision();
}
}
throw new IllegalStateException("Cannot preserve lease: key changed concurrently; retry the save");
}
private static Object delete(JsonObject params) throws Exception {
@@ -262,6 +350,13 @@ public final class EtcdAgent {
return kv;
}
private static Client requireClient() {
if (client == null) {
throw new IllegalStateException("Not connected");
}
return client;
}
private static void closeClient() {
if (kv != null) {
kv.close();
@@ -1,8 +1,24 @@
package com.dbx.agent.etcd;
import com.google.protobuf.ByteString;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.KV;
import io.etcd.jetcd.Txn;
import io.etcd.jetcd.api.KeyValue;
import io.etcd.jetcd.api.RangeResponse;
import io.etcd.jetcd.api.ResponseHeader;
import io.etcd.jetcd.kv.GetResponse;
import io.etcd.jetcd.kv.TxnResponse;
import java.lang.reflect.Proxy;
import java.nio.charset.StandardCharsets;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Deque;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -17,6 +33,7 @@ final class EtcdAgentTest {
Assertions.assertEquals(1, result.get("protocolVersion").getAsInt());
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_ttl\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"connect\"")));
}
@@ -52,4 +69,155 @@ final class EtcdAgentTest {
Assertions.assertEquals(-1, error.get("code").getAsInt());
Assertions.assertEquals("Not connected", error.get("message").getAsString());
}
@Test
void preserveLeaseRetriesWithTheLatestLeaseAfterConcurrentChange() throws Exception {
AtomicInteger getCalls = new AtomicInteger();
AtomicInteger txnCalls = new AtomicInteger();
KV active = scriptedKv(
getCalls,
txnCalls,
List.of(getResponse("/app/name", 10, 101), getResponse("/app/name", 11, 202)),
List.of(txnResponse(false, 10), txnResponse(true, 12))
);
long revision = EtcdAgent.putPreservingLease(active, bytes("/app/name"), bytes("updated"));
Assertions.assertEquals(12, revision);
Assertions.assertEquals(2, getCalls.get());
Assertions.assertEquals(2, txnCalls.get());
}
@Test
void preserveLeaseRejectsMissingOrUnleasedKeys() {
KV missing = scriptedKv(
new AtomicInteger(),
new AtomicInteger(),
List.of(emptyGetResponse()),
List.of()
);
KV unleased = scriptedKv(
new AtomicInteger(),
new AtomicInteger(),
List.of(getResponse("/app/name", 10, 0)),
List.of()
);
IllegalStateException missingError = Assertions.assertThrows(
IllegalStateException.class,
() -> EtcdAgent.putPreservingLease(missing, bytes("/app/name"), bytes("updated"))
);
IllegalStateException unleasedError = Assertions.assertThrows(
IllegalStateException.class,
() -> EtcdAgent.putPreservingLease(unleased, bytes("/app/name"), bytes("updated"))
);
Assertions.assertEquals("Cannot preserve lease: key does not exist or has no lease", missingError.getMessage());
Assertions.assertEquals("Cannot preserve lease: key does not exist or has no lease", unleasedError.getMessage());
}
@Test
void preserveLeaseFailsClearlyAfterBoundedConcurrentChanges() {
AtomicInteger getCalls = new AtomicInteger();
AtomicInteger txnCalls = new AtomicInteger();
KV active = scriptedKv(
getCalls,
txnCalls,
List.of(
getResponse("/app/name", 10, 101),
getResponse("/app/name", 11, 202),
getResponse("/app/name", 12, 303)
),
List.of(txnResponse(false, 10), txnResponse(false, 11), txnResponse(false, 12))
);
IllegalStateException error = Assertions.assertThrows(
IllegalStateException.class,
() -> EtcdAgent.putPreservingLease(active, bytes("/app/name"), bytes("updated"))
);
Assertions.assertEquals("Cannot preserve lease: key changed concurrently; retry the save", error.getMessage());
Assertions.assertEquals(3, getCalls.get());
Assertions.assertEquals(3, txnCalls.get());
}
private static KV scriptedKv(
AtomicInteger getCalls,
AtomicInteger txnCalls,
List<GetResponse> getResponses,
List<TxnResponse> txnResponses
) {
Deque<GetResponse> gets = new ArrayDeque<>(getResponses);
Deque<TxnResponse> txns = new ArrayDeque<>(txnResponses);
return (KV) Proxy.newProxyInstance(
KV.class.getClassLoader(),
new Class<?>[] { KV.class },
(proxy, method, args) -> {
if (method.getName().equals("get")) {
getCalls.incrementAndGet();
return CompletableFuture.completedFuture(gets.removeFirst());
}
if (method.getName().equals("txn")) {
return scriptedTxn(txnCalls, txns);
}
if (method.getName().equals("close")) {
return null;
}
throw new UnsupportedOperationException(method.getName());
}
);
}
private static Txn scriptedTxn(AtomicInteger txnCalls, Deque<TxnResponse> responses) {
return (Txn) Proxy.newProxyInstance(
Txn.class.getClassLoader(),
new Class<?>[] { Txn.class },
(proxy, method, args) -> {
if (Arrays.asList("If", "Then", "Else").contains(method.getName())) {
return proxy;
}
if (method.getName().equals("commit")) {
txnCalls.incrementAndGet();
return CompletableFuture.completedFuture(responses.removeFirst());
}
throw new UnsupportedOperationException(method.getName());
}
);
}
private static ByteSequence bytes(String value) {
return ByteSequence.from(value, StandardCharsets.UTF_8);
}
private static GetResponse getResponse(String key, long modRevision, long lease) {
KeyValue keyValue = KeyValue.newBuilder()
.setKey(ByteString.copyFromUtf8(key))
.setModRevision(modRevision)
.setLease(lease)
.build();
RangeResponse response = RangeResponse.newBuilder()
.setHeader(header(modRevision))
.addKvs(keyValue)
.build();
return new GetResponse(response, ByteSequence.EMPTY);
}
private static GetResponse emptyGetResponse() {
return new GetResponse(
RangeResponse.newBuilder().setHeader(header(1)).build(),
ByteSequence.EMPTY
);
}
private static TxnResponse txnResponse(boolean succeeded, long revision) {
io.etcd.jetcd.api.TxnResponse response = io.etcd.jetcd.api.TxnResponse.newBuilder()
.setHeader(header(revision))
.setSucceeded(succeeded)
.build();
return new TxnResponse(response, ByteSequence.EMPTY);
}
private static ResponseHeader header(long revision) {
return ResponseHeader.newBuilder().setRevision(revision).build();
}
}