feat(zookeeper): add metadata properties to agent
This commit is contained in:
1 parent
bf9ef4365b
commit
5f7a6096a9
3 files changed
+180
-20
No files matched your search
@@ -43,6 +43,8 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo
|
|||||||
| xugu | 虚谷 XuguDB | XuguDB Go native agent |
|
| xugu | 虚谷 XuguDB | XuguDB Go native agent |
|
||||||
| iotdb | Apache IoTDB | IoTDB JDBC |
|
| iotdb | Apache IoTDB | IoTDB JDBC |
|
||||||
| etcd | etcd | jetcd |
|
| etcd | etcd | jetcd |
|
||||||
|
| zookeeper | Apache ZooKeeper | Apache Curator |
|
||||||
|
|
||||||
|
|
||||||
## Multi-JRE Support
|
## Multi-JRE Support
|
||||||
|
|
||||||
|
|||||||
+51
-17
@@ -1,11 +1,14 @@
|
|||||||
package com.dbx.agent.zookeeper;
|
package com.dbx.agent.zookeeper;
|
||||||
|
|
||||||
import com.dbx.agent.AgentProtocol;
|
import com.dbx.agent.AgentProtocol;
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.*;
|
||||||
import com.google.gson.GsonBuilder;
|
import org.apache.curator.framework.CuratorFramework;
|
||||||
import com.google.gson.JsonElement;
|
import org.apache.curator.framework.CuratorFrameworkFactory;
|
||||||
import com.google.gson.JsonObject;
|
import org.apache.curator.retry.ExponentialBackoffRetry;
|
||||||
import com.google.gson.JsonParser;
|
import org.apache.zookeeper.CreateMode;
|
||||||
|
import org.apache.zookeeper.KeeperException;
|
||||||
|
import org.apache.zookeeper.data.Stat;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.InputStreamReader;
|
import java.io.InputStreamReader;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
@@ -13,19 +16,8 @@ import java.nio.charset.CharacterCodingException;
|
|||||||
import java.nio.charset.CharsetDecoder;
|
import java.nio.charset.CharsetDecoder;
|
||||||
import java.nio.charset.CodingErrorAction;
|
import java.nio.charset.CodingErrorAction;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.ArrayList;
|
import java.util.*;
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.Base64;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import org.apache.curator.framework.CuratorFramework;
|
|
||||||
import org.apache.curator.framework.CuratorFrameworkFactory;
|
|
||||||
import org.apache.curator.retry.ExponentialBackoffRetry;
|
|
||||||
import org.apache.zookeeper.KeeperException;
|
|
||||||
import org.apache.zookeeper.data.Stat;
|
|
||||||
|
|
||||||
public final class ZooKeeperAgent {
|
public final class ZooKeeperAgent {
|
||||||
private static final Gson GSON = new GsonBuilder().serializeNulls().create();
|
private static final Gson GSON = new GsonBuilder().serializeNulls().create();
|
||||||
@@ -196,6 +188,16 @@ public final class ZooKeeperAgent {
|
|||||||
requireMutablePath(path, "modified");
|
requireMutablePath(path, "modified");
|
||||||
byte[] bytes = parseValue(params.getAsJsonObject("value"));
|
byte[] bytes = parseValue(params.getAsJsonObject("value"));
|
||||||
|
|
||||||
|
String writeMode = stringOrDefault(params, "writeMode", "upsert");
|
||||||
|
return switch (writeMode) {
|
||||||
|
case "create" -> create(active, path, bytes, createMode(params));
|
||||||
|
case "update" -> update(active, path, bytes);
|
||||||
|
case "upsert" -> upsert(active, path, bytes);
|
||||||
|
default -> throw new IllegalArgumentException("Unsupported writeMode: " + writeMode);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Object upsert(CuratorFramework active, String path, byte[] bytes) throws Exception {
|
||||||
Stat stat = active.checkExists().forPath(path);
|
Stat stat = active.checkExists().forPath(path);
|
||||||
if (stat == null) {
|
if (stat == null) {
|
||||||
active.create().creatingParentsIfNeeded().forPath(path, bytes);
|
active.create().creatingParentsIfNeeded().forPath(path, bytes);
|
||||||
@@ -203,13 +205,43 @@ public final class ZooKeeperAgent {
|
|||||||
} else {
|
} else {
|
||||||
stat = active.setData().forPath(path, bytes);
|
stat = active.setData().forPath(path, bytes);
|
||||||
}
|
}
|
||||||
|
return putResult(stat);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Object create(CuratorFramework active, String path, byte[] bytes, CreateMode createMode) throws Exception {
|
||||||
|
String createdPath = active.create()
|
||||||
|
.creatingParentsIfNeeded()
|
||||||
|
.withMode(createMode)
|
||||||
|
.forPath(path, bytes);
|
||||||
|
Stat stat = active.checkExists().forPath(createdPath);
|
||||||
|
Map<String, Object> result = putResult(stat);
|
||||||
|
result.put("key", createdPath);
|
||||||
|
result.put("createdKey", createdPath);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Object update(CuratorFramework active, String path, byte[] bytes) throws Exception {
|
||||||
|
return putResult(active.setData().forPath(path, bytes));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Map<String, Object> putResult(Stat stat) {
|
||||||
Map<String, Object> result = new LinkedHashMap<>();
|
Map<String, Object> result = new LinkedHashMap<>();
|
||||||
result.put("version", stat.getVersion());
|
result.put("version", stat.getVersion());
|
||||||
result.put("mtime", stat.getMtime());
|
result.put("mtime", stat.getMtime());
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static CreateMode createMode(JsonObject params) {
|
||||||
|
String createMode = stringOrDefault(params, "createMode", "persistent");
|
||||||
|
return switch (createMode) {
|
||||||
|
case "persistent" -> CreateMode.PERSISTENT;
|
||||||
|
case "ephemeral" -> CreateMode.EPHEMERAL;
|
||||||
|
case "persistent_sequential" -> CreateMode.PERSISTENT_SEQUENTIAL;
|
||||||
|
case "ephemeral_sequential" -> CreateMode.EPHEMERAL_SEQUENTIAL;
|
||||||
|
default -> throw new IllegalArgumentException("Unsupported createMode: " + createMode);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private static Object delete(JsonObject params) throws Exception {
|
private static Object delete(JsonObject params) throws Exception {
|
||||||
CuratorFramework active = requireClient();
|
CuratorFramework active = requireClient();
|
||||||
String path = normalizePath(stringOrDefault(params, "key", ""));
|
String path = normalizePath(stringOrDefault(params, "key", ""));
|
||||||
@@ -326,9 +358,11 @@ public final class ZooKeeperAgent {
|
|||||||
Map<String, Object> metadata = new LinkedHashMap<>();
|
Map<String, Object> metadata = new LinkedHashMap<>();
|
||||||
long czxid = stat.getCzxid();
|
long czxid = stat.getCzxid();
|
||||||
long mzxid = stat.getMzxid();
|
long mzxid = stat.getMzxid();
|
||||||
|
long pzxid = stat.getPzxid();
|
||||||
int dataLength = stat.getDataLength();
|
int dataLength = stat.getDataLength();
|
||||||
metadata.put("czxid", czxid);
|
metadata.put("czxid", czxid);
|
||||||
metadata.put("mzxid", mzxid);
|
metadata.put("mzxid", mzxid);
|
||||||
|
metadata.put("pzxid", pzxid);
|
||||||
metadata.put("ctime", stat.getCtime());
|
metadata.put("ctime", stat.getCtime());
|
||||||
metadata.put("mtime", stat.getMtime());
|
metadata.put("mtime", stat.getMtime());
|
||||||
metadata.put("version", stat.getVersion());
|
metadata.put("version", stat.getVersion());
|
||||||
|
|||||||
+127
-3
@@ -4,13 +4,14 @@ import com.google.gson.JsonArray;
|
|||||||
import com.google.gson.JsonElement;
|
import com.google.gson.JsonElement;
|
||||||
import com.google.gson.JsonObject;
|
import com.google.gson.JsonObject;
|
||||||
import com.google.gson.JsonParser;
|
import com.google.gson.JsonParser;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import org.apache.curator.test.TestingServer;
|
import org.apache.curator.test.TestingServer;
|
||||||
import org.junit.jupiter.api.AfterEach;
|
import org.junit.jupiter.api.AfterEach;
|
||||||
import org.junit.jupiter.api.Assertions;
|
import org.junit.jupiter.api.Assertions;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
final class ZooKeeperAgentTest {
|
final class ZooKeeperAgentTest {
|
||||||
@AfterEach
|
@AfterEach
|
||||||
void disconnect() {
|
void disconnect() {
|
||||||
@@ -181,6 +182,122 @@ final class ZooKeeperAgentTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void kvPutCreatePersistentCreatesOnlyMissingZnode() throws Exception {
|
||||||
|
try (TestingServer server = new TestingServer()) {
|
||||||
|
connect(server);
|
||||||
|
|
||||||
|
JsonObject created = result(request(
|
||||||
|
2,
|
||||||
|
"kv_put",
|
||||||
|
"{\"key\":\"/app/created\",\"value\":{\"encoding\":\"utf8\",\"data\":\"dbx\"},"
|
||||||
|
+ "\"writeMode\":\"create\",\"createMode\":\"persistent\"}"
|
||||||
|
));
|
||||||
|
JsonObject get = result(request(3, "kv_get", "{\"key\":\"/app/created\"}"));
|
||||||
|
JsonObject duplicate = error(request(
|
||||||
|
4,
|
||||||
|
"kv_put",
|
||||||
|
"{\"key\":\"/app/created\",\"value\":{\"encoding\":\"utf8\",\"data\":\"again\"},"
|
||||||
|
+ "\"writeMode\":\"create\",\"createMode\":\"persistent\"}"
|
||||||
|
));
|
||||||
|
|
||||||
|
Assertions.assertEquals("/app/created", requiredString(created, "key"));
|
||||||
|
Assertions.assertEquals("/app/created", requiredString(created, "createdKey"));
|
||||||
|
Assertions.assertEquals("dbx", get.getAsJsonObject("value").get("data").getAsString());
|
||||||
|
Assertions.assertTrue(duplicate.get("message").getAsString().contains("NodeExists"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void kvPutCreateEphemeralCreatesEphemeralZnode() throws Exception {
|
||||||
|
try (TestingServer server = new TestingServer()) {
|
||||||
|
connect(server);
|
||||||
|
|
||||||
|
JsonObject created = result(request(
|
||||||
|
2,
|
||||||
|
"kv_put",
|
||||||
|
"{\"key\":\"/session/node\",\"value\":{\"encoding\":\"utf8\",\"data\":\"temporary\"},"
|
||||||
|
+ "\"writeMode\":\"create\",\"createMode\":\"ephemeral\"}"
|
||||||
|
));
|
||||||
|
JsonObject get = result(request(3, "kv_get", "{\"key\":\"/session/node\"}"));
|
||||||
|
|
||||||
|
Assertions.assertEquals("/session/node", requiredString(created, "createdKey"));
|
||||||
|
Assertions.assertEquals("temporary", get.getAsJsonObject("value").get("data").getAsString());
|
||||||
|
Assertions.assertTrue(get.getAsJsonObject("metadata").get("ephemeralOwner").getAsLong() > 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void kvPutCreatePersistentSequentialReturnsGeneratedPath() throws Exception {
|
||||||
|
try (TestingServer server = new TestingServer()) {
|
||||||
|
connect(server);
|
||||||
|
|
||||||
|
JsonObject created = result(request(
|
||||||
|
2,
|
||||||
|
"kv_put",
|
||||||
|
"{\"key\":\"/jobs/job-\",\"value\":{\"encoding\":\"utf8\",\"data\":\"one\"},"
|
||||||
|
+ "\"writeMode\":\"create\",\"createMode\":\"persistent_sequential\"}"
|
||||||
|
));
|
||||||
|
String createdKey = requiredString(created, "createdKey");
|
||||||
|
JsonObject get = result(request(3, "kv_get", "{\"key\":\"" + createdKey + "\"}"));
|
||||||
|
|
||||||
|
Assertions.assertEquals(createdKey, requiredString(created, "key"));
|
||||||
|
Assertions.assertTrue(createdKey.startsWith("/jobs/job-"));
|
||||||
|
Assertions.assertTrue(createdKey.length() > "/jobs/job-".length());
|
||||||
|
Assertions.assertEquals("one", get.getAsJsonObject("value").get("data").getAsString());
|
||||||
|
Assertions.assertEquals(0, get.getAsJsonObject("metadata").get("ephemeralOwner").getAsLong());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void kvPutCreateEphemeralSequentialReturnsGeneratedEphemeralPath() throws Exception {
|
||||||
|
try (TestingServer server = new TestingServer()) {
|
||||||
|
connect(server);
|
||||||
|
|
||||||
|
JsonObject created = result(request(
|
||||||
|
2,
|
||||||
|
"kv_put",
|
||||||
|
"{\"key\":\"/sessions/member-\",\"value\":{\"encoding\":\"utf8\",\"data\":\"online\"},"
|
||||||
|
+ "\"writeMode\":\"create\",\"createMode\":\"ephemeral_sequential\"}"
|
||||||
|
));
|
||||||
|
String createdKey = requiredString(created, "createdKey");
|
||||||
|
JsonObject get = result(request(3, "kv_get", "{\"key\":\"" + createdKey + "\"}"));
|
||||||
|
|
||||||
|
Assertions.assertEquals(createdKey, requiredString(created, "key"));
|
||||||
|
Assertions.assertTrue(createdKey.startsWith("/sessions/member-"));
|
||||||
|
Assertions.assertEquals("online", get.getAsJsonObject("value").get("data").getAsString());
|
||||||
|
Assertions.assertTrue(get.getAsJsonObject("metadata").get("ephemeralOwner").getAsLong() > 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void kvPutUpdateOnlyUpdatesExistingZnode() throws Exception {
|
||||||
|
try (TestingServer server = new TestingServer()) {
|
||||||
|
connect(server);
|
||||||
|
|
||||||
|
JsonObject missing = error(request(
|
||||||
|
2,
|
||||||
|
"kv_put",
|
||||||
|
"{\"key\":\"/app/missing\",\"value\":{\"encoding\":\"utf8\",\"data\":\"x\"},\"writeMode\":\"update\"}"
|
||||||
|
));
|
||||||
|
result(request(
|
||||||
|
3,
|
||||||
|
"kv_put",
|
||||||
|
"{\"key\":\"/app/name\",\"value\":{\"encoding\":\"utf8\",\"data\":\"before\"},\"writeMode\":\"create\"}"
|
||||||
|
));
|
||||||
|
JsonObject updated = result(request(
|
||||||
|
4,
|
||||||
|
"kv_put",
|
||||||
|
"{\"key\":\"/app/name\",\"value\":{\"encoding\":\"utf8\",\"data\":\"after\"},\"writeMode\":\"update\"}"
|
||||||
|
));
|
||||||
|
JsonObject value = result(request(5, "kv_get", "{\"key\":\"/app/name\"}")).getAsJsonObject("value");
|
||||||
|
|
||||||
|
Assertions.assertTrue(missing.get("message").getAsString().contains("NoNode"));
|
||||||
|
Assertions.assertTrue(updated.get("version").getAsInt() > 0);
|
||||||
|
Assertions.assertEquals("after", value.get("data").getAsString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void kvGetReturnsBase64ForBinaryData() throws Exception {
|
void kvGetReturnsBase64ForBinaryData() throws Exception {
|
||||||
try (TestingServer server = new TestingServer()) {
|
try (TestingServer server = new TestingServer()) {
|
||||||
@@ -382,7 +499,14 @@ final class ZooKeeperAgentTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static JsonObject error(String response) {
|
private static JsonObject error(String response) {
|
||||||
return JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error");
|
JsonObject object = JsonParser.parseString(response).getAsJsonObject();
|
||||||
|
Assertions.assertTrue(object.has("error"), "expected JSON-RPC error but got " + response);
|
||||||
|
return object.getAsJsonObject("error");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String requiredString(JsonObject object, String key) {
|
||||||
|
Assertions.assertTrue(object.has(key), "expected result to contain " + key);
|
||||||
|
return object.get(key).getAsString();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<String> listedKeys(JsonObject listResult) {
|
private static List<String> listedKeys(JsonObject listResult) {
|
||||||
|
|||||||
Reference in new issue
Block a user