From 8fce41e84404167f2cfc1c5ed1ca06c7cb34d385 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 16 Apr 2024 15:55:28 +0800 Subject: [PATCH 1/3] replica redirect read&write to master in standalone mode Signed-off-by: zhaozhao.zz --- src/cluster.c | 8 ------ src/config.c | 1 + src/server.c | 11 ++++++++ src/server.h | 1 + tests/integration/replica-redirect.tcl | 36 ++++++++++++++++++++++++++ valkey.conf | 12 +++++++++ 6 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 tests/integration/replica-redirect.tcl diff --git a/src/cluster.c b/src/cluster.c index 99c02cd86d..3bba0233a1 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -1461,20 +1461,12 @@ void askingCommand(client *c) { * In this mode slaves will not redirect clients as long as clients access * with read-only commands to keys that are served by the slave's master. */ void readonlyCommand(client *c) { - if (server.cluster_enabled == 0) { - addReplyError(c,"This instance has cluster support disabled"); - return; - } c->flags |= CLIENT_READONLY; addReply(c,shared.ok); } /* The READWRITE command just clears the READONLY command state. */ void readwriteCommand(client *c) { - if (server.cluster_enabled == 0) { - addReplyError(c,"This instance has cluster support disabled"); - return; - } c->flags &= ~CLIENT_READONLY; addReply(c,shared.ok); } diff --git a/src/config.c b/src/config.c index d3da3f94e5..b89d4abb54 100644 --- a/src/config.c +++ b/src/config.c @@ -3094,6 +3094,7 @@ standardConfig static_configs[] = { createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, 1, NULL, NULL), createBoolConfig("replica-read-only", "slave-read-only", DEBUG_CONFIG | MODIFIABLE_CONFIG, server.repl_slave_ro, 1, NULL, NULL), createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, 1, NULL, NULL), + createBoolConfig("replica-enable-redirect", NULL, MODIFIABLE_CONFIG, server.repl_replica_enable_redirect, 0, NULL, NULL), createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, updateJemallocBgThread), createBoolConfig("activedefrag", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.active_defrag_enabled, 0, isValidActiveDefrag, NULL), createBoolConfig("syslog-enabled", NULL, IMMUTABLE_CONFIG, server.syslog_enabled, 0, NULL, NULL), diff --git a/src/server.c b/src/server.c index e30338e452..94c295959e 100644 --- a/src/server.c +++ b/src/server.c @@ -4009,6 +4009,17 @@ int processCommand(client *c) { } } + if (!server.cluster_enabled && + server.masterhost && + !mustObeyClient(c) && + server.repl_replica_enable_redirect && + (is_write_command || + (is_read_command && !(c->flags & CLIENT_READONLY)))) { + addReplyErrorSds(c,sdscatprintf(sdsempty(), "-MOVED -1 %s:%d", + server.masterhost, server.masterport)); + return C_OK; + } + /* Disconnect some clients if total clients memory is too high. We do this * before key eviction, after the last command was executed and consumed * some client output buffer memory. */ diff --git a/src/server.h b/src/server.h index d891fccfb1..343808cdb4 100644 --- a/src/server.h +++ b/src/server.h @@ -1911,6 +1911,7 @@ struct valkeyServer { int repl_serve_stale_data; /* Serve stale data when link is down? */ int repl_slave_ro; /* Slave is read only? */ int repl_slave_ignore_maxmemory; /* If true slaves do not evict. */ + int repl_replica_enable_redirect; /* If true replica would redirect read&write commands to master. */ time_t repl_down_since; /* Unix time at which link with master went down */ int repl_disable_tcp_nodelay; /* Disable TCP_NODELAY after SYNC? */ int slave_priority; /* Reported in INFO and used by Sentinel. */ diff --git a/tests/integration/replica-redirect.tcl b/tests/integration/replica-redirect.tcl new file mode 100644 index 0000000000..510148ac3c --- /dev/null +++ b/tests/integration/replica-redirect.tcl @@ -0,0 +1,36 @@ +start_server {tags {needs:repl external:skip}} { + start_server {} { + set master_host [srv -1 host] + set master_port [srv -1 port] + + r replicaof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replicas not replicating from master" + } + + test {replica allow read command by default} { + r get foo + } {} + + test {replica reply READONLY error for write command by default} { + assert_error {READONLY*} {r set foo bar} + } + + test {replica redirect read and write command when enable replica-enable-redirect} { + r config set replica-enable-redirect yes + assert_error "MOVED -1 $master_host:$master_port" {r set foo bar} + assert_error "MOVED -1 $master_host:$master_port" {r get foo} + } + + test {non-data access commands are not redirected} { + r ping + } {PONG} + + test {replica allow read command in READONLY mode} { + r readonly + r get foo + } {} + } +} diff --git a/valkey.conf b/valkey.conf index b1d180893a..39862e6a30 100644 --- a/valkey.conf +++ b/valkey.conf @@ -817,6 +817,18 @@ replica-priority 100 # replica-announce-ip 5.5.5.5 # replica-announce-port 1234 +# You can configure a replica instance to redirect data access commands +# to its master or not, excluding commands such as: +# INFO, AUTH, ROLE, SUBSCRIBE, UNSUBSCRIBE, PUBLISH. +# +# When enabled, a replica instance will reply "-MOVED -1 master-ip:port" +# for data access commands. Normally these are write or read commands, if +# you want to run read commands while replica-enable-redirect is enabled, +# use the READONLY command first. +# +# This config only takes effect in standalone mode. +replica-enable-redirect no + ############################### KEYS TRACKING ################################# # The client side caching of values is assisted via server-side support. From 0d837657ba7aad564ee1f8dfedae38552392415a Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 22 Apr 2024 11:04:55 +0800 Subject: [PATCH 2/3] change to REDIRECT Signed-off-by: zhaozhao.zz --- src/server.c | 4 ++-- tests/integration/replica-redirect.tcl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server.c b/src/server.c index 94c295959e..6f73934976 100644 --- a/src/server.c +++ b/src/server.c @@ -4010,12 +4010,12 @@ int processCommand(client *c) { } if (!server.cluster_enabled && + server.repl_replica_enable_redirect && server.masterhost && !mustObeyClient(c) && - server.repl_replica_enable_redirect && (is_write_command || (is_read_command && !(c->flags & CLIENT_READONLY)))) { - addReplyErrorSds(c,sdscatprintf(sdsempty(), "-MOVED -1 %s:%d", + addReplyErrorSds(c,sdscatprintf(sdsempty(), "-REDIRECT %s:%d", server.masterhost, server.masterport)); return C_OK; } diff --git a/tests/integration/replica-redirect.tcl b/tests/integration/replica-redirect.tcl index 510148ac3c..7299ac53f7 100644 --- a/tests/integration/replica-redirect.tcl +++ b/tests/integration/replica-redirect.tcl @@ -20,8 +20,8 @@ start_server {tags {needs:repl external:skip}} { test {replica redirect read and write command when enable replica-enable-redirect} { r config set replica-enable-redirect yes - assert_error "MOVED -1 $master_host:$master_port" {r set foo bar} - assert_error "MOVED -1 $master_host:$master_port" {r get foo} + assert_error "REDIRECT $master_host:$master_port" {r set foo bar} + assert_error "REDIRECT $master_host:$master_port" {r get foo} } test {non-data access commands are not redirected} { From be838bce2cb8a67c3ebb299f4677ebe427c9153e Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Thu, 16 May 2024 13:55:17 +0800 Subject: [PATCH 3/3] redirect by per client's capability Signed-off-by: zhaozhao.zz --- src/commands.def | 23 +++++++++++++++++++++ src/commands/client-capa.json | 28 ++++++++++++++++++++++++++ src/config.c | 1 - src/networking.c | 8 ++++++++ src/server.c | 2 +- src/server.h | 5 ++++- tests/integration/replica-redirect.tcl | 4 ++-- valkey.conf | 12 ----------- 8 files changed, 66 insertions(+), 17 deletions(-) create mode 100644 src/commands/client-capa.json diff --git a/src/commands.def b/src/commands.def index bc5a1261f2..c849faa844 100644 --- a/src/commands.def +++ b/src/commands.def @@ -1083,6 +1083,28 @@ struct COMMAND_ARG CLIENT_CACHING_Args[] = { {MAKE_ARG("mode",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=CLIENT_CACHING_mode_Subargs}, }; +/********** CLIENT CAPA ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* CLIENT CAPA history */ +#define CLIENT_CAPA_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* CLIENT CAPA tips */ +#define CLIENT_CAPA_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* CLIENT CAPA key specs */ +#define CLIENT_CAPA_Keyspecs NULL +#endif + +/* CLIENT CAPA argument table */ +struct COMMAND_ARG CLIENT_CAPA_Args[] = { +{MAKE_ARG("capability",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)}, +}; + /********** CLIENT GETNAME ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -1543,6 +1565,7 @@ struct COMMAND_ARG CLIENT_UNBLOCK_Args[] = { /* CLIENT command table */ struct COMMAND_STRUCT CLIENT_Subcommands[] = { {MAKE_CMD("caching","Instructs the server whether to track the keys in the next request.","O(1)","6.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_CACHING_History,0,CLIENT_CACHING_Tips,0,clientCommand,3,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_CACHING_Keyspecs,0,NULL,1),.args=CLIENT_CACHING_Args}, +{MAKE_CMD("capa","A client claims its capability.","O(1)","8.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_CAPA_History,0,CLIENT_CAPA_Tips,0,clientCommand,3,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,CLIENT_CAPA_Keyspecs,0,NULL,1),.args=CLIENT_CAPA_Args}, {MAKE_CMD("getname","Returns the name of the connection.","O(1)","2.6.9",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_GETNAME_History,0,CLIENT_GETNAME_Tips,0,clientCommand,2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_GETNAME_Keyspecs,0,NULL,0)}, {MAKE_CMD("getredir","Returns the client ID to which the connection's tracking notifications are redirected.","O(1)","6.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_GETREDIR_History,0,CLIENT_GETREDIR_Tips,0,clientCommand,2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_GETREDIR_Keyspecs,0,NULL,0)}, {MAKE_CMD("help","Returns helpful text about the different subcommands.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_HELP_History,0,CLIENT_HELP_Tips,0,clientCommand,2,CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_HELP_Keyspecs,0,NULL,0)}, diff --git a/src/commands/client-capa.json b/src/commands/client-capa.json new file mode 100644 index 0000000000..e097298feb --- /dev/null +++ b/src/commands/client-capa.json @@ -0,0 +1,28 @@ +{ + "CAPA": { + "summary": "A client claims its capability.", + "complexity": "O(1)", + "group": "connection", + "since": "8.0.0", + "arity": 3, + "container": "CLIENT", + "function": "clientCommand", + "command_flags": [ + "NOSCRIPT", + "LOADING", + "STALE" + ], + "acl_categories": [ + "CONNECTION" + ], + "reply_schema": { + "const": "OK" + }, + "arguments": [ + { + "name": "capability", + "type": "string" + } + ] + } +} diff --git a/src/config.c b/src/config.c index 7e28a23526..38e6d1d2fb 100644 --- a/src/config.c +++ b/src/config.c @@ -3100,7 +3100,6 @@ standardConfig static_configs[] = { createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, 1, NULL, NULL), createBoolConfig("replica-read-only", "slave-read-only", DEBUG_CONFIG | MODIFIABLE_CONFIG, server.repl_slave_ro, 1, NULL, NULL), createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, 1, NULL, NULL), - createBoolConfig("replica-enable-redirect", NULL, MODIFIABLE_CONFIG, server.repl_replica_enable_redirect, 0, NULL, NULL), createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, updateJemallocBgThread), createBoolConfig("activedefrag", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.active_defrag_enabled, 0, isValidActiveDefrag, NULL), createBoolConfig("syslog-enabled", NULL, IMMUTABLE_CONFIG, server.syslog_enabled, 0, NULL, NULL), diff --git a/src/networking.c b/src/networking.c index 5aa02e8315..6950314098 100644 --- a/src/networking.c +++ b/src/networking.c @@ -166,6 +166,7 @@ client *createClient(connection *conn) { c->bulklen = -1; c->sentlen = 0; c->flags = 0; + c->capa = 0; c->slot = -1; c->ctime = c->lastinteraction = server.unixtime; c->duration = 0; @@ -3583,6 +3584,13 @@ NULL } else { addReplyErrorObject(c,shared.syntaxerr); } + } else if (!strcasecmp(c->argv[1]->ptr,"capa") && c->argc == 3) { + if (!strcasecmp(c->argv[2]->ptr,"redirect")) { + c->capa |= CLIENT_CAPA_REDIRECT; + addReply(c,shared.ok); + } else { + addReplyErrorObject(c,shared.syntaxerr); + } } else { addReplySubcommandSyntaxError(c); } diff --git a/src/server.c b/src/server.c index 5a0d8347bb..ef7e77d4e5 100644 --- a/src/server.c +++ b/src/server.c @@ -4040,7 +4040,7 @@ int processCommand(client *c) { } if (!server.cluster_enabled && - server.repl_replica_enable_redirect && + c->capa & CLIENT_CAPA_REDIRECT && server.masterhost && !mustObeyClient(c) && (is_write_command || diff --git a/src/server.h b/src/server.h index a4e903949c..b45c696513 100644 --- a/src/server.h +++ b/src/server.h @@ -404,6 +404,9 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; #define CLIENT_REPROCESSING_COMMAND (1ULL<<50) /* The client is re-processing the command. */ #define CLIENT_PREREPL_DONE (1ULL<<51) /* Indicate that pre-replication has been done on the client */ +/* Client capabilities */ +#define CLIENT_CAPA_REDIRECT (1<<0) /* Indicate that the client can handle redirection */ + /* Client block type (btype field in client structure) * if CLIENT_BLOCKED flag is set. */ typedef enum blocking_type { @@ -1164,6 +1167,7 @@ typedef struct { typedef struct client { uint64_t id; /* Client incremental unique ID. */ uint64_t flags; /* Client flags: CLIENT_* macros. */ + uint64_t capa; /* Client capabilities: CLIENT_CAPA* macros. */ connection *conn; int resp; /* RESP protocol version. Can be 2 or 3. */ serverDb *db; /* Pointer to currently SELECTed DB. */ @@ -1917,7 +1921,6 @@ struct valkeyServer { int repl_serve_stale_data; /* Serve stale data when link is down? */ int repl_slave_ro; /* Slave is read only? */ int repl_slave_ignore_maxmemory; /* If true slaves do not evict. */ - int repl_replica_enable_redirect; /* If true replica would redirect read&write commands to master. */ time_t repl_down_since; /* Unix time at which link with master went down */ int repl_disable_tcp_nodelay; /* Disable TCP_NODELAY after SYNC? */ int slave_priority; /* Reported in INFO and used by Sentinel. */ diff --git a/tests/integration/replica-redirect.tcl b/tests/integration/replica-redirect.tcl index 7299ac53f7..bba73c6b7d 100644 --- a/tests/integration/replica-redirect.tcl +++ b/tests/integration/replica-redirect.tcl @@ -18,8 +18,8 @@ start_server {tags {needs:repl external:skip}} { assert_error {READONLY*} {r set foo bar} } - test {replica redirect read and write command when enable replica-enable-redirect} { - r config set replica-enable-redirect yes + test {replica redirect read and write command after CLIENT CAPA REDIRECT} { + r client capa redirect assert_error "REDIRECT $master_host:$master_port" {r set foo bar} assert_error "REDIRECT $master_host:$master_port" {r get foo} } diff --git a/valkey.conf b/valkey.conf index 31ba1de284..7a3c4458cb 100644 --- a/valkey.conf +++ b/valkey.conf @@ -826,18 +826,6 @@ replica-priority 100 # replica-announce-ip 5.5.5.5 # replica-announce-port 1234 -# You can configure a replica instance to redirect data access commands -# to its master or not, excluding commands such as: -# INFO, AUTH, ROLE, SUBSCRIBE, UNSUBSCRIBE, PUBLISH. -# -# When enabled, a replica instance will reply "-MOVED -1 master-ip:port" -# for data access commands. Normally these are write or read commands, if -# you want to run read commands while replica-enable-redirect is enabled, -# use the READONLY command first. -# -# This config only takes effect in standalone mode. -replica-enable-redirect no - ############################### KEYS TRACKING ################################# # The client side caching of values is assisted via server-side support.