From df048d2e86b694ca75899999d8cfd85804b417b3 Mon Sep 17 00:00:00 2001 From: Chen Tianjie Date: Fri, 19 Apr 2024 11:16:07 +0800 Subject: [PATCH 1/5] Implement fat log. Signed-off-by: Chen Tianjie --- src/commands.def | 120 +++++++++++++++- src/commands/exec.json | 3 +- src/commands/fatlog-get.json | 69 +++++++++ src/commands/fatlog-help.json | 22 +++ src/commands/fatlog-len.json | 26 ++++ src/commands/fatlog-reset.json | 23 +++ src/commands/fatlog.json | 9 ++ src/commands/slowlog-get.json | 2 +- src/commands/slowlog-help.json | 2 +- src/commands/slowlog-len.json | 2 +- src/commands/slowlog-reset.json | 2 +- src/config.c | 4 +- src/module.c | 1 + src/networking.c | 7 + src/server.c | 22 ++- src/server.h | 10 +- src/slowlog.c | 133 +++++++++++------ src/slowlog.h | 15 +- tests/unit/fatlog.tcl | 244 ++++++++++++++++++++++++++++++++ 19 files changed, 653 insertions(+), 63 deletions(-) create mode 100644 src/commands/fatlog-get.json create mode 100644 src/commands/fatlog-help.json create mode 100644 src/commands/fatlog-len.json create mode 100644 src/commands/fatlog-reset.json create mode 100644 src/commands/fatlog.json create mode 100644 tests/unit/fatlog.tcl diff --git a/src/commands.def b/src/commands.def index c59cb01dc1..c5f038c4ef 100644 --- a/src/commands.def +++ b/src/commands.def @@ -6637,6 +6637,115 @@ struct COMMAND_ARG FAILOVER_Args[] = { {MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"TIMEOUT",NULL,NULL,CMD_ARG_OPTIONAL,0,NULL)}, }; +/********** FATLOG GET ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* FATLOG GET history */ +#define FATLOG_GET_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* FATLOG GET tips */ +const char *FATLOG_GET_Tips[] = { +"request_policy:all_nodes", +"nondeterministic_output", +}; +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* FATLOG GET key specs */ +#define FATLOG_GET_Keyspecs NULL +#endif + +/* FATLOG GET argument table */ +struct COMMAND_ARG FATLOG_GET_Args[] = { +{MAKE_ARG("count",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,0,NULL)}, +}; + +/********** FATLOG HELP ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* FATLOG HELP history */ +#define FATLOG_HELP_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* FATLOG HELP tips */ +#define FATLOG_HELP_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* FATLOG HELP key specs */ +#define FATLOG_HELP_Keyspecs NULL +#endif + +/********** FATLOG LEN ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* FATLOG LEN history */ +#define FATLOG_LEN_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* FATLOG LEN tips */ +const char *FATLOG_LEN_Tips[] = { +"request_policy:all_nodes", +"response_policy:agg_sum", +"nondeterministic_output", +}; +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* FATLOG LEN key specs */ +#define FATLOG_LEN_Keyspecs NULL +#endif + +/********** FATLOG RESET ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* FATLOG RESET history */ +#define FATLOG_RESET_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* FATLOG RESET tips */ +const char *FATLOG_RESET_Tips[] = { +"request_policy:all_nodes", +"response_policy:all_succeeded", +}; +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* FATLOG RESET key specs */ +#define FATLOG_RESET_Keyspecs NULL +#endif + +/* FATLOG command table */ +struct COMMAND_STRUCT FATLOG_Subcommands[] = { +{MAKE_CMD("get","Returns the fat log's entries.","O(N) where N is the number of entries returned","7.2.5",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_GET_History,0,FATLOG_GET_Tips,2,heavyLoadLogCommand,-2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,FATLOG_GET_Keyspecs,0,NULL,1),.args=FATLOG_GET_Args}, +{MAKE_CMD("help","Show helpful text about the different subcommands","O(1)","7.2.5",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_HELP_History,0,FATLOG_HELP_Tips,0,heavyLoadLogCommand,2,CMD_LOADING|CMD_STALE,0,FATLOG_HELP_Keyspecs,0,NULL,0)}, +{MAKE_CMD("len","Returns the number of entries in the fat log.","O(1)","7.2.5",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_LEN_History,0,FATLOG_LEN_Tips,3,heavyLoadLogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,FATLOG_LEN_Keyspecs,0,NULL,0)}, +{MAKE_CMD("reset","Clears all entries from the fat log.","O(N) where N is the number of entries in the fatlog","7.2.5",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_RESET_History,0,FATLOG_RESET_Tips,2,heavyLoadLogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,FATLOG_RESET_Keyspecs,0,NULL,0)}, +{0} +}; + +/********** FATLOG ********************/ + +#ifndef SKIP_CMD_HISTORY_TABLE +/* FATLOG history */ +#define FATLOG_History NULL +#endif + +#ifndef SKIP_CMD_TIPS_TABLE +/* FATLOG tips */ +#define FATLOG_Tips NULL +#endif + +#ifndef SKIP_CMD_KEY_SPECS_TABLE +/* FATLOG key specs */ +#define FATLOG_Keyspecs NULL +#endif + /********** FLUSHALL ********************/ #ifndef SKIP_CMD_HISTORY_TABLE @@ -7591,10 +7700,10 @@ const char *SLOWLOG_RESET_Tips[] = { /* SLOWLOG command table */ struct COMMAND_STRUCT SLOWLOG_Subcommands[] = { -{MAKE_CMD("get","Returns the slow log's entries.","O(N) where N is the number of entries returned","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_GET_History,1,SLOWLOG_GET_Tips,2,slowlogCommand,-2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_GET_Keyspecs,0,NULL,1),.args=SLOWLOG_GET_Args}, -{MAKE_CMD("help","Show helpful text about the different subcommands","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_HELP_History,0,SLOWLOG_HELP_Tips,0,slowlogCommand,2,CMD_LOADING|CMD_STALE,0,SLOWLOG_HELP_Keyspecs,0,NULL,0)}, -{MAKE_CMD("len","Returns the number of entries in the slow log.","O(1)","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_LEN_History,0,SLOWLOG_LEN_Tips,3,slowlogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_LEN_Keyspecs,0,NULL,0)}, -{MAKE_CMD("reset","Clears all entries from the slow log.","O(N) where N is the number of entries in the slowlog","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_RESET_History,0,SLOWLOG_RESET_Tips,2,slowlogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_RESET_Keyspecs,0,NULL,0)}, +{MAKE_CMD("get","Returns the slow log's entries.","O(N) where N is the number of entries returned","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_GET_History,1,SLOWLOG_GET_Tips,2,heavyLoadLogCommand,-2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_GET_Keyspecs,0,NULL,1),.args=SLOWLOG_GET_Args}, +{MAKE_CMD("help","Show helpful text about the different subcommands","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_HELP_History,0,SLOWLOG_HELP_Tips,0,heavyLoadLogCommand,2,CMD_LOADING|CMD_STALE,0,SLOWLOG_HELP_Keyspecs,0,NULL,0)}, +{MAKE_CMD("len","Returns the number of entries in the slow log.","O(1)","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_LEN_History,0,SLOWLOG_LEN_Tips,3,heavyLoadLogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_LEN_Keyspecs,0,NULL,0)}, +{MAKE_CMD("reset","Clears all entries from the slow log.","O(N) where N is the number of entries in the slowlog","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_RESET_History,0,SLOWLOG_RESET_Tips,2,heavyLoadLogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_RESET_Keyspecs,0,NULL,0)}, {0} }; @@ -10805,6 +10914,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("dbsize","Returns the number of keys in the database.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,DBSIZE_History,0,DBSIZE_Tips,2,dbsizeCommand,1,CMD_READONLY|CMD_FAST,ACL_CATEGORY_KEYSPACE,DBSIZE_Keyspecs,0,NULL,0)}, {MAKE_CMD("debug","A container for debugging commands.","Depends on subcommand.","1.0.0",CMD_DOC_SYSCMD,NULL,NULL,"server",COMMAND_GROUP_SERVER,DEBUG_History,0,DEBUG_Tips,0,debugCommand,-2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_PROTECTED,0,DEBUG_Keyspecs,0,NULL,0)}, {MAKE_CMD("failover","Starts a coordinated failover from a server to one of its replicas.","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FAILOVER_History,0,FAILOVER_Tips,0,failoverCommand,-1,CMD_ADMIN|CMD_NOSCRIPT|CMD_STALE,0,FAILOVER_Keyspecs,0,NULL,3),.args=FAILOVER_Args}, +{MAKE_CMD("fatlog","A container for fat log commands.","Depends on subcommand.","7.2.5",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_History,0,FATLOG_Tips,0,NULL,-2,0,0,FATLOG_Keyspecs,0,NULL,0),.subcommands=FATLOG_Subcommands}, {MAKE_CMD("flushall","Removes all keys from all databases.","O(N) where N is the total number of keys in all databases","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FLUSHALL_History,2,FLUSHALL_Tips,2,flushallCommand,-1,CMD_WRITE,ACL_CATEGORY_KEYSPACE|ACL_CATEGORY_DANGEROUS,FLUSHALL_Keyspecs,0,NULL,1),.args=FLUSHALL_Args}, {MAKE_CMD("flushdb","Remove all keys from the current database.","O(N) where N is the number of keys in the selected database","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FLUSHDB_History,2,FLUSHDB_Tips,2,flushdbCommand,-1,CMD_WRITE,ACL_CATEGORY_KEYSPACE|ACL_CATEGORY_DANGEROUS,FLUSHDB_Keyspecs,0,NULL,1),.args=FLUSHDB_Args}, {MAKE_CMD("info","Returns information and statistics about the server.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,INFO_History,1,INFO_Tips,3,infoCommand,-1,CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_DANGEROUS,INFO_Keyspecs,0,NULL,1),.args=INFO_Args}, @@ -10921,7 +11031,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("substr","Returns a substring from a string value.","O(N) where N is the length of the returned string. The complexity is ultimately determined by the returned length, but because creating a substring from an existing string is very cheap, it can be considered O(1) for small strings.","1.0.0",CMD_DOC_DEPRECATED,"`GETRANGE`","2.0.0","string",COMMAND_GROUP_STRING,SUBSTR_History,0,SUBSTR_Tips,0,getrangeCommand,4,CMD_READONLY,ACL_CATEGORY_STRING,SUBSTR_Keyspecs,1,NULL,3),.args=SUBSTR_Args}, /* transactions */ {MAKE_CMD("discard","Discards a transaction.","O(N), when N is the number of queued commands","2.0.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,DISCARD_History,0,DISCARD_Tips,0,discardCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_TRANSACTION,DISCARD_Keyspecs,0,NULL,0)}, -{MAKE_CMD("exec","Executes all commands in a transaction.","Depends on commands in the transaction","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,EXEC_History,0,EXEC_Tips,0,execCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SKIP_SLOWLOG,ACL_CATEGORY_TRANSACTION,EXEC_Keyspecs,0,NULL,0)}, +{MAKE_CMD("exec","Executes all commands in a transaction.","Depends on commands in the transaction","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,EXEC_History,0,EXEC_Tips,0,execCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SKIP_SLOWLOG|CMD_SKIP_FATLOG,ACL_CATEGORY_TRANSACTION,EXEC_Keyspecs,0,NULL,0)}, {MAKE_CMD("multi","Starts a transaction.","O(1)","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,MULTI_History,0,MULTI_Tips,0,multiCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_TRANSACTION,MULTI_Keyspecs,0,NULL,0)}, {MAKE_CMD("unwatch","Forgets about watched keys of a transaction.","O(1)","2.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,UNWATCH_History,0,UNWATCH_Tips,0,unwatchCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_TRANSACTION,UNWATCH_Keyspecs,0,NULL,0)}, {MAKE_CMD("watch","Monitors changes to keys to determine the execution of a transaction.","O(1) for every key.","2.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,WATCH_History,0,WATCH_Tips,0,watchCommand,-2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_TRANSACTION,WATCH_Keyspecs,1,NULL,1),.args=WATCH_Args}, diff --git a/src/commands/exec.json b/src/commands/exec.json index 5f03d76e08..b39849fedf 100644 --- a/src/commands/exec.json +++ b/src/commands/exec.json @@ -10,7 +10,8 @@ "NOSCRIPT", "LOADING", "STALE", - "SKIP_SLOWLOG" + "SKIP_SLOWLOG", + "SKIP_FATLOG" ], "acl_categories": [ "TRANSACTION" diff --git a/src/commands/fatlog-get.json b/src/commands/fatlog-get.json new file mode 100644 index 0000000000..181e79435c --- /dev/null +++ b/src/commands/fatlog-get.json @@ -0,0 +1,69 @@ +{ + "GET": { + "summary": "Returns the fat log's entries.", + "complexity": "O(N) where N is the number of entries returned", + "group": "server", + "since": "7.2.5", + "arity": -2, + "container": "FATLOG", + "function": "heavyLoadLogCommand", + "history": [], + "command_flags": [ + "ADMIN", + "LOADING", + "STALE" + ], + "command_tips": [ + "REQUEST_POLICY:ALL_NODES", + "NONDETERMINISTIC_OUTPUT" + ], + "reply_schema": { + "type": "array", + "description": "Entries from the slow log in chronological order.", + "uniqueItems": true, + "items": { + "type": "array", + "minItems": 6, + "maxItems": 6, + "items": [ + { + "type": "integer", + "description": "Fat log entry ID." + }, + { + "type": "integer", + "description": "The unix timestamp at which the logged command was processed.", + "minimum": 0 + }, + { + "type": "integer", + "description": "The size of the response to the query in bytes.", + "minimum": 0 + }, + { + "type": "array", + "description": "The arguments of the command.", + "items": { + "type": "string" + } + }, + { + "type": "string", + "description": "Client IP address and port." + }, + { + "type": "string", + "description": "Client name if set via the CLIENT SETNAME command." + } + ] + } + }, + "arguments": [ + { + "name": "count", + "type": "integer", + "optional": true + } + ] + } +} \ No newline at end of file diff --git a/src/commands/fatlog-help.json b/src/commands/fatlog-help.json new file mode 100644 index 0000000000..de739bd17d --- /dev/null +++ b/src/commands/fatlog-help.json @@ -0,0 +1,22 @@ +{ + "HELP": { + "summary": "Show helpful text about the different subcommands", + "complexity": "O(1)", + "group": "server", + "since": "7.2.5", + "arity": 2, + "container": "FATLOG", + "function": "heavyLoadLogCommand", + "command_flags": [ + "LOADING", + "STALE" + ], + "reply_schema": { + "type": "array", + "description": "Helpful text about subcommands.", + "items": { + "type": "string" + } + } + } +} \ No newline at end of file diff --git a/src/commands/fatlog-len.json b/src/commands/fatlog-len.json new file mode 100644 index 0000000000..b3418dbb3b --- /dev/null +++ b/src/commands/fatlog-len.json @@ -0,0 +1,26 @@ +{ + "LEN": { + "summary": "Returns the number of entries in the fat log.", + "complexity": "O(1)", + "group": "server", + "since": "7.2.5", + "arity": 2, + "container": "FATLOG", + "function": "heavyLoadLogCommand", + "command_flags": [ + "ADMIN", + "LOADING", + "STALE" + ], + "command_tips": [ + "REQUEST_POLICY:ALL_NODES", + "RESPONSE_POLICY:AGG_SUM", + "NONDETERMINISTIC_OUTPUT" + ], + "reply_schema": { + "type": "integer", + "description": "Number of entries in the fat log.", + "minimum": 0 + } + } +} \ No newline at end of file diff --git a/src/commands/fatlog-reset.json b/src/commands/fatlog-reset.json new file mode 100644 index 0000000000..ae9290aa97 --- /dev/null +++ b/src/commands/fatlog-reset.json @@ -0,0 +1,23 @@ +{ + "RESET": { + "summary": "Clears all entries from the fat log.", + "complexity": "O(N) where N is the number of entries in the fatlog", + "group": "server", + "since": "7.2.5", + "arity": 2, + "container": "FATLOG", + "function": "heavyLoadLogCommand", + "command_flags": [ + "ADMIN", + "LOADING", + "STALE" + ], + "command_tips": [ + "REQUEST_POLICY:ALL_NODES", + "RESPONSE_POLICY:ALL_SUCCEEDED" + ], + "reply_schema": { + "const": "OK" + } + } +} \ No newline at end of file diff --git a/src/commands/fatlog.json b/src/commands/fatlog.json new file mode 100644 index 0000000000..f5fe136fc6 --- /dev/null +++ b/src/commands/fatlog.json @@ -0,0 +1,9 @@ +{ + "FATLOG": { + "summary": "A container for fat log commands.", + "complexity": "Depends on subcommand.", + "group": "server", + "since": "7.2.5", + "arity": -2 + } +} \ No newline at end of file diff --git a/src/commands/slowlog-get.json b/src/commands/slowlog-get.json index ffc54b5454..5479f7e54b 100644 --- a/src/commands/slowlog-get.json +++ b/src/commands/slowlog-get.json @@ -6,7 +6,7 @@ "since": "2.2.12", "arity": -2, "container": "SLOWLOG", - "function": "slowlogCommand", + "function": "heavyLoadLogCommand", "history": [ [ "4.0.0", diff --git a/src/commands/slowlog-help.json b/src/commands/slowlog-help.json index dde8fd4598..688f18ee30 100644 --- a/src/commands/slowlog-help.json +++ b/src/commands/slowlog-help.json @@ -6,7 +6,7 @@ "since": "6.2.0", "arity": 2, "container": "SLOWLOG", - "function": "slowlogCommand", + "function": "heavyLoadLogCommand", "command_flags": [ "LOADING", "STALE" diff --git a/src/commands/slowlog-len.json b/src/commands/slowlog-len.json index 717a8ad416..a5c53fea84 100644 --- a/src/commands/slowlog-len.json +++ b/src/commands/slowlog-len.json @@ -6,7 +6,7 @@ "since": "2.2.12", "arity": 2, "container": "SLOWLOG", - "function": "slowlogCommand", + "function": "heavyLoadLogCommand", "command_flags": [ "ADMIN", "LOADING", diff --git a/src/commands/slowlog-reset.json b/src/commands/slowlog-reset.json index cfc1e4da7f..32d5213229 100644 --- a/src/commands/slowlog-reset.json +++ b/src/commands/slowlog-reset.json @@ -6,7 +6,7 @@ "since": "2.2.12", "arity": 2, "container": "SLOWLOG", - "function": "slowlogCommand", + "function": "heavyLoadLogCommand", "command_flags": [ "ADMIN", "LOADING", diff --git a/src/config.c b/src/config.c index e46d01cf6f..df04cb8fa5 100644 --- a/src/config.c +++ b/src/config.c @@ -3156,6 +3156,7 @@ standardConfig static_configs[] = { createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, 128, INTEGER_CONFIG, NULL, NULL), createULongConfig("acllog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.acllog_max_len, 128, INTEGER_CONFIG, NULL, NULL), + createULongConfig("fatlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.fatlog_max_len, 128, INTEGER_CONFIG, NULL, NULL), /* Long Long configs */ createLongLongConfig("busy-reply-threshold", "lua-time-limit", MODIFIABLE_CONFIG, 0, LONG_MAX, server.busy_reply_threshold, 5000, INTEGER_CONFIG, NULL, NULL), /* milliseconds */ @@ -3165,7 +3166,8 @@ standardConfig static_configs[] = { createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL), createLongLongConfig("proto-max-bulk-len", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, 1024 * 1024, LONG_MAX, server.proto_max_bulk_len, 512ll * 1024 * 1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */ createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL), - createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024 * 1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ + createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ + createLongLongConfig("fatlog-log-bigger-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.fatlog_log_bigger_than, 16*1024, INTEGER_CONFIG, NULL, NULL), /* Unsigned Long Long configs */ createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory), diff --git a/src/module.c b/src/module.c index ebb3d0e6c6..908e91362e 100644 --- a/src/module.c +++ b/src/module.c @@ -1151,6 +1151,7 @@ int64_t commandFlagsFromString(char *s) { else if (!strcasecmp(t,"allow-stale")) flags |= CMD_STALE; else if (!strcasecmp(t,"no-monitor")) flags |= CMD_SKIP_MONITOR; else if (!strcasecmp(t,"no-slowlog")) flags |= CMD_SKIP_SLOWLOG; + else if (!strcasecmp(t,"no-fatlog")) flags |= CMD_SKIP_FATLOG; else if (!strcasecmp(t,"fast")) flags |= CMD_FAST; else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH; else if (!strcasecmp(t,"may-replicate")) flags |= CMD_MAY_REPLICATE; diff --git a/src/networking.c b/src/networking.c index 9274f21c05..665518fe8c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -184,6 +184,7 @@ client *createClient(connection *conn) { c->reply = listCreate(); c->deferred_reply_errors = NULL; c->reply_bytes = 0; + c->cmd_reply_length = 0; c->obuf_soft_limit_reached_time = 0; listSetFreeMethod(c->reply, freeClientReplyValue); listSetDupMethod(c->reply, dupClientReplyValue); @@ -439,6 +440,9 @@ void _addReplyToBufferOrList(client *c, const char *s, size_t len) { * buffer offset (see function comment) */ reqresSaveClientReplyOffset(c); + /* Record reply length. */ + c->cmd_reply_length += len; + /* If we're processing a push message into the current client (i.e. executing PUBLISH * to a channel which we are subscribed to, then we wanna postpone that message to be added * after the command's reply (specifically important during multi-exec). the exception is @@ -791,6 +795,9 @@ void setDeferredReply(client *c, void *node, const char *s, size_t length) { if (node == NULL) return; serverAssert(!listNodeValue(ln)); + /* Record reply length. */ + c->cmd_reply_length += length; + /* Normally we fill this dummy NULL node, added by addReplyDeferredLen(), * with a new buffer structure containing the protocol needed to specify * the length of the array following. However sometimes there might be room diff --git a/src/server.c b/src/server.c index e01c125d9a..7330c55a55 100644 --- a/src/server.c +++ b/src/server.c @@ -2707,7 +2707,7 @@ void initServer(void) { serverPanic("Functions initialization failed, check the server logs."); exit(1); } - slowlogInit(); + heavyLoadLogInit(); latencyMonitorInit(); initSharedQueryBuf(); @@ -3254,6 +3254,18 @@ void slowlogPushCurrentCommand(client *c, struct serverCommand *cmd, ustime_t du slowlogPushEntryIfNeeded(c, argv, argc, duration); } +/* Log the last command a client executed into the fatlog. */ +void fatlogPushCurrentCommand(client *c, struct serverCommand *cmd, size_t size) { + /* Some commands may contain sensitive data that should not be available in the slowlog. */ + if (cmd->flags & CMD_SKIP_SLOWLOG) return; + + /* If command argument vector was rewritten, use the original + * arguments. */ + robj **argv = c->original_argv ? c->original_argv : c->argv; + int argc = c->original_argv ? c->original_argc : c->argc; + fatlogPushEntryIfNeeded(c,argv,argc,size); +} + /* This function is called in order to update the total command histogram duration. * The latency unit is nano-seconds. * If needed it will allocate the histogram memory and trim the duration to the upper/lower tracking limits*/ @@ -3445,6 +3457,9 @@ void call(client *c, int flags) { * re-processed. */ if (reprocessing_command) c->flags |= CLIENT_REPROCESSING_COMMAND; + /* To record how many reply bytes generated in this command. */ + c->cmd_reply_length = 0; + monotime monotonic_start = 0; if (monotonicGetType() == MONOTONIC_CLOCK_HW) monotonic_start = getMonotonicUs(); @@ -3503,7 +3518,10 @@ void call(client *c, int flags) { /* Log the command into the Slow log if needed. * If the client is blocked we will handle slowlog when it is unblocked. */ - if (update_command_stats && !(c->flags & CLIENT_BLOCKED)) slowlogPushCurrentCommand(c, real_cmd, c->duration); + if (update_command_stats && !(c->flags & CLIENT_BLOCKED)) { + slowlogPushCurrentCommand(c, real_cmd, c->duration); + fatlogPushCurrentCommand(c, real_cmd, c->cmd_reply_length); + } /* Send the command to clients in MONITOR mode if applicable, * since some administrative commands are considered too dangerous to be shown. diff --git a/src/server.h b/src/server.h index 66f2c81c4d..f4b38cfd49 100644 --- a/src/server.h +++ b/src/server.h @@ -1226,6 +1226,7 @@ typedef struct client { long bulklen; /* Length of bulk argument in multi bulk request. */ list *reply; /* List of reply objects to send to the client. */ unsigned long long reply_bytes; /* Tot bytes of objects in reply list. */ + unsigned long long cmd_reply_length; /* Reply length in bytes for one command. */ list *deferred_reply_errors; /* Used for module thread safe contexts. */ size_t sentlen; /* Amount of bytes already sent in the current buffer or object being sent. */ @@ -1726,6 +1727,10 @@ struct valkeyServer { long long slowlog_entry_id; /* SLOWLOG current entry ID */ long long slowlog_log_slower_than; /* SLOWLOG time limit (to get logged) */ unsigned long slowlog_max_len; /* SLOWLOG max number of items logged */ + list *fatlog; /* FATLOG list of commands */ + long long fatlog_entry_id; /* FATLOG current entry ID */ + long long fatlog_log_bigger_than; /* FATLOG size limit (to get logged) */ + unsigned long fatlog_max_len; /* FATLOG max number of items logged */ struct malloc_stats cron_malloc_stats; /* sampled in serverCron(). */ _Atomic long long stat_net_input_bytes; /* Bytes read from network. */ _Atomic long long stat_net_output_bytes; /* Bytes written to network. */ @@ -2324,6 +2329,8 @@ typedef int serverGetKeysProc(struct serverCommand *cmd, robj **argv, int argc, * CMD_SKIP_MONITOR: Do not automatically propagate the command on MONITOR. * * CMD_SKIP_SLOWLOG: Do not automatically propagate the command to the slowlog. + * + * CMD_SKIP_FATLOG: Do not automatically propagate the command to the fatlog. * * CMD_ASKING: Perform an implicit ASKING for this command, so the * command will be accepted in cluster mode if the slot is marked @@ -3125,6 +3132,7 @@ void preventCommandPropagation(client *c); void preventCommandAOF(client *c); void preventCommandReplication(client *c); void slowlogPushCurrentCommand(client *c, struct serverCommand *cmd, ustime_t duration); +void fatlogPushCurrentCommand(client *c, struct serverCommand *cmd, size_t size); void updateCommandLatencyHistogram(struct hdr_histogram **latency_histogram, int64_t duration_hist); int prepareForShutdown(int flags); void replyToClientsBlockedOnShutdown(void); @@ -3594,7 +3602,7 @@ void saveCommand(client *c); void bgsaveCommand(client *c); void bgrewriteaofCommand(client *c); void shutdownCommand(client *c); -void slowlogCommand(client *c); +void heavyLoadLogCommand(client *c); void moveCommand(client *c); void copyCommand(client *c); void renameCommand(client *c); diff --git a/src/slowlog.c b/src/slowlog.c index 2bf8e0185e..2f84132b62 100644 --- a/src/slowlog.c +++ b/src/slowlog.c @@ -5,8 +5,15 @@ * using the 'slowlog-log-slower-than' config directive, that is also * readable and writable using the CONFIG SET/GET command. * - * The slow queries log is actually not "logged" in the server log file - * but is accessible thanks to the SLOWLOG command. + * Similarly, fatlog remembers the latest N queries that has a response + * larger than K bytes. + * + * The size of the response to reach to be logged in the fat lof is set + * using the 'fatlog-log-bigger-than' config directive, that is also + * readable and writable using the CONFIG SET/GET command. + * + * Both logs are actually not "logged" in the Redis log file but are + * accessible thanks to the SLOWLOG/FATLOG command. * * ---------------------------------------------------------------------------- * @@ -43,11 +50,11 @@ /* Create a new slowlog entry. * Incrementing the ref count of all the objects retained is up to * this function. */ -slowlogEntry *slowlogCreateEntry(client *c, robj **argv, int argc, long long duration) { - slowlogEntry *se = zmalloc(sizeof(*se)); +heavyLoadLogEntry *heavyLoadLogCreateEntry(client *c, robj **argv, int argc, long long cost, long long id) { + heavyLoadLogEntry *se = zmalloc(sizeof(*se)); int j, slargc = argc; - if (slargc > SLOWLOG_ENTRY_MAX_ARGC) slargc = SLOWLOG_ENTRY_MAX_ARGC; + if (slargc > HEAVYLOAD_LOG_ENTRY_MAX_ARGC) slargc = HEAVYLOAD_LOG_ENTRY_MAX_ARGC; se->argc = slargc; se->argv = zmalloc(sizeof(robj *) * slargc); for (j = 0; j < slargc; j++) { @@ -80,8 +87,8 @@ slowlogEntry *slowlogCreateEntry(client *c, robj **argv, int argc, long long dur } } se->time = time(NULL); - se->duration = duration; - se->id = server.slowlog_entry_id++; + se->cost = cost; + se->id = id; se->peerid = sdsnew(getClientPeerId(c)); se->cname = c->name ? sdsnew(c->name->ptr) : sdsempty(); return se; @@ -91,8 +98,8 @@ slowlogEntry *slowlogCreateEntry(client *c, robj **argv, int argc, long long dur * function matches the one of the 'free' method of adlist.c. * * This function will take care to release all the retained object. */ -void slowlogFreeEntry(void *septr) { - slowlogEntry *se = septr; +void heavyLoadLogFreeEntry(void *septr) { + heavyLoadLogEntry *se = septr; int j; for (j = 0; j < se->argc; j++) decrRefCount(se->argv[j]); @@ -104,10 +111,13 @@ void slowlogFreeEntry(void *septr) { /* Initialize the slow log. This function should be called a single time * at server startup. */ -void slowlogInit(void) { +void heavyLoadLogInit(void) { server.slowlog = listCreate(); server.slowlog_entry_id = 0; - listSetFreeMethod(server.slowlog, slowlogFreeEntry); + listSetFreeMethod(server.slowlog, heavyLoadLogFreeEntry); + server.fatlog = listCreate(); + server.fatlog_entry_id = 0; + listSetFreeMethod(server.fatlog, heavyLoadLogFreeEntry); } /* Push a new entry into the slow log. @@ -116,23 +126,38 @@ void slowlogInit(void) { void slowlogPushEntryIfNeeded(client *c, robj **argv, int argc, long long duration) { if (server.slowlog_log_slower_than < 0 || server.slowlog_max_len == 0) return; /* Slowlog disabled */ if (duration >= server.slowlog_log_slower_than) - listAddNodeHead(server.slowlog, slowlogCreateEntry(c, argv, argc, duration)); + listAddNodeHead(server.slowlog, + heavyLoadLogCreateEntry(c, argv, argc, duration, server.slowlog_entry_id++)); /* Remove old entries if needed. */ while (listLength(server.slowlog) > server.slowlog_max_len) listDelNode(server.slowlog, listLast(server.slowlog)); } -/* Remove all the entries from the current slow log. */ -void slowlogReset(void) { - while (listLength(server.slowlog) > 0) listDelNode(server.slowlog, listLast(server.slowlog)); +/* Push a new entry into the fat log. + * This function will make sure to trim the fat log accordingly to the + * configured max length. */ +void fatlogPushEntryIfNeeded(client *c, robj **argv, int argc, long long cost) { + if (server.fatlog_log_bigger_than < 0 || server.fatlog_max_len == 0) return; /* Fatlog disabled */ + if (cost >= server.fatlog_log_bigger_than) + listAddNodeHead(server.fatlog, + heavyLoadLogCreateEntry(c, argv, argc, cost, server.fatlog_entry_id++)); + /* Remove old entries if needed. */ + while (listLength(server.fatlog) > server.fatlog_max_len) + listDelNode(server.fatlog,listLast(server.fatlog)); } -/* The SLOWLOG command. Implements all the subcommands needed to handle the - * slow log. */ -void slowlogCommand(client *c) { - if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr, "help")) { - /* clang-format off */ - const char *help[] = { +/* Remove all the entries from the current slow/fat log. */ +void heavyLoadLogReset(list *log_list) { + while (listLength(log_list) > 0) + listDelNode(log_list, listLast(log_list)); +} + +/* The SLOWLOG/FATLOG command. Implements all the subcommands needed to handle the + * slow/fat log. */ +void heavyLoadLogCommand(client *c) { + const char **help; + list *log_list; + const char *slowlog_help[] = { "GET []", " Return top entries from the slowlog (default: 10, -1 mean all).", " Entries are made of:", @@ -144,18 +169,41 @@ void slowlogCommand(client *c) { " Reset the slowlog.", NULL }; - /* clang-format on */ + const char *fatlog_help[] = { +"GET []", +" Return top entries from the fatlog (default: 10, -1 mean all).", +" Entries are made of:", +" id, timestamp, size in bytes, arguments array, client IP and port,", +" client name", +"LEN", +" Return the length of the fatlog.", +"RESET", +" Reset the fatlog.", +NULL + }; + + if (!strcasecmp(c->argv[0]->ptr,"slowlog")){ + help = slowlog_help; + log_list = server.slowlog; + } else { + help = fatlog_help; + log_list = server.fatlog; + } + + if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { addReplyHelp(c, help); - } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr, "reset")) { - slowlogReset(); - addReply(c, shared.ok); - } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr, "len")) { - addReplyLongLong(c, listLength(server.slowlog)); - } else if ((c->argc == 2 || c->argc == 3) && !strcasecmp(c->argv[1]->ptr, "get")) { + } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"reset")) { + heavyLoadLogReset(log_list); + addReply(c,shared.ok); + } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"len")) { + addReplyLongLong(c,listLength(log_list)); + } else if ((c->argc == 2 || c->argc == 3) && + !strcasecmp(c->argv[1]->ptr,"get")) + { long count = 10; listIter li; listNode *ln; - slowlogEntry *se; + heavyLoadLogEntry *se; if (c->argc == 3) { /* Consume count arg. */ @@ -165,29 +213,30 @@ NULL if (count == -1) { /* We treat -1 as a special value, which means to get all slow logs. - * Simply set count to the length of server.slowlog.*/ - count = listLength(server.slowlog); + * Simply set count to the length of log_list.*/ + count = listLength(log_list); } } - if (count > (long)listLength(server.slowlog)) { - count = listLength(server.slowlog); + if (count > (long)listLength(log_list)) { + count = listLength(log_list); } addReplyArrayLen(c, count); - listRewind(server.slowlog, &li); + listRewind(log_list, &li); while (count--) { int j; ln = listNext(&li); se = ln->value; - addReplyArrayLen(c, 6); - addReplyLongLong(c, se->id); - addReplyLongLong(c, se->time); - addReplyLongLong(c, se->duration); - addReplyArrayLen(c, se->argc); - for (j = 0; j < se->argc; j++) addReplyBulk(c, se->argv[j]); - addReplyBulkCBuffer(c, se->peerid, sdslen(se->peerid)); - addReplyBulkCBuffer(c, se->cname, sdslen(se->cname)); + addReplyArrayLen(c,6); + addReplyLongLong(c,se->id); + addReplyLongLong(c,se->time); + addReplyLongLong(c,se->cost); + addReplyArrayLen(c,se->argc); + for (j = 0; j < se->argc; j++) + addReplyBulk(c,se->argv[j]); + addReplyBulkCBuffer(c,se->peerid,sdslen(se->peerid)); + addReplyBulkCBuffer(c,se->cname,sdslen(se->cname)); } } else { addReplySubcommandSyntaxError(c); diff --git a/src/slowlog.h b/src/slowlog.h index 2372b9f6ea..185fa0917b 100644 --- a/src/slowlog.h +++ b/src/slowlog.h @@ -27,8 +27,8 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#ifndef __SLOWLOG_H__ -#define __SLOWLOG_H__ +#ifndef __HEAVY_LOAD_LOG_H__ +#define __HEAVY_LOAD_LOG_H__ #include "server.h" @@ -36,18 +36,19 @@ #define SLOWLOG_ENTRY_MAX_STRING 128 /* This structure defines an entry inside the slow log list */ -typedef struct slowlogEntry { +typedef struct heavyLoadLogEntry { robj **argv; int argc; long long id; /* Unique entry identifier. */ - long long duration; /* Time spent by the query, in microseconds. */ + long long cost; /* Time spent by the query, in microseconds, or memmory used by respons packet, in bytes. */ time_t time; /* Unix time at which the query was executed. */ sds cname; /* Client name. */ sds peerid; /* Client network address. */ -} slowlogEntry; +} heavyLoadLogEntry; /* Exported API */ -void slowlogInit(void); +void heavyLoadLogInit(void); void slowlogPushEntryIfNeeded(client *c, robj **argv, int argc, long long duration); +void fatlogPushEntryIfNeeded(client *c, robj **argv, int argc, long long cost); -#endif /* __SLOWLOG_H__ */ +#endif /* __HEAVY_LOAD_LOG_H__ */ diff --git a/tests/unit/fatlog.tcl b/tests/unit/fatlog.tcl new file mode 100644 index 0000000000..0c031fa278 --- /dev/null +++ b/tests/unit/fatlog.tcl @@ -0,0 +1,244 @@ +start_server {tags {"fatlog"}} { + + test {FATLOG - check that it starts with an empty log} { + if {$::external} { + r fatlog reset + } + r fatlog len + } {0} + + test {FATLOG - only logs commands having bigger response than threshold} { + r config set fatlog-log-bigger-than 10 + r set a abc + r get a + assert_equal [r fatlog len] 0 + r set a abcdeabcde1 + r get a + assert_equal [r fatlog len] 1 + } + + test {FATLOG - max entries is correctly handled} { + r config set fatlog-log-bigger-than 0 + r config set fatlog-max-len 10 + for {set i 0} {$i < 100} {incr i} { + r ping + } + r fatlog len + } {10} + + test {FATLOG - GET optional argument to limit output len works} { + assert_equal 5 [llength [r fatlog get 5]] + assert_equal 10 [llength [r fatlog get -1]] + assert_equal 10 [llength [r fatlog get 20]] + } + + test {FATLOG - RESET subcommand works} { + r config set fatlog-log-bigger-than 100000 + r fatlog reset + r fatlog len + } {0} + + test {FATLOG - logged entry sanity check} { + r config set fatlog-log-bigger-than 10 + r client setname foobar + r get a + set e [lindex [r fatlog get] 0] + assert_equal [llength $e] 6 + if {!$::external} { + assert_equal [lindex $e 0] 107 + } + assert_equal [expr {[lindex $e 2] > 10}] 1 + assert_equal [lindex $e 3] {get a} + assert_equal {foobar} [lindex $e 5] + } + + test {FATLOG - Certain commands are omitted that contain sensitive information} { + r config set fatlog-max-len 100 + r config set fatlog-log-bigger-than 0 + r fatlog reset + catch {r acl setuser "fatlog test user" +get +set} _ + r config set masteruser "" + r config set masterauth "" + r config set requirepass "" + r config set tls-key-file-pass "" + r config set tls-client-key-file-pass "" + r acl setuser fatlog-test-user +get +set + r acl getuser fatlog-test-user + r acl deluser fatlog-test-user non-existing-user + r config set fatlog-log-bigger-than 0 + r config set fatlog-log-bigger-than -1 + set fatlog_resp [r fatlog get -1] + + # Make sure normal configs work, but the two sensitive + # commands are omitted or redacted + assert_equal 11 [llength $fatlog_resp] + assert_equal {fatlog reset} [lindex [lindex $fatlog_resp 10] 3] + assert_equal {acl setuser (redacted) (redacted) (redacted)} [lindex [lindex $fatlog_resp 9] 3] + assert_equal {config set masteruser (redacted)} [lindex [lindex $fatlog_resp 8] 3] + assert_equal {config set masterauth (redacted)} [lindex [lindex $fatlog_resp 7] 3] + assert_equal {config set requirepass (redacted)} [lindex [lindex $fatlog_resp 6] 3] + assert_equal {config set tls-key-file-pass (redacted)} [lindex [lindex $fatlog_resp 5] 3] + assert_equal {config set tls-client-key-file-pass (redacted)} [lindex [lindex $fatlog_resp 4] 3] + assert_equal {acl setuser (redacted) (redacted) (redacted)} [lindex [lindex $fatlog_resp 3] 3] + assert_equal {acl getuser (redacted)} [lindex [lindex $fatlog_resp 2] 3] + assert_equal {acl deluser (redacted) (redacted)} [lindex [lindex $fatlog_resp 1] 3] + assert_equal {config set fatlog-log-bigger-than 0} [lindex [lindex $fatlog_resp 0] 3] + } {} {needs:repl} + + test {FATLOG - Some commands can redact sensitive fields} { + r config set fatlog-log-bigger-than 0 + r fatlog reset + r migrate [srv 0 host] [srv 0 port] key 9 5000 + r migrate [srv 0 host] [srv 0 port] key 9 5000 AUTH user + r migrate [srv 0 host] [srv 0 port] key 9 5000 AUTH2 user password + r config set fatlog-log-bigger-than -1 + set fatlog_resp [r fatlog get] + + # Make sure all 3 commands were logged, but the sensitive fields are omitted + assert_equal 4 [llength $fatlog_resp] + assert_match {* key 9 5000} [lindex [lindex $fatlog_resp 2] 3] + assert_match {* key 9 5000 AUTH (redacted)} [lindex [lindex $fatlog_resp 1] 3] + assert_match {* key 9 5000 AUTH2 (redacted) (redacted)} [lindex [lindex $fatlog_resp 0] 3] + } {} {needs:repl} + + test {FATLOG - Rewritten commands are logged as their original command} { + r config set fatlog-log-bigger-than 0 + + # Test rewriting client arguments + r sadd set a b c d e + r fatlog reset + + # SPOP is rewritten as DEL when all keys are removed + r spop set 10 + assert_equal {spop set 10} [lindex [lindex [r fatlog get] 0] 3] + + # Test replacing client arguments + r fatlog reset + + # GEOADD is replicated as ZADD + r geoadd cool-cities -122.33207 47.60621 Seattle + assert_equal {geoadd cool-cities -122.33207 47.60621 Seattle} [lindex [lindex [r fatlog get] 0] 3] + + # Test replacing a single command argument + r set A 5 + r fatlog reset + + # GETSET is replicated as SET + r getset a 5 + assert_equal {getset a 5} [lindex [lindex [r fatlog get] 0] 3] + + # INCRBYFLOAT calls rewrite multiple times, so it's a special case + r set A 0 + r fatlog reset + + # INCRBYFLOAT is replicated as SET + r INCRBYFLOAT A 1.0 + assert_equal {INCRBYFLOAT A 1.0} [lindex [lindex [r fatlog get] 0] 3] + + # blocked BLPOP is replicated as LPOP + set rd [valkey_deferring_client] + $rd blpop l 0 + wait_for_blocked_clients_count 1 50 100 + r multi + r lpush l foo + r fatlog reset + r exec + $rd read + $rd close + assert_equal {blpop l 0} [lindex [lindex [r fatlog get] 0] 3] + } + + test {FATLOG - commands with too many arguments are trimmed} { + r config set fatlog-log-bigger-than 0 + r fatlog reset + r sadd set 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 + set e [lindex [r fatlog get] end-1] + lindex $e 3 + } {sadd set 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 {... (2 more arguments)}} + + test {FATLOG - too long arguments are trimmed} { + r config set fatlog-log-bigger-than 0 + r fatlog reset + set arg [string repeat A 129] + r sadd set foo $arg + set e [lindex [r fatlog get] end-1] + lindex $e 3 + } {sadd set foo {AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA... (1 more bytes)}} + + test {FATLOG - EXEC is not logged, just executed commands} { + r config set fatlog-log-bigger-than 0 + r fatlog reset + r multi + r get a + r exec + r config set fatlog-log-bigger-than 10 + assert_equal [r fatlog len] 3 + assert_equal [lindex [lindex [r fatlog get] 0] 3] {get a} + } + + test {FATLOG - can clean older entries} { + r client setname lastentry_client + r config set fatlog-max-len 1 + r get b + assert {[llength [r fatlog get]] == 1} + set e [lindex [r fatlog get] 0] + assert_equal {lastentry_client} [lindex $e 5] + } + + test {FATLOG - can be disabled} { + r config set fatlog-max-len 1 + r config set fatlog-log-bigger-than 10 + r set a abcdeabcde1 + r fatlog reset + r get a + assert_equal [r fatlog len] 1 + r config set fatlog-log-bigger-than -1 + r fatlog reset + r get a + assert_equal [r fatlog len] 0 + } + + test {FATLOG - count must be >= -1} { + assert_error "ERR count should be greater than or equal to -1" {r fatlog get -2} + assert_error "ERR count should be greater than or equal to -1" {r fatlog get -222} + } + + test {FATLOG - get all slow logs} { + r config set fatlog-log-bigger-than 0 + r config set fatlog-max-len 3 + r fatlog reset + + r set key test + r sadd set a b c + r incr num + r lpush list a + + assert_equal [r fatlog len] 3 + assert_equal 0 [llength [r fatlog get 0]] + assert_equal 1 [llength [r fatlog get 1]] + assert_equal 3 [llength [r fatlog get -1]] + assert_equal 3 [llength [r fatlog get 3]] + } + + test {FATLOG - blocking command is reported only after unblocked} { + # Cleanup first + r del mylist + # create a test client + set rd [valkey_deferring_client] + + # config the fatlog and reset + r config set fatlog-log-bigger-than 0 + r config set fatlog-max-len 110 + r fatlog reset + + $rd BLPOP mylist 0 + wait_for_blocked_clients_count 1 50 20 + assert_equal 0 [llength [regexp -all -inline (?=BLPOP) [r fatlog get]]] + + r LPUSH mylist 1 + wait_for_blocked_clients_count 0 50 20 + assert_equal 1 [llength [regexp -all -inline (?=BLPOP) [r fatlog get]]] + + $rd close + } +} From 8845052d04e8cb96410cff393c4cbac1e7518439 Mon Sep 17 00:00:00 2001 From: Chen Tianjie Date: Fri, 19 Apr 2024 17:53:47 +0800 Subject: [PATCH 2/5] Rename file and correct typo. Signed-off-by: Chen Tianjie --- src/blocked.c | 2 +- src/{slowlog.c => heavyloadlog.c} | 0 src/{slowlog.h => heavyloadlog.h} | 4 ++-- src/module.c | 2 +- src/server.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename src/{slowlog.c => heavyloadlog.c} (100%) rename src/{slowlog.h => heavyloadlog.h} (95%) diff --git a/src/blocked.c b/src/blocked.c index 85ef9170a0..acea59fd07 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -61,7 +61,7 @@ */ #include "server.h" -#include "slowlog.h" +#include "heavyloadlog.h" #include "latency.h" #include "monotonic.h" diff --git a/src/slowlog.c b/src/heavyloadlog.c similarity index 100% rename from src/slowlog.c rename to src/heavyloadlog.c diff --git a/src/slowlog.h b/src/heavyloadlog.h similarity index 95% rename from src/slowlog.h rename to src/heavyloadlog.h index 185fa0917b..39fed3320d 100644 --- a/src/slowlog.h +++ b/src/heavyloadlog.h @@ -35,12 +35,12 @@ #define SLOWLOG_ENTRY_MAX_ARGC 32 #define SLOWLOG_ENTRY_MAX_STRING 128 -/* This structure defines an entry inside the slow log list */ +/* This structure defines an entry inside the slow/fat log list */ typedef struct heavyLoadLogEntry { robj **argv; int argc; long long id; /* Unique entry identifier. */ - long long cost; /* Time spent by the query, in microseconds, or memmory used by respons packet, in bytes. */ + long long cost; /* Time spent by the query, in microseconds, or memory used by response packet, in bytes. */ time_t time; /* Unix time at which the query was executed. */ sds cname; /* Client name. */ sds peerid; /* Client network address. */ diff --git a/src/module.c b/src/module.c index 908e91362e..98b32428db 100644 --- a/src/module.c +++ b/src/module.c @@ -53,7 +53,7 @@ #include "server.h" #include "cluster.h" -#include "slowlog.h" +#include "heavyloadlog.h" #include "rdb.h" #include "monotonic.h" #include "script.h" diff --git a/src/server.c b/src/server.c index 7330c55a55..b7253f54ad 100644 --- a/src/server.c +++ b/src/server.c @@ -30,7 +30,7 @@ #include "server.h" #include "monotonic.h" #include "cluster.h" -#include "slowlog.h" +#include "heavyloadlog.h" #include "bio.h" #include "latency.h" #include "mt19937-64.h" From 4be215bb37ed0253fca312aabd425839008075f8 Mon Sep 17 00:00:00 2001 From: Chen Tianjie Date: Wed, 29 May 2024 18:00:22 +0800 Subject: [PATCH 3/5] Rename. Fix details. Signed-off-by: Chen Tianjie --- src/blocked.c | 2 +- src/commands.def | 20 ++++----- src/commands/exec.json | 3 +- src/commands/fatlog-get.json | 4 +- src/commands/fatlog-help.json | 4 +- src/commands/fatlog-len.json | 4 +- src/commands/fatlog-reset.json | 4 +- src/commands/fatlog.json | 2 +- src/commands/slowlog-get.json | 2 +- src/commands/slowlog-help.json | 2 +- src/commands/slowlog-len.json | 2 +- src/commands/slowlog-reset.json | 2 +- src/config.c | 4 +- src/module.c | 3 +- src/server.c | 8 ++-- src/server.h | 6 +-- src/{heavyloadlog.c => slowlog.c} | 73 ++++++++++++++----------------- src/{heavyloadlog.h => slowlog.h} | 22 +++++----- tests/unit/fatlog.tcl | 4 +- 19 files changed, 79 insertions(+), 92 deletions(-) rename src/{heavyloadlog.c => slowlog.c} (78%) rename src/{heavyloadlog.h => slowlog.h} (79%) diff --git a/src/blocked.c b/src/blocked.c index acea59fd07..85ef9170a0 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -61,7 +61,7 @@ */ #include "server.h" -#include "heavyloadlog.h" +#include "slowlog.h" #include "latency.h" #include "monotonic.h" diff --git a/src/commands.def b/src/commands.def index c5f038c4ef..12a63e9347 100644 --- a/src/commands.def +++ b/src/commands.def @@ -6722,10 +6722,10 @@ const char *FATLOG_RESET_Tips[] = { /* FATLOG command table */ struct COMMAND_STRUCT FATLOG_Subcommands[] = { -{MAKE_CMD("get","Returns the fat log's entries.","O(N) where N is the number of entries returned","7.2.5",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_GET_History,0,FATLOG_GET_Tips,2,heavyLoadLogCommand,-2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,FATLOG_GET_Keyspecs,0,NULL,1),.args=FATLOG_GET_Args}, -{MAKE_CMD("help","Show helpful text about the different subcommands","O(1)","7.2.5",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_HELP_History,0,FATLOG_HELP_Tips,0,heavyLoadLogCommand,2,CMD_LOADING|CMD_STALE,0,FATLOG_HELP_Keyspecs,0,NULL,0)}, -{MAKE_CMD("len","Returns the number of entries in the fat log.","O(1)","7.2.5",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_LEN_History,0,FATLOG_LEN_Tips,3,heavyLoadLogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,FATLOG_LEN_Keyspecs,0,NULL,0)}, -{MAKE_CMD("reset","Clears all entries from the fat log.","O(N) where N is the number of entries in the fatlog","7.2.5",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_RESET_History,0,FATLOG_RESET_Tips,2,heavyLoadLogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,FATLOG_RESET_Keyspecs,0,NULL,0)}, +{MAKE_CMD("get","Returns the fat log's entries.","O(N) where N is the number of entries returned","8.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_GET_History,0,FATLOG_GET_Tips,2,slowlogCommand,-2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,FATLOG_GET_Keyspecs,0,NULL,1),.args=FATLOG_GET_Args}, +{MAKE_CMD("help","Show helpful text about the different subcommands","O(1)","8.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_HELP_History,0,FATLOG_HELP_Tips,0,slowlogCommand,2,CMD_LOADING|CMD_STALE,0,FATLOG_HELP_Keyspecs,0,NULL,0)}, +{MAKE_CMD("len","Returns the number of entries in the fat log.","O(1)","8.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_LEN_History,0,FATLOG_LEN_Tips,3,slowlogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,FATLOG_LEN_Keyspecs,0,NULL,0)}, +{MAKE_CMD("reset","Clears all entries from the fat log.","O(N) where N is the number of entries in the fatlog","8.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_RESET_History,0,FATLOG_RESET_Tips,2,slowlogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,FATLOG_RESET_Keyspecs,0,NULL,0)}, {0} }; @@ -7700,10 +7700,10 @@ const char *SLOWLOG_RESET_Tips[] = { /* SLOWLOG command table */ struct COMMAND_STRUCT SLOWLOG_Subcommands[] = { -{MAKE_CMD("get","Returns the slow log's entries.","O(N) where N is the number of entries returned","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_GET_History,1,SLOWLOG_GET_Tips,2,heavyLoadLogCommand,-2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_GET_Keyspecs,0,NULL,1),.args=SLOWLOG_GET_Args}, -{MAKE_CMD("help","Show helpful text about the different subcommands","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_HELP_History,0,SLOWLOG_HELP_Tips,0,heavyLoadLogCommand,2,CMD_LOADING|CMD_STALE,0,SLOWLOG_HELP_Keyspecs,0,NULL,0)}, -{MAKE_CMD("len","Returns the number of entries in the slow log.","O(1)","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_LEN_History,0,SLOWLOG_LEN_Tips,3,heavyLoadLogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_LEN_Keyspecs,0,NULL,0)}, -{MAKE_CMD("reset","Clears all entries from the slow log.","O(N) where N is the number of entries in the slowlog","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_RESET_History,0,SLOWLOG_RESET_Tips,2,heavyLoadLogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_RESET_Keyspecs,0,NULL,0)}, +{MAKE_CMD("get","Returns the slow log's entries.","O(N) where N is the number of entries returned","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_GET_History,1,SLOWLOG_GET_Tips,2,slowlogCommand,-2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_GET_Keyspecs,0,NULL,1),.args=SLOWLOG_GET_Args}, +{MAKE_CMD("help","Show helpful text about the different subcommands","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_HELP_History,0,SLOWLOG_HELP_Tips,0,slowlogCommand,2,CMD_LOADING|CMD_STALE,0,SLOWLOG_HELP_Keyspecs,0,NULL,0)}, +{MAKE_CMD("len","Returns the number of entries in the slow log.","O(1)","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_LEN_History,0,SLOWLOG_LEN_Tips,3,slowlogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_LEN_Keyspecs,0,NULL,0)}, +{MAKE_CMD("reset","Clears all entries from the slow log.","O(N) where N is the number of entries in the slowlog","2.2.12",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,SLOWLOG_RESET_History,0,SLOWLOG_RESET_Tips,2,slowlogCommand,2,CMD_ADMIN|CMD_LOADING|CMD_STALE,0,SLOWLOG_RESET_Keyspecs,0,NULL,0)}, {0} }; @@ -10914,7 +10914,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("dbsize","Returns the number of keys in the database.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,DBSIZE_History,0,DBSIZE_Tips,2,dbsizeCommand,1,CMD_READONLY|CMD_FAST,ACL_CATEGORY_KEYSPACE,DBSIZE_Keyspecs,0,NULL,0)}, {MAKE_CMD("debug","A container for debugging commands.","Depends on subcommand.","1.0.0",CMD_DOC_SYSCMD,NULL,NULL,"server",COMMAND_GROUP_SERVER,DEBUG_History,0,DEBUG_Tips,0,debugCommand,-2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_PROTECTED,0,DEBUG_Keyspecs,0,NULL,0)}, {MAKE_CMD("failover","Starts a coordinated failover from a server to one of its replicas.","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FAILOVER_History,0,FAILOVER_Tips,0,failoverCommand,-1,CMD_ADMIN|CMD_NOSCRIPT|CMD_STALE,0,FAILOVER_Keyspecs,0,NULL,3),.args=FAILOVER_Args}, -{MAKE_CMD("fatlog","A container for fat log commands.","Depends on subcommand.","7.2.5",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_History,0,FATLOG_Tips,0,NULL,-2,0,0,FATLOG_Keyspecs,0,NULL,0),.subcommands=FATLOG_Subcommands}, +{MAKE_CMD("fatlog","A container for fat log commands.","Depends on subcommand.","8.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FATLOG_History,0,FATLOG_Tips,0,NULL,-2,0,0,FATLOG_Keyspecs,0,NULL,0),.subcommands=FATLOG_Subcommands}, {MAKE_CMD("flushall","Removes all keys from all databases.","O(N) where N is the total number of keys in all databases","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FLUSHALL_History,2,FLUSHALL_Tips,2,flushallCommand,-1,CMD_WRITE,ACL_CATEGORY_KEYSPACE|ACL_CATEGORY_DANGEROUS,FLUSHALL_Keyspecs,0,NULL,1),.args=FLUSHALL_Args}, {MAKE_CMD("flushdb","Remove all keys from the current database.","O(N) where N is the number of keys in the selected database","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,FLUSHDB_History,2,FLUSHDB_Tips,2,flushdbCommand,-1,CMD_WRITE,ACL_CATEGORY_KEYSPACE|ACL_CATEGORY_DANGEROUS,FLUSHDB_Keyspecs,0,NULL,1),.args=FLUSHDB_Args}, {MAKE_CMD("info","Returns information and statistics about the server.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"server",COMMAND_GROUP_SERVER,INFO_History,1,INFO_Tips,3,infoCommand,-1,CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_DANGEROUS,INFO_Keyspecs,0,NULL,1),.args=INFO_Args}, @@ -11031,7 +11031,7 @@ struct COMMAND_STRUCT serverCommandTable[] = { {MAKE_CMD("substr","Returns a substring from a string value.","O(N) where N is the length of the returned string. The complexity is ultimately determined by the returned length, but because creating a substring from an existing string is very cheap, it can be considered O(1) for small strings.","1.0.0",CMD_DOC_DEPRECATED,"`GETRANGE`","2.0.0","string",COMMAND_GROUP_STRING,SUBSTR_History,0,SUBSTR_Tips,0,getrangeCommand,4,CMD_READONLY,ACL_CATEGORY_STRING,SUBSTR_Keyspecs,1,NULL,3),.args=SUBSTR_Args}, /* transactions */ {MAKE_CMD("discard","Discards a transaction.","O(N), when N is the number of queued commands","2.0.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,DISCARD_History,0,DISCARD_Tips,0,discardCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_TRANSACTION,DISCARD_Keyspecs,0,NULL,0)}, -{MAKE_CMD("exec","Executes all commands in a transaction.","Depends on commands in the transaction","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,EXEC_History,0,EXEC_Tips,0,execCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SKIP_SLOWLOG|CMD_SKIP_FATLOG,ACL_CATEGORY_TRANSACTION,EXEC_Keyspecs,0,NULL,0)}, +{MAKE_CMD("exec","Executes all commands in a transaction.","Depends on commands in the transaction","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,EXEC_History,0,EXEC_Tips,0,execCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SKIP_SLOWLOG,ACL_CATEGORY_TRANSACTION,EXEC_Keyspecs,0,NULL,0)}, {MAKE_CMD("multi","Starts a transaction.","O(1)","1.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,MULTI_History,0,MULTI_Tips,0,multiCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_TRANSACTION,MULTI_Keyspecs,0,NULL,0)}, {MAKE_CMD("unwatch","Forgets about watched keys of a transaction.","O(1)","2.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,UNWATCH_History,0,UNWATCH_Tips,0,unwatchCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_TRANSACTION,UNWATCH_Keyspecs,0,NULL,0)}, {MAKE_CMD("watch","Monitors changes to keys to determine the execution of a transaction.","O(1) for every key.","2.2.0",CMD_DOC_NONE,NULL,NULL,"transactions",COMMAND_GROUP_TRANSACTIONS,WATCH_History,0,WATCH_Tips,0,watchCommand,-2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_ALLOW_BUSY,ACL_CATEGORY_TRANSACTION,WATCH_Keyspecs,1,NULL,1),.args=WATCH_Args}, diff --git a/src/commands/exec.json b/src/commands/exec.json index b39849fedf..5f03d76e08 100644 --- a/src/commands/exec.json +++ b/src/commands/exec.json @@ -10,8 +10,7 @@ "NOSCRIPT", "LOADING", "STALE", - "SKIP_SLOWLOG", - "SKIP_FATLOG" + "SKIP_SLOWLOG" ], "acl_categories": [ "TRANSACTION" diff --git a/src/commands/fatlog-get.json b/src/commands/fatlog-get.json index 181e79435c..16870a8be2 100644 --- a/src/commands/fatlog-get.json +++ b/src/commands/fatlog-get.json @@ -3,10 +3,10 @@ "summary": "Returns the fat log's entries.", "complexity": "O(N) where N is the number of entries returned", "group": "server", - "since": "7.2.5", + "since": "8.0.0", "arity": -2, "container": "FATLOG", - "function": "heavyLoadLogCommand", + "function": "slowlogCommand", "history": [], "command_flags": [ "ADMIN", diff --git a/src/commands/fatlog-help.json b/src/commands/fatlog-help.json index de739bd17d..b3dc00dfb0 100644 --- a/src/commands/fatlog-help.json +++ b/src/commands/fatlog-help.json @@ -3,10 +3,10 @@ "summary": "Show helpful text about the different subcommands", "complexity": "O(1)", "group": "server", - "since": "7.2.5", + "since": "8.0.0", "arity": 2, "container": "FATLOG", - "function": "heavyLoadLogCommand", + "function": "slowlogCommand", "command_flags": [ "LOADING", "STALE" diff --git a/src/commands/fatlog-len.json b/src/commands/fatlog-len.json index b3418dbb3b..32d477ddcd 100644 --- a/src/commands/fatlog-len.json +++ b/src/commands/fatlog-len.json @@ -3,10 +3,10 @@ "summary": "Returns the number of entries in the fat log.", "complexity": "O(1)", "group": "server", - "since": "7.2.5", + "since": "8.0.0", "arity": 2, "container": "FATLOG", - "function": "heavyLoadLogCommand", + "function": "slowlogCommand", "command_flags": [ "ADMIN", "LOADING", diff --git a/src/commands/fatlog-reset.json b/src/commands/fatlog-reset.json index ae9290aa97..38f401dbda 100644 --- a/src/commands/fatlog-reset.json +++ b/src/commands/fatlog-reset.json @@ -3,10 +3,10 @@ "summary": "Clears all entries from the fat log.", "complexity": "O(N) where N is the number of entries in the fatlog", "group": "server", - "since": "7.2.5", + "since": "8.0.0", "arity": 2, "container": "FATLOG", - "function": "heavyLoadLogCommand", + "function": "slowlogCommand", "command_flags": [ "ADMIN", "LOADING", diff --git a/src/commands/fatlog.json b/src/commands/fatlog.json index f5fe136fc6..954fbdca97 100644 --- a/src/commands/fatlog.json +++ b/src/commands/fatlog.json @@ -3,7 +3,7 @@ "summary": "A container for fat log commands.", "complexity": "Depends on subcommand.", "group": "server", - "since": "7.2.5", + "since": "8.0.0", "arity": -2 } } \ No newline at end of file diff --git a/src/commands/slowlog-get.json b/src/commands/slowlog-get.json index 5479f7e54b..ffc54b5454 100644 --- a/src/commands/slowlog-get.json +++ b/src/commands/slowlog-get.json @@ -6,7 +6,7 @@ "since": "2.2.12", "arity": -2, "container": "SLOWLOG", - "function": "heavyLoadLogCommand", + "function": "slowlogCommand", "history": [ [ "4.0.0", diff --git a/src/commands/slowlog-help.json b/src/commands/slowlog-help.json index 688f18ee30..dde8fd4598 100644 --- a/src/commands/slowlog-help.json +++ b/src/commands/slowlog-help.json @@ -6,7 +6,7 @@ "since": "6.2.0", "arity": 2, "container": "SLOWLOG", - "function": "heavyLoadLogCommand", + "function": "slowlogCommand", "command_flags": [ "LOADING", "STALE" diff --git a/src/commands/slowlog-len.json b/src/commands/slowlog-len.json index a5c53fea84..717a8ad416 100644 --- a/src/commands/slowlog-len.json +++ b/src/commands/slowlog-len.json @@ -6,7 +6,7 @@ "since": "2.2.12", "arity": 2, "container": "SLOWLOG", - "function": "heavyLoadLogCommand", + "function": "slowlogCommand", "command_flags": [ "ADMIN", "LOADING", diff --git a/src/commands/slowlog-reset.json b/src/commands/slowlog-reset.json index 32d5213229..cfc1e4da7f 100644 --- a/src/commands/slowlog-reset.json +++ b/src/commands/slowlog-reset.json @@ -6,7 +6,7 @@ "since": "2.2.12", "arity": 2, "container": "SLOWLOG", - "function": "heavyLoadLogCommand", + "function": "slowlogCommand", "command_flags": [ "ADMIN", "LOADING", diff --git a/src/config.c b/src/config.c index df04cb8fa5..c3b709129d 100644 --- a/src/config.c +++ b/src/config.c @@ -3166,8 +3166,8 @@ standardConfig static_configs[] = { createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL), createLongLongConfig("proto-max-bulk-len", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, 1024 * 1024, LONG_MAX, server.proto_max_bulk_len, 512ll * 1024 * 1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */ createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL), - createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ - createLongLongConfig("fatlog-log-bigger-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.fatlog_log_bigger_than, 16*1024, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024 * 1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ + createLongLongConfig("fatlog-log-bigger-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.fatlog_log_bigger_than, 1024 * 1024, MEMORY_CONFIG, NULL, NULL), /* Unsigned Long Long configs */ createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory), diff --git a/src/module.c b/src/module.c index 98b32428db..ebb3d0e6c6 100644 --- a/src/module.c +++ b/src/module.c @@ -53,7 +53,7 @@ #include "server.h" #include "cluster.h" -#include "heavyloadlog.h" +#include "slowlog.h" #include "rdb.h" #include "monotonic.h" #include "script.h" @@ -1151,7 +1151,6 @@ int64_t commandFlagsFromString(char *s) { else if (!strcasecmp(t,"allow-stale")) flags |= CMD_STALE; else if (!strcasecmp(t,"no-monitor")) flags |= CMD_SKIP_MONITOR; else if (!strcasecmp(t,"no-slowlog")) flags |= CMD_SKIP_SLOWLOG; - else if (!strcasecmp(t,"no-fatlog")) flags |= CMD_SKIP_FATLOG; else if (!strcasecmp(t,"fast")) flags |= CMD_FAST; else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH; else if (!strcasecmp(t,"may-replicate")) flags |= CMD_MAY_REPLICATE; diff --git a/src/server.c b/src/server.c index b7253f54ad..5e1cb9b817 100644 --- a/src/server.c +++ b/src/server.c @@ -30,7 +30,7 @@ #include "server.h" #include "monotonic.h" #include "cluster.h" -#include "heavyloadlog.h" +#include "slowlog.h" #include "bio.h" #include "latency.h" #include "mt19937-64.h" @@ -2707,7 +2707,7 @@ void initServer(void) { serverPanic("Functions initialization failed, check the server logs."); exit(1); } - heavyLoadLogInit(); + slowlogInit(); latencyMonitorInit(); initSharedQueryBuf(); @@ -3256,14 +3256,14 @@ void slowlogPushCurrentCommand(client *c, struct serverCommand *cmd, ustime_t du /* Log the last command a client executed into the fatlog. */ void fatlogPushCurrentCommand(client *c, struct serverCommand *cmd, size_t size) { - /* Some commands may contain sensitive data that should not be available in the slowlog. */ + /* Some commands may contain sensitive data that should not be available in the fatlog. */ if (cmd->flags & CMD_SKIP_SLOWLOG) return; /* If command argument vector was rewritten, use the original * arguments. */ robj **argv = c->original_argv ? c->original_argv : c->argv; int argc = c->original_argv ? c->original_argc : c->argc; - fatlogPushEntryIfNeeded(c,argv,argc,size); + fatlogPushEntryIfNeeded(c, argv, argc, size); } /* This function is called in order to update the total command histogram duration. diff --git a/src/server.h b/src/server.h index f4b38cfd49..403e3f5cef 100644 --- a/src/server.h +++ b/src/server.h @@ -2328,9 +2328,7 @@ typedef int serverGetKeysProc(struct serverCommand *cmd, robj **argv, int argc, * * CMD_SKIP_MONITOR: Do not automatically propagate the command on MONITOR. * - * CMD_SKIP_SLOWLOG: Do not automatically propagate the command to the slowlog. - * - * CMD_SKIP_FATLOG: Do not automatically propagate the command to the fatlog. + * CMD_SKIP_SLOWLOG: Do not automatically propagate the command to the slowlog and fatlog. * * CMD_ASKING: Perform an implicit ASKING for this command, so the * command will be accepted in cluster mode if the slot is marked @@ -3602,7 +3600,7 @@ void saveCommand(client *c); void bgsaveCommand(client *c); void bgrewriteaofCommand(client *c); void shutdownCommand(client *c); -void heavyLoadLogCommand(client *c); +void slowlogCommand(client *c); void moveCommand(client *c); void copyCommand(client *c); void renameCommand(client *c); diff --git a/src/heavyloadlog.c b/src/slowlog.c similarity index 78% rename from src/heavyloadlog.c rename to src/slowlog.c index 2f84132b62..fd17843b81 100644 --- a/src/heavyloadlog.c +++ b/src/slowlog.c @@ -8,7 +8,7 @@ * Similarly, fatlog remembers the latest N queries that has a response * larger than K bytes. * - * The size of the response to reach to be logged in the fat lof is set + * The size of the response to reach to be logged in the fat log is set * using the 'fatlog-log-bigger-than' config directive, that is also * readable and writable using the CONFIG SET/GET command. * @@ -50,11 +50,11 @@ /* Create a new slowlog entry. * Incrementing the ref count of all the objects retained is up to * this function. */ -heavyLoadLogEntry *heavyLoadLogCreateEntry(client *c, robj **argv, int argc, long long cost, long long id) { - heavyLoadLogEntry *se = zmalloc(sizeof(*se)); +slowlogEntry *slowlogCreateEntry(client *c, robj **argv, int argc, long long cost, long long id) { + slowlogEntry *se = zmalloc(sizeof(*se)); int j, slargc = argc; - if (slargc > HEAVYLOAD_LOG_ENTRY_MAX_ARGC) slargc = HEAVYLOAD_LOG_ENTRY_MAX_ARGC; + if (slargc > SLOWLOG_ENTRY_MAX_ARGC) slargc = SLOWLOG_ENTRY_MAX_ARGC; se->argc = slargc; se->argv = zmalloc(sizeof(robj *) * slargc); for (j = 0; j < slargc; j++) { @@ -98,8 +98,8 @@ heavyLoadLogEntry *heavyLoadLogCreateEntry(client *c, robj **argv, int argc, lon * function matches the one of the 'free' method of adlist.c. * * This function will take care to release all the retained object. */ -void heavyLoadLogFreeEntry(void *septr) { - heavyLoadLogEntry *se = septr; +void slowlogFreeEntry(void *septr) { + slowlogEntry *se = septr; int j; for (j = 0; j < se->argc; j++) decrRefCount(se->argv[j]); @@ -111,13 +111,13 @@ void heavyLoadLogFreeEntry(void *septr) { /* Initialize the slow log. This function should be called a single time * at server startup. */ -void heavyLoadLogInit(void) { +void slowlogInit(void) { server.slowlog = listCreate(); server.slowlog_entry_id = 0; - listSetFreeMethod(server.slowlog, heavyLoadLogFreeEntry); + listSetFreeMethod(server.slowlog, slowlogFreeEntry); server.fatlog = listCreate(); server.fatlog_entry_id = 0; - listSetFreeMethod(server.fatlog, heavyLoadLogFreeEntry); + listSetFreeMethod(server.fatlog, slowlogFreeEntry); } /* Push a new entry into the slow log. @@ -126,8 +126,7 @@ void heavyLoadLogInit(void) { void slowlogPushEntryIfNeeded(client *c, robj **argv, int argc, long long duration) { if (server.slowlog_log_slower_than < 0 || server.slowlog_max_len == 0) return; /* Slowlog disabled */ if (duration >= server.slowlog_log_slower_than) - listAddNodeHead(server.slowlog, - heavyLoadLogCreateEntry(c, argv, argc, duration, server.slowlog_entry_id++)); + listAddNodeHead(server.slowlog, slowlogCreateEntry(c, argv, argc, duration, server.slowlog_entry_id++)); /* Remove old entries if needed. */ while (listLength(server.slowlog) > server.slowlog_max_len) listDelNode(server.slowlog, listLast(server.slowlog)); @@ -137,24 +136,21 @@ void slowlogPushEntryIfNeeded(client *c, robj **argv, int argc, long long durati * This function will make sure to trim the fat log accordingly to the * configured max length. */ void fatlogPushEntryIfNeeded(client *c, robj **argv, int argc, long long cost) { - if (server.fatlog_log_bigger_than < 0 || server.fatlog_max_len == 0) return; /* Fatlog disabled */ + if (server.fatlog_max_len == 0) return; /* Fatlog disabled */ if (cost >= server.fatlog_log_bigger_than) - listAddNodeHead(server.fatlog, - heavyLoadLogCreateEntry(c, argv, argc, cost, server.fatlog_entry_id++)); + listAddNodeHead(server.fatlog, slowlogCreateEntry(c, argv, argc, cost, server.fatlog_entry_id++)); /* Remove old entries if needed. */ - while (listLength(server.fatlog) > server.fatlog_max_len) - listDelNode(server.fatlog,listLast(server.fatlog)); + while (listLength(server.fatlog) > server.fatlog_max_len) listDelNode(server.fatlog, listLast(server.fatlog)); } /* Remove all the entries from the current slow/fat log. */ -void heavyLoadLogReset(list *log_list) { - while (listLength(log_list) > 0) - listDelNode(log_list, listLast(log_list)); +void slowlogReset(list *log_list) { + while (listLength(log_list) > 0) listDelNode(log_list, listLast(log_list)); } /* The SLOWLOG/FATLOG command. Implements all the subcommands needed to handle the * slow/fat log. */ -void heavyLoadLogCommand(client *c) { +void slowlogCommand(client *c) { const char **help; list *log_list; const char *slowlog_help[] = { @@ -182,7 +178,7 @@ NULL NULL }; - if (!strcasecmp(c->argv[0]->ptr,"slowlog")){ + if (!strcasecmp(c->argv[0]->ptr, "slowlog")) { help = slowlog_help; log_list = server.slowlog; } else { @@ -190,20 +186,18 @@ NULL log_list = server.fatlog; } - if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { + if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr, "help")) { addReplyHelp(c, help); - } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"reset")) { - heavyLoadLogReset(log_list); - addReply(c,shared.ok); - } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"len")) { - addReplyLongLong(c,listLength(log_list)); - } else if ((c->argc == 2 || c->argc == 3) && - !strcasecmp(c->argv[1]->ptr,"get")) - { + } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr, "reset")) { + slowlogReset(log_list); + addReply(c, shared.ok); + } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr, "len")) { + addReplyLongLong(c, listLength(log_list)); + } else if ((c->argc == 2 || c->argc == 3) && !strcasecmp(c->argv[1]->ptr, "get")) { long count = 10; listIter li; listNode *ln; - heavyLoadLogEntry *se; + slowlogEntry *se; if (c->argc == 3) { /* Consume count arg. */ @@ -228,15 +222,14 @@ NULL ln = listNext(&li); se = ln->value; - addReplyArrayLen(c,6); - addReplyLongLong(c,se->id); - addReplyLongLong(c,se->time); - addReplyLongLong(c,se->cost); - addReplyArrayLen(c,se->argc); - for (j = 0; j < se->argc; j++) - addReplyBulk(c,se->argv[j]); - addReplyBulkCBuffer(c,se->peerid,sdslen(se->peerid)); - addReplyBulkCBuffer(c,se->cname,sdslen(se->cname)); + addReplyArrayLen(c, 6); + addReplyLongLong(c, se->id); + addReplyLongLong(c, se->time); + addReplyLongLong(c, se->cost); + addReplyArrayLen(c, se->argc); + for (j = 0; j < se->argc; j++) addReplyBulk(c, se->argv[j]); + addReplyBulkCBuffer(c, se->peerid, sdslen(se->peerid)); + addReplyBulkCBuffer(c, se->cname, sdslen(se->cname)); } } else { addReplySubcommandSyntaxError(c); diff --git a/src/heavyloadlog.h b/src/slowlog.h similarity index 79% rename from src/heavyloadlog.h rename to src/slowlog.h index 39fed3320d..8dd803b177 100644 --- a/src/heavyloadlog.h +++ b/src/slowlog.h @@ -27,8 +27,8 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#ifndef __HEAVY_LOAD_LOG_H__ -#define __HEAVY_LOAD_LOG_H__ +#ifndef __SLOWLOG_H__ +#define __SLOWLOG_H__ #include "server.h" @@ -36,19 +36,19 @@ #define SLOWLOG_ENTRY_MAX_STRING 128 /* This structure defines an entry inside the slow/fat log list */ -typedef struct heavyLoadLogEntry { +typedef struct slowlogEntry { robj **argv; int argc; - long long id; /* Unique entry identifier. */ - long long cost; /* Time spent by the query, in microseconds, or memory used by response packet, in bytes. */ - time_t time; /* Unix time at which the query was executed. */ - sds cname; /* Client name. */ - sds peerid; /* Client network address. */ -} heavyLoadLogEntry; + long long id; /* Unique entry identifier. */ + long long cost; /* Time spent by the query, in microseconds, or memory used by response packet, in bytes. */ + time_t time; /* Unix time at which the query was executed. */ + sds cname; /* Client name. */ + sds peerid; /* Client network address. */ +} slowlogEntry; /* Exported API */ -void heavyLoadLogInit(void); +void slowlogInit(void); void slowlogPushEntryIfNeeded(client *c, robj **argv, int argc, long long duration); void fatlogPushEntryIfNeeded(client *c, robj **argv, int argc, long long cost); -#endif /* __HEAVY_LOAD_LOG_H__ */ +#endif /* __SLOWLOG_H__ */ diff --git a/tests/unit/fatlog.tcl b/tests/unit/fatlog.tcl index 0c031fa278..50a83c986b 100644 --- a/tests/unit/fatlog.tcl +++ b/tests/unit/fatlog.tcl @@ -66,7 +66,6 @@ start_server {tags {"fatlog"}} { r acl getuser fatlog-test-user r acl deluser fatlog-test-user non-existing-user r config set fatlog-log-bigger-than 0 - r config set fatlog-log-bigger-than -1 set fatlog_resp [r fatlog get -1] # Make sure normal configs work, but the two sensitive @@ -91,7 +90,6 @@ start_server {tags {"fatlog"}} { r migrate [srv 0 host] [srv 0 port] key 9 5000 r migrate [srv 0 host] [srv 0 port] key 9 5000 AUTH user r migrate [srv 0 host] [srv 0 port] key 9 5000 AUTH2 user password - r config set fatlog-log-bigger-than -1 set fatlog_resp [r fatlog get] # Make sure all 3 commands were logged, but the sensitive fields are omitted @@ -192,7 +190,7 @@ start_server {tags {"fatlog"}} { r fatlog reset r get a assert_equal [r fatlog len] 1 - r config set fatlog-log-bigger-than -1 + r config set fatlog-max-len 0 r fatlog reset r get a assert_equal [r fatlog len] 0 From f78492ef3e9c6c61c026adb9e2371c0d12bcdc73 Mon Sep 17 00:00:00 2001 From: Chen Tianjie Date: Wed, 29 May 2024 18:42:49 +0800 Subject: [PATCH 4/5] Run clang format. Signed-off-by: Chen Tianjie --- src/slowlog.c | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/src/slowlog.c b/src/slowlog.c index fd17843b81..bbe266a9a2 100644 --- a/src/slowlog.c +++ b/src/slowlog.c @@ -153,30 +153,26 @@ void slowlogReset(list *log_list) { void slowlogCommand(client *c) { const char **help; list *log_list; - const char *slowlog_help[] = { -"GET []", -" Return top entries from the slowlog (default: 10, -1 mean all).", -" Entries are made of:", -" id, timestamp, time in microseconds, arguments array, client IP and port,", -" client name", -"LEN", -" Return the length of the slowlog.", -"RESET", -" Reset the slowlog.", -NULL - }; - const char *fatlog_help[] = { -"GET []", -" Return top entries from the fatlog (default: 10, -1 mean all).", -" Entries are made of:", -" id, timestamp, size in bytes, arguments array, client IP and port,", -" client name", -"LEN", -" Return the length of the fatlog.", -"RESET", -" Reset the fatlog.", -NULL - }; + const char *slowlog_help[] = {"GET []", + " Return top entries from the slowlog (default: 10, -1 mean all).", + " Entries are made of:", + " id, timestamp, time in microseconds, arguments array, client IP and port,", + " client name", + "LEN", + " Return the length of the slowlog.", + "RESET", + " Reset the slowlog.", + NULL}; + const char *fatlog_help[] = {"GET []", + " Return top entries from the fatlog (default: 10, -1 mean all).", + " Entries are made of:", + " id, timestamp, size in bytes, arguments array, client IP and port,", + " client name", + "LEN", + " Return the length of the fatlog.", + "RESET", + " Reset the fatlog.", + NULL}; if (!strcasecmp(c->argv[0]->ptr, "slowlog")) { help = slowlog_help; From a0cf493572100e47d600140a16fba7825a6226ca Mon Sep 17 00:00:00 2001 From: Chen Tianjie Date: Thu, 30 May 2024 09:43:14 +0800 Subject: [PATCH 5/5] Fatlog count input big packet as well. Signed-off-by: Chen Tianjie --- src/server.c | 4 +++ tests/unit/fatlog.tcl | 62 +++++++++++++++++++++---------------------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/server.c b/src/server.c index 5e1cb9b817..88f0817fe3 100644 --- a/src/server.c +++ b/src/server.c @@ -3457,6 +3457,10 @@ void call(client *c, int flags) { * re-processed. */ if (reprocessing_command) c->flags |= CLIENT_REPROCESSING_COMMAND; + if (update_command_stats) { + fatlogPushCurrentCommand(c, real_cmd, c->argv_len_sum); + } + /* To record how many reply bytes generated in this command. */ c->cmd_reply_length = 0; diff --git a/tests/unit/fatlog.tcl b/tests/unit/fatlog.tcl index 50a83c986b..49ec0645ca 100644 --- a/tests/unit/fatlog.tcl +++ b/tests/unit/fatlog.tcl @@ -8,13 +8,14 @@ start_server {tags {"fatlog"}} { } {0} test {FATLOG - only logs commands having bigger response than threshold} { - r config set fatlog-log-bigger-than 10 + r config set fatlog-log-bigger-than 25 r set a abc r get a assert_equal [r fatlog len] 0 - r set a abcdeabcde1 - r get a + r set a 12345678901234567890123456 assert_equal [r fatlog len] 1 + r get a + assert_equal [r fatlog len] 2 } test {FATLOG - max entries is correctly handled} { @@ -39,14 +40,11 @@ start_server {tags {"fatlog"}} { } {0} test {FATLOG - logged entry sanity check} { - r config set fatlog-log-bigger-than 10 + r config set fatlog-log-bigger-than 25 r client setname foobar r get a set e [lindex [r fatlog get] 0] assert_equal [llength $e] 6 - if {!$::external} { - assert_equal [lindex $e 0] 107 - } assert_equal [expr {[lindex $e 2] > 10}] 1 assert_equal [lindex $e 3] {get a} assert_equal {foobar} [lindex $e 5] @@ -70,18 +68,18 @@ start_server {tags {"fatlog"}} { # Make sure normal configs work, but the two sensitive # commands are omitted or redacted - assert_equal 11 [llength $fatlog_resp] - assert_equal {fatlog reset} [lindex [lindex $fatlog_resp 10] 3] - assert_equal {acl setuser (redacted) (redacted) (redacted)} [lindex [lindex $fatlog_resp 9] 3] - assert_equal {config set masteruser (redacted)} [lindex [lindex $fatlog_resp 8] 3] - assert_equal {config set masterauth (redacted)} [lindex [lindex $fatlog_resp 7] 3] - assert_equal {config set requirepass (redacted)} [lindex [lindex $fatlog_resp 6] 3] - assert_equal {config set tls-key-file-pass (redacted)} [lindex [lindex $fatlog_resp 5] 3] - assert_equal {config set tls-client-key-file-pass (redacted)} [lindex [lindex $fatlog_resp 4] 3] - assert_equal {acl setuser (redacted) (redacted) (redacted)} [lindex [lindex $fatlog_resp 3] 3] - assert_equal {acl getuser (redacted)} [lindex [lindex $fatlog_resp 2] 3] - assert_equal {acl deluser (redacted) (redacted)} [lindex [lindex $fatlog_resp 1] 3] - assert_equal {config set fatlog-log-bigger-than 0} [lindex [lindex $fatlog_resp 0] 3] + assert_equal 22 [llength $fatlog_resp] + assert_equal {fatlog reset} [lindex [lindex $fatlog_resp 21] 3] + assert_equal {acl setuser (redacted) (redacted) (redacted)} [lindex [lindex $fatlog_resp 19] 3] + assert_equal {config set masteruser (redacted)} [lindex [lindex $fatlog_resp 17] 3] + assert_equal {config set masterauth (redacted)} [lindex [lindex $fatlog_resp 15] 3] + assert_equal {config set requirepass (redacted)} [lindex [lindex $fatlog_resp 13] 3] + assert_equal {config set tls-key-file-pass (redacted)} [lindex [lindex $fatlog_resp 11] 3] + assert_equal {config set tls-client-key-file-pass (redacted)} [lindex [lindex $fatlog_resp 9] 3] + assert_equal {acl setuser (redacted) (redacted) (redacted)} [lindex [lindex $fatlog_resp 7] 3] + assert_equal {acl getuser (redacted)} [lindex [lindex $fatlog_resp 5] 3] + assert_equal {acl deluser (redacted) (redacted)} [lindex [lindex $fatlog_resp 3] 3] + assert_equal {config set fatlog-log-bigger-than 0} [lindex [lindex $fatlog_resp 1] 3] } {} {needs:repl} test {FATLOG - Some commands can redact sensitive fields} { @@ -93,10 +91,10 @@ start_server {tags {"fatlog"}} { set fatlog_resp [r fatlog get] # Make sure all 3 commands were logged, but the sensitive fields are omitted - assert_equal 4 [llength $fatlog_resp] - assert_match {* key 9 5000} [lindex [lindex $fatlog_resp 2] 3] - assert_match {* key 9 5000 AUTH (redacted)} [lindex [lindex $fatlog_resp 1] 3] - assert_match {* key 9 5000 AUTH2 (redacted) (redacted)} [lindex [lindex $fatlog_resp 0] 3] + assert_equal 8 [llength $fatlog_resp] + assert_match {* key 9 5000} [lindex [lindex $fatlog_resp 5] 3] + assert_match {* key 9 5000 AUTH (redacted)} [lindex [lindex $fatlog_resp 3] 3] + assert_match {* key 9 5000 AUTH2 (redacted) (redacted)} [lindex [lindex $fatlog_resp 1] 3] } {} {needs:repl} test {FATLOG - Rewritten commands are logged as their original command} { @@ -108,14 +106,14 @@ start_server {tags {"fatlog"}} { # SPOP is rewritten as DEL when all keys are removed r spop set 10 - assert_equal {spop set 10} [lindex [lindex [r fatlog get] 0] 3] + assert_equal {spop set 10} [lindex [lindex [r fatlog get] 1] 3] # Test replacing client arguments r fatlog reset # GEOADD is replicated as ZADD r geoadd cool-cities -122.33207 47.60621 Seattle - assert_equal {geoadd cool-cities -122.33207 47.60621 Seattle} [lindex [lindex [r fatlog get] 0] 3] + assert_equal {geoadd cool-cities -122.33207 47.60621 Seattle} [lindex [lindex [r fatlog get] 1] 3] # Test replacing a single command argument r set A 5 @@ -123,7 +121,7 @@ start_server {tags {"fatlog"}} { # GETSET is replicated as SET r getset a 5 - assert_equal {getset a 5} [lindex [lindex [r fatlog get] 0] 3] + assert_equal {getset a 5} [lindex [lindex [r fatlog get] 1] 3] # INCRBYFLOAT calls rewrite multiple times, so it's a special case r set A 0 @@ -131,7 +129,7 @@ start_server {tags {"fatlog"}} { # INCRBYFLOAT is replicated as SET r INCRBYFLOAT A 1.0 - assert_equal {INCRBYFLOAT A 1.0} [lindex [lindex [r fatlog get] 0] 3] + assert_equal {INCRBYFLOAT A 1.0} [lindex [lindex [r fatlog get] 1] 3] # blocked BLPOP is replicated as LPOP set rd [valkey_deferring_client] @@ -143,7 +141,7 @@ start_server {tags {"fatlog"}} { r exec $rd read $rd close - assert_equal {blpop l 0} [lindex [lindex [r fatlog get] 0] 3] + assert_equal {blpop l 0} [lindex [lindex [r fatlog get] 1] 3] } test {FATLOG - commands with too many arguments are trimmed} { @@ -170,8 +168,8 @@ start_server {tags {"fatlog"}} { r get a r exec r config set fatlog-log-bigger-than 10 - assert_equal [r fatlog len] 3 - assert_equal [lindex [lindex [r fatlog get] 0] 3] {get a} + assert_equal [r fatlog len] 6 + assert_equal [lindex [lindex [r fatlog get] 1] 3] {get a} } test {FATLOG - can clean older entries} { @@ -231,11 +229,11 @@ start_server {tags {"fatlog"}} { $rd BLPOP mylist 0 wait_for_blocked_clients_count 1 50 20 - assert_equal 0 [llength [regexp -all -inline (?=BLPOP) [r fatlog get]]] + assert_equal 1 [llength [regexp -all -inline (?=BLPOP) [r fatlog get]]] r LPUSH mylist 1 wait_for_blocked_clients_count 0 50 20 - assert_equal 1 [llength [regexp -all -inline (?=BLPOP) [r fatlog get]]] + assert_equal 2 [llength [regexp -all -inline (?=BLPOP) [r fatlog get]]] $rd close }