app/vmbackup: introduce flags to take basic auth credentials and snapshot authkey

docs/CHANGELOG.md

@@ -30,6 +30,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 
 ## tip
 
+* FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/): allow flag based configuration for basic auth credentials and snapshot auth key via `snapshot.basicAuthUsername`, `snapshot.basicAuthPassword` and `snapshot.authKey` options respectively for `snapshot.createURL` and `snapshot.deleteURL`. See [these docs](https://docs.victoriametrics.com/cluster-victoriametrics/#list-of-command-line-flags-for-vmstorage) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5973).
 * FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): support selecting of multiple instances on the dashboard. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5869) for details.
 * FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): properly display version in the Stats row for the custom builds of VictoriaMetrics.
 * FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): add `Network Usage` panel to `Resource Usage` row.

docs/Cluster-VictoriaMetrics.md

@@ -1826,7 +1826,7 @@ Below is the output for `/path/to/vmstorage -help`:
   -smallMergeConcurrency int
      Deprecated: this flag does nothing
   -snapshotAuthKey value
-     authKey, which must be passed in query string to /snapshot* pages
+     authKey, which must be passed in query string or via 'X-AuthKey' http header to /snapshot* pages
      Flag value can be read from the given file when using -snapshotAuthKey=file:///abs/path/to/file or -snapshotAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -snapshotAuthKey=http://host/path or -snapshotAuthKey=https://host/path
   -snapshotCreateTimeout duration
      Deprecated: this flag does nothing

docs/vmbackup.md

@@ -443,6 +443,14 @@ Run `vmbackup -help` in order to see all the available options:
      VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set
   -snapshot.deleteURL string
      VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. All created snapshots will be automatically deleted. Example: http://victoriametrics:8428/snapshot/delete
+  -snapshot.basicAuthUsername string
+     Optional basic auth username to use for connections to -snapshotCreateURL.
+     Flag value can be read from the given file when using -snapshot.basicAuthUsername=file:///abs/path/to/file or -snapshot.basicAuthUsername=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -snapshot.basicAuthUsername=http://host/path or -snapshot.basicAuthUsername=https://host/path
+  -snapshot.basicAuthPassword string
+     Optional basic auth password to use for connections to -snapshotCreateURL.
+     Flag value can be read from the given file when using -snapshot.basicAuthPassword=file:///abs/path/to/file or -snapshot.basicAuthPassword=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -snapshot.basicAuthPassword=http://host/path or -snapshot.basicAuthPassword=https://host/path
+  -snapshot.authKey string
+     Optional authKey to be passed as 'X-AuthKey' HTTP headder for the connections to -snapshotCreateURL and -snapshot.deleteURL. Flag value can be read from the given file when using -snapshot.authKey=file:///abs/path/to/file or -snapshot.authKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -snapshot.authKey=http://host/path or -snapshot.authKey=https://host/path
   -snapshot.tlsCAFile string
      Optional path to TLS CA file to use for verifying connections to -snapshotCreateURL. By default, system CA is used
   -snapshot.tlsCertFile string

lib/httpserver/httpserver.go

@@ -431,6 +431,13 @@
 		return CheckBasicAuth(w, r)
 	}
 	if r.FormValue("authKey") != flagValue {
+		// Check and allow the request if the header 'X-AuthKey' matches the flagValue
+		// Currently, this is only applicable for snapshotAuthKey. Below condition can be modified to remove
+		// the check for flagName if 'X-AuthKey' header is to be allowed for other authKeys related flags (for instance reloadAuthKey)
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5973
+		if flagName == "snapshotAuthKey" && r.Header.Get("X-AuthKey") == flagValue {
+			return true
+		}
 		authKeyRequestErrors.Inc()
 		http.Error(w, fmt.Sprintf("The provided authKey doesn't match -%s", flagName), http.StatusUnauthorized)
 		return false

lib/httputils/url.go

@@ -0,0 +1,34 @@
+package httputils
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+)
+
+var secretWordsRe = regexp.MustCompile("auth|pass|key|secret|token")
+
+// RedactedURL redacts sensitive information like basic auth credentials
+// and query parameters containing sensitive info from a URL object (*url.URL).
+// It searches for query parameter names  containing words commonly associated
+// with authentication credentials (like "auth", "pass", "key", "secret", or "token").
+// These words are matched in a case-insensitive manner. If there is a match, such sensitive information will be mased with 'xxxxx'
+func RedactedURL(u *url.URL) string {
+	if u == nil {
+		return ""
+	}
+	ru := *u
+	values := ru.Query()
+	for k, vs := range values {
+		if secretWordsRe.MatchString(strings.ToLower(k)) {
+			for i := range vs {
+				vs[i] = "xxxxx"
+			}
+		}
+	}
+	ru.RawQuery = values.Encode()
+	if _, has := ru.User.Password(); has {
+		ru.User = url.UserPassword("xxxxx", "xxxxx")
+	}
+	return ru.String()
+}

lib/httputils/url_test.go

@@ -0,0 +1,59 @@
+package httputils
+
+import (
+	"net/url"
+	"testing"
+)
+
+func TestRedactedURL(t *testing.T) {
+	tests := []struct {
+		name     string
+		inputURL string
+		expected string
+	}{
+		{
+			name:     "empty URL",
+			inputURL: "",
+			expected: "",
+		},
+		{
+			name:     "no secrets",
+			inputURL: "https://example.com/path/to/resource",
+			expected: "https://example.com/path/to/resource",
+		},
+		{
+			name:     "secret query parameter",
+			inputURL: "https://example.com/path/to/resource?authKey=foobar",
+			expected: "https://example.com/path/to/resource?authKey=xxxxx",
+		},
+		{
+			name:     "secret query parameters (case insensitive)",
+			inputURL: "https://example.com/path/to/resource?TOKEN=foobar",
+			expected: "https://example.com/path/to/resource?TOKEN=xxxxx",
+		},
+		{
+			name:     "with basic auth secrets",
+			inputURL: "https://username:secretPassword@example.com/path/to/resource",
+			expected: "https://xxxxx:xxxxx@example.com/path/to/resource",
+		},
+		{
+			name:     "with basic auth and query parameters secrets",
+			inputURL: "https://username:secretPassword@example.com/path/to/resource?authKey=foobar",
+			expected: "https://xxxxx:xxxxx@example.com/path/to/resource?authKey=xxxxx",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			parsedURL, err := url.Parse(test.inputURL)
+			if err != nil {
+				t.Errorf("Unexpected error %v", err)
+				return
+			}
+			actual := RedactedURL(parsedURL)
+			if actual != test.expected {
+				t.Errorf("Expected: %s, Actual: %s", test.expected, actual)
+			}
+		})
+	}
+}

lib/snapshot/snapshot.go

@@ -1,14 +1,17 @@
 package snapshot
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
@@ -19,6 +22,9 @@
 	tlsKeyFile            = flag.String("snapshot.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -snapshotCreateURL")
 	tlsCAFile             = flag.String("snapshot.tlsCAFile", "", `Optional path to TLS CA file to use for verifying connections to -snapshotCreateURL. By default, system CA is used`)
 	tlsServerName         = flag.String("snapshot.tlsServerName", "", `Optional TLS server name to use for connections to -snapshotCreateURL. By default, the server name from -snapshotCreateURL is used`)
+	basicAuthUser         = flagutil.NewPassword("snapshot.basicAuthUsername", `Optional basic auth username to use for connections to -snapshotCreateURL`)
+	basicAuthPassword     = flagutil.NewPassword("snapshot.basicAuthPassword", `Optional basic auth password to use for connections to -snapshotCreateURL`)
+	snapshotAuthKey       = flagutil.NewPassword("snapshot.authKey", `Optional authKey to be passed as 'X-AuthKey' HTTP headder for the connections to -snapshotCreateURL`)
 )
 
 type snapshot struct {
@@ -42,7 +48,12 @@
 	}
 	hc := &http.Client{Transport: tr}
 
-	resp, err := hc.Get(u.String())
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return "", err
+	}
+	addAuthHeaders(req)
+	resp, err := hc.Do(req)
 	if err != nil {
 		return "", err
 	}
@@ -51,13 +62,13 @@
 		return "", err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("unexpected status code returned from %q: %d; expecting %d; response body: %q", u.Redacted(), resp.StatusCode, http.StatusOK, body)
+		return "", fmt.Errorf("unexpected status code returned from %q: %d; expecting %d; response body: %q", httputils.RedactedURL(u), resp.StatusCode, http.StatusOK, body)
 	}
 
 	snap := snapshot{}
 	err = json.Unmarshal(body, &snap)
 	if err != nil {
-		return "", fmt.Errorf("cannot parse JSON response from %q: %w; response body: %q", u.Redacted(), err, body)
+		return "", fmt.Errorf("cannot parse JSON response from %q: %w; response body: %q", httputils.RedactedURL(u), err, body)
 	}
 
 	if snap.Status == "ok" {
@@ -67,7 +78,7 @@
 	if snap.Status == "error" {
 		return "", errors.New(snap.Msg)
 	}
-	return "", fmt.Errorf("Unkown status: %v", snap.Status)
+	return "", fmt.Errorf("Unknown status: %v", snap.Status)
 }
 
 // Delete deletes a snapshot via the provided api endpoint
@@ -86,7 +97,13 @@
 		return err
 	}
 	hc := &http.Client{Transport: tr}
-	resp, err := hc.PostForm(u.String(), formData)
+	req, err := http.NewRequest("POST", u.String(), strings.NewReader(formData.Encode()))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	addAuthHeaders(req)
+	resp, err := hc.Do(req)
 	if err != nil {
 		return err
 	}
@@ -95,13 +112,13 @@
 		return err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("unexpected status code returned from %q: %d; expecting %d; response body: %q", u.Redacted(), resp.StatusCode, http.StatusOK, body)
+		return fmt.Errorf("unexpected status code returned from %q: %d; expecting %d; response body: %q", httputils.RedactedURL(u), resp.StatusCode, http.StatusOK, body)
 	}
 
 	snap := snapshot{}
 	err = json.Unmarshal(body, &snap)
 	if err != nil {
-		return fmt.Errorf("cannot parse JSON response from %q: %w; response body: %q", u.Redacted(), err, body)
+		return fmt.Errorf("cannot parse JSON response from %q: %w; response body: %q", httputils.RedactedURL(u), err, body)
 	}
 
 	if snap.Status == "ok" {
@@ -111,5 +128,19 @@
 	if snap.Status == "error" {
 		return errors.New(snap.Msg)
 	}
-	return fmt.Errorf("Unkown status: %v", snap.Status)
+	return fmt.Errorf("Unknown status: %v", snap.Status)
+}
+
+// addBasicAuthHeader adds basic auth  and X-AuthKey (for snapshot authkey) header to request
+// if their corresponding flags snapshot.basicAuthUsername, snapshot.basicAuthPassword and snapshot.authKey flags are set.
+func addAuthHeaders(req *http.Request) {
+	if basicAuthUser.Get() != "" {
+		auth := basicAuthUser.Get() + ":" + basicAuthPassword.Get()
+		authHeader := base64.StdEncoding.EncodeToString([]byte(auth))
+		req.Header.Set("Authorization", "Basic "+authHeader)
+	}
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5973
+	if snapshotAuthKey.Get() != "" {
+		req.Header.Set("X-AuthKey", snapshotAuthKey.Get())
+	}
 }