Query: make endpoint heartbeat interval configurable

* make endpoint heartbat interval configurable * random cleanups of endpoint set Signed-off-by: Michael Hoffmann <[email protected]>
thanos-io · Jun 1, 2024 · 7a4b9a8 · 7a4b9a8
1 parent 863d914
commit 7a4b9a8
Show file tree

Hide file tree

Showing 5 changed files with 152 additions and 143 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#7317](https://github.com/thanos-io/thanos/pull/7317) Tracing: allow specifying resource attributes for the OTLP configuration.
 - [#7367](https://github.com/thanos-io/thanos/pull/7367) Store Gateway: log request ID in request logs.
 - [#7361](https://github.com/thanos-io/thanos/pull/7361) Query: *breaking :warning:* pass query stats from remote execution from server to client. We changed the protobuf of the QueryAPI, if you use `query.mode=distributed` you need to update your client (upper level Queriers) first, before updating leaf Queriers (servers).
+- [#7407](https://github.com/thanos-io/thanos/pull/7407) Query: make endpoint heartbeat interval configurable
 
 ### Changed
 

diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go
@@ -180,6 +180,7 @@ func registerQuery(app *extkingpin.App) {
 	unhealthyStoreTimeout := extkingpin.ModelDuration(cmd.Flag("store.unhealthy-timeout", "Timeout before an unhealthy store is cleaned from the store UI page.").Default("5m"))
 
 	endpointInfoTimeout := extkingpin.ModelDuration(cmd.Flag("endpoint.info-timeout", "Timeout of gRPC Info requests.").Default("5s").Hidden())
+	endpointInfoInterval := extkingpin.ModelDuration(cmd.Flag("endpoint.info-interval", "Interval to refresh gRPC Info requests.").Default("5s").Hidden())
 
 	enableAutodownsampling := cmd.Flag("query.auto-downsampling", "Enable automatic adjustment (step / 5) to what source of data should be used in store gateways if no max_source_resolution param is specified.").
 		Default("false").Bool()
@@ -347,6 +348,7 @@ func registerQuery(app *extkingpin.App) {
 			*dnsSDResolver,
 			time.Duration(*unhealthyStoreTimeout),
 			time.Duration(*endpointInfoTimeout),
+			time.Duration(*endpointInfoInterval),
 			time.Duration(*instantDefaultMaxSourceResolution),
 			*defaultMetadataTimeRange,
 			*strictStores,
@@ -429,6 +431,7 @@ func runQuery(
 	dnsSDResolver string,
 	unhealthyStoreTimeout time.Duration,
 	endpointInfoTimeout time.Duration,
+	endpointInfoInterval time.Duration,
 	instantDefaultMaxSourceResolution time.Duration,
 	defaultMetadataTimeRange time.Duration,
 	strictStores []string,
@@ -548,6 +551,7 @@ func runQuery(
 			dialOpts,
 			unhealthyStoreTimeout,
 			endpointInfoTimeout,
+			endpointInfoInterval,
 			queryConnMetricLabels...,
 		)
 
@@ -863,10 +867,10 @@ func prepareEndpointSet(
 	dialOpts []grpc.DialOption,
 	unhealthyStoreTimeout time.Duration,
 	endpointInfoTimeout time.Duration,
+	endpointInfoInterval time.Duration,
 	queryConnMetricLabels ...string,
 ) *query.EndpointSet {
 	endpointSet := query.NewEndpointSet(
-		time.Now,
 		logger,
 		reg,
 		func() (specs []*query.GRPCEndpointSpec) {
@@ -913,7 +917,7 @@ func prepareEndpointSet(
 	{
 		ctx, cancel := context.WithCancel(context.Background())
 		g.Add(func() error {
-			return runutil.Repeat(5*time.Second, ctx.Done(), func() error {
+			return runutil.Repeat(endpointInfoInterval, ctx.Done(), func() error {
 				endpointSet.Update(ctx)
 				return nil
 			})

diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go
@@ -440,6 +440,7 @@ func runRule(
 			dialOpts,
 			5*time.Minute,
 			5*time.Second,
+			5*time.Second,
 		)
 
 		// Periodically update the GRPC addresses from query config by resolving them using DNS SD if necessary.