manager_test.go 28 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package discovery

import (
17
	"context"
Krasi Georgiev committed
18
	"fmt"
19
	"reflect"
20
	"sort"
Krasi Georgiev committed
21
	"strconv"
22
	"testing"
Bora Tunca committed
23
	"time"
24

25
	"github.com/go-kit/kit/log"
26
	client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
27
	"github.com/prometheus/common/model"
28
	"github.com/prometheus/prometheus/discovery/targetgroup"
29
	"github.com/prometheus/prometheus/util/testutil"
30 31
)

32 33 34 35
func TestMain(m *testing.M) {
	testutil.TolerantVerifyLeak(m)
}

36 37
// TestTargetUpdatesOrder checks that the target updates are received in the expected order.
func TestTargetUpdatesOrder(t *testing.T) {
Krasi Georgiev committed
38

39
	// The order by which the updates are send is determined by the interval passed to the mock discovery adapter
Krasi Georgiev committed
40 41
	// Final targets array is ordered alphabetically by the name of the discoverer.
	// For example discoverer "A" with targets "t2,t3" and discoverer "B" with targets "t1,t2" will result in "t2,t3,t1,t2" after the merge.
42
	testCases := []struct {
Krasi Georgiev committed
43 44
		title           string
		updates         map[string][]update
45
		expectedTargets [][]*targetgroup.Group
46 47 48 49 50 51
	}{
		{
			title: "Single TP no updates",
			updates: map[string][]update{
				"tp1": {},
			},
Krasi Georgiev committed
52
			expectedTargets: nil,
Bora Tunca committed
53
		},
54
		{
55
			title: "Multiple TPs no updates",
56 57 58 59 60
			updates: map[string][]update{
				"tp1": {},
				"tp2": {},
				"tp3": {},
			},
Krasi Georgiev committed
61
			expectedTargets: nil,
Bora Tunca committed
62
		},
63 64 65 66 67
		{
			title: "Single TP empty initials",
			updates: map[string][]update{
				"tp1": {
					{
68
						targetGroups: []targetgroup.Group{},
69
						interval:     5 * time.Millisecond,
70 71 72
					},
				},
			},
73
			expectedTargets: [][]*targetgroup.Group{
74 75
				{},
			},
Bora Tunca committed
76
		},
77 78 79 80 81
		{
			title: "Multiple TPs empty initials",
			updates: map[string][]update{
				"tp1": {
					{
82
						targetGroups: []targetgroup.Group{},
83
						interval:     5 * time.Millisecond,
84 85 86 87
					},
				},
				"tp2": {
					{
88
						targetGroups: []targetgroup.Group{},
89
						interval:     200 * time.Millisecond,
90 91 92 93
					},
				},
				"tp3": {
					{
94
						targetGroups: []targetgroup.Group{},
95
						interval:     100 * time.Millisecond,
96 97 98
					},
				},
			},
99
			expectedTargets: [][]*targetgroup.Group{
Krasi Georgiev committed
100
				{},
101 102 103 104 105 106 107 108 109
				{},
				{},
			},
		},
		{
			title: "Single TP initials only",
			updates: map[string][]update{
				"tp1": {
					{
110
						targetGroups: []targetgroup.Group{
Krasi Georgiev committed
111
							{
112
								Source:  "tp1_group1",
Krasi Georgiev committed
113 114 115
								Targets: []model.LabelSet{{"__instance__": "1"}},
							},
							{
116
								Source:  "tp1_group2",
Krasi Georgiev committed
117 118
								Targets: []model.LabelSet{{"__instance__": "2"}},
							}},
119 120 121
					},
				},
			},
122
			expectedTargets: [][]*targetgroup.Group{
123 124
				{
					{
125
						Source:  "tp1_group1",
126 127 128
						Targets: []model.LabelSet{{"__instance__": "1"}},
					},
					{
129
						Source:  "tp1_group2",
130 131 132
						Targets: []model.LabelSet{{"__instance__": "2"}},
					},
				},
133 134 135 136 137 138 139
			},
		},
		{
			title: "Multiple TPs initials only",
			updates: map[string][]update{
				"tp1": {
					{
140
						targetGroups: []targetgroup.Group{
Krasi Georgiev committed
141
							{
142
								Source:  "tp1_group1",
Krasi Georgiev committed
143
								Targets: []model.LabelSet{{"__instance__": "1"}},
144 145
							},
							{
146
								Source:  "tp1_group2",
Krasi Georgiev committed
147 148 149
								Targets: []model.LabelSet{{"__instance__": "2"}},
							},
						},
150 151 152 153
					},
				},
				"tp2": {
					{
154
						targetGroups: []targetgroup.Group{
155
							{
156
								Source:  "tp2_group1",
157 158 159
								Targets: []model.LabelSet{{"__instance__": "3"}},
							},
						},
160
						interval: 10 * time.Millisecond,
161 162 163
					},
				},
			},
164
			expectedTargets: [][]*targetgroup.Group{
Krasi Georgiev committed
165
				{
166
					{
167
						Source:  "tp1_group1",
Krasi Georgiev committed
168
						Targets: []model.LabelSet{{"__instance__": "1"}},
169 170
					},
					{
171
						Source:  "tp1_group2",
Krasi Georgiev committed
172
						Targets: []model.LabelSet{{"__instance__": "2"}},
173
					},
Krasi Georgiev committed
174
				}, {
175
					{
176
						Source:  "tp1_group1",
Krasi Georgiev committed
177
						Targets: []model.LabelSet{{"__instance__": "1"}},
178 179
					},
					{
180
						Source:  "tp1_group2",
Krasi Georgiev committed
181
						Targets: []model.LabelSet{{"__instance__": "2"}},
182 183
					},
					{
184
						Source:  "tp2_group1",
Krasi Georgiev committed
185
						Targets: []model.LabelSet{{"__instance__": "3"}},
186 187 188 189 190 191 192 193 194
					},
				},
			},
		},
		{
			title: "Single TP initials followed by empty updates",
			updates: map[string][]update{
				"tp1": {
					{
195
						targetGroups: []targetgroup.Group{
Krasi Georgiev committed
196
							{
197
								Source:  "tp1_group1",
Krasi Georgiev committed
198 199 200
								Targets: []model.LabelSet{{"__instance__": "1"}},
							},
							{
201
								Source:  "tp1_group2",
Krasi Georgiev committed
202 203 204 205
								Targets: []model.LabelSet{{"__instance__": "2"}},
							},
						},
						interval: 0,
206 207
					},
					{
208 209 210 211 212 213 214 215 216 217
						targetGroups: []targetgroup.Group{
							{
								Source:  "tp1_group1",
								Targets: []model.LabelSet{},
							},
							{
								Source:  "tp1_group2",
								Targets: []model.LabelSet{},
							},
						},
218
						interval: 10 * time.Millisecond,
219 220 221
					},
				},
			},
222
			expectedTargets: [][]*targetgroup.Group{
Krasi Georgiev committed
223
				{
224
					{
225
						Source:  "tp1_group1",
Krasi Georgiev committed
226
						Targets: []model.LabelSet{{"__instance__": "1"}},
227 228
					},
					{
229
						Source:  "tp1_group2",
Krasi Georgiev committed
230
						Targets: []model.LabelSet{{"__instance__": "2"}},
231 232
					},
				},
233 234 235 236 237 238 239 240 241 242
				{
					{
						Source:  "tp1_group1",
						Targets: []model.LabelSet{},
					},
					{
						Source:  "tp1_group2",
						Targets: []model.LabelSet{},
					},
				},
243 244 245
			},
		},
		{
Krasi Georgiev committed
246
			title: "Single TP initials and new groups",
247 248 249
			updates: map[string][]update{
				"tp1": {
					{
250
						targetGroups: []targetgroup.Group{
Krasi Georgiev committed
251
							{
252
								Source:  "tp1_group1",
Krasi Georgiev committed
253 254 255
								Targets: []model.LabelSet{{"__instance__": "1"}},
							},
							{
256
								Source:  "tp1_group2",
Krasi Georgiev committed
257 258 259 260
								Targets: []model.LabelSet{{"__instance__": "2"}},
							},
						},
						interval: 0,
261 262
					},
					{
263
						targetGroups: []targetgroup.Group{
Krasi Georgiev committed
264
							{
265
								Source:  "tp1_group1",
Krasi Georgiev committed
266 267 268
								Targets: []model.LabelSet{{"__instance__": "3"}},
							},
							{
269
								Source:  "tp1_group2",
Krasi Georgiev committed
270 271
								Targets: []model.LabelSet{{"__instance__": "4"}},
							},
272 273 274 275
							{
								Source:  "tp1_group3",
								Targets: []model.LabelSet{{"__instance__": "1"}},
							},
Krasi Georgiev committed
276
						},
277
						interval: 10 * time.Millisecond,
278 279 280
					},
				},
			},
281
			expectedTargets: [][]*targetgroup.Group{
Krasi Georgiev committed
282
				{
283
					{
284
						Source:  "tp1_group1",
Krasi Georgiev committed
285
						Targets: []model.LabelSet{{"__instance__": "1"}},
286 287
					},
					{
288
						Source:  "tp1_group2",
Krasi Georgiev committed
289
						Targets: []model.LabelSet{{"__instance__": "2"}},
290 291
					},
				},
Krasi Georgiev committed
292
				{
293
					{
294
						Source:  "tp1_group1",
Krasi Georgiev committed
295
						Targets: []model.LabelSet{{"__instance__": "3"}},
296 297
					},
					{
298
						Source:  "tp1_group2",
Krasi Georgiev committed
299
						Targets: []model.LabelSet{{"__instance__": "4"}},
300
					},
301 302 303 304
					{
						Source:  "tp1_group3",
						Targets: []model.LabelSet{{"__instance__": "1"}},
					},
305 306 307 308
				},
			},
		},
		{
Krasi Georgiev committed
309
			title: "Multiple TPs initials and new groups",
310 311 312
			updates: map[string][]update{
				"tp1": {
					{
313
						targetGroups: []targetgroup.Group{
Krasi Georgiev committed
314
							{
315
								Source:  "tp1_group1",
Krasi Georgiev committed
316 317 318
								Targets: []model.LabelSet{{"__instance__": "1"}},
							},
							{
319
								Source:  "tp1_group2",
Krasi Georgiev committed
320 321 322
								Targets: []model.LabelSet{{"__instance__": "2"}},
							},
						},
323
						interval: 10 * time.Millisecond,
324 325
					},
					{
326
						targetGroups: []targetgroup.Group{
Krasi Georgiev committed
327
							{
328
								Source:  "tp1_group3",
Krasi Georgiev committed
329 330 331
								Targets: []model.LabelSet{{"__instance__": "3"}},
							},
							{
332
								Source:  "tp1_group4",
Krasi Georgiev committed
333 334 335
								Targets: []model.LabelSet{{"__instance__": "4"}},
							},
						},
336
						interval: 500 * time.Millisecond,
337 338 339 340
					},
				},
				"tp2": {
					{
341
						targetGroups: []targetgroup.Group{
Krasi Georgiev committed
342
							{
343
								Source:  "tp2_group1",
Krasi Georgiev committed
344 345 346
								Targets: []model.LabelSet{{"__instance__": "5"}},
							},
							{
347
								Source:  "tp2_group2",
Krasi Georgiev committed
348 349 350
								Targets: []model.LabelSet{{"__instance__": "6"}},
							},
						},
351
						interval: 100 * time.Millisecond,
352 353
					},
					{
354
						targetGroups: []targetgroup.Group{
Krasi Georgiev committed
355
							{
356
								Source:  "tp2_group3",
Krasi Georgiev committed
357 358 359
								Targets: []model.LabelSet{{"__instance__": "7"}},
							},
							{
360
								Source:  "tp2_group4",
Krasi Georgiev committed
361 362 363
								Targets: []model.LabelSet{{"__instance__": "8"}},
							},
						},
364
						interval: 10 * time.Millisecond,
365 366 367
					},
				},
			},
368
			expectedTargets: [][]*targetgroup.Group{
Krasi Georgiev committed
369
				{
370
					{
371
						Source:  "tp1_group1",
Krasi Georgiev committed
372
						Targets: []model.LabelSet{{"__instance__": "1"}},
373 374
					},
					{
375
						Source:  "tp1_group2",
Krasi Georgiev committed
376
						Targets: []model.LabelSet{{"__instance__": "2"}},
377 378
					},
				},
Krasi Georgiev committed
379
				{
380
					{
381
						Source:  "tp1_group1",
Krasi Georgiev committed
382
						Targets: []model.LabelSet{{"__instance__": "1"}},
383 384
					},
					{
385
						Source:  "tp1_group2",
Krasi Georgiev committed
386
						Targets: []model.LabelSet{{"__instance__": "2"}},
387 388
					},
					{
389
						Source:  "tp2_group1",
Krasi Georgiev committed
390
						Targets: []model.LabelSet{{"__instance__": "5"}},
391 392
					},
					{
393
						Source:  "tp2_group2",
Krasi Georgiev committed
394
						Targets: []model.LabelSet{{"__instance__": "6"}},
395 396
					},
				},
Krasi Georgiev committed
397
				{
398
					{
399
						Source:  "tp1_group1",
Krasi Georgiev committed
400
						Targets: []model.LabelSet{{"__instance__": "1"}},
401 402
					},
					{
403
						Source:  "tp1_group2",
Krasi Georgiev committed
404
						Targets: []model.LabelSet{{"__instance__": "2"}},
405 406
					},
					{
407 408 409 410 411 412 413 414 415
						Source:  "tp2_group1",
						Targets: []model.LabelSet{{"__instance__": "5"}},
					},
					{
						Source:  "tp2_group2",
						Targets: []model.LabelSet{{"__instance__": "6"}},
					},
					{
						Source:  "tp2_group3",
Krasi Georgiev committed
416
						Targets: []model.LabelSet{{"__instance__": "7"}},
417 418
					},
					{
419
						Source:  "tp2_group4",
Krasi Georgiev committed
420
						Targets: []model.LabelSet{{"__instance__": "8"}},
421 422
					},
				},
Krasi Georgiev committed
423
				{
424
					{
425 426 427 428 429 430 431 432 433
						Source:  "tp1_group1",
						Targets: []model.LabelSet{{"__instance__": "1"}},
					},
					{
						Source:  "tp1_group2",
						Targets: []model.LabelSet{{"__instance__": "2"}},
					},
					{
						Source:  "tp1_group3",
Krasi Georgiev committed
434
						Targets: []model.LabelSet{{"__instance__": "3"}},
435 436
					},
					{
437
						Source:  "tp1_group4",
Krasi Georgiev committed
438
						Targets: []model.LabelSet{{"__instance__": "4"}},
439 440
					},
					{
441 442 443 444 445 446 447 448 449
						Source:  "tp2_group1",
						Targets: []model.LabelSet{{"__instance__": "5"}},
					},
					{
						Source:  "tp2_group2",
						Targets: []model.LabelSet{{"__instance__": "6"}},
					},
					{
						Source:  "tp2_group3",
Krasi Georgiev committed
450
						Targets: []model.LabelSet{{"__instance__": "7"}},
451 452
					},
					{
453
						Source:  "tp2_group4",
Krasi Georgiev committed
454
						Targets: []model.LabelSet{{"__instance__": "8"}},
455 456 457
					},
				},
			},
Bora Tunca committed
458
		},
459
		{
460
			title: "One TP initials arrive after other TP updates.",
461 462 463
			updates: map[string][]update{
				"tp1": {
					{
464
						targetGroups: []targetgroup.Group{
465
							{
466
								Source:  "tp1_group1",
Krasi Georgiev committed
467
								Targets: []model.LabelSet{{"__instance__": "1"}},
468 469
							},
							{
470
								Source:  "tp1_group2",
Krasi Georgiev committed
471
								Targets: []model.LabelSet{{"__instance__": "2"}},
472 473
							},
						},
474
						interval: 10 * time.Millisecond,
475 476
					},
					{
477
						targetGroups: []targetgroup.Group{
478
							{
479
								Source:  "tp1_group1",
Krasi Georgiev committed
480 481 482
								Targets: []model.LabelSet{{"__instance__": "3"}},
							},
							{
483
								Source:  "tp1_group2",
Krasi Georgiev committed
484
								Targets: []model.LabelSet{{"__instance__": "4"}},
485 486
							},
						},
487
						interval: 150 * time.Millisecond,
488 489
					},
				},
Krasi Georgiev committed
490
				"tp2": {
491
					{
492
						targetGroups: []targetgroup.Group{
493
							{
494
								Source:  "tp2_group1",
Krasi Georgiev committed
495
								Targets: []model.LabelSet{{"__instance__": "5"}},
496 497
							},
							{
498
								Source:  "tp2_group2",
Krasi Georgiev committed
499
								Targets: []model.LabelSet{{"__instance__": "6"}},
500 501
							},
						},
502
						interval: 200 * time.Millisecond,
503 504
					},
					{
505
						targetGroups: []targetgroup.Group{
506
							{
507
								Source:  "tp2_group1",
Krasi Georgiev committed
508 509 510
								Targets: []model.LabelSet{{"__instance__": "7"}},
							},
							{
511
								Source:  "tp2_group2",
Krasi Georgiev committed
512
								Targets: []model.LabelSet{{"__instance__": "8"}},
513 514
							},
						},
515
						interval: 100 * time.Millisecond,
516 517
					},
				},
Krasi Georgiev committed
518
			},
519
			expectedTargets: [][]*targetgroup.Group{
Krasi Georgiev committed
520
				{
521
					{
522
						Source:  "tp1_group1",
Krasi Georgiev committed
523
						Targets: []model.LabelSet{{"__instance__": "1"}},
524 525
					},
					{
526
						Source:  "tp1_group2",
Krasi Georgiev committed
527
						Targets: []model.LabelSet{{"__instance__": "2"}},
528
					},
Krasi Georgiev committed
529 530
				},
				{
531
					{
532
						Source:  "tp1_group1",
Krasi Georgiev committed
533 534 535
						Targets: []model.LabelSet{{"__instance__": "3"}},
					},
					{
536
						Source:  "tp1_group2",
Krasi Georgiev committed
537
						Targets: []model.LabelSet{{"__instance__": "4"}},
538 539
					},
				},
Krasi Georgiev committed
540
				{
541
					{
542
						Source:  "tp1_group1",
Krasi Georgiev committed
543
						Targets: []model.LabelSet{{"__instance__": "3"}},
544 545
					},
					{
546
						Source:  "tp1_group2",
Krasi Georgiev committed
547
						Targets: []model.LabelSet{{"__instance__": "4"}},
548 549
					},
					{
550
						Source:  "tp2_group1",
Krasi Georgiev committed
551
						Targets: []model.LabelSet{{"__instance__": "5"}},
552 553
					},
					{
554
						Source:  "tp2_group2",
Krasi Georgiev committed
555
						Targets: []model.LabelSet{{"__instance__": "6"}},
556 557
					},
				},
Krasi Georgiev committed
558
				{
559
					{
560
						Source:  "tp1_group1",
Krasi Georgiev committed
561
						Targets: []model.LabelSet{{"__instance__": "3"}},
562 563
					},
					{
564
						Source:  "tp1_group2",
Krasi Georgiev committed
565
						Targets: []model.LabelSet{{"__instance__": "4"}},
566 567
					},
					{
568
						Source:  "tp2_group1",
Krasi Georgiev committed
569 570 571
						Targets: []model.LabelSet{{"__instance__": "7"}},
					},
					{
572
						Source:  "tp2_group2",
Krasi Georgiev committed
573
						Targets: []model.LabelSet{{"__instance__": "8"}},
574 575 576 577
					},
				},
			},
		},
Krasi Georgiev committed
578

579
		{
580
			title: "Single TP empty update in between",
581 582 583
			updates: map[string][]update{
				"tp1": {
					{
584
						targetGroups: []targetgroup.Group{
585
							{
586
								Source:  "tp1_group1",
Krasi Georgiev committed
587
								Targets: []model.LabelSet{{"__instance__": "1"}},
588 589
							},
							{
590
								Source:  "tp1_group2",
Krasi Georgiev committed
591
								Targets: []model.LabelSet{{"__instance__": "2"}},
592 593
							},
						},
594
						interval: 30 * time.Millisecond,
595 596
					},
					{
597 598 599 600 601 602 603 604 605 606
						targetGroups: []targetgroup.Group{
							{
								Source:  "tp1_group1",
								Targets: []model.LabelSet{},
							},
							{
								Source:  "tp1_group2",
								Targets: []model.LabelSet{},
							},
						},
607
						interval: 10 * time.Millisecond,
608 609
					},
					{
610
						targetGroups: []targetgroup.Group{
611
							{
612
								Source:  "tp1_group1",
Krasi Georgiev committed
613
								Targets: []model.LabelSet{{"__instance__": "3"}},
614 615
							},
							{
616
								Source:  "tp1_group2",
Krasi Georgiev committed
617
								Targets: []model.LabelSet{{"__instance__": "4"}},
618 619
							},
						},
620
						interval: 300 * time.Millisecond,
621 622
					},
				},
Krasi Georgiev committed
623
			},
624
			expectedTargets: [][]*targetgroup.Group{
Krasi Georgiev committed
625
				{
626
					{
627
						Source:  "tp1_group1",
Krasi Georgiev committed
628
						Targets: []model.LabelSet{{"__instance__": "1"}},
629 630
					},
					{
631
						Source:  "tp1_group2",
Krasi Georgiev committed
632 633 634 635 636
						Targets: []model.LabelSet{{"__instance__": "2"}},
					},
				},
				{
					{
637 638 639 640 641 642 643 644 645 646 647
						Source:  "tp1_group1",
						Targets: []model.LabelSet{},
					},
					{
						Source:  "tp1_group2",
						Targets: []model.LabelSet{},
					},
				},
				{
					{
						Source:  "tp1_group1",
Krasi Georgiev committed
648 649 650
						Targets: []model.LabelSet{{"__instance__": "3"}},
					},
					{
651
						Source:  "tp1_group2",
Krasi Georgiev committed
652
						Targets: []model.LabelSet{{"__instance__": "4"}},
653 654 655 656 657 658
					},
				},
			},
		},
	}

659 660 661
	for i, tc := range testCases {
		tc := tc
		t.Run(tc.title, func(t *testing.T) {
662
			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
663
			defer cancel()
664

665 666 667 668 669 670
			discoveryManager := NewManager(ctx, log.NewNopLogger())
			discoveryManager.updatert = 100 * time.Millisecond

			var totalUpdatesCount int
			provUpdates := make(chan []*targetgroup.Group)
			for _, up := range tc.updates {
671
				go newMockDiscoveryProvider(up...).Run(ctx, provUpdates)
672
				if len(up) > 0 {
673
					totalUpdatesCount += len(up)
674
				}
Bora Tunca committed
675
			}
676

677 678
			for x := 0; x < totalUpdatesCount; x++ {
				select {
679
				case <-ctx.Done():
680
					t.Fatalf("%d: no update arrived within the timeout limit", x)
681 682 683 684 685 686 687 688 689
				case tgs := <-provUpdates:
					discoveryManager.updateGroup(poolKey{setName: strconv.Itoa(i), provider: tc.title}, tgs)
					for _, got := range discoveryManager.allGroups() {
						assertEqualGroups(t, got, tc.expectedTargets[x], func(got, expected string) string {
							return fmt.Sprintf("%d: \ntargets mismatch \ngot: %v \nexpected: %v",
								x,
								got,
								expected)
						})
Krasi Georgiev committed
690
					}
Bora Tunca committed
691 692
				}
			}
693 694 695 696 697 698 699 700 701 702 703 704 705
		})
	}
}

func assertEqualGroups(t *testing.T, got, expected []*targetgroup.Group, msg func(got, expected string) string) {
	t.Helper()
	format := func(groups []*targetgroup.Group) string {
		var s string
		for i, group := range groups {
			if i > 0 {
				s += ","
			}
			s += group.Source + ":" + fmt.Sprint(group.Targets)
Bora Tunca committed
706
		}
707 708 709 710 711 712 713 714 715
		return s
	}

	// Need to sort by the groups's source as the received order is not guaranteed.
	sort.Sort(byGroupSource(got))
	sort.Sort(byGroupSource(expected))

	if !reflect.DeepEqual(got, expected) {
		t.Errorf(msg(format(got), format(expected)))
Krasi Georgiev committed
716
	}
717

Krasi Georgiev committed
718
}
719

720 721 722 723 724 725 726 727 728 729 730 731 732
func staticConfig(addrs ...string) StaticConfig {
	var cfg StaticConfig
	for i, addr := range addrs {
		cfg = append(cfg, &targetgroup.Group{
			Source: fmt.Sprint(i),
			Targets: []model.LabelSet{
				{model.AddressLabel: model.LabelValue(addr)},
			},
		})
	}
	return cfg
}

733
func verifyPresence(t *testing.T, tSets map[poolKey]map[string]*targetgroup.Group, poolKey poolKey, label string, present bool) {
734
	t.Helper()
735 736 737 738
	if _, ok := tSets[poolKey]; !ok {
		t.Fatalf("'%s' should be present in Pool keys: %v", poolKey, tSets)
		return
	}
739

740 741 742
	match := false
	var mergedTargets string
	for _, targetGroup := range tSets[poolKey] {
743

744 745 746 747
		for _, l := range targetGroup.Targets {
			mergedTargets = mergedTargets + " " + l.String()
			if l.String() == label {
				match = true
Krasi Georgiev committed
748
			}
749
		}
750 751 752 753 754 755

	}
	if match != present {
		msg := ""
		if !present {
			msg = "not"
756
		}
757
		t.Fatalf("%q should %s be present in Targets labels: %q", label, msg, mergedTargets)
758
	}
759
}
760

761
func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) {
762 763
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
764
	discoveryManager := NewManager(ctx, log.NewNopLogger())
765
	discoveryManager.updatert = 100 * time.Millisecond
Krasi Georgiev committed
766
	go discoveryManager.Run()
767

768
	c := map[string]Configs{
769
		"prometheus": {
770
			staticConfig("foo:9090", "bar:9090"),
771
		},
Krasi Georgiev committed
772 773
	}
	discoveryManager.ApplyConfig(c)
774

775
	<-discoveryManager.SyncCh()
776 777
	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", true)
778

779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
	c["prometheus"] = Configs{
		staticConfig("foo:9090"),
	}
	discoveryManager.ApplyConfig(c)

	<-discoveryManager.SyncCh()
	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", false)
}

func TestDiscovererConfigs(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	discoveryManager := NewManager(ctx, log.NewNopLogger())
	discoveryManager.updatert = 100 * time.Millisecond
	go discoveryManager.Run()

	c := map[string]Configs{
		"prometheus": {
			staticConfig("foo:9090", "bar:9090"),
			staticConfig("baz:9090"),
800
		},
Krasi Georgiev committed
801 802
	}
	discoveryManager.ApplyConfig(c)
Bora Tunca committed
803

804
	<-discoveryManager.SyncCh()
805 806 807
	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"bar:9090\"}", true)
	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/1"}, "{__address__=\"baz:9090\"}", true)
808 809
}

810 811 812 813 814 815 816 817 818 819
// TestTargetSetRecreatesEmptyStaticConfigs ensures that reloading a config file after
// removing all targets from the static_configs sends an update with empty targetGroups.
// This is required to signal the receiver that this target set has no current targets.
func TestTargetSetRecreatesEmptyStaticConfigs(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	discoveryManager := NewManager(ctx, log.NewNopLogger())
	discoveryManager.updatert = 100 * time.Millisecond
	go discoveryManager.Run()

820
	c := map[string]Configs{
821
		"prometheus": {
822
			staticConfig("foo:9090"),
823
		},
824 825 826 827
	}
	discoveryManager.ApplyConfig(c)

	<-discoveryManager.SyncCh()
828
	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
829

830 831
	c["prometheus"] = Configs{
		StaticConfig{{}},
832 833 834 835 836
	}
	discoveryManager.ApplyConfig(c)

	<-discoveryManager.SyncCh()

837
	pkey := poolKey{setName: "prometheus", provider: "static/0"}
838 839 840 841 842 843 844 845 846 847 848 849 850 851
	targetGroups, ok := discoveryManager.targets[pkey]
	if !ok {
		t.Fatalf("'%v' should be present in target groups", pkey)
	}
	group, ok := targetGroups[""]
	if !ok {
		t.Fatalf("missing '' key in target groups %v", targetGroups)
	}

	if len(group.Targets) != 0 {
		t.Fatalf("Invalid number of targets: expected 0, got %d", len(group.Targets))
	}
}

852 853 854 855
func TestIdenticalConfigurationsAreCoalesced(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	discoveryManager := NewManager(ctx, nil)
856
	discoveryManager.updatert = 100 * time.Millisecond
857 858
	go discoveryManager.Run()

859
	c := map[string]Configs{
860
		"prometheus": {
861
			staticConfig("foo:9090"),
862
		},
863
		"prometheus2": {
864
			staticConfig("foo:9090"),
865
		},
866 867 868 869
	}
	discoveryManager.ApplyConfig(c)

	<-discoveryManager.SyncCh()
870 871
	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
	verifyPresence(t, discoveryManager.targets, poolKey{setName: "prometheus2", provider: "static/0"}, "{__address__=\"foo:9090\"}", true)
872 873 874
	if len(discoveryManager.providers) != 1 {
		t.Fatalf("Invalid number of providers: expected 1, got %d", len(discoveryManager.providers))
	}
Bora Tunca committed
875 876
}

877 878 879
func TestApplyConfigDoesNotModifyStaticTargets(t *testing.T) {
	originalConfig := Configs{
		staticConfig("foo:9090", "bar:9090", "baz:9090"),
880
	}
881 882
	processedConfig := Configs{
		staticConfig("foo:9090", "bar:9090", "baz:9090"),
883 884 885
	}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
886
	discoveryManager := NewManager(ctx, log.NewNopLogger())
887
	discoveryManager.updatert = 100 * time.Millisecond
888 889
	go discoveryManager.Run()

890 891
	cfgs := map[string]Configs{
		"prometheus": processedConfig,
892
	}
893
	discoveryManager.ApplyConfig(cfgs)
894 895
	<-discoveryManager.SyncCh()

896 897
	for _, cfg := range cfgs {
		if !reflect.DeepEqual(originalConfig, cfg) {
898
			t.Fatalf("discovery manager modified static config \n  expected: %v\n  got: %v\n",
899
				originalConfig, cfg)
900 901 902 903
		}
	}
}

904 905 906 907 908
type errorConfig struct{ err error }

func (e errorConfig) Name() string                                        { return "error" }
func (e errorConfig) NewDiscoverer(DiscovererOptions) (Discoverer, error) { return nil, e.err }

909 910 911 912 913 914 915
func TestGaugeFailedConfigs(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	discoveryManager := NewManager(ctx, log.NewNopLogger())
	discoveryManager.updatert = 100 * time.Millisecond
	go discoveryManager.Run()

916
	c := map[string]Configs{
917
		"prometheus": {
918 919 920
			errorConfig{fmt.Errorf("tests error 0")},
			errorConfig{fmt.Errorf("tests error 1")},
			errorConfig{fmt.Errorf("tests error 2")},
921
		},
922 923 924 925
	}
	discoveryManager.ApplyConfig(c)
	<-discoveryManager.SyncCh()

926
	failedCount := client_testutil.ToFloat64(failedConfigs)
927 928 929 930
	if failedCount != 3 {
		t.Fatalf("Expected to have 3 failed configs, got: %v", failedCount)
	}

931 932
	c["prometheus"] = Configs{
		staticConfig("foo:9090"),
933 934 935 936
	}
	discoveryManager.ApplyConfig(c)
	<-discoveryManager.SyncCh()

937
	failedCount = client_testutil.ToFloat64(failedConfigs)
938 939 940 941 942 943
	if failedCount != 0 {
		t.Fatalf("Expected to get no failed config, got: %v", failedCount)
	}

}

944 945 946 947 948 949 950 951 952 953
func TestCoordinationWithReceiver(t *testing.T) {
	updateDelay := 100 * time.Millisecond

	type expect struct {
		delay time.Duration
		tgs   map[string][]*targetgroup.Group
	}

	testCases := []struct {
		title     string
954
		providers map[string]Discoverer
955 956
		expected  []expect
	}{
957
		{
958
			title: "Receiver should get all updates even when one provider closes its channel",
959 960 961 962 963 964 965 966 967 968 969
			providers: map[string]Discoverer{
				"once1": &onceProvider{
					tgs: []*targetgroup.Group{
						{
							Source:  "tg1",
							Targets: []model.LabelSet{{"__instance__": "1"}},
						},
					},
				},
				"mock1": newMockDiscoveryProvider(
					update{
970
						interval: 2 * updateDelay,
971 972 973 974 975 976 977 978 979 980 981 982
						targetGroups: []targetgroup.Group{
							{
								Source:  "tg2",
								Targets: []model.LabelSet{{"__instance__": "2"}},
							},
						},
					},
				),
			},
			expected: []expect{
				{
					tgs: map[string][]*targetgroup.Group{
Matt Layher committed
983
						"once1": {
984 985 986 987 988 989 990 991 992
							{
								Source:  "tg1",
								Targets: []model.LabelSet{{"__instance__": "1"}},
							},
						},
					},
				},
				{
					tgs: map[string][]*targetgroup.Group{
Matt Layher committed
993
						"once1": {
994 995 996 997 998
							{
								Source:  "tg1",
								Targets: []model.LabelSet{{"__instance__": "1"}},
							},
						},
Matt Layher committed
999
						"mock1": {
1000 1001 1002 1003 1004 1005 1006 1007 1008
							{
								Source:  "tg2",
								Targets: []model.LabelSet{{"__instance__": "2"}},
							},
						},
					},
				},
			},
		},
1009
		{
1010
			title: "Receiver should get all updates even when the channel is blocked",
1011 1012
			providers: map[string]Discoverer{
				"mock1": newMockDiscoveryProvider(
1013 1014 1015 1016 1017 1018 1019 1020 1021
					update{
						targetGroups: []targetgroup.Group{
							{
								Source:  "tg1",
								Targets: []model.LabelSet{{"__instance__": "1"}},
							},
						},
					},
					update{
1022
						interval: 4 * updateDelay,
1023 1024 1025 1026 1027 1028 1029
						targetGroups: []targetgroup.Group{
							{
								Source:  "tg2",
								Targets: []model.LabelSet{{"__instance__": "2"}},
							},
						},
					},
1030
				),
1031 1032 1033 1034 1035
			},
			expected: []expect{
				{
					delay: 2 * updateDelay,
					tgs: map[string][]*targetgroup.Group{
Matt Layher committed
1036
						"mock1": {
1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
							{
								Source:  "tg1",
								Targets: []model.LabelSet{{"__instance__": "1"}},
							},
						},
					},
				},
				{
					delay: 4 * updateDelay,
					tgs: map[string][]*targetgroup.Group{
Matt Layher committed
1047
						"mock1": {
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
							{
								Source:  "tg1",
								Targets: []model.LabelSet{{"__instance__": "1"}},
							},
							{
								Source:  "tg2",
								Targets: []model.LabelSet{{"__instance__": "2"}},
							},
						},
					},
				},
			},
		},
	}

	for _, tc := range testCases {
		tc := tc
		t.Run(tc.title, func(t *testing.T) {
1066
			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)