Unverified Commit de1c1243 by Frederic Branczyk Committed by GitHub

Merge pull request #8187 from brancz/cp-8176

Cut v2.22.2
parents 00f16d1a 84b186b3
## 2.22.2 / 2020-11-16
* [BUGFIX] Fix race condition in syncing/stopping/reloading scrapers. #8176
## 2.22.1 / 2020-11-03 ## 2.22.1 / 2020-11-03
* [BUGFIX] Fix potential "mmap: invalid argument" errors in loading the head chunks, after an unclean shutdown, by performing read repairs. #8061 * [BUGFIX] Fix potential "mmap: invalid argument" errors in loading the head chunks, after an unclean shutdown, by performing read repairs. #8061
......
...@@ -191,23 +191,20 @@ func init() { ...@@ -191,23 +191,20 @@ func init() {
type scrapePool struct { type scrapePool struct {
appendable storage.Appendable appendable storage.Appendable
logger log.Logger logger log.Logger
cancel context.CancelFunc
// targetMtx protects activeTargets and droppedTargets from concurrent reads // mtx must not be taken after targetMtx.
// and writes. Only one of Sync/stop/reload may be called at once due to mtx sync.Mutex
// manager.mtxScrape so we only need to protect from concurrent reads from config *config.ScrapeConfig
// the ActiveTargets and DroppedTargets methods. This allows those two client *http.Client
// methods to always complete without having to wait on scrape loops to gracefull stop. loops map[uint64]loop
targetMtx sync.Mutex targetLimitHit bool // Internal state to speed up the target_limit checks.
config *config.ScrapeConfig targetMtx sync.Mutex
client *http.Client // activeTargets and loops must always be synchronized to have the same
// Targets and loops must always be synchronized to have the same
// set of hashes. // set of hashes.
activeTargets map[uint64]*Target activeTargets map[uint64]*Target
droppedTargets []*Target droppedTargets []*Target
loops map[uint64]loop
cancel context.CancelFunc
targetLimitHit bool // Internal state to speed up the target_limit checks.
// Constructor for new scrape loops. This is settable for testing convenience. // Constructor for new scrape loops. This is settable for testing convenience.
newLoop func(scrapeLoopOptions) loop newLoop func(scrapeLoopOptions) loop
...@@ -297,6 +294,8 @@ func (sp *scrapePool) DroppedTargets() []*Target { ...@@ -297,6 +294,8 @@ func (sp *scrapePool) DroppedTargets() []*Target {
// stop terminates all scrape loops and returns after they all terminated. // stop terminates all scrape loops and returns after they all terminated.
func (sp *scrapePool) stop() { func (sp *scrapePool) stop() {
sp.mtx.Lock()
defer sp.mtx.Unlock()
sp.cancel() sp.cancel()
var wg sync.WaitGroup var wg sync.WaitGroup
...@@ -331,6 +330,8 @@ func (sp *scrapePool) stop() { ...@@ -331,6 +330,8 @@ func (sp *scrapePool) stop() {
// but all scrape loops are restarted with the new scrape configuration. // but all scrape loops are restarted with the new scrape configuration.
// This method returns after all scrape loops that were stopped have stopped scraping. // This method returns after all scrape loops that were stopped have stopped scraping.
func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error {
sp.mtx.Lock()
defer sp.mtx.Unlock()
targetScrapePoolReloads.Inc() targetScrapePoolReloads.Inc()
start := time.Now() start := time.Now()
...@@ -407,6 +408,8 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { ...@@ -407,6 +408,8 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error {
// Sync converts target groups into actual scrape targets and synchronizes // Sync converts target groups into actual scrape targets and synchronizes
// the currently running scraper with the resulting set and returns all scraped and dropped targets. // the currently running scraper with the resulting set and returns all scraped and dropped targets.
func (sp *scrapePool) Sync(tgs []*targetgroup.Group) { func (sp *scrapePool) Sync(tgs []*targetgroup.Group) {
sp.mtx.Lock()
defer sp.mtx.Unlock()
start := time.Now() start := time.Now()
sp.targetMtx.Lock() sp.targetMtx.Lock()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment