Unverified Commit 4e6a94a2 by Andy Bursavich Committed by GitHub

Invert service discovery dependencies (#7701)

This also fixes a bug in query_log_file, which now is relative to the config file like all other paths.
Signed-off-by: 's avatarAndy Bursavich <abursavich@gmail.com>
parent 274dce9d
......@@ -53,7 +53,6 @@ import (
promlogflag "github.com/prometheus/common/promlog/flag"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery"
sd_config "github.com/prometheus/prometheus/discovery/config"
"github.com/prometheus/prometheus/notifier"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/logging"
......@@ -67,6 +66,8 @@ import (
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/util/strutil"
"github.com/prometheus/prometheus/web"
_ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations.
)
var (
......@@ -466,9 +467,9 @@ func main() {
}, {
name: "scrape_sd",
reloader: func(cfg *config.Config) error {
c := make(map[string]sd_config.ServiceDiscoveryConfig)
c := make(map[string]discovery.Configs)
for _, v := range cfg.ScrapeConfigs {
c[v.JobName] = v.ServiceDiscoveryConfig
c[v.JobName] = v.ServiceDiscoveryConfigs
}
return discoveryManagerScrape.ApplyConfig(c)
},
......@@ -478,9 +479,9 @@ func main() {
}, {
name: "notify_sd",
reloader: func(cfg *config.Config) error {
c := make(map[string]sd_config.ServiceDiscoveryConfig)
c := make(map[string]discovery.Configs)
for k, v := range cfg.AlertingConfig.AlertmanagerConfigs.ToMap() {
c[k] = v.ServiceDiscoveryConfig
c[k] = v.ServiceDiscoveryConfigs
}
return discoveryManagerNotify.ApplyConfig(c)
},
......
......@@ -40,7 +40,11 @@ import (
"gopkg.in/alecthomas/kingpin.v2"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/discovery/file"
"github.com/prometheus/prometheus/discovery/kubernetes"
"github.com/prometheus/prometheus/pkg/rulefmt"
_ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations.
)
func main() {
......@@ -263,24 +267,25 @@ func checkConfig(filename string) ([]string, error) {
return nil, err
}
for _, kd := range scfg.ServiceDiscoveryConfig.KubernetesSDConfigs {
if err := checkTLSConfig(kd.HTTPClientConfig.TLSConfig); err != nil {
return nil, err
}
}
for _, filesd := range scfg.ServiceDiscoveryConfig.FileSDConfigs {
for _, file := range filesd.Files {
files, err := filepath.Glob(file)
if err != nil {
for _, c := range scfg.ServiceDiscoveryConfigs {
switch c := c.(type) {
case *kubernetes.SDConfig:
if err := checkTLSConfig(c.HTTPClientConfig.TLSConfig); err != nil {
return nil, err
}
if len(files) != 0 {
// There was at least one match for the glob and we can assume checkFileExists
// for all matches would pass, we can continue the loop.
continue
case *file.SDConfig:
for _, file := range c.Files {
files, err := filepath.Glob(file)
if err != nil {
return nil, err
}
if len(files) != 0 {
// There was at least one match for the glob and we can assume checkFileExists
// for all matches would pass, we can continue the loop.
continue
}
fmt.Printf(" WARNING: file %q for file_sd in scrape job %q does not exist\n", file, scfg.JobName)
}
fmt.Printf(" WARNING: file %q for file_sd in scrape job %q does not exist\n", file, scfg.JobName)
}
}
}
......
......@@ -24,5 +24,4 @@ var ruleFilesExpectedConf = &Config{
"testdata/rules/second.rules",
"/absolute/third.rules",
},
original: "",
}
......@@ -22,5 +22,4 @@ var ruleFilesExpectedConf = &Config{
"testdata\\rules\\second.rules",
"c:\\absolute\\third.rules",
},
original: "",
}
alerting:
alertmanagers:
- scheme: https
file_sd_configs:
- files:
- foo/*.slow.json
- foo/*.slow.yml
refresh_interval: 10m
- files:
- bar/*.yaml
static_configs:
- targets:
- 1.2.3.4:9093
- 1.2.3.5:9093
- 1.2.3.6:9093
scrape_configs:
- job_name: foo
static_configs:
- targets:
- localhost:9090
- localhost:9191
labels:
my: label
your: label
- job_name: bar
azure_sd_configs:
- environment: AzurePublicCloud
authentication_method: OAuth
subscription_id: 11AAAA11-A11A-111A-A111-1111A1111A11
tenant_id: BBBB222B-B2B2-2B22-B222-2BB2222BB2B2
client_id: 333333CC-3C33-3333-CCC3-33C3CCCCC33C
client_secret: <secret>
port: 9100
consul_sd_configs:
- server: localhost:1234
token: <secret>
services: [nginx, cache, mysql]
tags: [canary, v1]
node_meta:
rack: "123"
allow_stale: true
scheme: https
tls_config:
ca_file: valid_ca_file
cert_file: valid_cert_file
key_file: valid_key_file
digitalocean_sd_configs:
- bearer_token: <secret>
dockerswarm_sd_configs:
- host: http://127.0.0.1:2375
role: nodes
dns_sd_configs:
- refresh_interval: 15s
names:
- first.dns.address.domain.com
- second.dns.address.domain.com
- names:
- first.dns.address.domain.com
ec2_sd_configs:
- region: us-east-1
access_key: access
secret_key: <secret>
profile: profile
filters:
- name: tag:environment
values:
- prod
- name: tag:service
values:
- web
- db
file_sd_configs:
- files:
- single/file.yml
kubernetes_sd_configs:
- role: endpoints
api_server: https://localhost:1234
tls_config:
cert_file: valid_cert_file
key_file: valid_key_file
basic_auth:
username: username
password: <secret>
- role: endpoints
api_server: https://localhost:1234
namespaces:
names:
- default
basic_auth:
username: username
password_file: valid_password_file
marathon_sd_configs:
- servers:
- https://marathon.example.com:443
auth_token: <secret>
tls_config:
cert_file: valid_cert_file
key_file: valid_key_file
nerve_sd_configs:
- servers:
- localhost
paths:
- /monitoring
openstack_sd_configs:
- role: instance
region: RegionOne
port: 80
refresh_interval: 1m
tls_config:
ca_file: valid_ca_file
cert_file: valid_cert_file
key_file: valid_key_file
static_configs:
- targets:
- localhost:9093
triton_sd_configs:
- account: testAccount
dns_suffix: triton.example.com
endpoint: triton.example.com
port: 9163
refresh_interval: 1m
version: 1
tls_config:
cert_file: valid_cert_file
key_file: valid_key_file
......@@ -146,85 +146,115 @@ both cases.
For example if we had a discovery mechanism and it retrieves the following groups:
```
```go
[]targetgroup.Group{
{
Targets: []model.LabelSet{
{
"__instance__": "10.11.150.1:7870",
"hostname": "demo-target-1",
"test": "simple-test",
},
{
"__instance__": "10.11.150.4:7870",
"hostname": "demo-target-2",
"test": "simple-test",
},
},
Labels: map[LabelName][LabelValue] {
"job": "mysql",
},
"Source": "file1",
},
{
Targets: []model.LabelSet{
{
"__instance__": "10.11.122.11:6001",
"hostname": "demo-postgres-1",
"test": "simple-test",
},
{
"__instance__": "10.11.122.15:6001",
"hostname": "demo-postgres-2",
"test": "simple-test",
},
},
Labels: map[LabelName][LabelValue] {
"job": "postgres",
},
"Source": "file2",
},
{
Targets: []model.LabelSet{
{
"__instance__": "10.11.150.1:7870",
"hostname": "demo-target-1",
"test": "simple-test",
},
{
"__instance__": "10.11.150.4:7870",
"hostname": "demo-target-2",
"test": "simple-test",
},
},
Labels: model.LabelSet{
"job": "mysql",
},
"Source": "file1",
},
{
Targets: []model.LabelSet{
{
"__instance__": "10.11.122.11:6001",
"hostname": "demo-postgres-1",
"test": "simple-test",
},
{
"__instance__": "10.11.122.15:6001",
"hostname": "demo-postgres-2",
"test": "simple-test",
},
},
Labels: model.LabelSet{
"job": "postgres",
},
"Source": "file2",
},
}
```
Here there are two target groups one group with source `file1` and another with `file2`. The grouping is implementation specific and could even be one target per group. But, one has to make sure every target group sent by an SD instance should have a `Source` which is unique across all the target groups of that SD instance.
In this case, both the target groups are sent down the channel the first time `Run()` is called. Now, for an update, we need to send the whole _changed_ target group down the channel. i.e, if the target with `hostname: demo-postgres-2` goes away, we send:
```
```go
&targetgroup.Group{
Targets: []model.LabelSet{
{
"__instance__": "10.11.122.11:6001",
"hostname": "demo-postgres-1",
"test": "simple-test",
},
},
Labels: map[LabelName][LabelValue] {
"job": "postgres",
},
"Source": "file2",
Targets: []model.LabelSet{
{
"__instance__": "10.11.122.11:6001",
"hostname": "demo-postgres-1",
"test": "simple-test",
},
},
Labels: model.LabelSet{
"job": "postgres",
},
"Source": "file2",
}
```
down the channel.
If all the targets in a group go away, we need to send the target groups with empty `Targets` down the channel. i.e, if all targets with `job: postgres` go away, we send:
```
```go
&targetgroup.Group{
Targets: nil,
"Source": "file2",
Targets: nil,
"Source": "file2",
}
```
down the channel.
### The Config interface
Now that your service discovery mechanism is ready to discover targets, you must help
Prometheus discover it. This is done by implementing the `discovery.Config` interface
and registering it with `discovery.RegisterConfig` in an init function of your package.
```go
type Config interface {
// Name returns the name of the discovery mechanism.
Name() string
// NewDiscoverer returns a Discoverer for the Config
// with the given DiscovererOptions.
NewDiscoverer(DiscovererOptions) (Discoverer, error)
}
type DiscovererOptions struct {
Logger log.Logger
}
```
The value returned by `Name()` should be short, descriptive, lowercase, and unique.
It's used to tag the provided `Logger` and as the part of the YAML key for your SD
mechanism's list of configs in `scrape_config` and `alertmanager_config`
(e.g. `${NAME}_sd_configs`).
### New Service Discovery Check List
Here are some non-obvious parts of adding service discoveries that need to be verified:
- Check for `nil` SDConfigs in `discovery/config/config.go`.
- Validate that discovery configs can be DeepEqualled by adding them to
`config/testdata/conf.good.yml` and to the associated tests.
- If there is a TLSConfig or HTTPClientConfig, add them to
`resolveFilepaths` in `config/config.go`.
- If the config contains file paths directly or indirectly (e.g. with a TLSConfig or
HTTPClientConfig field), then it must implement `config.DirectorySetter`.
- Import your SD package from `prometheus/discovery/install`. The install package is
imported from `main` to register all builtin SD mechanisms.
- List the service discovery in both `<scrape_config>` and
`<alertmanager_config>` in `docs/configuration/configuration.md`.
......
......@@ -33,6 +33,7 @@ import (
config_util "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery"
"github.com/prometheus/prometheus/discovery/refresh"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/strutil"
......@@ -64,6 +65,10 @@ var DefaultSDConfig = SDConfig{
AuthenticationMethod: authMethodOAuth,
}
func init() {
discovery.RegisterConfig(&SDConfig{})
}
// SDConfig is the configuration for Azure based service discovery.
type SDConfig struct {
Environment string `yaml:"environment,omitempty"`
......@@ -76,6 +81,14 @@ type SDConfig struct {
AuthenticationMethod string `yaml:"authentication_method,omitempty"`
}
// Name returns the name of the Config.
func (*SDConfig) Name() string { return "azure" }
// NewDiscoverer returns a Discoverer for the Config.
func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
return NewDiscovery(c, opts.Logger), nil
}
func validateAuthParam(param, name string) error {
if len(param) == 0 {
return errors.Errorf("azure SD configuration requires a %s", name)
......
// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"github.com/pkg/errors"
"github.com/prometheus/prometheus/discovery/azure"
"github.com/prometheus/prometheus/discovery/consul"
"github.com/prometheus/prometheus/discovery/digitalocean"
"github.com/prometheus/prometheus/discovery/dns"
"github.com/prometheus/prometheus/discovery/dockerswarm"
"github.com/prometheus/prometheus/discovery/ec2"
"github.com/prometheus/prometheus/discovery/file"
"github.com/prometheus/prometheus/discovery/gce"
"github.com/prometheus/prometheus/discovery/kubernetes"
"github.com/prometheus/prometheus/discovery/marathon"
"github.com/prometheus/prometheus/discovery/openstack"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/discovery/triton"
"github.com/prometheus/prometheus/discovery/zookeeper"
)
// ServiceDiscoveryConfig configures lists of different service discovery mechanisms.
type ServiceDiscoveryConfig struct {
// List of labeled target groups for this job.
StaticConfigs []*targetgroup.Group `yaml:"static_configs,omitempty"`
// List of DNS service discovery configurations.
DNSSDConfigs []*dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
// List of file service discovery configurations.
FileSDConfigs []*file.SDConfig `yaml:"file_sd_configs,omitempty"`
// List of Consul service discovery configurations.
ConsulSDConfigs []*consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
// List of DigitalOcean service discovery configurations.
DigitalOceanSDConfigs []*digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
// List of Docker Swarm service discovery configurations.
DockerSwarmSDConfigs []*dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
// List of Serverset service discovery configurations.
ServersetSDConfigs []*zookeeper.ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"`
// NerveSDConfigs is a list of Nerve service discovery configurations.
NerveSDConfigs []*zookeeper.NerveSDConfig `yaml:"nerve_sd_configs,omitempty"`
// MarathonSDConfigs is a list of Marathon service discovery configurations.
MarathonSDConfigs []*marathon.SDConfig `yaml:"marathon_sd_configs,omitempty"`
// List of Kubernetes service discovery configurations.
KubernetesSDConfigs []*kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
// List of GCE service discovery configurations.
GCESDConfigs []*gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
// List of EC2 service discovery configurations.
EC2SDConfigs []*ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"`
// List of OpenStack service discovery configurations.
OpenstackSDConfigs []*openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
// List of Azure service discovery configurations.
AzureSDConfigs []*azure.SDConfig `yaml:"azure_sd_configs,omitempty"`
// List of Triton service discovery configurations.
TritonSDConfigs []*triton.SDConfig `yaml:"triton_sd_configs,omitempty"`
}
// Validate validates the ServiceDiscoveryConfig.
func (c *ServiceDiscoveryConfig) Validate() error {
for _, cfg := range c.AzureSDConfigs {
if cfg == nil {
return errors.New("empty or null section in azure_sd_configs")
}
}
for _, cfg := range c.ConsulSDConfigs {
if cfg == nil {
return errors.New("empty or null section in consul_sd_configs")
}
}
for _, cfg := range c.DigitalOceanSDConfigs {
if cfg == nil {
return errors.New("empty or null section in digitalocean_sd_configs")
}
}
for _, cfg := range c.DockerSwarmSDConfigs {
if cfg == nil {
return errors.New("empty or null section in dockerswarm_sd_configs")
}
}
for _, cfg := range c.DNSSDConfigs {
if cfg == nil {
return errors.New("empty or null section in dns_sd_configs")
}
}
for _, cfg := range c.EC2SDConfigs {
if cfg == nil {
return errors.New("empty or null section in ec2_sd_configs")
}
}
for _, cfg := range c.FileSDConfigs {
if cfg == nil {
return errors.New("empty or null section in file_sd_configs")
}
}
for _, cfg := range c.GCESDConfigs {
if cfg == nil {
return errors.New("empty or null section in gce_sd_configs")
}
}
for _, cfg := range c.KubernetesSDConfigs {
if cfg == nil {
return errors.New("empty or null section in kubernetes_sd_configs")
}
}
for _, cfg := range c.MarathonSDConfigs {
if cfg == nil {
return errors.New("empty or null section in marathon_sd_configs")
}
}
for _, cfg := range c.NerveSDConfigs {
if cfg == nil {
return errors.New("empty or null section in nerve_sd_configs")
}
}
for _, cfg := range c.OpenstackSDConfigs {
if cfg == nil {
return errors.New("empty or null section in openstack_sd_configs")
}
}
for _, cfg := range c.ServersetSDConfigs {
if cfg == nil {
return errors.New("empty or null section in serverset_sd_configs")
}
}
for _, cfg := range c.StaticConfigs {
if cfg == nil {
return errors.New("empty or null section in static_configs")
}
}
for _, cfg := range c.TritonSDConfigs {
if cfg == nil {
return errors.New("empty or null section in triton_sd_configs")
}
}
return nil
}
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"fmt"
"reflect"
"strings"
"testing"
"github.com/prometheus/prometheus/util/testutil"
"gopkg.in/yaml.v2"
)
func TestForNilSDConfig(t *testing.T) {
// Get all the yaml fields names of the ServiceDiscoveryConfig struct.
s := reflect.ValueOf(ServiceDiscoveryConfig{})
configType := s.Type()
n := s.NumField()
fieldsSlice := make([]string, n)
for i := 0; i < n; i++ {
field := configType.Field(i)
tag := field.Tag.Get("yaml")
tag = strings.Split(tag, ",")[0]
fieldsSlice = append(fieldsSlice, tag)
}
// Unmarshall all possible yaml keys and validate errors check upon nil
// SD config.
for _, f := range fieldsSlice {
if f == "" {
continue
}
t.Run(f, func(t *testing.T) {
c := &ServiceDiscoveryConfig{}
err := yaml.Unmarshal([]byte(fmt.Sprintf(`
---
%s:
-
`, f)), c)
testutil.Ok(t, err)
err = c.Validate()
testutil.NotOk(t, err)
testutil.Equals(t, fmt.Sprintf("empty or null section in %s", f), err.Error())
})
}
}
......@@ -28,9 +28,10 @@ import (
conntrack "github.com/mwitkow/go-conntrack"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
config_util "github.com/prometheus/common/config"
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/discovery"
"github.com/prometheus/prometheus/discovery/targetgroup"
"github.com/prometheus/prometheus/util/strutil"
)
......@@ -99,15 +100,21 @@ var (
}
)
func init() {
discovery.RegisterConfig(&SDConfig{})
prometheus.MustRegister(rpcFailuresCount)
prometheus.MustRegister(rpcDuration)
}
// SDConfig is the configuration for Consul service discovery.
type SDConfig struct {
Server string `yaml:"server,omitempty"`
Token config_util.Secret `yaml:"token,omitempty"`
Datacenter string `yaml:"datacenter,omitempty"`
TagSeparator string `yaml:"tag_separator,omitempty"`
Scheme string `yaml:"scheme,omitempty"`
Username string `yaml:"username,omitempty"`
Password config_util.Secret `yaml:"password,omitempty"`
Server string `yaml:"server,omitempty"`
Token config.Secret `yaml:"token,omitempty"`
Datacenter string `yaml:"datacenter,omitempty"`
TagSeparator string `yaml:"tag_separator,omitempty"`
Scheme string `yaml:"scheme,omitempty"`
Username string `yaml:"username,omitempty"`
Password config.Secret `yaml:"password,omitempty"`
// See https://www.consul.io/docs/internals/consensus.html#consistency-modes,
// stale reads are a lot cheaper and are a necessity if you have >5k targets.
......@@ -127,7 +134,20 @@ type SDConfig struct {
// Desired node metadata.