dns.go 11 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dns

import (
17
	"context"
18 19 20 21 22 23
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

24 25
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
26
	"github.com/miekg/dns"
27
	"github.com/pkg/errors"
28 29
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"
30

31
	"github.com/prometheus/prometheus/discovery"
32
	"github.com/prometheus/prometheus/discovery/refresh"
33
	"github.com/prometheus/prometheus/discovery/targetgroup"
34 35 36 37 38
)

const (
	resolvConf = "/etc/resolv.conf"

39 40 41 42
	dnsNameLabel            = model.MetaLabelPrefix + "dns_name"
	dnsSrvRecordPrefix      = model.MetaLabelPrefix + "dns_srv_record_"
	dnsSrvRecordTargetLabel = dnsSrvRecordPrefix + "target"
	dnsSrvRecordPortLabel   = dnsSrvRecordPrefix + "port"
43 44 45 46 47 48 49 50 51

	// Constants for instrumentation.
	namespace = "prometheus"
)

var (
	dnsSDLookupsCount = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: namespace,
52
			Name:      "sd_dns_lookups_total",
53 54 55 56 57
			Help:      "The number of DNS-SD lookups.",
		})
	dnsSDLookupFailuresCount = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: namespace,
58
			Name:      "sd_dns_lookup_failures_total",
59 60
			Help:      "The number of DNS-SD lookup failures.",
		})
61 62 63 64 65 66

	// DefaultSDConfig is the default DNS SD configuration.
	DefaultSDConfig = SDConfig{
		RefreshInterval: model.Duration(30 * time.Second),
		Type:            "SRV",
	}
67 68
)

69 70 71 72 73 74
func init() {
	discovery.RegisterConfig(&SDConfig{})
	prometheus.MustRegister(dnsSDLookupFailuresCount)
	prometheus.MustRegister(dnsSDLookupsCount)
}

75 76 77 78 79 80 81 82
// SDConfig is the configuration for DNS based service discovery.
type SDConfig struct {
	Names           []string       `yaml:"names"`
	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
	Type            string         `yaml:"type"`
	Port            int            `yaml:"port"` // Ignored for SRV records
}

83 84 85 86 87 88 89 90
// Name returns the name of the Config.
func (*SDConfig) Name() string { return "dns" }

// NewDiscoverer returns a Discoverer for the Config.
func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
	return NewDiscovery(*c, opts.Logger), nil
}

91 92 93 94 95 96 97 98 99
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	*c = DefaultSDConfig
	type plain SDConfig
	err := unmarshal((*plain)(c))
	if err != nil {
		return err
	}
	if len(c.Names) == 0 {
100
		return errors.New("DNS-SD config must contain at least one SRV record name")
101 102 103 104 105
	}
	switch strings.ToUpper(c.Type) {
	case "SRV":
	case "A", "AAAA":
		if c.Port == 0 {
106
			return errors.New("a port is required in DNS-SD configs for all record types except SRV")
107 108
		}
	default:
109
		return errors.Errorf("invalid DNS-SD records type %s", c.Type)
110 111 112 113
	}
	return nil
}

114
// Discovery periodically performs DNS-SD requests. It implements
115
// the Discoverer interface.
116
type Discovery struct {
117 118 119 120 121
	*refresh.Discovery
	names  []string
	port   int
	qtype  uint16
	logger log.Logger
122 123

	lookupFn func(name string, qtype uint16, logger log.Logger) (*dns.Msg, error)
124 125 126
}

// NewDiscovery returns a new Discovery which periodically refreshes its targets.
127
func NewDiscovery(conf SDConfig, logger log.Logger) *Discovery {
128 129 130 131
	if logger == nil {
		logger = log.NewNopLogger()
	}

132 133 134 135 136 137 138 139 140
	qtype := dns.TypeSRV
	switch strings.ToUpper(conf.Type) {
	case "A":
		qtype = dns.TypeA
	case "AAAA":
		qtype = dns.TypeAAAA
	case "SRV":
		qtype = dns.TypeSRV
	}
141
	d := &Discovery{
142 143 144 145 146
		names:    conf.Names,
		qtype:    qtype,
		port:     conf.Port,
		logger:   logger,
		lookupFn: lookupWithSearchPath,
147
	}
148 149 150 151 152 153 154
	d.Discovery = refresh.NewDiscovery(
		logger,
		"dns",
		time.Duration(conf.RefreshInterval),
		d.refresh,
	)
	return d
155 156
}

157 158 159 160
func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
	var (
		wg  sync.WaitGroup
		ch  = make(chan *targetgroup.Group)
161
		tgs = make([]*targetgroup.Group, 0, len(d.names))
162
	)
163

164 165
	wg.Add(len(d.names))
	for _, name := range d.names {
166
		go func(n string) {
167
			if err := d.refreshOne(ctx, n, ch); err != nil && err != context.Canceled {
168
				level.Error(d.logger).Log("msg", "Error refreshing DNS targets", "err", err)
169 170 171 172 173
			}
			wg.Done()
		}(name)
	}

174 175 176 177 178 179 180 181 182
	go func() {
		wg.Wait()
		close(ch)
	}()

	for tg := range ch {
		tgs = append(tgs, tg)
	}
	return tgs, nil
183 184
}

185
func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targetgroup.Group) error {
186
	response, err := d.lookupFn(name, d.qtype, d.logger)
187 188 189 190 191 192
	dnsSDLookupsCount.Inc()
	if err != nil {
		dnsSDLookupFailuresCount.Inc()
		return err
	}

193
	tg := &targetgroup.Group{}
194 195 196
	hostPort := func(a string, p int) model.LabelValue {
		return model.LabelValue(net.JoinHostPort(a, fmt.Sprintf("%d", p)))
	}
197 198

	for _, record := range response.Answer {
199 200
		var target, dnsSrvRecordTarget, dnsSrvRecordPort model.LabelValue

201 202
		switch addr := record.(type) {
		case *dns.SRV:
203 204 205
			dnsSrvRecordTarget = model.LabelValue(addr.Target)
			dnsSrvRecordPort = model.LabelValue(fmt.Sprintf("%d", addr.Port))

206 207 208
			// Remove the final dot from rooted DNS names to make them look more usual.
			addr.Target = strings.TrimRight(addr.Target, ".")

209
			target = hostPort(addr.Target, int(addr.Port))
210
		case *dns.A:
211
			target = hostPort(addr.A.String(), d.port)
212
		case *dns.AAAA:
213
			target = hostPort(addr.AAAA.String(), d.port)
214 215
		case *dns.CNAME:
			// CNAME responses can occur with "Type: A" dns_sd_config requests.
216
			continue
217
		default:
218
			level.Warn(d.logger).Log("msg", "Invalid record", "record", record)
219 220 221
			continue
		}
		tg.Targets = append(tg.Targets, model.LabelSet{
222 223 224 225
			model.AddressLabel:      target,
			dnsNameLabel:            model.LabelValue(name),
			dnsSrvRecordTargetLabel: dnsSrvRecordTarget,
			dnsSrvRecordPortLabel:   dnsSrvRecordPort,
226 227 228 229 230 231 232
		})
	}

	tg.Source = name
	select {
	case <-ctx.Done():
		return ctx.Err()
233
	case ch <- tg:
234 235 236 237 238
	}

	return nil
}

239 240 241 242 243
// lookupWithSearchPath tries to get an answer for various permutations of
// the given name, appending the system-configured search path as necessary.
//
// There are three possible outcomes:
//
Ben Kochie committed
244
// 1. One of the permutations of the given name is recognized as
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
//    "valid" by the DNS, in which case we consider ourselves "done"
//    and that answer is returned.  Note that, due to the way the DNS
//    handles "name has resource records, but none of the specified type",
//    the answer received may have an empty set of results.
//
// 2.  All of the permutations of the given name are responded to by one of
//    the servers in the "nameservers" list with the answer "that name does
//    not exist" (NXDOMAIN).  In that case, it can be considered
//    pseudo-authoritative that there are no records for that name.
//
// 3.  One or more of the names was responded to by all servers with some
//    sort of error indication.  In that case, we can't know if, in fact,
//    there are records for the name or not, so whatever state the
//    configuration is in, we should keep it that way until we know for
//    sure (by, presumably, all the names getting answers in the future).
//
// Outcomes 1 and 2 are indicated by a valid response message (possibly an
// empty one) and no error.  Outcome 3 is indicated by an error return.  The
// error will be generic-looking, because trying to return all the errors
// returned by the combination of all name permutations and servers is a
// nightmare.
func lookupWithSearchPath(name string, qtype uint16, logger log.Logger) (*dns.Msg, error) {
267 268
	conf, err := dns.ClientConfigFromFile(resolvConf)
	if err != nil {
269
		return nil, errors.Wrap(err, "could not load resolv.conf")
270 271
	}

272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
	allResponsesValid := true

	for _, lname := range conf.NameList(name) {
		response, err := lookupFromAnyServer(lname, qtype, conf, logger)

		if err != nil {
			// We can't go home yet, because a later name
			// may give us a valid, successful answer.  However
			// we can no longer say "this name definitely doesn't
			// exist", because we did not get that answer for
			// at least one name.
			allResponsesValid = false
		} else if response.Rcode == dns.RcodeSuccess {
			// Outcome 1: GOLD!
			return response, nil
		}
	}

	if allResponsesValid {
		// Outcome 2: everyone says NXDOMAIN, that's good enough for me
		return &dns.Msg{}, nil
	}
	// Outcome 3: boned.
295
	return nil, errors.Errorf("could not resolve %q: all servers responded with errors to at least one search domain", name)
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
}

// lookupFromAnyServer uses all configured servers to try and resolve a specific
// name.  If a viable answer is received from a server, then it is
// immediately returned, otherwise the other servers in the config are
// tried, and if none of them return a viable answer, an error is returned.
//
// A "viable answer" is one which indicates either:
//
// 1. "yes, I know that name, and here are its records of the requested type"
//    (RCODE==SUCCESS, ANCOUNT > 0);
// 2. "yes, I know that name, but it has no records of the requested type"
//    (RCODE==SUCCESS, ANCOUNT==0); or
// 3. "I know that name doesn't exist" (RCODE==NXDOMAIN).
//
// A non-viable answer is "anything else", which encompasses both various
// system-level problems (like network timeouts) and also
// valid-but-unexpected DNS responses (SERVFAIL, REFUSED, etc).
func lookupFromAnyServer(name string, qtype uint16, conf *dns.ClientConfig, logger log.Logger) (*dns.Msg, error) {
315 316 317 318
	client := &dns.Client{}

	for _, server := range conf.Servers {
		servAddr := net.JoinHostPort(server, conf.Port)
319
		msg, err := askServerForName(name, qtype, client, servAddr, true)
320
		if err != nil {
321
			level.Warn(logger).Log("msg", "DNS resolution failed", "server", server, "name", name, "err", err)
322 323 324 325 326 327
			continue
		}

		if msg.Rcode == dns.RcodeSuccess || msg.Rcode == dns.RcodeNameError {
			// We have our answer.  Time to go home.
			return msg, nil
328 329
		}
	}
330

331
	return nil, errors.Errorf("could not resolve %s: no servers returned a viable answer", name)
332 333
}

334
// askServerForName makes a request to a specific DNS server for a specific
335 336
// name (and qtype).  Retries with TCP in the event of response truncation,
// but otherwise just sends back whatever the server gave, whether that be a
337 338
// valid-looking response, or an error.
func askServerForName(name string, queryType uint16, client *dns.Client, servAddr string, edns bool) (*dns.Msg, error) {
339 340
	msg := &dns.Msg{}

341
	msg.SetQuestion(dns.Fqdn(name), queryType)
342
	if edns {
343
		msg.SetEdns0(dns.DefaultMsgSize, false)
344 345 346
	}

	response, _, err := client.Exchange(msg, servAddr)
347 348 349 350 351
	if err != nil {
		return nil, err
	}

	if response.Truncated {
352
		if client.Net == "tcp" {
353
			return nil, errors.New("got truncated message on TCP (64kiB limit exceeded?)")
354
		}
355 356 357

		client.Net = "tcp"
		return askServerForName(name, queryType, client, servAddr, false)
358
	}
359

360 361
	return response, nil
}