deal with not responding nameservers (offline or unreachable)
if a nameserver does not respond, we flag it in the database (available = False) and stop talking to it for 5 minutes. after 5 minutes, we flag it available = True again and retry talking to it (and if not successful, flag it as unavailable again). this avoids a unresponsive UI as we do a lot of ns queries from the templates to get v4 and v6 addresses of our hosts from master nameservers. we log a warning when setting unavailable and a info when setting available.
This commit is contained in:
Thomas Waldmann
parent
a53ef40e46
commit
6bf1f57fc1
@ -4,6 +4,18 @@ Misc. DNS related code: query, dynamic update, etc.
|
||||
Usually, higher level code wants to call the add/update/delete functions.
|
||||
"""
|
||||
|
||||
# time to wait for dns name resolving [s]
|
||||
RESOLVER_TIMEOUT = 5.0
|
||||
|
||||
# time to wait for dns name updating [s]
|
||||
UPDATE_TIMEOUT = 20.0
|
||||
|
||||
# time after we retry to reach a previously unreachable ns [s]
|
||||
UNAVAILABLE_RETRY = 300.0
|
||||
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -16,6 +28,7 @@ import dns.tsig
|
||||
import dns.tsigkeyring
|
||||
|
||||
from django.conf import settings
|
||||
from django.utils.timezone import now
|
||||
|
||||
|
||||
class SameIpError(ValueError):
|
||||
@ -25,6 +38,13 @@ class SameIpError(ValueError):
|
||||
"""
|
||||
|
||||
|
||||
class NameServerNotAvailable(Exception):
|
||||
"""
|
||||
raised if some nameserver was flagged as not available,
|
||||
but we tried using it.
|
||||
"""
|
||||
|
||||
|
||||
def check_ip(ipaddr, keys=('ipv4', 'ipv6')):
|
||||
"""
|
||||
Check if a string is a valid ip address and also
|
||||
@ -128,7 +148,7 @@ def query_ns(qname, rdtype, origin=None):
|
||||
:type rdtype: int or str
|
||||
:param origin: origin zone
|
||||
:type origin: str or None
|
||||
:return: IP (as str)
|
||||
:return: IP (as str) or "-" if ns is not available
|
||||
"""
|
||||
origin, name = parse_name(qname, origin)
|
||||
origin_str = str(origin)
|
||||
@ -138,8 +158,16 @@ def query_ns(qname, rdtype, origin=None):
|
||||
# want into the documented attributes:
|
||||
resolver.nameservers = [nameserver, ]
|
||||
resolver.search = [dns.name.from_text(settings.BASEDOMAIN), ]
|
||||
answer = resolver.query(qname, rdtype)
|
||||
return str(list(answer)[0])
|
||||
resolver.lifetime = RESOLVER_TIMEOUT
|
||||
try:
|
||||
answer = resolver.query(qname, rdtype)
|
||||
ip = str(list(answer)[0])
|
||||
return ip
|
||||
except (dns.resolver.Timeout, dns.resolver.NoNameservers): # socket.error also?
|
||||
logger.warning("timeout when querying for name '%s' in zone '%s' with rdtype '%s'." % (
|
||||
name, origin, rdtype))
|
||||
set_ns_availability(origin, False)
|
||||
raise
|
||||
|
||||
|
||||
def parse_name(fqdn, origin=None):
|
||||
@ -169,9 +197,20 @@ def get_ns_info(origin):
|
||||
|
||||
:param origin: zone we are dealing with, must be with trailing dot
|
||||
:return: master nameserver, update key, update algo
|
||||
:raises: NameServerNotAvailable if ns was flagged unavailable in the db
|
||||
"""
|
||||
from .models import Domain
|
||||
d = Domain.objects.get(domain=origin.rstrip('.'))
|
||||
domain = origin.rstrip('.')
|
||||
d = Domain.objects.get(domain=domain)
|
||||
if not d.available:
|
||||
if d.last_update + timedelta(seconds=UNAVAILABLE_RETRY) > now():
|
||||
# if there are troubles with a nameserver, we set available=False
|
||||
# and stop trying working with that nameserver for a while
|
||||
raise NameServerNotAvailable("nameserver for domain %s at IP %s was flagged unavailable" % (
|
||||
domain, d.nameserver_ip, ))
|
||||
else:
|
||||
# retry timeout is over, set it available again
|
||||
set_ns_availability(origin, True)
|
||||
algorithm = getattr(dns.tsig, d.nameserver_update_algorithm)
|
||||
return d.nameserver_ip, d.nameserver_update_key, algorithm
|
||||
|
||||
@ -205,5 +244,31 @@ def update_ns(fqdn, rdtype='A', ipaddr=None, origin=None, action='upd', ttl=60):
|
||||
upd.replace(name, ttl, rdtype, ipaddr)
|
||||
logger.debug("performing %s for name %s and origin %s with rdtype %s and ipaddr %s" % (
|
||||
action, name, origin, rdtype, ipaddr))
|
||||
response = dns.query.tcp(upd, nameserver)
|
||||
return response
|
||||
try:
|
||||
response = dns.query.tcp(upd, nameserver, timeout=UPDATE_TIMEOUT)
|
||||
return response
|
||||
except dns.exception.Timeout:
|
||||
logger.warning("timeout when performing %s for name %s and origin %s with rdtype %s and ipaddr %s" % (
|
||||
action, name, origin, rdtype, ipaddr))
|
||||
set_ns_availability(origin, False)
|
||||
raise
|
||||
|
||||
|
||||
def set_ns_availability(domain, available):
|
||||
"""
|
||||
Set availability of the master nameserver for <domain>.
|
||||
|
||||
As each Timeout takes quite a while, we want to avoid it.
|
||||
|
||||
:param domain: domain object or string, may end with "."
|
||||
:param available: True/False for availability of ns
|
||||
"""
|
||||
from .models import Domain
|
||||
domain = str(domain).rstrip('.')
|
||||
d = Domain.objects.get(domain=domain)
|
||||
d.available = available
|
||||
d.save()
|
||||
if available:
|
||||
logger.info("set zone '%s' to available" % domain)
|
||||
else:
|
||||
logger.warning("set zone '%s' to unavailable" % domain)
|
||||
|
||||
@ -0,0 +1,94 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import datetime
|
||||
from south.db import db
|
||||
from south.v2 import SchemaMigration
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(SchemaMigration):
|
||||
|
||||
def forwards(self, orm):
|
||||
# Adding field 'Domain.available'
|
||||
db.add_column(u'main_domain', 'available',
|
||||
self.gf('django.db.models.fields.BooleanField')(default=True),
|
||||
keep_default=False)
|
||||
|
||||
|
||||
def backwards(self, orm):
|
||||
# Deleting field 'Domain.available'
|
||||
db.delete_column(u'main_domain', 'available')
|
||||
|
||||
|
||||
models = {
|
||||
u'auth.group': {
|
||||
'Meta': {'object_name': 'Group'},
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
|
||||
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
|
||||
},
|
||||
u'auth.permission': {
|
||||
'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
|
||||
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
|
||||
},
|
||||
u'auth.user': {
|
||||
'Meta': {'object_name': 'User'},
|
||||
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
|
||||
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
|
||||
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
|
||||
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
|
||||
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
|
||||
},
|
||||
u'contenttypes.contenttype': {
|
||||
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
|
||||
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
|
||||
},
|
||||
u'main.blacklisteddomain': {
|
||||
'Meta': {'object_name': 'BlacklistedDomain'},
|
||||
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
|
||||
'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}),
|
||||
'domain': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'last_update': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
|
||||
},
|
||||
u'main.domain': {
|
||||
'Meta': {'object_name': 'Domain'},
|
||||
'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
'available_for_everyone': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
|
||||
'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}),
|
||||
'domain': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'last_update': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
|
||||
'nameserver_ip': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39'}),
|
||||
'nameserver_update_algorithm': ('django.db.models.fields.CharField', [], {'default': "'HMAC_SHA512'", 'max_length': '256'}),
|
||||
'nameserver_update_key': ('django.db.models.fields.CharField', [], {'max_length': '256'})
|
||||
},
|
||||
u'main.host': {
|
||||
'Meta': {'unique_together': "(('subdomain', 'domain'),)", 'object_name': 'Host'},
|
||||
'comment': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '256', 'null': 'True', 'blank': 'True'}),
|
||||
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
|
||||
'created_by': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'hosts'", 'to': u"orm['auth.User']"}),
|
||||
'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['main.Domain']"}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'last_api_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'last_update': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
|
||||
'subdomain': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
|
||||
'update_secret': ('django.db.models.fields.CharField', [], {'max_length': '256'})
|
||||
}
|
||||
}
|
||||
|
||||
complete_apps = ['main']
|
||||
@ -52,7 +52,11 @@ class Domain(models.Model):
|
||||
nameserver_update_key = models.CharField(max_length=256)
|
||||
nameserver_update_algorithm = models.CharField(
|
||||
max_length=256, default='HMAC_SHA512', choices=UPDATE_ALGORITHMS)
|
||||
# XXX rename available_for_everyone to public
|
||||
available_for_everyone = models.BooleanField(default=False)
|
||||
# available means "nameserver for domain operating and reachable" -
|
||||
# gets set to False if we have trouble reaching the nameserver
|
||||
available = models.BooleanField(default=True)
|
||||
|
||||
last_update = models.DateTimeField(auto_now=True)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
@ -102,14 +106,16 @@ class Host(models.Model):
|
||||
def getIPv4(self):
|
||||
try:
|
||||
return dnstools.query_ns(self.get_fqdn(), 'A', origin=self.domain.domain)
|
||||
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.resolver.Timeout):
|
||||
return ''
|
||||
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.resolver.Timeout,
|
||||
dnstools.NameServerNotAvailable):
|
||||
return 'error'
|
||||
|
||||
def getIPv6(self):
|
||||
try:
|
||||
return dnstools.query_ns(self.get_fqdn(), 'AAAA', origin=self.domain.domain)
|
||||
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.resolver.Timeout):
|
||||
return ''
|
||||
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.resolver.Timeout,
|
||||
dnstools.NameServerNotAvailable):
|
||||
return 'error'
|
||||
|
||||
def poke(self):
|
||||
self.last_api_update = datetime.now()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user