serve /robots.txt

nsupdate/main/urls.py

@@ -1,7 +1,8 @@
 from django.conf.urls import patterns, url
 from django.views.generic import TemplateView
 from main.views import (
-    HomeView, OverviewView, HostView, DeleteHostView, AboutView, HelpView, GenerateSecretView)
+    HomeView, OverviewView, HostView, DeleteHostView, AboutView, HelpView, GenerateSecretView,
+    RobotsTxtView, )
 from api.views import (
     MyIpView, DetectIpView, NicUpdateView, AuthorizedNicUpdateView)
 
@@ -23,4 +24,5 @@ urlpatterns = patterns(
     url(r'^nic/update$', NicUpdateView),
     url(r'^nic/update_authorized$',
         AuthorizedNicUpdateView, name='nic_update_authorized'),
+    url(r'^robots.txt$', RobotsTxtView),
 )

nsupdate/main/views.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from django.views.generic import TemplateView, CreateView
 from django.views.generic.edit import UpdateView, DeleteView
-from django.http import HttpResponseRedirect
+from django.http import HttpResponse, HttpResponseRedirect
 from django.contrib.auth.decorators import login_required
 from django.contrib import messages
 from django.utils.decorators import method_decorator
@@ -152,3 +152,21 @@ class DeleteHostView(DeleteView):
         context['nav_overview'] = True
         context['hosts'] = Host.objects.filter(created_by=self.request.user)
         return context
+
+
+def RobotsTxtView(request):
+    """
+    Dynamically serve robots.txt content.
+    If you like, you can optimize this by statically serving this by your web server.
+
+    :param request: django request object
+    :return: HttpResponse object
+    """
+    content = """\
+User-agent: *
+Crawl-delay: 10
+Disallow: /accounts/
+Disallow: /myip/
+Disallow: /nic/update/
+"""
+    return HttpResponse(content, content_type="text/plain")