software/powerdns: add multidomain support

Allow each instance to support a zone list instead of a single zone.
Each slave is set for only one defined zone.

software/powerdns/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = 20c37ea06a8fa405bc02470d5115fd11
 
 [template-dns-replicate]
 _update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
-md5sum = c2bd424f588ad57d37f4cf1329734fb6
+md5sum = 72ce30bee3b8a9da8ac9be7eb65d83a2
 
 [iso-list]
 _update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd
@@ -34,4 +34,4 @@ md5sum = c4dc8c141d81b92d92cdb82ca67a13ee
 
 [template-zones-file]
 _update_hash_filename_ = template/zones-file.yml.jinja2
-md5sum = 03037141ad1d3467ae878c9798724f70
+md5sum = 612de569ac3d1e8cc10b830683ff92ae

software/powerdns/instance-powerdns-input-schema.json

@@ -33,12 +33,12 @@
       "default": "",
       "type": "string"
     },
-    "zone": {
+    "supported-zone-list": {
       "title": "Zone",
       "description": "Zone to be handled by the DNS cluster",
       "type": "string",
       "default": "domain.com",
-      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
+      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}(\\s([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6})*$"
     },
     "server-admin": {
       "title": "Zone Administrator Email",
@@ -50,7 +50,7 @@
       "title": "DNS domains template string",
       "description": "Template used to generate DNS domain name",
       "type": "string",
-      "default": "ns%s. + zone"
+      "default": "ns%s.domain.com"
     },
     "monitor-interface-url": {
       "title": "Monitor Web Interface URL",

software/powerdns/instance-powerdns-replicate.cfg.jinja2

@@ -33,9 +33,9 @@ context =
 {% endif -%}
 
 ## DNS set up
-{% set zone = slapparameter_dict.pop('zone', 'domain.com') %}
+{%- set supported_zone_list = slapparameter_dict.pop('supported-zone-list', 'domain.com').split() %}
 {% set server_admin = slapparameter_dict.pop('server-admin', 'admin@domain.com') %}
-{% set dns_name_template_string = slapparameter_dict.pop('dns-name-template-string', 'ns%s.' + zone) %}
+{% set dns_name_template_string = slapparameter_dict.pop('dns-name-template-string', 'ns%s.domain.com') %}
 
 # Here we request individualy each dns.
 # The presence of sla parameters is checked and added if found
@@ -65,7 +65,7 @@ name = {{dns_name}}
 {%   if state_key in slapparameter_dict %}
 state = {{ slapparameter_dict.pop(state_key) }}
 {%   endif%}
-config-zone = {{ zone }}
+config-supported-zone-list = {{ ' '.join(supported_zone_list) }}
 config-soa = {{ "%s,%s" % (dns_domain, server_admin) }}
 {%   for parameter in sla_parameters -%}
 sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
@@ -74,11 +74,9 @@ sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
 [{{promise_section_title}}]
 <= monitor-promise-base
 module = check_port_listening
-name = pdns-port-listening.py
-{%   set ipv6 = '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' -%}
-config-hostname = {{ipv6}}
-{%   set port = '${' ~ request_section_title ~ ':connection-powerdns-port}' -%}
-config-port = {{port}}
+name = {{promise_section_title}}.py
+config-hostname = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
+config-port = {{ '${' ~ request_section_title ~ ':connection-powerdns-port}' }}
 
 {%  do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%}
 {% endfor -%}

software/powerdns/slave-instance-powerdns-input-schema.json

@@ -8,6 +8,13 @@
       "description": "Record for the configuration",
       "type": "string"
     },
+    "applicable-zone": {
+      "title": "Applicable Zone",
+      "description": "Zone to which this record belongs. You can put only one zone here. If the record belongs to several zones, you should create several slaves.",
+      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
+      "default": "domain.com",
+      "type": "string"
+    },
     "origin": {
       "title": "Origin",
       "description": "Used to qualify RR in the configuration. i.e.: if your origin is a.example.com and the RR for Europe is 'eu' the european clients will use eu.a.example.com",

software/powerdns/template/zones-file.yml.jinja2

 # See https://doc.powerdns.com/authoritative/backends/geoip.html
 
 {%- set slave_instance_list = json_module.loads(slapparameter_dict.get('extra_slave_instance_list', '[]')) %}
-{%- set zone = slapparameter_dict.get('zone', 'example.com') %}
+{%- set supported_zone_list = slapparameter_dict.get('supported-zone-list', 'example.com').split() %}
 
 {%- macro disambiguate_domain_name(a, b) %}
 {#- See http://www.dns-sd.org/trailingdotsindomainnames.html #}
@@ -13,6 +13,8 @@
 {%- endmacro %}
 
 domains:
+{%- for zone in supported_zone_list %}
+
 - domain: {{ zone }}
   # TODO: what value for ttl?
   ttl: 300
@@ -48,6 +50,7 @@ domains:
 
     {%- for slave in slave_instance_list %}
 
+    {%- if slave['applicable-zone'] == zone %}
     {%-   set origin = slave['origin'] %}
     {%-   set unique_slave_id = slave['slave_reference'] %}
     {#-   Set the RR to use for each region, as described in
@@ -77,13 +80,15 @@ domains:
       - cname: {{ disambiguate_domain_name(rr_dict[region], origin) }}
     {%-     endfor %}
     {%-   endfor %}
+    {%- endif %}
     {%- endfor %}
 
   services:
     {%- for slave in slave_instance_list %}
+    {%- if slave['applicable-zone'] == zone %}
     {%-   set origin = slave['origin'] %}
     {%-   set unique_slave_id = slave['slave_reference'] %}
-    {{ disambiguate_domain_name(slave['record'], zone) }}:
+    {{ disambiguate_domain_name(slave['record'], slave['applicable-zone']) }}:
       {#- Note: Placeholders (i.e. "country." and "continent.") are used to avoid
           possible name collisions, e.g.:
           - %cc for American Samoa is 'as'
@@ -94,4 +99,6 @@ domains:
       {%- for ip_range, country_code in china %}
       {{ ip_range }}: {{ country_code }}.country.{{ unique_slave_id }}
       {%- endfor %}
+    {%- endif %}
     {%- endfor %}
+{%- endfor %}

software/powerdns/test/test.py

@@ -31,11 +31,14 @@ import dns.query
 import http.client
 import os
 import requests
+import unittest
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
 
+skip = unittest.skip('port conflit between powerdns instances')
+
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
@@ -59,42 +62,40 @@ WEST_ASIAN_SUBNET = '46.70.0.0'
 class PowerDNSTestCase(SlapOSInstanceTestCase):
   # power dns uses sockets and need shorter paths on test nodes.
   __partition_reference__ = 'pdns'
-  default_zone = 'domain.com'
+  default_supported_zone = 'domain.com'
 
   # focus to test connexion parameters only depending on PowerDNS
-  def getPowerDNSParameterDict(self, parameter_dict):
-    new_parameter_dict = {}
-    for key, value in parameter_dict.items():
-      if key in [
-        'ns-record',
-        'ns1-port',
-        'ns1-ipv6',
-        'slave-amount',
-      ]:
-        new_parameter_dict[key] = value
-    return new_parameter_dict
-
-  def getPowerDNSConnexionParameterDict(self):
+  def getPowerDNSParameterDict(self, parameter_dict, dns_quantity):
+    selected_key_list = ['ns-record', ]
+    for replicate_nb in range(1, dns_quantity + 1):
+      selected_key_list.append('ns%s-port' % replicate_nb)
+      selected_key_list.append('ns%s-ipv6' % replicate_nb)
+    selected_key_list.append('slave-amount')
+
+    return {
+      k: v for k, v in parameter_dict.items() if k in selected_key_list
+    }
+
+  def getPowerDNSConnexionParameterDict(self, dns_quantity=1):
     return self.getPowerDNSParameterDict(
-      self.requestDefaultInstance().getConnectionParameterDict()
+      self.requestDefaultInstance().getConnectionParameterDict(),
+      dns_quantity
     )
 
-  def _test_parameter_dict(self, zone=None, dns_quantity=1, slave_amount=0):
-    if zone is None:
-      zone = self.default_zone
+  def _test_parameter_dict(self, dns_quantity=1, slave_amount=0):
 
     parameter_dict = self.getPowerDNSConnexionParameterDict()
     expected_dict = {
       'ns-record': '',
     }
 
-    ns_record = ''
+    ns_record = []
     for replicate_nb in range(1, dns_quantity + 1):
-      prefix = 'ns%s' % replicate_nb
-      ns_record += prefix + '.%s' % zone
-      expected_dict[prefix + '-port'] = str(DNS_PORT)
-      expected_dict[prefix + '-ipv6'] = self._ipv6_address
-    expected_dict['ns-record'] = ns_record
+      ns_id = 'ns%s' % replicate_nb
+      ns_record.append(ns_id + '.' + self.default_supported_zone)
+      expected_dict[ns_id + '-port'] = str(DNS_PORT)
+      expected_dict[ns_id + '-ipv6'] = self._ipv6_address
+    expected_dict['ns-record'] = ','.join(ns_record)
     expected_dict['slave-amount'] = str(slave_amount)
 
     self.assertEqual(expected_dict, parameter_dict)
@@ -149,24 +150,11 @@ class TestMasterRequest(PowerDNSTestCase):
     self._test_parameter_dict()
 
 
-class TestMasterRequestDomain(PowerDNSTestCase):
-
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {
-      'zone': 'toto.example.com',
-    }
-
-  def test(self):
-    self._test_parameter_dict(zone=self.getInstanceParameterDict()['zone'])
-
-
 class PowerDNSSlaveTestCase(PowerDNSTestCase):
 
   @classmethod
-  def requestDefaultInstance(cls):
-    default_instance = super(PowerDNSSlaveTestCase, cls)\
-        .requestDefaultInstance()
+  def requestDefaultInstance(cls, state='started'):
+    default_instance = super().requestDefaultInstance(state=state)
     cls.requestSlaves()
     return default_instance
 
@@ -202,49 +190,19 @@ class PowerDNSSlaveTestCase(PowerDNSTestCase):
       ).getConnectionParameterDict())
     return parameter_dict_list
 
-  @classmethod
-  def getSlaveParameterDictDict(cls):
-    return {
-      'slave-pdns1': {
-        'record': 'test1',
-        'origin': 'nexedi.com',
-        'default': 'test1.com.',
-        'africa': 'test1africa.com.',
-        'china-telecom': 'test1china-telecom.com.',
-        'china-unicom': 'test1china-unicom.com.',
-        'china-mobile': 'test1china-mobile.com.',
-        'east-asia': 'test1east-asia.com.',
-        'europe': 'test1europe.com.',
-        'hong-kong': 'test1hong-kong.com.',
-        'japan': 'test1japan.com.',
-        'north-america': 'test1north-america.com.',
-        'oceania': 'test1oceania.com.',
-        'south-america': 'test1south-america.com.',
-        'west-asia': 'test1west-asia.com.',
-      },
-      'slave-pdns2': {
-        'record': 'test2',
-        'origin': 'nexedi.com',
-        'default': 'test2.com.',
-        'china-telecom': 'test2china-telecom.com.',
-        'europe': 'test2europe.com.',
-        'japan': 'test2japan.com.',
-      }
-    }
-
   def dns_query(self, domain_name, subnet):
     message = dns.message.make_query(domain_name, 'A')
     client_subnet_option = dns.edns.ECSOption(subnet)
     message.use_edns(options=[client_subnet_option])
     answer = dns.query.udp(message, self._ipv6_address, port=DNS_PORT)
-    return answer.find_rrset(
+    return answer.get_rrset(
             dns.message.ANSWER,
             dns.name.from_text(domain_name),
             dns.rdataclass.IN,
             dns.rdatatype.CNAME
           ).to_text().split()[-1]
 
-  def _test_dns_resolver(self, zone):
+  def _test_dns_resolver(self):
     slave_parameter_dict_dict = self.getSlaveParameterDictDict()
     subnet_dict = {
       'africa': AFRICAN_SUBNET,
@@ -277,39 +235,195 @@ class PowerDNSSlaveTestCase(PowerDNSTestCase):
 
     for slave_name in slave_parameter_dict_dict:
       slave_parameter_dict = slave_parameter_dict_dict[slave_name]
-      domain_name = '%s.%s' % (slave_parameter_dict['record'], zone)
+      domain_name = '%s.%s' % (
+        slave_parameter_dict['record'], slave_parameter_dict['applicable-zone']
+      )
       for region in subnet_dict:
         self.assertEqual(
-          slave_parameter_dict.pop(
+          slave_parameter_dict.get(
             region,
             '%s.%s.' % (default_rr_dict[region], slave_parameter_dict['origin'])
           ),
           self.dns_query(domain_name, subnet_dict[region])
         )
 
-  def _test(self, zone=None):
-    if zone is None:
-      zone = self.default_zone
+  def _test_slaves(self, dns_quantity=1):
     self._test_parameter_dict(
-      zone=zone,
+      dns_quantity=dns_quantity,
       slave_amount=len(self.getSlaveParameterDictDict())
     )
-    self._test_dns_resolver(zone)
+    self._test_dns_resolver()
 
 
 class TestSlaveRequest(PowerDNSSlaveTestCase):
 
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    return {
+      'slave-test1.domain.com': {
+        'record': 'test1',
+        'applicable-zone': 'domain.com',
+        'origin': 'nexedi.com',
+        'default': 'test1.com.',
+        'africa': 'test1africa.com.',
+        'china-telecom': 'test1china-telecom.com.',
+        'china-unicom': 'test1china-unicom.com.',
+        'china-mobile': 'test1china-mobile.com.',
+        'east-asia': 'test1east-asia.com.',
+        'europe': 'test1europe.com.',
+        'hong-kong': 'test1hong-kong.com.',
+        'japan': 'test1japan.com.',
+        'north-america': 'test1north-america.com.',
+        'oceania': 'test1oceania.com.',
+        'south-america': 'test1south-america.com.',
+        'west-asia': 'test1west-asia.com.',
+      },
+      'slave-test2.domain.com': {
+        'record': 'test2',
+        'applicable-zone': 'domain.com',
+        'origin': 'nexedi.com',
+        'default': 'test2.com.',
+        'china-telecom': 'test2china-telecom.com.',
+        'europe': 'test2europe.com.',
+        'japan': 'test2japan.com.',
+      },
+    }
+
   def test(self):
-    self._test()
+    self._test_slaves()
 
 
-class TestSlaveRequestDomain(PowerDNSSlaveTestCase):
+class TestSlaveRequestSingleDomain(TestSlaveRequest):
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    return {
+      'slave-test1.toto.example.com': {
+        'record': 'test1',
+        'applicable-zone': 'toto.example.com',
+        'origin': 'nexedi.com',
+        'default': 'test1.com.',
+        'africa': 'test1africa.com.',
+        'china-telecom': 'test1china-telecom.com.',
+        'china-unicom': 'test1china-unicom.com.',
+        'china-mobile': 'test1china-mobile.com.',
+        'east-asia': 'test1east-asia.com.',
+        'europe': 'test1europe.com.',
+        'hong-kong': 'test1hong-kong.com.',
+        'japan': 'test1japan.com.',
+        'north-america': 'test1north-america.com.',
+        'oceania': 'test1oceania.com.',
+        'south-america': 'test1south-america.com.',
+        'west-asia': 'test1west-asia.com.',
+      },
+      'slave-test2.toto.example.com': {
+        'record': 'test2',
+        'applicable-zone': 'toto.example.com',
+        'origin': 'nexedi.com',
+        'default': 'test2.com.',
+        'china-telecom': 'test2china-telecom.com.',
+        'europe': 'test2europe.com.',
+        'japan': 'test2japan.com.',
+      },
+    }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'supported-zone-list': 'toto.example.com',
+    }
+
+
+class TestSlaveRequestDomains(TestSlaveRequest):
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    return {
+      'slave-test1.toto.example.com': {
+        'record': 'test1',
+        'applicable-zone': 'toto.example.com',
+        'origin': 'nexedi.com',
+        'default': 'test1.com.',
+        'africa': 'test1africa.com.',
+        'china-telecom': 'test1china-telecom.com.',
+        'china-unicom': 'test1china-unicom.com.',
+        'china-mobile': 'test1china-mobile.com.',
+        'east-asia': 'test1east-asia.com.',
+        'europe': 'test1europe.com.',
+        'hong-kong': 'test1hong-kong.com.',
+        'japan': 'test1japan.com.',
+        'north-america': 'test1north-america.com.',
+        'oceania': 'test1oceania.com.',
+        'south-america': 'test1south-america.com.',
+        'west-asia': 'test1west-asia.com.',
+      },
+      'slave-test2.toto.example.com': {
+        'record': 'test2',
+        'applicable-zone': 'toto.example.com',
+        'origin': 'nexedi.com',
+        'default': 'test2.com.',
+        'china-telecom': 'test2china-telecom.com.',
+        'europe': 'test2europe.com.',
+        'japan': 'test2japan.com.',
+      },
+      'slave-test1.tata.example.com': {
+        'record': 'test1',
+        'applicable-zone': 'tata.example.com',
+        'origin': 'nexedi.com',
+        'default': 'test1.com.',
+        'china-telecom': 'test1china-telecom.com.',
+        'europe': 'test1europe.com.',
+        'japan': 'test1japan.com.',
+      },
+      'slave-test4.tata.example.com': {
+        'record': 'test4',
+        'applicable-zone': 'tata.example.com',
+        'origin': 'nexedi.com',
+        'default': 'test4".com.',
+        'africa': 'test4africa.com.',
+        'china-telecom': 'test4china-telecom.com.',
+        'china-unicom': 'test4china-unicom.com.',
+        'china-mobile': 'test4china-mobile.com.',
+        'east-asia': 'test4east-asia.com.',
+        'europe': 'test4europe.com.',
+        'hong-kong': 'test4hong-kong.com.',
+        'japan': 'test4japan.com.',
+        'north-america': 'test4north-america.com.',
+        'oceania': 'test4oceania.com.',
+        'south-america': 'test4south-america.com.',
+        'west-asia': 'test4west-asia.com.',
+      },
+      'slave-test5.tata.example.com': {
+        'record': 'test5',
+        'applicable-zone': 'tata.example.com',
+        'origin': 'nexedi.com',
+        'default': 'test5.com.',
+        'china-telecom': 'test5china-telecom.com.',
+        'europe': 'test5europe.com.',
+        'japan': 'test5japan.com.',
+      },
+    }
 
   @classmethod
   def getInstanceParameterDict(cls):
     return {
-      'zone': 'toto.example.com',
+      'supported-zone-list': 'toto.example.com tata.example.com',
+    }
+
+
+# Because all powerdns instances run under the same ip address during tests,
+# there is a port conflict between these instances
+@skip
+class TestMultipleInstances(TestSlaveRequestDomains):
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      '-dns-quantity': '2',
+      'supported-zone-list': 'toto.example.com tata.example.com',
     }
 
   def test(self):
-    self._test(zone=self.getInstanceParameterDict()['zone'])
+    self._test_slaves(
+      dns_quantity=int(self.getInstanceParameterDict()['-dns-quantity'])
+    )