software/powerdns: add tests

Tests added:
- connexion parameters
- monitor url
- slaves instanciation

Adds connexion parameters:
- ipv6
- port
- ipv4

Upgrade dnspython from 1.15.0 to 1.16.0

Removes unused parameters in instance-powerdns-replicate.cfg.jinja:
- private-ipv4
- public-ipv4
- domain

software/powerdns/buildout.hash.cfg

@@ -18,15 +18,15 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62
 
 [template-powerdns]
 filename = instance-powerdns.cfg
-md5sum = 9cd4e436fa432f37b9f8f4de8350581b
+md5sum = 0920200cb05a68b1b4a161a927d9488f
 
 [template-pdns-configuration]
 _update_hash_filename_ = template/pdns.conf.jinja2
-md5sum = e45d72de87b4adb89c195ba463be4077
+md5sum = 20c37ea06a8fa405bc02470d5115fd11
 
 [template-dns-replicate]
 _update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
-md5sum = a23e241a236f90ae1afbb5bd5ba0b32d
+md5sum = c2bd424f588ad57d37f4cf1329734fb6
 
 [iso-list]
 _update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd

software/powerdns/instance-powerdns-replicate.cfg.jinja2

@@ -43,6 +43,7 @@ context =
 {%   set dns_name = 'ns%s' % i -%}
 {%   set dns_domain = dns_name_template_string % i %}
 {%   set request_section_title = 'request-%s' % dns_name -%}
+{%   set promise_section_title = 'promise-listen-port-%s' % dns_name -%}
 {%   set sla_key = "-sla-%s-" % i -%}
 {%   set sla_key_length = sla_key | length %}
 {%   set sla_parameters = [] %}
@@ -55,6 +56,7 @@ context =
 {%   do dns_domain_list.append(dns_domain) -%}
 {%   do dns_section_list.append(request_section_title) -%}
 {%   do part_list.append(request_section_title) -%}
+{%   do part_list.append(promise_section_title) -%}
 
 [{{request_section_title}}]
 <= replicate
@@ -69,6 +71,15 @@ config-soa = {{ "%s,%s" % (dns_domain, server_admin) }}
 sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
 {%   endfor -%}
 
+[{{promise_section_title}}]
+<= monitor-promise-base
+module = check_port_listening
+name = pdns-port-listening.py
+{%   set ipv6 = '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' -%}
+config-hostname = {{ipv6}}
+{%   set port = '${' ~ request_section_title ~ ':connection-powerdns-port}' -%}
+config-port = {{port}}
+
 {%  do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%}
 {% endfor -%}
 
@@ -101,7 +112,7 @@ software-url = {{ slapparameter_dict.pop(dns_software_url_key) }}
 software-url = ${slap-connection:software-release-url}
 {% endif %}
 software-type = {{dns_type}}
-return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url
+return = slave-instance-information-list monitor-base-url powerdns-ipv6 powerdns-port powerdns-ipv4
 config-server-admin = {{ server_admin }}
 config-ns-record = {{ ns_record }}
 {% for parameter, value in slapparameter_dict.items() -%}
@@ -114,12 +125,17 @@ config-monitor-password = ${monitor-htpasswd:passwd}
 
 [publish-information]
 recipe = slapos.cookbook:publish
-domain = {{ slapparameter_dict.get('domain') }}
 slave-amount = {{ slave_instance_list | length }}
 ns-record = {{ ns_record }}
 monitor-url = ${monitor-publish-parameters:monitor-url}
 monitor-user = ${monitor-publish-parameters:monitor-user}
 monitor-password = ${monitor-publish-parameters:monitor-password}
+{% for dns_name, dns_section in zip(dns_list, dns_section_list) -%}
+{%   set dns_ipv6 = '${' ~ dns_section ~ ':connection-powerdns-ipv6}' -%}
+{%   set dns_port = '${' ~ dns_section ~ ':connection-powerdns-port}' -%}
+{{ dns_name }}-ipv6 = {{ dns_ipv6 }}
+{{ dns_name }}-port = {{ dns_port }}
+{% endfor -%}
 {% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
 {% if monitor_interface_url -%}
 monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}

software/powerdns/instance-powerdns.cfg

@@ -55,7 +55,7 @@ context =
 #
 [pdns]
 configuration = $${pdns-directory:configuration}/pdns.conf
-local-ipv4 = $${instance-parameter:ipv4-random}
+ipv4 = $${instance-parameter:ipv4-random}
 ipv6 = $${instance-parameter:ipv6-random}
 port = 5353
 socket-directory = $${pdns-directory:socket-directory}
@@ -139,7 +139,7 @@ extra-context =
 <= monitor-promise-base
 module = check_port_listening
 name = pdns-port-listening.py
-config-hostname = $${pdns:local-ipv4}
+config-hostname = $${pdns:ipv4}
 config-port = $${pdns:port}
 
 [publish-connection-informations]
@@ -147,6 +147,9 @@ recipe = slapos.cookbook:publish
 monitor-url = $${monitor-publish-parameters:monitor-url}
 monitor-user = $${monitor-publish-parameters:monitor-user}
 monitor-password = $${monitor-publish-parameters:monitor-password}
+powerdns-ipv4 = $${pdns:ipv4}
+powerdns-ipv6 = $${pdns:ipv6}
+powerdns-port = $${pdns:port}
 
 [buildout]
 parts =

software/powerdns/software.cfg

@@ -55,7 +55,6 @@ PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
 slapos.recipe.template = 4.4
-dnspython = 1.15.0
 passlib = 1.7.1
 GitPython = 2.1.11
 lockfile = 0.12.2

software/powerdns/template/pdns.conf.jinja2

 # -------------------------------------------------------------------------
 
 # Configure ip/port binding
-local-address={{ pdns.get('local-ipv4') }}
+local-address={{ pdns.get('ipv4') }}
 local-ipv6={{ pdns.get('ipv6') }}
 local-port={{ pdns.get('port') }}
 
@@ -10,7 +10,7 @@ socket-dir={{ pdns.get('socket-directory') }}
 
 # Monitoring
 webserver=yes
-webserver-address={{ pdns.get('local-ipv4') }}
+webserver-address={{ pdns.get('ipv4') }}
 webserver-port={{ pdns.get('webserver-port') }}
 
 # These totally disable query+packet caching for all zones.  This is necessary

software/powerdns/test/setup.py

@@ -47,6 +47,7 @@ setup(name=name,
           'erp5.util',
           'supervisor',
           'psutil',
+          'dnspython',
       ],
       zip_safe=True,
       test_suite='test',

software/powerdns/test/test.py

@@ -25,7 +25,12 @@
 #
 ##############################################################################
 
+import dns.edns
+import dns.message
+import dns.query
+import http.client
 import os
+import requests
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -35,10 +40,64 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
+DNS_PORT = 5353
+
+AFRICAN_SUBNET = '41.0.0.0'
+CHINA_TELECOM_SUBNET = '1.0.32.0'
+CHINA_UNICOM_SUBNET = '116.181.0.0'
+CHINA_MOBILE_SUBNET = '112.21.42.5'
+EAST_ASIAN_SUBNET = '1.11.0.0'
+EUROPEAN_SUBNET = '5.42.160.0'
+HONG_KONG_SUBNET = '1.32.192.0'
+JAPANESE_SUBNET = '1.0.16.0'
+NORTH_AMERICAN_SUBNET = '3.0.0.0'
+OCEANIAN_SUBNET = '1.120.0.0'
+SOUTH_AMERICAN_SUBNET = '45.70.188.0'
+WEST_ASIAN_SUBNET = '46.70.0.0'
+
 
 class PowerDNSTestCase(SlapOSInstanceTestCase):
   # power dns uses sockets and need shorter paths on test nodes.
   __partition_reference__ = 'pdns'
+  default_zone = 'domain.com'
+
+  # focus to test connexion parameters only depending on PowerDNS
+  def getPowerDNSParameterDict(self, parameter_dict):
+    new_parameter_dict = {}
+    for key, value in parameter_dict.items():
+      if key in [
+        'ns-record',
+        'ns1-port',
+        'ns1-ipv6',
+        'slave-amount',
+      ]:
+        new_parameter_dict[key] = value
+    return new_parameter_dict
+
+  def getPowerDNSConnexionParameterDict(self):
+    return self.getPowerDNSParameterDict(
+      self.requestDefaultInstance().getConnectionParameterDict()
+    )
+
+  def _test_parameter_dict(self, zone=None, dns_quantity=1, slave_amount=0):
+    if zone is None:
+      zone = self.default_zone
+
+    parameter_dict = self.getPowerDNSConnexionParameterDict()
+    expected_dict = {
+      'ns-record': '',
+    }
+
+    ns_record = ''
+    for replicate_nb in range(1, dns_quantity + 1):
+      prefix = 'ns%s' % replicate_nb
+      ns_record += prefix + '.%s' % zone
+      expected_dict[prefix + '-port'] = str(DNS_PORT)
+      expected_dict[prefix + '-ipv6'] = self._ipv6_address
+    expected_dict['ns-record'] = ns_record
+    expected_dict['slave-amount'] = str(slave_amount)
+
+    self.assertEqual(expected_dict, parameter_dict)
 
 
 class ServicesTestCase(PowerDNSTestCase):
@@ -63,3 +122,194 @@ class ServicesTestCase(PowerDNSTestCase):
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_names)
+
+
+class TestMonitorAccess(PowerDNSTestCase):
+
+  def test(self):
+    connection_parameter_dict = self.requestDefaultInstance()\
+      .getConnectionParameterDict()
+    monitor_base_url = connection_parameter_dict.get('monitor-url')
+    result = requests.get(
+      monitor_base_url, verify=False, auth=(
+        connection_parameter_dict.get('monitor-user'),
+        connection_parameter_dict.get('monitor-password')
+      )
+    )
+
+    self.assertEqual(
+      http.client.OK,
+      result.status_code
+    )
+
+
+class TestMasterRequest(PowerDNSTestCase):
+
+  def test(self):
+    self._test_parameter_dict()
+
+
+class TestMasterRequestDomain(PowerDNSTestCase):
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'zone': 'toto.example.com',
+    }
+
+  def test(self):
+    self._test_parameter_dict(zone=self.getInstanceParameterDict()['zone'])
+
+
+class PowerDNSSlaveTestCase(PowerDNSTestCase):
+
+  @classmethod
+  def requestDefaultInstance(cls):
+    default_instance = super(PowerDNSSlaveTestCase, cls)\
+        .requestDefaultInstance()
+    cls.requestSlaves()
+    return default_instance
+
+  @classmethod
+  def requestSlaves(cls):
+    software_url = cls.getSoftwareURL()
+    software_type = cls.getInstanceSoftwareType()
+    for slave_reference, partition_parameter_kw in cls\
+            .getSlaveParameterDictDict().items():
+      cls.logger.debug(
+        'requesting slave "%s" software:%s parameters:%s',
+        slave_reference, software_url, partition_parameter_kw)
+      cls.slap.request(
+        software_release=software_url,
+        software_type=software_type,
+        partition_reference=slave_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        shared=True
+      )
+
+  @classmethod
+  def getSlaveConnectionParameterDictList(cls):
+    parameter_dict_list = []
+
+    for slave_reference, partition_parameter_kw in cls\
+            .getSlaveParameterDictDict().items():
+      parameter_dict_list.append(cls.slap.request(
+        software_release=cls.getSoftwareURL(),
+        software_type=cls.getInstanceSoftwareType(),
+        partition_reference=slave_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        shared=True
+      ).getConnectionParameterDict())
+    return parameter_dict_list
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    return {
+      'slave-pdns1': {
+        'record': 'test1',
+        'origin': 'nexedi.com',
+        'default': 'test1.com.',
+        'africa': 'test1africa.com.',
+        'china-telecom': 'test1china-telecom.com.',
+        'china-unicom': 'test1china-unicom.com.',
+        'china-mobile': 'test1china-mobile.com.',
+        'east-asia': 'test1east-asia.com.',
+        'europe': 'test1europe.com.',
+        'hong-kong': 'test1hong-kong.com.',
+        'japan': 'test1japan.com.',
+        'north-america': 'test1north-america.com.',
+        'oceania': 'test1oceania.com.',
+        'south-america': 'test1south-america.com.',
+        'west-asia': 'test1west-asia.com.',
+      },
+      'slave-pdns2': {
+        'record': 'test2',
+        'origin': 'nexedi.com',
+        'default': 'test2.com.',
+        'china-telecom': 'test2china-telecom.com.',
+        'europe': 'test2europe.com.',
+        'japan': 'test2japan.com.',
+      }
+    }
+
+  def dns_query(self, domain_name, subnet):
+    message = dns.message.make_query(domain_name, 'A')
+    client_subnet_option = dns.edns.ECSOption(subnet)
+    message.use_edns(options=[client_subnet_option])
+    answer = dns.query.udp(message, self._ipv6_address, port=DNS_PORT)
+    return answer.find_rrset(
+            dns.message.ANSWER,
+            dns.name.from_text(domain_name),
+            dns.rdataclass.IN,
+            dns.rdatatype.CNAME
+          ).to_text().split()[-1]
+
+  def _test_dns_resolver(self, zone):
+    slave_parameter_dict_dict = self.getSlaveParameterDictDict()
+    subnet_dict = {
+      'africa': AFRICAN_SUBNET,
+      'china-telecom': CHINA_TELECOM_SUBNET,
+      'china-unicom': CHINA_UNICOM_SUBNET,
+      'china-mobile': CHINA_MOBILE_SUBNET,
+      'east-asia': EAST_ASIAN_SUBNET,
+      'europe': EUROPEAN_SUBNET,
+      'hong-kong': HONG_KONG_SUBNET,
+      'japan': JAPANESE_SUBNET,
+      'north-america': NORTH_AMERICAN_SUBNET,
+      'oceania': OCEANIAN_SUBNET,
+      'south-america': SOUTH_AMERICAN_SUBNET,
+      'west-asia': WEST_ASIAN_SUBNET,
+    }
+    default_rr_dict = {
+      'europe': 'eu',
+      'africa': 'af',
+      'south-america': 'sa',
+      'north-america': 'na',
+      'china-telecom': 'cn-t',
+      'china-unicom': 'cn-u',
+      'china-mobile': 'cn-m',
+      'japan': 'jp',
+      'hong-kong': 'hk',
+      'east-asia': 'as',
+      'west-asia': 'eu',
+      'oceania': 'oc',
+    }
+
+    for slave_name in slave_parameter_dict_dict:
+      slave_parameter_dict = slave_parameter_dict_dict[slave_name]
+      domain_name = '%s.%s' % (slave_parameter_dict['record'], zone)
+      for region in subnet_dict:
+        self.assertEqual(
+          slave_parameter_dict.pop(
+            region,
+            '%s.%s.' % (default_rr_dict[region], slave_parameter_dict['origin'])
+          ),
+          self.dns_query(domain_name, subnet_dict[region])
+        )
+
+  def _test(self, zone=None):
+    if zone is None:
+      zone = self.default_zone
+    self._test_parameter_dict(
+      zone=zone,
+      slave_amount=len(self.getSlaveParameterDictDict())
+    )
+    self._test_dns_resolver(zone)
+
+
+class TestSlaveRequest(PowerDNSSlaveTestCase):
+
+  def test(self):
+    self._test()
+
+
+class TestSlaveRequestDomain(PowerDNSSlaveTestCase):
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'zone': 'toto.example.com',
+    }
+
+  def test(self):
+    self._test(zone=self.getInstanceParameterDict()['zone'])

stack/slapos.cfg

@@ -247,7 +247,7 @@ croniter = 0.3.25
 
 # Required by:
 # slapos.toolbox==0.94
-dnspython = 1.15.0
+dnspython = 1.16.0
 
 # Required by:
 # cryptography==1.8.1