From 340a7609dc50474d4479c1e1a3567efdd3da9f5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Mon, 1 Feb 2021 02:52:18 +0100
Subject: [PATCH] software/*: version up urllib3, requests and dependencies
Several softwares were using fixed versions, but versions are also defined in
stack/slapos.cfg, so let's use from stack/slapos.cfg which will simplify
version management.
This urllib3 has visible changes for us, it no longer send the request path
"as is", but normalizes the . and .. which is not what we want in
caddy-frontend test, so we adjust the test to use prepared requests, which is
the recommended approach for this kind of low level needs. Since prepared
requests does not set default user agent "python-requests", tests checking that
user agent is in log files have been adjusted to use a dedicated test user agent
header.
Another change is that certificates are now verified by default, so we adjust
seleniumserver tests to disable certificate validation. jstestnode, already
had a patch regarding certificate validation, because we wanted certificates
to be validated before it was the default. We just update this patch a bit
because we want to keep the optional behaviours of validating against a
specific CA or to disable validation.
---
software/caddy-frontend/software.cfg | 9 ------
software/caddy-frontend/test/test.py | 37 +++++++++++++++---------
software/erp5/test/test/test_balancer.py | 2 +-
software/jstestnode/buildout.hash.cfg | 2 +-
software/jstestnode/runTestSuite.in | 4 +--
software/jstestnode/software.cfg | 3 --
software/monitor/software.cfg | 5 ----
software/seleniumserver/test/test.py | 9 +++++-
software/slapos-sr-testing/software.cfg | 15 +---------
9 files changed, 37 insertions(+), 49 deletions(-)
diff --git a/software/caddy-frontend/software.cfg b/software/caddy-frontend/software.cfg
index efd1809f1..ef9ed2e20 100644
--- a/software/caddy-frontend/software.cfg
+++ b/software/caddy-frontend/software.cfg
@@ -228,15 +228,6 @@ mode = 0644
[versions]
# Modern KeDiFa requires zc.lockfile
zc.lockfile = 1.4
-# Versions pinned for kedifa need urllib3 >= 1.18
-urllib3 = 1.24
-requests = 2.20.0
-certifi = 2018.10.15
-idna = 2.7
-chardet = 3.0.4
-# ipaddress is patching IPAddress so IPv6 match works
-ipaddress = 1.0.22
-# Versions pinned for kedifa need urllib3 >= 1.18
validators = 0.12.2
PyRSS2Gen = 1.1
diff --git a/software/caddy-frontend/test/test.py b/software/caddy-frontend/test/test.py
index 9aa33325f..b43838604 100644
--- a/software/caddy-frontend/test/test.py
+++ b/software/caddy-frontend/test/test.py
@@ -419,13 +419,21 @@ def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
try:
add_custom_dns(domain, port, TEST_IP)
socket.getaddrinfo = new_getaddrinfo
- return session.get(
- 'https://%s:%s/%s' % (domain, port, path),
- verify=False,
- allow_redirects=False,
- headers=headers,
- cookies=cookies
- )
+ # Use a prepared request, to disable path normalization.
+ # We need this because some test checks requests with paths like
+ # /test-path/deep/.././deeper but we don't want the client to send
+ # /test-path/deeper
+ # See also https://github.com/psf/requests/issues/5289
+ url = 'https://%s:%s/%s' % (domain, port, path)
+ req = requests.Request(
+ method='GET',
+ url=url,
+ headers=headers,
+ cookies=cookies,
+ )
+ prepped = req.prepare()
+ prepped.url = url
+ return session.send(prepped, verify=False, allow_redirects=False)
finally:
socket.getaddrinfo = socket_getaddrinfo
@@ -447,11 +455,13 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
new_source = source.SourceAddressAdapter(source_ip)
session.mount('http://', new_source)
session.mount('https://', new_source)
- return session.get(
- 'http://%s:%s/%s' % (TEST_IP, port, path),
- headers=headers,
- allow_redirects=False,
- )
+
+ # Use a prepared request, to disable path normalization.
+ url = 'http://%s:%s/%s' % (TEST_IP, port, path)
+ req = requests.Request(method='GET', url=url, headers=headers)
+ prepped = req.prepare()
+ prepped.url = url
+ return session.send(prepped, allow_redirects=False)
class TestHandler(BaseHTTPRequestHandler):
@@ -1821,6 +1831,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
+ 'User-Agent': 'TEST USER AGENT',
}
)
@@ -1854,7 +1865,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - - ' \
r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2} \+\d{4}\] ' \
r'"GET \/test-path\/deep\/..\/.\/deeper HTTP\/1.1" \d{3} ' \
- r'\d+ "-" "python-requests.*" \d+'
+ r'\d+ "-" "TEST USER AGENT" \d+'
self.assertRegexpMatches(
open(log_file, 'r').readlines()[-1],
diff --git a/software/erp5/test/test/test_balancer.py b/software/erp5/test/test/test_balancer.py
index 4dcfe2acd..a9ab9c6eb 100644
--- a/software/erp5/test/test/test_balancer.py
+++ b/software/erp5/test/test/test_balancer.py
@@ -750,7 +750,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
).json()
self.assertNotEqual(result['Incoming Headers'].get('x-forwarded-for', '').split(', ')[0], '1.2.3.4')
balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default-auth']
- with self.assertRaises(OpenSSL.SSL.Error):
+ with self.assertRaisesRegexp(Exception, "certificate required"):
requests.get(
balancer_url,
headers={'X-Forwarded-For': '1.2.3.4'},
diff --git a/software/jstestnode/buildout.hash.cfg b/software/jstestnode/buildout.hash.cfg
index 54b3d085d..d08486019 100644
--- a/software/jstestnode/buildout.hash.cfg
+++ b/software/jstestnode/buildout.hash.cfg
@@ -27,4 +27,4 @@ md5sum = 98faa5ad8cfb23a11d97a459078a1d05
[template-runTestSuite]
filename = runTestSuite.in
-md5sum = bb3f053b6cdb0a8888e9d32e63085ed5
+md5sum = 73ef758825563b7a6a1d660d4d5229b8
diff --git a/software/jstestnode/runTestSuite.in b/software/jstestnode/runTestSuite.in
index 23e3bdc9f..bbfcdf1ef 100644
--- a/software/jstestnode/runTestSuite.in
+++ b/software/jstestnode/runTestSuite.in
@@ -97,8 +97,8 @@ def main():
executable_path='${geckodriver:location}')
else:
assert target == 'selenium-server', "Unsupported target {}".format(test_runner['target'])
- # use a remote connection which verifies TLS certificate
- # workaround for https://github.com/SeleniumHQ/selenium/issues/6534
+ # use a remote connection which optionally verifies TLS certificate
+ # and optionally against a user provided CA.
executor = RemoteConnection(test_runner['server-url'], keep_alive=True)
cert_reqs = 'CERT_REQUIRED'
ca_certs = certifi.where()
diff --git a/software/jstestnode/software.cfg b/software/jstestnode/software.cfg
index c67c5611a..310c7995c 100644
--- a/software/jstestnode/software.cfg
+++ b/software/jstestnode/software.cfg
@@ -117,7 +117,4 @@ output = ${buildout:directory}/template-nginx.cfg.in
output = ${buildout:directory}/runTestSuite.in
[versions]
-
selenium = 3.14.1
-urllib3 = 1.24
-certifi = 2018.10.15
diff --git a/software/monitor/software.cfg b/software/monitor/software.cfg
index 98de4a4e7..42648c75b 100644
--- a/software/monitor/software.cfg
+++ b/software/monitor/software.cfg
@@ -81,12 +81,7 @@ surykatka = 0.5.0
# For surykatka 0.5.0
click = 7.0
-certifi = 2019.11.28
-chardet = 3.0.4
dnspython = 1.16.0
forcediphttpsadapter = 1.0.1
-idna = 2.8
miniupnpc = 2.0.2
peewee = 3.13.1
-requests = 2.22.0
-urllib3 = 1.25.8
diff --git a/software/seleniumserver/test/test.py b/software/seleniumserver/test/test.py
index 3596c3c99..9eb7d302d 100644
--- a/software/seleniumserver/test/test.py
+++ b/software/seleniumserver/test/test.py
@@ -46,8 +46,10 @@ from PIL import Image
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.webdriver.remote.remote_connection import RemoteConnection
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
+import urllib3
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase, ManagedHTTPServer
@@ -318,8 +320,13 @@ class TestFrontend(WebServerMixin, SeleniumServerTestCase):
self.assertEqual('selenium', parsed.username)
self.assertTrue(parsed.password)
+ # XXX we are using a self signed certificate, but selenium 3.141.0 does
+ # not expose API to ignore certificate verification
+ executor = RemoteConnection(webdriver_url, keep_alive=True)
+ executor._conn = urllib3.PoolManager(cert_reqs='CERT_NONE', ca_certs=None)
+
driver = webdriver.Remote(
- command_executor=webdriver_url,
+ command_executor=executor,
desired_capabilities=DesiredCapabilities.CHROME)
driver.get(self.server_url)
diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg
index 41a200726..3cb4bb030 100644
--- a/software/slapos-sr-testing/software.cfg
+++ b/software/slapos-sr-testing/software.cfg
@@ -298,16 +298,7 @@ selenium = 3.141.0
# Patched eggs
PyPDF2 = 1.26.0+SlapOSPatched001
-# modern versions for SSL fixed support
-requests = 2.20.0
-certifi = 2018.10.15
-idna = 2.7
-chardet = 3.0.4
-
-# ipaddress is patching IPAddress so IPv6 match works
-ipaddress = 1.0.22
-
-# cacuase and its dependencies
+# caucase and its dependencies
caucase = 0.9.4
pem = 18.2.0
PyJWT = 1.6.4
@@ -315,10 +306,6 @@ PyJWT = 1.6.4
# Django 1.11 is python 2 compatible
Django = 1.11
-# Required by:
-# selenium==3.141.0
-urllib3 = 1.24.1
-
mock = 2.0.0
testfixtures = 6.11
funcsigs = 1.0.2
--
2.30.9