cli.updater: Retry on network error.
Maybe name resolution is flapping, or server is unreachable or misbehaving... This must not cause caucase-update to exit.
Showing
... | ... | @@ -24,8 +24,10 @@ import datetime |
import httplib | ||
import json | ||
import os | ||
import socket | ||
import struct | ||
import sys | ||
import traceback | ||
from cryptography import x509 | ||
from cryptography.hazmat.backends import default_backend | ||
from . import utils | ||
... | ... | @@ -36,6 +38,35 @@ from .client import ( |
HTTPSOnlyCaucaseClient, | ||
) | ||
class RetryingCaucaseClient(CaucaseClient): | ||
_until = utils.until | ||
def _request(self, connection, method, url, body=None, headers=None): | ||
while True: | ||
try: | ||
return super(RetryingCaucaseClient, self)._request( | ||
connection=connection, | ||
method=method, | ||
url=url, | ||
body=body, | ||
headers=headers, | ||
) | ||
except ( | ||
socket.error, | ||
# Note: all exceptions below inherit from httplib.HTTPException, | ||
# but so do errors which are either sign of code bugs | ||
# (ImproperConnectionState) or sign of garbage values provided by | ||
# caller/user, and these should be let through. | ||
httplib.BadStatusLine, | ||
httplib.LineTooLong, | ||
httplib.UnknownProtocol, | ||
httplib.IncompleteRead, | ||
): | ||
connection.close() # Resets HTTPConnection state machine. | ||
print 'Got a network error, retrying in a bit...' | ||
traceback.print_exc() | ||
self._until(datetime.datetime.now() + datetime.timedelta(0, 10)) | ||
|
||
_cryptography_backend = default_backend() | ||
STATUS_ERROR = 1 | ||
... | ... | @@ -693,11 +724,11 @@ def updater(argv=None, until=utils.until): |
}[args.mode] | ||
threshold = datetime.timedelta(args.threshold, 0) | ||
max_sleep = datetime.timedelta(args.max_sleep, 0) | ||
updated = CaucaseClient.updateCAFile( | ||
updated = RetryingCaucaseClient.updateCAFile( | ||
cas_url, | ||
args.cas_ca, | ||
) and args.cas_ca == args.ca | ||
client = CaucaseClient( | ||
client = RetryingCaucaseClient( | ||
ca_url=ca_url, | ||
ca_crt_pem_list=utils.getCertList(args.cas_ca) | ||
) | ||
... | ... | @@ -733,15 +764,15 @@ def updater(argv=None, until=utils.until): |
) | ||
now = until(next_deadline) | ||
next_deadline = now + max_sleep | ||
if args.cas_ca != args.ca and CaucaseClient.updateCAFile( | ||
if args.cas_ca != args.ca and RetryingCaucaseClient.updateCAFile( | ||
cas_url, | ||
args.cas_ca, | ||
): | ||
client = CaucaseClient( | ||
client = RetryingCaucaseClient( | ||
ca_url=ca_url, | ||
ca_crt_pem_list=utils.getCertList(args.cas_ca) | ||
) | ||
if CaucaseClient.updateCAFile(ca_url, args.ca): | ||
if RetryingCaucaseClient.updateCAFile(ca_url, args.ca): | ||
print 'Got new CA' | ||
updated = True | ||
# Note: CRL expiration should happen several time during CA renewal | ||
... | ... | @@ -751,7 +782,7 @@ def updater(argv=None, until=utils.until): |
utils.load_ca_certificate(x) | ||
for x in utils.getCertList(args.ca) | ||
] | ||
if CaucaseClient.updateCRLFile(ca_url, args.crl, ca_crt_list): | ||
if RetryingCaucaseClient.updateCRLFile(ca_url, args.crl, ca_crt_list): | ||
print 'Got new CRL' | ||
updated = True | ||
next_deadline = min( | ||
... | ... |