Commit bf91273f authored by Ophélie Gagnard's avatar Ophélie Gagnard

Downlaod: Add "<<" and ">>" operators.

"<<" and ">>" are like "<=" and ">=" except they select the
closest value and accept "Infinity" for integer comparisons.
parent a0105b6d
...@@ -477,14 +477,15 @@ class NetworkcacheClient(object): ...@@ -477,14 +477,15 @@ class NetworkcacheClient(object):
class NetworkcacheFilter(object): class NetworkcacheFilter(object):
special_word_mapping = {"max":max, "min":max} parse_criterion = re.compile("(<<|>>|[<>]=?|==)").split
parse_criterion = re.compile("([<>]=?|==|:)").split
operator_mapping = { operator_mapping = {
">=": operator.ge, ">=": operator.ge,
"<=": operator.le, "<=": operator.le,
">": operator.gt, ">": operator.gt,
"<": operator.lt, "<": operator.lt,
"==": operator.eq, "==": operator.eq,
">>": operator.ge,
"<<": operator.le,
} }
def __init__(self, criterion_list=()): def __init__(self, criterion_list=()):
...@@ -500,42 +501,41 @@ class NetworkcacheFilter(object): ...@@ -500,42 +501,41 @@ class NetworkcacheFilter(object):
raise NetworkcacheException( raise NetworkcacheException(
'Could not parse criterion: missing or invalid separator (%s)' 'Could not parse criterion: missing or invalid separator (%s)'
% criterion) % criterion)
if parsed_criterion[1] != ':': parsed_criterion[2] = json.loads(parsed_criterion[2])
parsed_criterion[2] = json.loads(parsed_criterion[2])
elif parsed_criterion[2] not in self.special_word_mapping:
raise NetworkcacheException('Unknown special word %r'
% parsed_criterion[2])
parsed_criterion_list.append(parsed_criterion) parsed_criterion_list.append(parsed_criterion)
self.criterion_list = parsed_criterion_list self.criterion_list = parsed_criterion_list
else: else:
raise NetworkcacheException('Invalid criteria: %s' % criterion_list) raise NetworkcacheException('Invalid criteria: %s' % criterion_list)
def __call__(self, data_dict_iterator): def __call__(self, data_dict_list):
''' Return a list of shadir entries that match given criteria ''' Return a list of shadir entries that match given criteria
''' '''
# converting generator into list because the min/max case whould exhaust it def safe_op(data_dict):
data_dict_list = list(data_dict_iterator) try:
return _op(data_dict[key], value)
except TypeError:
logger.info('Comparison failed: %s %s %s with types: %s %s',
data_dict[key], op, value,
type(data_dict[key]), type(value))
for key, op, value in self.criterion_list: for key, op, value in self.criterion_list:
data_dict_list = [data_dict for data_dict in data_dict_list if key in data_dict] data_dict_list = [data_dict for data_dict in data_dict_list
if key in data_dict]
if not data_dict_list: if not data_dict_list:
break break
if op == ":": _op = self.operator_mapping[op]
extremum = self.special_word_mapping[value]( if op in ("<<", ">>"):
data_dict[key] for data_dict in data_dict_list)
data_dict_list = [data_dict for data_dict in data_dict_list
if data_dict[key] == extremum]
else:
filtered_data_dict_list = [] filtered_data_dict_list = []
for data_dict in data_dict_list: for data_dict in data_dict_list:
try: if safe_op(data_dict):
if self.operator_mapping[op](data_dict[key], value): if filtered_data_dict_list and _op(filtered_data_dict_list[0][key], data_dict[key]):
filtered_data_dict_list = [data_dict]
elif filtered_data_dict_list and filtered_data_dict_list[0][key] == data_dict[key]:
filtered_data_dict_list.append(data_dict)
elif not filtered_data_dict_list:
filtered_data_dict_list.append(data_dict) filtered_data_dict_list.append(data_dict)
except TypeError:
logger.info('Comparison failed: %s %s %s'
' with types: %s %s',
data_dict[key], op, value,
type(data_dict[key]), type(value))
data_dict_list = filtered_data_dict_list data_dict_list = filtered_data_dict_list
else:
data_dict_list = list(filter(safe_op, data_dict_list))
return data_dict_list return data_dict_list
class NetworkcacheException(Exception): class NetworkcacheException(Exception):
...@@ -610,8 +610,8 @@ def cmd_upload(*args): ...@@ -610,8 +610,8 @@ def cmd_upload(*args):
def cmd_download(*args): def cmd_download(*args):
parser = _newArgumentParser("URL of data to download.", key_help, True) parser = _newArgumentParser("URL of data to download.", key_help, True)
parser.add_argument('meta', nargs='*', metavar='KEY{>=,<=,==,<,>,:}VALUE', parser.add_argument('meta', nargs='*', metavar='KEY{>=,<=,==,<,>,>>,<<}VALUE',
help='Filter metadata. Each argument represents a filter with a comparison condition. The filters will be applied one by one with the arguments processed in the order of appearance.VALUE is expected to be a json dump of a comparable object in Python (strings included), except when the separator is ":" (in this case VALUE must be min or max).') help='Filter metadata. Each argument represents a filter with a comparison condition. The filters will be applied one by one with the arguments processed in the order of appearance. VALUE is expected to be a json dump of a comparable object in Python (strings included). "<<" and ">>" return the highest (resp. lowest) values lower (resp.higher) than the given value They accept "Infinity" and "-Infinity" for number comparisons.')
args = parser.parse_args(args or sys.argv[1:]) args = parser.parse_args(args or sys.argv[1:])
nc = NetworkcacheClient(args.config) nc = NetworkcacheClient(args.config)
if args.key: if args.key:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment