Skip to content

erp5
erp5
Commit cd0da5b4 authored by Jérome Perrin's avatar Jérome Perrin
Browse files
Options

WIP: Inventory api: interpolation + group by time sequence 

!6 rebased and in a big
commit so that we can start using that
SimulationTool: implement "linear" flow API

testInventoryAPI: silent pyflakes warnings

more interpolation implementation

inventory_list interpolation flow: support at_date & to_date

more and more interpolation API

interpolation

interpolation

group_by_time_sequence_list

testInventoryAPI: move inventory valuation methods in a dedicated test class

This should not be in test inventory list.

test: move Tracking API to a dedicated test module

inventory api: dirty way of disabling cache if keys such as node_category are used.

interpolation: security on SimulationTool_zGetInterpolationMethod

SimulationTool_zGetInterpolationMethod is called in restricted context
of Resource_zGetMovementHistoryList so it needs to be usable by
anonymous.

ZSQL Method does not read parameters from REQUEST, so it looks safe.

inventory_api: rely on catalog to add slot_index in the query

instead of implementing this logic in ZSQL's DTML
parent c14bc53e
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1325 additions and 17 deletions
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Resource_zGetInventoryList.sql
View file @ cd0da5b4
<dtml-let interpolation_ratio="SimulationTool_zGetInterpolationMethod(
stock_table_id=stock_table_id,
interpolation_method=interpolation_method,
interpolation_method_from_date=interpolation_method_from_date,
interpolation_method_to_date=interpolation_method_to_date,
interpolation_method_at_date=interpolation_method_at_date,
group_by_time_sequence_list=group_by_time_sequence_list,
src__=1)">
SELECT
<dtml-if expr="precision is not None">
SUM(ROUND(<dtml-var stock_table_id>.quantity
<dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>)) AS inventory,
SUM(ROUND(<dtml-var stock_table_id>.quantity
<dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>)) AS total_quantity,
SUM(ROUND(
<dtml-var stock_table_id>.quantity
<dtml-if transformed_uid> * transformation.quantity</dtml-if>
* <dtml-var interpolation_ratio>, <dtml-var precision>)) AS inventory,
SUM(ROUND(
<dtml-var stock_table_id>.quantity
<dtml-if transformed_uid> * transformation.quantity</dtml-if>
* <dtml-var interpolation_ratio>, <dtml-var precision>)) AS total_quantity,
<dtml-if convert_quantity_result>
SUM(ROUND(<dtml-var stock_table_id>.quantity * measure.quantity
<dtml-if quantity_unit_uid> / quantity_unit_conversion.quantity</dtml-if>
<dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>))
* <dtml-var interpolation_ratio>, <dtml-var precision>))
AS converted_quantity,
</dtml-if>
IFNULL(SUM(ROUND(<dtml-var stock_table_id>.total_price, <dtml-var precision>)), 0) AS total_price
IFNULL(SUM(ROUND(
<dtml-var stock_table_id>.total_price * <dtml-var interpolation_ratio>, <dtml-var precision>)), 0) AS total_price
<dtml-else>
SUM(<dtml-var stock_table_id>.quantity <dtml-if transformed_uid> * transformation.quantity</dtml-if>) AS inventory,
SUM(<dtml-var stock_table_id>.quantity <dtml-if transformed_uid> * transformation.quantity</dtml-if>) AS total_quantity,
SUM(<dtml-var stock_table_id>.quantity
<dtml-if transformed_uid> * transformation.quantity</dtml-if>
* <dtml-var interpolation_ratio>
) AS inventory,
SUM(<dtml-var stock_table_id>.quantity
<dtml-if transformed_uid> * transformation.quantity</dtml-if>
* <dtml-var interpolation_ratio>
) AS total_quantity,
<dtml-if convert_quantity_result>
ROUND(SUM(<dtml-var stock_table_id>.quantity * measure.quantity
<dtml-if quantity_unit_uid> / quantity_unit_conversion.quantity</dtml-if>
<dtml-if transformed_uid> * transformation.quantity</dtml-if>), 12)
<dtml-if transformed_uid> * transformation.quantity</dtml-if> * <dtml-var interpolation_ratio>), 12)
AS converted_quantity,
</dtml-if>
IFNULL(SUM(<dtml-var stock_table_id>.total_price), 0) AS total_price
IFNULL(SUM(<dtml-var stock_table_id>.total_price * <dtml-var interpolation_ratio>), 0) AS total_price
</dtml-if>
<dtml-if inventory_list>
,
......@@ -56,6 +77,7 @@ SELECT
COUNT(DISTINCT <dtml-var stock_table_id>.uid) AS stock_uid,
MAX(<dtml-var stock_table_id>.date) AS date
</dtml-if>
<dtml-if select_expression>, <dtml-var select_expression></dtml-if>
FROM
......@@ -69,6 +91,55 @@ FROM
</dtml-if>
</dtml-in>
, <dtml-var stock_table_id>
<dtml-if group_by_time_sequence_list>
RIGHT JOIN
( <dtml-in prefix="time_slot" expr="_.list(_.enumerate(group_by_time_sequence_list))">
SELECT
<dtml-sqlvar expr="time_slot_key" type="int"> slot_index,
<dtml-sqlvar expr="time_slot_item.get('from_date')" type="datetime" optional> slot_from_date,
<dtml-sqlvar expr="time_slot_item.get('at_date')" type="datetime" optional> slot_at_date,
<dtml-sqlvar expr="time_slot_item.get('to_date')" type="datetime" optional> slot_to_date
<dtml-unless time_slot_end>UNION ALL</dtml-unless>
</dtml-in> ) slots
ON
<dtml-if group_by_time_sequence_list>
(
( slot_from_date is not null AND
( slot_at_date is not null AND
GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND
LEAST(`stock`.`date`, `stock`.`mirror_date`) <= slot_at_date
) OR (
(
slot_to_date is not null AND
GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND
LEAST(`stock`.`date`, `stock`.`mirror_date`) < slot_to_date
) OR (
GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND
slot_at_date is null AND slot_to_date is null
)
)
) OR (
slot_from_date is null AND (
( slot_at_date is not null AND
( LEAST(`stock`.`date`, `stock`.`mirror_date`) <= slot_at_date )
) OR LEAST(`stock`.`date`, `stock`.`mirror_date`) < slot_to_date
)
)
)
<dtml-else>
(
( slot_from_date is null OR stock.date >= slot_from_date )
AND ( slot_at_date is null OR stock.date <= slot_at_date )
AND ( slot_to_date is null OR stock.date < slot_to_date )
)
</dtml-if>
</dtml-if>
</dtml-if>
<dtml-if quantity_unit_uid> <dtml-comment>XXX quantity unit conversion will not work when using implict_join=False</dtml-comment>
LEFT JOIN quantity_unit_conversion ON
......@@ -117,8 +188,10 @@ WHERE
GROUP BY
<dtml-if transformed_uid>transformation.transformed_uid,</dtml-if>
<dtml-var group_by_expression>
</dtml-if>
<dtml-if order_by_expression>
ORDER BY
<dtml-var order_by_expression>
</dtml-if>
</dtml-let>
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Resource_zGetInventoryList.xml
View file @ cd0da5b4
......@@ -46,7 +46,12 @@ convert_quantity_result\n
quantity_unit_uid\n
stock_table_id=stock\n
transformed_uid\n
transformed_variation_text</string> </value>
transformed_variation_text\n
group_by_time_sequence_list:list\n
interpolation_method\n
interpolation_method_from_date=not_applicable\n
interpolation_method_to_date=not_applicable\n
interpolation_method_at_date=not_applicable</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
......
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/SimulationTool_zGetInterpolationMethod.sql 0 → 100644
View file @ cd0da5b4
<dtml-if expr="interpolation_method == 'linear'">
<dtml-if group_by_time_sequence_list>
CASE
WHEN <dtml-var stock_table_id>.mirror_date = <dtml-var stock_table_id>.date THEN 1
ELSE (
UNIX_TIMESTAMP(
IFNULL(
LEAST(
IFNULL(slot_at_date, slot_to_date),
GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
),
GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
)
)
- UNIX_TIMESTAMP(
GREATEST(
IFNULL(
slot_from_date, TIMESTAMP(0)
),
LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
)
)
)
/ (
UNIX_TIMESTAMP(GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) -
UNIX_TIMESTAMP(LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) ) END
<dtml-else>
CASE
WHEN <dtml-var stock_table_id>.mirror_date = <dtml-var stock_table_id>.date THEN 1
ELSE (
UNIX_TIMESTAMP(LEAST(
<dtml-if interpolation_method_at_date>
<dtml-sqlvar interpolation_method_at_date type="datetime">
<dtml-else>
<dtml-sqlvar interpolation_method_to_date type="datetime">
</dtml-if>,
GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date) ))
- UNIX_TIMESTAMP(GREATEST(<dtml-sqlvar interpolation_method_from_date type="datetime">,
LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date))))
/ ( UNIX_TIMESTAMP(GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) -
UNIX_TIMESTAMP(LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) ) END
</dtml-if>
<dtml-elif expr="interpolation_method == 'all_or_nothing'">
CASE
WHEN (
-- movement contained in time frame
LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
>= <dtml-sqlvar interpolation_method_from_date type="datetime"> AND
GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
<dtml-if interpolation_method_at_date>
<= <dtml-sqlvar interpolation_method_at_date type="datetime">
<dtml-else>
< <dtml-sqlvar interpolation_method_to_date type="datetime">
</dtml-if>
) THEN 1
ELSE 0
END
<dtml-elif expr="interpolation_method == 'one_for_all'">
CASE
WHEN (
-- movement overlaps with time frame
GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
<= <dtml-sqlvar interpolation_method_from_date type="datetime"> OR
LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)
<dtml-if interpolation_method_at_date>
>= <dtml-sqlvar interpolation_method_at_date type="datetime">
<dtml-else>
> <dtml-sqlvar interpolation_method_to_date type="datetime">
</dtml-if>
) THEN 0
ELSE 1
END
<dtml-else>
1
</dtml-if>
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/SimulationTool_zGetInterpolationMethod.xml 0 → 100644
View file @ cd0da5b4
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Use_Database_Methods_Permission</string> </key>
<value>
<list>
<string>Anonymous</string>
</list>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>stock_table_id\r\n
interpolation_method\r\n
interpolation_method_from_date\r\n
interpolation_method_to_date\r\n
interpolation_method_at_date\r\n
group_by_time_sequence_list:list</string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>cmf_activity_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SimulationTool_zGetInterpolationMethod</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_core/ToolComponentTemplateItem/portal_components/tool.erp5.SimulationTool.py
View file @ cd0da5b4
......@@ -514,6 +514,15 @@ class SimulationTool(BaseTool):
from_table_dict[alias] = table
sql_kw.update(catalog_sql_kw)
sql_kw['from_table_list'] = from_table_dict.items()
# When group_by_time_sequence_list is used, the ZSQL method template
# will use a variable slot_index, we want to select it, group and order
# by it.
if sql_kw.get('group_by_time_sequence_list'):
new_kw['group_by_list'] = new_kw.get('group_by_list', []) + ['slot_index']
new_kw['order_by_list'] = new_kw.get('order_by_list', []) + [('slot_index', )]
new_kw.setdefault('select_dict', {})['slot_index'] = 'slot_index'
sql_kw.update(ctool.buildSQLQuery(**new_kw))
return sql_kw
def _generateKeywordDict(self,
......@@ -563,6 +572,8 @@ class SimulationTool(BaseTool):
omit_output=0,
omit_asset_increase=0,
omit_asset_decrease=0,
# interpolation_method
interpolation_method='default',
# group by
group_by_node=0,
group_by_node_category=0,
......@@ -599,6 +610,7 @@ class SimulationTool(BaseTool):
group_by_function_category=0,
group_by_function_category_strict_membership=0,
group_by_date=0,
group_by_time_sequence_list=None,
# sort_on
sort_on=None,
group_by=None,
......@@ -689,6 +701,43 @@ class SimulationTool(BaseTool):
date_dict['range'] = 'ngt'
if date_dict:
column_value_dict['date'] = date_dict
if interpolation_method != 'default':
if not group_by_time_sequence_list:
if not (from_date and (to_date or at_date)):
raise ValueError("date_range is required to use interpolation_method")
# if we consider flow, we also select movement whose mirror date is
# in the from_date/to_date range and movement whose
# start_date/stop_date contains the report range.
# The selected range is wider, but the selected movements will have an
# "interpolation_ratio" applied to their quantity and prices.
if to_date:
column_value_dict['date'] = ComplexQuery(
Query(date=(from_date, to_date), range='minmax'),
Query(mirror_date=(from_date, to_date), range='minmax'),
ComplexQuery(
Query(mirror_date=from_date, range='min'),
Query(date=to_date, range='max'),
logical_operator="AND"),
ComplexQuery(
Query(date=from_date, range='min'),
Query(mirror_date=to_date, range='max'),
logical_operator="AND"),
logical_operator="OR"
)
else:
column_value_dict['date'] = ComplexQuery(
Query(date=(from_date, at_date), range='minngt'),
Query(mirror_date=(from_date, at_date), range='minngt'),
ComplexQuery(
Query(mirror_date=from_date, range='min'),
Query(date=at_date, range='ngt'),
logical_operator="AND"),
ComplexQuery(
Query(date=from_date, range='min'),
Query(mirror_date=at_date, range='ngt'),
logical_operator="AND"),
logical_operator="OR"
)
else:
column_value_dict['date'] = {'query': [to_date], 'range': 'ngt'}
column_value_dict['mirror_date'] = {'query': [from_date], 'range': 'nlt'}
......@@ -980,6 +1029,7 @@ class SimulationTool(BaseTool):
new_kw['related_key_select_expression_list'] =\
related_key_select_expression_list
sql_kw['group_by_time_sequence_list'] = group_by_time_sequence_list
return sql_kw, new_kw
#######################################################
......@@ -1062,6 +1112,17 @@ class SimulationTool(BaseTool):
output_simulation_state - only take rows with specified simulation_state
and quantity < 0
interpolation_method - Method to consider movements when calculating flows.
* (default): Consider the movement decreases 100% of source stock on
start date and increase 100% of the destination node stock on stop
date.
* linear: consider the movement decreases source stock and increase
destination stock linearly between start date and stop date.
* all_or_nothing: consider only movement who starts after the beginning of
the query period and finishes after the end of the query period.
* one_for_all: consider the movement fully as long as it is partially
contained in the query period.
ignore_variation - do not take into account variation in inventory
calculation (useless on getInventory, but useful on
......@@ -1187,6 +1248,7 @@ class SimulationTool(BaseTool):
group_by_section_category=0,
group_by_section_category_strict_membership=0,
group_by_resource=None,
group_by_time_sequence_list=(),
group_by=None,
**ignored):
"""
......@@ -1206,7 +1268,8 @@ class SimulationTool(BaseTool):
group_by_function or group_by_mirror_section or group_by_payment or \
group_by_sub_variation or group_by_variation or \
group_by_movement or group_by_date or group_by_section_category or\
group_by_section_category_strict_membership:
group_by_section_category_strict_membership or \
group_by_time_sequence_list:
if group_by_resource is None:
group_by_resource = 1
new_group_by_dict['group_by_resource'] = group_by_resource
......@@ -1223,6 +1286,7 @@ class SimulationTool(BaseTool):
omit_simulation=0,
only_accountable=True,
default_stock_table='stock',
interpolation_method='default',
statistic=0, inventory_list=1,
precision=None, connection_id=None,
**kw):
......@@ -1300,6 +1364,7 @@ class SimulationTool(BaseTool):
'stock_table_id': default_stock_table,
'src__': src__,
'ignore_variation': ignore_variation,
'interpolation_method': interpolation_method,
'standardise': standardise,
'omit_simulation': omit_simulation,
'only_accountable': only_accountable,
......@@ -1313,7 +1378,8 @@ class SimulationTool(BaseTool):
# Get cached data
if getattr(self, "Resource_zGetInventoryCacheResult", None) is not None and \
optimisation__ and (not kw.get('from_date')) and \
'transformed_resource' not in kw:
'transformed_resource' not in kw\
and "group_by_time_sequence_list" not in kw:
# Here is the different kind of date
# from_date : >=
# to_date : <
......@@ -1343,7 +1409,7 @@ class SimulationTool(BaseTool):
kw['from_date'] = cached_date
else:
cached_result = []
sql_kw, new_kw = self._generateKeywordDict(**kw)
sql_kw, new_kw = self._generateKeywordDict(interpolation_method=interpolation_method, **kw)
# Copy kw content as _generateSQLKeywordDictFromKeywordDict
# remove some values from it
try:
......@@ -1356,6 +1422,25 @@ class SimulationTool(BaseTool):
stock_sql_kw = self._generateSQLKeywordDictFromKeywordDict(
table=default_stock_table, sql_kw=sql_kw, new_kw=new_kw_copy)
stock_sql_kw.update(base_inventory_kw)
# TODO: move in this interpolation_method handling in _generateSQLKeywordDictFromKeywordDict ?
if interpolation_method in ('linear', 'all_or_nothing', 'one_for_all'):
# XXX only DateTime instance are supported
from_date = kw.get('from_date')
if from_date:
from_date = from_date.toZone("UTC")
to_date = kw.get('to_date')
if to_date:
to_date = to_date.toZone("UTC")
at_date = kw.get('at_date')
if at_date:
at_date = at_date.toZone("UTC")
stock_sql_kw['interpolation_method_from_date'] = from_date
stock_sql_kw['interpolation_method_to_date'] = to_date
stock_sql_kw['interpolation_method_at_date'] = at_date
elif interpolation_method != 'default':
raise ValueError("Unsupported interpolation_method %r" % (interpolation_method,))
delta_result = self.Resource_zGetInventoryList(
**stock_sql_kw)
if src__:
......
product/ERP5/tests/testInventoryAPI.py
View file @ cd0da5b4
This diff is collapsed.
product/ERP5/tests/testTrackingAPI.py 0 → 100644
View file @ cd0da5b4
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7