accounting_l10n_fr: export FEC only with ASCII characters

in 5ae0508d (accounting_l10n_fr: Workaround encoding bugs
in Test Compta Demat, 2023-03-31) we replaced a few characters
that were known to be problematic, but while trying on some
other production data, we found new problematic characters.
This shown that this approach was too "optimistic" and that we
the safest way to have a file that is compatible with Test
Compta Demat is to use only ascii characters, this use
unicodedata to try to retain the original characters (so that
"Jérôme" becomes "Jerome") and also special case € to replace
it by "EUR"

bt5/erp5_accounting_l10n_fr/SkinTemplateItem/portal_skins/erp5_accounting_l10n_fr/AccountingTransactionModule_aggregateFrenchAccountingTransactionFile.py

 # coding: utf-8
+import unicodedata
 from cStringIO import StringIO
 import zipfile
 from Products.ERP5Type.Message import translateString
@@ -13,15 +14,13 @@ fec_file = context.AccountingTransactionModule_viewComptabiliteAsFECXML(
       at_date=at_date,
       result_list=result_list)
 if test_compta_demat_compatibility:
-  fec_file = (fec_file
-    # https://github.com/DGFiP/Test-Compta-Demat/issues/37
-    .replace(u"’", u"'")
-    .replace(u"Œ", u"OE")
-    .replace(u"œ", u"oe")
-    .replace(u"Ÿ", u"Y")
-    # https://github.com/DGFiP/Test-Compta-Demat/issues/39
-    .replace(u"€", u"EUR")
-)
+  # normalize all non ascii characters (é => e), while still keeping
+  # some "important" characters such as €
+  # https://github.com/DGFiP/Test-Compta-Demat/issues/37
+  # https://github.com/DGFiP/Test-Compta-Demat/issues/39
+  fec_file = unicodedata.normalize(
+    'NFKD', fec_file.replace(u"€", "EUR")
+  ).encode('ascii', 'ignore')
 
 zipbuffer = StringIO()
 zipfilename = at_date.strftime('FEC-%Y%m%d.zip')

bt5/erp5_accounting_l10n_fr/TestTemplateItem/portal_components/test.erp5.testAccounting_l10n_fr.py

@@ -137,7 +137,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     account_module = self.portal.account_module
     self._makeOne(
               portal_type='Purchase Invoice Transaction',
-              title='Première Écriture',
+              title='Premiere Ecriture',
               simulation_state='delivered',
               reference='1',
               source_section_value=self.organisation_module.supplier,
@@ -151,7 +151,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
 
     self._makeOne(
               portal_type='Sale Invoice Transaction',
-              title='Seconde Écriture',
+              title='Seconde Ecriture',
               simulation_state='delivered',
               reference='2',
               destination_section_value=self.organisation_module.client_2,
@@ -188,7 +188,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     account_module = self.portal.account_module
     self._makeOne(
               portal_type='Purchase Invoice Transaction',
-              title='Première Écriture',
+              title='Premiere Ecriture',
               simulation_state='delivered',
               ledger='accounting/general',
               reference='1',
@@ -203,7 +203,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
 
     self._makeOne(
               portal_type='Sale Invoice Transaction',
-              title='Seconde Écriture',
+              title='Seconde Ecriture',
               simulation_state='delivered',
               ledger='accounting/general',
               reference='2',
@@ -218,7 +218,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
 
     self._makeOne(
               portal_type='Sale Invoice Transaction',
-              title='Troisième Écriture',
+              title='Troisieme Ecriture',
               simulation_state='delivered',
               ledger='accounting/detailed',
               reference='3',
@@ -254,7 +254,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     journal = journal_list[0]
 
     ecriture_list = sorted([x.text.encode('utf-8') for x in journal.xpath(".//EcritureLib")])
-    self.assertEqual(['Première Écriture'], ecriture_list)
+    self.assertEqual(['Premiere Ecriture'], ecriture_list)
 
     debit_list = journal.xpath(".//Debit")
     self.assertEqual(3, len(debit_list))
@@ -270,7 +270,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     journal = journal_list[0]
 
     ecriture_list = sorted([x.text.encode('utf-8') for x in journal.xpath(".//EcritureLib")])
-    self.assertEqual(['Seconde Écriture'], ecriture_list)
+    self.assertEqual(['Seconde Ecriture'], ecriture_list)
 
     debit_list = journal.xpath(".//Debit")
     self.assertEqual(3, len(debit_list))
@@ -290,7 +290,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     journal = journal_list[0]
 
     ecriture_list = sorted([x.text.encode('utf-8') for x in journal.xpath(".//EcritureLib")])
-    self.assertEqual(['Première Écriture'], ecriture_list)
+    self.assertEqual(['Premiere Ecriture'], ecriture_list)
 
     debit_list = journal.xpath(".//Debit")
     self.assertEqual(3, len(debit_list))
@@ -306,7 +306,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     journal = journal_list[0]
 
     ecriture_list = sorted([x.text.encode('utf-8') for x in journal.xpath(".//EcritureLib")])
-    self.assertEqual(['Seconde Écriture', 'Troisième Écriture'], ecriture_list)
+    self.assertEqual(['Seconde Ecriture', 'Troisieme Ecriture'], ecriture_list)
 
     debit_list = journal.xpath(".//Debit")
     self.assertEqual(6, len(debit_list))
@@ -325,7 +325,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     journal = journal_list[0]
 
     ecriture_list = sorted([x.text.encode('utf-8') for x in journal.xpath(".//EcritureLib")])
-    self.assertEqual(['Première Écriture', 'Seconde Écriture'], ecriture_list)
+    self.assertEqual(['Premiere Ecriture', 'Seconde Ecriture'], ecriture_list)
 
     debit_list = journal.xpath(".//Debit")
     self.assertEqual(6, len(debit_list))
@@ -341,7 +341,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     journal = journal_list[0]
 
     ecriture_list = sorted([x.text.encode('utf-8') for x in journal.xpath(".//EcritureLib")])
-    self.assertEqual(['Troisième Écriture'], ecriture_list)
+    self.assertEqual(['Troisieme Ecriture'], ecriture_list)
 
     debit_list = journal.xpath(".//Debit")
     self.assertEqual(3, len(debit_list))
@@ -360,7 +360,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     journal = journal_list[0]
 
     ecriture_list = sorted([x.text.encode('utf-8') for x in journal.xpath(".//EcritureLib")])
-    self.assertEqual(['Première Écriture'], ecriture_list)
+    self.assertEqual(['Premiere Ecriture'], ecriture_list)
 
     debit_list = journal.xpath(".//Debit")
     self.assertEqual(3, len(debit_list))
@@ -376,7 +376,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     journal = journal_list[0]
 
     ecriture_list = sorted([x.text.encode('utf-8') for x in journal.xpath(".//EcritureLib")])
-    self.assertEqual(['Seconde Écriture'], ecriture_list)
+    self.assertEqual(['Seconde Ecriture'], ecriture_list)
 
     debit_list = journal.xpath(".//Debit")
     self.assertEqual(3, len(debit_list))
@@ -392,7 +392,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     journal = journal_list[0]
 
     ecriture_list = sorted([x.text.encode('utf-8') for x in journal.xpath(".//EcritureLib")])
-    self.assertEqual(['Troisième Écriture'], ecriture_list)
+    self.assertEqual(['Troisieme Ecriture'], ecriture_list)
 
     debit_list = journal.xpath(".//Debit")
     self.assertEqual(3, len(debit_list))
@@ -406,7 +406,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     account_module = self.portal.account_module
     invoice = self._makeOne(
               portal_type='Purchase Invoice Transaction',
-              title='Première Écriture',
+              title='Premiere Ecriture',
               simulation_state='delivered',
               reference='1',
               source_section_value=self.organisation_module.supplier,
@@ -438,7 +438,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     account_module = self.portal.account_module
     self._makeOne(
       portal_type='Purchase Invoice Transaction',
-      title='Des œufs, des Œufs, des Ÿ et des €',
+      title='Le libéllé c’est çà: œufs, des Œufs, des Ÿ et des €',
       simulation_state='delivered',
       reference='1',
       source_section_value=self.organisation_module.supplier,
@@ -464,7 +464,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     self.validateFECXML(tree)
     self.assertEqual(
       tree.xpath('//EcritureLib/text()'),
-      [u'Des oeufs, des OEufs, des Y et des EUR'])
+      [u'Le libelle cest ca: ufs, des ufs, des Y et des EUR'])
 
   def test_Skip0QuantityLines(self):
     # Don't include lines with 0 quantity in the output, because they are
@@ -545,7 +545,7 @@ class TestAccounting_l10n_fr(AccountingTestCase):
     account_module = self.portal.account_module
     invoice = self._makeOne(
       portal_type='Purchase Invoice Transaction',
-      title='Première Écriture',
+      title='Premiere Ecriture',
       simulation_state='delivered',
       source_section_value=self.organisation_module.supplier,
       stop_date=DateTime(2014, 2, 2),