Commit 3ff2d0a7 authored by Jérome Perrin's avatar Jérome Perrin

Support UTF-8 encoded CSV

/reviewed-on !18
parents 4912750b 3bdab08f
......@@ -48,5 +48,3 @@ class TestAllSupportedFormat(TestCase):
"""Tests if ffmpeg convets midi file"""
self.runConversionList(join('data', 'test.ogg'), "ogg", "midi", "audio/rtp-midi")
def test_suite():
return make_suite(TestAllSupportedFormat)
......@@ -50,5 +50,3 @@ class TestAllSupportedFormat(TestCase):
def testAllSupportedFormat(self):
self.runConversionList(self.ConversionScenarioList())
def test_suite():
return make_suite(TestAllSupportedFormat)
......@@ -69,5 +69,3 @@ class TestHandler(HandlerTestCase):
self.assertEquals(file_format, 'audio/x-wav')
def test_suite():
return make_suite(TestHandler)
......@@ -45,5 +45,3 @@ class TestInterface(unittest.TestCase):
('metadata_dict',))
def test_suite():
return make_suite(TestInterface)
......@@ -86,5 +86,3 @@ class TestServer(TestCase):
"""Test if metadata is inserted correctly into video files"""
self.runUpdateMetadataList(self.UpdateMetadataScenarioList())
def test_suite():
return make_suite(TestServer)
......@@ -61,5 +61,3 @@ class TestHandler(HandlerTestCase):
self.assertRaises(NotImplementedError, handler.setMetadata)
def test_suite():
return make_suite(TestHandler)
......@@ -74,5 +74,3 @@ class TestServer(TestCase):
self.runFaultGetMetadataList(self.FaultGetMetadataScenarioList())
def test_suite():
return make_suite(TestServer)
......@@ -138,16 +138,16 @@ class OpenOffice(Application):
removeDirectory(self.path_user_installation)
# Create command with all parameters to start the instance
self.command = [join(self.office_binary_path, self._bin_soffice),
'-headless',
'-invisible',
'-nocrashreport',
'-nologo',
'-nodefault',
'-norestore',
'-nofirststartwizard',
'-accept=socket,host=%s,port=%d;urp;' % (self.hostname, self.port),
'--headless',
'--invisible',
'--nocrashreport',
'--nologo',
'--nodefault',
'--norestore',
'--nofirststartwizard',
'--accept=socket,host=%s,port=%d;urp;' % (self.hostname, self.port),
'-env:UserInstallation=file://%s' % self.path_user_installation,
'-language=%s' % self.default_language,
'--language=%s' % self.default_language,
]
# To run soffice.bin, several environment variables should be set.
env = self.environment_dict.copy()
......
......@@ -54,9 +54,6 @@ class Handler(object):
def __init__(self, base_folder_url, data, source_format, **kw):
"""Creates document in file system and loads it in OOo."""
self.document = FileSystemDocument(base_folder_url,
data,
source_format)
self.zip = kw.get('zip', False)
self.uno_path = kw.get("uno_path", None)
self.office_binary_path = kw.get("office_binary_path", None)
......@@ -67,6 +64,22 @@ class Handler(object):
self.uno_path = environ.get("uno_path")
if not self.office_binary_path:
self.office_binary_path = environ.get("office_binary_path")
self._createDocument(base_folder_url, data, source_format)
def _createDocument(self, base_folder_url, data, source_format):
if source_format == 'csv':
# Cloudooo expect utf-8 encoded csv, but also tolerate latin9 for
# backward compatibility.
# The heuristic is "if it's not utf-8", let's assume it's iso-8859-15.
try:
unicode(data, 'utf-8')
except UnicodeDecodeError:
data = unicode(data, 'iso-8859-15').encode('utf-8')
logger.warn("csv data is not utf-8, assuming iso-8859-15")
self.document = FileSystemDocument(
base_folder_url,
data,
source_format)
def _getCommand(self, *args, **kw):
"""Transforms all parameters passed in a command"""
......
......@@ -28,8 +28,10 @@
##############################################################################
import sys
import csv
import codecs
import helper_util
from os.path import dirname
from os.path import dirname, splitext
from tempfile import mktemp
from base64 import decodestring, encodestring
from getopt import getopt, GetoptError
......@@ -144,6 +146,26 @@ class UnoConverter(object):
else:
return ()
def _getPropertyToImport(self, source_url):
"""Create the property for import filter, according to the extension of the file."""
_, extension = splitext(source_url)
if extension == '.csv':
# https://wiki.openoffice.org/wiki/Documentation/DevGuide/Spreadsheets/Filter_Options
# Try to sniff the csv delimiter
with codecs.open(source_url, 'rb', 'utf-8', errors="ignore") as csvfile:
try:
dialect = csv.Sniffer().sniff(csvfile.read(1024))
delimiter = ord(dialect.delimiter)
except csv.Error:
delimiter = ord(',')
return (
self._createProperty("FilterName", "Text - txt - csv (StarCalc)"),
self._createProperty("FilterOptions", "{delimiter},34,UTF-8".format(**locals())), )
return ()
def _load(self):
"""Create one document with basic properties
refresh argument tells to uno environment to
......@@ -154,7 +176,11 @@ class UnoConverter(object):
self.office_binary_path)
desktop = service_manager.createInstance("com.sun.star.frame.Desktop")
uno_url = self.systemPathToFileUrl(self.document_url)
uno_document = desktop.loadComponentFromURL(uno_url, "_blank", 0, ())
uno_document = desktop.loadComponentFromURL(
uno_url,
"_blank",
0,
self._getPropertyToImport(self.document_url))
if not uno_document:
raise AttributeError("This document can not be loaded or is empty")
if self.refresh:
......
Jérome,ジェローム
नमस्ते,여보세요
a b
1,3 c
\ No newline at end of file
......@@ -62,5 +62,3 @@ class TestAllFormats(TestCase):
for extension in extension_list:
self._testConvertFile(input_url, source_format, extension[0], None)
def test_suite():
return make_suite(TestAllFormats)
......@@ -75,5 +75,3 @@ class TestAllFormatsERP5Compatibility(TestCase):
self.fail('Failed Conversions:\n' + message)
def test_suite():
return make_suite(TestAllFormatsERP5Compatibility)
......@@ -60,5 +60,3 @@ class TestApplication(unittest.TestCase):
self.assertEquals(self.application.pid(), None)
def test_suite():
return make_suite(TestApplication)
......@@ -130,5 +130,3 @@ class TestFileSystemDocument(unittest.TestCase):
sorted(['logo.gif', 'test.htm']))
def test_suite():
return make_suite(TestFileSystemDocument)
......@@ -54,5 +54,3 @@ class TestFilter(unittest.TestCase):
self.assertTrue(self.filter.isPreferred())
def test_suite():
return make_suite(TestFilter)
......@@ -174,5 +174,3 @@ class TestOOGranulator(HandlerTestCase):
oogranulator = OOGranulator(data, 'odt')
self.assertEquals(['Title 1', 1], oogranulator.getChapterItem(1))
def test_suite():
return make_suite(TestOOGranulator)
......@@ -685,5 +685,3 @@ class TestHandler(HandlerTestCase):
('image/x-ms-bmp', 'BMP - Windows Bitmap'),
('text/html', 'HTML Document (Impress)') ])
def test_suite():
return make_suite(TestHandler)
......@@ -65,5 +65,3 @@ class TestHighLoad(TestCase):
self.assertTrue(all(result_list))
def test_suite():
return make_suite(TestHighLoad)
......@@ -181,5 +181,3 @@ class TestInterface(TestCase):
self.assertEquals(sorted(ILockable.names()), sorted(lockable_method_list))
def test_suite():
return make_suite(TestInterface)
......@@ -64,5 +64,3 @@ class TestLegacyInterface(TestCase):
self.assertEquals(self._getFileType(response_dict['data']),
'application/vnd.oasis.opendocument.text')
def test_suite():
return make_suite(TestLegacyInterface)
......@@ -306,5 +306,3 @@ class TestMimeMapper(HandlerTestCase):
self.assertEquals(filtername, "impress_html_Export")
def test_suite():
return make_suite(TestMimeMapper)
......@@ -72,5 +72,3 @@ class TestMonitorInit(HandlerTestCase):
True)
def test_suite():
return make_suite(TestMonitorInit)
......@@ -107,5 +107,3 @@ class TestMonitorMemory(unittest.TestCase):
self.assertEquals(type(memory_usage_int), IntType)
def test_suite():
return make_suite(TestMonitorMemory)
......@@ -60,5 +60,3 @@ class TestMonitorRequest(HandlerTestCase):
monitor_request.terminate()
def test_suite():
return make_suite(TestMonitorRequest)
......@@ -91,5 +91,3 @@ class TestMonitorTimeout(unittest.TestCase):
openoffice.release()
def test_suite():
return make_suite(TestMonitorTimeout)
......@@ -65,5 +65,3 @@ class TestOdfDocument(HandlerTestCase):
'document-content'))
def test_suite():
return make_suite(TestOdfDocument)
......@@ -117,5 +117,3 @@ class TestOpenOffice(HandlerTestCase):
self.assertFalse(second_openoffice.status())
def test_suite():
return make_suite(TestOpenOffice)
This diff is collapsed.
......@@ -87,5 +87,3 @@ class TestUnoConverter(HandlerTestCase):
self.assertEquals(exists(output_url), False)
def test_suite():
return make_suite(TestUnoConverter)
......@@ -118,5 +118,3 @@ class TestUnoMimeMapper(HandlerTestCase):
openoffice.start()
def test_suite():
return make_suite(TestUnoMimeMapper)
......@@ -62,9 +62,3 @@ class TestUtil(unittest.TestCase):
self.assertEquals(mimetypes.types_map.get(".3gp"), "video/3gpp")
def test_suite():
return make_suite(TestUtil)
if "__main__" == __name__:
suite = unittest.TestLoader().loadTestsFromTestCase(TestUtil)
unittest.TextTestRunner(verbosity=2).run(suite)
......@@ -78,5 +78,3 @@ class TestHandler(HandlerTestCase):
self.assertEquals(get("text/plain;ignored=param"), [])
self.assertEquals(get("text/plain;charset=UTF-8;ignored=param"), [])
def test_suite():
return make_suite(TestHandler)
......@@ -86,5 +86,3 @@ class TestServer(TestCase):
"""Test if metadata is inserted correctly in pdf file"""
self.runUpdateMetadataList(self.UpdateMetadataScenarioList())
def test_suite():
return make_suite(TestServer)
......@@ -90,5 +90,3 @@ class TestHandler(HandlerTestCase):
# Unhandled mimetypes
self.assertEquals(get("application/pdf;ignored=param"), [])
def test_suite():
return make_suite(TestHandler)
......@@ -49,5 +49,3 @@ class TestServer(TestCase):
(open(join('data', 'test_with_png_dataurl.html')).read(), 'html', 'xyz'),
]
def test_suite():
return make_suite(TestServer)
......@@ -133,5 +133,3 @@ class TestHandler(HandlerTestCase):
[("application/vnd.openxmlformats-officedocument.presentationml.presentation", "PowerPoint 2007 Presentation"),
('application/vnd.oasis.opendocument.presentation', 'ODF Presentation Document')])
def test_suite():
return make_suite(TestHandler)
......@@ -51,5 +51,3 @@ class TestServer(TestCase):
(open(join('data', 'test.xlsx')).read(), 'xlsx', 'xyz'),
]
def test_suite():
return make_suite(TestServer)
......@@ -70,15 +70,12 @@ def run():
config = ConfigParser()
config.read(server_cloudooo_conf)
module = __import__(test_name)
if not hasattr(module, "test_suite"):
exit("No test suite to run, exiting immediately")
DAEMON = getattr(module, 'DAEMON', False)
OPENOFFICE = getattr(module, 'OPENOFFICE', False)
TestRunner = backportUnittest.TextTestRunner
suite = unittest.TestSuite()
suite.addTest(module.test_suite())
suite = unittest.defaultTestLoader.loadTestsFromModule(module)
if DAEMON:
log_file = '%s/cloudooo_test.log' % config.get('app:main',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment