add: embulk

software/embulk/instance.cfg.in

+[buildout]
+parts =
+  embulk-service
+  embulk-config
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+var = ${buildout:directory}/var
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+promise = ${:etc}/promise/
+home = ${:etc}/home
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap_connection:computer_id}
+partition = ${slap_connection:partition_id}
+url = ${slap_connection:server_url}
+key = ${slap_connection:key_file}
+cert = ${slap_connection:cert_file}
+
+[embulk-service]
+recipe  = slapos.cookbook:wrapper
+wrapper-path    = ${directory:services}/embulk-service
+command-line    = {{ java_location }}/bin/java -jar {{ embulk_location }}/embulk.jar 
+                    run ${embulk-config:rendered} -b {{ embulkPlugins_location }}/plugins
+
+[embulk-config]
+recipe  = slapos.recipe.template:jinja2
+context =
+  key slapparameter_dict instance-parameter:configuration
+template = inline:{% raw -%} 
+  {{ slapparameter_dict['conf_text'] }} 
+  {%- endraw %}
+rendered = ${directory:etc}/config.yml
+mode = 0644

software/embulk/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../../component/java/buildout.cfg
+  ../../component/embulk/buildout.cfg
+
+parts =
+  slapos-cookbook
+  instance-profile
+  java
+  embulk
+  embulkPlugins
+
+[jinja-template]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+mode = 0644
+
+[instance-profile]
+recipe = slapos.recipe.template:jinja2
+md5sum = aa74da73dd13095098a6f15e2184cc2b
+template = ${:_profile_base_location_}/instance.cfg.in
+rendered = ${buildout:directory}/instance.cfg
+mode = 0644
+extensions = jinja2.ext.do
+context =
+  section buildout  buildout
+  key embulk_location embulk:location
+  key java_location java:location
+  key embulkPlugins_location embulkPlugins:location
+
+[java]
+<= java-re-8
+
+[embulkPlugins]
+recipe = slapos.recipe.build
+install =
+  import os
+  bin_dir = os.path.join(location, 'bin')
+  os.makedirs(bin_dir)
+  os.chdir("parts/embulkPlugins")
+  call(["${java:location}/bin/java", "-jar", "${embulk:location}/embulk.jar",
+          "mkbundle", "plugins"])
+  os.chdir("plugins")
+  f = open('Gemfile', 'w')
+  f.write("source 'https://rubygems.org/'\n")
+  f.write("gem 'embulk'\n")
+  f.write("gem 'embulk-input-filename'\n")
+  f.write("gem 'embulk-output-wendelin'\n")
+  f.write("gem 'embulk-parser-none-bin'\n")
+  f.write("gem 'embulk-formatter-single_value'\n")
+  f.close()
+  call(["${java:location}/bin/java", "-jar", "${embulk:location}/embulk.jar", "bundle"])
+
+[versions]
+slapos.recipe.template = 4.4
+rubygemsrecipe  = 0.2.2
+java = 8

software/embulk/test/README.md

+Test for Embulk software release

software/embulk/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.embulk'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' embulk",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/embulk/test/test.py

+import os
+import subprocess
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TransferDataTest(SlapOSInstanceTestCase):
+  
+  def create_embulk_config(self, config, data_in_file_name, data_out_path):
+    config_string = "exec:\n max_threads: 1\n min_output_tasks: 1\n" + \
+      "in:\n type: filename\n path_prefix: %s\n parser:\n  type: none-bin\n" + \
+      "out:\n type: file\n path_prefix: %s\n file_ext: wen\n formatter:\n  type: single_value"
+      
+    config_string = config_string % (data_in_file_name, data_out_path)
+    f = open(config,'w')
+    print >> f,config_string
+    f.close()
+  
+  def test_transfer_data(self):
+    pwd = os.getcwd()
+    config = pwd + "/config.yml"
+    data_in_file_name = pwd + "/data-in.wen"
+    data_out_path = pwd + "/data_out"
+    
+    data_in = "It works"
+    data_in_file = open(data_in_file_name, 'w')
+    data_in_file.write(data_in)
+    data_in_file.close()
+    
+    self.create_embulk_config(config, data_in_file_name, data_out_path)
+    subprocess.call(['{{ java_location }}/bin/java', '-jar', '{{ embulk_location }}/embulk.jar', 
+      'run', config, '-b', '{{ embulkPlugins_location }}/plugins'])
+    
+    data_out_path = data_out_path + "000.00.wen"
+    data_out_file = open(data_out_path, 'rb')
+    data_out = data_out_file.read()
+    data_out_file.close()
+
+    self.assertEqual(data_in, base64.b64decode(data_out))
+    
+    os.remove(config)
+    os.remove(data_in_file_name)
+    os.remove(data_out_path)
+    
+                    
\ No newline at end of file

software/slapos-sr-testing/software.cfg

@@ -152,6 +152,11 @@ setup = ${slapos-repository:location}/software/dream/test/
 egg = slapos.test.repman
 setup = ${slapos-repository:location}/software/repman/test/
 
+[slapos.test.embulk-setup]
+<= setup-develop-egg
+egg = slapos.test.embulk
+setup = ${slapos-repository:location}/software/embulk/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -264,6 +269,7 @@ extra =
   ${slapos.test.cloudooo-setup:setup}
   ${slapos.test.dream-setup:setup}
   ${slapos.test.repman-setup:setup}
+  ${slapos.test.embulk-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always