Commit 31c66e2b authored by Ekaterina's avatar Ekaterina

add: embulk

parent 698c3902
[buildout]
parts =
embulk-service
embulk-config
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
bin = ${buildout:directory}/bin
var = ${buildout:directory}/var
log = ${:var}/log
scripts = ${:etc}/run
services = ${:etc}/service
promise = ${:etc}/promise/
home = ${:etc}/home
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap_connection:computer_id}
partition = ${slap_connection:partition_id}
url = ${slap_connection:server_url}
key = ${slap_connection:key_file}
cert = ${slap_connection:cert_file}
[embulk-service]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services}/embulk-service
command-line = {{ java_location }}/bin/java -jar {{ embulk_location }}/embulk.jar
run ${embulk-config:rendered} -b {{ embulkPlugins_location }}/plugins
[embulk-config]
recipe = slapos.recipe.template:jinja2
context =
key slapparameter_dict instance-parameter:configuration
template = inline:{% raw -%}
{{ slapparameter_dict['conf_text'] }}
{%- endraw %}
rendered = ${directory:etc}/config.yml
mode = 0644
[buildout]
extends =
../../stack/slapos.cfg
../../component/java/buildout.cfg
../../component/embulk/buildout.cfg
parts =
slapos-cookbook
instance-profile
java
embulk
embulkPlugins
[jinja-template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
mode = 0644
[instance-profile]
recipe = slapos.recipe.template:jinja2
md5sum = aa74da73dd13095098a6f15e2184cc2b
template = ${:_profile_base_location_}/instance.cfg.in
rendered = ${buildout:directory}/instance.cfg
mode = 0644
extensions = jinja2.ext.do
context =
section buildout buildout
key embulk_location embulk:location
key java_location java:location
key embulkPlugins_location embulkPlugins:location
[java]
<= java-re-8
[embulkPlugins]
recipe = slapos.recipe.build
install =
import os
bin_dir = os.path.join(location, 'bin')
os.makedirs(bin_dir)
os.chdir("parts/embulkPlugins")
call(["${java:location}/bin/java", "-jar", "${embulk:location}/embulk.jar",
"mkbundle", "plugins"])
os.chdir("plugins")
f = open('Gemfile', 'w')
f.write("source 'https://rubygems.org/'\n")
f.write("gem 'embulk'\n")
f.write("gem 'embulk-input-filename'\n")
f.write("gem 'embulk-output-wendelin'\n")
f.write("gem 'embulk-parser-none-bin'\n")
f.write("gem 'embulk-formatter-single_value'\n")
f.close()
call(["${java:location}/bin/java", "-jar", "${embulk:location}/embulk.jar", "bundle"])
[versions]
slapos.recipe.template = 4.4
rubygemsrecipe = 0.2.2
java = 8
Test for Embulk software release
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.embulk'
with open("README.md") as f:
long_description = f.read()
setup(
name=name,
version=version,
description="Test for SlapOS' embulk",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'requests',
],
zip_safe=True,
test_suite='test',
)
import os
import subprocess
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TransferDataTest(SlapOSInstanceTestCase):
def create_embulk_config(self, config, data_in_file_name, data_out_path):
config_string = "exec:\n max_threads: 1\n min_output_tasks: 1\n" + \
"in:\n type: filename\n path_prefix: %s\n parser:\n type: none-bin\n" + \
"out:\n type: file\n path_prefix: %s\n file_ext: wen\n formatter:\n type: single_value"
config_string = config_string % (data_in_file_name, data_out_path)
f = open(config,'w')
print >> f,config_string
f.close()
def test_transfer_data(self):
pwd = os.getcwd()
config = pwd + "/config.yml"
data_in_file_name = pwd + "/data-in.wen"
data_out_path = pwd + "/data_out"
data_in = "It works"
data_in_file = open(data_in_file_name, 'w')
data_in_file.write(data_in)
data_in_file.close()
self.create_embulk_config(config, data_in_file_name, data_out_path)
subprocess.call(['{{ java_location }}/bin/java', '-jar', '{{ embulk_location }}/embulk.jar',
'run', config, '-b', '{{ embulkPlugins_location }}/plugins'])
data_out_path = data_out_path + "000.00.wen"
data_out_file = open(data_out_path, 'rb')
data_out = data_out_file.read()
data_out_file.close()
self.assertEqual(data_in, base64.b64decode(data_out))
os.remove(config)
os.remove(data_in_file_name)
os.remove(data_out_path)
\ No newline at end of file
...@@ -152,6 +152,11 @@ setup = ${slapos-repository:location}/software/dream/test/ ...@@ -152,6 +152,11 @@ setup = ${slapos-repository:location}/software/dream/test/
egg = slapos.test.repman egg = slapos.test.repman
setup = ${slapos-repository:location}/software/repman/test/ setup = ${slapos-repository:location}/software/repman/test/
[slapos.test.embulk-setup]
<= setup-develop-egg
egg = slapos.test.embulk
setup = ${slapos-repository:location}/software/embulk/test/
[slapos.core-repository] [slapos.core-repository]
<= git-clone-repository <= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git repository = https://lab.nexedi.com/nexedi/slapos.core.git
...@@ -264,6 +269,7 @@ extra = ...@@ -264,6 +269,7 @@ extra =
${slapos.test.cloudooo-setup:setup} ${slapos.test.cloudooo-setup:setup}
${slapos.test.dream-setup:setup} ${slapos.test.dream-setup:setup}
${slapos.test.repman-setup:setup} ${slapos.test.repman-setup:setup}
${slapos.test.embulk-setup:setup}
[versions] [versions]
# slapos.core is used from the clone always # slapos.core is used from the clone always
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment