Imported shacache.

slapos/slapos.tool.shacache/README.txt

+Introduction
+===================
+
+The network cache server is a NoSQL storage with a REST API.
+
+
+How does it works
+===================
+                          _______________
+                         /               \
+                         |               |
+                  ------>| NETWORKCACHED |
+                  |  ----|               |<-----
+                  |  |   \_______________/     |
+        GET /key  |  |                         |
+                  |  | File                    | PUT / <- data
+                __|__v______             ______|_____
+               |            |           |            |
+               |   Client   |           |   Client   |
+               |____________|           |____________|
+
+
+Basically, the networkcached archives the files through HTTP PUT method.
+When a client want to download the file it just need to provide the key
+value and the server will send a response with the file data.
+
+API:
+ PUT / : 
+   parameter: file uploaded 
+   Used to upload/modify an entry
+
+ GET /<key>
+   Return raw content
+   Raise HTTP error (404) if key does not exist
+
+
+Installation
+==============
+$ python2.6 setup.py install
+
+Now it is time to create the 'networkcached.conf' file in /etc/networkcached.conf
+directory, using your preferred text editor (gedit, kate, vim.).
+Follow text shall be put in this file:
+
+[networkcached]
+host = 127.0.0.1
+port = 5001
+cache_base_folder = /var/cache/networkcached/
+
+Run the server:
+# networkcached networkcached.conf
+
+
+Setup Develoment Environment
+===============================
+
+$ mkdir -p ~/networkcached/downloads
+$ cd ~/networkcached
+
+Now it is time to create 'buildout.cfg' file in ~/networkcached directory,
+using your preferred text editor (gedit, kate, vim.).
+Follow text shall be put in this file:
+
+[buildout]
+extensions = mr.developer
+auto-checkout = slapos.tool.networkcached
+download-cache = /nexedi/buildout-networkcached/downloads
+eggs-directory = /nexedi/buildout-networkcached/eggs
+
+parts =
+ networkcached
+
+[sources]
+slapos.tool.networkcached = svn https://svn.erp5.org/repos/vifib/trunk/utils/slapos.tool.networkcached
+
+[networkcached]
+recipe = zc.recipe.egg
+eggs = 
+  slapos.tool.networkcached
+
+Now you bootstrap the buildout:
+$ python -S -c 'import urllib;print urllib.urlopen(\
+     "http://svn.zope.org/*checkout*/zc.buildout/trunk/bootstrap/bootstrap.py"\
+     ).read()' | python -S -
+
+Run the buildout:
+$ bin/buildout -v
+
+Now it is time to create the 'networkcached.conf' file in ~/networkcached
+directory, using your preferred text editor (gedit, kate, vim.).
+Follow text shall be put in this file:
+
+[networkcached]
+host = 127.0.0.1
+port = 5001
+cache_base_folder = ~/networkcached/networkcached-database
+
+Now you can start your networkcached server:
+$ bin/networkcached networkcached.conf
+or 
+$ bin/networkcached -d networkcached-database -a 127.0.0.1 -p 5002

slapos/slapos.tool.shacache/setup.cfg

+[egg_info]
+tag_build = .dev
+tag_svn_revision = 1

slapos/slapos.tool.shacache/setup.py

+from setuptools import setup, find_packages
+import os
+
+name = "slapos.tool.networkcached"
+version = '1.1'
+
+def read(*rnames):
+  return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+
+long_description=(
+        read('README.txt')
+        + '\n' +
+        read('CHANGES.txt')
+    )
+
+setup(
+    name = name,
+    version = version,
+    description = "networkcached - The networkcached is cache provider.",
+    long_description=long_description,
+    license = "GPLv3",
+    keywords = "vifib slapos cache",
+    classifiers=[
+      ],
+    packages = find_packages('src'),
+    include_package_data = True,
+    package_dir = {'':'src'},
+    namespace_packages = ['slapos', 'slapos.tool'],
+    install_requires = [
+      'Flask', # used to create this
+      'setuptools', # namespaces
+    ],
+    zip_safe=False,
+    entry_points = """
+    [console_scripts]
+    networkcached = %(name)s:main
+    runNetorkcacheTest = %(name)s.test:run
+    """ % dict(name=name),
+    )

slapos/slapos.tool.shacache/src/slapos/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.tool.shacache/src/slapos/tool/__init__.py

+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
+

slapos/slapos.tool.shacache/src/slapos/tool/networkcached/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 ViFiB SARL and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+
+import hashlib
+import os
+import sys
+import traceback
+from flask import Flask, request, helpers, abort
+from config import NetworkcacheConfiguration, NetworkcacheParser
+
+
+app = Flask(__name__)
+
+
+@app.route('/', methods=['PUT'])
+def put():
+  """ Save the file on the cache server. """
+  if getattr(request, 'data', None) is None:
+    abort(400, 'PUT requires data.')
+
+  cache_base_folder = app.config.get('CACHE_BASE_FOLDER')
+  file_name = hashlib.sha512(request.data).hexdigest()
+  try:
+    f = open(os.path.join(cache_base_folder, file_name), "w+")
+    try:
+      f.write(request.data)
+    finally:
+      f.close()
+  except:
+    app.logger.info(traceback.format_exc())
+    abort(500, "Faile to upload the file.")
+
+  return 'Success'
+
+
+@app.route('/<key>', methods=['GET'])
+def get(key):
+  """ Return the file if found, otherwise 404 is raised. """
+  cache_base_folder = app.config.get('CACHE_BASE_FOLDER')
+  file_path = os.path.join(cache_base_folder, key)
+  if not os.path.exists(file_path):
+    abort(404, 'File not found.')
+
+  return helpers.send_file(file_path)
+
+
+def main():
+  "Run default configuration."
+  usage = "usage: %s [options] CONFIGURATION_FILE" % sys.argv[0]
+
+  try:
+    # Parse arguments
+    configuration = NetworkcacheConfiguration()
+    configuration.setConfig(*NetworkcacheParser(usage=usage).check_args())
+
+    app.config['CACHE_BASE_FOLDER'] = configuration.cache_base_folder
+    app.run(host=configuration.host, port=int(configuration.port), debug=True)
+    return_code = 0
+  except SystemExit, err:
+    # Catch exception raise by optparse
+    return_code = err
+
+  sys.exit(return_code)

slapos/slapos.tool.shacache/src/slapos/tool/networkcached/config.py

+##############################################################################
+#
+# Copyright (c) 2010 ViFiB SARL and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+
+import os
+import logging
+import logging.handlers
+import ConfigParser
+from optparse import OptionParser, Option
+
+
+class NetworkcacheParser(OptionParser):
+  """
+  Parse all arguments.
+  """
+  def __init__(self, usage=None, version=None):
+    """
+    Initialize all options possibles.
+    """
+    OptionParser.__init__(self, usage=usage, version=version,
+                          option_list=[
+      Option("-l", "--log_file",
+             help="The path to the log file used by the script.",
+             type=str),
+      Option("-v", "--verbose",
+             default=False,
+             action="store_true",
+             help="Verbose output."),
+      Option("-c", "--console",
+             default=False,
+             action="store_true",
+             help="Console output."),
+      Option("-d", "--cache_base_folder",
+             type=str,
+             help="Folder to store the files."),
+      Option("-a", "--host",
+             type=str,
+             help="Host address."),
+      Option("-p", "--port",
+             type=str,
+             help="Port number."),
+    ])
+
+  def check_args(self):
+    """
+      Check arguments
+    """
+    (options, args) = self.parse_args()
+    configuration_file_path = []
+
+    if len(args) > 1:
+      self.error("Incorrect number of arguments")
+
+    if len(args) == 1:
+      configuration_file_path = args[0]
+
+    return options, configuration_file_path
+
+
+class NetworkcacheConfiguration:
+  def setConfig(self, option_dict, configuration_file_path):
+    """
+    Set options given by parameters.
+    """
+    # Set options parameters
+    for option, value in option_dict.__dict__.items():
+      setattr(self, option, value)
+
+    # Load configuration file
+    if configuration_file_path:
+      configuration_parser = ConfigParser.SafeConfigParser()
+      configuration_parser.read(configuration_file_path)
+      # Merges the arguments and configuration
+      for section in ("networkcached",):
+        configuration_dict = dict(configuration_parser.items(section))
+        for key in configuration_dict:
+          if not getattr(self, key, None):
+            setattr(self, key, configuration_dict[key])
+
+    # set up logging
+    self.logger = logging.getLogger("networkcached")
+    self.logger.setLevel(logging.INFO)
+    if self.console:
+      self.logger.addHandler(logging.StreamHandler())
+
+    # cache base folder is required
+    if not self.cache_base_folder:
+      raise ValueError('cache_base_folder is required.')
+
+    if self.log_file:
+      if not os.path.isdir(os.path.dirname(self.log_file)):
+        # fallback to console only if directory for logs does not exists and
+        # continue to run
+        raise ValueError('Please create directory %r to store %r log file' % (
+          os.path.dirname(self.log_file), self.log_file))
+      else:
+        file_handler = logging.FileHandler(self.log_file)
+        format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        file_handler.setFormatter(logging.Formatter(format))
+        self.logger.addHandler(file_handler)
+        self.logger.info('Configured logging to file %r' % self.log_file)
+
+    self.logger.info("Started.")
+    if self.verbose:
+      self.logger.setLevel(logging.DEBUG)
+      self.logger.debug("Verbose mode enabled.")

slapos/slapos.tool.shacache/src/slapos/tool/networkcached/networkcached.conf.sample

+[networkcached]
+host = 127.0.0.1
+port = 5001
+cache_base_folder = /opt/slapos/networkcached

slapos/slapos.tool.shacache/src/slapos/tool/networkcached/test.py

+##############################################################################
+#
+# Copyright (c) 2010 ViFiB SARL and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+
+import hashlib
+import os
+import shutil
+import slapos.tool.networkcached as networkcached
+import tempfile
+import unittest
+
+
+class NetworkcacheTestCase(unittest.TestCase):
+
+  def setUp(self):
+    networkcached.app.config['CACHE_BASE_FOLDER'] = tempfile.mkdtemp()
+    self.cache_base_folder = networkcached.app.config['CACHE_BASE_FOLDER']
+    self.app = networkcached.app.test_client()
+    self.file_content = 'test file content.'
+
+  def tearDown(self):
+    shutil.rmtree(self.cache_base_folder)
+
+  def create_file_into_cache_base_folder(self):
+    """
+      Create a binary file into the directory.
+    """
+    file_name = hashlib.sha512(self.file_content).hexdigest()
+    file_path = os.path.join(self.cache_base_folder, file_name)
+    f = open(file_path, 'w')
+    try:
+      f.write(self.file_content)
+    finally:
+      f.close()
+    return file_path
+
+  def test_get_non_existing_file(self):
+    """
+      When the client tries to download a file which does not exists, it must
+      return a 404 error.
+    """
+    result = self.app.get('/does_not_exists')
+    self.assertEquals('404 NOT FOUND', result.status)
+    self.assertTrue('File not found.' in result.data)
+
+  def test_get_existing_file(self):
+    """
+      Check if the file is returned if it exists.
+    """
+    file_path = self.create_file_into_cache_base_folder()
+    result = self.app.get('/%s' % file_path.split('/')[-1])
+    self.assertEquals('200 OK', result.status)
+    self.assertEquals('application/octet-stream',
+                            result.headers['Content-Type'])
+    self.assertEquals(self.file_content, result.data)
+
+  def test_puti_file(self):
+    """
+      Check if the file will be upload just fine.
+    """
+    file_content = 'file does not exist yet'
+    file_name = hashlib.sha512(file_content).hexdigest()
+
+    result = self.app.put('/', data=file_content)
+    self.assertEquals('200 OK', result.status)
+    self.assertEquals('Success', result.data)
+
+    f = open(os.path.join(self.cache_base_folder, file_name), 'r')
+    try:
+      data = f.read()
+    finally:
+      f.close()
+    self.assertEquals(file_content, data)
+
+
+def run():
+  unittest.main(module=networkcached.test)
+
+if __name__ == '__main__':
+  run()