Merge branch 'dz-system-hooks-plugins' into 'master'

Add ability to use external plugins as system hooks

See merge request gitlab-org/gitlab-ce!17003

.codeclimate.yml

@@ -45,3 +45,4 @@ exclude_paths:
 - log/
 - backups/
 - coverage-javascript/
+- plugins/

.gitignore

@@ -66,3 +66,4 @@ eslint-report.html
 /locale/**/LC_MESSAGES
 /locale/**/*.time_stamp
 /.rspec
+/plugins/*

.rubocop.yml

@@ -17,6 +17,7 @@ AllCops:
     - 'bin/**/*'
     - 'generator_templates/**/*'
     - 'builds/**/*'
+    - 'plugins/**/*'
   CacheRootDirectory: tmp
 
 # This cop checks whether some constant value isn't a

app/services/system_hooks_service.rb

@@ -11,6 +11,8 @@ class SystemHooksService
     SystemHook.hooks_for(hooks_scope).find_each do |hook|
       hook.async_execute(data, 'system_hooks')
     end
+
+    Gitlab::Plugin.execute_all_async(data)
   end
 
   private

app/workers/all_queues.yml

@@ -84,6 +84,7 @@
 - new_note
 - pages
 - pages_domain_verification
+- plugin
 - post_receive
 - process_commit
 - project_cache

app/workers/plugin_worker.rb

+class PluginWorker
+  include ApplicationWorker
+
+  sidekiq_options retry: false
+
+  def perform(file_name, data)
+    success, message = Gitlab::Plugin.execute(file_name, data)
+
+    unless success
+      Gitlab::PluginLogger.error("Plugin Error => #{file_name}: #{message}")
+    end
+
+    true
+  end
+end

changelogs/unreleased/dz-system-hooks-plugins.yml

+---
+title: Add ability to use external plugins as an alternative to system hooks
+merge_request: 17003
+author:
+type: added

config/sidekiq_queues.yml

@@ -68,3 +68,4 @@
   - [project_migrate_hashed_storage, 1]
   - [storage_migrator, 1]
   - [pages_domain_verification, 1]
+  - [plugin, 1]

doc/administration/plugins.md

+# Plugins
+
+**Note:** Plugins must be configured on the filesystem of the GitLab
+server. Only GitLab server administrators will be able to complete these tasks.
+Please explore [system hooks] or [webhooks] as an option if you do not
+have filesystem access. 
+
+Introduced in GitLab 10.6.
+
+A plugin will run on each event so it's up to you to filter events or projects within a plugin code. You can have as many plugins as you want. Each plugin will be triggered by GitLab asynchronously in case of an event. For a list of events please see [system hooks] documentation.
+
+## Setup
+
+Plugins must be placed directly into `plugins` directory, subdirectories will be ignored. 
+There is an `example` directory inside `plugins` where you can find some basic examples. 
+
+Follow the steps below to set up a custom hook:
+
+1. On the GitLab server, navigate to the project's plugin directory.
+   For an installation from source the path is usually
+   `/home/git/gitlab/plugins/`. For Omnibus installs the path is
+   usually `/opt/gitlab/embedded/service/gitlab-rails/plugins`.
+1. Inside the `plugins` directory, create a file with a name of your choice, but without spaces or special characters.
+1. Make the hook file executable and make sure it's owned by the git user.
+1. Write the code to make the plugin function as expected. Plugin can be
+   in any language. Ensure the 'shebang' at the top properly reflects the language
+   type. For example, if the script is in Ruby the shebang will probably be
+   `#!/usr/bin/env ruby`.
+1. The data to the plugin will be provided as JSON on STDIN. It will be exactly same as one for [system hooks]
+
+That's it! Assuming the plugin code is properly implemented the hook will fire
+as appropriate. Plugins file list is updated for each event. There is no need to restart GitLab to apply a new plugin.
+
+If a plugin executes with non-zero exit code or GitLab fails to execute it, a
+message will be logged to `plugin.log`.
+
+## Validation
+
+Writing own plugin can be tricky and its easier if you can check it without altering the system. 
+We provided a rake task you can use with staging environment to test your plugin before using it in production. 
+The rake task will use a sample data and execute each of plugins. By output you should be able to determine if 
+system sees your plugin and if it was executed without errors.
+
+```bash
+# Omnibus installations
+sudo gitlab-rake plugins:validate
+
+# Installations from source
+bundle exec rake plugins:validate RAILS_ENV=production
+```
+
+Example of output can be next: 
+
+```
+-> bundle exec rake plugins:validate RAILS_ENV=production
+Validating plugins from /plugins directory
+* /home/git/gitlab/plugins/save_to_file.clj succeed (zero exit code)
+* /home/git/gitlab/plugins/save_to_file.rb failure (non-zero exit code)
+```
+
+[hooks]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#Server-Side-Hooks
+[system hooks]: ../system_hooks/system_hooks.md
+[webhooks]: ../user/project/integrations/webhooks.md
+[5073]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5073
+[93]: https://gitlab.com/gitlab-org/gitlab-shell/merge_requests/93
+

lib/gitlab/plugin.rb

+module Gitlab
+  module Plugin
+    def self.files
+      Dir.glob(Rails.root.join('plugins/*')).select do |entry|
+        File.file?(entry)
+      end
+    end
+
+    def self.execute_all_async(data)
+      args = files.map { |file| [file, data] }
+
+      PluginWorker.bulk_perform_async(args)
+    end
+
+    def self.execute(file, data)
+      result = Gitlab::Popen.popen_with_detail([file]) do |stdin|
+        stdin.write(data.to_json)
+      end
+
+      exit_status = result.status&.exitstatus
+      [exit_status.zero?, result.stderr]
+    rescue => e
+      [false, e.message]
+    end
+  end
+end

lib/gitlab/plugin_logger.rb

+module Gitlab
+  class PluginLogger < Gitlab::Logger
+    def self.file_name_noext
+      'plugin'
+    end
+  end
+end

lib/tasks/plugins.rake

+namespace :plugins do
+  desc 'Validate existing plugins'
+  task validate: :environment do
+    puts 'Validating plugins from /plugins directory'
+
+    Gitlab::Plugin.files.each do |file|
+      success, message = Gitlab::Plugin.execute(file, Gitlab::DataBuilder::Push::SAMPLE_DATA)
+
+      if success
+        puts "* #{file} succeed (zero exit code)."
+      else
+        puts "* #{file} failure (non-zero exit code). #{message}"
+      end
+    end
+  end
+end

plugins/examples/save_to_file.clj

+#!/usr/bin/env clojure
+(let [in (slurp *in*)]
+  (spit "/tmp/clj-data.txt" in))

plugins/examples/save_to_file.rb

+#!/usr/bin/env ruby
+x = STDIN.read
+File.write('/tmp/rb-data.txt', x)

spec/lib/gitlab/plugin_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Plugin do
+  describe '.execute' do
+    let(:data) { Gitlab::DataBuilder::Push::SAMPLE_DATA }
+    let(:plugin) { Rails.root.join('plugins', 'test.rb') }
+    let(:tmp_file) { Tempfile.new('plugin-dump') }
+    let(:result) { described_class.execute(plugin.to_s, data) }
+    let(:success) { result.first }
+    let(:message) { result.last }
+
+    let(:plugin_source) do
+      <<~EOS
+        #!/usr/bin/env ruby
+        x = STDIN.read
+        File.write('#{tmp_file.path}', x)
+      EOS
+    end
+
+    before do
+      File.write(plugin, plugin_source)
+    end
+
+    after do
+      FileUtils.rm(plugin)
+    end
+
+    context 'successful execution' do
+      before do
+        File.chmod(0o777, plugin)
+      end
+
+      after do
+        tmp_file.close!
+      end
+
+      it { expect(success).to be true }
+      it { expect(message).to be_empty }
+
+      it 'ensures plugin received data via stdin' do
+        result
+
+        expect(File.read(tmp_file.path)).to eq(data.to_json)
+      end
+    end
+
+    context 'non-executable' do
+      it { expect(success).to be false }
+      it { expect(message).to include('Permission denied') }
+    end
+
+    context 'non-zero exit' do
+      let(:plugin_source) do
+        <<~EOS
+          #!/usr/bin/env ruby
+          exit 1
+        EOS
+      end
+
+      before do
+        File.chmod(0o777, plugin)
+      end
+
+      it { expect(success).to be false }
+      it { expect(message).to be_empty }
+    end
+  end
+end

spec/workers/plugin_worker_spec.rb

+require 'spec_helper'
+
+describe PluginWorker do
+  include RepoHelpers
+
+  let(:filename) { 'my_plugin.rb' }
+  let(:data) { { 'event_name' => 'project_create' } }
+
+  subject { described_class.new }
+
+  describe '#perform' do
+    it 'executes Gitlab::Plugin with expected values' do
+      allow(Gitlab::Plugin).to receive(:execute).with(filename, data).and_return([true, ''])
+
+      expect(subject.perform(filename, data)).to be_truthy
+    end
+
+    it 'logs message in case of plugin execution failure' do
+      allow(Gitlab::Plugin).to receive(:execute).with(filename, data).and_return([false, 'permission denied'])
+
+      expect(Gitlab::PluginLogger).to receive(:error)
+      expect(subject.perform(filename, data)).to be_truthy
+    end
+  end
+end