Improve multipart middleware

* don't try to close `nil` objects
* include `Dir.tmpdir` in the `Handler#allowed_paths`

This allows us to simplify multipart spec.
Make it more readable by using clear contexts and minimize the reading
noise using shared context and examples

lib/gitlab/middleware/multipart.rb

@@ -57,7 +57,8 @@ module Gitlab
 
           yield
         ensure
-          @open_files.each(&:close)
+          @open_files.compact
+                     .each(&:close)
         end
 
         # This function calls itself recursively
@@ -122,6 +123,7 @@ module Gitlab
 
         def allowed_paths
           [
+            Dir.tmpdir,
             ::FileUploader.root,
             ::Gitlab.config.uploads.storage_path,
             ::JobArtifactUploader.workhorse_upload_path,

lib/uploaded_file.rb

@@ -52,8 +52,7 @@ class UploadedFile
     elsif path.present?
       file_path = File.realpath(path)
 
-      paths = Array(upload_paths) << Dir.tmpdir
-      unless self.allowed_path?(file_path, paths.compact)
+      unless self.allowed_path?(file_path, Array(upload_paths).compact)
         raise InvalidPathError, "insecure path used '#{file_path}'"
       end
     end

spec/lib/gitlab/middleware/multipart/handler_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Middleware::Multipart::Handler do
+  using RSpec::Parameterized::TableSyntax
+
+  let_it_be(:env) { Rack::MockRequest.env_for('/', method: 'post', params: {}) }
+  let_it_be(:message) { { 'rewritten_fields' => {} } }
+
+  describe '#allowed_paths' do
+    let_it_be(:expected_allowed_paths) do
+      [
+        Dir.tmpdir,
+        ::FileUploader.root,
+        ::Gitlab.config.uploads.storage_path,
+        ::JobArtifactUploader.workhorse_upload_path,
+        ::LfsObjectUploader.workhorse_upload_path,
+        File.join(Rails.root, 'public/uploads/tmp')
+      ]
+    end
+
+    let_it_be(:expected_with_packages_path) { expected_allowed_paths + [::Packages::PackageFileUploader.workhorse_upload_path] }
+
+    subject { described_class.new(env, message).send(:allowed_paths) }
+
+    where(:package_features_enabled, :object_storage_enabled, :direct_upload_enabled, :expected_paths) do
+      false | false | true  | :expected_allowed_paths
+      false | false | false | :expected_allowed_paths
+      false | true  | true  | :expected_allowed_paths
+      false | true  | false | :expected_allowed_paths
+      true  | false | true  | :expected_with_packages_path
+      true  | false | false | :expected_with_packages_path
+      true  | true  | true  | :expected_allowed_paths
+      true  | true  | false | :expected_with_packages_path
+    end
+
+    with_them do
+      before do
+        stub_config(packages: {
+          enabled: package_features_enabled,
+          object_store: {
+            enabled: object_storage_enabled,
+            direct_upload: direct_upload_enabled
+          },
+          storage_path: '/any/dir'
+        })
+      end
+
+      it { is_expected.to eq(send(expected_paths)) }
+    end
+  end
+end

spec/lib/uploaded_file_spec.rb

@@ -23,7 +23,7 @@ RSpec.describe UploadedFile do
     end
 
     subject do
-      described_class.from_params(params, :file, upload_path, file_path_override)
+      described_class.from_params(params, :file, [upload_path, Dir.tmpdir], file_path_override)
     end
 
     context 'when valid file is specified' do

spec/support/helpers/multipart_helpers.rb

+# frozen_string_literal: true
+
+module MultipartHelpers
+  def post_env(rewritten_fields:, params:, secret:, issuer:)
+    token = JWT.encode({ 'iss' => issuer, 'rewritten_fields' => rewritten_fields }, secret, 'HS256')
+    Rack::MockRequest.env_for(
+      '/',
+      method: 'post',
+      params: params,
+      described_class::RACK_ENV_KEY => token
+    )
+  end
+
+  # This function assumes a `mode` variable to be set
+  def upload_parameters_for(filepath: nil, key: nil, filename: 'filename', remote_id: 'remote_id')
+    result = {
+      "#{key}.name" => filename,
+      "#{key}.type" => "application/octet-stream",
+      "#{key}.sha256" => "1234567890"
+    }
+
+    case mode
+    when :local
+      result["#{key}.path"] = filepath
+    when :remote
+      result["#{key}.remote_id"] = remote_id
+      result["#{key}.size"] = 3.megabytes
+    else
+      raise ArgumentError, "can't handle #{mode} mode"
+    end
+
+    result
+  end
+
+  # This function assumes a `mode` variable to be set
+  def rewritten_fields_hash(hash)
+    if mode == :remote
+      # For remote uploads, workhorse still submits rewritten_fields,
+      # but all the values are empty strings.
+      hash.keys.each { |k| hash[k] = '' }
+    end
+
+    hash
+  end
+
+  def expect_uploaded_files(uploaded_file_expectations)
+    expect(app).to receive(:call) do |env|
+      Array.wrap(uploaded_file_expectations).each do |expectation|
+        file = get_params(env).dig(*expectation[:params_path])
+        expect_uploaded_file(file, expectation)
+      end
+    end
+  end
+
+  # This function assumes a `mode` variable to be set
+  def expect_uploaded_file(file, expectation)
+    expect(file).to be_a(::UploadedFile)
+    expect(file.original_filename).to eq(expectation[:original_filename])
+    expect(file.sha256).to eq('1234567890')
+
+    case mode
+    when :local
+      expect(file.path).to eq(File.realpath(expectation[:filepath]))
+      expect(file.remote_id).to be_nil
+      expect(file.size).to eq(expectation[:size])
+    when :remote
+      expect(file.remote_id).to eq(expectation[:remote_id])
+      expect(file.path).to be_nil
+      expect(file.size).to eq(3.megabytes)
+    else
+      raise ArgumentError, "can't handle #{mode} mode"
+    end
+  end
+
+  # Rails doesn't combine the GET/POST parameters in
+  # ActionDispatch::HTTP::Parameters if action_dispatch.request.parameters is set:
+  # https://github.com/rails/rails/blob/aea6423f013ca48f7704c70deadf2cd6ac7d70a1/actionpack/lib/action_dispatch/http/parameters.rb#L41
+  def get_params(env)
+    req = ActionDispatch::Request.new(env)
+    req.GET.merge(req.POST)
+  end
+end

spec/support/shared_contexts/lib/gitlab/middleware/multipart_shared_contexts.rb

 # frozen_string_literal: true
 
-RSpec.shared_context 'multipart middleware context' do
-  let(:app) { double(:app) }
-  let(:middleware) { described_class.new(app) }
-  let(:original_filename) { 'filename' }
-
-  # Rails 5 doesn't combine the GET/POST parameters in
-  # ActionDispatch::HTTP::Parameters if action_dispatch.request.parameters is set:
-  # https://github.com/rails/rails/blob/aea6423f013ca48f7704c70deadf2cd6ac7d70a1/actionpack/lib/action_dispatch/http/parameters.rb#L41
-  def get_params(env)
-    req = ActionDispatch::Request.new(env)
-    req.GET.merge(req.POST)
+# This context provides one temporary file for the multipart spec
+#
+# Here are the available variables:
+# - uploaded_file
+# - uploaded_filepath
+# - filename
+# - remote_id
+RSpec.shared_context 'with one temporary file for multipart' do |within_tmp_sub_dir: false|
+  let(:uploaded_filepath) { uploaded_file.path }
+
+  around do |example|
+    Tempfile.open('uploaded_file2') do |tempfile|
+      @uploaded_file = tempfile
+      @filename = 'test_file.png'
+      @remote_id = 'remote_id'
+
+      example.run
+    end
   end
 
-  def post_env(rewritten_fields, params, secret, issuer)
-    token = JWT.encode({ 'iss' => issuer, 'rewritten_fields' => rewritten_fields }, secret, 'HS256')
-    Rack::MockRequest.env_for(
-      '/',
-      method: 'post',
-      params: params,
-      described_class::RACK_ENV_KEY => token
-    )
+  attr_reader :uploaded_file, :filename, :remote_id
+end
+
+# This context provides two temporary files for the multipart spec
+#
+# Here are the available variables:
+# - uploaded_file
+# - uploaded_filepath
+# - filename
+# - remote_id
+# - tmp_sub_dir (only when using within_tmp_sub_dir: true)
+# - uploaded_file2
+# - uploaded_filepath2
+# - filename2
+# - remote_id2
+RSpec.shared_context 'with two temporary files for multipart' do
+  include_context 'with one temporary file for multipart'
+
+  let(:uploaded_filepath2) { uploaded_file2.path }
+
+  around do |example|
+    Tempfile.open('uploaded_file2') do |tempfile|
+      @uploaded_file2 = tempfile
+      @filename2 = 'test_file2.png'
+      @remote_id2 = 'remote_id2'
+
+      example.run
+    end
   end
 
-  def with_tmp_dir(uploads_sub_dir, storage_path = '')
-    Dir.mktmpdir do |dir|
-      upload_dir = File.join(dir, storage_path, uploads_sub_dir)
-      FileUtils.mkdir_p(upload_dir)
+  attr_reader :uploaded_file2, :filename2, :remote_id2
+end
+
+# This context provides three temporary files for the multipart spec
+#
+# Here are the available variables:
+# - uploaded_file
+# - uploaded_filepath
+# - filename
+# - remote_id
+# - tmp_sub_dir (only when using within_tmp_sub_dir: true)
+# - uploaded_file2
+# - uploaded_filepath2
+# - filename2
+# - remote_id2
+# - uploaded_file3
+# - uploaded_filepath3
+# - filename3
+# - remote_id3
+RSpec.shared_context 'with three temporary files for multipart' do
+  include_context 'with two temporary files for multipart'
 
-      allow(Rails).to receive(:root).and_return(dir)
-      allow(Dir).to receive(:tmpdir).and_return(File.join(Dir.tmpdir, 'tmpsubdir'))
-      allow(GitlabUploader).to receive(:root).and_return(File.join(dir, storage_path))
+  let(:uploaded_filepath3) { uploaded_file3.path }
 
-      Tempfile.open('top-level', upload_dir) do |tempfile|
-        env = post_env({ 'file' => tempfile.path }, { 'file.name' => original_filename, 'file.path' => tempfile.path }, Gitlab::Workhorse.secret, 'gitlab-workhorse')
+  around do |example|
+    Tempfile.open('uploaded_file3') do |tempfile|
+      @uploaded_file3 = tempfile
+      @filename3 = 'test_file3.png'
+      @remote_id3 = 'remote_id3'
 
-        yield dir, env
-      end
+      example.run
     end
   end
+
+  attr_reader :uploaded_file3, :filename3, :remote_id3
 end

spec/support/shared_examples/lib/gitlab/middleware/multipart_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'handling all upload parameters conditions' do
+  context 'one root parameter' do
+    include_context 'with one temporary file for multipart'
+
+    let(:rewritten_fields) { rewritten_fields_hash('file' => uploaded_filepath) }
+    let(:params) { upload_parameters_for(filepath: uploaded_filepath, key: 'file', filename: filename, remote_id: remote_id) }
+
+    it 'builds an UploadedFile' do
+      expect_uploaded_files(filepath: uploaded_filepath, original_filename: filename, remote_id: remote_id, size: uploaded_file.size, params_path: %w(file))
+
+      subject
+    end
+  end
+
+  context 'two root parameters' do
+    include_context 'with two temporary files for multipart'
+
+    let(:rewritten_fields) { rewritten_fields_hash('file1' => uploaded_filepath, 'file2' => uploaded_filepath2) }
+    let(:params) do
+      upload_parameters_for(filepath: uploaded_filepath, key: 'file1', filename: filename, remote_id: remote_id).merge(
+        upload_parameters_for(filepath: uploaded_filepath2, key: 'file2', filename: filename2, remote_id: remote_id2)
+      )
+    end
+
+    it 'builds UploadedFiles' do
+      expect_uploaded_files([
+        { filepath: uploaded_filepath, original_filename: filename, remote_id: remote_id, size: uploaded_file.size, params_path: %w(file1) },
+        { filepath: uploaded_filepath2, original_filename: filename2, remote_id: remote_id2, size: uploaded_file2.size, params_path: %w(file2) }
+      ])
+
+      subject
+    end
+  end
+
+  context 'one nested parameter' do
+    include_context 'with one temporary file for multipart'
+
+    let(:rewritten_fields) { rewritten_fields_hash('user[avatar]' => uploaded_filepath) }
+    let(:params) { { 'user' => { 'avatar' => upload_parameters_for(filepath: uploaded_filepath, filename: filename, remote_id: remote_id) } } }
+
+    it 'builds an UploadedFile' do
+      expect_uploaded_files(filepath: uploaded_filepath, original_filename: filename, remote_id: remote_id, size: uploaded_file.size, params_path: %w(user avatar))
+
+      subject
+    end
+  end
+
+  context 'two nested parameters' do
+    include_context 'with two temporary files for multipart'
+
+    let(:rewritten_fields) { rewritten_fields_hash('user[avatar]' => uploaded_filepath, 'user[screenshot]' => uploaded_filepath2) }
+    let(:params) do
+      {
+        'user' => {
+          'avatar' => upload_parameters_for(filepath: uploaded_filepath, filename: filename, remote_id: remote_id),
+          'screenshot' => upload_parameters_for(filepath: uploaded_filepath2, filename: filename2, remote_id: remote_id2)
+        }
+      }
+    end
+
+    it 'builds UploadedFiles' do
+      expect_uploaded_files([
+        { filepath: uploaded_filepath, original_filename: filename, remote_id: remote_id, size: uploaded_file.size, params_path: %w(user avatar) },
+        { filepath: uploaded_filepath2, original_filename: filename2, remote_id: remote_id2, size: uploaded_file2.size, params_path: %w(user screenshot) }
+      ])
+
+      subject
+    end
+  end
+
+  context 'one deeply nested parameter' do
+    include_context 'with one temporary file for multipart'
+
+    let(:rewritten_fields) { rewritten_fields_hash('user[avatar][bananas]' => uploaded_filepath) }
+    let(:params) { { 'user' => { 'avatar' => { 'bananas' => upload_parameters_for(filepath: uploaded_filepath, filename: filename, remote_id: remote_id) } } } }
+
+    it 'builds an UploadedFile' do
+      expect_uploaded_files(filepath: uploaded_file, original_filename: filename, remote_id: remote_id, size: uploaded_file.size, params_path: %w(user avatar bananas))
+
+      subject
+    end
+  end
+
+  context 'two deeply nested parameters' do
+    include_context 'with two temporary files for multipart'
+
+    let(:rewritten_fields) { rewritten_fields_hash('user[avatar][bananas]' => uploaded_filepath, 'user[friend][ananas]' => uploaded_filepath2) }
+    let(:params) do
+      {
+        'user' => {
+          'avatar' => {
+            'bananas' => upload_parameters_for(filepath: uploaded_filepath, filename: filename, remote_id: remote_id)
+          },
+          'friend' => {
+            'ananas' => upload_parameters_for(filepath: uploaded_filepath2, filename: filename2, remote_id: remote_id2)
+          }
+        }
+      }
+    end
+
+    it 'builds UploadedFiles' do
+      expect_uploaded_files([
+        { filepath: uploaded_file, original_filename: filename, remote_id: remote_id, size: uploaded_file.size, params_path: %w(user avatar bananas) },
+        { filepath: uploaded_file2, original_filename: filename2, remote_id: remote_id2, size: uploaded_file2.size, params_path: %w(user friend ananas) }
+      ])
+
+      subject
+    end
+  end
+
+  context 'three parameters nested at different levels' do
+    include_context 'with three temporary files for multipart'
+
+    let(:rewritten_fields) do
+      rewritten_fields_hash(
+        'file' => uploaded_filepath,
+        'user[avatar]' => uploaded_filepath2,
+        'user[friend][avatar]' => uploaded_filepath3
+      )
+    end
+
+    let(:params) do
+      upload_parameters_for(filepath: uploaded_filepath, filename: filename, key: 'file', remote_id: remote_id).merge(
+        'user' => {
+          'avatar' => upload_parameters_for(filepath: uploaded_filepath2, filename: filename2, remote_id: remote_id2),
+          'friend' => {
+            'avatar' => upload_parameters_for(filepath: uploaded_filepath3, filename: filename3, remote_id: remote_id3)
+          }
+        }
+      )
+    end
+
+    it 'builds UploadedFiles' do
+      expect_uploaded_files([
+        { filepath: uploaded_filepath, original_filename: filename, remote_id: remote_id, size: uploaded_file.size, params_path: %w(file) },
+        { filepath: uploaded_filepath2, original_filename: filename2, remote_id: remote_id2, size: uploaded_file2.size, params_path: %w(user avatar) },
+        { filepath: uploaded_filepath3, original_filename: filename3, remote_id: remote_id3, size: uploaded_file3.size, params_path: %w(user friend avatar) }
+      ])
+
+      subject
+    end
+  end
+end