Add latest changes from gitlab-org/gitlab@13-5-stable-ee

GITLAB_SHELL_VERSION

-13.10.0
+13.11.0

app/assets/javascripts/jobs/components/log/line.vue

 <script>
-import linkifyHtml from 'linkifyjs/html';
-import { sanitize } from '~/lib/dompurify';
-import { isAbsolute } from '~/lib/utils/url_utility';
 import LineNumber from './line_number.vue';
 
-const linkifyOptions = {
-  attributes: {
-    // eslint-disable-next-line @gitlab/require-i18n-strings
-    rel: 'nofollow noopener',
-  },
-  className: 'gl-reset-color!',
-  defaultProtocol: 'https',
-  validate: {
-    email: false,
-    url(value) {
-      return isAbsolute(value);
-    },
-  },
-};
-
 export default {
   functional: true,
   props: {
@@ -35,15 +17,13 @@ export default {
     const { line, path } = props;
 
     const chars = line.content.map(content => {
-      const linkfied = linkifyHtml(content.text, linkifyOptions);
-      return h('span', {
-        class: ['gl-white-space-pre-wrap', content.style],
-        domProps: {
-          innerHTML: sanitize(linkfied, {
-            ALLOWED_TAGS: ['a'],
-          }),
+      return h(
+        'span',
+        {
+          class: ['gl-white-space-pre-wrap', content.style],
         },
-      });
+        content.text,
+      );
     });
 
     return h('div', { class: 'js-line log-line' }, [

changelogs/unreleased/update-gitlab-shell-13-11-0.yml

+---
+title: Update GitLab Shell to v13.11.0
+merge_request: 45660
+author:
+type: other

doc/api/repositories.md

@@ -112,10 +112,12 @@ Parameters:
 
 ## Get file archive
 
+> Support for [including Git LFS blobs](../topics/git/lfs/index.md#lfs-objects-in-project-archives) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15079) in GitLab 13.5.
+
 Get an archive of the repository. This endpoint can be accessed without
 authentication if the repository is publicly accessible.
 
-This endpoint has a rate limit threshold of 5 requests per minute.
+This endpoint has a rate limit threshold of 5 requests per minute for GitLab.com users.
 
 ```plaintext
 GET /projects/:id/repository/archive[.format]

doc/development/lfs.md

@@ -10,3 +10,74 @@ and the slides on [Google Slides](https://docs.google.com/presentation/d/1E-aw6-
 and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/07a89257a140db067bdfb484aecd35e1/Git_LFS_Deep_Dive__Create_.pdf).
 Everything covered in this deep dive was accurate as of GitLab 11.10, and while specific
 details may have changed since then, it should still serve as a good introduction.
+
+## Including LFS blobs in project archives
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15079) in GitLab 13.5.
+
+The following diagram illustrates how GitLab resolves LFS files for project archives:
+
+```mermaid
+sequenceDiagram
+    autonumber
+    Client->>+Workhorse: GET /group/project/-/archive/master.zip
+    Workhorse->>+Rails: GET /group/project/-/archive/master.zip
+    Rails->>+Workhorse: Gitlab-Workhorse-Send-Data git-archive
+    Workhorse->>Gitaly: SendArchiveRequest
+    Gitaly->>Git: git archive master
+    Git->>Smudge: OID 12345
+    Smudge->>+Workhorse: GET /internal/api/v4/lfs?oid=12345&gl_repository=project-1234
+    Workhorse->>+Rails: GET /internal/api/v4/lfs?oid=12345&gl_repository=project-1234
+    Rails->>+Workhorse: Gitlab-Workhorse-Send-Data send-url
+    Workhorse->>Smudge: <LFS data>
+    Smudge->>Git: <LFS data>
+    Git->>Gitaly: <streamed data>
+    Gitaly->>Workhorse: <streamed data>
+    Workhorse->>Client: master.zip
+```
+
+1. The user requests the project archive from the UI.
+1. Workhorse forwards this request to Rails.
+1. If the user is authorized to download the archive, Rails replies with
+an HTTP header of `Gitlab-Workhorse-Send-Data` with a base64-encoded
+JSON payload prefaced with `git-archive`. This payload includes the
+`SendArchiveRequest` binary message, which is encoded again in base64.
+1. Workhorse decodes the `Gitlab-Workhorse-Send-Data` payload. If the
+archive already exists in the archive cache, Workhorse sends that
+file. Otherwise, Workhorse sends the `SendArchiveRequest` to the
+appropriate Gitaly server.
+1. The Gitaly server will call `git archive <ref>` to begin generating
+the Git archive on-the-fly. If the `include_lfs_blobs` flag is enabled,
+Gitaly enables a custom LFS smudge filter via the `-c
+filter.lfs.smudge=/path/to/gitaly-lfs-smudge` Git option.
+1. When `git` identifies a possible LFS pointer using the
+`.gitattributes` file, `git` calls `gitaly-lfs-smudge` and provides the
+LFS pointer via the standard input. Gitaly provides `GL_PROJECT_PATH`
+and `GL_INTERNAL_CONFIG` as environment variables to enable lookup of
+the LFS object.
+1. If a valid LFS pointer is decoded, `gitaly-lfs-smudge` makes an
+internal API call to Workhorse to download the LFS object from GitLab.
+1. Workhorse forwards this request to Rails. If the LFS object exists
+and is associated with the project, Rails sends `ArchivePath` either
+with a path where the LFS object resides (for local disk) or a
+pre-signed URL (when object storage is enabled) via the
+`Gitlab-Workhorse-Send-Data` HTTP header with a payload prefaced with
+`send-url`.
+1. Workhorse retrieves the file and send it to the `gitaly-lfs-smudge`
+process, which writes the contents to the standard output.
+1. `git` reads this output and sends it back to the Gitaly process.
+1. Gitaly sends the data back to Rails.
+1. The archive data is sent back to the client.
+
+In step 7, the `gitaly-lfs-smudge` filter must talk to Workhorse, not to
+Rails, or an invalid LFS blob will be saved. To support this, GitLab
+13.5 [changed the default Omnibus configuration to have Gitaly talk to
+the Workhorse](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4592)
+instead of Rails.
+
+One side effect of this change: the correlation ID of the original
+request is not preserved for the internal API requests made by Gitaly
+(or `gitaly-lfs-smudge`), such as the one made in step 8. The
+correlation IDs for those API requests will be random values until [this
+Workhorse issue](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/309) is
+resolved.

doc/topics/git/lfs/index.md

@@ -42,7 +42,6 @@ Documentation for GitLab instance administrators is under [LFS administration do
   credentials store is recommended
 - Git LFS always assumes HTTPS so if you have GitLab server on HTTP you will have
   to add the URL to Git configuration manually (see [troubleshooting](#troubleshooting))
-- Files added using Git LFS are [not included in the archives created using "download zip" functionality](https://gitlab.com/gitlab-org/gitlab/-/issues/15079)
 
 NOTE: **Note:**
 With 8.12 GitLab added LFS support to SSH. The Git LFS communication
@@ -112,6 +111,51 @@ To remove objects from LFS:
 
 See the documentation on [File Locking](../../../user/project/file_lock.md).
 
+## LFS objects in project archives
+
+> - Support for including Git LFS blobs inside [project source downloads](../../../user/project/repository/index.md) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15079) in GitLab 13.5.
+> - It's [deployed behind a feature flag](../../../user/feature_flags.md), disabled by default.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-lfs-objects-in-project-archives). **(CORE ONLY)**
+
+CAUTION: **Warning:**
+This feature might not be available to you. Check the **version history** note above for details.
+
+Prior to GitLab 13.5, [project source
+downloads](../../../user/project/repository/index.md) would include Git
+LFS pointers instead of the actual objects. For example, LFS pointers
+look like the following:
+
+```markdown
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ea5dd307f195f449f0e08234183b82e92c3d5f4cff11c2a6bb014f9e0de12aa
+size 177735
+```
+
+Starting with GitLab 13.5, these pointers are converted to the uploaded
+LFS object if the `include_lfs_blobs_in_archive` feature flag is
+enabled.
+
+Technical details about how this works can be found in the [development documentation for LFS](../../../development/lfs.md#including-lfs-blobs-in-project-archives).
+
+### Enable or disable LFS objects in project archives
+
+_LFS objects in project archives_ is under development and not ready for production use. It is
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md)
+can enable it.
+
+To enable it:
+
+```ruby
+Feature.enable(:include_lfs_blobs_in_archive)
+```
+
+To disable it:
+
+```ruby
+Feature.disable(:include_lfs_blobs_in_archive)
+```
+
 ## Troubleshooting
 
 ### error: Repository or object not found

doc/user/project/repository/index.md

@@ -247,6 +247,7 @@ used for cloning your project. The button is only shown on macOS.
 ## Download Source Code
 
 > Support for directory download was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/24704) in GitLab 11.11.
+> Support for [including Git LFS blobs](../../../topics/git/lfs#lfs-objects-in-project-archives) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15079) in GitLab 13.5.
 
 The source code stored in a repository can be downloaded from the UI.
 By clicking the download icon, a dropdown will open with links to download the following:

package.json

@@ -105,7 +105,6 @@
     "jszip": "^3.1.3",
     "jszip-utils": "^0.0.2",
     "katex": "^0.10.0",
-    "linkifyjs": "^2.1.9",
     "lodash": "^4.17.20",
     "marked": "^0.3.12",
     "mermaid": "^8.5.2",

spec/frontend/jobs/components/log/line_spec.js

@@ -2,25 +2,21 @@ import { shallowMount } from '@vue/test-utils';
 import Line from '~/jobs/components/log/line.vue';
 import LineNumber from '~/jobs/components/log/line_number.vue';
 
-const httpUrl = 'http://example.com';
-const httpsUrl = 'https://example.com';
-
-const mockProps = ({ text = 'Running with gitlab-runner 12.1.0 (de7731dd)' } = {}) => ({
-  line: {
-    content: [
-      {
-        text,
-        style: 'term-fg-l-green',
-      },
-    ],
-    lineNumber: 0,
-  },
-  path: '/jashkenas/underscore/-/jobs/335',
-});
-
 describe('Job Log Line', () => {
   let wrapper;
-  let data;
+
+  const data = {
+    line: {
+      content: [
+        {
+          text: 'Running with gitlab-runner 12.1.0 (de7731dd)',
+          style: 'term-fg-l-green',
+        },
+      ],
+      lineNumber: 0,
+    },
+    path: '/jashkenas/underscore/-/jobs/335',
+  };
 
   const createComponent = (props = {}) => {
     wrapper = shallowMount(Line, {
@@ -31,7 +27,6 @@ describe('Job Log Line', () => {
   };
 
   beforeEach(() => {
-    data = mockProps();
     createComponent(data);
   });
 
@@ -50,38 +45,4 @@ describe('Job Log Line', () => {
   it('renders the provided style as a class attribute', () => {
     expect(wrapper.find('span').classes()).toContain(data.line.content[0].style);
   });
-
-  describe('when the line contains a link', () => {
-    const findLink = () => wrapper.find('span a');
-
-    it('renders an http link', () => {
-      createComponent(mockProps({ text: httpUrl }));
-
-      expect(findLink().text()).toBe(httpUrl);
-      expect(findLink().attributes().href).toEqual(httpUrl);
-    });
-
-    it('renders an https link', () => {
-      createComponent(mockProps({ text: httpsUrl }));
-
-      expect(findLink().text()).toBe(httpsUrl);
-      expect(findLink().attributes().href).toEqual(httpsUrl);
-    });
-
-    it('renders a link with rel nofollow and noopener', () => {
-      createComponent(mockProps({ text: httpsUrl }));
-
-      expect(findLink().attributes().rel).toBe('nofollow noopener');
-    });
-
-    test.each`
-      type           | text
-      ${'ftp'}       | ${'ftp://example.com/file'}
-      ${'email'}     | ${'email@example.com'}
-      ${'no scheme'} | ${'example.com/page'}
-    `('does not render a $type link', ({ text }) => {
-      createComponent(mockProps({ text }));
-      expect(findLink().exists()).toBe(false);
-    });
-  });
 });

yarn.lock

@@ -7550,11 +7550,6 @@ linkify-it@^2.0.0:
   dependencies:
     uc.micro "^1.0.1"
 
-linkifyjs@^2.1.9:
-  version "2.1.9"
-  resolved "https://registry.yarnpkg.com/linkifyjs/-/linkifyjs-2.1.9.tgz#af06e45a2866ff06c4766582590d098a4d584702"
-  integrity sha512-74ivurkK6WHvHFozVaGtQWV38FzBwSTGNmJolEgFp7QgR2bl6ArUWlvT4GcHKbPe1z3nWYi+VUdDZk16zDOVug==
-
 load-json-file@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0"