Merge branch 'changelog-api' into 'master'

Add API for generating changelogs in GitLab

See merge request gitlab-org/gitlab!52116

app/finders/merge_requests/oldest_per_commit_finder.rb

+# frozen_string_literal: true
+
+module MergeRequests
+  # OldestPerCommitFinder is used to retrieve the oldest merge requests for
+  # every given commit, grouped per commit SHA.
+  #
+  # This finder is useful when you need to efficiently retrieve the first/oldest
+  # merge requests for multiple commits, and you want to do so in batches;
+  # instead of running a query for every commit.
+  class OldestPerCommitFinder
+    def initialize(project)
+      @project = project
+    end
+
+    # Returns a Hash that maps a commit ID to the oldest merge request that
+    # introduced that commit.
+    def execute(commits)
+      id_rows = MergeRequestDiffCommit
+        .oldest_merge_request_id_per_commit(@project.id, commits.map(&:id))
+
+      mrs = MergeRequest
+        .preload_target_project
+        .id_in(id_rows.map { |r| r[:merge_request_id] })
+        .index_by(&:id)
+
+      id_rows.each_with_object({}) do |row, hash|
+        if (mr = mrs[row[:merge_request_id]])
+          hash[row[:sha]] = mr
+        end
+      end
+    end
+  end
+end

app/models/merge_request_diff_commit.rb

@@ -35,4 +35,23 @@ class MergeRequestDiffCommit < ApplicationRecord
 
     Gitlab::Database.bulk_insert(self.table_name, rows) # rubocop:disable Gitlab/BulkInsert
   end
+
+  def self.oldest_merge_request_id_per_commit(project_id, shas)
+    # This method is defined here and not on MergeRequest, otherwise the SHA
+    # values used in the WHERE below won't be encoded correctly.
+    select(['merge_request_diff_commits.sha AS sha', 'min(merge_requests.id) AS merge_request_id'])
+      .joins(:merge_request_diff)
+      .joins(
+        'INNER JOIN merge_requests ' \
+          'ON merge_requests.latest_merge_request_diff_id = merge_request_diffs.id'
+      )
+      .where(sha: shas)
+      .where(
+        merge_requests: {
+          target_project_id: project_id,
+          state_id: MergeRequest.available_states[:merged]
+        }
+      )
+      .group(:sha)
+  end
 end

app/services/repositories/changelog_service.rb

+# frozen_string_literal: true
+
+module Repositories
+  # A service class for generating a changelog section.
+  class ChangelogService
+    DEFAULT_TRAILER = 'Changelog'
+    DEFAULT_FILE = 'CHANGELOG.md'
+
+    # The `project` specifies the `Project` to generate the changelog section
+    # for.
+    #
+    # The `user` argument specifies a `User` to use for committing the changes
+    # to the Git repository.
+    #
+    # The `version` arguments must be a version `String` using semantic
+    # versioning as the format.
+    #
+    # The arguments `from` and `to` must specify a Git ref or SHA to use for
+    # fetching the commits to include in the changelog. The SHA/ref set in the
+    # `from` argument isn't included in the list.
+    #
+    # The `date` argument specifies the date of the release, and defaults to the
+    # current time/date.
+    #
+    # The `branch` argument specifies the branch to commit the changes to. The
+    # branch must already exist.
+    #
+    # The `trailer` argument is the Git trailer to use for determining what
+    # commits to include in the changelog.
+    #
+    # The `file` arguments specifies the name/path of the file to commit the
+    # changes to. If the file doesn't exist, it's created automatically.
+    #
+    # The `message` argument specifies the commit message to use when committing
+    # the changelog changes.
+    #
+    # rubocop: disable Metrics/ParameterLists
+    def initialize(
+      project,
+      user,
+      version:,
+      from:,
+      to:,
+      date: DateTime.now,
+      branch: project.default_branch_or_master,
+      trailer: DEFAULT_TRAILER,
+      file: DEFAULT_FILE,
+      message: "Add changelog for version #{version}"
+    )
+      @project = project
+      @user = user
+      @version = version
+      @from = from
+      @to = to
+      @date = date
+      @branch = branch
+      @trailer = trailer
+      @file = file
+      @message = message
+    end
+    # rubocop: enable Metrics/ParameterLists
+
+    def execute
+      # For every entry we want to only include the merge request that
+      # originally introduced the commit, which is the oldest merge request that
+      # contains the commit. We fetch there merge requests in batches, reducing
+      # the number of SQL queries needed to get this data.
+      mrs_finder = MergeRequests::OldestPerCommitFinder.new(@project)
+      config = Gitlab::Changelog::Config.from_git(@project)
+      release = Gitlab::Changelog::Release
+        .new(version: @version, date: @date, config: config)
+
+      commits =
+        CommitsWithTrailerFinder.new(project: @project, from: @from, to: @to)
+
+      commits.each_page(@trailer) do |page|
+        mrs = mrs_finder.execute(page)
+
+        # Preload the authors. This ensures we only need a single SQL query per
+        # batch of commits, instead of needing a query for every commit.
+        page.each(&:lazy_author)
+
+        page.each do |commit|
+          release.add_entry(
+            title: commit.title,
+            commit: commit,
+            category: commit.trailers.fetch(@trailer),
+            author: commit.author,
+            merge_request: mrs[commit.id]
+          )
+        end
+      end
+
+      Gitlab::Changelog::Committer
+        .new(@project, @user)
+        .commit(release: release, file: @file, branch: @branch, message: @message)
+    end
+  end
+end

changelogs/unreleased/changelog-api.yml

+---
+title: Add API for generating Markdown changelogs
+merge_request: 52116
+author:
+type: added

config/feature_flags/development/changelog_api.yml

+---
+name: changelog_api
+introduced_by_url: '13.9'
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/300043
+milestone: '13.9'
+type: development
+group: group::source code
+default_enabled: false

db/migrate/20210127143025_add_oldest_merge_requests_index.rb

+# frozen_string_literal: true
+
+class AddOldestMergeRequestsIndex < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::SchemaHelpers
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  INDEX = 'index_on_merge_requests_for_latest_diffs'
+
+  def up
+    return if index_exists_by_name?('merge_requests', INDEX)
+
+    execute "CREATE INDEX CONCURRENTLY #{INDEX} ON merge_requests " \
+      'USING btree (target_project_id) INCLUDE (id, latest_merge_request_diff_id)'
+
+    create_comment(
+      'INDEX',
+      INDEX,
+      'Index used to efficiently obtain the oldest merge request for a commit SHA'
+    )
+  end
+
+  def down
+    return unless index_exists_by_name?('merge_requests', INDEX)
+
+    execute "DROP INDEX CONCURRENTLY #{INDEX}"
+  end
+end

db/schema_migrations/20210127143025

+c173ba86340efe39977f1b319d1ebcead634e3bfe819a30e230fb4f81766f28a
\ No newline at end of file

db/structure.sql

@@ -22487,6 +22487,10 @@ CREATE UNIQUE INDEX index_on_instance_statistics_recorded_at_and_identifier ON a
 
 CREATE INDEX index_on_label_links_all_columns ON label_links USING btree (target_id, label_id, target_type);
 
+CREATE INDEX index_on_merge_requests_for_latest_diffs ON merge_requests USING btree (target_project_id) INCLUDE (id, latest_merge_request_diff_id);
+
+COMMENT ON INDEX index_on_merge_requests_for_latest_diffs IS 'Index used to efficiently obtain the oldest merge request for a commit SHA';
+
 CREATE INDEX index_on_namespaces_lower_name ON namespaces USING btree (lower((name)::text));
 
 CREATE INDEX index_on_namespaces_lower_path ON namespaces USING btree (lower((path)::text));

doc/api/repositories.md

@@ -5,7 +5,7 @@ info: "To determine the technical writer assigned to the Stage/Group associated
 type: reference, api
 ---
 
-# Repositories API
+# Repositories API **(CORE)**
 
 ## List repository tree
 
@@ -18,14 +18,15 @@ This command provides essentially the same functionality as the `git ls-tree` co
 GET /projects/:id/repository/tree
 ```
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `path` (optional) - The path inside repository. Used to get content of subdirectories
-- `ref` (optional) - The name of a repository branch or tag or if not given the default branch
-- `recursive` (optional) - Boolean value used to get a recursive tree (false by default)
-- `per_page` (optional) - Number of results to show per page. If not specified, defaults to `20`.
-  Read more on [pagination](README.md#pagination).
+| Attribute   | Type           | Required | Description |
+| :---------- | :------------- | :------- | :---------- |
+| `id`        | integer/string | no       | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `path`      | string         | yes      | The path inside repository. Used to get content of subdirectories. |
+| `ref`       | string         | yes      | The name of a repository branch or tag or if not given the default branch. |
+| `recursive` | boolean        | yes      | Boolean value used to get a recursive tree (false by default). |
+| `per_page`  | integer        | yes      | Number of results to show per page. If not specified, defaults to `20`. [Learn more on pagination](README.md#pagination). |
 
 ```json
 [
@@ -91,10 +92,12 @@ without authentication if the repository is publicly accessible.
 GET /projects/:id/repository/blobs/:sha
 ```
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `sha` (required) - The blob SHA
+| Attribute | Type           | Required | Description |
+| :-------- | :------------- | :------- | :---------- |
+| `id`      | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `sha`     | string         | yes      | The blob SHA. |
 
 ## Raw blob content
 
@@ -105,10 +108,12 @@ without authentication if the repository is publicly accessible.
 GET /projects/:id/repository/blobs/:sha/raw
 ```
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `sha` (required) - The blob SHA
+| Attribute | Type     | Required | Description |
+| :-------- | :------- | :------- | :---------- |
+| `id`      | datatype | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `sha`     | datatype | yes      | The blob SHA. |
 
 ## Get file archive
 
@@ -128,10 +133,14 @@ GET /projects/:id/repository/archive[.format]
 `bz2`, `tar`, and `zip`. For example, specifying `archive.zip`
 would send an archive in ZIP format.
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `sha` (optional) - The commit SHA to download. A tag, branch reference, or SHA can be used. This defaults to the tip of the default branch if not specified. For example:
+| Attribute   | Type           | Required | Description           |
+|:------------|:---------------|:---------|:----------------------|
+| `id`        | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `sha`       | string         | no       | The commit SHA to download. A tag, branch reference, or SHA can be used. This defaults to the tip of the default branch if not specified. |
+
+Example request:
 
 ```shell
 curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.com/api/v4/projects/<project_id>/repository/archive?sha=<commit_sha>"
@@ -146,21 +155,22 @@ publicly accessible. Note that diffs could have an empty diff string if [diff li
 GET /projects/:id/repository/compare
 ```
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `from` (required) - the commit SHA or branch name
-- `to` (required) - the commit SHA or branch name
-- `straight` (optional) - comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)'. Default is `false`.
+| Attribute  | Type           | Required | Description |
+| :--------- | :------------- | :------- | :---------- |
+| `id`       | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `from`     | string         | yes      | The commit SHA or branch name. |
+| `to`       | string         | yes      | The commit SHA or branch name. |
+| `straight` | boolean        | no       | Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)'. Default is `false`. |
 
 ```plaintext
 GET /projects/:id/repository/compare?from=master&to=feature
 ```
 
-Response:
+Example response:
 
 ```json
-
 {
   "commit": {
     "id": "12d65c8dd2b2676fa3ac47d955accc085a37a9c1",
@@ -203,15 +213,17 @@ GET /projects/:id/repository/contributors
 ```
 
 WARNING:
-The `additions` and `deletions` attributes are deprecated [as of GitLab 13.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39653) because they [always return `0`](https://gitlab.com/gitlab-org/gitlab/-/issues/233119).
+The `additions` and `deletions` attributes are deprecated [as of GitLab 13.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39653), because they [always return `0`](https://gitlab.com/gitlab-org/gitlab/-/issues/233119).
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `order_by` (optional) - Return contributors ordered by `name`, `email`, or `commits` (orders by commit date) fields. Default is `commits`
-- `sort` (optional) - Return contributors sorted in `asc` or `desc` order. Default is `asc`
+| Attribute  | Type           | Required | Description |
+| :--------- | :------------- | :------- | :---------- |
+| `id`       | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `order_by` | string         | no       | Return contributors ordered by `name`, `email`, or `commits` (orders by commit date) fields. Default is `commits`. |
+| `sort`     | string         | no       | Return contributors sorted in `asc` or `desc` order. Default is `asc`. |
 
-Response:
+Example response:
 
 ```json
 [{
@@ -237,10 +249,12 @@ Get the common ancestor for 2 or more refs (commit SHAs, branch names or tags).
 GET /projects/:id/repository/merge_base
 ```
 
-| Attribute | Type | Required | Description |
-| --------- | ---- | -------- | ----------- |
-| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
-| `refs` | array | yes | The refs to find the common ancestor of, multiple refs can be passed |
+| Attribute | Type           | Required | Description |
+| --------- | -------------- | -------- | ------------------------------------------------------------------------------- |
+| `id`      | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
+| `refs`    | array          | yes      | The refs to find the common ancestor of, multiple refs can be passed            |
+
+Example request:
 
 ```shell
 curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/merge_base?refs[]=304d257dcb821665ab5110318fc58a007bd104ed&refs[]=0031876facac3f2b2702a0e53a26e89939a42209"
@@ -264,3 +278,252 @@ Example response:
   "committed_date": "2014-02-27T08:03:18.000Z"
 }
 ```
+
+## Generate changelog data
+
+> - [Introduced](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/351) in GitLab 13.9.
+> - It's [deployed behind a feature flag](../user/feature_flags.md), disabled by default.
+> - It's disabled on GitLab.com.
+> - It's not yet recommended for production use.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-generating-changelog-data).
+
+WARNING:
+This feature might not be available to you. Check the **version history** note above for details.
+
+Generate changelog data based on commits in a repository.
+
+Given a version (using semantic versioning) and a range of commits,
+GitLab generates a changelog for all commits that use a particular
+[Git trailer](https://git-scm.com/docs/git-interpret-trailers).
+
+The output of this process is a new section in a changelog file in the Git
+repository of the given project. The output format is in Markdown, and can be
+customized.
+
+```plaintext
+POST /projects/:id/repository/changelog
+```
+
+Supported attributes:
+
+| Attribute | Type     | Required   | Description |
+| :-------- | :------- | :--------- | :---------- |
+| `version` | string   | yes | The version to generate the changelog for. The format must follow [semantic versioning](https://semver.org/). |
+| `from`    | string   | yes | The start of the range of commits (as a SHA) to use for generating the changelog. This commit itself isn't included in the list. |
+| `to`      | string   | yes | The end of the range of commits (as a SHA) to use for the changelog. This commit _is_ included in the list. |
+| `date`    | datetime | no | The date and time of the release, defaults to the current time. |
+| `branch`  | string   | no | The branch to commit the changelog changes to, defaults to the project's default branch. |
+| `trailer` | string   | no | The Git trailer to use for including commits, defaults to `Changelog`. |
+| `file`    | string   | no | The file to commit the changes to, defaults to `CHANGELOG.md`. |
+| `message` | string   | no | The commit message to produce when committing the changes, defaults to `Add changelog for version X` where X is the value of the `version` argument. |
+
+### How it works
+
+Changelogs are generated based on commit titles. Commits are only included if
+they contain a specific Git trailer. GitLab uses the value of this trailer to
+categorize the changes.
+
+GitLab uses Git trailers, because Git trailers are
+supported by Git out of the box. We use commits as input, as this is the only
+source of data every project uses. In addition, commits can be retrieved when
+operating on a mirror. This is important for GitLab itself, because during a security
+release we might need to include changes from both public projects and private
+security mirrors.
+
+Changelogs are generated by taking the title of the commits to include and using
+these as the changelog entries. You can enrich entries with additional data,
+such as a link to the merge request or details about the commit author. You can
+[customize the format of a changelog](#customize-the-changelog-output) section with a template.
+
+### Customize the changelog output
+
+The output is customized using a YAML configuration file stored in your
+project's Git repository. This file must reside in
+`.gitlab/changelog_config.yml`.
+
+You can set the following variables in this file:
+
+- `date_format`: the date format to use in the title of the newly added
+  changelog data. This uses regular `strftime` formatting.
+- `template`: a custom template to use for generating the changelog data.
+- `categories`: a hash that maps raw category names to the names to use in the
+  changelog.
+
+Using the default settings, generating a changelog results in a section along
+the lines of the following:
+
+```markdown
+## 1.0.0 (2021-01-05)
+
+### Features (4 changes)
+
+- [Feature 1](gitlab-org/gitlab@123abc) by @alice ([merge request](gitlab-org/gitlab!123))
+- [Feature 2](gitlab-org/gitlab@456abc) ([merge request](gitlab-org/gitlab!456))
+- [Feature 3](gitlab-org/gitlab@234abc) by @steve
+- [Feature 4](gitlab-org/gitlab@456)
+```
+
+Each section starts with a title that contains the version and release date.
+While the format of the date can be customized, the rest of the title can't be
+changed. When adding a new section, GitLab parses these titles to determine
+where in the file the new section should be placed. GitLab sorts sections
+according to their versions, not their dates.
+
+Each section can have categories, each with their
+corresponding changes. In the above example, "Features" is one such category.
+You can customize the format of these sections.
+
+The section names are derived from the values of the Git trailer used to include
+or exclude commits.
+
+For example, if the trailer to use is called `Changelog`,
+and its value is `feature`, then the commit is grouped in the `feature`
+category. The names of these raw values might differ from what you want to
+show in a changelog, you can remap them. Let's say we use the `Changelog`
+trailer and developers use the following values: `feature`, `bug`, and
+`performance`.
+
+You can remap these using the following YAML configuration file:
+
+```yaml
+---
+categories:
+  feature: Features
+  bug: Bug fixes
+  performance: Performance improvements
+```
+
+When generating the changelog data, the category titles are then `### Features`,
+`### Bug fixes`, and `### Performance improvements`.
+
+### Custom templates
+
+The category sections are generated using a template. The default template is as
+follows:
+
+```plaintext
+{% if categories %}
+{% each categories %}
+### {{ title }} ({% if single_change %}1 change{% else %}{{ count }} changes{% end %})
+
+{% each entries %}
+- [{{ title }}]({{ commit.reference }})\
+{% if author.contributor %} by {{ author.reference }}{% end %}\
+{% if merge_request %} ([merge request]({{ merge_request.reference }})){% end %}
+{% end %}
+
+{% end %}
+{% else %}
+No changes.
+{% end %}
+```
+
+The `{% ... %}` tags are for statements, and `{{ ... }}` is used for printing
+data. Statements must be terminated using a `{% end %}` tag. Both the `if` and
+`each` statements require a single argument.
+
+For example, if we have a variable `valid`, and we want to display "yes"
+when this value is true, and display "nope" otherwise. We can do so as follows:
+
+```plaintext
+{% if valid %}
+yes
+{% else %}
+nope
+{% end %}
+```
+
+The use of `else` is optional. A value is considered true when it's a non-empty
+value or boolean `true`. Empty arrays and hashes are considered false.
+
+Looping is done using `each`, and variables inside a loop are scoped to it.
+Referring to the current value in a loop is done using the variable tag `{{ it
+}}`. Other variables read their value from the current loop value. Take
+this template for example:
+
+```plaintext
+{% each users %}
+{{name}}
+{% end %}
+```
+
+Assuming `users` is an array of objects, each with a `name` field, this would
+then print the name of every user.
+
+Using variable tags, you can access nested objects. For example, `{{
+users.0.name }}` prints the name of the first user in the `users` variable.
+
+If a line ends in a backslash, the next newline is ignored. This allows you to
+wrap code across multiple lines, without introducing unnecessary newlines in the
+Markdown output.
+
+You can specify a custom template in your configuration like so:
+
+```yaml
+---
+template: >
+  {% if categories %}
+  {% each categories %}
+  ### {{ title }}
+
+  {% each entries %}
+  - [{{ title }}]({{ commit.reference }})\
+  {% if author.contributor %} by {{ author.reference }}{% end %}
+  {% end %}
+
+  {% end %}
+  {% else %}
+  No changes.
+  {% end %}
+```
+
+### Template data
+
+At the top level, the following variable is available:
+
+- `categories`: an array of objects, one for every changelog category.
+
+In a category, the following variables are available:
+
+- `title`: the title of the category (after it has been remapped).
+- `count`: the number of entries in this category.
+- `single_change`: a boolean that indicates if there is only one change (`true`),
+  or multiple changes (`false`).
+- `entries`: the entries that belong to this category.
+
+In an entry, the following variables are available (here `foo.bar` means that
+`bar` is a sub-field of `foo`):
+
+- `title`: the title of the changelog entry (this is the commit title).
+- `commit.reference`: a reference to the commit, for example,
+  `gitlab-org/gitlab@0a4cdd86ab31748ba6dac0f69a8653f206e5cfc7`.
+- `commit.trailers`: an object containing all the Git trailers that were present
+  in the commit body.
+- `author.reference`: a reference to the commit author (for example, `@alice`).
+- `author.contributor`: a boolean set to `true` when the author is an external
+  contributor, otherwise this is set to `false`.
+- `merge_request.reference`: a reference to the merge request that first
+  introduced the change (for example, `gitlab-org/gitlab!50063`).
+
+The `author` and `merge_request` objects might not be present if the data couldn't
+be determined (for example, when a commit was created without a corresponding merge
+request).
+
+### Enable or disable generating changelog data **(CORE ONLY)**
+
+This feature is under development and not ready for production use. It is
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md)
+can enable it.
+
+To enable it for a project:
+
+```ruby
+Feature.enable(:changelog_api, Project.find(id_of_the_project))
+```
+
+To disable it for a project:
+
+```ruby
+Feature.disable(:changelog_api, Project.find(id_of_the_project))
+```

lib/api/repositories.rb

@@ -170,6 +170,67 @@ module API
           not_found!("Merge Base")
         end
       end
+
+      desc 'Generates a changelog section for a release' do
+        detail 'This feature was introduced in GitLab 13.9'
+      end
+      params do
+        requires :version,
+          type: String,
+          regexp: Gitlab::Regex.unbounded_semver_regex,
+          desc: 'The version of the release, using the semantic versioning format'
+
+        requires :from,
+          type: String,
+          desc: 'The first commit in the range of commits to use for the changelog'
+
+        requires :to,
+          type: String,
+          desc: 'The last commit in the range of commits to use for the changelog'
+
+        optional :date,
+          type: DateTime,
+          desc: 'The date and time of the release'
+
+        optional :branch,
+          type: String,
+          desc: 'The branch to commit the changelog changes to'
+
+        optional :trailer,
+          type: String,
+          desc: 'The Git trailer to use for determining if commits are to be included in the changelog',
+          default: ::Repositories::ChangelogService::DEFAULT_TRAILER
+
+        optional :file,
+          type: String,
+          desc: 'The file to commit the changelog changes to',
+          default: ::Repositories::ChangelogService::DEFAULT_FILE
+
+        optional :message,
+          type: String,
+          desc: 'The commit message to use when committing the changelog'
+      end
+      post ':id/repository/changelog' do
+        not_found! unless Feature.enabled?(:changelog_api, user_project)
+
+        branch = params[:branch] || user_project.default_branch_or_master
+        access = Gitlab::UserAccess.new(current_user, container: user_project)
+
+        unless access.can_push_to_branch?(branch)
+          forbidden!("You are not allowed to commit a changelog on this branch")
+        end
+
+        service = ::Repositories::ChangelogService.new(
+          user_project,
+          current_user,
+          **declared_params(include_missing: false)
+        )
+
+        service.execute
+        status(200)
+      rescue => ex
+        render_api_error!("Failed to generate the changelog: #{ex.message}", 500)
+      end
     end
   end
 end

lib/gitlab/changelog/committer.rb

@@ -26,7 +26,13 @@ module Gitlab
         # scratch, otherwise we may end up throwing away changes. As such, all
         # the logic is contained within the retry block.
         Retriable.retriable(on: CommitError) do
-          commit = @project.commit(branch)
+          commit = Gitlab::Git::Commit.last_for_path(
+            @project.repository,
+            branch,
+            file,
+            literal_pathspec: true
+          )
+
           content = blob_content(file, commit)
 
           # If the release has already been added (e.g. concurrently by another

lib/gitlab/changelog/config.rb

@@ -37,7 +37,10 @@ module Gitlab
         end
 
         if (template = hash['template'])
-          config.template = Template::Compiler.new.compile(template)
+          # We use the full namespace here (and further down) as otherwise Rails
+          # may use the wrong constant when autoloading is used.
+          config.template =
+            ::Gitlab::Changelog::Template::Compiler.new.compile(template)
         end
 
         if (categories = hash['categories'])
@@ -54,7 +57,8 @@ module Gitlab
       def initialize(project)
         @project = project
         @date_format = DEFAULT_DATE_FORMAT
-        @template = Template::Compiler.new.compile(DEFAULT_TEMPLATE)
+        @template =
+          ::Gitlab::Changelog::Template::Compiler.new.compile(DEFAULT_TEMPLATE)
         @categories = {}
       end
 

lib/gitlab/changelog/template/compiler.rb

@@ -98,19 +98,27 @@ module Gitlab
         ESCAPED_NEWLINE = /\\\n$/.freeze
 
         # The start tag for ERB tags. These tags will be escaped, preventing
-        # users FROM USING erb DIRECTLY.
-        ERB_START_TAG = '<%'
+        # users from using ERB directly.
+        ERB_START_TAG = /<\\?\s*\\?\s*%/.freeze
 
         def compile(template)
           transformed_lines = ['<% it = variables %>']
 
+          # ERB tags must be stripped here, otherwise a user may introduce ERB
+          # tags by making clever use of whitespace. See
+          # https://gitlab.com/gitlab-org/gitlab/-/issues/300224 for more
+          # information.
+          template = template.gsub(ERB_START_TAG, '<%%')
+
           template.each_line { |line| transformed_lines << transform(line) }
-          Template.new(transformed_lines.join)
+
+          # We use the full namespace here as otherwise Rails may use the wrong
+          # constant when autoloading is used.
+          ::Gitlab::Changelog::Template::Template.new(transformed_lines.join)
         end
 
         def transform(line)
           line.gsub!(ESCAPED_NEWLINE, '')
-          line.gsub!(ERB_START_TAG, '<%%')
 
           # This replacement ensures that "end" blocks on their own lines
           # don't add extra newlines. Using an ERB -%> tag sadly swallows too

spec/finders/merge_requests/oldest_per_commit_finder_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe MergeRequests::OldestPerCommitFinder do
+  describe '#execute' do
+    it 'returns a Hash mapping commit SHAs to their oldest merge requests' do
+      project = create(:project)
+      mr1 = create(:merge_request, :merged, target_project: project)
+      mr2 = create(:merge_request, :merged, target_project: project)
+      mr1_diff = create(:merge_request_diff, merge_request: mr1)
+      mr2_diff = create(:merge_request_diff, merge_request: mr2)
+      sha1 = Digest::SHA1.hexdigest('foo')
+      sha2 = Digest::SHA1.hexdigest('bar')
+
+      create(:merge_request_diff_commit, merge_request_diff: mr1_diff, sha: sha1)
+      create(:merge_request_diff_commit, merge_request_diff: mr2_diff, sha: sha1)
+      create(
+        :merge_request_diff_commit,
+        merge_request_diff: mr2_diff,
+        sha: sha2,
+        relative_order: 1
+      )
+
+      commits = [double(:commit, id: sha1), double(:commit, id: sha2)]
+
+      expect(described_class.new(project).execute(commits)).to eq(
+        sha1 => mr1,
+        sha2 => mr2
+      )
+    end
+
+    it 'skips merge requests that are not merged' do
+      mr = create(:merge_request)
+      mr_diff = create(:merge_request_diff, merge_request: mr)
+      sha = Digest::SHA1.hexdigest('foo')
+
+      create(:merge_request_diff_commit, merge_request_diff: mr_diff, sha: sha)
+
+      commits = [double(:commit, id: sha)]
+
+      expect(described_class.new(mr.target_project).execute(commits))
+        .to be_empty
+    end
+  end
+end

spec/lib/gitlab/changelog/committer_spec.rb

@@ -86,5 +86,43 @@ RSpec.describe Gitlab::Changelog::Committer do
         end.not_to raise_error
       end
     end
+
+    context "when the changelog changes before saving the changes" do
+      it 'raises a CommitError' do
+        release1 = Gitlab::Changelog::Release
+          .new(version: '1.0.0', date: Time.utc(2020, 1, 1), config: config)
+
+        release2 = Gitlab::Changelog::Release
+          .new(version: '2.0.0', date: Time.utc(2020, 1, 1), config: config)
+
+        # This creates the initial commit we'll later use to see if the
+        # changelog changed before saving our changes.
+        committer.commit(
+          release: release1,
+          file: 'CHANGELOG.md',
+          branch: 'master',
+          message: 'Initial commit'
+        )
+
+        allow(Gitlab::Git::Commit)
+          .to receive(:last_for_path)
+          .with(
+            project.repository,
+            'master',
+            'CHANGELOG.md',
+            literal_pathspec: true
+          )
+          .and_return(double(:commit, sha: 'foo'))
+
+        expect do
+          committer.commit(
+            release: release2,
+            file: 'CHANGELOG.md',
+            branch: 'master',
+            message: 'Test commit'
+          )
+        end.to raise_error(described_class::CommitError)
+      end
+    end
   end
 end

spec/lib/gitlab/changelog/template/compiler_spec.rb

@@ -125,5 +125,12 @@ RSpec.describe Gitlab::Changelog::Template::Compiler do
 
       expect(compile(input)).to eq(input)
     end
+
+    it 'ignores malicious code that makes use of whitespace' do
+      input = "x<\\\n%::Kernel.system(\"id\")%>"
+
+      expect(Kernel).not_to receive(:system).with('id')
+      expect(compile(input)).to eq('x<%::Kernel.system("id")%>')
+    end
   end
 end

spec/requests/api/repositories_spec.rb

@@ -610,4 +610,102 @@ RSpec.describe API::Repositories do
       end
     end
   end
+
+  describe 'POST /projects/:id/repository/changelog' do
+    context 'when the changelog_api feature flag is enabled' do
+      it 'generates the changelog for a version' do
+        spy = instance_spy(Repositories::ChangelogService)
+
+        allow(Repositories::ChangelogService)
+          .to receive(:new)
+          .with(
+            project,
+            user,
+            version: '1.0.0',
+            from: 'foo',
+            to: 'bar',
+            date: DateTime.new(2020, 1, 1),
+            branch: 'kittens',
+            trailer: 'Foo',
+            file: 'FOO.md',
+            message: 'Commit message'
+          )
+          .and_return(spy)
+
+        allow(spy).to receive(:execute)
+
+        post(
+          api("/projects/#{project.id}/repository/changelog", user),
+          params: {
+            version: '1.0.0',
+            from: 'foo',
+            to: 'bar',
+            date: '2020-01-01',
+            branch: 'kittens',
+            trailer: 'Foo',
+            file: 'FOO.md',
+            message: 'Commit message'
+          }
+        )
+
+        expect(response).to have_gitlab_http_status(:ok)
+      end
+
+      it 'produces an error when generating the changelog fails' do
+        spy = instance_spy(Repositories::ChangelogService)
+
+        allow(Repositories::ChangelogService)
+          .to receive(:new)
+          .with(
+            project,
+            user,
+            version: '1.0.0',
+            from: 'foo',
+            to: 'bar',
+            date: DateTime.new(2020, 1, 1),
+            branch: 'kittens',
+            trailer: 'Foo',
+            file: 'FOO.md',
+            message: 'Commit message'
+          )
+          .and_return(spy)
+
+        allow(spy)
+          .to receive(:execute)
+          .and_raise(Gitlab::Changelog::Committer::CommitError.new('oops'))
+
+        post(
+          api("/projects/#{project.id}/repository/changelog", user),
+          params: {
+            version: '1.0.0',
+            from: 'foo',
+            to: 'bar',
+            date: '2020-01-01',
+            branch: 'kittens',
+            trailer: 'Foo',
+            file: 'FOO.md',
+            message: 'Commit message'
+          }
+        )
+
+        expect(response).to have_gitlab_http_status(:internal_server_error)
+        expect(json_response['message']).to eq('Failed to generate the changelog: oops')
+      end
+    end
+
+    context 'when the changelog_api feature flag is disabled' do
+      before do
+        stub_feature_flags(changelog_api: false)
+      end
+
+      it 'responds with a 404 Not Found' do
+        post(
+          api("/projects/#{project.id}/repository/changelog", user),
+          params: { version: '1.0.0', from: 'foo', to: 'bar' }
+        )
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+  end
 end

spec/services/repositories/changelog_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Repositories::ChangelogService do
+  describe '#execute' do
+    it 'generates and commits a changelog section' do
+      project = create(:project, :empty_repo)
+      creator = project.creator
+      author1 = create(:user)
+      author2 = create(:user)
+
+      project.add_maintainer(author1)
+      project.add_maintainer(author2)
+
+      mr1 = create(:merge_request, :merged, target_project: project)
+      mr2 = create(:merge_request, :merged, target_project: project)
+
+      # The range of commits ignores the first commit, but includes the last
+      # commit. To ensure both the commits below are included, we must create an
+      # extra commit.
+      #
+      # In the real world, the start commit of the range will be the last commit
+      # of the previous release, so ignoring that is expected and desired.
+      sha1 = create_commit(
+        project,
+        creator,
+        commit_message: 'Initial commit',
+        actions: [{ action: 'create', content: 'test', file_path: 'README.md' }]
+      )
+
+      sha2 = create_commit(
+        project,
+        author1,
+        commit_message: "Title 1\n\nChangelog: feature",
+        actions: [{ action: 'create', content: 'foo', file_path: 'a.txt' }]
+      )
+
+      sha3 = create_commit(
+        project,
+        author2,
+        commit_message: "Title 2\n\nChangelog: feature",
+        actions: [{ action: 'create', content: 'bar', file_path: 'b.txt' }]
+      )
+
+      commit1 = project.commit(sha2)
+      commit2 = project.commit(sha3)
+
+      allow(MergeRequestDiffCommit)
+        .to receive(:oldest_merge_request_id_per_commit)
+        .with(project.id, [commit2.id, commit1.id])
+        .and_return([
+          { sha: sha2, merge_request_id: mr1.id },
+          { sha: sha3, merge_request_id: mr2.id }
+        ])
+
+      recorder = ActiveRecord::QueryRecorder.new do
+        described_class
+          .new(project, creator, version: '1.0.0', from: sha1, to: sha3)
+          .execute
+      end
+
+      changelog = project.repository.blob_at('master', 'CHANGELOG.md')&.data
+
+      expect(recorder.count).to eq(10)
+      expect(changelog).to include('Title 1', 'Title 2')
+    end
+  end
+
+  def create_commit(project, user, params)
+    params = { start_branch: 'master', branch_name: 'master' }.merge(params)
+    Files::MultiService.new(project, user, params).execute.fetch(:result)
+  end
+end