Fix releasenotes generation

.ci/ReleaseChangelog.java

@@ -16,80 +16,142 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.time.Instant;
-import java.time.ZoneOffset;
+import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Locale;
 import java.util.OptionalInt;
 import java.util.function.Predicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class ReleaseChangelog {
 
     public static void main(String[] args) throws IOException {
-        if (args.length != 2) {
-            System.out.println("Expected exactly two arguments: <ChangelogFile> <VersionToRelease>");
+        if (args.length != 3) {
+            System.out.println("Expected exactly three arguments: <ChangelogFile> <ReleaseNotesPath> <VersionToRelease>");
             System.exit(-1);
         }
-        Path fileName = Paths.get(args[0]);
-        String version = args[1].trim();
-        if (!version.matches("\\d+\\.\\d+\\.\\d+")) {
-            System.out.println("Version must be in the format x.x.x but was not: " + version);
-            System.exit(-1);
+        Path nextChangelogFile = Paths.get(args[0]);
+        Path releaseNotesDir = Paths.get(args[1]);
+        Path releaseNotesFile = releaseNotesDir.resolve("index.md");
+        Path deprecationsFile = releaseNotesDir.resolve("deprecations.md");
+        VersionNumber version = VersionNumber.parse(args[2].trim());
+
+        Lines nextChangelogLines = new Lines(Files.readAllLines(nextChangelogFile, StandardCharsets.UTF_8));
+        Lines fixes = nextChangelogLines.cutLinesBetween("<!--FIXES-START-->", "<!--FIXES-END-->");
+        Lines enhancements = nextChangelogLines.cutLinesBetween("<!--ENHANCEMENTS-START-->", "<!--ENHANCEMENTS-END-->");
+        Lines deprecations = nextChangelogLines.cutLinesBetween("<!--DEPRECATIONS-START-->", "<!--DEPRECATIONS-END-->");
+
+
+        var formatter = DateTimeFormatter.ofPattern("LLLL d, yyyy", Locale.ENGLISH);
+        String releaseDateLine = "**Release date:** " + formatter.format(LocalDate.now());
+
+        Lines allReleaseNotes = new Lines(Files.readAllLines(releaseNotesFile, StandardCharsets.UTF_8));
+        int insertBeforeLine = findHeadingOfPreviousVersion(allReleaseNotes, version);
+        allReleaseNotes.insert(generateReleaseNotes(version, releaseDateLine, enhancements, fixes), insertBeforeLine);
+
+        if (!deprecations.isEmpty()) {
+            Lines allDeprecations = new Lines(Files.readAllLines(deprecationsFile, StandardCharsets.UTF_8));
+            int insertDepsBeforeLine = findHeadingOfPreviousVersion(allDeprecations, version);
+            allDeprecations.insert(generateDeprecations(version, releaseDateLine, deprecations), insertDepsBeforeLine);
+            Files.writeString(deprecationsFile, allDeprecations + "\n", StandardCharsets.UTF_8);
         }
-        Lines f = new Lines(Files.readAllLines(fileName, StandardCharsets.UTF_8));
-        int unreleasedStart = f.findLine(str -> str.trim().equals("=== Unreleased"), 0).orElseThrow() + 1;
-        int unreleasedEnd = f.findLine(str -> str.startsWith("[[release-notes-"), unreleasedStart).orElseThrow();
-
-        Lines changes = f.cut(unreleasedStart, unreleasedEnd);
-        f.insert(new Lines(List.of("")), unreleasedStart); //add a blank line below unreleased heading
-        changes.trim();
+        Files.writeString(releaseNotesFile, allReleaseNotes + "\n", StandardCharsets.UTF_8);
+        Files.writeString(nextChangelogFile, nextChangelogLines + "\n", StandardCharsets.UTF_8);
+    }
 
-        // a few sanity checks
-        if (changes.lineCount() == 0) {
-            System.out.println("Unreleased changes are empty, there must be something wrong!");
-            System.exit(-1);
+    private static Lines generateReleaseNotes(VersionNumber version, String releaseDateLine, Lines enhancements, Lines fixes) {
+        Lines result = new Lines()
+            .append("## " + version.dotStr() + " [elastic-apm-java-agent-" + version.dashStr() + "-release-notes]")
+            .append(releaseDateLine);
+        if (!enhancements.isEmpty()) {
+            result
+                .append("")
+                .append("### Features and enhancements [elastic-apm-java-agent-" + version.dashStr() + "-features-enhancements]")
+                .append(enhancements);
         }
-        OptionalInt wrongIndentedHeading = changes.findLine(str -> str.matches("^==?=?=?[^=].*"), 0);
-        if (wrongIndentedHeading.isPresent()) {
-            System.out.println("Found heading which is too high level (must be at least =====) within changes: "
-                + changes.getLine(wrongIndentedHeading.getAsInt()));
-            System.exit(-1);
+        if (!fixes.isEmpty()) {
+            result
+                .append("")
+                .append("### Fixes [elastic-apm-java-agent-" + version.dashStr() + "-fixes]")
+                .append(fixes);
         }
+        result.append("");
+        return result;
+    }
+
 
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy/MM/dd")
-            .withZone(ZoneOffset.UTC);
+    private static Lines generateDeprecations(VersionNumber version, String releaseDateLine, Lines deprecations) {
+        return new Lines()
+            .append("## " + version.dotStr() + " [elastic-apm-java-agent-" + version.dashStr() + "-deprecations]")
+            .append(releaseDateLine)
+            .append("")
+            .append(deprecations)
+            .append("");
+    }
 
-        changes.insert(new Lines(List.of(
-            "",
-            String.format("[[release-notes-%s]]", version),
-            String.format("==== %s - %s", version, formatter.format(Instant.now())),
-            ""
-        )), 0);
+    static int findHeadingOfPreviousVersion(Lines lines, VersionNumber version) {
+        Pattern headingPattern = Pattern.compile("## (\\d+\\.\\d+\\.\\d+) .*");
+        Comparator<VersionNumber> comp = VersionNumber.comparator();
+        int currentBestLineNo = -1;
+        VersionNumber currentBestVersion = null;
+        for (int i = 0; i < lines.lineCount(); i++) {
+            Matcher matcher = headingPattern.matcher(lines.getLine(i));
+            if (matcher.matches()) {
+                VersionNumber headingForVersion = VersionNumber.parse(matcher.group(1));
+                if (comp.compare(headingForVersion, version) < 0
+                    && (currentBestVersion == null || comp.compare(headingForVersion, currentBestVersion) > 0)) {
+                    currentBestLineNo = i;
+                    currentBestVersion = headingForVersion;
+                }
+            }
+        }
+        return currentBestLineNo;
+    }
 
-        int majorVersion = Integer.parseInt(version.split("\\.")[0]);
-        String majorHeading = String.format("=== Java Agent version %d.x", majorVersion);
-        OptionalInt sectionStart = f.findLine(str -> str.trim().equals(majorHeading), 0);
-        if (sectionStart.isEmpty()) {
-            System.out.println("Could not find heading for major version: " + majorHeading);
-            System.out.println("Is this a new major version? If yes, please add an empty section for it manually to the Changelog");
-            System.exit(-1);
+    record VersionNumber(int major, int minor, int patch) {
+        public static VersionNumber parse(String versionString) {
+            if (!versionString.matches("\\d+\\.\\d+\\.\\d+")) {
+                throw new IllegalArgumentException("Version must be in the format x.x.x but was not: " + versionString);
+            }
+            String[] parts = versionString.split("\\.");
+            return new VersionNumber(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]), Integer.parseInt(parts[2]));
+        }
+
+        static Comparator<VersionNumber> comparator() {
+            return Comparator
+                .comparing(VersionNumber::major)
+                .thenComparing(VersionNumber::minor)
+                .thenComparing(VersionNumber::patch);
+        }
+
+        String dashSt() {
+            return major + "-" + minor + "-" + patch;
         }
 
-        f.insert(changes, sectionStart.getAsInt() + 1);
+        String dotStr() {
+            return major + "." + minor + "." + patch;
+        }
 
-        Files.writeString(fileName, f.toString() + "\n", StandardCharsets.UTF_8);
     }
 
     static class Lines {
 
         private final List<String> lines;
 
+        public Lines() {
+            this.lines = new ArrayList<>();
+        }
+
         public Lines(List<String> lines) {
             this.lines = new ArrayList<>(lines);
         }
@@ -98,6 +160,22 @@ int lineCount() {
             return lines.size();
         }
 
+        boolean isEmpty() {
+            return lines.isEmpty();
+        }
+
+        Lines cutLinesBetween(String startLine, String endLine) {
+            int start = findLine(l -> l.trim().equals(startLine), 0)
+                .orElseThrow(() -> new IllegalStateException("Expected line '" + startLine + "' to exist"));
+            int end = findLine(l -> l.trim().equals(endLine), start + 1)
+                .orElseThrow(() -> new IllegalStateException("Expected line '" + endLine + "' to exist after '" + startLine + "'"));
+            Lines result = cut(start + 1, end).trim();
+
+            lines.add(start + 1, "");
+
+            return result;
+        }
+
         OptionalInt findLine(Predicate<String> condition, int startAt) {
             for (int i = startAt; i < lines.size(); i++) {
                 if (condition.test(lines.get(i))) {
@@ -119,16 +197,27 @@ void insert(Lines other, int insertAt) {
             this.lines.addAll(insertAt, other.lines);
         }
 
+        Lines append(String line) {
+            lines.add(line);
+            return this;
+        }
+
+        Lines append(Lines toAppend) {
+            lines.addAll(toAppend.lines);
+            return this;
+        }
+
         /**
          * Trims lines consisting of only blanks at the top and bottom
          */
-        void trim() {
+        Lines trim() {
             while (!lines.isEmpty() && lines.get(0).matches("\\s*")) {
                 lines.remove(0);
             }
             while (!lines.isEmpty() && lines.get(lines.size() - 1).matches("\\s*")) {
                 lines.remove(lines.size() - 1);
             }
+            return this;
         }
 
         @Override

.ci/release/pre-release.sh

@@ -19,4 +19,4 @@ echo "Set release version"
 ./mvnw -V versions:set -DprocessAllModules=true -DgenerateBackupPoms=false -DnewVersion="${RELEASE_VERSION}"
 
 echo "Prepare changelog for release"
-java "${BASE_PROJECT}/.ci/ReleaseChangelog.java" CHANGELOG.asciidoc "${RELEASE_VERSION}"
+java "${BASE_PROJECT}/.ci/ReleaseChangelog.java" CHANGELOG.next-release.md docs/release-notes "${RELEASE_VERSION}"

.github/pull_request_template.md

@@ -13,21 +13,21 @@ Just ignore the checkboxes of categories that don't apply.
 -->
 
 - [ ] This is an enhancement of existing features, or a new feature in existing plugins
-  - [ ] I have updated [CHANGELOG.asciidoc](https://github.com/elastic/apm-agent-java/blob/main/CHANGELOG.asciidoc)
+  - [ ] I have updated [CHANGELOG.next-release.md](https://github.com/elastic/apm-agent-java/blob/main/CHANGELOG.next-release.md)
   - [ ] I have added tests that prove my fix is effective or that my feature works
   - [ ] Added an API method or config option? Document in which version this will be introduced
   - [ ] I have made corresponding changes to the documentation
 - [ ] This is a bugfix
-  - [ ] I have updated [CHANGELOG.asciidoc](https://github.com/elastic/apm-agent-java/blob/main/CHANGELOG.asciidoc)
+  - [ ] I have updated [CHANGELOG.next-release.md](https://github.com/elastic/apm-agent-java/blob/main/CHANGELOG.next-release.md)
   - [ ] I have added tests that would fail without this fix
 - [ ] This is a new plugin
-  - [ ] I have updated [CHANGELOG.asciidoc](https://github.com/elastic/apm-agent-java/blob/main/CHANGELOG.asciidoc)
+  - [ ] I have updated [CHANGELOG.next-release.md](https://github.com/elastic/apm-agent-java/blob/main/CHANGELOG.next-release.md)
   - [ ] My code follows the [style guidelines of this project](https://github.com/elastic/apm-agent-java/blob/main/CONTRIBUTING.md#java-language-formatting-guidelines)
   - [ ] I have made corresponding changes to the documentation
   - [ ] I have added tests that prove my fix is effective or that my feature works
   - [ ] New and existing [**unit** tests](https://github.com/elastic/apm-agent-java/blob/main/CONTRIBUTING.md#testing) pass locally with my changes
-  - [ ] I have updated [supported-technologies.asciidoc](https://github.com/elastic/apm-agent-java/blob/main/docs/supported-technologies.asciidoc)
+  - [ ] I have updated [supported-technologies.md](https://github.com/elastic/apm-agent-java/blob/main/docs/reference/supported-technologies.md)
   - [ ] Added an API method or config option? Document in which version this will be introduced
   - [ ] Added an instrumentation plugin? Describe how you made sure that old, non-supported versions are not instrumented by accident.
 - [ ] This is something else
-  - [ ] I have updated [CHANGELOG.asciidoc](https://github.com/elastic/apm-agent-java/blob/main/CHANGELOG.asciidoc)
+  - [ ] I have updated [CHANGELOG.next-release.md](https://github.com/elastic/apm-agent-java/blob/main/CHANGELOG.next-release.md)

CHANGELOG.next-release.md

@@ -0,0 +1,21 @@
+This file contains all changes which are not released yet.
+<!--
+ Note that the content between the marker comment lines (e.g. FIXES-START/END) will be automatically
+ moved into the docs/release-notes markdown files on release (via the .ci/ReleaseChangelog.java script).
+ Simply add the changes as bullet points into those sections, empty lines will be ignored. Example:
+
+* Description of the change - [#1234](https://github.com/elastic/apm-agent-java/pull/1234)
+-->
+
+# Fixes
+<!--FIXES-START-->
+
+<!--FIXES-END-->
+# Features and enhancements
+<!--ENHANCEMENTS-START-->
+
+<!--ENHANCEMENTS-END-->
+# Deprecations
+<!--DEPRECATIONS-START-->
+
+<!--DEPRECATIONS-END-->