From fc2a2416472bda6454b0ad9ab83378465ec07e3d Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Thu, 6 Aug 2026 11:17:33 -0300 Subject: [PATCH 1/5] Changelog: add temporary migrate-from-web command --- config/migrate-from-web.yml | 18 + docs/cli-schema.json | 102 +++++ .../Migration/MigrateFromWebScope.cs | 127 ++++++ .../Migration/ReleaseNotesPageParser.cs | 307 +++++++++++++++ .../Migration/WebMigrationService.cs | 360 ++++++++++++++++++ .../docs-builder/Commands/ChangelogCommand.cs | 56 +++ 6 files changed, 970 insertions(+) create mode 100644 config/migrate-from-web.yml create mode 100644 src/services/Elastic.Changelog/Migration/MigrateFromWebScope.cs create mode 100644 src/services/Elastic.Changelog/Migration/ReleaseNotesPageParser.cs create mode 100644 src/services/Elastic.Changelog/Migration/WebMigrationService.cs diff --git a/config/migrate-from-web.yml b/config/migrate-from-web.yml new file mode 100644 index 0000000000..3474ad9772 --- /dev/null +++ b/config/migrate-from-web.yml @@ -0,0 +1,18 @@ +# Scope/cutoff list for the TEMPORARY `docs-builder changelog migrate-from-web` command +# (elastic/docs-eng-team#736). Delete this file together with the command once the migration +# rollout (elastic/docs-eng-team#683) completes. +# +# Each entry maps a product id (the `bundle/{product}/` S3 prefix, see config/products.yml) to: +# owner/repo — the repository whose docs back the published release notes +# path — repository-relative path of the release-notes Markdown page +# ref — pinned commit SHA at which the Markdown is fetched (reproducible runs) +# cutoff — inclusive upper version bound; releases above it are owned by the live pipeline +products: + edot-java: + owner: elastic + repo: elastic-otel-java + path: docs/release-notes/index.md + # Last commit before the repo switched to native docs-builder bundle YAMLs (#1023): + # the final hand-authored state of the published release-notes Markdown. + ref: 9a61ce4faaf08e272c433a083bcc6f0e96d80e0a + cutoff: 1.10.0 diff --git a/docs/cli-schema.json b/docs/cli-schema.json index baa460f695..312ef8b5c6 100644 --- a/docs/cli-schema.json +++ b/docs/cli-schema.json @@ -3815,6 +3815,108 @@ } ] }, + { + "path": [ + "changelog" + ], + "name": "migrate-from-web", + "summary": "TEMPORARY: One-off migration of published release notes into the S3 bundle store; removed after elastic/docs-eng-team#683.", + "notes": "Fetches the release-notes Markdown that backs the published pages (at the pinned git ref in the\nchecked-in scope config), maps it to the existing bundle YAML shape, and uploads to\nbundle/{product}/ with create-only semantics (If-None-Match: *) \u2014 existing keys are\nskipped, never overwritten. Prints a per-key run report (created / skipped / failed with reason and\nobject ETag) suitable for pasting into the tracking issue.\n\nScope is always explicit: the product must have an entry in config/migrate-from-web.yml\n(product id \u2192 source repo, release-notes path, pinned ref, version cutoff). Nothing runs implicitly\nfor all products. Tracked by elastic/docs-eng-team#736.", + "usage": "docs-builder changelog migrate-from-web \u003Cproduct\u003E [options]", + "examples": [], + "parameters": [ + { + "role": "positional", + "name": "product", + "type": "string", + "required": true, + "summary": "Product id to migrate (e.g. \u0022edot-java\u0022). Must have an entry in the checked-in scope config." + }, + { + "role": "flag", + "name": "s3-bucket-name", + "type": "string", + "required": false, + "summary": "Destination S3 bucket. Required unless --dry-run; when provided with --dry-run, existing keys are still inspected so the report distinguishes would-create from skipped." + }, + { + "role": "flag", + "name": "config", + "type": "string", + "required": false, + "summary": "Path to the scope config. Defaults to config/migrate-from-web.yml in the current directory (the copy checked into the docs-builder repository).", + "validations": [ + { + "kind": "rejectSymbolicLinks" + }, + { + "kind": "existing" + }, + { + "kind": "fileExtensions", + "values": [ + "yml", + "yaml" + ] + } + ] + }, + { + "role": "flag", + "name": "versions", + "type": "array", + "required": false, + "summary": "Optional: restrict the run to specific versions (comma-separated or repeated). Versions above the configured cutoff are always skipped.", + "repeatable": true, + "elementType": "string" + }, + { + "role": "dryRun", + "name": "dry-run", + "type": "boolean", + "required": false, + "summary": "Do everything except the S3 writes and report what would be created.", + "defaultValue": "false" + }, + { + "role": "flag", + "name": "log-level", + "shortName": "l", + "type": "enum", + "required": false, + "summary": "Minimum log level. Default: information", + "enumValues": [ + "trace", + "debug", + "information", + "warning", + "error", + "critical", + "none" + ] + }, + { + "role": "flag", + "name": "config-source", + "shortName": "c", + "type": "enum", + "required": false, + "summary": "Override the configuration source: local, remote", + "enumValues": [ + "local", + "remote", + "embedded" + ] + }, + { + "role": "flag", + "name": "skip-private-repositories", + "type": "boolean", + "required": false, + "summary": "Skip cloning private repositories" + } + ] + }, { "path": [ "changelog" diff --git a/src/services/Elastic.Changelog/Migration/MigrateFromWebScope.cs b/src/services/Elastic.Changelog/Migration/MigrateFromWebScope.cs new file mode 100644 index 0000000000..b294f2fe01 --- /dev/null +++ b/src/services/Elastic.Changelog/Migration/MigrateFromWebScope.cs @@ -0,0 +1,127 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Elastic.Documentation.Configuration.ReleaseNotes; +using Elastic.Documentation.Diagnostics; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Elastic.Changelog.Migration; + +/// +/// TEMPORARY (elastic/docs-eng-team#736): the checked-in scope/cutoff list for +/// changelog migrate-from-web. Maps a product id to the repository, path, and pinned git ref +/// of the published release-notes Markdown, plus the inclusive version cutoff for the migration. +/// Delete together with the command once the rollout (elastic/docs-eng-team#683) completes. +/// +public sealed class MigrateFromWebScope +{ + public required string ProductId { get; init; } + public required string Owner { get; init; } + public required string Repo { get; init; } + public required string Path { get; init; } + public required string Ref { get; init; } + public required string Cutoff { get; init; } + + /// + /// Loads and validates the scope entry for from the checked-in + /// scope config at , or null (with errors emitted) when the file, + /// product, or any required field is missing. + /// + public static MigrateFromWebScope? Load(IDiagnosticsCollector collector, IFileSystem fileSystem, string configPath, string productId) + { + if (!fileSystem.File.Exists(configPath)) + { + collector.EmitError(configPath, "Scope config not found. The migrate-from-web scope list is checked into the docs-builder repository (config/migrate-from-web.yml); pass --config when running from elsewhere."); + return null; + } + + MigrateFromWebConfigDto? dto; + try + { + var deserializer = new StaticDeserializerBuilder(new MigrationYamlContext()) + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build(); + dto = deserializer.Deserialize(fileSystem.File.ReadAllText(configPath)); + } + catch (Exception ex) when (ex is YamlDotNet.Core.YamlException or IOException) + { + collector.EmitError(configPath, $"Could not parse scope config: {ex.Message}", ex); + return null; + } + + if (dto?.Products is null || !dto.Products.TryGetValue(productId, out var product) || product is null) + { + var known = dto?.Products is { Count: > 0 } ? string.Join(", ", dto.Products.Keys.Order(StringComparer.Ordinal)) : ""; + collector.EmitError(configPath, $"Product '{productId}' is not in the migrate-from-web scope config. Configured products: {known}. Add an entry before running the migration."); + return null; + } + + if (!ChangelogKeys.IsValidProduct(productId)) + { + collector.EmitError(configPath, $"Product id '{productId}' is not a valid bundle key segment (must match [a-zA-Z0-9_-]+)."); + return null; + } + + var missing = new List(); + if (string.IsNullOrWhiteSpace(product.Owner)) + missing.Add("owner"); + if (string.IsNullOrWhiteSpace(product.Repo)) + missing.Add("repo"); + if (string.IsNullOrWhiteSpace(product.Path)) + missing.Add("path"); + if (string.IsNullOrWhiteSpace(product.Ref)) + missing.Add("ref"); + if (string.IsNullOrWhiteSpace(product.Cutoff)) + missing.Add("cutoff"); + + if (missing.Count > 0) + { + collector.EmitError(configPath, $"Scope entry for '{productId}' is missing required field(s): {string.Join(", ", missing)}."); + return null; + } + + return new MigrateFromWebScope + { + ProductId = productId, + Owner = product.Owner!, + Repo = product.Repo!, + Path = product.Path!, + Ref = product.Ref!, + Cutoff = product.Cutoff! + }; + } +} + +/// Root DTO of the checked-in migrate-from-web scope config (product id → source/cutoff). +public sealed class MigrateFromWebConfigDto +{ + public Dictionary? Products { get; set; } +} + +/// One product's scope: where its published release-notes Markdown lives and the migration cutoff. +public sealed class MigrateFromWebProductDto +{ + /// GitHub owner of the source repository (e.g. elastic). + public string? Owner { get; set; } + + /// Source repository name (e.g. elastic-otel-java). + public string? Repo { get; set; } + + /// Repository-relative path of the release-notes Markdown page. + public string? Path { get; set; } + + /// Pinned git ref (commit SHA) at which the Markdown is fetched. + public string? Ref { get; set; } + + /// Inclusive upper version bound; releases above it belong to the live pipeline. + public string? Cutoff { get; set; } +} + +/// Source-generated YAML context for the migrate-from-web scope config (AOT-safe, no reflection). +[YamlStaticContext] +[YamlSerializable(typeof(MigrateFromWebConfigDto))] +[YamlSerializable(typeof(MigrateFromWebProductDto))] +public partial class MigrationYamlContext; diff --git a/src/services/Elastic.Changelog/Migration/ReleaseNotesPageParser.cs b/src/services/Elastic.Changelog/Migration/ReleaseNotesPageParser.cs new file mode 100644 index 0000000000..41d4896743 --- /dev/null +++ b/src/services/Elastic.Changelog/Migration/ReleaseNotesPageParser.cs @@ -0,0 +1,307 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; +using Elastic.Documentation; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.ReleaseNotes; +using Elastic.Documentation.Versions; + +namespace Elastic.Changelog.Migration; + +/// A single release parsed from a published release-notes Markdown page, mapped to the existing bundle shape. +public sealed record MigratedRelease +{ + public required string Version { get; init; } + public required Bundle Bundle { get; init; } +} + +/// +/// TEMPORARY (elastic/docs-eng-team#736): parses a hand-authored release-notes Markdown page — +/// ## {version} sections with typed ### {section} subsections and bullet entries — +/// into the existing shape with inline entries. No new schema is introduced. +/// Delete together with the migrate-from-web command once the rollout (elastic/docs-eng-team#683) completes. +/// +public static partial class ReleaseNotesPageParser +{ + [GeneratedRegex(@"^##\s+(?\S+)(?:\s+\[[^\]]*\])?\s*$")] + private static partial Regex VersionHeadingRegex(); + + [GeneratedRegex(@"^###\s+(?.+?)(?:\s*\[[^\]]*\])?\s*$")] + private static partial Regex SubsectionHeadingRegex(); + + [GeneratedRegex(@"^\*\*Release date:?\*\*:?\s*(?<date>.+?)\s*$")] + private static partial Regex ReleaseDateRegex(); + + [GeneratedRegex(@"\[#?\d+\]\((?<url>https://github\.com/[^\s)]+/pull/\d+)\)")] + private static partial Regex PrLinkRegex(); + + [GeneratedRegex(@"(?<=^|[\s(])#(?<number>\d+)\b")] + private static partial Regex BarePrRefRegex(); + + /// <summary> + /// Parses <paramref name="markdown"/> into one <see cref="MigratedRelease"/> per <c>## {version}</c> + /// section. Content that cannot be mapped to typed entries is preserved verbatim in the bundle + /// description so no published content is dropped; anything ambiguous emits a warning on + /// <paramref name="collector"/> so the operator can review it before uploading. + /// </summary> + public static IReadOnlyList<MigratedRelease> Parse( + IDiagnosticsCollector collector, + string markdown, + string sourceId, + MigrateFromWebScope scope) + { + var lines = markdown.Replace("\r\n", "\n", StringComparison.Ordinal).Split('\n'); + var releases = new List<MigratedRelease>(); + SectionBuilder? section = null; + + var index = SkipFrontmatter(lines); + for (; index < lines.Length; index++) + { + var line = lines[index]; + + // MyST comment lines never contribute content (used for authoring templates like "% ## version.next"). + if (line.TrimStart().StartsWith('%')) + continue; + + var versionMatch = VersionHeadingRegex().Match(line); + if (versionMatch.Success) + { + AddCompleted(releases, section); + section = StartSection(collector, versionMatch.Groups["version"].Value, sourceId, scope); + continue; + } + + // Content before the first "## {version}" heading is the page intro, not release content. + section?.ConsumeLine(collector, line); + } + + AddCompleted(releases, section); + return releases; + } + + private static int SkipFrontmatter(string[] lines) + { + if (lines.Length == 0 || lines[0].TrimEnd() != "---") + return 0; + + for (var i = 1; i < lines.Length; i++) + { + if (lines[i].TrimEnd() == "---") + return i + 1; + } + + return 0; + } + + private static SectionBuilder? StartSection(IDiagnosticsCollector collector, string version, string sourceId, MigrateFromWebScope scope) + { + // A heading token that is neither a version nor a date (e.g. a prose heading) is not a release + // section; skip it entirely rather than fabricating a bundle for it. + if (VersionOrDate.Parse(version).Raw is not null) + { + collector.EmitWarning(sourceId, $"Skipping section '## {version}': heading is not a recognizable version or date."); + return null; + } + + return new SectionBuilder(version, sourceId, scope); + } + + private static void AddCompleted(List<MigratedRelease> releases, SectionBuilder? section) + { + if (section?.Build() is { } release) + releases.Add(release); + } + + /// <summary>Accumulates one <c>## {version}</c> section's date, description, and typed entries.</summary> + private sealed class SectionBuilder(string version, string sourceId, MigrateFromWebScope scope) + { + private readonly StringBuilder _description = new(); + private readonly List<BundledEntry> _entries = []; + private DateOnly? _releaseDate; + private ChangelogEntryType? _entryType; + private bool _collectingEntries; + private int _lastEntryIndex = -1; + + public void ConsumeLine(IDiagnosticsCollector collector, string line) + { + var subsectionMatch = SubsectionHeadingRegex().Match(line); + if (subsectionMatch.Success) + { + ConsumeSubsectionHeading(collector, line, subsectionMatch.Groups["title"].Value); + return; + } + + var dateMatch = ReleaseDateRegex().Match(line); + if (dateMatch.Success && TryParseReleaseDate(dateMatch.Groups["date"].Value, out var date)) + { + _releaseDate = date; + return; + } + + if (dateMatch.Success) + collector.EmitWarning(sourceId, $"Could not parse release date '{dateMatch.Groups["date"].Value}' for {version}; keeping the line as description text."); + + ConsumeContentLine(line); + } + + private void ConsumeSubsectionHeading(IDiagnosticsCollector collector, string line, string title) + { + var type = ResolveSectionType(title); + if (type is null) + { + // Unrecognized subsections flow into the description verbatim (heading included) so the + // published content is preserved even when it cannot be mapped to typed entries. + collector.EmitWarning(sourceId, $"Unrecognized subsection '### {title.Trim()}' under {version}; preserving it in the bundle description."); + _entryType = null; + _collectingEntries = false; + AppendDescriptionLine(line); + return; + } + + _entryType = type; + _collectingEntries = true; + } + + private void ConsumeContentLine(string line) + { + var isBlank = string.IsNullOrWhiteSpace(line); + var trimmed = line.TrimStart(); + var isBullet = trimmed.StartsWith("* ", StringComparison.Ordinal) || trimmed.StartsWith("- ", StringComparison.Ordinal); + var isIndented = !isBlank && line.Length > 0 && (line[0] == ' ' || line[0] == '\t'); + + if (_collectingEntries) + { + if (isBlank) + return; + + if (isBullet && !isIndented) + { + _entries.Add(ParseEntry(trimmed[2..], _entryType!.Value, scope)); + _lastEntryIndex = _entries.Count - 1; + return; + } + + // A wrapped bullet continues the previous entry. + if (isIndented && _lastEntryIndex >= 0) + { + var entry = _entries[_lastEntryIndex]; + _entries[_lastEntryIndex] = MergeContinuation(entry, trimmed, scope); + return; + } + + // First plain paragraph after the entry list ends entry collection: trailing prose + // (and any lists inside it) belongs to the description, not to the entries. + _collectingEntries = false; + _lastEntryIndex = -1; + } + + AppendDescriptionLine(line); + } + + private void AppendDescriptionLine(string line) + { + if (_description.Length == 0 && string.IsNullOrWhiteSpace(line)) + return; + _ = _description.Append(line.TrimEnd()).Append('\n'); + } + + public MigratedRelease Build() + { + var description = _description.ToString().Trim(); + return new MigratedRelease + { + Version = version, + Bundle = new Bundle + { + Products = + [ + new BundledProduct + { + ProductId = scope.ProductId, + Target = version, + Lifecycle = Lifecycle.Ga, + Repo = scope.Repo, + Owner = scope.Owner + } + ], + Description = description.Length > 0 ? description : null, + ReleaseDate = _releaseDate, + Entries = _entries + } + }; + } + } + + private static ChangelogEntryType? ResolveSectionType(string heading) => + heading.Trim().ToLowerInvariant() switch + { + "features and enhancements" or "features" or "enhancements" => ChangelogEntryType.Enhancement, + "fixes" or "bug fixes" => ChangelogEntryType.BugFix, + "breaking changes" => ChangelogEntryType.BreakingChange, + "deprecations" => ChangelogEntryType.Deprecation, + "known issues" => ChangelogEntryType.KnownIssue, + "security" or "security updates" => ChangelogEntryType.Security, + _ => null + }; + + private static bool TryParseReleaseDate(string text, out DateOnly date) + { + string[] formats = ["MMMM d, yyyy", "yyyy-MM-dd"]; + return DateOnly.TryParseExact(text.Trim().TrimEnd('.'), formats, CultureInfo.InvariantCulture, DateTimeStyles.None, out date); + } + + private static BundledEntry ParseEntry(string text, ChangelogEntryType type, MigrateFromWebScope scope) + { + var (title, prs) = ExtractPrReferences(text, scope); + return new BundledEntry + { + Type = type, + Title = title, + Products = [new ProductReference { ProductId = scope.ProductId }], + Prs = prs.Count > 0 ? prs : null + }; + } + + private static BundledEntry MergeContinuation(BundledEntry entry, string continuation, MigrateFromWebScope scope) + { + var (title, prs) = ExtractPrReferences($"{entry.Title} {continuation}", scope); + var mergedPrs = (entry.Prs ?? []).Concat(prs).Distinct(StringComparer.Ordinal).ToList(); + return entry with { Title = title, Prs = mergedPrs.Count > 0 ? mergedPrs : null }; + } + + /// <summary> + /// Extracts PR references from bullet text — Markdown links like <c>[#899](…/pull/899)</c> or + /// <c>[835](…/pull/835)</c>, and bare <c>#958</c> refs resolved against the scope's repository — + /// returning the cleaned-up title and the collected PR URLs. + /// </summary> + private static (string Title, List<string> Prs) ExtractPrReferences(string text, MigrateFromWebScope scope) + { + var prs = new List<string>(); + + var title = PrLinkRegex().Replace(text, m => + { + prs.Add(m.Groups["url"].Value); + return string.Empty; + }); + + title = BarePrRefRegex().Replace(title, m => + { + prs.Add($"https://github.com/{scope.Owner}/{scope.Repo}/pull/{m.Groups["number"].Value}"); + return string.Empty; + }); + + return (NormalizeTitle(title), prs); + } + + private static string NormalizeTitle(string title) + { + // Removing PR tokens can leave empty parentheses and dangling separators behind. + title = title.Replace("()", string.Empty, StringComparison.Ordinal); + var collapsed = string.Join(' ', title.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + return collapsed.TrimEnd(' ', '-', '–', '—', ':', ',', ';'); + } +} diff --git a/src/services/Elastic.Changelog/Migration/WebMigrationService.cs b/src/services/Elastic.Changelog/Migration/WebMigrationService.cs new file mode 100644 index 0000000000..63e30f1ac8 --- /dev/null +++ b/src/services/Elastic.Changelog/Migration/WebMigrationService.cs @@ -0,0 +1,360 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO.Abstractions; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using Amazon.S3; +using Amazon.S3.Model; +using Elastic.Changelog.Uploading; +using Elastic.Documentation; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.ReleaseNotes; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Integrations.S3; +using Elastic.Documentation.Services; +using Elastic.Documentation.Versions; +using Microsoft.Extensions.Logging; +using Nullean.ScopedFileSystem; + +namespace Elastic.Changelog.Migration; + +public record MigrateFromWebArguments +{ + /// <summary>Product id to migrate; must have an entry in the checked-in scope config.</summary> + public required string Product { get; init; } + + /// <summary>Destination S3 bucket. Optional in dry-run mode (no S3 access at all when omitted).</summary> + public string S3BucketName { get; init; } = ""; + + /// <summary>When true, does everything except the S3 writes and reports what would be created.</summary> + public bool DryRun { get; init; } + + /// <summary>Path to the scope config. Defaults to the checked-in <c>config/migrate-from-web.yml</c>.</summary> + public string? Config { get; init; } + + /// <summary>Optional exact-version filter; when set, only these versions are migrated.</summary> + public IReadOnlyList<string> Versions { get; init; } = []; +} + +/// <summary>Per-key outcome of a migration run, printed as the run report / paper trail.</summary> +public sealed record MigrationKeyResult(string Key, string Outcome, string? ETag, string Detail); + +/// <summary> +/// TEMPORARY (elastic/docs-eng-team#736): one-off migration of published release notes into the +/// S3 bundle store. Fetches the release-notes Markdown that backs the published pages (pinned ref, +/// raw.githubusercontent.com), maps it to the existing bundle YAML shape, and uploads with +/// create-only semantics (<c>If-None-Match: *</c>) — existing keys are skipped, never overwritten. +/// Delete once the migration rollout (elastic/docs-eng-team#683) completes. +/// </summary> +public class WebMigrationService( + ILoggerFactory logFactory, + ScopedFileSystem? fileSystem = null, + IAmazonS3? s3Client = null, + HttpMessageHandler? httpMessageHandler = null +) : IService +{ + public const string DefaultConfigPath = "config/migrate-from-web.yml"; + + private const string OutcomeCreated = "created"; + private const string OutcomeWouldCreate = "would-create"; + private const string OutcomeSkipped = "skipped"; + private const string OutcomeFailed = "failed"; + + private readonly ILogger _logger = logFactory.CreateLogger<WebMigrationService>(); + private readonly IFileSystem _fileSystem = fileSystem ?? FileSystemFactory.RealWrite; + + /// <summary>Per-key results of the most recent run; exposed for tests.</summary> + internal IReadOnlyList<MigrationKeyResult> LastResults { get; private set; } = []; + + public async Task<bool> MigrateFromWeb(IDiagnosticsCollector collector, MigrateFromWebArguments args, Cancel ctx) + { + var configPath = string.IsNullOrWhiteSpace(args.Config) + ? _fileSystem.Path.GetFullPath(DefaultConfigPath) + : args.Config; + + var scope = MigrateFromWebScope.Load(collector, _fileSystem, configPath, args.Product); + if (scope is null) + return false; + + var sourceUrl = $"https://raw.githubusercontent.com/{scope.Owner}/{scope.Repo}/{scope.Ref}/{scope.Path}"; + var markdown = await FetchMarkdown(collector, sourceUrl, ctx); + if (markdown is null) + return false; + + var releases = ReleaseNotesPageParser.Parse(collector, markdown, sourceUrl, scope); + if (releases.Count == 0) + { + collector.EmitError(sourceUrl, "No release sections were parsed from the published release notes; refusing to continue with an empty scope."); + return false; + } + + _logger.LogInformation("Parsed {Count} release section(s) from {Url}", releases.Count, sourceUrl); + + var (inScope, results) = ApplyScopeFilters(releases, scope, args.Versions); + var staged = StageBundles(collector, scope, inScope); + if (collector.Errors > 0) + return false; + + var uploadResults = await UploadCreateOnly(collector, args, staged, ctx); + results.AddRange(uploadResults); + LastResults = results; + + var failed = results.Count(r => r.Outcome == OutcomeFailed); + var report = FormatReport(scope, args, results); + Console.WriteLine(report); + + if (failed > 0) + collector.EmitError(string.Empty, $"{failed} key(s) failed to migrate; see the run report above."); + + return failed == 0; + } + + private async Task<string?> FetchMarkdown(IDiagnosticsCollector collector, string sourceUrl, Cancel ctx) + { + try + { + using var client = httpMessageHandler is null + ? new HttpClient { Timeout = TimeSpan.FromSeconds(30) } + : new HttpClient(httpMessageHandler, disposeHandler: false) { Timeout = TimeSpan.FromSeconds(30) }; + client.DefaultRequestHeaders.Add("User-Agent", "docs-builder"); + + using var response = await client.GetAsync(sourceUrl, ctx); + if (!response.IsSuccessStatusCode) + { + collector.EmitError(sourceUrl, $"Fetching published release notes failed with HTTP {(int)response.StatusCode} ({response.StatusCode})."); + return null; + } + + return await response.Content.ReadAsStringAsync(ctx); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + collector.EmitError(sourceUrl, $"Fetching published release notes failed: {ex.Message}", ex); + return null; + } + } + + /// <summary> + /// Partitions parsed releases into the in-scope set and out-of-scope report lines. Versions above + /// the configured cutoff belong to the live pipeline; an explicit <c>--versions</c> selection + /// narrows the scope further. + /// </summary> + private static (List<MigratedRelease> InScope, List<MigrationKeyResult> Results) ApplyScopeFilters( + IReadOnlyList<MigratedRelease> releases, + MigrateFromWebScope scope, + IReadOnlyList<string> versions) + { + var cutoff = VersionOrDate.Parse(scope.Cutoff); + var selection = versions.Count > 0 ? new HashSet<string>(versions, StringComparer.OrdinalIgnoreCase) : null; + + var inScope = new List<MigratedRelease>(); + var results = new List<MigrationKeyResult>(); + foreach (var release in releases) + { + var key = ChangelogKeys.BundleFileKey(scope.ProductId, $"{release.Version}.yaml"); + if (VersionOrDate.Parse(release.Version).CompareTo(cutoff) > 0) + { + results.Add(new MigrationKeyResult(key, OutcomeSkipped, null, $"beyond cutoff {scope.Cutoff}; owned by the live pipeline")); + continue; + } + + if (selection is not null && !selection.Contains(release.Version)) + { + results.Add(new MigrationKeyResult(key, OutcomeSkipped, null, "not in --versions selection")); + continue; + } + + inScope.Add(release); + } + + return (inScope, results); + } + + private sealed record StagedBundle(string Key, string LocalPath, string LocalETag); + + /// <summary> + /// Serializes each in-scope release to bundle YAML and stages it in a temp directory so the + /// existing registry-refresh machinery (which reads local files) can be reused as-is. + /// </summary> + [SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms", Justification = "MD5 matches the S3 single-part ETag, used for content comparison only")] + private List<StagedBundle> StageBundles(IDiagnosticsCollector collector, MigrateFromWebScope scope, IReadOnlyList<MigratedRelease> releases) + { + var stagingDir = _fileSystem.Path.Join(_fileSystem.Path.GetTempPath(), "docs-builder-migrate-from-web", scope.ProductId); + var staged = new List<StagedBundle>(releases.Count); + + try + { + _ = _fileSystem.Directory.CreateDirectory(stagingDir); + foreach (var release in releases) + { + var yaml = ReleaseNotesSerialization.SerializeBundle(release.Bundle); + var bytes = Encoding.UTF8.GetBytes(yaml); + var localPath = _fileSystem.Path.Join(stagingDir, $"{release.Version}.yaml"); + _fileSystem.File.WriteAllBytes(localPath, bytes); + + var key = ChangelogKeys.BundleFileKey(scope.ProductId, $"{release.Version}.yaml"); + staged.Add(new StagedBundle(key, localPath, Convert.ToHexStringLower(MD5.HashData(bytes)))); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + collector.EmitError(stagingDir, $"Could not stage mapped bundles: {ex.Message}", ex); + } + + return staged; + } + + private async Task<List<MigrationKeyResult>> UploadCreateOnly( + IDiagnosticsCollector collector, + MigrateFromWebArguments args, + IReadOnlyList<StagedBundle> staged, + Cancel ctx) + { + // A credential-free dry run: without a bucket there is nothing to compare against, so every + // in-scope key is reported as would-create. + if (args.DryRun && string.IsNullOrWhiteSpace(args.S3BucketName)) + return [.. staged.Select(s => new MigrationKeyResult(s.Key, OutcomeWouldCreate, s.LocalETag, "no bucket specified; existence not checked"))]; + + using var defaultClient = s3Client is null ? new AmazonS3Client() : null; + var client = s3Client ?? defaultClient!; + + var results = new List<MigrationKeyResult>(staged.Count); + var uploadedTargets = new List<UploadTarget>(); + foreach (var bundle in staged) + { + ctx.ThrowIfCancellationRequested(); + var result = await MigrateKey(client, args, bundle, ctx); + results.Add(result); + if (result.Outcome is OutcomeCreated || (result.Outcome is OutcomeSkipped && result.ETag == bundle.LocalETag)) + uploadedTargets.Add(new UploadTarget(bundle.LocalPath, bundle.Key)); + } + + var created = results.Count(r => r.Outcome == OutcomeCreated); + if (!args.DryRun && created > 0) + await RefreshRegistry(collector, client, args.S3BucketName, uploadedTargets, ctx); + + return results; + } + + private async Task<MigrationKeyResult> MigrateKey(IAmazonS3 client, MigrateFromWebArguments args, StagedBundle bundle, Cancel ctx) + { + try + { + var remoteEtag = await GetRemoteEtag(client, args.S3BucketName, bundle.Key, ctx); + if (remoteEtag is not null) + { + var detail = remoteEtag == bundle.LocalETag + ? "already exists with identical content" + : "already exists with different content; never overwritten"; + return new MigrationKeyResult(bundle.Key, OutcomeSkipped, remoteEtag, detail); + } + + if (args.DryRun) + return new MigrationKeyResult(bundle.Key, OutcomeWouldCreate, bundle.LocalETag, "dry run; no write performed"); + + // The conditional PUT is the actual race guard: a key created between the inspection above + // and this write surfaces as a 412 and is skipped, never overwritten. + var response = await PutCreateOnly(client, args.S3BucketName, bundle, ctx); + return new MigrationKeyResult(bundle.Key, OutcomeCreated, response.ETag?.Trim('"'), ""); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.PreconditionFailed) + { + return new MigrationKeyResult(bundle.Key, OutcomeSkipped, null, "created concurrently by another writer; never overwritten"); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogError(ex, "Failed to migrate {Key}", bundle.Key); + return new MigrationKeyResult(bundle.Key, OutcomeFailed, null, ex.Message); + } + } + + private static async Task<string?> GetRemoteEtag(IAmazonS3 client, string bucketName, string key, Cancel ctx) + { + try + { + var response = await client.GetObjectMetadataAsync(new GetObjectMetadataRequest + { + BucketName = bucketName, + Key = key + }, ctx); + return response.ETag.Trim('"'); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } + + private async Task<PutObjectResponse> PutCreateOnly(IAmazonS3 client, string bucketName, StagedBundle bundle, Cancel ctx) + { + _logger.LogInformation("Creating s3://{Bucket}/{Key} (If-None-Match: *)", bucketName, bundle.Key); + await using var stream = _fileSystem.FileStream.New(bundle.LocalPath, FileMode.Open, FileAccess.Read, FileShare.Read); + var request = new PutObjectRequest + { + BucketName = bucketName, + Key = bundle.Key, + InputStream = stream, + ChecksumAlgorithm = ChecksumAlgorithm.SHA256, + IfNoneMatch = "*" + }; + return await client.PutObjectAsync(request, ctx); + } + + /// <summary> + /// Refreshes <c>bundle/{product}/registry.json</c> with the keys this run created (plus keys that + /// already existed with identical content), reusing the live pipeline's registry machinery. + /// Best-effort, matching live uploads: the bundles themselves are already in S3. + /// </summary> + private async Task RefreshRegistry( + IDiagnosticsCollector collector, + IAmazonS3 client, + string bucketName, + IReadOnlyList<UploadTarget> targets, + Cancel ctx) + { + try + { + var etagCalculator = new S3EtagCalculator(logFactory, _fileSystem); + var builder = new RegistryBuilder(logFactory, _fileSystem, client, etagCalculator, bucketName); + var result = await builder.RefreshAsync(collector, targets, ctx, RegistryScope.Bundle); + _logger.LogInformation("Registry refresh: {Updated} updated, {Unchanged} unchanged, {Failed} failed", + result.Updated, result.Unchanged, result.Failed); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Registry refresh failed; migrated bundles are in S3 but the manifest may be stale"); + collector.EmitWarning(string.Empty, $"Failed to refresh registry manifest: {ex.Message}"); + } + } + + /// <summary>Formats the run report — one line per key — in a form that can be pasted into the tracking issue.</summary> + public static string FormatReport(MigrateFromWebScope scope, MigrateFromWebArguments args, IReadOnlyList<MigrationKeyResult> results) + { + var sb = new StringBuilder(); + _ = sb.AppendLine("### Run report: changelog migrate-from-web"); + _ = sb.AppendLine(); + _ = sb.AppendLine(CultureInfo.InvariantCulture, $"- product: `{scope.ProductId}`"); + _ = sb.AppendLine(CultureInfo.InvariantCulture, $"- source: `{scope.Owner}/{scope.Repo}@{scope.Ref}` `{scope.Path}`"); + _ = sb.AppendLine(CultureInfo.InvariantCulture, $"- cutoff: `{scope.Cutoff}`"); + _ = sb.AppendLine(CultureInfo.InvariantCulture, $"- mode: {(args.DryRun ? "dry-run (no S3 writes)" : $"upload to `{args.S3BucketName}`")}"); + _ = sb.AppendLine(); + _ = sb.AppendLine("| key | outcome | etag | detail |"); + _ = sb.AppendLine("|---|---|---|---|"); + + foreach (var result in results.OrderBy(r => r.Key, StringComparer.Ordinal)) + _ = sb.AppendLine(CultureInfo.InvariantCulture, $"| `{result.Key}` | {result.Outcome} | {(result.ETag is null ? "" : $"`{result.ETag}`")} | {result.Detail} |"); + + var counts = results + .GroupBy(r => r.Outcome) + .OrderBy(g => g.Key, StringComparer.Ordinal) + .Select(g => $"{g.Key} {g.Count()}"); + _ = sb.AppendLine(); + _ = sb.AppendLine(CultureInfo.InvariantCulture, $"totals: {string.Join(", ", counts)}"); + return sb.ToString(); + } +} diff --git a/src/tooling/docs-builder/Commands/ChangelogCommand.cs b/src/tooling/docs-builder/Commands/ChangelogCommand.cs index bff225100f..3acffec863 100644 --- a/src/tooling/docs-builder/Commands/ChangelogCommand.cs +++ b/src/tooling/docs-builder/Commands/ChangelogCommand.cs @@ -17,6 +17,7 @@ using Elastic.Changelog.Evaluation; using Elastic.Changelog.GitHub; using Elastic.Changelog.GithubRelease; +using Elastic.Changelog.Migration; using Elastic.Changelog.Rendering; using Elastic.Changelog.Uploading; using Elastic.Changelog.Utilities; @@ -1725,6 +1726,61 @@ static async (s, c, state, ct) => await s.ResolveDeployedAsync(c, state, ct) is return await serviceInvoker.InvokeAsync(ctx); } + /// <summary>TEMPORARY: One-off migration of published release notes into the S3 bundle store; removed after elastic/docs-eng-team#683.</summary> + /// <remarks> + /// <para>Fetches the release-notes Markdown that backs the published pages (at the pinned git ref in the + /// checked-in scope config), maps it to the existing bundle YAML shape, and uploads to + /// <c>bundle/{product}/</c> with create-only semantics (<c>If-None-Match: *</c>) — existing keys are + /// skipped, never overwritten. Prints a per-key run report (created / skipped / failed with reason and + /// object ETag) suitable for pasting into the tracking issue.</para> + /// <para>Scope is always explicit: the product must have an entry in <c>config/migrate-from-web.yml</c> + /// (product id → source repo, release-notes path, pinned ref, version cutoff). Nothing runs implicitly + /// for all products. Tracked by elastic/docs-eng-team#736.</para> + /// </remarks> + /// <param name="product">Product id to migrate (e.g. "edot-java"). Must have an entry in the checked-in scope config.</param> + /// <param name="s3BucketName">Destination S3 bucket. Required unless --dry-run; when provided with --dry-run, existing keys are still inspected so the report distinguishes would-create from skipped.</param> + /// <param name="config">Path to the scope config. Defaults to config/migrate-from-web.yml in the current directory (the copy checked into the docs-builder repository).</param> + /// <param name="versions">Optional: restrict the run to specific versions (comma-separated or repeated). Versions above the configured cutoff are always skipped.</param> + /// <param name="dryRun">Do everything except the S3 writes and report what would be created.</param> + /// <param name="ctx">Cancellation token</param> + [NoOptionsInjection] + public async Task<int> MigrateFromWeb( + [Argument] string product, + string s3BucketName = "", + [Existing, ExpandUserProfile, RejectSymbolicLinks, FileExtensions(Extensions = "yml,yaml")] FileInfo? config = null, + string[]? versions = null, + [DryRun] bool dryRun = false, + CancellationToken ct = default + ) + { + var ctx = ct; + await using var serviceInvoker = new ServiceInvoker(collector); + + if (!dryRun && string.IsNullOrWhiteSpace(s3BucketName)) + { + collector.EmitError(string.Empty, "--s3-bucket-name is required unless --dry-run is specified."); + _ = collector.StartAsync(ctx); + await collector.WaitForDrain(); + await collector.StopAsync(ctx); + return 1; + } + + var service = new WebMigrationService(logFactory, FileSystemFactory.RealWrite); + var args = new MigrateFromWebArguments + { + Product = product, + S3BucketName = s3BucketName, + DryRun = dryRun, + Config = config?.FullName, + Versions = ExpandCommaSeparated(versions) + }; + + serviceInvoker.AddCommand(service, args, + static async (s, c, state, ct) => await s.MigrateFromWeb(c, state, ct) + ); + return await serviceInvoker.InvokeAsync(ctx); + } + /// <summary>Resolves the authoring repo/owner/branch for uploads (CLI flags > <c>bundle.{repo,owner}</c> > git); owner falls back to the <c>owner/</c> prefix of repo (<see cref="ChangelogRepoOwnerResolver"/>) before git, reducing the repo to a single path segment.</summary> private async Task<(string? Repo, string? Owner, string? Branch)> ResolveUploadRepoOwnerBranch(string? repoCli, string? ownerCli, string? branchCli, string? configPath, string? uploadDirectory, CancellationToken ctx) { From d92a3d6243ef514b3ec03829d5d68e98dbd07646 Mon Sep 17 00:00:00 2001 From: Felipe Cotti <felipe.cotti@elastic.co> Date: Thu, 6 Aug 2026 11:17:34 -0300 Subject: [PATCH 2/5] Changelog: test and document migrate-from-web --- docs/cli/changelog/cmd-migrate-from-web.md | 84 +++++ .../Migration/ReleaseNotesFixture.cs | 98 +++++ .../Migration/ReleaseNotesPageParserTests.cs | 181 ++++++++++ .../Migration/WebMigrationServiceTests.cs | 335 ++++++++++++++++++ 4 files changed, 698 insertions(+) create mode 100644 docs/cli/changelog/cmd-migrate-from-web.md create mode 100644 tests/Elastic.Changelog.Tests/Migration/ReleaseNotesFixture.cs create mode 100644 tests/Elastic.Changelog.Tests/Migration/ReleaseNotesPageParserTests.cs create mode 100644 tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs diff --git a/docs/cli/changelog/cmd-migrate-from-web.md b/docs/cli/changelog/cmd-migrate-from-web.md new file mode 100644 index 0000000000..b45abf86ff --- /dev/null +++ b/docs/cli/changelog/cmd-migrate-from-web.md @@ -0,0 +1,84 @@ +## Description + +:::{warning} +This command is **temporary**. It exists solely to migrate release notes that were published before the changelog pipeline existed into the S3 bundle store, and it will be deleted once the migration rollout ([docs-eng-team#683](https://github.com/elastic/docs-eng-team/issues/683)) completes. Do not build workflows on top of it. +::: + +One-off migration of already-published release notes into the S3 bundle store. For each product in scope, the command: + +1. Fetches the release-notes Markdown that backs the published pages — from `raw.githubusercontent.com` at the pinned commit recorded in the scope config, not by scraping live site HTML. +2. Parses each `## {version}` section (typed `### …` subsections become entries; prose is preserved as the bundle description) and maps it to the **existing** bundle YAML shape that [](/cli/changelog/upload.md) publishes. No new schema is introduced. +3. Uploads each release to `bundle/{product}/{version}.yaml` with **create-only** semantics (`If-None-Match: *`): keys that already exist are skipped and never overwritten, so the migration can never clobber bundles produced by the live pipeline. +4. Prints a per-key run report (created / skipped / failed, with the reason and object ETag) suitable for pasting into the tracking issue. + +The scope is always explicit. The product must have an entry in the checked-in scope config (`config/migrate-from-web.yml` in the docs-builder repository); nothing runs implicitly for all products. + +## Scope config + +Each entry maps a product id (the `bundle/{product}/` S3 prefix, see `config/products.yml`) to the source of its published release notes and a version cutoff: + +```yaml +products: + edot-java: + owner: elastic # GitHub owner of the source repository + repo: elastic-otel-java # source repository + path: docs/release-notes/index.md # repo-relative path of the release-notes page + ref: 9a61ce4faaf08e272c433a083bcc6f0e96d80e0a # pinned commit SHA (reproducible runs) + cutoff: 1.10.0 # inclusive; releases above it belong to the live pipeline +``` + +Releases above the cutoff are always skipped — they are owned by the live changelog pipeline. Use `--versions` to narrow a run to specific versions below the cutoff. + +## Requirements + +Uploads use the same AWS SDK credential chain, region, and IAM permissions as [](/cli/changelog/upload.md). No credentials are needed for `--dry-run` without `--s3-bucket-name`. + +## Run report + +The report lists one line per key with its outcome: + +| Outcome | Meaning | +| ------- | ------- | +| `created` | The key did not exist and was written (conditional PUT succeeded). | +| `would-create` | Dry run only: the key would be written. | +| `skipped` | The key already exists (identical or different content — never overwritten), was created concurrently by another writer, is beyond the cutoff, or is not in the `--versions` selection. | +| `failed` | The write failed; the reason is included and the command exits non-zero. | + +After a run that created keys, the command refreshes `bundle/{product}/registry.json` (best-effort, like `changelog upload`). + +## Examples + +### Dry run without credentials + +Parse, map, and report what would be created — no S3 access at all: + +```sh +docs-builder changelog migrate-from-web edot-java --dry-run +``` + +### Dry run against the real bucket + +Also checks which keys already exist, so the report distinguishes `would-create` from `skipped`: + +```sh +docs-builder changelog migrate-from-web edot-java \ + --dry-run \ + --s3-bucket-name my-changelog-bundles +``` + +### Perform the migration + +```sh +docs-builder changelog migrate-from-web edot-java \ + --s3-bucket-name my-changelog-bundles +``` + +Re-running the same command is safe: every existing key is reported as `skipped` and the run is a no-op. + +### Migrate specific versions only + +```sh +docs-builder changelog migrate-from-web edot-java \ + --s3-bucket-name my-changelog-bundles \ + --versions 1.9.0,1.10.0 +``` diff --git a/tests/Elastic.Changelog.Tests/Migration/ReleaseNotesFixture.cs b/tests/Elastic.Changelog.Tests/Migration/ReleaseNotesFixture.cs new file mode 100644 index 0000000000..d5a528d2a9 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Migration/ReleaseNotesFixture.cs @@ -0,0 +1,98 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Changelog.Migration; + +namespace Elastic.Changelog.Tests.Migration; + +/// <summary> +/// A realistic release-notes Markdown fixture, modeled on the published EDOT Java page +/// (elastic/elastic-otel-java docs/release-notes/index.md before the repo switched to +/// native bundle YAMLs). Exercises frontmatter, comment templates, prose-only releases, +/// typed subsections, PR-reference variants, trailing prose, and a post-cutoff release. +/// </summary> +public static class ReleaseNotesFixture +{ + public static MigrateFromWebScope Scope { get; } = new() + { + ProductId = "edot-java", + Owner = "elastic", + Repo = "elastic-otel-java", + Path = "docs/release-notes/index.md", + Ref = "9a61ce4faaf08e272c433a083bcc6f0e96d80e0a", + Cutoff = "1.10.0" + }; + + // language=markdown + public const string Markdown = """ + --- + navigation_title: EDOT Java + description: Release notes for Elastic Distribution of OpenTelemetry Java. + products: + - id: edot-sdk + --- + + # Elastic Distribution of OpenTelemetry Java release notes [edot-java-release-notes] + + Review the changes, fixes, and more in each version. + + % Release notes include only features, enhancements, and fixes. + + % ## version.next [edot-java-X.X.X-release-notes] + + % ### Features and enhancements [edot-java-X.X.X-features-enhancements] + % * + + ## 2.0.0 [edot-java-2-0-0-release-notes] + **Release date:** May 1, 2026 + + ### Features and enhancements [edot-java-2-0-0-features-enhancements] + * A release owned by the live pipeline #1200 + + ## 1.10.0 [edot-java-1-10-0-release-notes] + **Release date:** March 24, 2026 + + The 1.10.0 release contains fixes for potential security vulnerabilities. + Refer to our [security advisory](https://discuss.elastic.co/t/example/385700) for more details. + + This release is based on the following upstream versions: + + * opentelemetry-javaagent: [2.26.1](https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/tag/v2.26.1) + * opentelemetry-sdk: [1.60.1](https://github.com/open-telemetry/opentelemetry-java/releases/tag/v1.60.1) + + ## 1.9.0 [edot-java-1-9-0-release-notes] + **Release date:** February 9, 2026 + + ### Breaking changes [edot-java-1-9-0-fixes] + - universal profiling is disabled by default #958 + + ### Deprecations [edot-java-1-9-0-deprecations] + * The legacy exporter is deprecated #960 + + ## 1.7.0 [edot-java-1-7-0-release-notes] + **Release date:** November 5, 2025 + + ### Features and enhancements [edot-java-1-7-0-features-enhancements] + * Inferred spans can now be disabled and re-enabled via central config - [#838](https://github.com/elastic/elastic-otel-java/pull/838) + * The agent config is now logged on startup - [835](https://github.com/elastic/elastic-otel-java/pull/835) + * add header support for OpAMP integration [#848](https://github.com/elastic/elastic-otel-java/pull/848) + + This release is based on the following upstream versions: + + * opentelemetry-javaagent: [2.21.0](https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/tag/v2.21.0) + + ### Known issues [edot-java-1-7-0-known-issues] + * OpAMP header support can fail on restart #850 + + ## 1.4.1 [edot-java-1.4.1-release-notes] + + ### Fixes [edot-java-1.4.1-fixes] + + * Fixed `otel.exporter.otlp.metrics.temporality.preference` config option having no effect. + + ### Upgrade notes [edot-java-1.4.1-upgrade-notes] + + Re-run the installer after upgrading. + """; +} diff --git a/tests/Elastic.Changelog.Tests/Migration/ReleaseNotesPageParserTests.cs b/tests/Elastic.Changelog.Tests/Migration/ReleaseNotesPageParserTests.cs new file mode 100644 index 0000000000..150a135be4 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Migration/ReleaseNotesPageParserTests.cs @@ -0,0 +1,181 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using AwesomeAssertions; +using Elastic.Changelog.Migration; +using Elastic.Documentation; +using Elastic.Documentation.Configuration.ReleaseNotes; +using Elastic.Documentation.ReleaseNotes; + +namespace Elastic.Changelog.Tests.Migration; + +[SuppressMessage("Usage", "CA1001:Types that own disposable fields should be disposable")] +public class ReleaseNotesPageParserTests(ITestOutputHelper output) +{ + private readonly TestDiagnosticsCollector _collector = new(output); + + private IReadOnlyList<MigratedRelease> ParseFixture() => + ReleaseNotesPageParser.Parse(_collector, ReleaseNotesFixture.Markdown, "fixture.md", ReleaseNotesFixture.Scope); + + private MigratedRelease ParseFixtureVersion(string version) + { + var release = ParseFixture().SingleOrDefault(r => r.Version == version); + release.Should().NotBeNull(); + return release; + } + + [Fact] + public void Parse_RealisticPage_ParsesEveryVersionSection() + { + var releases = ParseFixture(); + + releases.Select(r => r.Version).Should().Equal("2.0.0", "1.10.0", "1.9.0", "1.7.0", "1.4.1"); + _collector.Errors.Should().Be(0); + } + + [Fact] + public void Parse_VersionSection_MapsProductTargetLifecycleAndReleaseDate() + { + var release = ParseFixtureVersion("1.9.0"); + + var product = release.Bundle.Products.Should().ContainSingle().Subject; + product.ProductId.Should().Be("edot-java"); + product.Target.Should().Be("1.9.0"); + product.Lifecycle.Should().Be(Lifecycle.Ga); + product.Repo.Should().Be("elastic-otel-java"); + product.Owner.Should().Be("elastic"); + release.Bundle.ReleaseDate.Should().Be(new DateOnly(2026, 2, 9)); + } + + [Fact] + public void Parse_TypedSubsections_MapToEntryTypes() + { + var release = ParseFixtureVersion("1.9.0"); + + release.Bundle.Entries.Should().HaveCount(2); + release.Bundle.Entries[0].Type.Should().Be(ChangelogEntryType.BreakingChange); + release.Bundle.Entries[1].Type.Should().Be(ChangelogEntryType.Deprecation); + + ParseFixtureVersion("1.7.0").Bundle.Entries.Should() + .Contain(e => e.Type == ChangelogEntryType.Enhancement) + .And.Contain(e => e.Type == ChangelogEntryType.KnownIssue); + + ParseFixtureVersion("1.4.1").Bundle.Entries.Should() + .ContainSingle().Which.Type.Should().Be(ChangelogEntryType.BugFix); + } + + [Fact] + public void Parse_BarePrReference_ResolvesAgainstScopeRepoAndCleansTitle() + { + var entry = ParseFixtureVersion("1.9.0").Bundle.Entries[0]; + + entry.Title.Should().Be("universal profiling is disabled by default"); + entry.Prs.Should().Equal("https://github.com/elastic/elastic-otel-java/pull/958"); + } + + [Theory] + [InlineData(0, "Inferred spans can now be disabled and re-enabled via central config", "https://github.com/elastic/elastic-otel-java/pull/838")] + [InlineData(1, "The agent config is now logged on startup", "https://github.com/elastic/elastic-otel-java/pull/835")] + [InlineData(2, "add header support for OpAMP integration", "https://github.com/elastic/elastic-otel-java/pull/848")] + public void Parse_MarkdownPrLinkVariants_ExtractUrlAndCleanTitle(int index, string expectedTitle, string expectedPr) + { + var entries = ParseFixtureVersion("1.7.0").Bundle.Entries; + + entries[index].Title.Should().Be(expectedTitle); + entries[index].Prs.Should().Equal(expectedPr); + } + + [Fact] + public void Parse_EntryWithoutPrReference_HasNoPrs() + { + var entry = ParseFixtureVersion("1.4.1").Bundle.Entries.Single(e => e.Type == ChangelogEntryType.BugFix); + + entry.Title.Should().Be("Fixed `otel.exporter.otlp.metrics.temporality.preference` config option having no effect."); + entry.Prs.Should().BeNull(); + } + + [Fact] + public void Parse_EntryProducts_CarryTheScopeProduct() + { + var entries = ParseFixtureVersion("1.7.0").Bundle.Entries; + + entries.Should().AllSatisfy(e => + e.Products.Should().ContainSingle().Which.ProductId.Should().Be("edot-java")); + } + + [Fact] + public void Parse_TrailingProseAfterEntries_GoesToDescriptionNotEntries() + { + var release = ParseFixtureVersion("1.7.0"); + + // The upstream-versions list after the enhancement bullets is prose, not entries. + release.Bundle.Entries.Should().HaveCount(4, "three enhancements plus one known issue"); + release.Bundle.Description.Should().Contain("This release is based on the following upstream versions:"); + release.Bundle.Description.Should().Contain("opentelemetry-javaagent: [2.21.0]"); + } + + [Fact] + public void Parse_ProseOnlyRelease_ProducesDescriptionOnlyBundle() + { + var release = ParseFixtureVersion("1.10.0"); + + release.Bundle.Entries.Should().BeEmpty(); + release.Bundle.ReleaseDate.Should().Be(new DateOnly(2026, 3, 24)); + release.Bundle.Description.Should().Contain("fixes for potential security vulnerabilities"); + release.Bundle.Description.Should().Contain("opentelemetry-sdk: [1.60.1]"); + } + + [Fact] + public void Parse_UnrecognizedSubsection_PreservedInDescriptionWithWarning() + { + var release = ParseFixtureVersion("1.4.1"); + + release.Bundle.Description.Should().Contain("### Upgrade notes"); + release.Bundle.Description.Should().Contain("Re-run the installer after upgrading."); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("Unrecognized subsection")); + } + + [Fact] + public void Parse_CommentTemplateLines_NeverProduceContent() + { + var releases = ParseFixture(); + + releases.Should().NotContain(r => r.Version == "version.next"); + releases.Should().AllSatisfy(r => r.Bundle.Description?.Should().NotContain("% ")); + } + + [Fact] + public void Parse_NonVersionHeading_SkippedWithWarning() + { + var markdown = """ + ## Overview [some-anchor] + Not release content. + + ## 1.0.0 [v1] + ### Fixes [v1-fixes] + * A fix #1 + """; + + var releases = ReleaseNotesPageParser.Parse(_collector, markdown, "fixture.md", ReleaseNotesFixture.Scope); + + releases.Should().ContainSingle().Which.Version.Should().Be("1.0.0"); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("not a recognizable version")); + } + + [Fact] + public void Parse_MappedBundle_SerializesToLoadableBundleYaml() + { + var release = ParseFixtureVersion("1.9.0"); + + var yaml = ReleaseNotesSerialization.SerializeBundle(release.Bundle); + var roundTripped = ReleaseNotesSerialization.DeserializeBundle(yaml); + + yaml.Should().Contain("release-date: 2026-02-09"); + roundTripped.Products.Should().ContainSingle().Which.Target.Should().Be("1.9.0"); + roundTripped.Entries.Should().HaveCount(2); + roundTripped.Entries[0].Type.Should().Be(ChangelogEntryType.BreakingChange); + roundTripped.Entries[0].Prs.Should().Equal("https://github.com/elastic/elastic-otel-java/pull/958"); + } +} diff --git a/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs b/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs new file mode 100644 index 0000000000..9240e93b93 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs @@ -0,0 +1,335 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using System.IO.Abstractions.TestingHelpers; +using System.Net; +using System.Security.Cryptography; +using Amazon.S3; +using Amazon.S3.Model; +using AwesomeAssertions; +using Elastic.Changelog.Migration; +using Elastic.Documentation.Configuration; +using FakeItEasy; +using Microsoft.Extensions.Logging.Abstractions; +using Nullean.ScopedFileSystem; + +namespace Elastic.Changelog.Tests.Migration; + +[SuppressMessage("Usage", "CA1001:Types that own disposable fields should be disposable")] +[SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms", Justification = "MD5 mirrors the S3 single-part ETag the service compares against")] +public class WebMigrationServiceTests +{ + private const string Bucket = "test-bucket"; + private static readonly string[] InScopeVersions = ["1.10.0", "1.9.0", "1.7.0", "1.4.1"]; + + private readonly MockFileSystem _mockFileSystem; + private readonly ScopedFileSystem _fileSystem; + private readonly IAmazonS3 _s3Client = A.Fake<IAmazonS3>(); + private readonly TestDiagnosticsCollector _collector; + private readonly string _configPath; + private readonly StubHandler _httpHandler; + + public WebMigrationServiceTests(ITestOutputHelper output) + { + _mockFileSystem = new MockFileSystem(new MockFileSystemOptions + { + CurrentDirectory = Paths.WorkingDirectoryRoot.FullName + }); + _fileSystem = FileSystemFactory.ScopeCurrentWorkingDirectoryForWrite(_mockFileSystem); + _collector = new TestDiagnosticsCollector(output); + _httpHandler = new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReleaseNotesFixture.Markdown) + }); + + _configPath = _mockFileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, "migrate-from-web.yml"); + // language=yaml + _mockFileSystem.AddFile(_configPath, new MockFileData(""" + products: + edot-java: + owner: elastic + repo: elastic-otel-java + path: docs/release-notes/index.md + ref: 9a61ce4faaf08e272c433a083bcc6f0e96d80e0a + cutoff: 1.10.0 + """)); + } + + private WebMigrationService CreateService() => + new(NullLoggerFactory.Instance, _fileSystem, _s3Client, _httpHandler); + + private MigrateFromWebArguments Args(bool dryRun = false, string bucket = Bucket, string[]? versions = null) => new() + { + Product = "edot-java", + S3BucketName = bucket, + DryRun = dryRun, + Config = _configPath, + Versions = versions ?? [] + }; + + private static string Key(string version) => $"bundle/edot-java/{version}.yaml"; + + /// <summary>Fakes an empty bucket: every HEAD/GET misses, every PUT succeeds and records the body's MD5.</summary> + private Dictionary<string, string> FakeEmptyBucket() + { + var putEtags = new Dictionary<string, string>(StringComparer.Ordinal); + + A.CallTo(() => _s3Client.GetObjectMetadataAsync(A<GetObjectMetadataRequest>._, A<CancellationToken>._)) + .Throws(new AmazonS3Exception("Not Found") { StatusCode = HttpStatusCode.NotFound }); + A.CallTo(() => _s3Client.GetObjectAsync(A<GetObjectRequest>._, A<CancellationToken>._)) + .Throws(new AmazonS3Exception("Not Found") { StatusCode = HttpStatusCode.NotFound }); + A.CallTo(() => _s3Client.PutObjectAsync(A<PutObjectRequest>._, A<CancellationToken>._)) + .ReturnsLazily((PutObjectRequest request, CancellationToken _) => + { + using var buffer = new MemoryStream(); + request.InputStream.CopyTo(buffer); + var etag = Convert.ToHexStringLower(MD5.HashData(buffer.ToArray())); + putEtags[request.Key] = etag; + return new PutObjectResponse { ETag = $"\"{etag}\"" }; + }); + + return putEtags; + } + + /// <summary>Fakes a bucket where the given keys already exist with the given ETags.</summary> + private void FakeExistingKeys(IReadOnlyDictionary<string, string> etags) => + A.CallTo(() => _s3Client.GetObjectMetadataAsync(A<GetObjectMetadataRequest>._, A<CancellationToken>._)) + .ReturnsLazily((GetObjectMetadataRequest request, CancellationToken _) => + etags.TryGetValue(request.Key, out var etag) + ? new GetObjectMetadataResponse { ETag = $"\"{etag}\"" } + : throw new AmazonS3Exception("Not Found") { StatusCode = HttpStatusCode.NotFound }); + + [Fact] + public async Task FirstRun_CreatesEveryInScopeKeyWithCreateOnlySemantics() + { + _ = FakeEmptyBucket(); + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args(), ct); + + result.Should().BeTrue(); + _collector.Errors.Should().Be(0); + + foreach (var version in InScopeVersions) + { + A.CallTo(() => _s3Client.PutObjectAsync( + A<PutObjectRequest>.That.Matches(r => r.Key == Key(version) && r.BucketName == Bucket && r.IfNoneMatch == "*"), + A<CancellationToken>._ + )).MustHaveHappenedOnceExactly(); + } + + service.LastResults.Where(r => r.Outcome == "created").Should().HaveCount(InScopeVersions.Length); + service.LastResults.Should().AllSatisfy(r => r.Outcome.Should().NotBe("failed")); + } + + [Fact] + public async Task FirstRun_RefreshesTheProductRegistryManifest() + { + _ = FakeEmptyBucket(); + var ct = TestContext.Current.CancellationToken; + + var result = await CreateService().MigrateFromWeb(_collector, Args(), ct); + + result.Should().BeTrue(); + A.CallTo(() => _s3Client.PutObjectAsync( + A<PutObjectRequest>.That.Matches(r => r.Key == "bundle/edot-java/registry.json"), + A<CancellationToken>._ + )).MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task SecondRun_OverSameScope_IsANoOpWithAllSkips() + { + var putEtags = FakeEmptyBucket(); + var ct = TestContext.Current.CancellationToken; + _ = await CreateService().MigrateFromWeb(_collector, Args(), ct); + putEtags.Should().NotBeEmpty(); + + // Second run against a bucket that now contains exactly what the first run created. + Fake.ClearRecordedCalls(_s3Client); + FakeExistingKeys(putEtags); + + var service = CreateService(); + var result = await service.MigrateFromWeb(_collector, Args(), ct); + + result.Should().BeTrue(); + _collector.Errors.Should().Be(0); + A.CallTo(() => _s3Client.PutObjectAsync(A<PutObjectRequest>._, A<CancellationToken>._)) + .MustNotHaveHappened(); + service.LastResults.Where(r => r.Detail.Contains("identical content")).Should().HaveCount(InScopeVersions.Length); + } + + [Fact] + public async Task ExistingKeyWithDifferentContent_IsSkippedAndNeverOverwritten() + { + _ = FakeEmptyBucket(); + FakeExistingKeys(InScopeVersions.ToDictionary(Key, _ => "0000aaaa0000aaaa0000aaaa0000aaaa")); + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args(), ct); + + result.Should().BeTrue("skipping existing keys is the expected safe outcome, not a failure"); + A.CallTo(() => _s3Client.PutObjectAsync(A<PutObjectRequest>._, A<CancellationToken>._)) + .MustNotHaveHappened(); + service.LastResults.Where(r => r.Detail.Contains("different content")).Should().HaveCount(InScopeVersions.Length); + } + + [Fact] + public async Task ConcurrentCreate_PreconditionFailed_IsReportedAsSkipNotFailure() + { + _ = FakeEmptyBucket(); + // The key appears between the HEAD check and the conditional PUT: S3 answers 412. + A.CallTo(() => _s3Client.PutObjectAsync(A<PutObjectRequest>._, A<CancellationToken>._)) + .Throws(new AmazonS3Exception("Precondition Failed") { StatusCode = HttpStatusCode.PreconditionFailed }); + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args(), ct); + + result.Should().BeTrue(); + service.LastResults.Where(r => r.Detail.Contains("concurrently")).Should().HaveCount(InScopeVersions.Length); + } + + [Fact] + public async Task PutFailure_IsReportedPerKeyAndFailsTheRun() + { + _ = FakeEmptyBucket(); + A.CallTo(() => _s3Client.PutObjectAsync(A<PutObjectRequest>._, A<CancellationToken>._)) + .Throws(new AmazonS3Exception("Access Denied") { StatusCode = HttpStatusCode.Forbidden }); + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args(), ct); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + service.LastResults.Where(r => r.Outcome == "failed" && r.Detail.Contains("Access Denied")) + .Should().HaveCount(InScopeVersions.Length); + } + + [Fact] + public async Task VersionsBeyondTheCutoff_AreSkippedAndNeverUploaded() + { + _ = FakeEmptyBucket(); + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args(), ct); + + result.Should().BeTrue(); + // 2.0.0 > cutoff 1.10.0: owned by the live pipeline. + A.CallTo(() => _s3Client.PutObjectAsync( + A<PutObjectRequest>.That.Matches(r => r.Key == Key("2.0.0")), A<CancellationToken>._ + )).MustNotHaveHappened(); + var cutoffResult = service.LastResults.Should().ContainSingle(r => r.Key == Key("2.0.0")).Subject; + cutoffResult.Outcome.Should().Be("skipped"); + cutoffResult.Detail.Should().Contain("beyond cutoff 1.10.0"); + } + + [Fact] + public async Task VersionsFilter_RestrictsTheRunToTheSelection() + { + _ = FakeEmptyBucket(); + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args(versions: ["1.9.0"]), ct); + + result.Should().BeTrue(); + A.CallTo(() => _s3Client.PutObjectAsync( + A<PutObjectRequest>.That.Matches(r => r.Key == Key("1.9.0")), A<CancellationToken>._ + )).MustHaveHappenedOnceExactly(); + service.LastResults.Where(r => r.Outcome == "created").Should().ContainSingle(); + service.LastResults.Where(r => r.Detail.Contains("--versions")).Should().HaveCount(InScopeVersions.Length - 1); + } + + [Fact] + public async Task DryRunWithoutBucket_TouchesNoS3AtAll() + { + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args(dryRun: true, bucket: ""), ct); + + result.Should().BeTrue(); + _collector.Errors.Should().Be(0); + A.CallTo(_s3Client).MustNotHaveHappened(); + service.LastResults.Where(r => r.Outcome == "would-create").Should().HaveCount(InScopeVersions.Length); + service.LastResults.Where(r => r.Outcome == "would-create").Should().AllSatisfy(r => r.ETag.Should().NotBeNullOrEmpty()); + } + + [Fact] + public async Task DryRunWithBucket_InspectsExistenceButNeverWrites() + { + _ = FakeEmptyBucket(); + FakeExistingKeys(new Dictionary<string, string> { [Key("1.9.0")] = "0000aaaa0000aaaa0000aaaa0000aaaa" }); + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args(dryRun: true), ct); + + result.Should().BeTrue(); + A.CallTo(() => _s3Client.PutObjectAsync(A<PutObjectRequest>._, A<CancellationToken>._)) + .MustNotHaveHappened(); + service.LastResults.Where(r => r.Outcome == "would-create").Should().HaveCount(InScopeVersions.Length - 1); + service.LastResults.Should().ContainSingle(r => r.Key == Key("1.9.0") && r.Outcome == "skipped"); + } + + [Fact] + public async Task ProductNotInScopeConfig_FailsWithoutAnyNetworkAccess() + { + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args() with { Product = "not-configured" }, ct); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + _httpHandler.RequestedPaths.Should().BeEmpty(); + A.CallTo(_s3Client).MustNotHaveHappened(); + } + + [Fact] + public async Task FetchFailure_FailsWithoutAnyS3Calls() + { + var handler = new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); + var service = new WebMigrationService(NullLoggerFactory.Instance, _fileSystem, _s3Client, handler); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args(), ct); + + result.Should().BeFalse(); + _collector.Errors.Should().BeGreaterThan(0); + A.CallTo(_s3Client).MustNotHaveHappened(); + } + + [Fact] + public async Task FetchesTheMarkdownFromThePinnedRefOnRawGithubusercontent() + { + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + _ = await service.MigrateFromWeb(_collector, Args(dryRun: true, bucket: ""), ct); + + _httpHandler.RequestedPaths.Should().Equal( + "/elastic/elastic-otel-java/9a61ce4faaf08e272c433a083bcc6f0e96d80e0a/docs/release-notes/index.md"); + } + + private sealed class StubHandler(Func<HttpRequestMessage, HttpResponseMessage> responder) : HttpMessageHandler + { + public List<string> RequestedPaths { get; } = []; + + protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken) + { + RequestedPaths.Add(request.RequestUri!.AbsolutePath); + return responder(request); + } + + protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(Send(request, cancellationToken)); + } +} From f8a1f8539644890c257e73c4b7bf0f8264dcdc58 Mon Sep 17 00:00:00 2001 From: Felipe Cotti <felipe.cotti@elastic.co> Date: Thu, 6 Aug 2026 12:44:07 -0300 Subject: [PATCH 3/5] Changelog: migrate-from-web writes bundles only (no registry refresh) Restacked on the scrubber-ownership chain: the scrubber Lambda owns the public bundle/{product}/registry.json manifests and the shallow per-tree maps (#3738), and the client-side refresh machinery is retired (#3760). The migration command now writes YAML bundle objects only; the S3 events those creates emit trigger the reconciliation that materializes the manifests. Tests assert no registry.json is ever PUT. --- docs/cli/changelog/cmd-migrate-from-web.md | 2 +- .../Migration/WebMigrationService.cs | 47 +++---------------- .../Migration/WebMigrationServiceTests.cs | 9 ++-- 3 files changed, 14 insertions(+), 44 deletions(-) diff --git a/docs/cli/changelog/cmd-migrate-from-web.md b/docs/cli/changelog/cmd-migrate-from-web.md index b45abf86ff..c614d58834 100644 --- a/docs/cli/changelog/cmd-migrate-from-web.md +++ b/docs/cli/changelog/cmd-migrate-from-web.md @@ -44,7 +44,7 @@ The report lists one line per key with its outcome: | `skipped` | The key already exists (identical or different content — never overwritten), was created concurrently by another writer, is beyond the cutoff, or is not in the `--versions` selection. | | `failed` | The write failed; the reason is included and the command exits non-zero. | -After a run that created keys, the command refreshes `bundle/{product}/registry.json` (best-effort, like `changelog upload`). +The command writes YAML bundle objects only — never a `registry.json`. The scrubber Lambda owns the public `bundle/{product}/registry.json` manifests and the shallow per-tree maps, reconciling them from the S3 events these creates emit ([#3738](https://github.com/elastic/docs-builder/pull/3738), [#3760](https://github.com/elastic/docs-builder/pull/3760)). ## Examples diff --git a/src/services/Elastic.Changelog/Migration/WebMigrationService.cs b/src/services/Elastic.Changelog/Migration/WebMigrationService.cs index 63e30f1ac8..3222660140 100644 --- a/src/services/Elastic.Changelog/Migration/WebMigrationService.cs +++ b/src/services/Elastic.Changelog/Migration/WebMigrationService.cs @@ -10,12 +10,10 @@ using System.Text; using Amazon.S3; using Amazon.S3.Model; -using Elastic.Changelog.Uploading; using Elastic.Documentation; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.ReleaseNotes; using Elastic.Documentation.Diagnostics; -using Elastic.Documentation.Integrations.S3; using Elastic.Documentation.Services; using Elastic.Documentation.Versions; using Microsoft.Extensions.Logging; @@ -100,7 +98,7 @@ public async Task<bool> MigrateFromWeb(IDiagnosticsCollector collector, MigrateF if (collector.Errors > 0) return false; - var uploadResults = await UploadCreateOnly(collector, args, staged, ctx); + var uploadResults = await UploadCreateOnly(args, staged, ctx); results.AddRange(uploadResults); LastResults = results; @@ -178,8 +176,8 @@ private static (List<MigratedRelease> InScope, List<MigrationKeyResult> Results) private sealed record StagedBundle(string Key, string LocalPath, string LocalETag); /// <summary> - /// Serializes each in-scope release to bundle YAML and stages it in a temp directory so the - /// existing registry-refresh machinery (which reads local files) can be reused as-is. + /// Serializes each in-scope release to bundle YAML and stages it in a temp directory, computing + /// the local single-part ETag used to distinguish identical from divergent remote content. /// </summary> [SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms", Justification = "MD5 matches the S3 single-part ETag, used for content comparison only")] private List<StagedBundle> StageBundles(IDiagnosticsCollector collector, MigrateFromWebScope scope, IReadOnlyList<MigratedRelease> releases) @@ -210,7 +208,6 @@ private List<StagedBundle> StageBundles(IDiagnosticsCollector collector, Migrate } private async Task<List<MigrationKeyResult>> UploadCreateOnly( - IDiagnosticsCollector collector, MigrateFromWebArguments args, IReadOnlyList<StagedBundle> staged, Cancel ctx) @@ -223,21 +220,18 @@ private async Task<List<MigrationKeyResult>> UploadCreateOnly( using var defaultClient = s3Client is null ? new AmazonS3Client() : null; var client = s3Client ?? defaultClient!; + // No registry write, by design: the scrubber Lambda owns the public bundle/{product}/ + // registry.json manifests and the shallow per-tree maps, reconciling them from the S3 + // events these creates emit (elastic/docs-builder#3738); the client-side refresh is + // retired (elastic/docs-builder#3760). var results = new List<MigrationKeyResult>(staged.Count); - var uploadedTargets = new List<UploadTarget>(); foreach (var bundle in staged) { ctx.ThrowIfCancellationRequested(); var result = await MigrateKey(client, args, bundle, ctx); results.Add(result); - if (result.Outcome is OutcomeCreated || (result.Outcome is OutcomeSkipped && result.ETag == bundle.LocalETag)) - uploadedTargets.Add(new UploadTarget(bundle.LocalPath, bundle.Key)); } - var created = results.Count(r => r.Outcome == OutcomeCreated); - if (!args.DryRun && created > 0) - await RefreshRegistry(collector, client, args.S3BucketName, uploadedTargets, ctx); - return results; } @@ -305,33 +299,6 @@ private async Task<PutObjectResponse> PutCreateOnly(IAmazonS3 client, string buc return await client.PutObjectAsync(request, ctx); } - /// <summary> - /// Refreshes <c>bundle/{product}/registry.json</c> with the keys this run created (plus keys that - /// already existed with identical content), reusing the live pipeline's registry machinery. - /// Best-effort, matching live uploads: the bundles themselves are already in S3. - /// </summary> - private async Task RefreshRegistry( - IDiagnosticsCollector collector, - IAmazonS3 client, - string bucketName, - IReadOnlyList<UploadTarget> targets, - Cancel ctx) - { - try - { - var etagCalculator = new S3EtagCalculator(logFactory, _fileSystem); - var builder = new RegistryBuilder(logFactory, _fileSystem, client, etagCalculator, bucketName); - var result = await builder.RefreshAsync(collector, targets, ctx, RegistryScope.Bundle); - _logger.LogInformation("Registry refresh: {Updated} updated, {Unchanged} unchanged, {Failed} failed", - result.Updated, result.Unchanged, result.Failed); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, "Registry refresh failed; migrated bundles are in S3 but the manifest may be stale"); - collector.EmitWarning(string.Empty, $"Failed to refresh registry manifest: {ex.Message}"); - } - } - /// <summary>Formats the run report — one line per key — in a form that can be pasted into the tracking issue.</summary> public static string FormatReport(MigrateFromWebScope scope, MigrateFromWebArguments args, IReadOnlyList<MigrationKeyResult> results) { diff --git a/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs b/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs index 9240e93b93..5c6ba47137 100644 --- a/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs +++ b/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs @@ -126,8 +126,11 @@ public async Task FirstRun_CreatesEveryInScopeKeyWithCreateOnlySemantics() } [Fact] - public async Task FirstRun_RefreshesTheProductRegistryManifest() + public async Task Run_NeverWritesARegistryManifest() { + // The scrubber Lambda owns the public manifests and shallow maps (elastic/docs-builder#3738); + // the client-side refresh is retired (elastic/docs-builder#3760). Migration writes YAML + // bundle objects only — the S3 events those creates emit trigger the reconciliation. _ = FakeEmptyBucket(); var ct = TestContext.Current.CancellationToken; @@ -135,9 +138,9 @@ public async Task FirstRun_RefreshesTheProductRegistryManifest() result.Should().BeTrue(); A.CallTo(() => _s3Client.PutObjectAsync( - A<PutObjectRequest>.That.Matches(r => r.Key == "bundle/edot-java/registry.json"), + A<PutObjectRequest>.That.Matches(r => r.Key.EndsWith("registry.json", StringComparison.Ordinal)), A<CancellationToken>._ - )).MustHaveHappenedOnceExactly(); + )).MustNotHaveHappened(); } [Fact] From 6ee86897facb3b0174e8f6a1df0680e8ab074299 Mon Sep 17 00:00:00 2001 From: Felipe Cotti <felipe.cotti@elastic.co> Date: Mon, 10 Aug 2026 14:58:12 -0300 Subject: [PATCH 4/5] Changelog: migrate-from-web covers the scope table, --products narrows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback (#3794): drop the standing config surface. The former config/migrate-from-web.yml becomes a checked-in table in the command itself (MigrateFromWebScope.All) — temporary tooling state, added per rollout wave and deleted with the command. A run covers every product in the table by default; --products x,y,z narrows it for tests and pilots. The page→product mapping stays explicit because bundle product ids appear in no published metadata: page frontmatter carries the site taxonomy, not bundle ids. One product's failure no longer aborts the run: each product migrates independently and the run fails at the end if any of them did. --- config/migrate-from-web.yml | 18 --- docs/cli-schema.json | 40 ++--- docs/cli/changelog/cmd-migrate-from-web.md | 44 +++--- .../Migration/MigrateFromWebScope.cs | 148 +++++++----------- .../Migration/WebMigrationService.cs | 79 ++++++---- .../docs-builder/Commands/ChangelogCommand.cs | 22 ++- .../Migration/WebMigrationServiceTests.cs | 37 ++--- 7 files changed, 170 insertions(+), 218 deletions(-) delete mode 100644 config/migrate-from-web.yml diff --git a/config/migrate-from-web.yml b/config/migrate-from-web.yml deleted file mode 100644 index 3474ad9772..0000000000 --- a/config/migrate-from-web.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Scope/cutoff list for the TEMPORARY `docs-builder changelog migrate-from-web` command -# (elastic/docs-eng-team#736). Delete this file together with the command once the migration -# rollout (elastic/docs-eng-team#683) completes. -# -# Each entry maps a product id (the `bundle/{product}/` S3 prefix, see config/products.yml) to: -# owner/repo — the repository whose docs back the published release notes -# path — repository-relative path of the release-notes Markdown page -# ref — pinned commit SHA at which the Markdown is fetched (reproducible runs) -# cutoff — inclusive upper version bound; releases above it are owned by the live pipeline -products: - edot-java: - owner: elastic - repo: elastic-otel-java - path: docs/release-notes/index.md - # Last commit before the repo switched to native docs-builder bundle YAMLs (#1023): - # the final hand-authored state of the published release-notes Markdown. - ref: 9a61ce4faaf08e272c433a083bcc6f0e96d80e0a - cutoff: 1.10.0 diff --git a/docs/cli-schema.json b/docs/cli-schema.json index 312ef8b5c6..734552cd92 100644 --- a/docs/cli-schema.json +++ b/docs/cli-schema.json @@ -3821,52 +3821,32 @@ ], "name": "migrate-from-web", "summary": "TEMPORARY: One-off migration of published release notes into the S3 bundle store; removed after elastic/docs-eng-team#683.", - "notes": "Fetches the release-notes Markdown that backs the published pages (at the pinned git ref in the\nchecked-in scope config), maps it to the existing bundle YAML shape, and uploads to\nbundle/{product}/ with create-only semantics (If-None-Match: *) \u2014 existing keys are\nskipped, never overwritten. Prints a per-key run report (created / skipped / failed with reason and\nobject ETag) suitable for pasting into the tracking issue.\n\nScope is always explicit: the product must have an entry in config/migrate-from-web.yml\n(product id \u2192 source repo, release-notes path, pinned ref, version cutoff). Nothing runs implicitly\nfor all products. Tracked by elastic/docs-eng-team#736.", - "usage": "docs-builder changelog migrate-from-web \u003Cproduct\u003E [options]", + "notes": "Fetches the release-notes Markdown that backs the published pages (at the pinned git ref in the\nchecked-in scope table), maps it to the existing bundle YAML shape, and uploads to\nbundle/{product}/ with create-only semantics (If-None-Match: *) \u2014 existing keys are\nskipped, never overwritten. Prints a per-key run report (created / skipped / failed with reason and\nobject ETag) suitable for pasting into the tracking issue.\n\nMigrates every product in the checked-in scope table by default. The table lives in code\n(MigrateFromWebScope.All: product id \u2192 source repo, release-notes path, pinned ref, version\ncutoff) and grows per rollout wave; use --products to narrow a run for tests and pilots.\nTracked by elastic/docs-eng-team#736.", + "usage": "docs-builder changelog migrate-from-web [options]", "examples": [], "parameters": [ - { - "role": "positional", - "name": "product", - "type": "string", - "required": true, - "summary": "Product id to migrate (e.g. \u0022edot-java\u0022). Must have an entry in the checked-in scope config." - }, { "role": "flag", - "name": "s3-bucket-name", - "type": "string", + "name": "products", + "type": "array", "required": false, - "summary": "Destination S3 bucket. Required unless --dry-run; when provided with --dry-run, existing keys are still inspected so the report distinguishes would-create from skipped." + "summary": "Optional: restrict the run to specific product ids (comma-separated or repeated), e.g. \u0022edot-java\u0022. Defaults to every product in the checked-in scope table.", + "repeatable": true, + "elementType": "string" }, { "role": "flag", - "name": "config", + "name": "s3-bucket-name", "type": "string", "required": false, - "summary": "Path to the scope config. Defaults to config/migrate-from-web.yml in the current directory (the copy checked into the docs-builder repository).", - "validations": [ - { - "kind": "rejectSymbolicLinks" - }, - { - "kind": "existing" - }, - { - "kind": "fileExtensions", - "values": [ - "yml", - "yaml" - ] - } - ] + "summary": "Destination S3 bucket. Required unless --dry-run; when provided with --dry-run, existing keys are still inspected so the report distinguishes would-create from skipped." }, { "role": "flag", "name": "versions", "type": "array", "required": false, - "summary": "Optional: restrict the run to specific versions (comma-separated or repeated). Versions above the configured cutoff are always skipped.", + "summary": "Optional: restrict the run to specific versions (comma-separated or repeated). Versions above a product\u0027s cutoff are always skipped.", "repeatable": true, "elementType": "string" }, diff --git a/docs/cli/changelog/cmd-migrate-from-web.md b/docs/cli/changelog/cmd-migrate-from-web.md index c614d58834..c853e5b7c6 100644 --- a/docs/cli/changelog/cmd-migrate-from-web.md +++ b/docs/cli/changelog/cmd-migrate-from-web.md @@ -6,28 +6,27 @@ This command is **temporary**. It exists solely to migrate release notes that we One-off migration of already-published release notes into the S3 bundle store. For each product in scope, the command: -1. Fetches the release-notes Markdown that backs the published pages — from `raw.githubusercontent.com` at the pinned commit recorded in the scope config, not by scraping live site HTML. +1. Fetches the release-notes Markdown that backs the published pages — from `raw.githubusercontent.com` at the pinned commit recorded in the scope table, not by scraping live site HTML. 2. Parses each `## {version}` section (typed `### …` subsections become entries; prose is preserved as the bundle description) and maps it to the **existing** bundle YAML shape that [](/cli/changelog/upload.md) publishes. No new schema is introduced. 3. Uploads each release to `bundle/{product}/{version}.yaml` with **create-only** semantics (`If-None-Match: *`): keys that already exist are skipped and never overwritten, so the migration can never clobber bundles produced by the live pipeline. 4. Prints a per-key run report (created / skipped / failed, with the reason and object ETag) suitable for pasting into the tracking issue. -The scope is always explicit. The product must have an entry in the checked-in scope config (`config/migrate-from-web.yml` in the docs-builder repository); nothing runs implicitly for all products. +By default the command migrates **every product in the checked-in scope table**; use `--products` to narrow a run for tests and pilots. -## Scope config +## Migration scope -Each entry maps a product id (the `bundle/{product}/` S3 prefix, see `config/products.yml`) to the source of its published release notes and a version cutoff: +The scope table is checked into the command itself (`MigrateFromWebScope.All` in the docs-builder repository) rather than into a config file — it is temporary tooling state, added per rollout wave and deleted with the command. Each entry maps a product id (the `bundle/{product}/` S3 prefix, see `config/products.yml`) to the source of its published release notes and a version cutoff: -```yaml -products: - edot-java: - owner: elastic # GitHub owner of the source repository - repo: elastic-otel-java # source repository - path: docs/release-notes/index.md # repo-relative path of the release-notes page - ref: 9a61ce4faaf08e272c433a083bcc6f0e96d80e0a # pinned commit SHA (reproducible runs) - cutoff: 1.10.0 # inclusive; releases above it belong to the live pipeline -``` +| Field | Meaning | +| ----- | ------- | +| `Owner` / `Repo` | GitHub repository whose docs back the published release notes. | +| `Path` | Repo-relative path of the release-notes Markdown page. | +| `Ref` | Pinned commit SHA at which the Markdown is fetched (reproducible runs). | +| `Cutoff` | Inclusive upper version bound; releases above it belong to the live pipeline. | + +The page→product mapping is deliberately explicit: bundle product ids appear in no published metadata (page frontmatter carries the site taxonomy, not bundle ids), so deriving it automatically is not possible. Adding a product to the migration is a small PR against the table. -Releases above the cutoff are always skipped — they are owned by the live changelog pipeline. Use `--versions` to narrow a run to specific versions below the cutoff. +Releases above a product's cutoff are always skipped — they are owned by the live changelog pipeline. Use `--versions` to narrow a run to specific versions below the cutoff. ## Requirements @@ -53,7 +52,7 @@ The command writes YAML bundle objects only — never a `registry.json`. The scr Parse, map, and report what would be created — no S3 access at all: ```sh -docs-builder changelog migrate-from-web edot-java --dry-run +docs-builder changelog migrate-from-web --dry-run ``` ### Dry run against the real bucket @@ -61,7 +60,7 @@ docs-builder changelog migrate-from-web edot-java --dry-run Also checks which keys already exist, so the report distinguishes `would-create` from `skipped`: ```sh -docs-builder changelog migrate-from-web edot-java \ +docs-builder changelog migrate-from-web \ --dry-run \ --s3-bucket-name my-changelog-bundles ``` @@ -69,16 +68,25 @@ docs-builder changelog migrate-from-web edot-java \ ### Perform the migration ```sh -docs-builder changelog migrate-from-web edot-java \ +docs-builder changelog migrate-from-web \ --s3-bucket-name my-changelog-bundles ``` Re-running the same command is safe: every existing key is reported as `skipped` and the run is a no-op. +### Migrate a single product (pilots and tests) + +```sh +docs-builder changelog migrate-from-web \ + --products edot-java \ + --s3-bucket-name my-changelog-bundles +``` + ### Migrate specific versions only ```sh -docs-builder changelog migrate-from-web edot-java \ +docs-builder changelog migrate-from-web \ + --products edot-java \ --s3-bucket-name my-changelog-bundles \ --versions 1.9.0,1.10.0 ``` diff --git a/src/services/Elastic.Changelog/Migration/MigrateFromWebScope.cs b/src/services/Elastic.Changelog/Migration/MigrateFromWebScope.cs index b294f2fe01..6919b552e2 100644 --- a/src/services/Elastic.Changelog/Migration/MigrateFromWebScope.cs +++ b/src/services/Elastic.Changelog/Migration/MigrateFromWebScope.cs @@ -2,126 +2,86 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information -using System.IO.Abstractions; -using Elastic.Documentation.Configuration.ReleaseNotes; +using System.Collections.Immutable; using Elastic.Documentation.Diagnostics; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; namespace Elastic.Changelog.Migration; /// <summary> -/// TEMPORARY (elastic/docs-eng-team#736): the checked-in scope/cutoff list for -/// <c>changelog migrate-from-web</c>. Maps a product id to the repository, path, and pinned git ref -/// of the published release-notes Markdown, plus the inclusive version cutoff for the migration. -/// Delete together with the command once the rollout (elastic/docs-eng-team#683) completes. +/// TEMPORARY (elastic/docs-eng-team#736): one product's migration scope — where its published +/// release-notes Markdown lives (owner/repo/path at a pinned ref) and the inclusive version cutoff. +/// The checked-in table below replaces the former <c>config/migrate-from-web.yml</c> (dropped on +/// review: no standing config surface for a one-off tool). It grows per rollout wave +/// (elastic/docs-eng-team#683) and is deleted together with the command once the rollout completes. /// </summary> -public sealed class MigrateFromWebScope +public sealed record MigrateFromWebScope { public required string ProductId { get; init; } + + /// <summary>GitHub owner of the source repository (e.g. <c>elastic</c>).</summary> public required string Owner { get; init; } + + /// <summary>Source repository name (e.g. <c>elastic-otel-java</c>).</summary> public required string Repo { get; init; } + + /// <summary>Repository-relative path of the release-notes Markdown page.</summary> public required string Path { get; init; } + + /// <summary>Pinned git ref (commit SHA) at which the Markdown is fetched (reproducible runs).</summary> public required string Ref { get; init; } + + /// <summary>Inclusive upper version bound; releases above it belong to the live pipeline.</summary> public required string Cutoff { get; init; } /// <summary> - /// Loads and validates the scope entry for <paramref name="productId"/> from the checked-in - /// scope config at <paramref name="configPath"/>, or null (with errors emitted) when the file, - /// product, or any required field is missing. + /// Every product the migration knows how to source. The page→product mapping is deliberately + /// checked in rather than derived: bundle product ids appear in no published metadata (page + /// frontmatter carries the site taxonomy, not bundle ids), so each entry pins its source + /// explicitly. A run covers the whole table unless narrowed with <c>--products</c>. /// </summary> - public static MigrateFromWebScope? Load(IDiagnosticsCollector collector, IFileSystem fileSystem, string configPath, string productId) - { - if (!fileSystem.File.Exists(configPath)) + public static ImmutableArray<MigrateFromWebScope> All { get; } = + [ + new() { - collector.EmitError(configPath, "Scope config not found. The migrate-from-web scope list is checked into the docs-builder repository (config/migrate-from-web.yml); pass --config when running from elsewhere."); - return null; + ProductId = "edot-java", + Owner = "elastic", + Repo = "elastic-otel-java", + Path = "docs/release-notes/index.md", + // Last commit before the repo switched to native docs-builder bundle YAMLs (#1023): + // the final hand-authored state of the published release-notes Markdown. + Ref = "9a61ce4faaf08e272c433a083bcc6f0e96d80e0a", + Cutoff = "1.10.0" } + ]; - MigrateFromWebConfigDto? dto; - try - { - var deserializer = new StaticDeserializerBuilder(new MigrationYamlContext()) - .WithNamingConvention(UnderscoredNamingConvention.Instance) - .Build(); - dto = deserializer.Deserialize<MigrateFromWebConfigDto>(fileSystem.File.ReadAllText(configPath)); - } - catch (Exception ex) when (ex is YamlDotNet.Core.YamlException or IOException) - { - collector.EmitError(configPath, $"Could not parse scope config: {ex.Message}", ex); - return null; - } - - if (dto?.Products is null || !dto.Products.TryGetValue(productId, out var product) || product is null) - { - var known = dto?.Products is { Count: > 0 } ? string.Join(", ", dto.Products.Keys.Order(StringComparer.Ordinal)) : "<none>"; - collector.EmitError(configPath, $"Product '{productId}' is not in the migrate-from-web scope config. Configured products: {known}. Add an entry before running the migration."); - return null; - } + /// <summary> + /// Resolves a <c>--products</c> selection against the table — the whole table when the selection + /// is empty — or null (with an error emitted) when any requested id is unknown. + /// </summary> + public static IReadOnlyList<MigrateFromWebScope>? Select(IDiagnosticsCollector collector, IReadOnlyList<string> products) + { + if (products.Count == 0) + return All; - if (!ChangelogKeys.IsValidProduct(productId)) + var byId = All.ToDictionary(s => s.ProductId, StringComparer.Ordinal); + var selected = new List<MigrateFromWebScope>(products.Count); + var unknown = new List<string>(); + foreach (var product in products.Distinct(StringComparer.Ordinal)) { - collector.EmitError(configPath, $"Product id '{productId}' is not a valid bundle key segment (must match [a-zA-Z0-9_-]+)."); - return null; + if (byId.TryGetValue(product, out var scope)) + selected.Add(scope); + else + unknown.Add(product); } - var missing = new List<string>(); - if (string.IsNullOrWhiteSpace(product.Owner)) - missing.Add("owner"); - if (string.IsNullOrWhiteSpace(product.Repo)) - missing.Add("repo"); - if (string.IsNullOrWhiteSpace(product.Path)) - missing.Add("path"); - if (string.IsNullOrWhiteSpace(product.Ref)) - missing.Add("ref"); - if (string.IsNullOrWhiteSpace(product.Cutoff)) - missing.Add("cutoff"); - - if (missing.Count > 0) + if (unknown.Count > 0) { - collector.EmitError(configPath, $"Scope entry for '{productId}' is missing required field(s): {string.Join(", ", missing)}."); + var known = string.Join(", ", All.Select(s => s.ProductId).Order(StringComparer.Ordinal)); + collector.EmitError(string.Empty, + $"Unknown product id(s) in --products: {string.Join(", ", unknown)}. Products in the checked-in migration scope: {known}. Add an entry to MigrateFromWebScope.All before running the migration."); return null; } - return new MigrateFromWebScope - { - ProductId = productId, - Owner = product.Owner!, - Repo = product.Repo!, - Path = product.Path!, - Ref = product.Ref!, - Cutoff = product.Cutoff! - }; + return selected; } } - -/// <summary>Root DTO of the checked-in migrate-from-web scope config (product id → source/cutoff).</summary> -public sealed class MigrateFromWebConfigDto -{ - public Dictionary<string, MigrateFromWebProductDto?>? Products { get; set; } -} - -/// <summary>One product's scope: where its published release-notes Markdown lives and the migration cutoff.</summary> -public sealed class MigrateFromWebProductDto -{ - /// <summary>GitHub owner of the source repository (e.g. <c>elastic</c>).</summary> - public string? Owner { get; set; } - - /// <summary>Source repository name (e.g. <c>elastic-otel-java</c>).</summary> - public string? Repo { get; set; } - - /// <summary>Repository-relative path of the release-notes Markdown page.</summary> - public string? Path { get; set; } - - /// <summary>Pinned git ref (commit SHA) at which the Markdown is fetched.</summary> - public string? Ref { get; set; } - - /// <summary>Inclusive upper version bound; releases above it belong to the live pipeline.</summary> - public string? Cutoff { get; set; } -} - -/// <summary>Source-generated YAML context for the migrate-from-web scope config (AOT-safe, no reflection).</summary> -[YamlStaticContext] -[YamlSerializable(typeof(MigrateFromWebConfigDto))] -[YamlSerializable(typeof(MigrateFromWebProductDto))] -public partial class MigrationYamlContext; diff --git a/src/services/Elastic.Changelog/Migration/WebMigrationService.cs b/src/services/Elastic.Changelog/Migration/WebMigrationService.cs index 3222660140..5b9aae036e 100644 --- a/src/services/Elastic.Changelog/Migration/WebMigrationService.cs +++ b/src/services/Elastic.Changelog/Migration/WebMigrationService.cs @@ -23,8 +23,8 @@ namespace Elastic.Changelog.Migration; public record MigrateFromWebArguments { - /// <summary>Product id to migrate; must have an entry in the checked-in scope config.</summary> - public required string Product { get; init; } + /// <summary>Optional product-id selection; empty covers every product in the checked-in scope table.</summary> + public IReadOnlyList<string> Products { get; init; } = []; /// <summary>Destination S3 bucket. Optional in dry-run mode (no S3 access at all when omitted).</summary> public string S3BucketName { get; init; } = ""; @@ -32,9 +32,6 @@ public record MigrateFromWebArguments /// <summary>When true, does everything except the S3 writes and reports what would be created.</summary> public bool DryRun { get; init; } - /// <summary>Path to the scope config. Defaults to the checked-in <c>config/migrate-from-web.yml</c>.</summary> - public string? Config { get; init; } - /// <summary>Optional exact-version filter; when set, only these versions are migrated.</summary> public IReadOnlyList<string> Versions { get; init; } = []; } @@ -56,8 +53,6 @@ public class WebMigrationService( HttpMessageHandler? httpMessageHandler = null ) : IService { - public const string DefaultConfigPath = "config/migrate-from-web.yml"; - private const string OutcomeCreated = "created"; private const string OutcomeWouldCreate = "would-create"; private const string OutcomeSkipped = "skipped"; @@ -71,45 +66,73 @@ public class WebMigrationService( public async Task<bool> MigrateFromWeb(IDiagnosticsCollector collector, MigrateFromWebArguments args, Cancel ctx) { - var configPath = string.IsNullOrWhiteSpace(args.Config) - ? _fileSystem.Path.GetFullPath(DefaultConfigPath) - : args.Config; - - var scope = MigrateFromWebScope.Load(collector, _fileSystem, configPath, args.Product); - if (scope is null) + var scopes = MigrateFromWebScope.Select(collector, args.Products); + if (scopes is null) return false; + // One product's failure never blocks the others: the default run covers the whole table, + // so a broken source page should still let every other product migrate — the run itself + // fails at the end so nothing goes unnoticed. + var allResults = new List<MigrationKeyResult>(); + var report = new StringBuilder(); + var failedProducts = 0; + foreach (var scope in scopes) + { + ctx.ThrowIfCancellationRequested(); + var results = await MigrateProduct(collector, args, scope, ctx); + if (results is null) + { + failedProducts++; + continue; + } + + allResults.AddRange(results); + _ = report.Append(FormatReport(scope, args, results)); + _ = report.AppendLine(); + } + + LastResults = allResults; + Console.Write(report.ToString()); + + var failed = allResults.Count(r => r.Outcome == OutcomeFailed); + if (failed > 0) + collector.EmitError(string.Empty, $"{failed} key(s) failed to migrate; see the run report above."); + if (failedProducts > 0) + collector.EmitError(string.Empty, $"{failedProducts} product(s) failed before upload; see the errors above."); + + return failed == 0 && failedProducts == 0; + } + + /// <summary> + /// Runs one product's fetch → parse → filter → stage → upload chain. Returns null (with errors + /// emitted) when the product fails before the upload phase — the caller continues with the + /// remaining products and fails the run at the end. + /// </summary> + private async Task<List<MigrationKeyResult>?> MigrateProduct(IDiagnosticsCollector collector, MigrateFromWebArguments args, MigrateFromWebScope scope, Cancel ctx) + { var sourceUrl = $"https://raw.githubusercontent.com/{scope.Owner}/{scope.Repo}/{scope.Ref}/{scope.Path}"; var markdown = await FetchMarkdown(collector, sourceUrl, ctx); if (markdown is null) - return false; + return null; var releases = ReleaseNotesPageParser.Parse(collector, markdown, sourceUrl, scope); if (releases.Count == 0) { - collector.EmitError(sourceUrl, "No release sections were parsed from the published release notes; refusing to continue with an empty scope."); - return false; + collector.EmitError(sourceUrl, $"No release sections were parsed from the published release notes for '{scope.ProductId}'; refusing to continue with an empty scope."); + return null; } _logger.LogInformation("Parsed {Count} release section(s) from {Url}", releases.Count, sourceUrl); var (inScope, results) = ApplyScopeFilters(releases, scope, args.Versions); + var errorsBeforeStaging = collector.Errors; var staged = StageBundles(collector, scope, inScope); - if (collector.Errors > 0) - return false; + if (collector.Errors > errorsBeforeStaging) + return null; var uploadResults = await UploadCreateOnly(args, staged, ctx); results.AddRange(uploadResults); - LastResults = results; - - var failed = results.Count(r => r.Outcome == OutcomeFailed); - var report = FormatReport(scope, args, results); - Console.WriteLine(report); - - if (failed > 0) - collector.EmitError(string.Empty, $"{failed} key(s) failed to migrate; see the run report above."); - - return failed == 0; + return results; } private async Task<string?> FetchMarkdown(IDiagnosticsCollector collector, string sourceUrl, Cancel ctx) diff --git a/src/tooling/docs-builder/Commands/ChangelogCommand.cs b/src/tooling/docs-builder/Commands/ChangelogCommand.cs index 3acffec863..7215517b38 100644 --- a/src/tooling/docs-builder/Commands/ChangelogCommand.cs +++ b/src/tooling/docs-builder/Commands/ChangelogCommand.cs @@ -1729,25 +1729,24 @@ static async (s, c, state, ct) => await s.ResolveDeployedAsync(c, state, ct) is /// <summary>TEMPORARY: One-off migration of published release notes into the S3 bundle store; removed after elastic/docs-eng-team#683.</summary> /// <remarks> /// <para>Fetches the release-notes Markdown that backs the published pages (at the pinned git ref in the - /// checked-in scope config), maps it to the existing bundle YAML shape, and uploads to + /// checked-in scope table), maps it to the existing bundle YAML shape, and uploads to /// <c>bundle/{product}/</c> with create-only semantics (<c>If-None-Match: *</c>) — existing keys are /// skipped, never overwritten. Prints a per-key run report (created / skipped / failed with reason and /// object ETag) suitable for pasting into the tracking issue.</para> - /// <para>Scope is always explicit: the product must have an entry in <c>config/migrate-from-web.yml</c> - /// (product id → source repo, release-notes path, pinned ref, version cutoff). Nothing runs implicitly - /// for all products. Tracked by elastic/docs-eng-team#736.</para> + /// <para>Migrates every product in the checked-in scope table by default. The table lives in code + /// (<c>MigrateFromWebScope.All</c>: product id → source repo, release-notes path, pinned ref, version + /// cutoff) and grows per rollout wave; use <c>--products</c> to narrow a run for tests and pilots. + /// Tracked by elastic/docs-eng-team#736.</para> /// </remarks> - /// <param name="product">Product id to migrate (e.g. "edot-java"). Must have an entry in the checked-in scope config.</param> + /// <param name="products">Optional: restrict the run to specific product ids (comma-separated or repeated), e.g. "edot-java". Defaults to every product in the checked-in scope table.</param> /// <param name="s3BucketName">Destination S3 bucket. Required unless --dry-run; when provided with --dry-run, existing keys are still inspected so the report distinguishes would-create from skipped.</param> - /// <param name="config">Path to the scope config. Defaults to config/migrate-from-web.yml in the current directory (the copy checked into the docs-builder repository).</param> - /// <param name="versions">Optional: restrict the run to specific versions (comma-separated or repeated). Versions above the configured cutoff are always skipped.</param> + /// <param name="versions">Optional: restrict the run to specific versions (comma-separated or repeated). Versions above a product's cutoff are always skipped.</param> /// <param name="dryRun">Do everything except the S3 writes and report what would be created.</param> - /// <param name="ctx">Cancellation token</param> + /// <param name="ct">Cancellation token</param> [NoOptionsInjection] public async Task<int> MigrateFromWeb( - [Argument] string product, + string[]? products = null, string s3BucketName = "", - [Existing, ExpandUserProfile, RejectSymbolicLinks, FileExtensions(Extensions = "yml,yaml")] FileInfo? config = null, string[]? versions = null, [DryRun] bool dryRun = false, CancellationToken ct = default @@ -1768,10 +1767,9 @@ public async Task<int> MigrateFromWeb( var service = new WebMigrationService(logFactory, FileSystemFactory.RealWrite); var args = new MigrateFromWebArguments { - Product = product, + Products = ExpandCommaSeparated(products), S3BucketName = s3BucketName, DryRun = dryRun, - Config = config?.FullName, Versions = ExpandCommaSeparated(versions) }; diff --git a/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs b/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs index 5c6ba47137..b6cf414155 100644 --- a/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs +++ b/tests/Elastic.Changelog.Tests/Migration/WebMigrationServiceTests.cs @@ -28,7 +28,6 @@ public class WebMigrationServiceTests private readonly ScopedFileSystem _fileSystem; private readonly IAmazonS3 _s3Client = A.Fake<IAmazonS3>(); private readonly TestDiagnosticsCollector _collector; - private readonly string _configPath; private readonly StubHandler _httpHandler; public WebMigrationServiceTests(ITestOutputHelper output) @@ -43,29 +42,16 @@ public WebMigrationServiceTests(ITestOutputHelper output) { Content = new StringContent(ReleaseNotesFixture.Markdown) }); - - _configPath = _mockFileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, "migrate-from-web.yml"); - // language=yaml - _mockFileSystem.AddFile(_configPath, new MockFileData(""" - products: - edot-java: - owner: elastic - repo: elastic-otel-java - path: docs/release-notes/index.md - ref: 9a61ce4faaf08e272c433a083bcc6f0e96d80e0a - cutoff: 1.10.0 - """)); } private WebMigrationService CreateService() => new(NullLoggerFactory.Instance, _fileSystem, _s3Client, _httpHandler); - private MigrateFromWebArguments Args(bool dryRun = false, string bucket = Bucket, string[]? versions = null) => new() + // Default arguments cover the whole checked-in scope table (today: edot-java only). + private static MigrateFromWebArguments Args(bool dryRun = false, string bucket = Bucket, string[]? versions = null) => new() { - Product = "edot-java", S3BucketName = bucket, DryRun = dryRun, - Config = _configPath, Versions = versions ?? [] }; @@ -283,12 +269,12 @@ public async Task DryRunWithBucket_InspectsExistenceButNeverWrites() } [Fact] - public async Task ProductNotInScopeConfig_FailsWithoutAnyNetworkAccess() + public async Task ProductNotInScopeTable_FailsWithoutAnyNetworkAccess() { var service = CreateService(); var ct = TestContext.Current.CancellationToken; - var result = await service.MigrateFromWeb(_collector, Args() with { Product = "not-configured" }, ct); + var result = await service.MigrateFromWeb(_collector, Args() with { Products = ["not-configured"] }, ct); result.Should().BeFalse(); _collector.Errors.Should().BeGreaterThan(0); @@ -296,6 +282,21 @@ public async Task ProductNotInScopeConfig_FailsWithoutAnyNetworkAccess() A.CallTo(_s3Client).MustNotHaveHappened(); } + [Fact] + public async Task ProductsFilter_SelectsOnlyTheRequestedTableEntries() + { + _ = FakeEmptyBucket(); + var service = CreateService(); + var ct = TestContext.Current.CancellationToken; + + var result = await service.MigrateFromWeb(_collector, Args() with { Products = ["edot-java"] }, ct); + + result.Should().BeTrue(); + _collector.Errors.Should().Be(0); + service.LastResults.Where(r => r.Outcome == "created").Should().HaveCount(InScopeVersions.Length); + service.LastResults.Should().AllSatisfy(r => r.Key.Should().StartWith("bundle/edot-java/")); + } + [Fact] public async Task FetchFailure_FailsWithoutAnyS3Calls() { From 0f8f214853cc7b33d6f26fe2f5f89cc53df57d88 Mon Sep 17 00:00:00 2001 From: Felipe Cotti <felipe.cotti@elastic.co> Date: Thu, 13 Aug 2026 04:19:18 -0300 Subject: [PATCH 5/5] FileSystemFactory: extend the mock-temp scope workaround to Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MockFileSystem hardcodes its temp path on every OS (C:\temp on Windows, unix-ified /temp elsewhere) instead of calling Path.GetTempPath(), while AllowedSpecialFolder.Temp resolves the real temp. The existing workaround only covered non-Windows, assuming the two coincide there — they don't: GitHub runners' real temp is under the user profile, so the first test to write through mockFs.Path.GetTempPath() on Windows (migrate-from-web's bundle staging) failed scope validation. Add the inner mock's temp as an explicit root unconditionally; drop once TestableIO#1454 ships. --- .../FileSystemFactory.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Elastic.Documentation.Tooling/FileSystemFactory.cs b/src/Elastic.Documentation.Tooling/FileSystemFactory.cs index 5ba4746dfb..9df45537b8 100644 --- a/src/Elastic.Documentation.Tooling/FileSystemFactory.cs +++ b/src/Elastic.Documentation.Tooling/FileSystemFactory.cs @@ -139,10 +139,11 @@ public static ScopedFileSystem ScopeCurrentWorkingDirectory(IFileSystem inner, I // Builds write options that include AllowedSpecialFolders.Temp PLUS the inner FS's own // GetTempPath() as an explicit root — but only when the inner FS is MockFileSystem. // - // On non-Windows MockFileSystem hardcodes a Unix-ified path ("/temp/", derived from "C:\temp") - // instead of calling System.IO.Path.GetTempPath(). AllowedSpecialFolder.Temp uses the real - // GetTempPath() (e.g. "/tmp/" on Linux), so the two diverge and scope validation fails for any - // path created via mockFs.Path.GetTempPath(). + // MockFileSystem hardcodes its temp path ("C:\temp" on Windows, unix-ified to "/temp/" + // elsewhere) instead of calling System.IO.Path.GetTempPath(). AllowedSpecialFolder.Temp uses + // the real GetTempPath() (e.g. "/tmp/" on Linux, "C:\Users\<user>\AppData\Local\Temp" on + // Windows), so the two diverge on every OS and scope validation fails for any path created + // via mockFs.Path.GetTempPath(). // // Fix tracked upstream: https://github.com/TestableIO/System.IO.Abstractions/pull/1454 // Once that ships and we update the package reference we can drop this workaround. @@ -153,9 +154,9 @@ private static ScopedFileSystemOptions BuildWriteOptions(IFileSystem inner, para { var allRoots = roots.ToList(); var innerType = inner is ScopedFileSystem sf ? sf.InnerType : inner.GetType(); - if (!OperatingSystem.IsWindows() && innerType.Name.Contains("Mock", StringComparison.OrdinalIgnoreCase)) + if (innerType.Name.Contains("Mock", StringComparison.OrdinalIgnoreCase)) { - // Cover MockFileSystem's unixified hardcoded temp path + // Cover MockFileSystem's hardcoded temp path var innerTemp = inner.Path.GetTempPath().TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); if (!string.IsNullOrEmpty(innerTemp) && !allRoots.Contains(innerTemp, StringComparer.OrdinalIgnoreCase)) allRoots.Add(innerTemp);