diff --git a/README.md b/README.md index aa79ab33..d35f4257 100644 --- a/README.md +++ b/README.md @@ -37,22 +37,32 @@ For lists of builds see: ## Docker images -Docker images for Bob's Elixir and Erlang builds are built periodically. Bob checks for new Elixir and Erlang releases every 15 minutes and builds images for any new versions it discovers. The list of operating system distributions we base the images on can be found here: https://github.com/hexpm/bob/blob/main/lib/bob/job/docker_checker.ex#L5. +Docker images for Bob's Elixir and Erlang builds are built periodically. Bob checks for new Elixir and Erlang releases every 15 minutes and builds images for any new versions it discovers. The list of operating system distributions we base the images on is defined by `builds/0` in https://github.com/hexpm/bob/blob/main/lib/bob/job/docker_checker.ex. + +Bob automatically builds the latest patch release of every version family: the newest release per Erlang `major.minor` line (for example only `26.2.5.21` out of the `26.2.x` line) and the newest Elixir patch release per minor version, for every compatible OTP major. Older patch versions are not built automatically but can be requested manually, see below. Tagged images are never changed, that means `hexpm/erlang@22.0-alpine-3.11.2` will always target the tag `OTP-22.0` and won't update when `OTP-22.0.1` is released. -Erlang builds are found at https://hub.docker.com/r/hexpm/erlang, they use the versioning scheme `${OTP_VER}-${OS_NAME}-${OS_VER}` for tags. Builds for all releases since OTP 17 are provided. +Erlang builds are found at https://hub.docker.com/r/hexpm/erlang, they use the versioning scheme `${OTP_VER}-${OS_NAME}-${OS_VER}` for tags. -Elixir builds are found at https://hub.docker.com/r/hexpm/elixir, they use the versioning scheme `${ELIXIR_VER}-erlang-${OTP_VER}-${OS_NAME}-${OS_VER}`. Builds for all major releases since Elixir 1.0.0 are provided. Images are built for all pairs of compatible Elixir and OTP versions. +Elixir builds are found at https://hub.docker.com/r/hexpm/elixir, they use the versioning scheme `${ELIXIR_VER}-erlang-${OTP_VER}-${OS_NAME}-${OS_VER}`. Builds are provided for the following OS and architectures using the docker `OS/ARCH` scheme: * `linux/amd64` * `linux/arm64/v8` +All provided images and tags can be searched at https://bob.hex.pm/docker. + ### Examples of image names * `hexpm/erlang:22.2.8-alpine-3.11.3` * `hexpm/erlang:22.2-ubuntu-bionic-20200219` * `hexpm/elixir:1.10.2-erlang-22.2.8-alpine-3.11.3` * `hexpm/elixir:1.10.0-erlang-22.2-ubuntu-bionic-20200219` + +### Requesting builds + +If you need a specific patch version that is not built automatically, request it at https://bob.hex.pm/request. Sign in with your hex.pm account, then pick the image kind (Erlang or Elixir), the OS and base image version, and the exact Erlang (and Elixir) version. The image is built for both architectures, including the multi-arch manifest; when an Elixir image needs an Erlang base image that does not exist yet, the base is built first and the Elixir image follows automatically. + +Requested builds are limited to ten image builds per user per hour. Build progress can be followed on the jobs dashboard at https://bob.hex.pm. diff --git a/assets/css/app.css b/assets/css/app.css index e7301a37..dbba12a3 100644 --- a/assets/css/app.css +++ b/assets/css/app.css @@ -72,13 +72,13 @@ pre, } .flash { - margin: 10px auto 0; - max-width: 1240px; + margin: 0 0 18px; border-radius: 8px; background: var(--color-grey-800); color: #fff; - padding: 8px 12px; + padding: 11px 14px; font-size: 13px; + line-height: 1.35; } .bob { @@ -126,14 +126,6 @@ pre, letter-spacing: 0; } -.bob-brand__tag { - color: var(--color-grey-400); - font-size: 12px; - font-weight: 500; - padding-left: 11px; - border-left: 1px solid var(--color-grey-700); -} - .bob-nav__tabs { display: flex; align-items: center; @@ -287,6 +279,18 @@ pre, color: var(--color-grey-400); } +.nav-user { + color: var(--color-grey-300); + font-family: var(--font-mono); + font-size: 12.5px; +} + +.req-intro { + padding: 14px 20px 0; + font-size: 13.5px; + color: var(--color-grey-600); +} + .bob-main { flex: 1; } @@ -1079,7 +1083,6 @@ pre, } @media (max-width: 820px) { - .bob-brand__tag, .nav-ext { display: none; } diff --git a/config/config.exs b/config/config.exs index 8cc60f70..c8d70650 100644 --- a/config/config.exs +++ b/config/config.exs @@ -50,7 +50,8 @@ config :bob, master?: true, parallel_jobs: 1, local_jobs: [], - remote_jobs: [] + remote_jobs: [], + hexpm_impl: Bob.Hexpm.Impl config :mime, :types, %{ "application/vnd.bob+erlang" => ["erlang"] diff --git a/config/dev.exs b/config/dev.exs index 5e7f6895..e0f14875 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -2,7 +2,10 @@ import Config config :bob, master_schedule: [], - agent_schedule: [] + agent_schedule: [], + hexpm_url: "http://localhost:4000", + oauth_client_id: "b0b00000-0000-4000-8000-000000000b0b", + oauth_client_secret: "dev_secret_for_testing" config :bob, Bob.Repo, username: "postgres", diff --git a/config/runtime.exs b/config/runtime.exs index 3566302d..a3c30c93 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -16,7 +16,10 @@ if config_env() == :prod do master?: System.fetch_env!("BOB_WHO") == "master", parallel_jobs: String.to_integer(System.fetch_env!("BOB_PARALLEL_JOBS")), local_jobs: jobs_fun.("BOB_LOCAL_JOBS"), - remote_jobs: jobs_fun.("BOB_REMOTE_JOBS") + remote_jobs: jobs_fun.("BOB_REMOTE_JOBS"), + hexpm_url: System.get_env("BOB_HEXPM_URL", "https://hex.pm"), + oauth_client_id: System.fetch_env!("BOB_OAUTH_CLIENT_ID"), + oauth_client_secret: System.fetch_env!("BOB_OAUTH_CLIENT_SECRET") config :ex_aws, access_key_id: System.fetch_env!("BOB_S3_ACCESS_KEY"), diff --git a/config/test.exs b/config/test.exs index f43b4b78..7f177e89 100644 --- a/config/test.exs +++ b/config/test.exs @@ -2,7 +2,12 @@ import Config config :bob, master_schedule: [], - agent_schedule: [] + agent_schedule: [], + github: Bob.FakeGitHub, + hexpm_impl: Bob.FakeHexpm, + hexpm_url: "http://localhost:4000", + oauth_client_id: "b0b00000-0000-4000-8000-000000000b0b", + oauth_client_secret: "test_secret" config :ex_aws, access_key_id: "test", diff --git a/lib/bob/application.ex b/lib/bob/application.ex index 65b26aab..26bb7e19 100644 --- a/lib/bob/application.ex +++ b/lib/bob/application.ex @@ -17,6 +17,7 @@ defmodule Bob.Application do {Finch, name: Bob.Finch}, {Task.Supervisor, [name: Bob.Tasks]}, {Phoenix.PubSub, name: Bob.PubSub}, + Bob.Cache, Bob.DockerHub.RateLimiter, Bob.DockerHub.Auth, runner_spec(), diff --git a/lib/bob/artifacts.ex b/lib/bob/artifacts.ex index 56c9c80d..68712b1f 100644 --- a/lib/bob/artifacts.ex +++ b/lib/bob/artifacts.ex @@ -407,6 +407,19 @@ defmodule Bob.Artifacts do |> MapSet.new() end + @doc """ + The architectures a single tag covers, or `nil` if the tag is absent. Used to + tell whether a manifest tag already spans every architecture. + """ + def docker_tag_archs(repo, tag) do + Repo.one( + from(d in DockerTag, + where: d.repo == ^repo and d.tag == ^tag, + select: d.archs + ) + ) + end + @doc """ `{repo, tag}` for every tag of `repos` whose search metadata carries one of `os_versions`. Scopes the checker's erlang reads to current base images — @@ -471,18 +484,37 @@ defmodule Bob.Artifacts do def replace_base_image_tags(repo, tags) do now = DateTime.utc_now() + tags = Enum.uniq(tags) + rows = Enum.map(tags, &%{repo: repo, tag: &1, inserted_at: now, updated_at: now}) - rows = - tags |> Enum.uniq() |> Enum.map(&%{repo: repo, tag: &1, inserted_at: now, updated_at: now}) - + # Prune vanished tags but keep existing rows untouched, so inserted_at marks + # when a base image first appeared rather than the last reconcile. Repo.transaction(fn -> - Repo.delete_all(from(b in BaseImageTag, where: b.repo == ^repo)) - Repo.insert_all(BaseImageTag, rows) + Repo.delete_all(from(b in BaseImageTag, where: b.repo == ^repo and b.tag not in ^tags)) + Repo.insert_all(BaseImageTag, rows, on_conflict: :nothing, conflict_target: [:repo, :tag]) end) :ok end + @doc """ + `{os, tag}` for every base image first seen on or after `cutoff`. A recent + base image is one of the components that makes a build worth triggering. + """ + def recent_base_image_versions(cutoff) do + Repo.all( + from(b in BaseImageTag, + where: b.inserted_at >= ^cutoff, + select: {b.repo, b.tag} + ) + ) + |> Enum.flat_map(fn + {"library/" <> os, tag} -> [{os, tag}] + _other -> [] + end) + |> MapSet.new() + end + def upsert(attrs) do %Artifact{} |> Artifact.changeset(attrs) diff --git a/lib/bob/build_requests.ex b/lib/bob/build_requests.ex new file mode 100644 index 00000000..7b04a2ce --- /dev/null +++ b/lib/bob/build_requests.ex @@ -0,0 +1,193 @@ +defmodule Bob.BuildRequests do + import Ecto.Query + + require Logger + + alias Bob.BuildRequests.BuildRequest + alias Bob.Repo + + @hourly_build_limit 10 + + def hourly_build_limit(), do: @hourly_build_limit + + def submit(attrs) do + request = struct!(BuildRequest, attrs) + + with :ok <- validate_combo(request) do + case Bob.Job.DockerChecker.request_jobs(request) do + [] -> + {:ok, :already_built} + + jobs -> + submit_jobs(request, attrs, jobs) + end + end + end + + defp submit_jobs(request, attrs, jobs) do + builds_count = builds_count(request.kind, jobs) + + # Serialize a user's concurrent submits (e.g. two tabs) with a per-user + # advisory lock so the limit check and insert can't interleave and both + # slip past the cap. + result = + Repo.transaction(fn -> + lock_user(request.username) + + if over_limit?(request.username, builds_count) do + Repo.rollback(:rate_limited) + else + {:ok, created} = create(Map.put(attrs, :builds_count, builds_count)) + created + end + end) + + case result do + {:ok, created} -> + # Enqueue outside the lock; if it fails the request stays pending and + # the checker's reconciliation enqueues the jobs on its next cycle. + Bob.Queue.add_many(jobs) + + Logger.info( + "BUILD REQUEST by #{created.username}: #{created.kind} #{request_target(created)}" + ) + + {:ok, created} + + {:error, :rate_limited} -> + {:error, :rate_limited} + end + end + + defp lock_user(username) do + Repo.query!("SELECT pg_advisory_xact_lock(hashtext($1))", [username]) + end + + defp validate_combo(%{kind: "erlang"} = request) do + if Bob.Job.DockerChecker.valid_erlang_build?(request.erlang, request.os, request.os_version) do + :ok + else + {:error, :invalid_combo} + end + end + + defp validate_combo(%{kind: "elixir"} = request) do + otp_major = request.erlang |> String.split(".") |> hd() + + if request.elixir != nil and + Bob.Job.DockerChecker.valid_elixir_build?( + request.elixir, + otp_major, + request.erlang, + request.os, + request.os_version + ) do + :ok + else + {:error, :invalid_combo} + end + end + + # Only image builds count against the hourly limit; manifest assembly is + # cheap. An erlang job under an elixir request implies a follow-up elixir + # build on the same arch, so it counts as two. + defp builds_count("erlang", jobs) do + Enum.count(jobs, &match?({{Bob.Job.BuildDockerErlang, _arch}, _args}, &1)) + end + + defp builds_count("elixir", jobs) do + Enum.reduce(jobs, 0, fn + {{Bob.Job.BuildDockerErlang, _arch}, _args}, acc -> acc + 2 + {{Bob.Job.BuildDockerElixir, _arch}, _args}, acc -> acc + 1 + _other, acc -> acc + end) + end + + defp over_limit?(username, builds_count) do + hour_ago = DateTime.add(DateTime.utc_now(), -1, :hour) + builds_count_for_user_since(username, hour_ago) + builds_count > @hourly_build_limit + end + + defp request_target(%{kind: "erlang"} = request) do + "#{request.erlang}-#{request.os}-#{request.os_version}" + end + + defp request_target(%{kind: "elixir"} = request) do + "#{request.elixir}-erlang-#{request.erlang}-#{request.os}-#{request.os_version}" + end + + def create(attrs) do + %BuildRequest{} + |> BuildRequest.changeset(attrs) + |> Repo.insert() + end + + def pending() do + Repo.all(pending_query()) + end + + def pending(limit) when is_integer(limit) and limit > 0 do + Repo.all( + from(request in pending_query(), + limit: ^limit + ) + ) + end + + def complete(request), do: set_state(request, "completed") + + def expire(request), do: set_state(request, "expired") + + def builds_count_for_user_since(username, since) do + Repo.one( + from(request in BuildRequest, + where: request.username == ^username and request.inserted_at >= ^since, + select: coalesce(sum(request.builds_count), 0) + ) + ) + end + + def recent(limit, offset) do + Repo.all( + from(request in BuildRequest, + order_by: [desc: request.inserted_at], + limit: ^limit, + offset: ^offset + ) + ) + end + + def count() do + Repo.aggregate(BuildRequest, :count) + end + + @doc """ + Deletes finished requests older than `older_than_seconds`. Pending requests + are left alone; the checker reconciliation expires or completes them first. + """ + def prune(older_than_seconds) do + cutoff = DateTime.add(DateTime.utc_now(), -older_than_seconds, :second) + + {count, _} = + Repo.delete_all( + from(request in BuildRequest, + where: request.state in ["completed", "expired"] and request.inserted_at < ^cutoff + ) + ) + + count + end + + defp pending_query() do + from(request in BuildRequest, + where: request.state == "pending", + order_by: [asc: request.inserted_at] + ) + end + + defp set_state(request, state) do + request + |> Ecto.Changeset.change(state: state) + |> Repo.update!() + end +end diff --git a/lib/bob/build_requests/build_request.ex b/lib/bob/build_requests/build_request.ex new file mode 100644 index 00000000..d95fff51 --- /dev/null +++ b/lib/bob/build_requests/build_request.ex @@ -0,0 +1,27 @@ +defmodule Bob.BuildRequests.BuildRequest do + use Ecto.Schema + + import Ecto.Changeset + + @fields [:username, :kind, :elixir, :erlang, :os, :os_version, :builds_count] + @required [:username, :kind, :erlang, :os, :os_version, :builds_count] + + schema "build_requests" do + field(:username, :string) + field(:kind, :string) + field(:elixir, :string) + field(:erlang, :string) + field(:os, :string) + field(:os_version, :string) + field(:builds_count, :integer) + field(:state, :string, default: "pending") + timestamps(type: :utc_datetime_usec) + end + + def changeset(build_request, attrs) do + build_request + |> cast(attrs, @fields) + |> validate_required(@required) + |> validate_inclusion(:kind, ~w(erlang elixir)) + end +end diff --git a/lib/bob/cache.ex b/lib/bob/cache.ex new file mode 100644 index 00000000..7b4c5865 --- /dev/null +++ b/lib/bob/cache.ex @@ -0,0 +1,39 @@ +defmodule Bob.Cache do + @moduledoc """ + Read-through cache with per-key TTLs for values that are expensive to fetch + (external version metadata, refs) but fine to serve slightly stale. + """ + + use GenServer + + @table __MODULE__ + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + def fetch(key, ttl_seconds, fun) do + now = System.system_time(:second) + + case :ets.lookup(@table, key) do + [{^key, value, expires_at}] when expires_at > now -> + value + + _ -> + value = fun.() + :ets.insert(@table, {key, value, now + ttl_seconds}) + value + end + end + + def clear() do + :ets.delete_all_objects(@table) + :ok + end + + @impl true + def init(_opts) do + :ets.new(@table, [:named_table, :public, :set, read_concurrency: true]) + {:ok, nil} + end +end diff --git a/lib/bob/github.ex b/lib/bob/github.ex index 9e5f055b..751f9fb3 100644 --- a/lib/bob/github.ex +++ b/lib/bob/github.ex @@ -3,8 +3,31 @@ defmodule Bob.GitHub do def fetch_repo_refs(repo) do branches = github_request(@github_url <> "repos/#{repo}/branches?per_page=100") - tags = github_request(@github_url <> "repos/#{repo}/tags?per_page=100") - response_to_refs(branches) ++ response_to_refs(tags) + response_to_refs(branches) ++ fetch_repo_tags(repo) + end + + def fetch_repo_tags(repo) do + repo + |> tags_url() + |> github_request() + |> response_to_refs() + end + + def fetch_repo_releases(repo) do + repo + |> releases_url() + |> github_request() + end + + # Releases come back newest-first, so the first page is enough to find + # anything published recently without paging through years of history. + def fetch_recent_releases(repo) do + {body, _headers} = + repo + |> releases_url() + |> github_get() + + body end defp response_to_refs(response) do @@ -14,21 +37,25 @@ defmodule Bob.GitHub do end defp github_request(url) do + {body, headers} = github_get(url) + + if url = next_link(headers) do + body ++ github_request(url) + else + body + end + end + + defp github_get(url) do user = Application.get_env(:bob, :github_user) token = Application.get_env(:bob, :github_token) - opts = [basic_auth: {user, token}] + opts = if user && token, do: [basic_auth: {user, token}], else: [] {:ok, 200, headers, body} = Bob.HTTP.retry("GitHub #{url}", fn -> Bob.HTTP.request(:get, url, [], "", opts) end) - body = JSON.decode!(body) - - if url = next_link(headers) do - body ++ github_request(url) - else - body - end + {JSON.decode!(body), headers} end defp next_link(headers) do @@ -46,4 +73,8 @@ defmodule Bob.GitHub do end end) end + + defp tags_url(repo), do: @github_url <> "repos/#{repo}/tags?per_page=100" + + defp releases_url(repo), do: @github_url <> "repos/#{repo}/releases?per_page=100" end diff --git a/lib/bob/hexpm.ex b/lib/bob/hexpm.ex new file mode 100644 index 00000000..f85c7be3 --- /dev/null +++ b/lib/bob/hexpm.ex @@ -0,0 +1,29 @@ +defmodule Bob.Hexpm do + @callback exchange_code( + code :: String.t(), + code_verifier :: String.t(), + redirect_uri :: String.t() + ) :: + {:ok, map()} | {:error, term()} + @callback refresh_token(refresh_token :: String.t()) :: {:ok, map()} | {:error, term()} + @callback revoke_token(token :: String.t()) :: :ok | {:error, term()} + @callback get_current_user(access_token :: String.t()) :: {:ok, map()} | {:error, term()} + + def exchange_code(code, code_verifier, redirect_uri) do + impl().exchange_code(code, code_verifier, redirect_uri) + end + + def refresh_token(refresh_token) do + impl().refresh_token(refresh_token) + end + + def revoke_token(token) do + impl().revoke_token(token) + end + + def get_current_user(access_token) do + impl().get_current_user(access_token) + end + + defp impl(), do: Application.fetch_env!(:bob, :hexpm_impl) +end diff --git a/lib/bob/hexpm/impl.ex b/lib/bob/hexpm/impl.ex new file mode 100644 index 00000000..f19af4a2 --- /dev/null +++ b/lib/bob/hexpm/impl.ex @@ -0,0 +1,78 @@ +defmodule Bob.Hexpm.Impl do + @behaviour Bob.Hexpm + + # Token requests are not retried: authorization codes are single-use, so a + # replay after a flaky response would be rejected by hexpm anyway. + @impl true + def exchange_code(code, code_verifier, redirect_uri) do + token_request(%{ + "grant_type" => "authorization_code", + "code" => code, + "redirect_uri" => redirect_uri, + "client_id" => Bob.OAuth.client_id(), + "client_secret" => Bob.OAuth.client_secret(), + "code_verifier" => code_verifier, + "name" => "bob.hex.pm" + }) + end + + @impl true + def refresh_token(refresh_token) do + token_request(%{ + "grant_type" => "refresh_token", + "refresh_token" => refresh_token, + "client_id" => Bob.OAuth.client_id(), + "client_secret" => Bob.OAuth.client_secret() + }) + end + + @impl true + def revoke_token(token) do + url = Bob.OAuth.hexpm_url() <> "/api/oauth/revoke" + headers = [{"content-type", "application/json"}] + + body = + JSON.encode!(%{ + "token" => token, + "client_id" => Bob.OAuth.client_id(), + "client_secret" => Bob.OAuth.client_secret() + }) + + case Bob.HTTP.request(:post, url, headers, body) do + {:ok, status, _headers, _body} when status in 200..299 -> :ok + {:ok, status, _headers, body} -> {:error, {status, body}} + {:error, reason} -> {:error, reason} + end + end + + @impl true + def get_current_user(access_token) do + url = Bob.OAuth.hexpm_url() <> "/api/users/me" + headers = [{"authorization", "Bearer " <> access_token}, {"accept", "application/json"}] + + :get + |> Bob.HTTP.request(url, headers, "") + |> read_response() + end + + defp token_request(body) do + url = Bob.OAuth.hexpm_url() <> "/api/oauth/token" + headers = [{"content-type", "application/json"}] + + :post + |> Bob.HTTP.request(url, headers, JSON.encode!(body)) + |> read_response() + end + + defp read_response({:ok, status, _headers, body}) when status in 200..299 do + {:ok, JSON.decode!(body)} + end + + defp read_response({:ok, status, _headers, body}) do + {:error, {status, body}} + end + + defp read_response({:error, reason}) do + {:error, reason} + end +end diff --git a/lib/bob/job/docker_checker.ex b/lib/bob/job/docker_checker.ex index 94c4cfbf..686bc5fc 100644 --- a/lib/bob/job/docker_checker.ex +++ b/lib/bob/job/docker_checker.ex @@ -3,6 +3,7 @@ defmodule Bob.Job.DockerChecker do @erlang_tag_regex ~r"^(.+)-(alpine|ubuntu|debian)-(.+)$" @elixir_tag_regex ~r"^(.+)-erlang-(.+)-(alpine|ubuntu|debian)-(.+)$" + @elixir_release_asset_regex ~r"^elixir-otp-(\d+)\.zip$" @archs ["amd64", "arm64"] @erlang_arch_repos Enum.map(@archs, &"hexpm/erlang-#{&1}") @@ -55,11 +56,13 @@ defmodule Bob.Job.DockerChecker do def run() do erlang() elixir() + requests() manifest() end def run(:erlang), do: erlang() def run(:elixir), do: elixir() + def run(:requests), do: requests() def run(:manifest), do: manifest() def priority(), do: 1 @@ -67,7 +70,13 @@ defmodule Bob.Job.DockerChecker do def concurrency(), do: :shared def erlang() do + recent_os = recent_base_image_versions() + recent_otp = recent_erlang_versions() + expected_erlang_tags() + |> Enum.filter(fn {erlang, os, os_version, _arch} -> + MapSet.member?(recent_os, {os, os_version}) or MapSet.member?(recent_otp, erlang) + end) |> Enum.group_by(fn {_erlang, _os, _os_version, arch} -> arch end) |> Enum.flat_map(fn {arch, expected} -> present = @@ -87,7 +96,7 @@ defmodule Bob.Job.DockerChecker do defp erlang_tag_name({erlang, os, os_version, _arch}), do: "#{erlang}-#{os}-#{os_version}" def expected_erlang_tags() do - refs = erlang_refs() + refs = latest_erlang_refs(erlang_refs()) Stream.flat_map(builds(), fn {os, os_versions} -> Stream.flat_map(refs, fn ref -> @@ -113,6 +122,20 @@ defmodule Bob.Job.DockerChecker do end) end + def archs(), do: @archs + + def erlang_versions() do + Enum.map(erlang_refs(), fn "OTP-" <> version -> version end) + end + + # Whether the build rules allow this erlang/os/os_version combination on all archs. + def valid_erlang_build?(erlang, os, os_version) do + ref = "OTP-" <> erlang + + build_erlang_ref?(os, ref) and build_erlang_ref?(os, os_version, ref) and + Enum.all?(@archs, &build_erlang_ref?(&1, os, os_version, ref)) + end + defp build_erlang_ref?(_os, "OTP-18.0-rc2"), do: false defp build_erlang_ref?("alpine", "OTP-17" <> _), do: false @@ -221,13 +244,96 @@ defmodule Bob.Job.DockerChecker do end defp erlang_refs() do - "erlang/otp" - |> Bob.GitHub.fetch_repo_refs() + {:repo_refs, "erlang/otp"} + |> cached_github(fn -> github().fetch_repo_refs("erlang/otp") end) |> Enum.map(fn {ref_name, _ref} -> ref_name end) |> Enum.filter(&String.starts_with?(&1, "OTP-")) |> Enum.sort(&(cmp_erlang_tags(&1, &2) != :lt)) end + defp github(), do: Application.get_env(:bob, :github, Bob.GitHub) + + # A run() calls erlang() then elixir(), and both reach for the same refs and + # release lists, so cache the raw GitHub responses briefly to collapse the + # duplicate calls within a cycle. The TTL is far below the 15-minute schedule, + # so each cycle still fetches fresh. + @github_cache_ttl 60 + + defp cached_github(key, fun) do + Bob.Cache.fetch({__MODULE__, key}, @github_cache_ttl, fun) + end + + # Auto-builds only fire when one of the image's components — its base image, + # OTP, or Elixir — was released within this window. Old combinations that + # Docker Hub prunes for lack of pulls then stay pruned instead of being + # rebuilt every cycle; users can still request them explicitly. + @build_freshness_days 30 + + defp freshness_cutoff() do + DateTime.add(DateTime.utc_now(), -@build_freshness_days, :day) + end + + defp recent_base_image_versions() do + Bob.Artifacts.recent_base_image_versions(freshness_cutoff()) + end + + defp recent_erlang_versions() do + cutoff = freshness_cutoff() + + {:recent_releases, "erlang/otp"} + |> cached_github(fn -> github().fetch_recent_releases("erlang/otp") end) + |> Enum.flat_map(fn + %{"tag_name" => "OTP-" <> version, "published_at" => published_at} -> + if release_recent?(published_at, cutoff), do: [version], else: [] + + _other -> + [] + end) + |> MapSet.new() + end + + defp recent_elixir_versions() do + cutoff = freshness_cutoff() + + {:recent_releases, "elixir-lang/elixir"} + |> cached_github(fn -> github().fetch_recent_releases("elixir-lang/elixir") end) + |> Enum.flat_map(fn + %{"tag_name" => "v" <> version, "published_at" => published_at} -> + if release_recent?(published_at, cutoff), do: [version], else: [] + + _other -> + [] + end) + |> MapSet.new() + end + + defp release_recent?(published_at, cutoff) when is_binary(published_at) do + case DateTime.from_iso8601(published_at) do + {:ok, datetime, _offset} -> DateTime.compare(datetime, cutoff) != :lt + _error -> false + end + end + + defp release_recent?(_published_at, _cutoff), do: false + + # Keeps only the newest ref per OTP major.minor line. Stable releases rank + # above prereleases within a line, so RCs drop out once a stable lands. + # Selection happens before the per-OS filters: a line whose newest ref an OS + # rule rejects gets no older fallback. + def latest_erlang_refs(refs) do + refs + |> Enum.sort(&(cmp_erlang_tags(&1, &2) != :lt)) + |> Enum.uniq_by(fn "OTP-" <> version -> + version |> to_matchable() |> elem(0) |> Enum.take(2) + end) + end + + defp latest_erlang_versions() do + erlang_refs() + |> latest_erlang_refs() + |> MapSet.new(fn "OTP-" <> version -> version end) + end + defp cmp_erlang_tags("OTP-" <> left, "OTP-" <> right) do cmp_erlang_components(to_matchable(left), to_matchable(right)) end @@ -270,7 +376,15 @@ defmodule Bob.Job.DockerChecker do end def elixir() do + recent_os = recent_base_image_versions() + recent_otp = recent_erlang_versions() + recent_elixir = recent_elixir_versions() + expected_elixir_tags() + |> Enum.filter(fn {elixir, erlang, os, os_version, _arch} -> + MapSet.member?(recent_os, {os, os_version}) or MapSet.member?(recent_otp, erlang) or + MapSet.member?(recent_elixir, elixir) + end) |> Enum.group_by(fn {_elixir, _erlang, _os, _os_version, arch} -> arch end) |> Enum.flat_map(fn {arch, expected} -> present = @@ -293,10 +407,12 @@ defmodule Bob.Job.DockerChecker do def expected_elixir_tags() do builds = builds() - refs = elixir_builds() + refs = latest_elixir_builds(elixir_builds()) + latest_erlang = latest_erlang_versions() Stream.flat_map(current_erlang_tags(builds), fn {erlang, os, os_version, erlang_arch} -> - if not skip_elixir_for_erlang?(erlang) and os_version in builds[os] do + if MapSet.member?(latest_erlang, erlang) and not skip_elixir_for_erlang?(erlang) and + os_version in builds[os] do Stream.flat_map(refs, fn {"v" <> elixir, otp_major} -> if compatible_elixir_and_erlang?(otp_major, erlang) do [{elixir, erlang, os, os_version, erlang_arch}] @@ -325,21 +441,30 @@ defmodule Bob.Job.DockerChecker do end def elixir_builds() do - "builds/elixir" - |> Bob.Store.fetch_built_refs() - |> Stream.map(fn {build_name, _ref} -> build_name end) - |> Stream.map(&split_elixir_build/1) + {:repo_releases, "elixir-lang/elixir"} + |> cached_github(fn -> github().fetch_repo_releases("elixir-lang/elixir") end) + |> Stream.flat_map(&release_elixir_builds/1) |> Stream.filter(&build_elixir_ref?/1) |> Enum.sort(&cmp_elixir_tags/2) - |> Enum.reject(fn {_elixir, otp} -> otp == nil end) |> Enum.reject(fn {"v" <> elixir, _otp} -> skip_elixir?(elixir) end) end - defp split_elixir_build(build_name) do - case String.split(build_name, "-otp-") do - [elixir, major_otp] -> {elixir, major_otp} - [elixir] -> {elixir, nil} - end + # Whether the build rules allow this elixir build on this exact erlang/os/os_version. + def valid_elixir_build?(elixir, otp_major, erlang, os, os_version) do + build_elixir_ref?({"v" <> elixir, otp_major}) and not skip_elixir?(elixir) and + not skip_elixir_for_erlang?(erlang) and compatible_elixir_and_erlang?(otp_major, erlang) and + valid_erlang_build?(erlang, os, os_version) + end + + # Keeps only the newest build per {elixir major.minor, otp major} pair. + # Stable releases rank above prereleases within a pair. + def latest_elixir_builds(builds) do + builds + |> Enum.sort(&cmp_elixir_tags/2) + |> Enum.uniq_by(fn {"v" <> elixir, otp} -> + version = Version.parse!(normalize_version(elixir)) + {version.major, version.minor, otp} + end) end defp cmp_elixir_tags({"v" <> elixir_left, otp_left}, {"v" <> elixir_right, otp_right}) do @@ -373,6 +498,15 @@ defmodule Bob.Job.DockerChecker do end end + defp release_elixir_builds(%{"tag_name" => tag_name, "assets" => assets}) do + Enum.flat_map(assets, fn %{"name" => name} -> + case Regex.run(@elixir_release_asset_regex, name, capture: :all_but_first) do + [otp_major] -> [{tag_name, otp_major}] + _other -> [] + end + end) + end + defp compatible_elixir_and_erlang?(otp_major, erlang) do String.starts_with?(erlang, otp_major <> ".") end @@ -388,6 +522,101 @@ defmodule Bob.Job.DockerChecker do Version.compare(normalize_version(elixir), "1.10.0-0") == :lt end + # Requests that still have no tags after this long are unbuildable; expiring + # them stops the reconciliation from re-enqueueing doomed jobs forever. + @request_ttl_days 14 + + # User-requested builds are one-time: pinned to the os_version current at + # request time and reconciled here until every target tag exists. Requests + # whose os_version rotated out of builds() expire instead. + def requests() do + builds = builds() + + Enum.each(Bob.BuildRequests.pending(), fn request -> + cond do + request.os_version not in Map.get(builds, request.os, []) -> + Bob.BuildRequests.expire(request) + + DateTime.diff(DateTime.utc_now(), request.inserted_at, :day) >= @request_ttl_days -> + Bob.BuildRequests.expire(request) + + true -> + case request_jobs(request) do + [] -> Bob.BuildRequests.complete(request) + jobs -> Bob.Queue.add_many(jobs) + end + end + end) + end + + def request_jobs(%{kind: "erlang"} = request) do + %{erlang: erlang, os: os, os_version: os_version} = request + tag = "#{erlang}-#{os}-#{os_version}" + + arch_jobs = + Enum.flat_map(@archs, fn arch -> + if tag_present?("hexpm/erlang-#{arch}", tag) do + [] + else + [{{Bob.Job.BuildDockerErlang, arch}, [erlang, os, os_version]}] + end + end) + + arch_jobs ++ manifest_jobs(arch_jobs, "erlang", tag, {erlang, os, os_version}) + end + + # The elixir image is built FROM the erlang image, and failed jobs are not + # retried by the queue, so the elixir job is only enqueued once its base tag + # exists; until then the erlang build is enqueued and the next reconciliation + # cycle picks the request up again. + def request_jobs(%{kind: "elixir"} = request) do + %{elixir: elixir, erlang: erlang, os: os, os_version: os_version} = request + erlang_tag = "#{erlang}-#{os}-#{os_version}" + elixir_tag = "#{elixir}-erlang-#{erlang}-#{os}-#{os_version}" + + arch_jobs = + Enum.flat_map(@archs, fn arch -> + cond do + tag_present?("hexpm/elixir-#{arch}", elixir_tag) -> + [] + + tag_present?("hexpm/erlang-#{arch}", erlang_tag) -> + [{{Bob.Job.BuildDockerElixir, arch}, [elixir, erlang, os, os_version]}] + + true -> + [{{Bob.Job.BuildDockerErlang, arch}, [erlang, os, os_version]}] + end + end) + + arch_jobs ++ manifest_jobs(arch_jobs, "elixir", elixir_tag, {elixir, erlang, os, os_version}) + end + + # A request is only done once its manifest spans every architecture, so while + # the per-arch tags are still building we wait, and once they exist we enqueue + # the manifest ourselves rather than relying solely on manifest/0. + defp manifest_jobs(pending_arch_jobs, _kind, _tag, _key) when pending_arch_jobs != [], do: [] + + defp manifest_jobs([], kind, tag, key) do + if manifest_complete?(kind, tag) do + [] + else + [{Bob.Job.DockerManifest, [kind, key]}] + end + end + + defp manifest_complete?(kind, tag) do + case Bob.Artifacts.docker_tag_archs("hexpm/#{kind}", tag) do + nil -> false + archs -> Enum.all?(@archs, &(&1 in archs)) + end + end + + defp tag_present?(repo, tag) do + repo + |> Bob.Artifacts.docker_tags_present([tag]) + |> MapSet.member?(tag) + end + def manifest() do check_manifests("erlang") check_manifests("elixir") diff --git a/lib/bob/oauth.ex b/lib/bob/oauth.ex new file mode 100644 index 00000000..0d315fef --- /dev/null +++ b/lib/bob/oauth.ex @@ -0,0 +1,43 @@ +defmodule Bob.OAuth do + @moduledoc """ + OAuth 2.0 Authorization Code with PKCE helpers for authenticating against hexpm. + """ + + def generate_code_verifier() do + :crypto.strong_rand_bytes(32) + |> Base.url_encode64(padding: false) + end + + def generate_code_challenge(verifier) do + :crypto.hash(:sha256, verifier) + |> Base.url_encode64(padding: false) + end + + def generate_state() do + :crypto.strong_rand_bytes(16) + |> Base.url_encode64(padding: false) + end + + def authorization_url(opts) do + state = Keyword.fetch!(opts, :state) + code_challenge = Keyword.fetch!(opts, :code_challenge) + redirect_uri = Keyword.fetch!(opts, :redirect_uri) + + query = + URI.encode_query(%{ + "response_type" => "code", + "client_id" => client_id(), + "redirect_uri" => redirect_uri, + "scope" => "api:read", + "state" => state, + "code_challenge" => code_challenge, + "code_challenge_method" => "S256" + }) + + "#{hexpm_url()}/oauth/authorize?#{query}" + end + + def hexpm_url(), do: Application.fetch_env!(:bob, :hexpm_url) + def client_id(), do: Application.fetch_env!(:bob, :oauth_client_id) + def client_secret(), do: Application.fetch_env!(:bob, :oauth_client_secret) +end diff --git a/lib/bob/queue/maintenance.ex b/lib/bob/queue/maintenance.ex index c672d3ba..32522b95 100644 --- a/lib/bob/queue/maintenance.ex +++ b/lib/bob/queue/maintenance.ex @@ -2,8 +2,9 @@ defmodule Bob.Queue.Maintenance do @moduledoc """ Periodic queue maintenance, guarded by a Postgres advisory lock so exactly one master instance does the work: requeues stale running jobs (failing them - once their requeues are exhausted), prunes expired backoff rows, and prunes - job history that is older than the retention window or beyond the row cap. + once their requeues are exhausted), prunes expired backoff rows, prunes job + history that is older than the retention window or beyond the row cap, and + prunes old build requests. """ use GenServer @@ -22,6 +23,7 @@ defmodule Bob.Queue.Maintenance do @backoff_expiry_seconds 7 * 24 * 60 * 60 @history_retention_seconds 90 * 24 * 60 * 60 @history_max_jobs 10_000 + @build_request_retention_seconds 90 * 24 * 60 * 60 @advisory_lock_key 4_771_001 def start_link([]) do @@ -49,6 +51,7 @@ defmodule Bob.Queue.Maintenance do sweep_stale_running() prune_failures() prune_history() + prune_build_requests() end end) @@ -118,6 +121,10 @@ defmodule Bob.Queue.Maintenance do Application.get_env(:bob, :history_max_jobs, @history_max_jobs) end + defp prune_build_requests() do + Bob.BuildRequests.prune(@build_request_retention_seconds) + end + defp schedule_tick() do Process.send_after(self(), :tick, @interval_seconds * 1000) end diff --git a/lib/bob/store.ex b/lib/bob/store.ex index 999f24e3..eac655bd 100644 --- a/lib/bob/store.ex +++ b/lib/bob/store.ex @@ -1,21 +1,6 @@ defmodule Bob.Store do @bucket "s3.hex.pm" - # TODO: Use S3 object metadata - def fetch_built_refs(build_path) do - path = Path.join(build_path, "builds.txt") - - case ExAws.S3.get_object(@bucket, path, []) |> ExAws.request() do - {:ok, %{body: body}} -> - body - |> String.split("\n", trim: true) - |> Map.new(&line_to_ref/1) - - {:error, {:http_error, 404, _}} -> - %{} - end - end - def fetch_file(path) do %{body: body} = ExAws.S3.get_object(@bucket, path, []) |> ExAws.request!() body @@ -38,9 +23,4 @@ defmodule Bob.Store do |> ExAws.stream!() |> Stream.map(&Map.get(&1, :key)) end - - defp line_to_ref(line) do - destructure [ref_name, ref], String.split(line, " ", trim: true) - {ref_name, ref} - end end diff --git a/lib/bob_web/components/core_components.ex b/lib/bob_web/components/core_components.ex index cf5e8d91..6ebca154 100644 --- a/lib/bob_web/components/core_components.ex +++ b/lib/bob_web/components/core_components.ex @@ -228,13 +228,17 @@ defmodule BobWeb.CoreComponents do attr(:value, :string, default: "") attr(:options, :list, required: true) attr(:disabled, :boolean, default: false) + # nil renders a selectable "any" option (filters); a string renders a + # non-selectable placeholder (forms that demand an explicit choice). + attr(:prompt, :string, default: nil) def filter_select(assigns) do ~H"""