bug: configure an OS whose /etc is assembled at boot without breaking it

crates/osmodifier/src/users.rs

@@ -5,7 +5,7 @@
 
 use std::{fs, os::unix::fs::PermissionsExt, path::Path};
 
-use anyhow::{bail, Context, Error};
+use anyhow::{bail, ensure, Context, Error};
 use log::{debug, info};
 use osutils::dependencies::Dependency;
 
@@ -30,8 +30,30 @@ const PASSWD_FIELD_SHELL: usize = 6;
 const SSH_DIR_MODE: u32 = 0o700;
 const AUTHORIZED_KEYS_MODE: u32 = 0o600;
 
+/// The user database `useradd` and friends read and write.
+const PASSWD_PATH: &str = "/etc/passwd";
+
 /// Add or update all configured users.
 pub fn add_or_update_users(ctx: &OsModifierContext, users: &[MICUser]) -> Result<(), Error> {
+    if users.is_empty() {
+        return Ok(());
+    }
+
+    // `useradd` creates a user database from scratch when none is present,
+    // holding only the users it is asked to add. On an image that assembles
+    // /etc at boot rather than shipping it, that file replaces the real one
+    // and the system loses every account it had, root included, leaving it
+    // unbootable. Refuse instead: without an existing database there is
+    // nothing meaningful to add users to.
+    let passwd_path = ctx.path(PASSWD_PATH);
+    ensure!(
+        passwd_path.exists(),
+        "Cannot configure users: '{}' does not exist in the target OS. Creating it would \
+        replace the user database the image provides elsewhere, leaving the system with only \
+        the configured users and no way to boot.",
+        passwd_path.display()
+    );
+
     for user in users {
         add_or_update_user(ctx, user)
             .with_context(|| format!("Failed to configure user '{}'", user.name))?;
@@ -538,3 +560,63 @@ fn set_startup_command(ctx: &OsModifierContext, username: &str, cmd: &str) -> Re
 
     atomic_write_file(&passwd_path, &result)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use tempfile::TempDir;
+
+    /// An empty user list is a no-op, and must not require a user database.
+    #[test]
+    fn test_add_or_update_users_empty_is_noop() {
+        let root = TempDir::new().unwrap();
+        let ctx = OsModifierContext {
+            root: root.path().to_path_buf(),
+        };
+        add_or_update_users(&ctx, &[]).unwrap();
+    }
+
+    /// Configuring users against a root with no user database is refused.
+    ///
+    /// `useradd` would otherwise create one holding only the configured users,
+    /// replacing the database an image assembles at boot and leaving the system
+    /// unbootable.
+    #[test]
+    fn test_add_or_update_users_without_passwd_is_refused() {
+        let root = TempDir::new().unwrap();
+        let ctx = OsModifierContext {
+            root: root.path().to_path_buf(),
+        };
+        let user = MICUser {
+            name: "someone".to_string(),
+            uid: None,
+            password: None,
+            password_expires_days: None,
+            ssh_public_keys: Vec::new(),
+            primary_group: None,
+            secondary_groups: Vec::new(),
+            startup_command: None,
+            home_directory: None,
+        };
+
+        let err = add_or_update_users(&ctx, std::slice::from_ref(&user)).unwrap_err();
+        assert!(
+            err.to_string().contains("does not exist in the target OS"),
+            "got: {err}"
+        );
+
+        // With a database present the guard lets the operation through.
+        fs::create_dir_all(root.path().join("etc")).unwrap();
+        fs::write(
+            root.path().join("etc/passwd"),
+            "root:x:0:0::/root:/bin/bash\n",
+        )
+        .unwrap();
+        let err = add_or_update_users(&ctx, &[user]).unwrap_err();
+        assert!(
+            !err.to_string().contains("does not exist in the target OS"),
+            "guard should have passed, got: {err}"
+        );
+    }
+}

crates/osutils/src/overlay.rs

@@ -1,4 +1,7 @@
-use std::path::{Path, PathBuf};
+use std::{
+    fs,
+    path::{Path, PathBuf},
+};
 
 use anyhow::{Context, Error};
 use tempfile::TempDir;
@@ -67,6 +70,101 @@ impl EphemeralOverlay {
     }
 }
 
+/// An overlay mounted from explicitly chosen layers.
+///
+/// Unlike [`EphemeralOverlay`], whose upper layer is a temporary directory
+/// discarded on unmount, this mounts a caller-provided upper layer, so writes
+/// made through the overlay persist in it. That makes the merged view -- and
+/// crucially overlayfs copy-up, which duplicates a lower-layer file into the
+/// upper layer on first write -- available to code that would otherwise see
+/// only the upper layer's contents.
+pub struct LayeredOverlay {
+    target_path: PathBuf,
+    work_dir: PathBuf,
+}
+
+impl LayeredOverlay {
+    /// Mounts an overlay over `target_path` with the given layers.
+    ///
+    /// `work_dir` must be on the same filesystem as `upper_dir`, as overlayfs
+    /// requires. It is created if missing, as are the other directories.
+    /// `options` are appended to the layer options, for callers that need to
+    /// match the mount options another party uses for the same overlay.
+    pub fn mount(
+        target_path: impl AsRef<Path>,
+        lower_dir: impl AsRef<Path>,
+        upper_dir: impl AsRef<Path>,
+        work_dir: impl AsRef<Path>,
+        options: Option<&str>,
+    ) -> Result<Self, Error> {
+        let target_path = target_path.as_ref();
+        let (lower_dir, upper_dir, work_dir) =
+            (lower_dir.as_ref(), upper_dir.as_ref(), work_dir.as_ref());
+
+        for dir in [upper_dir, work_dir] {
+            files::create_dirs(dir)
+                .with_context(|| format!("Failed to create overlay dir '{}'", dir.display()))?;
+        }
+
+        let path_str = |p: &Path| -> Result<String, Error> {
+            Ok(p.to_str()
+                .with_context(|| format!("Failed to decode '{}'", p.display()))?
+                .to_owned())
+        };
+
+        let mut opts = format!(
+            "lowerdir={},upperdir={},workdir={}",
+            path_str(lower_dir)?,
+            path_str(upper_dir)?,
+            path_str(work_dir)?,
+        );
+        if let Some(options) = options {
+            opts.push(',');
+            opts.push_str(options);
+        }
+
+        Dependency::Mount
+            .cmd()
+            .arg("-t")
+            .arg("overlay")
+            .arg("overlay")
+            .arg("-o")
+            .arg(opts)
+            .arg(target_path)
+            .run_and_check()
+            .with_context(|| format!("Failed to mount overlay on '{}'", target_path.display()))?;
+
+        Ok(Self {
+            target_path: target_path.to_owned(),
+            work_dir: work_dir.to_owned(),
+        })
+    }
+
+    /// Unmounts the overlay, leaving both layers in place.
+    ///
+    /// The work directory is removed, since it is overlayfs bookkeeping rather
+    /// than content and is meaningless once the overlay is gone.
+    pub fn unmount(self) -> Result<(), Error> {
+        Dependency::Umount
+            .cmd()
+            .arg(&self.target_path)
+            .run_and_check()
+            .with_context(|| {
+                format!(
+                    "Failed to unmount overlay on '{}'",
+                    self.target_path.display()
+                )
+            })?;
+
+        fs::remove_dir_all(&self.work_dir).with_context(|| {
+            format!(
+                "Failed to remove overlay work directory '{}'",
+                self.work_dir.display()
+            )
+        })
+    }
+}
+
 #[cfg(feature = "functional-test")]
 #[cfg_attr(not(test), allow(unused_imports, dead_code))]
 mod functional_test {

crates/sysdefs/src/acl.rs

@@ -1,7 +1,8 @@
 //! Azure Container Linux (ACL) system definitions.
 //!
 //! Fixed PARTUUIDs and partition type UUIDs for the ACL UKI disk layout,
-//! sourced from acl-scripts disk_layout_uki.json.
+//! sourced from acl-scripts disk_layout_uki.json, and the layout of the
+//! `/etc` overlay ACL assembles at boot.
 
 use uuid::{uuid, Uuid};
 
@@ -13,3 +14,24 @@ pub const ACL_USR_B_PARTUUID: Uuid = uuid!("e03dd35c-7c2d-4a47-b3fe-27f15780a57c
 
 /// ACL USR partition type UUID.
 pub const ACL_USR_PARTITION_TYPE_UUID: Uuid = uuid!("5dfbf5f4-2848-4bac-aa5e-0d9a20b745a6");
+
+/// Directory on the sealed `/usr` holding the factory contents of `/etc`.
+///
+/// ACL does not ship an `/etc` on its root filesystem. Its initrd instead
+/// mounts an overlay over `/etc` whose lower layer is this directory and whose
+/// upper layer is the root filesystem's own `/etc`, so the factory files are
+/// visible while modifications land on the root filesystem. See the
+/// `99setup-root` dracut module in the ACL image.
+pub const ACL_ETC_LOWER_DIR: &str = "/usr/share/distro/etc";
+
+/// Working directory the `/etc` overlay requires, on the root filesystem.
+///
+/// overlayfs requires the work directory to live on the same filesystem as the
+/// upper layer. ACL's initrd uses this path for the same reason.
+pub const ACL_ETC_WORK_DIR: &str = "/.etc-work";
+
+/// Mount options ACL's initrd uses for the `/etc` overlay.
+///
+/// Matching them keeps copy-up behaving the same whether the overlay was
+/// mounted by the initrd or by servicing.
+pub const ACL_ETC_OVERLAY_OPTIONS: &str = "redirect_dir=on,metacopy=off";

crates/trident/src/engine/ab_update.rs

@@ -85,11 +85,23 @@ pub(super) fn stage_update(
 
     engine::provision(subsystems, &ctx, newroot_mount.path())?;
 
+    // Give the chroot the same view of /etc the booted system will have, for
+    // images that assemble it at boot rather than shipping it.
+    let etc_overlay = engine::mount_etc_overlay(&ctx, newroot_mount.path())?;
+
     debug!("Entering '{}' chroot", newroot_mount.path().display());
     let result = chroot::enter_update_chroot(newroot_mount.path())
         .message("Failed to enter chroot")?
         .execute_and_exit(|| engine::configure(subsystems, &ctx));
 
+    if let Some(etc_overlay) = etc_overlay {
+        if let Err(e) = etc_overlay.unmount() {
+            // The newroot cannot be unmounted while the overlay holds it, so
+            // report rather than swallow.
+            warn!("Failed to unmount the /etc overlay: {e:?}");
+        }
+    }
+
     if let Err(original_error) = result {
         if let Err(e) = newroot_mount.unmount_all() {
             warn!("While handling an earlier error: {e:?}");

crates/trident/src/engine/clean_install.rs

@@ -227,11 +227,23 @@ fn stage_clean_install(
 
     engine::provision(subsystems, &ctx, newroot_mount.path())?;
 
+    // Give the chroot the same view of /etc the booted system will have, for
+    // images that assemble it at boot rather than shipping it.
+    let etc_overlay = engine::mount_etc_overlay(&ctx, newroot_mount.path())?;
+
     debug!("Entering '{}' chroot", newroot_mount.path().display());
     let result = chroot::enter_update_chroot(newroot_mount.path())
         .message("Failed to enter chroot")?
         .execute_and_exit(|| engine::configure(subsystems, &ctx));
 
+    if let Some(etc_overlay) = etc_overlay {
+        if let Err(e) = etc_overlay.unmount() {
+            // The newroot cannot be unmounted while the overlay holds it, so
+            // report rather than swallow.
+            warn!("Failed to unmount the /etc overlay: {e:?}");
+        }
+    }
+
     if let Some(mut monitor) = monitor {
         // If the monitor was created successfully, stop it after execution
         if let Err(e) = monitor.stop() {

crates/trident/src/engine/mod.rs

@@ -7,11 +7,12 @@ use std::{
 use chrono::Utc;
 use log::{debug, info, trace, warn};
 
-use osutils::path::join_relative;
+use osutils::{overlay::LayeredOverlay, path::join_relative};
+use sysdefs::acl::{ACL_ETC_LOWER_DIR, ACL_ETC_OVERLAY_OPTIONS, ACL_ETC_WORK_DIR};
 use trident_api::{
     config::Storage,
     constants,
-    error::{InternalError, TridentError, TridentResultExt},
+    error::{InternalError, ReportError, ServicingError, TridentError, TridentResultExt},
     is_default,
     status::{ServicingState, ServicingType},
     storage_graph::graph::StorageGraph,
@@ -325,6 +326,66 @@ fn prepare(subsystems: &mut [Box<dyn Subsystem>], ctx: &EngineContext) -> Result
     Ok(())
 }
 
+/// Directory the OS configuration lives in, relative to a mounted root.
+const ETC_PATH: &str = "/etc";
+
+/// Mounts ACL's `/etc` overlay over the new root, if the image needs it.
+///
+/// ACL ships no `/etc` on its root filesystem: its initrd assembles one at boot
+/// as an overlay, with the factory contents on the sealed `/usr` as the lower
+/// layer and the root filesystem's own `/etc` as the upper. Servicing runs
+/// before any of that has happened, so a chroot into the new root sees an empty
+/// `/etc`, and anything that rewrites a file the image provides -- the user
+/// database above all -- writes a replacement rather than an edit. At boot that
+/// replacement becomes the upper layer and hides the factory file.
+///
+/// Mounting the same overlay here makes the chroot see what the booted system
+/// will. Writes still land in the root filesystem's `/etc`, exactly as before,
+/// but a modified file is copied up from the factory version first, so it stays
+/// complete.
+///
+/// Returns `None` when the image does not use this layout, in which case the
+/// new root's `/etc` is used directly.
+pub(super) fn mount_etc_overlay(
+    ctx: &EngineContext,
+    newroot_path: &Path,
+) -> Result<Option<LayeredOverlay>, TridentError> {
+    if !ctx.image_distro().is_acl() {
+        return Ok(None);
+    }
+
+    let etc_path = join_relative(newroot_path, ETC_PATH);
+    let lower_dir = join_relative(newroot_path, ACL_ETC_LOWER_DIR);
+    if !lower_dir.exists() {
+        // An ACL image that does not ship the factory directory has nothing to
+        // overlay; leave /etc alone rather than inventing a layer.
+        debug!(
+            "Image reports as ACL but has no '{}', using '{}' directly",
+            lower_dir.display(),
+            etc_path.display()
+        );
+        return Ok(None);
+    }
+
+    let work_dir = join_relative(newroot_path, ACL_ETC_WORK_DIR);
+    debug!(
+        "Mounting ACL /etc overlay on '{}' with lower layer '{}'",
+        etc_path.display(),
+        lower_dir.display()
+    );
+
+    LayeredOverlay::mount(
+        &etc_path,
+        &lower_dir,
+        &etc_path,
+        &work_dir,
+        Some(ACL_ETC_OVERLAY_OPTIONS),
+    )
+    .structured(ServicingError::MountNewroot)
+    .message("Failed to mount the /etc overlay on the new root")
+    .map(Some)
+}
+
 fn provision(
     subsystems: &mut [Box<dyn Subsystem>],
     ctx: &EngineContext,