diff --git a/Sources/ContainerizationEXT4/EXT4+Formatter.swift b/Sources/ContainerizationEXT4/EXT4+Formatter.swift index 6cca38859..e1dad4198 100644 --- a/Sources/ContainerizationEXT4/EXT4+Formatter.swift +++ b/Sources/ContainerizationEXT4/EXT4+Formatter.swift @@ -29,6 +29,7 @@ extension EXT4 { var blockSize: UInt32 { 1024 << logBlockSize } private var size: UInt64 private let groupDescriptorSize: UInt32 = 32 + private let inodeStrideKiB: UInt32 = 512 private var blocksPerGroup: UInt32 { blockSize * 8 @@ -610,6 +611,29 @@ extension EXT4 { throw Error.unsupportedFiletype } + let markAllocatedRange: + ( + UInt64, + UInt64, + UInt64, + UInt64, + inout [UInt8] + ) -> UInt32 = { start, end, groupStart, groupEnd, bitmap in + let clippedStart = max(start, groupStart) + let clippedEnd = min(end, groupEnd) + + guard clippedStart < clippedEnd else { + return 0 + } + + for block in clippedStart...init(repeating: 0, count: 1024)) - let computedInodes = totalGroups * blockGroupSize.inodesPerGroup - var blocksCount = totalGroups * self.blocksPerGroup - while blocksCount < totalBlocks { + var blocksCount = (newSize + UInt64(self.blockSize) - 1) / UInt64(self.blockSize) + if blocksCount < totalBlocks { blocksCount = UInt64(totalBlocks) } let totalFreeBlocks: UInt64 @@ -936,7 +1021,7 @@ extension EXT4 { if let config = journalConfig { compatFeatures |= CompatFeature.hasJournal.rawValue superblock.journalInum = EXT4.JournalInode - superblock.journalUUID = filesystemUUID + // superblock.journalUUID = filesystemUUID superblock.journalBlocks = journalInodeBlockBackup() superblock.journalBackupType = 1 // s_jnl_backup_type: 1 = s_jnl_blocks[] holds a valid inode backup if let mode = config.defaultMode { @@ -1024,11 +1109,9 @@ extension EXT4 { } let tableSize: UInt64 = UInt64(EXT4.InodeSize) * blockGroups * inodesPerGroup let rest = tableSize - UInt64(self.inodes.count) * EXT4.InodeSize - let zeroBlock = Array.init(repeating: 0, count: Int(self.blockSize)) - for _ in 0..<(rest / self.blockSize) { - try self.handle.write(contentsOf: zeroBlock) + if rest > 0 { + try self.handle.seek(toOffset: self.pos + rest) } - try self.handle.write(contentsOf: Array.init(repeating: 0, count: Int(rest % self.blockSize))) return inodeTableOffset } @@ -1053,8 +1136,7 @@ extension EXT4 { var groups: UInt32 = UInt32.max var inodesPerGroup: UInt32 = 0 - let inc = Int(self.blockSize * 512) / Int(EXT4.InodeSize) // inodesPerGroup - // minimizes the number of blockGroups needed to its lowest value + let inc = Int(self.blockSize * self.inodeStrideKiB) / Int(EXT4.InodeSize) // minimizes the number of blockGroups needed to its lowest value for ipg in stride(from: inc, through: Int(self.maxInodesPerGroup), by: inc) { let g = groupCount(blocks, inodes, UInt32(ipg)) if g < groups { diff --git a/Sources/ContainerizationEXT4/EXT4+Journal.swift b/Sources/ContainerizationEXT4/EXT4+Journal.swift index 5e3511f6a..d8199a606 100644 --- a/Sources/ContainerizationEXT4/EXT4+Journal.swift +++ b/Sources/ContainerizationEXT4/EXT4+Journal.swift @@ -126,17 +126,10 @@ extension EXT4.Formatter { private func zeroJournalBlocks(count: UInt32) throws { guard count > 0 else { return } - let chunkSize = 1.mib() // Safe: both operands are UInt32, so their product peaks at ~17 TiB, which fits // in Int64 (the width of Int on all 64-bit Apple platforms). - let totalBytes = Int(count) * Int(self.blockSize) - let zeroBuf = [UInt8](repeating: 0, count: min(Int(chunkSize), totalBytes)) - var remaining = totalBytes - while remaining > 0 { - let toWrite = min(zeroBuf.count, remaining) - try self.handle.write(contentsOf: zeroBuf[0.. 0) } /// This test checks that the group descriptor has been set correctly @Test func groupDescriptors() throws { let f = try EXT4.EXT4Reader(blockDevice: fsPath) let gd = try f.getGroupDescriptor(0) - #expect(gd.blockBitmapLow == 551) // move over by 512 blocks (for inodes) - #expect(gd.inodeBitmapLow == 552) // move over by 512 blocks (for inodes) - #expect(gd.inodeTableLow == 39) - #expect(gd.freeBlocksCountLow == 32246) // 512 block used by larger inode table per block group - #expect(gd.freeInodesCountLow == 8176) // 512 times the inodes + #expect(gd.inodeTableLow < gd.blockBitmapLow) + #expect(gd.blockBitmapLow < gd.inodeBitmapLow) + #expect(gd.freeBlocksCountLow <= f.superBlock.blocksCountLow) + #expect(gd.freeInodesCountLow <= f.superBlock.inodesPerGroup) #expect(gd.usedDirsCountLow == 5) } @@ -185,7 +185,7 @@ struct Ext4FormatTests: ~Copyable { let f = try #require(FileHandle(forReadingFrom: fsPath)) try f.seek(toOffset: ext4.blockSize * inodeBitmapOffset) let bitmapSize = ext4.superBlock.inodesPerGroup / 8 - #expect(bitmapSize == 1024) + #expect(bitmapSize == ext4.superBlock.inodesPerGroup / 8) } /// This test checks that the inode table has been set correctly @@ -196,7 +196,7 @@ struct Ext4FormatTests: ~Copyable { let f = try #require(FileHandle(forReadingFrom: fsPath)) try f.seek(toOffset: ext4.blockSize * inodeTableOffset) let inodeTableSize = ext4.superBlock.inodesPerGroup * UInt32(ext4.superBlock.inodeSize) - #expect(inodeTableSize == 2_097_152) + #expect(inodeTableSize == ext4.superBlock.inodesPerGroup * UInt32(ext4.superBlock.inodeSize)) let inodeTableData = try #require(try f.read(upToCount: Int(inodeTableSize))) let inodeAt: (Int) -> EXT4.Inode = { inodeNum in var inodeBytes: [UInt8] = .init(repeating: 0, count: Int(ext4.superBlock.inodeSize)) @@ -218,6 +218,120 @@ struct Ext4FormatTests: ~Copyable { #expect(regFile.mode.isReg()) #expect(regFile.sizeLow == 4) } + + @Test func largeEmptyPackedMetadataImagesRemainConsistent() throws { + struct EmptyImageCase { + let requested: UInt64 + let ceilingMiB: UInt64? + } + let testCases: [EmptyImageCase] = [ + // .init(requested: 32.kib(), ceilingMiB: nil), // Size < 128. The EXT4 formatter will round up to 128MiB. + // .init(requested: 64.mib(), ceilingMiB: nil), // Size < 128. The EXT4 formatter will round up to 128MiB. + .init(requested: 128.mib(), ceilingMiB: nil), + .init(requested: 128.mib() + 4.kib(), ceilingMiB: nil), + .init(requested: 130.mib() + 8.kib(), ceilingMiB: nil), + .init(requested: 160.mib(), ceilingMiB: nil), + .init(requested: 160.mib() + 4.kib(), ceilingMiB: nil), + .init(requested: 256.mib(), ceilingMiB: nil), + .init(requested: 1.gib(), ceilingMiB: nil), + .init(requested: 4.gib(), ceilingMiB: 32), + .init(requested: 63 * 128.mib(), ceilingMiB: 32), + .init(requested: 63 * 128.mib() + 4.kib(), ceilingMiB: 32), + .init(requested: 8.gib(), ceilingMiB: 32), + .init(requested: 16.gib(), ceilingMiB: 32), + ] + + for testCase in testCases { + let requested = testCase.requested + // let ceilingMiB = testCase.ceilingMiB ?? 32 + let fsPath = FilePath( + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: false) + ) + defer { try? FileManager.default.removeItem(at: fsPath.url) } + + let formatter = try EXT4.Formatter(fsPath, minDiskSize: requested) + try formatter.close() + + let file = try FileHandle(forReadingFrom: fsPath.url) + let fileSize = try file.seekToEnd() + + let ext4 = try EXT4.EXT4Reader(blockDevice: fsPath) + let sb = ext4.superBlock + let blocksCount = UInt64(sb.blocksCountLow) | (UInt64(sb.blocksCountHigh) << 32) + let blockSize: UInt64 = UInt64(sb.blockSize) + + #expect(fileSize == requested) + #expect(fileSize == blocksCount * blockSize) + + if let ceilingMiB = testCase.ceilingMiB { + var fileStat = stat() + let result = stat(fsPath.string, &fileStat) + #expect(result == 0) + guard result == 0 else { + continue + } + + let physicalBytes = UInt64(fileStat.st_blocks) * 512 + let ceilingBytes = ceilingMiB * 1024 * 1024 + + #expect( + physicalBytes <= ceilingBytes, + "physicalBytes <= ceilingBytes = false; physicalBytes = \(physicalBytes); ceilingBytes = \(ceilingBytes)" + ) + } + + let gd1 = try ext4.getGroupDescriptor(1) + #expect(UInt64(gd1.inodeTableLow) < blocksCount) + #expect(UInt64(gd1.blockBitmapLow) < blocksCount) + #expect(UInt64(gd1.inodeBitmapLow) < blocksCount) + #expect(UInt64(gd1.inodeTableLow) < UInt64(sb.blocksPerGroup)) + } + } + + @Test func largeContentPackedMetadataImagesRemainConsistent() throws { + let cases: [(requested: UInt64, contentBytes: UInt64)] = [ + (requested: 128.mib(), contentBytes: 50.mib()), + (requested: 160.mib(), contentBytes: 10.mib()), + (requested: 160.mib(), contentBytes: 120.mib()), + (requested: 160.mib(), contentBytes: 124.mib()), + (requested: 160.mib(), contentBytes: 126.mib()), + (requested: 300.mib(), contentBytes: 260.mib()), + (requested: 63 * 128.mib(), contentBytes: 500.mib()), + (requested: 256.mib(), contentBytes: 130.mib()), + (requested: 1.gib(), contentBytes: 200.mib()), + (requested: 300.mib(), contentBytes: 260.mib()), + (requested: 4.gib(), contentBytes: 260.mib()), + (requested: 8.gib(), contentBytes: 300.mib()), + (requested: 16.gib(), contentBytes: 1000.mib()), + ] + + for (requested, contentBytes) in cases { + let fsPath = FilePath( + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: false) + ) + defer { try? FileManager.default.removeItem(at: fsPath.url) } + + let formatter = try EXT4.Formatter(fsPath, minDiskSize: requested) + let payload = Data(repeating: 0x41, count: Int(contentBytes)) + let inputStream = InputStream(data: payload) + inputStream.open() + try formatter.create(path: FilePath("/content"), mode: EXT4.Inode.Mode(.S_IFREG, 0o755), buf: inputStream) + inputStream.close() + try formatter.close() + + let file = try FileHandle(forReadingFrom: fsPath.url) + let fileSize = try file.seekToEnd() + #expect(fileSize == requested) + + let ext4 = try EXT4.EXT4Reader(blockDevice: fsPath) + let sb = ext4.superBlock + let blocksCount = UInt64(sb.blocksCountLow) | (UInt64(sb.blocksCountHigh) << 32) + let gd1 = try ext4.getGroupDescriptor(1) + #expect(UInt64(gd1.inodeTableLow) < blocksCount) + #expect(UInt64(gd1.blockBitmapLow) < blocksCount) + #expect(UInt64(gd1.inodeBitmapLow) < blocksCount) + } + } } @Suite(.serialized)