refactor: move arc stats call to a separate function to be less obtrusive

Signed-off-by: Hudson Gerwing <[email protected]>

Author: Hudson Gerwing

psutil/__init__.py

@@ -1961,7 +1961,7 @@ def cpu_freq(percpu=False):
 # =====================================================================
 
 
-def virtual_memory(include_zfs_arc=False):
+def virtual_memory():
     """Return statistics about system memory usage as a namedtuple
     including the following fields, expressed in bytes:
 
@@ -2014,7 +2014,7 @@ def virtual_memory(include_zfs_arc=False):
     On Windows 'available' and 'free' are the same.
     """
     global _TOTAL_PHYMEM
-    ret = _psplatform.virtual_memory(include_zfs_arc)
+    ret = _psplatform.virtual_memory()
     # cached for later use in Process.memory_percent()
     _TOTAL_PHYMEM = ret.total
     return ret
@@ -2036,6 +2036,14 @@ def swap_memory():
     return _psplatform.swap_memory()
 
 
+def apply_zfs_arcstats(vm_stats):
+    """Apply ZFS ARC stats to virtual memory stats."""
+    # Only applicable to linux distros
+    if LINUX:
+        return _psplatform.apply_zfs_arcstats(vm_stats)
+    raise NotImplementedError("ZFS ARC stats are only available on Linux")
+
+
 # =====================================================================
 # --- disks/paritions related functions
 # =====================================================================

psutil/_pslinux.py

@@ -171,10 +171,6 @@ class IOPriority(enum.IntEnum):
 svmem = namedtuple(
     'svmem', ['total', 'available', 'percent', 'used', 'free',
               'active', 'inactive', 'buffers', 'cached', 'shared', 'slab'])
-# psutil.zfs_arc_stats()
-szfsarc = namedtuple(
-    'szfsarc', ['enabled', 'min', 'max', 'compressed', 'uncompressed',
-                'size', 'header', 'anon', 'mfu', 'mru', 'other'])
 # psutil.disk_io_counters()
 sdiskio = namedtuple(
     'sdiskio', ['read_count', 'write_count',
@@ -419,15 +415,13 @@ def calculate_avail_vmem(mems):
     return int(avail)
 
 
-def virtual_memory(include_zfs_arc: bool = False):
+def virtual_memory():
     """Report virtual memory stats.
     This implementation mimics procps-ng-3.3.12, aka "free" CLI tool:
     https://gitlab.com/procps-ng/procps/blob/
         24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L778-791
     The returned values are supposed to match both "free" and "vmstat -s"
     CLI tools.
-    If specifying the `include_zfs_arc` parameter, the implementation mimics
-    "htop" CLI tool instead of "free" CLI tool.
     """
     missing_fields = []
     mems = {}
@@ -529,29 +523,7 @@ def virtual_memory(include_zfs_arc: bool = False):
         #     24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L764
         avail = free
 
-    # ZFS ARC memory consumption is not reported by /proc/meminfo.
-    # Specifying `include_zfs_arc` will include reclaimable ZFS ARC
-    # memory in the returned values.
-    # N.B. this will make psutil match the output of "htop" instead
-    # of "free" CLI tool.
-    # See:
-    # https://www.reddit.com/r/zfs/comments/ha0p7f/understanding_arcstat_and_free/
-    # https://github.com/openzfs/zfs/issues/10255
-    if include_zfs_arc:
-        zfs = zfs_arc_stats()
-
-        # When accounting for zfs memory, we need to keep track of "shared"
-        # So "used" memory is more relevant than "available"
-        used += shared
-        cached -= shared
-        if zfs.enabled:
-            shrinkable_size = max(zfs.size - zfs.min, 0)
-            used -= shrinkable_size
-            cached += shrinkable_size
-            avail += shrinkable_size
-        percent = usage_percent(used, total, round_=1)
-    else:
-        percent = usage_percent((total - avail), total, round_=1)
+    percent = usage_percent((total - avail), total, round_=1)
 
     # Warn about missing metrics which are set to 0.
     if missing_fields:
@@ -631,19 +603,11 @@ def swap_memory():
     return _common.sswap(total, used, free, percent, sin, sout)
 
 
-def zfs_arc_stats():
-    """Return ZFS ARC (Adaptive Replacement Cache) stats."""
-    missing_fields = []
+def apply_zfs_arcstats(vm_stats: svmem):
+    """Apply ZFS ARC (Adaptive Replacement Cache) stats to
+    input virtual memory call results"""
     mems = {}
 
-    def get_if_available(key):
-        try:
-            return mems[key]
-        except KeyError:
-            missing_fields.append(key)
-            return 0
-
-
     with open_binary('%s/spl/kstat/zfs/arcstats' % get_procfs_path()) as f:
         for line in f:
             fields = line.split()
@@ -653,44 +617,35 @@ def get_if_available(key):
                 # Not a key: value line
                 continue
 
-    zfs_min = get_if_available(b'c_min')
-    zfs_max = get_if_available(b'c_max')
-    zfs_compressed = get_if_available(b'compressed_size')
-    zfs_uncompressed = get_if_available(b'uncompressed_size')
-    zfs_size = get_if_available(b'size')
-    zfs_header = get_if_available(b'hdr_size')
-    dbuf_size = get_if_available(b'dbuf_size')
-    dnode_size = get_if_available(b'dnode_size')
-    bonus_size = get_if_available(b'bonus_size')
-    zfs_anon = get_if_available(b'anon_size')
-    zfs_mfu = get_if_available(b'mfu_size')
-    zfs_mru = get_if_available(b'mru_size')
-
-    zfs_enabled = zfs_size > 0
-    zfs_other = dbuf_size + dnode_size + bonus_size if all([dbuf_size, dnode_size, bonus_size]) else 0
-
-    # Warn about missing metrics which are set to 0.
-    if missing_fields:
-        msg = "%s memory stats couldn't be determined and %s set to 0" % (
-            ", ".join(missing_fields),
-            "was" if len(missing_fields) == 1 else "were",
-        )
+    try:
+        zfs_min = mems[b'c_min']
+        zfs_size = mems[b'size']
+    except KeyError:
+        msg = ("ZFS ARC memory stats couldn't be determined, "
+               "no modification made to virtual memory stats")
         warnings.warn(msg, RuntimeWarning, stacklevel=2)
+        zfs_min = zfs_size = 0
 
-    return szfsarc(
-        zfs_enabled,
-        zfs_min,
-        zfs_max,
-        zfs_compressed,
-        zfs_uncompressed,
-        zfs_size,
-        zfs_header,
-        zfs_anon,
-        zfs_mfu,
-        zfs_mru,
-        zfs_other
-    )
+    # ZFS ARC memory consumption is not reported by /proc/meminfo.
+    # Running this func will include reclaimable ZFS ARC
+    # memory in the returned values.
+    # N.B. this will make psutil match the output of "htop" instead
+    # of "free" CLI tool.
+    # See:
+    # https://www.reddit.com/r/zfs/comments/ha0p7f/understanding_arcstat_and_free/
+    # https://github.com/openzfs/zfs/issues/10255
 
+    # When accounting for zfs memory, we need to keep track of "shared"
+    # So "used" memory is more relevant than "available"
+    vm_stats.used += vm_stats.shared
+    vm_stats.cached -= vm_stats.shared
+    shrinkable_size = max(zfs_size - zfs_min, 0)
+    vm_stats.used -= shrinkable_size
+    vm_stats.cached += shrinkable_size
+    vm_stats.available += shrinkable_size
+    vm_stats.percent = usage_percent(vm_stats.used, vm_stats.total, round_=1)
+
+    return vm_stats
 
 
 # =====================================================================