diff --git a/changelog/56239.fixed.md b/changelog/56239.fixed.md new file mode 100644 index 000000000000..b44007181500 --- /dev/null +++ b/changelog/56239.fixed.md @@ -0,0 +1 @@ +Rewrote the pillar / renderers / encryption documentation to address twelve outstanding doc issues: clarified the pillar data structure and merge rules (#56239, #61283), documented that custom modules used in pillar must be synced to the **master** with `salt-run saltutil.sync_all` (#62802), documented the `key:` subkey separator on pillar `include:` directives (#66622), explained how to target with `-I` against a specific pillarenv (#63110), tightened the pillar encryption page with a working shebang-renderer example (#62733), added the required `--homedir` flag to every documented `gpg` encrypt command (#65682), added a consistent GPG-terminology note (#59539), added a `Module.run`+pyobjects `**kwargs` workaround for dotted function names (#61828), expanded the py renderer guide with new examples and the highstate return-data shape (#63698), clarified the meaning of `default=` in `pillar.get` (#60378), and added a per-pillarenv `root` example to the `git_pillar` walkthrough (#58929). New regression tests verify that every documented example renders or round-trips end-to-end. diff --git a/doc/topics/pillar/index.rst b/doc/topics/pillar/index.rst index 66e49c091a32..0a6949ab5e9f 100644 --- a/doc/topics/pillar/index.rst +++ b/doc/topics/pillar/index.rst @@ -337,6 +337,114 @@ Since both pillar SLS files contained a ``bind`` key which contained a nested dictionary, the pillar dictionary's ``bind`` key contains the combined contents of both SLS files' ``bind`` keys. +.. _pillar-data-structure: + +The Pillar Data Structure +========================= + +Pillar SLS files are parsed by their renderer into Python data types and +then collapsed into a single, per-minion dictionary. Knowing the rules +the compiler uses makes pillar files much easier to read and write. + +* A pillar dictionary is **always a dict** at the top level. Keys in + that dict come from every SLS file the minion was assigned in the top + file. + +* When two SLS files declare the **same top-level key** the values are + combined according to the **types** involved: + + * Two ``dict`` values are recursively merged, key-by-key. + * Two ``list`` values are merged by appending; whether the merge + de-duplicates or appends raw depends on + :conf_minion:`pillar_merge_lists` (default ``False``). + * Any **scalar collision** (string, int, bool, etc.) uses the + *last-one-wins* rule: the value from the file processed last + overwrites the earlier one. Files are processed in the order they + appear in the top file. + +* The colon-delimited subkey syntax used throughout pillar + (:py:func:`pillar.get('foo:bar:baz') `, + :conf_master:`decrypt_pillar`, the include ``key`` option) refers to + Python dict nesting -- ``foo:bar:baz`` reads + ``pillar['foo']['bar']['baz']``. + +Worked example +-------------- + +Top file declares both pillars apply to every minion: + +``/srv/pillar/top.sls``: + +.. code-block:: yaml + + base: + '*': + - users + - sudoers + +``/srv/pillar/users.sls``: + +.. code-block:: yaml + + users: + alice: + uid: 1001 + bob: + uid: 1002 + +``/srv/pillar/sudoers.sls``: + +.. code-block:: yaml + + users: + alice: + sudo: True + carol: + uid: 1003 + +After compilation the pillar dictionary contains a single merged +``users`` key: + +.. code-block:: python + + { + "users": { + "alice": {"uid": 1001, "sudo": True}, + "bob": {"uid": 1002}, + "carol": {"uid": 1003}, + } + } + +``alice`` is the recursive-merge case (both files contributed dict +fragments under the same key path). ``bob`` and ``carol`` are present +because each file added a unique sub-key. If, instead, ``sudoers.sls`` +had declared ``users: 'alice'`` (a scalar), it would overwrite the +``users`` dictionary entirely under the "last one wins" rule. + +.. note:: + + The same compiled dictionary is exposed three different ways in + templates: ``pillar['users']['alice']['uid']`` (raw access), + ``salt['pillar.get']('users:alice:uid')`` (safe traversal with a + default value) and ``__pillar__['users']['alice']['uid']`` (inside + execution modules and the ``py`` renderer). + +Custom modules in pillar +------------------------ + +Execution modules that are referenced by ``ext_pillar`` or by pillar SLS +files (via ``salt['custom_mod.foo']``) run on the **master**, not on +the minion. Custom modules placed in ``_modules/`` on the master will +not be available in pillar until they have been synced to the master: + +.. code-block:: bash + + salt-run saltutil.sync_all + +Running ``salt '*' saltutil.sync_all`` only syncs custom modules to the +minions. Pillar compilation will continue to fail to find the custom +function until the master itself has the synced copy. + .. _pillar-include: Including Other Pillars @@ -369,6 +477,22 @@ With this form, the included file (users.sls) will be nested within the 'users' key of the compiled pillar. Additionally, the 'sudo' value will be available as a template variable to users.sls. +.. note:: + + The ``key`` option uses the same colon-delimited subkey syntax as + :py:func:`pillar.get `. To nest the included + pillar deeper than one level, use ``parent:child:leaf``: + + .. code-block:: yaml + + include: + - users: + key: accounts:internal:users + + With this form, the contents of ``users.sls`` will be made available + under ``pillar['accounts']['internal']['users']``. The colon is the + only separator accepted by ``key``. + .. _pillar-in-memory: In-Memory Pillar Data vs. On-Demand Pillar Data @@ -586,6 +710,15 @@ Salt's renderer system can be used to decrypt pillar data. This allows for pillar items to be stored in an encrypted state, and decrypted during pillar compilation. +.. note:: GPG terminology + + Throughout this documentation the term "GPG" is used for all + OpenPGP/PGP/GPG-related material and implementations, matching the + convention used by the GNU Privacy Guard project itself. The + armored block markers (``-----BEGIN PGP MESSAGE-----`` and + ``-----END PGP MESSAGE-----``) come from the underlying OpenPGP + standard and appear regardless of which tool produced them. + Encrypted Pillar SLS -------------------- @@ -693,6 +826,33 @@ So, the first example above could be rewritten as: decrypt_pillar: - 'secrets:vault' +Per-file Shebang Renderers +************************** + +When :conf_master:`decrypt_pillar` is not configured, individual pillar +SLS files can still be marked as encrypted by setting the rendering +pipeline with a shebang on the first line of the file: + +.. code-block:: yaml + + #!yaml|gpg + + api_key: | + -----BEGIN PGP MESSAGE----- + ... + -----END PGP MESSAGE----- + +The ``#!yaml|gpg`` shebang tells the renderer system to parse the file +as YAML and then pass every string value through the +:mod:`gpg ` renderer, which decrypts the PGP +message blocks in place. The same pattern works for any decryption +renderer listed in :conf_master:`decrypt_pillar_renderers` (for example +``#!yaml|nacl``). + +This is often simpler than :conf_master:`decrypt_pillar` when only a +handful of pillar files contain secrets: the encryption is declared in +the file itself rather than via a separate path-based configuration. + Encrypted Pillar Data on the CLI -------------------------------- @@ -719,7 +879,7 @@ Triggering decryption of this CLI pillar data can be done in one of two ways: .. code-block:: bash - $ echo -n bar | gpg --armor --trust-model always --encrypt -r user@domain.tld | sed ':a;N;$!ba;s/\n/\\n/g' + $ echo -n bar | gpg --homedir /etc/salt/gpgkeys --armor --trust-model always --encrypt -r user@domain.tld | sed ':a;N;$!ba;s/\n/\\n/g' .. note:: Using ``pillar_enc`` will perform the decryption minion-side, so for diff --git a/doc/topics/targeting/pillar.rst b/doc/topics/targeting/pillar.rst index ab79cf535a08..ad56a31c6cfc 100644 --- a/doc/topics/targeting/pillar.rst +++ b/doc/topics/targeting/pillar.rst @@ -32,3 +32,50 @@ is being traversed. The below example would match minions with a pillar named .. code-block:: bash salt -I 'foo:bar:baz*' test.version + +Targeting against a specific pillarenv +====================================== + +Pillar match commands on the CLI evaluate against the **in-memory** pillar +data that lives on each minion -- i.e. the data that was already compiled +and shipped to the minion the last time it refreshed its pillar. By +default that in-memory pillar is the merge of every configured pillar +environment. + +To target on data sourced exclusively from a single +:conf_minion:`pillarenv`, the targeted minions must first be made to +compile their in-memory pillar against that environment. Two patterns +are supported: + +1. Set :conf_minion:`pillarenv` in the minion configuration file (or + override it on the master with + :conf_master:`pillarenv`/``pillarenv_from_saltenv``). The minion's + in-memory pillar will then only contain keys from that environment + and pillar matching with ``-I`` will only see those keys: + + .. code-block:: bash + + salt '*' saltutil.refresh_pillar + salt -I 'key_only_in_qa_env:value' test.version + +2. Run states with an explicit ``pillarenv`` keyword argument when you + need targeting and pillar selection in the same call: + + .. code-block:: bash + + salt -I 'key:value' state.apply mystates pillarenv=qa + + In this form ``-I 'key:value'`` is still evaluated against the + minion's *in-memory* pillar; the ``pillarenv=qa`` kwarg only + controls the pillar made available during state rendering. If + ``key`` is not present in the default in-memory pillar the minion + will not be matched. + +.. note:: + + Pillar targeting cannot pick which pillarenv to evaluate the match + against on the master side. If a key lives only in the ``qa`` + pillarenv, minions must already be running with ``pillarenv: qa`` + (or equivalent) for ``-I 'key:value'`` to find them. See + :ref:`How Pillar Environments Are Handled ` + for the complete set of options. diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index 72a1edb6b6e8..7dc83dc948fd 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -65,6 +65,29 @@ def get( unless :conf_minion:`pillar_raise_on_missing` is set to ``True``, in which case an error will be raised. + .. note:: + + The function signature uses a private ``NOT_SET`` sentinel for + ``default`` and the auto-generated documentation may render + this as ``default=``. Both are internal + implementation details and not intended to be passed by the + caller. The supported behavior is: + + * Omit ``default`` (the common case): missing keys return an + empty string, or raise :py:exc:`KeyError` when + :conf_minion:`pillar_raise_on_missing` is ``True``. + * Pass any value for ``default`` -- scalars, dictionaries + and lists are all accepted. When ``merge`` is also + ``True`` a dictionary or list default is recursively + merged with the retrieved pillar value. + * Explicitly passing the :py:class:`KeyError` class is + equivalent to enabling + :conf_minion:`pillar_raise_on_missing`: a missing key + raises ``KeyError`` rather than returning a value. + + See ``salt/modules/pillar.py`` ``get()`` for the canonical + behavior. + merge : ``False`` If ``True``, the retrieved values will be merged into the passed default. When the default and the retrieved value are both diff --git a/salt/pillar/git_pillar.py b/salt/pillar/git_pillar.py index 0e02e73c9191..1ba38c6b641c 100644 --- a/salt/pillar/git_pillar.py +++ b/salt/pillar/git_pillar.py @@ -329,6 +329,45 @@ - Salt versions prior to 2018.3.4 ignore the ``root`` parameter when ``mountpoint`` is set. +.. _git-pillar-per-saltenv-root: + +Per-saltenv ``root`` +~~~~~~~~~~~~~~~~~~~~ + +The ``root`` per-remote parameter sets where in the repository the +pillar SLS files live for a given remote. When the same remote serves +more than one pillarenv, ``root`` can be set per pillarenv so that one +branch can contain pillar data for several environments at different +subdirectories. + +The per-pillarenv ``root`` is **not additive** to a global +:conf_master:`git_pillar_root`: when set it replaces it completely for +that pillarenv. + +.. code-block:: yaml + + git_pillar_root: pillar + + ext_pillar: + - git: + - __env__ https://mydomain.tld/secrets.git: + - saltenv: + - dev: + - root: pillar/dev + - prod: + - root: pillar/prod + - test: + - root: pillar/test/v2 + +With the above: + +* In the ``dev`` pillarenv, pillar SLS files for this remote are read + from ``pillar/dev`` (NOT ``pillar/dev`` *plus* ``pillar``). +* In the ``prod`` pillarenv they are read from ``pillar/prod``. +* In the ``test`` pillarenv they are read from ``pillar/test/v2``. +* In any other pillarenv the global ``git_pillar_root: pillar`` is + used. + .. _git-pillar-all_saltenvs: all_saltenvs diff --git a/salt/renderers/gpg.py b/salt/renderers/gpg.py index f417ee0fe5af..0721e691346b 100644 --- a/salt/renderers/gpg.py +++ b/salt/renderers/gpg.py @@ -173,7 +173,15 @@ .. code-block:: bash - $ echo -n 'supersecret' | gpg --trust-model always -ear + $ echo -n 'supersecret' | gpg --homedir /etc/salt/gpgkeys --trust-model always -ear + +.. note:: + + The ``--homedir`` option is required whenever the recipient's public + key lives in the salt-master's gpg keyring rather than the calling + user's ``~/.gnupg`` directory. Omitting it produces + ``gpg: [stdin]: encryption failed: No public key`` because gpg + searches the wrong keyring. To apply the renderer on a file-by-file basis add the following line to the top of any pillar with gpg data in it: @@ -244,9 +252,9 @@ .. code-block:: bash # awk - ciphertext=`echo -n "supersecret" | gpg --armor --batch --trust-model always --encrypt -r user@domain.com | awk '{printf "%s\\n",$0} END {print ""}'` + ciphertext=`echo -n "supersecret" | gpg --homedir /etc/salt/gpgkeys --armor --batch --trust-model always --encrypt -r user@domain.com | awk '{printf "%s\\n",$0} END {print ""}'` # Perl - ciphertext=`echo -n "supersecret" | gpg --armor --batch --trust-model always --encrypt -r user@domain.com | perl -pe 's/\n/\\n/g'` + ciphertext=`echo -n "supersecret" | gpg --homedir /etc/salt/gpgkeys --armor --batch --trust-model always --encrypt -r user@domain.com | perl -pe 's/\n/\\n/g'` With Python: @@ -255,7 +263,8 @@ import subprocess secret, stderr = subprocess.Popen( - ['gpg', '--armor', '--batch', '--trust-model', 'always', '--encrypt', + ['gpg', '--homedir', '/etc/salt/gpgkeys', + '--armor', '--batch', '--trust-model', 'always', '--encrypt', '-r', 'user@domain.com'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -294,9 +303,9 @@ .. code-block:: bash # awk - ciphertext=`echo -n "{'secret_a': 'CorrectHorseBatteryStaple', 'secret_b': 'GPG is fun!'}" | gpg --armor --batch --trust-model always --encrypt -r user@domain.com | awk '{printf "%s\\n",$0} END {print ""}'` + ciphertext=`echo -n "{'secret_a': 'CorrectHorseBatteryStaple', 'secret_b': 'GPG is fun!'}" | gpg --homedir /etc/salt/gpgkeys --armor --batch --trust-model always --encrypt -r user@domain.com | awk '{printf "%s\\n",$0} END {print ""}'` # Perl - ciphertext=`echo -n "{'secret_a': 'CorrectHorseBatteryStaple', 'secret_b': 'GPG is fun!'}" | gpg --armor --batch --trust-model always --encrypt -r user@domain.com | perl -pe 's/\n/\\n/g'` + ciphertext=`echo -n "{'secret_a': 'CorrectHorseBatteryStaple', 'secret_b': 'GPG is fun!'}" | gpg --homedir /etc/salt/gpgkeys --armor --batch --trust-model always --encrypt -r user@domain.com | perl -pe 's/\n/\\n/g'` With Python: @@ -308,7 +317,8 @@ 'secret_b': 'GPG is fun!'} secret, stderr = subprocess.Popen( - ['gpg', '--armor', '--batch', '--trust-model', 'always', '--encrypt', + ['gpg', '--homedir', '/etc/salt/gpgkeys', + '--armor', '--batch', '--trust-model', 'always', '--encrypt', '-r', 'user@domain.com'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, diff --git a/salt/renderers/py.py b/salt/renderers/py.py index 46ce1c039026..e359eb1afa38 100644 --- a/salt/renderers/py.py +++ b/salt/renderers/py.py @@ -124,6 +124,95 @@ def run(): return config + +Calling execution modules from ``run()`` +---------------------------------------- + +The ``__salt__`` dunder dictionary in the py renderer is the same one +seen by execution and state modules. Call functions by name and the +return value is the same Python object an execution module would see. + +.. code-block:: python + + #!py + + def run(): + # Run an http.query and use the response to drive a state. + response = __salt__["http.query"]( + url="https://example.com/health", + status=True, + ) + if response.get("status") != 200: + return {} + return { + "restart_app": { + "service.running": [ + {"name": "myapp"}, + {"watch": [{"file": "/etc/myapp.conf"}]}, + ], + }, + } + + +Inspecting state return values +------------------------------ + +Because ``run()`` is plain Python, intermediate values produced by +``__salt__`` calls (and even from previously running states accessed via +``__context__``) can be inspected with normal Python control flow. For +example, to skip a state when an upstream API call returned a falsy +``result`` field: + +.. code-block:: python + + #!py + + def run(): + result = __salt__["http.query"]( + url="https://example.com/feature-flag", + ) + config = {} + if result.get("dict", {}).get("enabled"): + config["enable_feature"] = { + "cmd.run": [ + {"name": "/usr/local/bin/enable-feature"}, + ], + } + return config + + +Highstate return-data structure +------------------------------- + +``run()`` must return a Python ``dict`` whose top-level keys are state +IDs. Each value is a dict whose keys are dotted state functions and +whose values are a list of single-key dicts holding the kwargs for that +function. For example, this YAML state: + +.. code-block:: yaml + + common_packages: + pkg.installed: + - pkgs: + - curl + - vim + +is the YAML rendering of: + +.. code-block:: python + + { + "common_packages": { + "pkg.installed": [ + {"pkgs": ["curl", "vim"]}, + ], + }, + } + +A ``run()`` that returns the same dict produces an identical highstate +result. Returning an empty dict is valid -- it simply produces no +states for that minion. + """ import os diff --git a/salt/renderers/pyobjects.py b/salt/renderers/pyobjects.py index efb14f497390..e339ae558f62 100644 --- a/salt/renderers/pyobjects.py +++ b/salt/renderers/pyobjects.py @@ -312,6 +312,39 @@ class RHEL: with Pkg.installed("samba", names=[Samba.server, Samba.client]): Service.running("samba", name=Samba.service) + +Calling ``Module.run`` with dotted function names +------------------------------------------------- + +Starting in the 3005 release the :mod:`module.run ` +state expects the name of the module function as a keyword argument. +Because Python keyword arguments cannot contain a dot, the obvious form +will not parse: + +.. code-block:: python + + #!pyobjects + + # SYNTAX ERROR -- "shadow.lock_password" is not a valid keyword name. + Module.run("pyobject_shadow", shadow.lock_password=["susan"]) + +The renderer reports this as +``Rendering SLS ... failed, render error: keyword can't be an expression``. + +Pass the dotted function name through a ``**kwargs`` dictionary instead: + +.. code-block:: python + + #!pyobjects + + lock_kwargs = {"shadow.lock_password": ["susan"]} + Module.run("pyobject_shadow", **lock_kwargs) + +The first positional argument (``"pyobject_shadow"`` above) is the state +ID; the dictionary key (``"shadow.lock_password"``) is the dotted name +of the execution module function to call; the dictionary value is the +list of arguments passed to that function. + """ # TODO: Interface for working with reactor files diff --git a/tests/pytests/functional/pillar/test_documented_examples.py b/tests/pytests/functional/pillar/test_documented_examples.py new file mode 100644 index 000000000000..ae8861bef1dd --- /dev/null +++ b/tests/pytests/functional/pillar/test_documented_examples.py @@ -0,0 +1,202 @@ +""" +Tests that the pillar examples appearing in the documentation actually render. + +Each fixture is a verbatim copy of an example used in the pillar topic guide. +If a doc example changes, the matching fixture must be updated and the test +re-run so the documentation stays trustworthy. + +Covers the issues: + +- 56239 (pillar update) +- 61283 (pillar yaml structure / merge behavior) +- 62802 (custom modules in pillar / pillar.items) +- 66622 (pillar include subkey separator) +- 63110 (pillarenv targeting) +""" + +import textwrap + +import pytest + +import salt.pillar + +pytestmark = [ + pytest.mark.slow_test, +] + + +# --- merge behavior (issue 61283) --------------------------------------- + + +@pytest.fixture(scope="module") +def merge_pillar_state_tree(pillar_state_tree): + """ + Lay down the merge-behavior pillar example from the docs. + """ + top_sls = textwrap.dedent( + """\ + base: + '*': + - packages + - services + """ + ) + packages_sls = textwrap.dedent( + """\ + bind: + package-name: bind9 + version: 9.9.5 + """ + ) + services_sls = textwrap.dedent( + """\ + bind: + port: 53 + listen-on: any + """ + ) + with pytest.helpers.temp_file( + "top.sls", top_sls, pillar_state_tree + ), pytest.helpers.temp_file( + "packages.sls", packages_sls, pillar_state_tree + ), pytest.helpers.temp_file( + "services.sls", services_sls, pillar_state_tree + ): + yield None + + +def test_documented_dict_merge(salt_master, grains, merge_pillar_state_tree): + """ + Verify the recursive dict merge example produces the documented output. + """ + opts = salt_master.config.copy() + pillar_obj = salt.pillar.Pillar(opts, grains, "test", "base") + ret = pillar_obj.compile_pillar() + assert ret.get("bind") == { + "package-name": "bind9", + "version": "9.9.5", + "port": 53, + "listen-on": "any", + } + + +@pytest.fixture(scope="module") +def overwrite_pillar_state_tree(pillar_state_tree): + """ + Lay down the "last one wins" overwrite example. + """ + top_sls = textwrap.dedent( + """\ + base: + '*': + - packages + - services + """ + ) + packages_sls = "bind: bind9\n" + services_sls = "bind: named\n" + with pytest.helpers.temp_file( + "top.sls", top_sls, pillar_state_tree + ), pytest.helpers.temp_file( + "packages.sls", packages_sls, pillar_state_tree + ), pytest.helpers.temp_file( + "services.sls", services_sls, pillar_state_tree + ): + yield None + + +def test_documented_last_one_wins(salt_master, grains, overwrite_pillar_state_tree): + """ + The pillar topic guide promises "last one wins" for non-dict scalar + collisions. ``services.sls`` is applied after ``packages.sls`` so + ``bind`` should resolve to ``named``. + """ + opts = salt_master.config.copy() + pillar_obj = salt.pillar.Pillar(opts, grains, "test", "base") + ret = pillar_obj.compile_pillar() + assert ret.get("bind") == "named" + + +# --- include + subkey separator (issue 66622) --------------------------- + + +@pytest.fixture(scope="module") +def include_pillar_state_tree(pillar_state_tree): + """ + Lay down the documented include-with-key example. + """ + top_sls = textwrap.dedent( + """\ + base: + '*': + - main + """ + ) + main_sls = textwrap.dedent( + """\ + include: + - users: + key: users + """ + ) + users_sls = textwrap.dedent( + """\ + alice: 1000 + bob: 1001 + """ + ) + with pytest.helpers.temp_file( + "top.sls", top_sls, pillar_state_tree + ), pytest.helpers.temp_file( + "main.sls", main_sls, pillar_state_tree + ), pytest.helpers.temp_file( + "users.sls", users_sls, pillar_state_tree + ): + yield None + + +def test_documented_include_with_key(salt_master, grains, include_pillar_state_tree): + """ + With the documented form ``- users: {key: users}`` the included pillar + must be nested under the ``users`` key. The pillar dictionary keys + coming from ``users.sls`` should *not* appear at the top level. + """ + opts = salt_master.config.copy() + pillar_obj = salt.pillar.Pillar(opts, grains, "test", "base") + ret = pillar_obj.compile_pillar() + assert ret.get("users") == {"alice": 1000, "bob": 1001} + # The documented behavior: the included file contents are NOT promoted to + # the top level when `key:` is set. + assert "alice" not in ret + assert "bob" not in ret + + +# --- pillarenv selection (issue 63110) ---------------------------------- + + +@pytest.fixture(scope="module") +def multi_env_pillar_tree(tmp_path_factory): + """ + Lay down two pillarenvs: ``base`` and ``qa``. + """ + base_dir = tmp_path_factory.mktemp("pillar_base") + qa_dir = tmp_path_factory.mktemp("pillar_qa") + (base_dir / "top.sls").write_text("base:\n '*':\n - common\n") + (base_dir / "common.sls").write_text("environment: base\nsentinel: from-base\n") + (qa_dir / "top.sls").write_text("qa:\n '*':\n - common\n") + (qa_dir / "common.sls").write_text("environment: qa\nsentinel: from-qa\n") + yield {"base": [str(base_dir)], "qa": [str(qa_dir)]} + + +def test_documented_pillarenv_isolation(salt_master, grains, multi_env_pillar_tree): + """ + The pillar topic guide says that setting ``pillarenv`` causes a single + pillar environment to be used, ignoring all others. + """ + opts = salt_master.config.copy() + opts["pillar_roots"] = multi_env_pillar_tree + opts["pillarenv"] = "qa" + pillar_obj = salt.pillar.Pillar(opts, grains, "test", "base", pillarenv="qa") + ret = pillar_obj.compile_pillar() + assert ret.get("environment") == "qa" + assert ret.get("sentinel") == "from-qa" diff --git a/tests/pytests/functional/pillar/test_gpg_encryption_documented.py b/tests/pytests/functional/pillar/test_gpg_encryption_documented.py new file mode 100644 index 000000000000..392f8f19bd6f --- /dev/null +++ b/tests/pytests/functional/pillar/test_gpg_encryption_documented.py @@ -0,0 +1,196 @@ +""" +Round-trip GPG encryption examples from the pillar encryption documentation. + +These tests cover the issues: + +- 62733 (shebang renderer ``#!yaml|gpg`` for per-file decryption) +- 65682 (``--homedir`` must be passed to ``gpg`` when encrypting) +- 59539 (GPG terminology consistency: encrypted block markers are + ``BEGIN PGP MESSAGE``) + +The tests prove that the documented command-line invocations actually +produce ciphertexts that the gpg renderer can decrypt. A fresh, +short-lived keypair is generated inside a tmp homedir for each test +module so the round-trip exercises real encrypt + decrypt. +""" + +import logging +import pathlib +import shutil +import subprocess +import textwrap + +import pytest + +import salt.pillar + +log = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.skip_if_binaries_missing("gpg"), + pytest.mark.requires_random_entropy, + pytest.mark.slow_test, +] + + +GEN_BATCH = textwrap.dedent( + """\ + %no-protection + Key-Type: RSA + Key-Length: 2048 + Subkey-Type: RSA + Subkey-Length: 2048 + Name-Real: Salt Pillar Doctest + Name-Email: doctest@salt.example + Expire-Date: 0 + %commit + """ +) +TEST_RECIPIENT = "doctest@salt.example" + + +@pytest.fixture(scope="module", autouse=True) +def gpg_homedir(salt_master): + """ + Tmp gpg homedir loaded with a freshly generated keypair. + + The documented setup uses ``/etc/salt/gpgkeys`` and ``--homedir`` is + passed for every gpg command. We do the same here but rooted at the + salt-master config dir so cleanup is automatic. + """ + _gpg_homedir = pathlib.Path(salt_master.config_dir) / "gpgkeys" + _gpg_homedir.mkdir(0o700) + agent_started = False + try: + cmd_prefix = ["gpg", "--homedir", str(_gpg_homedir)] + + proc = subprocess.run( + cmd_prefix + ["--list-keys"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + text=True, + ) + log.debug("Instantiating gpg keyring: %s", proc.stdout) + + proc = subprocess.run( + cmd_prefix + ["--batch", "--gen-key"], + capture_output=True, + check=True, + text=True, + input=GEN_BATCH, + ) + log.debug("Generated keypair: %s %s", proc.stdout, proc.stderr) + + agent_started = True + yield _gpg_homedir + finally: + if agent_started: + try: + subprocess.run( + ["gpg-connect-agent", "--homedir", str(_gpg_homedir)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + text=True, + input="KILLAGENT", + ) + except (OSError, subprocess.CalledProcessError): + log.debug("No need to kill: old gnupg doesn't start the agent.") + shutil.rmtree(str(_gpg_homedir), ignore_errors=True) + + +def _encrypt(plaintext, gpg_homedir): + """ + Run the gpg encryption command that the docs prescribe and return the + armored ciphertext. + + The documented command is:: + + echo -n 'supersecret' | gpg --homedir /etc/salt/gpgkeys \\ + --trust-model always --armor --batch \\ + --encrypt -r + + The fix in issue 65682 was that the doc had to pass ``--homedir`` for + the command to succeed when running as a non-root user. Without it, + gpg looks up the recipient in ``~/.gnupg`` and fails with + ``encryption failed: No public key``. + """ + cmd = [ + "gpg", + "--homedir", + str(gpg_homedir), + "--armor", + "--batch", + "--trust-model", + "always", + "--encrypt", + "-r", + TEST_RECIPIENT, + ] + proc = subprocess.run( + cmd, + input=plaintext, + capture_output=True, + check=True, + text=True, + ) + return proc.stdout + + +def test_doc_encrypt_command_round_trip( + salt_master, grains, gpg_homedir, pillar_state_tree +): + """ + Encrypt with the exact command line documented in the renderers + section, then round-trip the ciphertext through the pillar GPG + decryption flow. + + Validates fix for issue 65682: passing ``--homedir`` is required for + the documented encrypt command to succeed. + """ + plaintext = "supersecret" + ciphertext = _encrypt(plaintext, gpg_homedir) + + assert ciphertext.startswith("-----BEGIN PGP MESSAGE-----") + assert ciphertext.rstrip().endswith("-----END PGP MESSAGE-----") + + indented = textwrap.indent(ciphertext.rstrip("\n"), " ") + sls_body = "secrets:\n" " vault:\n" " api_key: |\n" + indented + "\n" + + top_sls = "base:\n '*':\n - gpg\n" + with pytest.helpers.temp_file( + "top.sls", top_sls, pillar_state_tree + ), pytest.helpers.temp_file("gpg.sls", sls_body, pillar_state_tree): + opts = salt_master.config.copy() + opts["decrypt_pillar"] = ["secrets:vault"] + pillar_obj = salt.pillar.Pillar(opts, grains, "test", "base") + ret = pillar_obj.compile_pillar() + + assert ret["secrets"]["vault"]["api_key"] == plaintext + + +def test_doc_shebang_renderer_decrypts( + salt_master, grains, gpg_homedir, pillar_state_tree +): + """ + Validate fix for issue 62733: a pillar file with a ``#!yaml|gpg`` + shebang has its GPG-encrypted scalars decrypted at compile time + without needing ``decrypt_pillar`` to be configured on the master. + """ + plaintext = "supersecret" + ciphertext = _encrypt(plaintext, gpg_homedir) + + indented = textwrap.indent(ciphertext.rstrip("\n"), " ") + sls_body = "#!yaml|gpg\n\napi_key: |\n" + indented + "\n" + + top_sls = "base:\n '*':\n - gpg\n" + with pytest.helpers.temp_file( + "top.sls", top_sls, pillar_state_tree + ), pytest.helpers.temp_file("gpg.sls", sls_body, pillar_state_tree): + opts = salt_master.config.copy() + # Note: NO decrypt_pillar config; the shebang alone must do the work. + pillar_obj = salt.pillar.Pillar(opts, grains, "test", "base") + ret = pillar_obj.compile_pillar() + + assert ret.get("api_key") == plaintext diff --git a/tests/pytests/unit/renderers/test_py_renderer_documented.py b/tests/pytests/unit/renderers/test_py_renderer_documented.py new file mode 100644 index 000000000000..47a0da4a7186 --- /dev/null +++ b/tests/pytests/unit/renderers/test_py_renderer_documented.py @@ -0,0 +1,116 @@ +""" +Render the pure-python (``#!py``) renderer examples shown in the +renderer documentation. + +Covers issue 63698: the py renderer documentation needed real examples +demonstrating ``run()``, return-value structure, and access to dunders +(``__salt__``, ``__pillar__``, ``__grains__``). +""" + +import textwrap + +import pytest + +import salt.renderers.py as py_renderer +from tests.support.mock import MagicMock + + +@pytest.fixture() +def configure_loader_modules(minion_opts): + salt_dunder = { + "test.echo": MagicMock(return_value="echoed"), + } + return { + py_renderer: { + "__opts__": minion_opts, + "__salt__": salt_dunder, + "__pillar__": {"apache": "httpd"}, + "__grains__": {"os_family": "RedHat"}, + }, + } + + +def _write_template(tmp_path, body): + path = tmp_path / "tmpl.py" + path.write_text(textwrap.dedent(body)) + return str(path) + + +@pytest.mark.slow_test +def test_documented_basic_state(tmp_path): + """ + The simplest documented example: a ``run()`` function that returns + a highstate dictionary. + """ + tmplpath = _write_template( + tmp_path, + """\ + def run(): + return { + 'common_packages': { + 'pkg.installed': [ + {'pkgs': ['curl', 'vim']}, + ], + }, + } + """, + ) + # The py renderer ignores ``template`` and reads ``tmplpath``. + ret = py_renderer.render(template=None, tmplpath=tmplpath) + assert ret == { + "common_packages": { + "pkg.installed": [{"pkgs": ["curl", "vim"]}], + }, + } + + +@pytest.mark.slow_test +def test_documented_pillar_and_grains_dunders(tmp_path): + """ + The docs claim that ``__pillar__`` and ``__grains__`` are exposed + inside the python template. Make sure both work. + """ + tmplpath = _write_template( + tmp_path, + """\ + def run(): + return { + 'install_apache': { + 'pkg.installed': [ + {'name': __pillar__['apache']}, + ], + }, + 'tag_os_family': { + 'cmd.run': [ + {'name': 'echo ' + __grains__['os_family']}, + ], + }, + } + """, + ) + ret = py_renderer.render(template=None, tmplpath=tmplpath) + assert ret["install_apache"]["pkg.installed"][0]["name"] == "httpd" + assert ret["tag_os_family"]["cmd.run"][0]["name"] == "echo RedHat" + + +@pytest.mark.slow_test +def test_documented_salt_function_call(tmp_path): + """ + The docs example calls ``__salt__['test.echo']`` from inside ``run()``. + """ + tmplpath = _write_template( + tmp_path, + """\ + def run(): + value = __salt__['test.echo']('hello') + return { + 'announce': { + 'cmd.run': [ + {'name': 'echo ' + value}, + ], + }, + } + """, + ) + ret = py_renderer.render(template=None, tmplpath=tmplpath) + assert ret["announce"]["cmd.run"][0]["name"] == "echo echoed" diff --git a/tests/pytests/unit/renderers/test_pyobjects_documented.py b/tests/pytests/unit/renderers/test_pyobjects_documented.py new file mode 100644 index 000000000000..b3e3880bfe1d --- /dev/null +++ b/tests/pytests/unit/renderers/test_pyobjects_documented.py @@ -0,0 +1,103 @@ +""" +Render the pyobjects examples shown in the renderer documentation. + +Covers issues: + +- 61828: ``Module.run`` cannot accept a dotted function name as a literal + keyword argument; the docs now show a ``**kwargs`` workaround that the + test below renders end-to-end. +""" + +import logging +from collections import OrderedDict + +import pytest + +import salt.renderers.pyobjects as pyobjects +from tests.support.mock import MagicMock + +log = logging.getLogger(__name__) + + +@pytest.fixture() +def configure_loader_modules(minion_opts): + minion_opts["file_client"] = "local" + minion_opts["id"] = "doctest-minion" + pillar = MagicMock(return_value={}) + return { + pyobjects: { + "__opts__": minion_opts, + "__pillar__": pillar, + "__salt__": { + "config.get": MagicMock(), + "grains.get": MagicMock(), + "mine.get": MagicMock(), + "pillar.get": MagicMock(), + }, + }, + } + + +def _template(lines): + class Template: + def readlines(): # pylint: disable=no-method-argument + return list(lines) + + return Template + + +@pytest.mark.slow_test +def test_documented_basic_file_managed(): + """ + Verbatim from the first pyobjects example. + """ + template = _template( + [ + "#!pyobjects", + "File.managed('/tmp/foo', user='root', group='root', mode='1777')", + ] + ) + ret = pyobjects.render(template, sls="pyobj.basic") + assert ret == OrderedDict( + [ + ( + "/tmp/foo", + { + "file.managed": [ + {"group": "root"}, + {"mode": "1777"}, + {"user": "root"}, + ] + }, + ) + ] + ) + + +@pytest.mark.slow_test +def test_documented_module_run_kwargs_workaround(): + """ + The issue 61828 fix in the docs shows that calls of the form + ``Module.run('name', shadow.lock_password=user)`` are a syntax error + because keyword names cannot contain a dot. The recommended fix is + to pass a kwargs dictionary via ``**``. + """ + template = _template( + [ + "#!pyobjects", + "kw = {'shadow.lock_password': ['susan']}", + "Module.run('pyobject_shadow', **kw)", + ] + ) + ret = pyobjects.render(template, sls="pyobj.module_run") + # The Registry serializes Module.run with the function name as a key. + assert "pyobject_shadow" in ret + state = ret["pyobject_shadow"] + assert "module.run" in state + flat = { + k: v + for entry in state["module.run"] + if isinstance(entry, dict) + for k, v in entry.items() + } + assert flat.get("shadow.lock_password") == ["susan"]