Get ready for release 10.0.0

.github/workflows/macos.yml

@@ -21,16 +21,16 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install Mathics3 Pygments dependencies
       run: |
+        python -m pip install --upgrade pip setuptools wheel
         # We can comment out after next Mathics3-Scanner release
         # python -m pip install -e git+https://github.com/Mathics3/Mathics3-scanner#egg=Mathics3-Scanner[full]
-        git clone --depth 1 https://github.com/Mathics3/Mathics3-scanner.git
-        cd Mathics3-scanner/
-        pip install -e .
-        bash -x admin-tools/make-JSON-tables.sh
-        cd ..
+        # git clone --depth 1 https://github.com/Mathics3/Mathics3-scanner.git
+        # cd Mathics3-scanner/
+        # pip install -e .
+        # bash -x admin-tools/make-JSON-tables.sh
+        # cd ..
     - name: Install Mathics3 Pygments
       run: |
-        python -m pip install --upgrade pip setuptools wheel
         python -m pip install --no-build-isolation -e .
     - name: Test Mathics3 Pygments
       run: |

.github/workflows/ubuntu.yml

@@ -20,16 +20,16 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install Mathics3 Pygments dependencies
       run: |
+        python -m pip install --upgrade pip setuptools wheel
         # We can comment out after next Mathics3-Scanner release
         # python -m pip install -e git+https://github.com/Mathics3/Mathics3-scanner#egg=Mathics3-Scanner[full]
-        git clone --depth 1 https://github.com/Mathics3/Mathics3-scanner.git
-        cd Mathics3-scanner/
-        pip install -e .
-        bash -x admin-tools/make-JSON-tables.sh
-        cd ..
+        # git clone --depth 1 https://github.com/Mathics3/Mathics3-scanner.git
+        # cd Mathics3-scanner/
+        # pip install -e .
+        # bash -x admin-tools/make-JSON-tables.sh
+        # cd ..
     - name: Install Mathics3 Pygments
       run: |
-        python -m pip install --upgrade pip setuptools wheel
         python -m pip install --no-build-isolation -e .
     - name: Test Mathics3 Pygments
       run: |

.github/workflows/windows.yml

@@ -1,4 +1,4 @@
-name: Mathics (Windows)
+name: Mathics3 pygments (Windows)
 
 on:
   push:
@@ -21,16 +21,16 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install Mathics3 Pygments dependencies
       run: |
+        python -m pip install --upgrade pip setuptools wheel
         # We can comment out after next Mathics3-Scanner release
         # python -m pip install -e git+https://github.com/Mathics3/Mathics3-scanner#egg=Mathics3-Scanner[full]
-        git clone --depth 1 https://github.com/Mathics3/Mathics3-scanner.git
-        cd Mathics3-scanner/
-        pip install -e .
-        bash -x admin-tools/make-JSON-tables.sh
-        cd ..
+        # git clone --depth 1 https://github.com/Mathics3/Mathics3-scanner.git
+        # cd Mathics3-scanner/
+        # pip install -e .
+        # bash -x admin-tools/make-JSON-tables.sh
+        # cd ..
     - name: Install Mathics3 Pygments
       run: |
-        python -m pip install --upgrade pip setuptools wheel
         python -m pip install --no-build-isolation -e .
     - name: Test Mathics3 Pygments
       run: |

.pre-commit-config.yaml

@@ -23,7 +23,7 @@ repos:
         stages: [pre-commit]
         args: ["--profile", "black"]
 -   repo: https://github.com/psf/black
-    rev: 23.12.1
+    rev: 25.11.0
     hooks:
     - id: black
       language_version: python3

CHANGES.rst

@@ -1,12 +1,21 @@
 CHANGES
 =======
 
+10.0.0
+------
+
+"Mathics" has been renamed to "Mathics3".
+
+The bump in version number from 1.0.4 to 10.0.0 is to simplify understaning compatibility between all Mathics3 10 API components.
+
+Small administrative changes have been made for the ever-changing Python environment.
+
 1.0.4
 -----
 
 This is a Transitional release.
 
-Although there is a dependency on mathics-scanner, we don't use it since tables have changed. There is some code for reading the YAML takes from the project though. With a newer and better mathics-scanner, we should be able to revise this code, use the tokenizer from mathics-scanner, and simplify things greatly.
+Although there is a dependency on Mathics3-scanner, we don't use it since tables have changed. There is some code for reading the YAML takes from the project though. With a newer and better mathics-scanner, we should be able to revise this code, use the tokenizer from mathics-scanner, and simplify things greatly.
 
 1.0.3
 -----
@@ -44,8 +53,8 @@ Note: There are still a few tests from pygments-mathematica that fail.
 
 First public release. Is largely the same thing as
 pygments-mathematica, but we are geared for CLI use in mathicsscript and
-we get operator information from Mathics Scanner.
+we get operator information from Mathics3 Scanner.
 
 For uses outside of terminal use, pygments-mathematica is probably better.
 But over time I expect this will change since we will draw more power from
-Mathics Scanner.
+Mathics3 Scanner.

Makefile

@@ -52,7 +52,10 @@ pytest:
 rmChangeLog:
 	$(RM) ChangeLog || true
 
+#: Create ChangeLog from version control without corrections
+ChangeLog-without-corrections:
+	git log --pretty --numstat --summary | $(GIT2CL) >ChangeLog
+
 #: Create a ChangeLog from git via git log and git2cl
-ChangeLog: rmChangeLog
-	git log --pretty --numstat --summary | $(GIT2CL) >$@
+ChangeLog: rmChangeLog ChangeLog-without-corrections
 	patch ChangeLog < ChangeLog-spell-corrected.diff

README.md

@@ -1,37 +1,34 @@
 # Mathematica lexer and highlighter for Pygments
 
-[![mathics-pygments (OSX)](https://github.com/Mathics3/mathics-pygments/actions/workflows/osx.yaml/badge.svg)](https://github.com/Mathics3/mathics-pygments/actions/workflows/osx.yaml)
-[![mathics-pygments (ubuntu)](https://github.com/Mathics3/mathics-pygments/actions/workflows/ubuntu.yaml/badge.svg)](https://github.com/Mathics3/mathics-pygments/actions/workflows/ubuntu.yaml)
-![](https://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)
-[![Packaging status](https://repology.org/badge/tiny-repos/mathics-pygments.svg)](https://repology.org/project/mathics-pygments/versions)
-
+[![Mathics3-pygments (ubuntu)](https://github.com/Mathics3/Mathics3-pygments/actions/workflows/ubuntu.yml/badge.svg)](https://github.com/Mathics3/Mathics3-pygments/actions/workflows/ubuntu.yml)
+[![Mathics3-pygments (MacOS)](https://github.com/Mathics3/Mathics3-pygments/actions/workflows/macos.yml/badge.svg)](https://github.com/Mathics3/Mathics3-pygments/actions/workflows/macos.yml)
+![MIT license](https://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)
 
 
 A lexer and highlighter for [_Mathematica_](http://wolfram.com/mathematica)/Wolfram Language
  source code using the [pygments](http://pygments.org) engine.
 
- This code is based on [pygments-mathematica](https://pypi.org/project/pygments-mathematica/) but has been specially revised so it interacts with [mathicsscript](https://pypi.org/project/mathicsscript/). As such it works better that pygments-mathematica for this use, and possibly might not be as good in other contexts. Over time though this should get address.
+ This code is based on [pygments-mathematica](https://pypi.org/project/pygments-mathematica/) but has been specially revised so it interacts with [mathicsscript](https://pypi.org/project/mathicsscript/). As such, it works better than Pygments-Mathematica for this use, and possibly might not be as good in other contexts. Over time, though, this should get addressed.
 
- Another difference, is the fact that this package relegates knowledge about operator names to [Mathics-Scanner](https://pypi.org/project/Mathics-Scanner/).
+ Another difference is the fact that this package delegates knowledge about operator names to [Mathics-Scanner](https://pypi.org/project/Mathics-Scanner/).
 
 
 ## Features
 
 It can currently lex and highlight:
 
-  - All builtin functions in the ``System` `` context including Unicode symbols like `π` except those
-  that use characters from the private Unicode space (e.g. `\[FormalA]`).
-  - User defined symbols, including those in a context.
-  - All operators including Unicode operators like `∈` and `⊕`.
-  - Comments, including multi line and nested.
-  - Strings, including multi line and escaped quotes.
-  - Patterns, slots (including named slots `#name` introduced in version 10) and slot sequences.
-  - Message names (e.g. the `ivar` in `General::ivar`)
+  - All built-in functions in the `System` context, including Unicode symbols like `π`, except those that use characters from the private Unicode space (e.g. `\[FormalA]`).
+  - User-defined symbols, including those in a context.
+  - All operators, including Unicode operators like `∈` and `⊕`.
+  - Comments, including multi-line and nested.
+  - Strings, including multi-line and escaped quotes.
+  - Patterns, slots (including named slots `#name` introduced in version 10), and slot sequences.
+  - Message names (e.g., the `ivar` in `General::ivar`)
   - Numbers including base notation (e.g. `8 ^^ 23 == 19`) and scientific notation (e.g. `1 *^ 3 == 1000`).
-  - Local variables in `Block`, `With` and `Module`.
+  - Local variables in `Block`, `With`, and `Module`.
 
 ### Example:
-```
+```mathematica
 (* An example highlighting the features of
    this Pygments plugin for Mathematica *)
 lissajous::usage = "An example Lissajous curve.\n" <>
@@ -52,23 +49,20 @@ Run `pip install mathics-pygments` from the command line. That's it!
 
 ### From source code
 
-If you'd like to make modifications to the color scheme for personal use or if you'd like to try the
-most recent release that might not yet be available in PyPi, download and unzip the source code
-from the [latest release](https://github.com/Mathics3/mathics-pygments/releases/latest). After
-you've [installed Pygments](http://pygments.org/download/) (`pip install Pygments` works well
-if you already have python setup on your system), run the following from the repo's root directory:
+If you'd like to make modifications to the color scheme for personal use or if you'd like to try the most recent release that might not yet be available in PyPi, download and unzip the source code from the [latest release](https://github.com/Mathics3/Mathics3-pygments/releases/latest). After
+you've [installed Pygments](http://pygments.org/download/) (`pip install Pygments` works well if you already have Python set up on your system), run the following from the repo's root directory:
 
 ```bash
 python setup.py install
 ```
 
 ## Usage
 
-### Server-side syntax highlighting in Jekyll, Octopress and other static websites
+### Server-side syntax highlighting in Jekyll, Octopress, and other static websites
 
 To highlight _Mathematica_ code using this lexer, enclose the code between these liquid tags:
 
-```
+```Jinja
 {% highlight wl %}
 <your code here>
 {% endhighlight %}
@@ -83,9 +77,7 @@ require 'pygments'
 Pygments.start('<path to your python env>/site-packages/pygments/')
 ```
 
-> **NOTE:** Although this lexer is registered with the names `mathematica` and `mma` for use as language hints, the
-default lexer that ships with Pygments overrides this. Hence until this is incorporated into the main Pygments repository
-please use `wl` or `wolfram` or `wolfram-language` as the language hint.
+> **NOTE:** Although this lexer is registered with the names `mathematica` and `mma` for use as language hints, the default lexer that ships with Pygments overrides this. Hence, until this is incorporated into the main Pygments repository, please use `wl` or `wolfram` or `wolfram-language` as the language hint.
 
 ### Highlighting in LaTeX documents
 
@@ -122,7 +114,7 @@ Saving the above as `mma.tex` and running `xelatex --shell-escape mma.tex` shoul
 
 The [Pelican static generator](http://blog.getpelican.com/) is written in Python and uses Pygments by default. To use it there, you mark code blocks with the usual 4 spaces indent and you prepend it with `:::wl` if you are using Markdown
 
-```
+```wl
     :::wl
     FileNames["CodeGenerator.m", {$InstallationDirectory}, 4]
     (*
@@ -132,16 +124,15 @@ The [Pelican static generator](http://blog.getpelican.com/) is written in Python
 
 If you are using ReStructuredText, please mark your *Mathematica* code with
 
-```
+```RsT
 .. code-block:: wl
 
    <indented code block goes here>
 ```
 
 ### Command line usage
 
-The `pygmentize` command can be used to invoke this lexer and convert any _Mathematica_ file to an appropriately
-highlighted file in a different format. For example, to convert a file `package.m` to a HTML file, run
+The `pygmentize` command can be used to invoke this lexer and convert any _Mathematica_ file to an appropriately highlighted file in a different format. For example, to convert a file `package.m` to an HTML file, run
 
 ```bash
 pygmentize -O full,style=mathematica -f html -l wl -o package.html package.m
@@ -158,20 +149,17 @@ using the `sass` compiler as:
 make mma.css  # Same thing as: sass mma.scss mma.css
 ```
 
-For other applications including command line usage, the lexer ships with a style named `mathematica`.
+For other applications, including command line usage, the lexer ships with a style named `mathematica`.
 (See the arguments to the `pygmentize` command in the section above.) To use different colors, modify
 the style in `mathics_pygemnts/style.py` and run `python setup.py install` again.
 
 If you fancy the default style that ships with the _Mathematica_ notebook, use the `mathematica-notebook` scheme.
 
 ## Limitations
 
-It cannot highlight lexically and dynamically scoped variables (e.g. the `x` in `With[{x = 1}, x + 1]` or
-the `Plus` in `Block[{Plus = Times}, 2 + 3]`, etc.) consistently throughout their scope. This would require a
-parser that further processes the stream of tokens and builds an AST that captures the semantics of the language.
+It cannot highlight lexically and dynamically scoped variables (e.g., the `x` in `With[{x = 1}, x + 1]` or the `Plus` in `Block[{Plus = Times}, 2 + 3]`, etc.) consistently throughout their scope. This would require a parser that further processes the stream of tokens and builds an AST that captures the semantics of the language.
 
-This is currently not a high priority since it is non-trivial to implement it within the framework
-by Pygments, but I am interested in supporting this eventually, so collaborations/pull requests are welcome :)
+This is currently not a high priority since it is non-trivial to implement within the framework by Pygments, but I am interested in supporting this eventually, so collaborations/pull requests are welcome :)
 
 ## Acknowledgments
 

admin-tools/pyenv-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.6 3.7 3.8 pypy3.7 3.9 3.10 3.11 3.12 3.13'
+export PYVERSIONS='3.11 pypy3.11-7.3.20 3.12 3.13 3.14'

mathics_pygments/builtins.py

@@ -6550,7 +6550,7 @@ def get_datadir():
     "$WolframUUID",
 }
 
-# The Mathics parser can't handle Degree ("uB0") as an operator, so we'll add it here.
+# The Mathics3 parser can't handle Degree ("uB0") as an operator, so we'll add it here.
 UNICODE_OPERATORS = set(list(_data["unicode-operators"].keys()) + ["\u00b0"])
 
 UNICODE_OPERATORS_OLD = {

mathics_pygments/generate/build_pygments_tables.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# This scripts reads the data from named-characters and converts it to the
+# format used by the library internally
+
+import json
+import os.path as osp
+import sys
+from pathlib import Path
+from typing import Any, Dict, Set
+
+import click
+import mathics_scanner
+import yaml
+
+try:
+    from mathics_scanner.version import __version__
+except ImportError:
+    # When using build isolation
+    __version__ = "unknown"
+
+OPERATOR_FIELDS = [
+    "ascii-operators",
+    "unicode-operators",
+]
+
+DEFAULT_DATA_DIR = Path(osp.join(osp.dirname(mathics_scanner.__file__), "data"))
+
+
+def get_srcdir() -> str:
+    filename = osp.normcase(osp.dirname(osp.abspath(__file__)))
+    return osp.realpath(filename)
+
+
+def read(*rnames) -> str:
+    return open(osp.join(get_srcdir(), *rnames)).read()
+
+
+def compile_tables(
+    operator_data: Dict[str, dict], character_data: Dict[str, dict]
+) -> Dict[str, Any]:
+    """
+    Compiles the general table into the tables used internally by the library.
+    This facilitates fast access of this information by clients needing this
+    information.
+    """
+    ascii_operators: Set[str] = set()
+    unicode_operators: Dict[str, str] = {}
+
+    operator_names = {operator_name for operator_name in operator_data.keys()}
+    for name, character_info in character_data.items():
+        operator_name = character_info.get("operator-name")
+        if name in operator_names or operator_name:
+            if operator_name:
+                name = operator_name
+            if "unicode-equivalent" in character_info.keys():
+                unicode_operators[character_info["unicode-equivalent"]] = name
+            if "ascii" in character_info.keys():
+                ascii_operators.add(character_info["ascii"])
+
+    return {
+        "ascii-operators": sorted(ascii_operators),
+        "unicode-operators": unicode_operators,
+    }
+
+
+@click.command()
+@click.version_option(version=__version__)  # NOQA
+@click.option(
+    "--output",
+    "-o",
+    show_default=True,
+    type=click.Path(writable=True),
+    default=DEFAULT_DATA_DIR / "operators.json",
+)
+@click.argument(
+    "data_dir", type=click.Path(readable=True), default=DEFAULT_DATA_DIR, required=False
+)
+def main(output, data_dir):
+    with (
+        open(data_dir / "operators.yml", "r", encoding="utf8") as operator_f,
+        open(data_dir / "named-characters.yml", "r", encoding="utf8") as character_f,
+        open(output, "w") as o,
+    ):
+        # Load the YAML data.
+        operator_data = yaml.load(operator_f, Loader=yaml.FullLoader)
+        character_data = yaml.load(character_f, Loader=yaml.FullLoader)
+
+        # Precompile the tables.
+        data = compile_tables(operator_data, character_data)
+
+        # Dump the preprocessed dictionaries to disk as JSON.
+        json.dump(data, o)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])