Built-in assemblers for all ISAs, ISA registry, and simulator restructuring

.github/workflows/ci.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: [main, master]
   pull_request:
-    branches: [main, master]
+  workflow_dispatch:
 
 jobs:
   lint:
@@ -14,7 +14,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Install linting tools
         run: pip install ruff mypy
@@ -29,19 +29,84 @@ jobs:
         run: mypy mapachespim/ --ignore-missing-imports
 
   test:
-    runs-on: ubuntu-latest
+    name: test (${{ matrix.os }}, py${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.8", "3.10", "3.12"]
+        # macos-14 is Apple Silicon, which many students have.
+        os: [ubuntu-latest, windows-latest, macos-14]
+        python-version: ["3.9", "3.13"]
+        include:
+          - os: ubuntu-latest
+            python-version: "3.11"
     steps:
       - uses: actions/checkout@v4
 
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install package with dev dependencies
-        run: pip install -e ".[dev]"
+      - name: Build wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build --wheel --outdir dist
+
+      # Install the built wheel (not an editable checkout) so packaging
+      # mistakes, like files missing from the wheel, fail CI.
+      - name: Install wheel
+        shell: bash
+        run: pip install dist/*.whl pytest
+
+      - name: Smoke test the installed package outside the repository
+        shell: bash
+        working-directory: ${{ runner.temp }}
+        run: |
+          set -ex
+          python -c "import mapachespim, pathlib; p = pathlib.Path(mapachespim.__file__); assert 'site-packages' in str(p), p"
+          mapachespim --version
+          mapachespim-as --version
+          mapachespim -e riscv/hello_asm
+          mapachespim -e mips/hello_asm
+          mapachespim -e arm/hello_asm
+          mapachespim -e x86_64/hello_asm
+          mapachespim --copy-examples ex
+          mapachespim -e ex/riscv/fibonacci/fibonacci.s
+          mapachespim -e ex/mips/fibonacci/fibonacci.s
+          mapachespim -e ex/arm/fibonacci/fibonacci.s
+          mapachespim -e ex/x86_64/fibonacci/fibonacci.S
+          printf 'load riscv/fibonacci\nbreak fibonacci\nrun\nregs\nlist\nquit\n' | mapachespim
+
+      - name: Install GNU binutils for the assembler differential tests
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu binutils-aarch64-linux-gnu binutils-x86-64-linux-gnu
 
       - name: Run tests
-        run: pytest tests/ -v --tb=short
+        run: python -m pytest tests/ --tb=short
+
+  assembler-fuzz:
+    # The built-in assemblers must produce byte-identical
+    # output to GNU as. This runs the randomized comparison much deeper
+    # than the regular test job.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install GNU binutils
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu binutils-aarch64-linux-gnu binutils-x86-64-linux-gnu
+
+      - name: Install package
+        run: pip install -e ".[dev]"
+
+      - name: Compare with GNU as (50x fuzz scale)
+        env:
+          MAPACHESPIM_FUZZ_SCALE: "50"
+        run: python -m pytest tests/test_differential_*.py --tb=short

.github/workflows/release.yml

@@ -0,0 +1,55 @@
+name: Release
+
+# Publishes to PyPI when a GitHub release is published.
+# One-time setup is described in docs/RELEASING.md.
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Check the tag matches the package version
+        env:
+          TAG: ${{ github.event.release.tag_name }}
+        run: |
+          version=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
+          if [ "${TAG#v}" != "$version" ]; then
+            echo "Release tag $TAG does not match pyproject.toml version $version" >&2
+            exit 1
+          fi
+
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install --upgrade build twine
+          python -m build
+          python -m twine check --strict dist/*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/mapachespim/
+    permissions:
+      id-token: write  # PyPI trusted publishing; no API token needed
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -37,3 +37,8 @@ dist/
 
 # Temporary/debug files
 *.log
+
+# Coverage data
+.coverage
+.coverage.*
+htmlcov/

CHANGELOG.md

@@ -0,0 +1,81 @@
+# Changelog
+
+## Unreleased
+
+### Code structure
+- Each ISA is described once, in `mapachespim/isa.py`; the simulator, disassembler, ELF loader and
+  writer, assembler, console, and command-line tools all read their ISA-specific facts from it.
+  Emulator constants are in `engines.py` and each ISA's assembler hooks in `toolchain/targets.py`.
+  See `docs/dev/architecture.md`, which also explains how to add an ISA.
+- New `Simulator.run_until()` with `StopReason` and `RunResult`; the console uses it for `run`
+  and `-e`. New `Simulator.spec`, `symbols`, `get_sections()`, and `find_section()`.
+- Removed: `mapachespim.toolchain.memory_map` (use `mapachespim.memory_map`), the `SailSimulator`
+  alias, the per-ISA `*Config` classes, `STACK_TOP`/`STACK_SIZE`/`STACK_ADDR`, and
+  `mapachespim.elf_loader.Architecture`.
+- Tests that every registered ISA must pass: syscalls, register mapping, ELF output, and more.
+
+### Fixed
+- ARM64 registers x29 (fp) and x30 (lr) showed and changed the wrong (vector) registers.
+- ARM64 programs crashed on `read_int` and `read_char`: x0 was treated as a read-only register.
+- 32-bit ARM and RISC-V ELF files were run as if they were 64-bit programs; they are now refused.
+- ELF symbol tables written by the assembler listed local symbols after global ones and put every
+  symbol in `.text`. The example binaries are rebuilt.
+
+### Installation
+- All four ISAs are assembled by new built-in pure-Python assemblers, so every ISA assembles on
+  every platform. The Keystone dependency is gone: it had no prebuilt packages for ARM machines
+  such as Apple Silicon Macs, and its PyPI release could not assemble RISC-V at all.
+- The example programs are included in the package, so `load riscv/fibonacci` works after
+  `pip install`. `mapachespim --copy-examples DIR` copies them somewhere editable.
+- Python 3.9 or newer is required.
+
+### Console
+- `load` assembles `.s` files directly (with debug info) and loads bundled examples by name.
+- New `reload` command re-assembles after editing, keeping breakpoints on their labels.
+- New `examples` command lists the bundled programs.
+- `step` now performs syscalls (it used to skip them) and stops at program exit.
+- `reset` reloads the program, so it can be run again.
+- Errors explain what went wrong and where, e.g.
+  `Read from unmapped address 0x0 (at PC=0x80000004)` with the source line.
+- `disasm` walks x86-64 instructions by their real length and marks the PC.
+- `mem`, `disasm`, and `delete` accept labels; `regs decimal` is signed; the first `regs` after
+  loading marks registers changed since the start; `set show-changes off` works.
+- `mapachespim --version`; `-e` returns the program's exit code (124 on hitting `--max-steps`) and
+  accepts `.s` files and example names.
+
+### Simulator
+- `read_char` returned the second line of input; `print_int` printed negative MIPS values as
+  large unsigned numbers; unknown syscall numbers are now reported as errors; new SPIM `exit2` (17).
+- Address 0 is no longer mapped on ARM64, so null-pointer accesses fault.
+- The RISC-V sign-extended address alias now shares memory with the real address.
+- Loading a program starts from a fresh machine state and may switch ISA.
+
+### Assembler
+- The built-in assemblers produce byte-for-byte the same code as GNU `as` for all four ISAs,
+  verified by randomized differential tests against GNU binutils (run in CI). This changed some
+  pseudo-instruction expansions to GNU's: RISC-V `call`/`tail` are always `auipc`+`jalr`, `li` uses
+  GNU's algorithm; MIPS `la` is `lui`+`addiu`, and compare-branches, `abs`, `mul`/`div`/`rem` with
+  constants follow GNU. Out-of-range RISC-V branches are relaxed like GNU does.
+- MIPS `div`/`rem`/`divu`/`remu` with three operands now trap on division by zero and overflow
+  (as SPIM and GNU do), reported as "Division by zero"; `.set noreorder` is supported.
+- Fixed: `.align n` meant n bytes instead of 2^n on RISC-V/MIPS/ARM; MIPS data was little-endian
+  when the ISA came from `--isa`/`load prog.s mips32`; labels in `.word` etc. became 0; `\xNN` in
+  strings produced two bytes; custom sections were not written to the ELF; invalid values in data
+  directives were silently ignored; MIPS `.half`/`.word` are now auto-aligned like GNU `as`; `break`
+  codes were encoded in the wrong field.
+- Expressions follow GNU `as` rules (operators, precedence, octal `010`, `'c'` literals); `.`, and
+  `name = expression` (e.g. `len = . - msg`) are supported.
+- RISC-V `.equ` constants can be used as instruction operands; `add a0, a1, 5` is accepted as `addi`.
+- Debug info for big-endian targets (MIPS) was written in the wrong byte order, so source-level
+  debugging never worked for MIPS.
+- `.isa riscv64  # comment` is recognized.
+- ARM64: `ldr x0, =value` uses a literal pool (`.ltorg`/`.pool`) like GNU `as`; logical immediates,
+  `mov` aliases, and `//` comments follow GNU. The ARM examples were updated to GNU syntax.
+- x86-64: AT&T and Intel syntax (auto-detected, or `.att_syntax`/`.intel_syntax`), GNU's choice of
+  encodings, and GNU's jump relaxation (short jumps grow to 32-bit only when needed). `.word` is 2
+  bytes on x86, as in GNU `as`.
+- Alignment in code is padded with nops the way GNU `as` does on each ISA; the `.p2align`/`.balign`
+  maximum-skip argument works; `.fill` is supported; `;` separates statements; numeric local labels
+  (`1:` with `1b`/`1f`) work.
+- ISA aliases (`mips`, `x64`, `aarch64`) were treated as unknown ISAs by the directive parser, so
+  e.g. `--isa mips` produced little-endian data.