From 742d95ddea2fd39e05576f2993bab1f4f6c19d5c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 00:07:20 +0000 Subject: [PATCH 01/18] Fix simulator and console bugs found in code review Simulator: - Perform syscalls when single-stepping (step used to skip them, so stepping printed nothing and ran past exit) - read_char read two lines and returned the second; it now buffers one line and returns characters in order - print_int sign-extends 32-bit MIPS registers (-5 printed as 4294967291) - reset reloads the ELF instead of leaving memory empty - Fault messages name the access and address ("Read from unmapped address 0x50000000") instead of Unicorn's generic error - Stop mapping address 0 on ARM so null dereferences fault - Back the RISC-V sign-extended alias region with the same memory, so writes through lui-built addresses are visible at 0x8xxxxxxx - Track exit state and exit code; unknown syscalls are errors - Add SPIM exit2 (17); stdin/stdout are injectable for tests/grading - Loading a new ELF starts from a fresh machine and may change ISA - Move memory_map to the package top level so the simulator and assembler share one layout (stack placement now follows it) Console: - disasm walks x86-64 by real instruction length and marks the PC - step shows full x86 instruction bytes and keeps program output on its own line - run/step report exit code, fault reason, and source line - mem/disasm/delete accept symbol names - regs decimal is signed; set show-changes works; set completes output-spacing; list suggests mapachespim-as -g - Add --version and --max-steps; -e exits with the program's exit code, 1 on error, 124 when the instruction limit is hit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- mapachespim/console.py | 453 +++++++++++++-------- mapachespim/memory_map.py | 133 ++++++ mapachespim/toolchain/memory_map.py | 142 +------ mapachespim/unicorn_backend.py | 605 ++++++++++++++++++---------- tests/test_regressions.py | 232 +++++++++++ 5 files changed, 1069 insertions(+), 496 deletions(-) create mode 100644 mapachespim/memory_map.py create mode 100644 tests/test_regressions.py diff --git a/mapachespim/console.py b/mapachespim/console.py index 0949dc4..aad3165 100644 --- a/mapachespim/console.py +++ b/mapachespim/console.py @@ -388,6 +388,9 @@ def do_step(self, arg: str) -> None: self.print_error(f'Error: Invalid number "{arg}".') return + if self._report_if_exited(): + return + # Execute instructions for i in range(n_steps): pc = self.sim.get_pc() @@ -397,41 +400,20 @@ def do_step(self, arg: str) -> None: print(f"Breakpoint hit at {pc:#018x}", file=self.stdout) break - # Get instruction before executing (for display) - try: - instr_disasm = self.sim.disasm(pc) - instr_bytes = self.sim.read_mem(pc, 4) - instr_hex = "".join(f"{b:02x}" for b in instr_bytes) - except Exception: - instr_disasm = "" - instr_hex = "????????" + # Show the instruction before executing it, so any output it + # produces (e.g. a print syscall) appears after it + self._end_program_output() + print(self._format_instruction(pc), file=self.stdout) + self.stdout.flush() result = self.sim.step() - - if result == StepResult.HALT: - print(f"[{pc:#010x}] 0x{instr_hex} {instr_disasm}", file=self.stdout) - print("Program halted", file=self.stdout) - break - elif result == StepResult.ERROR: - print(f"[{pc:#010x}] 0x{instr_hex} {instr_disasm}", file=self.stdout) - print("Execution error", file=self.stdout) + should_terminate, reason = self.sim.check_termination(result) + if should_terminate: + self._report_stop(reason, pc) break - # Show the instruction that was executed - # Try to show symbol name - sym, offset = self.sim.addr_to_symbol(pc) - if sym and offset == 0: - print(f"[{pc:#010x}] 0x{instr_hex} {instr_disasm} <{sym}>", file=self.stdout) - elif sym: - print( - f"[{pc:#010x}] 0x{instr_hex} {instr_disasm} <{sym}+{offset}>", - file=self.stdout, - ) - else: - print(f"[{pc:#010x}] 0x{instr_hex} {instr_disasm}", file=self.stdout) - - # Note: prev_regs snapshot is now taken when registers are displayed, - # not after each step. This allows stars to accumulate across steps. + # Keep the next prompt off the line of any program output + self._end_program_output() def do_stepreg(self, arg: str) -> None: """Execute instructions and show registers @@ -506,65 +488,52 @@ def do_run(self, arg: str) -> None: self.print_error(f'Error: Invalid number "{arg}".') return + if self._report_if_exited(): + return + # Run with breakpoint/interrupt checking self._running = True self._interrupted = False steps_executed = 0 + stopped = False try: while max_steps == 0 or steps_executed < max_steps: # Check for interrupt if self._interrupted: self._interrupted = False + self._end_program_output() print(f"Interrupted after {steps_executed} instructions", file=self.stdout) + stopped = True break # Check for breakpoint (but skip on first iteration to allow continuing from a breakpoint) pc = self.sim.get_pc() if steps_executed > 0 and pc in self.breakpoints: + self._end_program_output() print( f"Breakpoint hit at {pc:#018x} after {steps_executed} instructions", file=self.stdout, ) + stopped = True break result = self.sim.step() steps_executed += 1 - # Check for termination using centralized logic should_terminate, reason = self.sim.check_termination(result) if should_terminate: - # Print appropriate termination message - if reason == "syscall_exit": - print( - f"Program exited via syscall after {steps_executed} instructions", - file=self.stdout, - ) - elif reason == "halt": - print( - f"Program halted after {steps_executed} instructions", file=self.stdout - ) - elif reason == "error": - print( - f"Execution error at {pc:#018x} after {steps_executed} instructions", - file=self.stdout, - ) - elif reason == "tohost": - print( - f"Program completed (tohost) after {steps_executed} instructions", - file=self.stdout, - ) + self._report_stop(reason, pc, steps_executed) + stopped = True break finally: self._running = False - if not self._interrupted and steps_executed > 0: - final_pc = self.sim.get_pc() - if max_steps > 0 and steps_executed >= max_steps: - print( - f"Executed {steps_executed} instructions (max limit reached)", file=self.stdout - ) - print(f"PC = {final_pc:#018x}", file=self.stdout) + self._end_program_output() + if max_steps > 0 and steps_executed >= max_steps and not stopped: + print(f"Executed {steps_executed} instructions (max limit reached)", file=self.stdout) + if steps_executed > 0 and not self.sim.exited: + print(f"PC = {self.sim.get_pc():#018x}", file=self.stdout) def do_continue(self, arg: str) -> None: """Continue execution after hitting a breakpoint @@ -595,43 +564,114 @@ def do_continue(self, arg: str) -> None: self.do_run("") def do_reset(self, arg: str) -> None: - """Reset the simulator to initial state + """Reset the program to its initial state Usage: reset - Resets the simulator state, clearing all register values and - resetting the program counter. The loaded program remains in - memory but you may need to reload it to reset the entry point. + Reloads the current program from disk: memory, registers, and the + program counter go back to how they were right after 'load', so + you can run the program again from the start. Breakpoints are kept. Examples: - load examples/test_simple/simple - step 5 # Execute some instructions - reset # Reset simulator state - load examples/test_simple/simple # Reload to restore entry point + load examples/riscv/fibonacci/fibonacci + run # Run to completion + reset # Start over + step # Step from the entry point again Tips: - - Breakpoints are preserved (use 'clear' to remove them) - - Memory contents may be preserved (depends on simulator state) - - Usually better to reload the file for a clean state - - Use to recover from error states + - Use 'clear' to remove breakpoints + - If you edited and re-assembled the program, 'reset' picks up + the new version """ - self.sim.reset() - print("Simulator reset.", file=self.stdout) - if self.loaded_file: - print('Program still loaded. Use "load" to reload if needed.', file=self.stdout) + if not self.loaded_file: + self.print_error('Error: No program loaded. Use "load " first.') + return + try: + self.sim.reset() + except Exception as e: + self.print_error(f"Error reloading {self.loaded_file}: {e}") + return + self.prev_regs = None + print( + f"Reset {self.loaded_file}. PC = {self.sim.get_pc():#018x}", + file=self.stdout, + ) + + # --- Execution helpers --- + + def _format_instruction(self, pc: int) -> str: + """Format one instruction as '[addr] 0xbytes disasm '""" + try: + text, size = self.sim.disasm_with_size(pc) + raw = self.sim.read_mem(pc, size) + instr_hex = "".join(f"{b:02x}" for b in raw) + except Exception: + text, instr_hex = "", "????????" + + sym, offset = self.sim.addr_to_symbol(pc) + if sym and offset == 0: + suffix = f" <{sym}>" + elif sym: + suffix = f" <{sym}+{offset}>" + else: + suffix = "" + return f"[{pc:#010x}] 0x{instr_hex} {text}{suffix}" + + def _end_program_output(self) -> None: + """Start a new line if the program's output left the cursor mid-line""" + if self.sim is not None and self.sim.output_needs_newline: + print(file=self.stdout) + self.sim.output_needs_newline = False + + def _report_if_exited(self) -> bool: + """If the program already exited, say so and return True""" + if self.sim.exited: + self.print_error( + f"The program has exited (code {self.sim.exit_code}). " + 'Use "reset" to run it again.' + ) + return True + return False + + def _report_stop(self, reason: Optional[str], pc: int, steps: Optional[int] = None) -> None: + """Print why execution stopped (exit, error, ...)""" + self._end_program_output() + after = f" after {steps} instructions" if steps is not None else "" + if reason == "syscall_exit": + print(f"Program exited with code {self.sim.exit_code}{after}", file=self.stdout) + elif reason == "halt": + print(f"Program halted{after}", file=self.stdout) + elif reason == "tohost": + print(f"Program completed (tohost){after}", file=self.stdout) + elif reason == "error": + detail = self.sim.last_error or "Execution error" + print(f"Error: {detail}", file=self.stdout) + print(f" {self._format_instruction(pc)}", file=self.stdout) + location = self.source_info.get_location(pc) + if location: + filename, line_num = location + lines = self.source_info.get_source_lines(filename, line_num, 1) + if lines: + print(f" {filename}:{line_num}: {lines[0][1].strip()}", file=self.stdout) # --- State Inspection --- def _format_reg_value(self, value: int, show_mode: str, leading_zeros_mode: str) -> str: """Format a register value according to display settings""" + sign = "" # only used by signed decimal if show_mode == "hex": # Format as hex with 0x prefix formatted = f"{value:016x}" prefix = "0x" elif show_mode == "decimal": - # Format as decimal (max 20 digits for 64-bit) - formatted = f"{value:020d}" + # Signed decimal, since that is how students think of values + # like -1 (max 20 digits for 64-bit) + bits = 32 if self._is_32bit() else 64 + if value & (1 << (bits - 1)): + value -= 1 << bits + sign = "-" if value < 0 else "" + formatted = f"{abs(value):0{20 - len(sign)}d}" prefix = "" elif show_mode == "binary": # Format as binary with 0b prefix @@ -657,9 +697,16 @@ def _format_reg_value(self, value: int, show_mode: str, leading_zeros_mode: str) # Replace leading zeros with dots stripped = formatted.lstrip("0") or "0" num_leading = len(formatted) - len(stripped) - formatted = "." * num_leading + stripped + formatted = "." * num_leading + sign + stripped + sign = "" + + return sign + prefix + formatted - return prefix + formatted + def _is_32bit(self) -> bool: + """True if the loaded ISA has 32-bit registers""" + from . import ISA + + return self.sim is not None and self.sim.get_isa() == ISA.MIPS def do_regs(self, arg: str) -> None: """Display all registers @@ -677,7 +724,7 @@ def do_regs(self, arg: str) -> None: Options (override current settings for this call only): hex - Show values in hexadecimal - decimal - Show values in decimal + decimal - Show values in signed decimal binary - Show values in binary default - Use default leading zeros (show for hex/binary, dot for decimal) show - Show all leading zeros @@ -761,7 +808,8 @@ def do_regs(self, arg: str) -> None: star = ( " ★ " if ( - self.prev_regs is not None + self.show_reg_changes + and self.prev_regs is not None and reg_num < len(self.prev_regs) and self.prev_regs[reg_num] != value ) @@ -816,7 +864,7 @@ def do_mem(self, arg: str) -> None: """Display memory contents in hex dump format Usage: - mem [length] + mem [length] Displays memory contents starting at the given address or section in hexadecimal format with ASCII sidebar. Default length is 256 @@ -824,6 +872,7 @@ def do_mem(self, arg: str) -> None: Arguments: address - Memory address in hex (0x...) or decimal + symbol - Label name from the symbol table (e.g., my_array) section - ELF section name (e.g., .text, .data, .rodata) length - Number of bytes to display (optional, default=256) @@ -832,6 +881,7 @@ def do_mem(self, arg: str) -> None: mem .data # Show .data section mem .rodata # Show read-only data section mem 0x80000000 64 # Show 64 bytes + mem my_array 32 # Show 32 bytes at label my_array Common Sections: .text - Executable code @@ -906,12 +956,14 @@ def do_mem(self, arg: str) -> None: self.print_error(f"Error reading section: {e}") return else: - # Parse as address - try: - addr = int(addr_or_section, 0) # Auto-detect base (0x for hex, etc.) - except ValueError: - self.print_error(f'Error: Invalid address "{addr_or_section}".') + # Parse as address or symbol name + parsed = self._parse_address(addr_or_section) + if parsed is None: + self.print_error( + f'Error: "{addr_or_section}" is not a valid address or known symbol.' + ) return + addr = parsed # Read and display memory try: @@ -950,14 +1002,14 @@ def do_disasm(self, arg: str) -> None: """Disassemble instructions at address Usage: - disasm [address|pc] [count] + disasm [address|symbol|pc] [count] Disassembles instructions starting at the given address. If no address is specified, defaults to the current PC. Default count is 10 instructions if not specified. Arguments: - address - Memory address in hex (0x...) or 'pc' for current PC + address - Memory address in hex (0x...), a symbol name, or 'pc' count - Number of instructions to disassemble (optional, default=10) Aliases: @@ -970,10 +1022,11 @@ def do_disasm(self, arg: str) -> None: disasm 0x80000000 # Disassemble 10 instructions disasm 0x80000000 5 # Disassemble 5 instructions d 0x80000000 # Using alias + disasm fibonacci # Disassemble a function by name Tips: - - Use 'pc' command to see current program counter value - - Instruction sizes: RISC-V/ARM64 = 4 bytes, x86-64 = variable + - '>' marks the instruction at the current PC + - Instruction sizes: RISC-V/ARM64/MIPS = 4 bytes, x86-64 = variable - Use 'mem ' to see raw instruction bytes """ parts = arg.split() if arg else [] @@ -996,12 +1049,12 @@ def do_disasm(self, arg: str) -> None: self.print_error(f'Error: Invalid count "{parts[1]}".') return else: - # Parse address - try: - addr = int(parts[0], 0) - except ValueError: - self.print_error(f'Error: Invalid address "{parts[0]}".') + # Parse address or symbol name + parsed = self._parse_address(parts[0]) + if parsed is None: + self.print_error(f'Error: "{parts[0]}" is not a valid address or known symbol.') return + addr = parsed # Parse count (default 10) count = 10 @@ -1015,16 +1068,21 @@ def do_disasm(self, arg: str) -> None: self.print_error(f'Error: Invalid count "{parts[1]}".') return - # Disassemble instructions + # Disassemble instructions (x86-64 instructions vary in length, so + # advance by each instruction's actual size) self._print_block_start() - for i in range(count): + instr_addr = addr + for _ in range(count): try: - instr_addr = addr + (i * 4) - disasm = self.sim.disasm(instr_addr) - print(f"[{instr_addr:#010x}] {disasm}", file=self.stdout) + disasm, size = self.sim.disasm_with_size(instr_addr) except Exception as e: print(f"[{instr_addr:#010x}] ", file=self.stdout) break + marker = ">" if instr_addr == self.sim.get_pc() else " " + print(f"[{instr_addr:#010x}]{marker} {disasm}", file=self.stdout) + if disasm == "": + break + instr_addr += size self._print_block_end() def do_list(self, arg: str) -> None: @@ -1049,7 +1107,7 @@ def do_list(self, arg: str) -> None: l # Using alias Tips: - - Compile with 'as -g' to include debug symbols + - Assemble with 'mapachespim-as -g' to include debug info - Source file must be in same directory as ELF file - Shows 10 lines by default - Current PC is marked with '# <-- PC: 0xXXXXXXXX' @@ -1065,9 +1123,8 @@ def do_list(self, arg: str) -> None: if not self.source_info.has_debug_info: self._print_block_start() print("No source information available.", file=self.stdout) - print("Compile your program with debug symbols (use -g flag):", file=self.stdout) - print(" as -g -o program.o program.s", file=self.stdout) - print(" ld -o program program.o", file=self.stdout) + print("Assemble your program with debug info (use the -g flag):", file=self.stdout) + print(" mapachespim-as -g program.s -o program", file=self.stdout) self._print_block_end() return @@ -1146,6 +1203,17 @@ def _show_source_lines( # --- Breakpoints --- + def _parse_address(self, text: str) -> Optional[int]: + """Parse a symbol name or a numeric address (hex or decimal)""" + if self.loaded_file: + addr = self.sim.lookup_symbol(text) + if addr is not None: + return addr + try: + return int(text, 0) + except ValueError: + return None + def do_break(self, arg: str) -> None: """Set a breakpoint at an address or symbol @@ -1182,21 +1250,15 @@ def do_break(self, arg: str) -> None: self.print_error("Error: Please specify an address or symbol name.") return - # First try to look up as symbol name - if self.loaded_file: - addr = self.sim.lookup_symbol(arg) - if addr is not None: - self.breakpoints.add(addr) - print(f"Breakpoint set at {arg} ({addr:#010x})", file=self.stdout) - return - - # If not a symbol, try to parse as address - try: - addr = int(arg, 0) - self.breakpoints.add(addr) - print(f"Breakpoint set at {addr:#010x}", file=self.stdout) - except ValueError: + addr = self._parse_address(arg) + if addr is None: self.print_error(f'Error: "{arg}" is not a valid address or known symbol.') + return + self.breakpoints.add(addr) + if self.loaded_file and self.sim.lookup_symbol(arg) is not None: + print(f"Breakpoint set at {arg} ({addr:#010x})", file=self.stdout) + else: + print(f"Breakpoint set at {addr:#010x}", file=self.stdout) def do_info(self, arg: str) -> None: """Show information about simulator state @@ -1322,13 +1384,14 @@ def do_delete(self, arg: str) -> None: """Delete a specific breakpoint Usage: - delete
+ delete Removes the breakpoint at the specified address. If no breakpoint exists at that address, a message is displayed. Arguments: address - Memory address in hex (0x...) or decimal + symbol - Function or label name used with 'break' Examples: break 0x80000010 # Set a breakpoint @@ -1339,21 +1402,20 @@ def do_delete(self, arg: str) -> None: Tips: - Use 'info breakpoints' to see all addresses with breakpoints - Use 'clear' to remove all breakpoints at once - - Address must match exactly (including 0x prefix if used) """ if not arg: self.print_error("Error: Please specify an address.") return - try: - addr = int(arg, 0) - if addr in self.breakpoints: - self.breakpoints.remove(addr) - print(f"Breakpoint removed at {addr:#018x}", file=self.stdout) - else: - print(f"No breakpoint at {addr:#018x}", file=self.stdout) - except ValueError: - self.print_error(f'Error: Invalid address "{arg}".') + addr = self._parse_address(arg) + if addr is None: + self.print_error(f'Error: "{arg}" is not a valid address or known symbol.') + return + if addr in self.breakpoints: + self.breakpoints.remove(addr) + print(f"Breakpoint removed at {addr:#018x}", file=self.stdout) + else: + print(f"No breakpoint at {addr:#018x}", file=self.stdout) def do_clear(self, arg: str) -> None: """Clear all breakpoints @@ -1421,7 +1483,7 @@ def do_set(self, arg: str) -> None: set # Show all current settings Options: - show-changes [on|off] - Show register changes after each step + show-changes [on|off] - Mark changed registers with ★ in regs regs-base [hex|decimal|binary] - Default format for register values regs-leading-zeros [default|show|cut|dot] - How to display leading zeros output-spacing [normal|compact] - Spacing around multi-line output @@ -1689,7 +1751,7 @@ def complete_set(self, text: str, line: str, begidx: int, endidx: int) -> List[s parts = line.split() if len(parts) <= 2: # Completing option name - options = ["show-changes", "regs-base", "regs-leading-zeros"] + options = ["show-changes", "regs-base", "regs-leading-zeros", "output-spacing"] if text: return [o for o in options if o.startswith(text)] return options @@ -1702,6 +1764,8 @@ def complete_set(self, text: str, line: str, begidx: int, endidx: int) -> List[s values = ["hex", "decimal", "binary"] elif option == "regs-leading-zeros": values = ["default", "show", "cut", "dot"] + elif option == "output-spacing": + values = ["normal", "compact"] else: return [] if text: @@ -1770,23 +1834,104 @@ def do_quickstart(self, arg: str) -> None: self._print_block_end() +DEFAULT_MAX_STEPS = 10_000_000 + + +def _execute(console: MapacheSPIMConsole, path: str, max_steps: int, verbose: bool) -> int: + """Run a program non-interactively and return the process exit status. + + The status is the program's exit code if it exits normally, 1 if the file + cannot be loaded or execution fails, and 124 (like ``timeout``) if the + instruction limit is reached. + """ + import io + + if not Path(path).exists(): + print(f"Error: File '{path}' not found", file=sys.stderr) + return 1 + + # Suppress console output during load (unless verbose) + captured_output = io.StringIO() + original_stdout = console.stdout + if not verbose: + console.stdout = captured_output + console.onecmd(f"load {path}") + console.stdout = original_stdout + + if not console.loaded_file: + captured = captured_output.getvalue().strip() + print(captured or f"Error: Failed to load '{path}'", file=sys.stderr) + return 1 + + sim = console.sim + steps = 0 + pc = sim.get_pc() + reason: Optional[str] = None + while steps < max_steps: + pc = sim.get_pc() + result = sim.step() + steps += 1 + done, reason = sim.check_termination(result) + if done: + break + else: + reason = None + + # Diagnostics go to stderr so stdout contains only the program's output + if sim.output_needs_newline: + sys.stdout.flush() + print(file=sys.stderr) + if verbose: + print(f"Program completed in {steps} steps", file=sys.stderr) + + if reason == "syscall_exit": + return sim.exit_code or 0 + if reason in ("halt", "tohost"): + return 0 + if reason == "error": + print(f"Error: {sim.last_error or 'execution error'}", file=sys.stderr) + print(f" {console._format_instruction(pc)}", file=sys.stderr) + return 1 + print( + f"Error: program did not exit within {max_steps} instructions " + "(infinite loop? use --max-steps to raise the limit)", + file=sys.stderr, + ) + return 124 + + def main() -> None: """Entry point for the console""" import argparse + from . import __version__ + parser = argparse.ArgumentParser( - description="MapacheSPIM - Interactive Multi-ISA Simulator (RISC-V, ARM64, x86-64)" + prog="mapachespim", + description="MapacheSPIM - Interactive Multi-ISA Simulator (RISC-V, ARM64, x86-64, MIPS32)", ) - parser.add_argument("file", nargs="?", help="ELF file to load on startup") + parser.add_argument("file", nargs="?", help="Program to load on startup") + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") parser.add_argument( "-v", "--verbose", action="store_true", help="Verbose mode (show extra messages)" ) parser.add_argument( - "-e", "--execute", action="store_true", - help="Execute program and exit (no interactive REPL)" + "-e", + "--execute", + action="store_true", + help="Run the program and exit with its exit code (no interactive console)", + ) + parser.add_argument( + "--max-steps", + type=int, + default=DEFAULT_MAX_STEPS, + metavar="N", + help=f"With -e, stop after N instructions (default {DEFAULT_MAX_STEPS:,})", ) args = parser.parse_args() + if args.max_steps < 1: + parser.error("--max-steps must be at least 1") # Create console console = MapacheSPIMConsole(verbose=args.verbose) @@ -1796,41 +1941,7 @@ def main() -> None: if not args.file: print("Error: -e/--execute requires a file argument", file=sys.stderr) sys.exit(1) - - # Check if file exists before loading - from pathlib import Path - if not Path(args.file).exists(): - print(f"Error: File '{args.file}' not found", file=sys.stderr) - sys.exit(1) - - # Suppress console output during load (unless verbose) - import io - captured_output = io.StringIO() - original_stdout = console.stdout - if not args.verbose: - console.stdout = captured_output - - console.onecmd(f"load {args.file}") - - console.stdout = original_stdout - - # Check if load succeeded - if not console.loaded_file: - # Show any captured error output - captured = captured_output.getvalue() - if captured.strip(): - print(captured.strip(), file=sys.stderr) - else: - print(f"Error: Failed to load '{args.file}'", file=sys.stderr) - sys.exit(1) - - if args.verbose: - print(captured_output.getvalue(), end='') - - steps = console.sim.run(max_steps=100000) - if args.verbose: - print(f"\nProgram completed in {steps} steps", file=sys.stderr) - sys.exit(0) + sys.exit(_execute(console, args.file, args.max_steps, args.verbose)) # Interactive mode: start REPL if args.file: diff --git a/mapachespim/memory_map.py b/mapachespim/memory_map.py new file mode 100644 index 0000000..856e6d6 --- /dev/null +++ b/mapachespim/memory_map.py @@ -0,0 +1,133 @@ +""" +ISA-specific memory layouts for MapacheSPIM. + +Each ISA has different conventional memory layouts for bare-metal programs. +The assembler places sections at these addresses and the simulator uses the +same values for its default memory map and initial stack pointer, so this +module is the single source of truth for both. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict + + +@dataclass(frozen=True) +class MemoryLayout: + """Memory layout configuration for an ISA.""" + + text_base: int + """Base address for .text (code) section.""" + + data_base: int + """Base address for .data section.""" + + rodata_base: int + """Base address for .rodata section.""" + + bss_base: int + """Base address for .bss section.""" + + stack_top: int + """Address just above the stack region (stack grows down). + + The simulator maps ``[stack_top - stack_size, stack_top)`` and starts the + stack pointer at ``stack_top - 8``.""" + + stack_size: int + """Size of stack region.""" + + is_64bit: bool + """True for 64-bit ISA, False for 32-bit.""" + + is_little_endian: bool + """True for little-endian, False for big-endian.""" + + def __str__(self) -> str: + bits = "64-bit" if self.is_64bit else "32-bit" + endian = "little-endian" if self.is_little_endian else "big-endian" + return f"MemoryLayout({bits}, {endian}, text=0x{self.text_base:x})" + + +# ISA-specific memory layouts +# These match the linker.ld files in examples/*/ + +RISCV64_LAYOUT = MemoryLayout( + text_base=0x80000000, # Traditional RISC-V bare-metal + data_base=0x80100000, + rodata_base=0x80080000, + bss_base=0x80180000, + stack_top=0x83F00000, + stack_size=0x100000, # 1MB stack + is_64bit=True, + is_little_endian=True, +) + +ARM64_LAYOUT = MemoryLayout( + text_base=0x10000000, # Unicorn default for ARM64 + data_base=0x10100000, + rodata_base=0x10080000, + bss_base=0x10180000, + stack_top=0x13F00000, + stack_size=0x100000, # 1MB stack + is_64bit=True, + is_little_endian=True, +) + +X86_64_LAYOUT = MemoryLayout( + text_base=0x400000, # Linux-like layout + data_base=0x500000, + rodata_base=0x480000, + bss_base=0x580000, + stack_top=0x7FFFFFFFE000, # Near top of user space + stack_size=0x100000, # 1MB stack + is_64bit=True, + is_little_endian=True, +) + +MIPS32_LAYOUT = MemoryLayout( + text_base=0x00400000, # Traditional MIPS user space + data_base=0x10000000, + rodata_base=0x00480000, + bss_base=0x10080000, + stack_top=0x80000000, # Just below kernel space (kseg0) + stack_size=0x100000, # 1MB stack + is_64bit=False, + is_little_endian=False, # MIPS is big-endian in MapacheSPIM +) + +# Map of ISA names to layouts +MEMORY_LAYOUTS: Dict[str, MemoryLayout] = { + "riscv64": RISCV64_LAYOUT, + "riscv": RISCV64_LAYOUT, # Alias + "arm64": ARM64_LAYOUT, + "aarch64": ARM64_LAYOUT, # Alias + "x86_64": X86_64_LAYOUT, + "x86-64": X86_64_LAYOUT, # Alias + "x64": X86_64_LAYOUT, # Alias + "mips32": MIPS32_LAYOUT, + "mips": MIPS32_LAYOUT, # Alias +} + + +def get_layout(isa: str) -> MemoryLayout: + """ + Get the memory layout for an ISA. + + Args: + isa: ISA name (e.g., "riscv64", "arm64", "x86_64", "mips32") + + Returns: + MemoryLayout for the specified ISA. + + Raises: + ValueError: If ISA is not recognized. + """ + isa_lower = isa.lower().replace("-", "_") + if isa_lower not in MEMORY_LAYOUTS: + valid = sorted(set(MEMORY_LAYOUTS.keys())) + raise ValueError( + f"Unknown ISA: {isa!r}. Valid options: {', '.join(valid)}" + ) + return MEMORY_LAYOUTS[isa_lower] diff --git a/mapachespim/toolchain/memory_map.py b/mapachespim/toolchain/memory_map.py index 54943c9..3e5c3da 100644 --- a/mapachespim/toolchain/memory_map.py +++ b/mapachespim/toolchain/memory_map.py @@ -1,128 +1,26 @@ """ -ISA-specific memory layouts for MapacheSPIM toolchain. +Re-export of :mod:`mapachespim.memory_map`. -Each ISA has different conventional memory layouts for bare-metal programs. -These layouts match the linker scripts used by the existing examples. +The memory layout is shared by the assembler and the simulator, so it lives +at the package top level. This module is kept for backward compatibility. """ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Dict - - -@dataclass(frozen=True) -class MemoryLayout: - """Memory layout configuration for an ISA.""" - - text_base: int - """Base address for .text (code) section.""" - - data_base: int - """Base address for .data section.""" - - rodata_base: int - """Base address for .rodata section.""" - - bss_base: int - """Base address for .bss section.""" - - stack_top: int - """Top of stack (stack grows down).""" - - stack_size: int - """Size of stack region.""" - - is_64bit: bool - """True for 64-bit ISA, False for 32-bit.""" - - is_little_endian: bool - """True for little-endian, False for big-endian.""" - - def __str__(self) -> str: - bits = "64-bit" if self.is_64bit else "32-bit" - endian = "little-endian" if self.is_little_endian else "big-endian" - return f"MemoryLayout({bits}, {endian}, text=0x{self.text_base:x})" - - -# ISA-specific memory layouts -# These match the linker.ld files in examples/*/ - -RISCV64_LAYOUT = MemoryLayout( - text_base=0x80000000, # Traditional RISC-V bare-metal - data_base=0x80100000, - rodata_base=0x80080000, - bss_base=0x80180000, - stack_top=0x83F00000, - stack_size=0x100000, # 1MB stack - is_64bit=True, - is_little_endian=True, -) - -ARM64_LAYOUT = MemoryLayout( - text_base=0x10000000, # Unicorn default for ARM64 - data_base=0x10100000, - rodata_base=0x10080000, - bss_base=0x10180000, - stack_top=0x13F00000, - stack_size=0x100000, # 1MB stack - is_64bit=True, - is_little_endian=True, +from ..memory_map import ( + ARM64_LAYOUT, + MEMORY_LAYOUTS, + MIPS32_LAYOUT, + RISCV64_LAYOUT, + X86_64_LAYOUT, + MemoryLayout, + get_layout, ) -X86_64_LAYOUT = MemoryLayout( - text_base=0x400000, # Linux-like layout - data_base=0x500000, - rodata_base=0x480000, - bss_base=0x580000, - stack_top=0x7FFFFFFFE000, # Near top of user space - stack_size=0x100000, # 1MB stack - is_64bit=True, - is_little_endian=True, -) - -MIPS32_LAYOUT = MemoryLayout( - text_base=0x00400000, # Traditional MIPS user space - data_base=0x10000000, - rodata_base=0x00480000, - bss_base=0x10080000, - stack_top=0x7FFFFFF0, - stack_size=0x100000, # 1MB stack - is_64bit=False, - is_little_endian=False, # MIPS is big-endian in MapacheSPIM -) - -# Map of ISA names to layouts -MEMORY_LAYOUTS: Dict[str, MemoryLayout] = { - "riscv64": RISCV64_LAYOUT, - "riscv": RISCV64_LAYOUT, # Alias - "arm64": ARM64_LAYOUT, - "aarch64": ARM64_LAYOUT, # Alias - "x86_64": X86_64_LAYOUT, - "x86-64": X86_64_LAYOUT, # Alias - "x64": X86_64_LAYOUT, # Alias - "mips32": MIPS32_LAYOUT, - "mips": MIPS32_LAYOUT, # Alias -} - - -def get_layout(isa: str) -> MemoryLayout: - """ - Get the memory layout for an ISA. - - Args: - isa: ISA name (e.g., "riscv64", "arm64", "x86_64", "mips32") - - Returns: - MemoryLayout for the specified ISA. - - Raises: - ValueError: If ISA is not recognized. - """ - isa_lower = isa.lower().replace("-", "_") - if isa_lower not in MEMORY_LAYOUTS: - valid = sorted(set(MEMORY_LAYOUTS.keys())) - raise ValueError( - f"Unknown ISA: {isa!r}. Valid options: {', '.join(valid)}" - ) - return MEMORY_LAYOUTS[isa_lower] +__all__ = [ + "ARM64_LAYOUT", + "MEMORY_LAYOUTS", + "MIPS32_LAYOUT", + "RISCV64_LAYOUT", + "X86_64_LAYOUT", + "MemoryLayout", + "get_layout", +] diff --git a/mapachespim/unicorn_backend.py b/mapachespim/unicorn_backend.py index 4acbdb9..13215dc 100644 --- a/mapachespim/unicorn_backend.py +++ b/mapachespim/unicorn_backend.py @@ -1,15 +1,17 @@ """ Python bindings for Unicorn Engine-based simulator -Provides a Pythonic interface to the Unicorn CPU emulator for RISC-V and ARM. -This replaces the SAIL-based backend with a more stable, battle-tested emulation engine. +Provides a Pythonic interface to the Unicorn CPU emulator for RISC-V, ARM64, +x86-64, and MIPS32. """ from __future__ import annotations +import ctypes import struct +import sys from enum import IntEnum -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Dict, List, Optional, TextIO, Tuple, Union if TYPE_CHECKING: from unicorn import Uc @@ -22,6 +24,9 @@ UC_ARCH_X86, UC_HOOK_CODE, UC_HOOK_MEM_UNMAPPED, + UC_MEM_FETCH_UNMAPPED, + UC_MEM_READ_UNMAPPED, + UC_MEM_WRITE_UNMAPPED, UC_MODE_32, UC_MODE_64, UC_MODE_ARM, @@ -29,8 +34,6 @@ UC_MODE_MIPS32, UC_MODE_RISCV64, UC_PROT_ALL, - UC_PROT_READ, - UC_PROT_WRITE, Uc, UcError, ) @@ -81,6 +84,7 @@ from .elf_loader import ISA as ELF_ISA from .elf_loader import ELFSegment, load_elf_file +from .memory_map import MemoryLayout, get_layout class ISA(IntEnum): @@ -103,12 +107,65 @@ class StepResult(IntEnum): ERROR = -1 -# Memory layout constants -# Stack grows downward from 0x83F00000 (this is the TOP of the stack) +# RISC-V memory layout constants (kept for backward compatibility; see memory_map) STACK_TOP = 0x83F00000 STACK_SIZE = 0x100000 # 1MB stack STACK_ADDR = STACK_TOP - STACK_SIZE # Bottom of stack +PAGE_SIZE = 0x1000 + +_UNMAPPED_ACCESS_KINDS = { + UC_MEM_READ_UNMAPPED: "Read from", + UC_MEM_WRITE_UNMAPPED: "Write to", + UC_MEM_FETCH_UNMAPPED: "Instruction fetch from", +} + +# Size of the region mapped at the start of each ISA's text/data areas so that +# small hand-written programs work without an ELF file. +DEFAULT_REGION_SIZE = 0x400000 + +_ISA_LAYOUT_NAMES = { + 0: "riscv64", # ISA.RISCV + 1: "arm64", # ISA.ARM + 2: "x86_64", # ISA.X86_64 + 3: "mips32", # ISA.MIPS +} + +# SPIM-compatible syscall numbers +SYSCALL_PRINT_INT = 1 +SYSCALL_PRINT_STRING = 4 +SYSCALL_READ_INT = 5 +SYSCALL_EXIT = 10 +SYSCALL_PRINT_CHAR = 11 +SYSCALL_READ_CHAR = 12 +SYSCALL_EXIT2 = 17 +SYSCALL_EXIT_CODE = 93 + +SUPPORTED_SYSCALLS = ( + SYSCALL_PRINT_INT, + SYSCALL_PRINT_STRING, + SYSCALL_READ_INT, + SYSCALL_EXIT, + SYSCALL_PRINT_CHAR, + SYSCALL_READ_CHAR, + SYSCALL_EXIT2, + SYSCALL_EXIT_CODE, +) + + +def _describe_uc_error(e: UcError) -> str: + """Turn a Unicorn error into a message a student can act on""" + text = str(e) + if "INSN_INVALID" in text: + return "Invalid instruction" + if "FETCH_PROT" in text or "FETCH_UNALIGNED" in text: + return "Cannot execute code at this address" + if "UNALIGNED" in text: + return "Unaligned memory access" + if "EXCEPTION" in text: + return "CPU exception (unhandled trap)" + return text + class ISAConfig: """Base class for ISA-specific configuration""" @@ -409,22 +466,56 @@ def disassemble_one(self, simulator: UnicornSimulator, addr: int) -> str: return f"{instr.mnemonic} {instr.op_str}".strip() # If disassembly failed, show raw bytes - if self._isa == ISA.X86_64: - # Show first 4 bytes for x86 - word = int.from_bytes(code[:4], byteorder="little") - else: - word = int.from_bytes(code, byteorder="little") + byteorder = "big" if self._isa == ISA.MIPS else "little" + word = int.from_bytes(code[:4], byteorder=byteorder) return f".word 0x{word:08x}" + def disassemble_with_size(self, simulator: UnicornSimulator, addr: int) -> Tuple[str, int]: + """ + Disassemble one instruction and report its length in bytes. + + x86-64 instructions are variable length, so callers that walk a range + of instructions must use the returned size to find the next one. + + Returns: + (text, size) - size is 4 for undecodable fixed-width instructions + and 1 for undecodable x86 bytes. + """ + read_size = 15 if self._isa == ISA.X86_64 else 4 + code = b"" + # Near the end of a mapped region fewer than 15 bytes may be readable + while read_size > 0: + try: + code = simulator.read_mem(addr, read_size) + break + except Exception: + read_size -= 1 + if not code: + return ("", 4) + + for instr in self._cs.disasm(code, addr, count=1): + return (f"{instr.mnemonic} {instr.op_str}".strip(), instr.size) + + if self._isa == ISA.X86_64: + return (f".byte 0x{code[0]:02x}", 1) + byteorder = "big" if self._isa == ISA.MIPS else "little" + return (f".word 0x{int.from_bytes(code[:4], byteorder=byteorder):08x}", 4) + class UnicornSimulator: """ - Unicorn Engine-based CPU emulator for RISC-V and ARM + Unicorn Engine-based CPU emulator for RISC-V, ARM64, x86-64, and MIPS32 - Provides step-by-step execution, register/memory access, and state inspection - compatible with the original SAIL-based backend. + Provides step-by-step execution, register/memory access, and state inspection. + + Program I/O from syscalls goes to ``stdout`` and comes from ``stdin``. Both + default to the process streams (looked up at call time, so pytest's capsys + and similar redirection work) and can be replaced, e.g. for autograding. """ + stdout: Optional[TextIO] + stdin: Optional[TextIO] + _isa: Optional[ISA] _config: Optional[ISAConfig] _uc: Optional[Uc] @@ -434,17 +525,26 @@ class UnicornSimulator: _entry_point: Optional[int] _pending_syscall: bool _last_error: Optional[str] + _elf_path: Optional[str] + _explicit_isa: bool + _mem_buffers: List[ctypes.Array] + _exited: bool + _exit_code: Optional[int] + _output_needs_newline: bool + _input_buffer: str + _tohost_addr: Optional[int] def __init__(self, isa: Optional[ISA] = None, config_file: Optional[str] = None) -> None: """ Initialize the simulator Args: - isa (ISA, optional): ISA to use (ISA.RISCV or ISA.ARM). + isa (ISA, optional): ISA to use (ISA.RISCV, ISA.ARM, ISA.X86_64, ISA.MIPS). If None, will be auto-detected from ELF file during load_elf() config_file (str, optional): Not used by Unicorn backend (for compatibility) """ self._isa = isa + self._explicit_isa = isa is not None self._config = None self._uc = None self._symbols = {} @@ -453,6 +553,15 @@ def __init__(self, isa: Optional[ISA] = None, config_file: Optional[str] = None) self._entry_point = None self._pending_syscall = False self._last_error = None + self._elf_path = None + self._mem_buffers = [] + self._exited = False + self._exit_code = None + self._output_needs_newline = False + self._input_buffer = "" + self._tohost_addr = None + self.stdout = None + self.stdin = None # If ISA is specified, initialize Unicorn now if isa is not None: @@ -476,6 +585,16 @@ def _init_unicorn(self, isa: ISA) -> None: self._uc = Uc(self._config.arch, self._config.mode) except UcError as e: raise RuntimeError(f"Failed to initialize Unicorn: {e}") + self._isa = isa + self._mem_buffers = [] + self._exited = False + self._exit_code = None + self._last_error = None + self._input_buffer = "" + self._pending_syscall = False + self._symbols = {} + self._addr_to_symbol = {} + self._tohost_addr = None # Create disassembler self._disasm = Disassembler(isa) @@ -507,138 +626,98 @@ def _install_unmapped_handler(self) -> None: def unmapped_handler( uc: Uc, access: int, address: int, size: int, value: int, user_data: Optional[object] ) -> bool: - self._last_error = f"Unmapped memory access at 0x{address:x}" + kind = _UNMAPPED_ACCESS_KINDS.get(access, "access") + self._last_error = f"{kind} unmapped address 0x{address:x}" return False # Don't handle it, let it error self._uc.hook_add(UC_HOOK_MEM_UNMAPPED, unmapped_handler) - def _map_default_memory(self) -> None: - """Map default memory regions for tests without ELF files""" - if self._config.arch == UC_ARCH_ARM64: - # Map 1MB at address 0 for ARM tests - try: - self._uc.mem_map(0x0, 0x100000, UC_PROT_ALL) - except UcError: - pass - elif self._config.arch == UC_ARCH_X86: - # Map main region (4MB from 0x400000) for x86-64 executables - try: - self._uc.mem_map(0x400000, 0x400000, UC_PROT_ALL) - except UcError: - pass - # Map stack region (2MB ending near 0x7FFFFFFFFFFF - typical Linux) - try: - self._uc.mem_map(0x7FFFFE00000, 0x200000, UC_PROT_ALL) - except UcError: - pass - elif self._config.arch == UC_ARCH_MIPS: - # Map main region (4MB from 0x00400000) for MIPS - # Note: 0x00400000 is the traditional MIPS user-space .text address - # 0x80000000 is kernel space (kseg0) and has special handling - try: - self._uc.mem_map(0x00400000, 0x400000, UC_PROT_ALL) - except UcError: - pass - # Map data region (4MB from 0x10000000) - traditional MIPS .data address - try: - self._uc.mem_map(0x10000000, 0x400000, UC_PROT_ALL) - except UcError: - pass - # Map stack region (near top of user space) - try: - self._uc.mem_map(0x7FE00000, 0x200000, UC_PROT_ALL) - except UcError: - pass - else: # RISC-V - # Map main region (4MB from 0x80000000) - try: - self._uc.mem_map(0x80000000, 0x400000, UC_PROT_ALL) - except UcError: - pass + @property + def layout(self) -> MemoryLayout: + """Memory layout for the current ISA""" + if self._isa is None: + raise RuntimeError("Simulator ISA not set") + return get_layout(_ISA_LAYOUT_NAMES[int(self._isa)]) - # Also map sign-extended version for RV64 lui addresses - # When lui loads values like 0x80100, it sign-extends to 0xFFFFFFFF80100000 - try: - self._uc.mem_map(0xFFFFFFFF80000000, 0x400000, UC_PROT_ALL) - except UcError: - pass + def _map_new(self, addr: int, size: int) -> None: + """Map a fresh, page-aligned region that does not overlap existing maps. - # Also map stack region for tests (include STACK_TOP itself) - try: - self._uc.mem_map(STACK_ADDR, STACK_SIZE + 0x100000, UC_PROT_ALL) - except UcError: - pass + On RV64, ``lui`` sign-extends, so an address like 0x80100000 built with + lui+addi becomes 0xFFFFFFFF80100000. Real hardware with a 32-bit + physical address space treats these as the same location, so any + region in [0x80000000, 0x100000000) is also mapped at its sign-extended + alias, backed by the same host memory. + """ + if self._isa == ISA.RISCV and 0x80000000 <= addr and addr + size <= 0x100000000: + buf = ctypes.create_string_buffer(size) + self._mem_buffers.append(buf) # keep host memory alive + ptr = ctypes.addressof(buf) + self._uc.mem_map_ptr(addr, size, UC_PROT_ALL, ptr) + self._uc.mem_map_ptr(addr | 0xFFFFFFFF00000000, size, UC_PROT_ALL, ptr) + else: + self._uc.mem_map(addr, size, UC_PROT_ALL) + + def _ensure_mapped(self, addr: int, size: int) -> None: + """Map every page in [addr, addr+size) that is not already mapped.""" + start = addr & ~(PAGE_SIZE - 1) + end = (addr + size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1) + if end <= start: + return + + cur = start + # mem_regions() yields (begin, end_inclusive, perms) + for begin, last, _perms in sorted(self._uc.mem_regions()): + region_end = last + 1 + if region_end <= cur or begin >= end: + continue + if begin > cur: + self._map_new(cur, begin - cur) + cur = max(cur, region_end) + if cur >= end: + return + if cur < end: + self._map_new(cur, end - cur) + + def _map_default_memory(self) -> None: + """Map the ISA's text/data area and stack so programs can run without an ELF. + + Address 0 is deliberately left unmapped on every ISA so that null + pointer dereferences fault instead of silently succeeding. + """ + layout = self.layout + self._ensure_mapped(layout.text_base, DEFAULT_REGION_SIZE) + if not layout.text_base <= layout.data_base < layout.text_base + DEFAULT_REGION_SIZE: + self._ensure_mapped(layout.data_base, DEFAULT_REGION_SIZE) + stack_bottom = layout.stack_top - layout.stack_size + stack_span = layout.stack_size + if self._isa == ISA.RISCV: + # RISC-V has historically mapped one page above stack_top too + stack_span += PAGE_SIZE + self._ensure_mapped(stack_bottom, stack_span) + # Start at the conventional code address rather than 0 + self.set_pc(layout.text_base) def _map_memory_for_segments(self, segments: List[ELFSegment]) -> None: - """Map memory regions for ELF segments with page alignment""" + """Map memory regions for ELF segments and copy in their contents""" for segment in segments: - # Calculate page-aligned region - page_size = 0x1000 # 4KB pages - page_aligned_addr = segment.vaddr & ~(page_size - 1) - end_addr = segment.vaddr + segment.memsz - page_aligned_end = (end_addr + page_size - 1) & ~(page_size - 1) - size = page_aligned_end - page_aligned_addr - - # Map memory with RWX permissions - try: - self._uc.mem_map(page_aligned_addr, size, UC_PROT_ALL) - except UcError: - # Region might overlap with already-mapped memory - # This is OK - Unicorn will handle it - pass + if segment.memsz == 0: + continue + self._ensure_mapped(segment.vaddr, segment.memsz) - # Write segment data if segment.data: self._uc.mem_write(segment.vaddr, segment.data) - # For RISC-V 64-bit, also write to sign-extended address - # This handles lui sign-extension when accessing addresses >= 0x80000000 - if self._isa == ISA.RISCV and segment.vaddr >= 0x80000000: - sign_extended_addr = segment.vaddr | 0xFFFFFFFF00000000 - try: - self._uc.mem_write(sign_extended_addr, segment.data) - except UcError: - pass # Memory might not be mapped there # Zero-fill BSS (if memsz > filesz) if segment.memsz > segment.filesz: bss_size = segment.memsz - segment.filesz bss_addr = segment.vaddr + segment.filesz self._uc.mem_write(bss_addr, b"\x00" * bss_size) - # Also zero-fill sign-extended BSS for RISC-V - if self._isa == ISA.RISCV and bss_addr >= 0x80000000: - sign_extended_bss = bss_addr | 0xFFFFFFFF00000000 - try: - self._uc.mem_write(sign_extended_bss, b"\x00" * bss_size) - except UcError: - pass def _setup_stack(self) -> None: - """Map and initialize stack memory""" - # MIPS requires stack in user segment (< 0x80000000) - # Addresses >= 0x80000000 are kernel-only in MIPS - if self._isa == ISA.MIPS: - mips_stack_top = 0x80000000 # Just below kernel space - mips_stack_size = 0x200000 # 2MB - mips_stack_addr = mips_stack_top - mips_stack_size - try: - self._uc.mem_map(mips_stack_addr, mips_stack_size, UC_PROT_READ | UC_PROT_WRITE) - except UcError: - pass # Already mapped in _map_default_memory - sp_reg = self._config.get_sp_reg() - self._uc.reg_write(sp_reg, mips_stack_top - 8) - return - - try: - self._uc.mem_map(STACK_ADDR, STACK_SIZE, UC_PROT_READ | UC_PROT_WRITE) - except UcError as e: - # Stack mapping can fail if already mapped - that's OK - if "already" not in str(e).lower() and "map" not in str(e).lower(): - raise RuntimeError(f"Failed to setup stack: {e}") - - # Set stack pointer to top of stack (grows downward) - # Note: Some programs set their own SP, which will override this - sp_reg = self._config.get_sp_reg() - self._uc.reg_write(sp_reg, STACK_TOP - 8) + """Map stack memory and point the stack pointer at its top""" + layout = self.layout + self._ensure_mapped(layout.stack_top - layout.stack_size, layout.stack_size) + self._uc.reg_write(self._config.get_sp_reg(), layout.stack_top - 8) def load_elf(self, elf_path: str) -> bool: """ @@ -665,14 +744,16 @@ def load_elf(self, elf_path: str) -> bool: else: raise RuntimeError(f"Unknown ISA in ELF file: {elf_path}") - # Initialize Unicorn if not already done - if self._isa is None: - self._isa = detected_isa - self._init_unicorn(detected_isa) - elif self._isa != detected_isa: + # A simulator created for a specific ISA only accepts that ISA; one + # created without an ISA adopts whatever each loaded ELF uses. + if self._explicit_isa and self._isa != detected_isa: raise RuntimeError( f"ELF ISA ({detected_isa.name}) doesn't match simulator ISA ({self._isa.name})" ) + # Start from a fresh machine so nothing from a previous program + # (memory, registers, exit state) leaks into this one. + self._init_unicorn(detected_isa) + self._elf_path = elf_path # Map memory for ELF segments self._map_memory_for_segments(elf_info.segments) @@ -689,6 +770,7 @@ def load_elf(self, elf_path: str) -> bool: # Build reverse mapping (addr -> symbol) self._addr_to_symbol = {addr: name for name, addr in self._symbols.items()} + self._tohost_addr = self._symbols.get("tohost") return True @@ -702,7 +784,12 @@ def step(self) -> StepResult: if self._uc is None: return StepResult.ERROR + # Once the program has exited there is nothing left to execute + if self._exited: + return StepResult.HALT + self._pending_syscall = False + self._last_error = None pc = self.get_pc() try: @@ -714,7 +801,13 @@ def step(self) -> StepResult: # The hook will stop execution if a syscall is detected self._uc.emu_start(pc, end_addr, count=1) except UcError as e: - self._last_error = f"Execution error at PC=0x{pc:x}: {e}" + # Prefer the specific description recorded by the unmapped-memory + # hook (it includes the faulting address) over Unicorn's message. + detail = self._last_error or _describe_uc_error(e) + self._last_error = f"{detail} (at PC=0x{pc:x})" + # Unicorn may leave PC anywhere after a fault; keep it on the + # faulting instruction so the user can inspect it. + self.set_pc(pc) return StepResult.ERROR # Check if we hit a syscall @@ -864,6 +957,18 @@ def disasm(self, addr: int) -> str: return self._disasm.disassemble_one(self, addr) + def disasm_with_size(self, addr: int) -> Tuple[str, int]: + """ + Disassemble instruction at address and return its size in bytes + + Returns: + tuple: (instruction text, instruction length in bytes) + """ + if self._disasm is None: + raise RuntimeError("Simulator not initialized") + + return self._disasm.disassemble_with_size(self, addr) + def get_symbols(self) -> Dict[str, int]: """ Get all symbols from the symbol table @@ -921,12 +1026,44 @@ def addr_to_symbol(self, addr: int) -> Tuple[Optional[str], Optional[int]]: return (None, None) def reset(self) -> None: - """Reset the simulator to initial state""" - # Unicorn doesn't have a native reset, so we reinitialize - if self._isa is not None: + """Reset the simulator to its initial state + + If an ELF file was loaded it is reloaded from disk, restoring memory, + registers, and the entry point, so the program can be run again. + """ + if self._elf_path is not None: + self.load_elf(self._elf_path) + elif self._isa is not None: + # Unicorn doesn't have a native reset, so we reinitialize self._init_unicorn(self._isa) - if self._entry_point is not None: - self.set_pc(self._entry_point) + + @property + def last_error(self) -> Optional[str]: + """Description of the most recent execution error, if any""" + return self._last_error + + @property + def exited(self) -> bool: + """True once the program has exited via an exit syscall""" + return self._exited + + @property + def exit_code(self) -> Optional[int]: + """Exit code passed to the exit syscall (None if not exited)""" + return self._exit_code + + @property + def output_needs_newline(self) -> bool: + """True if program output so far did not end with a newline + + Consoles use this to avoid printing their own messages on the same + line as program output. + """ + return self._output_needs_newline + + @output_needs_newline.setter + def output_needs_newline(self, value: bool) -> None: + self._output_needs_newline = value def get_all_regs(self) -> List[int]: """ @@ -1002,104 +1139,166 @@ def _get_syscall_regs(self) -> Tuple[int, int, int]: # RISC-V: a7=syscall# (x17), a0=arg0 (x10), return in a0 return (17, 10, 10) - def _handle_syscall(self) -> bool: + def _write_output(self, text: str) -> None: + """Write program output (from a print syscall)""" + if not text: + return + stream = self.stdout if self.stdout is not None else sys.stdout + stream.write(text) + stream.flush() + self._output_needs_newline = not text.endswith("\n") + + def _read_input_line(self) -> Optional[str]: + """Read one line of program input, without the newline (None on EOF)""" + # Make sure any prompt the program printed is visible first + out = self.stdout if self.stdout is not None else sys.stdout + out.flush() + # Pressing Enter echoes a newline, so the console is at a line start + self._output_needs_newline = False + if self.stdin is not None: + line = self.stdin.readline() + if line == "": + return None + return line.rstrip("\r\n") + try: + # input() gives line editing on interactive terminals + return input() + except EOFError: + return None + + def _signed(self, value: int) -> int: + """Interpret a register value as a signed integer of the ISA's width""" + bits = 32 if self._isa == ISA.MIPS else 64 + value &= (1 << bits) - 1 + if value & (1 << (bits - 1)): + value -= 1 << bits + return value + + def _read_c_string(self, addr: int, limit: int = 65536) -> str: + """Read a NUL-terminated string from simulated memory""" + data = bytearray() + while len(data) < limit: + # Read a chunk, shrinking near the end of a mapped region + chunk = b"" + size = 256 - (addr % 256) + while size > 0: + try: + chunk = self.read_mem(addr, size) + break + except RuntimeError: + size //= 2 + if not chunk: + if not data: + raise RuntimeError(f"print_string: address 0x{addr:x} is not mapped") + break + nul = chunk.find(0) + if nul >= 0: + data += chunk[:nul] + break + data += chunk + addr += len(chunk) + return data.decode("utf-8", errors="replace") + + def _handle_syscall(self) -> Optional[str]: """ Handle SPIM-compatible syscalls Returns: - bool: True if program should exit, False otherwise + None to continue, "syscall_exit" if the program exited, or "error" + (with ``last_error`` set) if the syscall could not be performed. """ syscall_reg, arg_reg, result_reg = self._get_syscall_regs() syscall_num = self.get_reg(syscall_reg) - if syscall_num == 1: - # print_int - Print integer in arg0 - value = self.get_reg(arg_reg) - # Convert to signed 64-bit for proper display - if value & (1 << 63): - value = value - (1 << 64) - print(value, end="") - - elif syscall_num == 4: - # print_string - Print null-terminated string at address in arg0 - addr = self.get_reg(arg_reg) - chars = [] - try: - while True: - byte = self.read_mem(addr, 1)[0] - if byte == 0: - break - chars.append(chr(byte)) - addr += 1 - if len(chars) > 4096: # Safety limit - break - print("".join(chars), end="") - except Exception: - pass + if syscall_num == SYSCALL_PRINT_INT: + self._write_output(str(self._signed(self.get_reg(arg_reg)))) - elif syscall_num == 5: - # read_int - Read integer from stdin, return in result reg + elif syscall_num == SYSCALL_PRINT_STRING: try: - value = int(input()) - self.set_reg(result_reg, value & 0xFFFFFFFFFFFFFFFF) - except Exception: - self.set_reg(result_reg, 0) - - elif syscall_num == 10: - # exit - Terminate program - return True - - elif syscall_num == 11: - # print_char - Print character in arg0 - char_code = self.get_reg(arg_reg) & 0xFF - print(chr(char_code), end="") - - elif syscall_num == 12: - # read_char - Read character from stdin, return in result reg + self._write_output(self._read_c_string(self.get_reg(arg_reg))) + except RuntimeError as e: + self._last_error = str(e) + return "error" + + elif syscall_num == SYSCALL_READ_INT: + # A partially consumed line (from read_char) is used first + if self._input_buffer.strip(): + line: Optional[str] = self._input_buffer + else: + line = self._read_input_line() + self._input_buffer = "" try: - char = input()[0] if input() else "\0" - self.set_reg(result_reg, ord(char)) - except Exception: - self.set_reg(result_reg, 0) + value = int((line or "").strip(), 0) + except ValueError: + value = 0 + self.set_reg(result_reg, value & 0xFFFFFFFFFFFFFFFF) + + elif syscall_num == SYSCALL_READ_CHAR: + # Characters come from a line buffer so "ab" yields a, b, \n + if not self._input_buffer: + line = self._read_input_line() + self._input_buffer = "" if line is None else line + "\n" + if self._input_buffer: + char, self._input_buffer = self._input_buffer[0], self._input_buffer[1:] + self.set_reg(result_reg, ord(char) & 0xFF) + else: + self.set_reg(result_reg, 0) # EOF + + elif syscall_num == SYSCALL_PRINT_CHAR: + self._write_output(chr(self.get_reg(arg_reg) & 0xFF)) + + elif syscall_num == SYSCALL_EXIT: + self._exited = True + self._exit_code = 0 + return "syscall_exit" + + elif syscall_num in (SYSCALL_EXIT2, SYSCALL_EXIT_CODE): + self._exited = True + self._exit_code = self._signed(self.get_reg(arg_reg)) & 0xFF + return "syscall_exit" - elif syscall_num == 93: - # exit_code - Exit with code in a0 - return True + else: + reg_name = self.get_reg_name(syscall_reg) + supported = ", ".join(str(n) for n in SUPPORTED_SYSCALLS) + self._last_error = ( + f"Unknown syscall number {self._signed(syscall_num)} in {reg_name} " + f"(supported: {supported})" + ) + return "error" - return False + return None def check_termination(self, step_result: StepResult) -> Tuple[bool, Optional[str]]: """ Check if program should terminate based on step result + Performs the syscall if ``step_result`` is SYSCALL. + Args: step_result: Result from step() Returns: - tuple: (should_terminate: bool, reason: str or None) + tuple: (should_terminate, reason) where reason is one of + "syscall_exit", "halt", "error", "tohost", or None """ - # Handle syscall - perform I/O and check for exit if step_result == StepResult.SYSCALL: - if self._handle_syscall(): - return (True, "syscall_exit") + reason = self._handle_syscall() + if reason is not None: + return (True, reason) - # Check for HALT if step_result == StepResult.HALT: return (True, "halt") - # Check for ERROR if step_result == StepResult.ERROR: return (True, "error") # Check for tohost write (HTIF mechanism) - tohost_addr = self.lookup_symbol("tohost") - if tohost_addr is not None: + if self._tohost_addr is not None: try: - tohost_bytes = self.read_mem(tohost_addr, 8) - tohost_value = int.from_bytes(tohost_bytes, byteorder="little", signed=False) - if tohost_value != 0: + tohost_bytes = self.read_mem(self._tohost_addr, 8) + if int.from_bytes(tohost_bytes, byteorder="little") != 0: return (True, "tohost") - except Exception: + except RuntimeError: pass return (False, None) diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 0000000..a9427ee --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,232 @@ +""" +Regression tests for simulator and console bugs. + +Programs here are written as raw machine code so the tests do not depend on +the assembler. +""" + +import io +import struct +import unittest +from pathlib import Path + +from mapachespim import ISA, Simulator, StepResult +from mapachespim.console import MapacheSPIMConsole + +EXAMPLES = Path(__file__).parent.parent / "examples" + +# RISC-V encodings +RV_ECALL = 0x00000073 + + +def rv_addi(rd: int, rs1: int, imm: int) -> int: + return ((imm & 0xFFF) << 20) | (rs1 << 15) | (0 << 12) | (rd << 7) | 0x13 + + +def load_words(sim: Simulator, words, big_endian: bool = False) -> int: + """Write instruction words at the current PC and return that address.""" + pc = sim.get_pc() + fmt = ">I" if big_endian else " str: + text = self.out.getvalue() + self.out.truncate(0) + self.out.seek(0) + return text + + def test_step_performs_syscalls(self): + """step used to skip syscalls, so nothing printed and exit was ignored""" + self.console.sim.stdout = self.out + self.console.onecmd(f"load {EXAMPLES / 'riscv/hello_asm/hello_asm'}") + self.output() + self.console.onecmd("step 100") + text = self.output() + self.assertIn("Hello", text) + self.assertIn("Program exited with code 0", text) + self.assertNotIn("Error", text) + + self.console.onecmd("step") + self.assertIn("has exited", self.output()) + + def test_x86_disasm_uses_instruction_sizes(self): + self.console.onecmd(f"load {EXAMPLES / 'x86_64/hello_asm/hello_asm'}") + self.output() + self.console.onecmd("disasm pc 3") + text = self.output() + self.assertIn("[0x00400000]> lea", text) + self.assertIn("[0x00400007] mov", text) + self.assertIn("[0x0040000e] syscall", text) + + def test_reset_command(self): + self.console.onecmd(f"load {EXAMPLES / 'riscv/fibonacci/fibonacci'}") + entry = self.console.sim.get_pc() + self.console.onecmd("step 5") + self.console.onecmd("reset") + self.assertEqual(self.console.sim.get_pc(), entry) + self.assertIn("Reset", self.output()) + + def test_run_error_shows_reason(self): + self.console.onecmd(f"load {EXAMPLES / 'riscv/fibonacci/fibonacci'}") + self.console.sim.set_pc(0x10) + self.output() + self.console.onecmd("run") + self.assertIn("Instruction fetch from unmapped address 0x10", self.output()) + + def test_regs_decimal_is_signed(self): + self.console.onecmd(f"load {EXAMPLES / 'riscv/fibonacci/fibonacci'}") + self.console.sim.set_reg(5, (-7) & 0xFFFFFFFFFFFFFFFF) + self.output() + self.console.onecmd("regs decimal") + self.assertIn("..................-7", self.output()) + + def test_show_changes_off(self): + self.console.onecmd(f"load {EXAMPLES / 'riscv/fibonacci/fibonacci'}") + self.console.onecmd("regs") + self.console.onecmd("step 3") + self.console.onecmd("set show-changes off") + self.output() + self.console.onecmd("regs") + self.assertNotIn("★", self.output()) + + def test_break_and_delete_by_symbol(self): + self.console.onecmd(f"load {EXAMPLES / 'riscv/fibonacci/fibonacci'}") + self.console.onecmd("break fibonacci") + self.assertEqual(len(self.console.breakpoints), 1) + self.console.onecmd("delete fibonacci") + self.assertEqual(len(self.console.breakpoints), 0) + + def test_complete_set_includes_output_spacing(self): + self.assertIn("output-spacing", self.console.complete_set("out", "set out", 4, 7)) + + +if __name__ == "__main__": + unittest.main() From f3036ed4959bd401cbb103fc121dfb18bea1d593 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 00:13:19 +0000 Subject: [PATCH 02/18] Add built-in RISC-V encoder and make Keystone optional The Keystone release on PyPI (0.9.2) has no RISC-V support, so RISC-V programs could not be assembled on any machine that installed MapacheSPIM from pip; this was the cause of all 25 failing tests. - Add toolchain/riscv.py: a table-driven RV64IM encoder with Zicsr, fence/ecall/ebreak, the standard pseudo-instructions, symbol expressions, and %hi/%lo. li now handles full 64-bit constants (the old lui+addi expansion was wrong for values like 0x7fffffff and anything above 32 bits) - Output is byte-identical to the committed example binaries, including debug info - Label layout for RISC-V uses the encoder's own sizing, and pass 2 now errors if any label moved instead of emitting bad offsets - .equ/.set constants can be used as instruction operands - Keystone is only installed automatically on x86 machines, where PyPI has prebuilt wheels; elsewhere (Apple Silicon, ARM Linux) it is an optional extra instead of a source build that fails. Assembling ARM64/x86-64/MIPS without it gives an actionable error - Tests that need Keystone are skipped (not failed) when it is absent - mapachespim-as --version reports the package version Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- mapachespim/toolchain/assembler.py | 318 +++++---------- mapachespim/toolchain/cli.py | 4 +- mapachespim/toolchain/riscv.py | 630 +++++++++++++++++++++++++++++ pyproject.toml | 9 +- tests/conftest.py | 25 ++ tests/test_assembler_accuracy.py | 8 +- tests/test_forward_references.py | 7 +- tests/test_isa_specification.py | 7 +- tests/test_riscv_encoder.py | 244 +++++++++++ tests/test_toolchain.py | 8 +- 10 files changed, 1010 insertions(+), 250 deletions(-) create mode 100644 mapachespim/toolchain/riscv.py create mode 100644 tests/conftest.py create mode 100644 tests/test_riscv_encoder.py diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index f91c1c8..d50dcbe 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -25,7 +25,8 @@ KEYSTONE_ERROR = f"Keystone native library failed: {e}" keystone = None -from .directives import DirectiveParser, LineType +from . import riscv +from .directives import DirectiveParser, LineType, SectionData from .dwarf import DWARFv2Builder from .elf_builder import ELFBuilder, Section, Symbol, STT_FUNC, STT_NOTYPE, STB_GLOBAL, STB_LOCAL from .memory_map import get_layout, MemoryLayout @@ -68,13 +69,11 @@ def success(self) -> bool: KS_ARCH_ARM64 = 2 KS_ARCH_MIPS = 3 KS_ARCH_X86 = 4 -KS_ARCH_RISCV = 10 # Added in recent Keystone KS_MODE_LITTLE_ENDIAN = 0 KS_MODE_BIG_ENDIAN = 0x40000000 KS_MODE_64 = 0x8 KS_MODE_32 = 0x4 -KS_MODE_RISCV64 = 0x8 KS_MODE_MIPS32 = 0x4 @@ -101,14 +100,15 @@ class Assembler: # ISA configuration mapping ISA_CONFIG = { + # RISC-V is encoded by the built-in pure-Python encoder (riscv.py) "riscv64": { - "arch": KS_ARCH_RISCV, - "mode": KS_MODE_RISCV64 | KS_MODE_LITTLE_ENDIAN, + "arch": None, + "mode": None, "instr_size": 4, }, "riscv": { - "arch": KS_ARCH_RISCV, - "mode": KS_MODE_RISCV64 | KS_MODE_LITTLE_ENDIAN, + "arch": None, + "mode": None, "instr_size": 4, }, "arm64": { @@ -159,16 +159,6 @@ def __init__(self, isa: str): ValueError: If ISA is not supported. ImportError: If Keystone is not available. """ - if not KEYSTONE_AVAILABLE: - msg = "Keystone Engine not available.\n" - if KEYSTONE_ERROR: - msg += f"Error: {KEYSTONE_ERROR}\n\n" - msg += "Install with: pip install keystone-engine\n\n" - msg += "On Apple Silicon, you may need to install via Homebrew:\n" - msg += " brew install keystone\n" - msg += " pip install keystone-engine" - raise ImportError(msg) - isa_lower = isa.lower().replace("-", "_") if isa_lower not in self.ISA_CONFIG: valid = sorted(set(k for k in self.ISA_CONFIG.keys() if "_" not in k)) @@ -177,13 +167,29 @@ def __init__(self, isa: str): self.isa = isa_lower self._config = self.ISA_CONFIG[isa_lower] self._layout = get_layout(isa_lower) + self._ks = None + + # RISC-V uses the built-in encoder; other ISAs need Keystone + if self._config["arch"] is None: + return + + if not KEYSTONE_AVAILABLE: + msg = f"Assembling {isa} requires the Keystone Engine, which is not installed.\n" + if KEYSTONE_ERROR: + msg += f"Error: {KEYSTONE_ERROR}\n\n" + msg += "Install it with: pip install 'mapachespim[keystone]'\n\n" + msg += "RISC-V programs can be assembled without Keystone." + raise ImportError(msg) - # Initialize Keystone try: self._ks = keystone.Ks(self._config["arch"], self._config["mode"]) except keystone.KsError as e: raise RuntimeError(f"Failed to initialize Keystone for {isa}: {e}") + @property + def _is_riscv(self) -> bool: + return self.isa in ("riscv64", "riscv") + def assemble( self, source: str, @@ -253,10 +259,14 @@ def assemble( # Calculate .text section label positions # Pass all_labels (which now contains data section labels) for pseudo-instruction sizing + # .equ/.set constants can be used as instruction operands but are + # not addresses, so they are kept out of the ELF symbol table + constants = dict(parser.constants) + text_section = sections.get(".text") if text_section: text_labels = self._calculate_text_labels( - text_section, self._layout.text_base, all_labels + text_section, self._layout.text_base, {**constants, **all_labels} ) all_labels.update(text_labels) @@ -264,7 +274,7 @@ def assemble( debug_lines: List[Tuple[int, int]] = [] if text_section: code, code_labels, asm_errors, section_debug_lines = self._assemble_section( - text_section, self._layout.text_base, all_labels + text_section, self._layout.text_base, {**constants, **all_labels} ) result.errors.extend(asm_errors) debug_lines = section_debug_lines @@ -342,7 +352,7 @@ def assemble( def _calculate_text_labels( self, - section: "SectionData", + section: SectionData, base_addr: int, data_labels: Optional[Dict[str, int]] = None, ) -> Dict[str, int]: @@ -371,6 +381,9 @@ def _calculate_text_labels( if instr_size is None: return self._calculate_x86_text_labels_iterative(section, base_addr) + if self._is_riscv: + return self._calculate_riscv_text_labels(section, base_addr, data_labels or {}) + # Fixed-size ISA: single pass with size estimation labels: Dict[str, int] = {} current_addr = base_addr @@ -390,9 +403,36 @@ def _calculate_text_labels( return labels + def _calculate_riscv_text_labels( + self, + section: SectionData, + base_addr: int, + data_labels: Dict[str, int], + ) -> Dict[str, int]: + """ + Calculate RISC-V label addresses using the encoder's own sizing. + + Only li's size depends on a value, so this converges in a pass or two; + the loop re-sizes with the labels found so far until they are stable. + """ + labels: Dict[str, int] = {} + for _ in range(8): + known = {**data_labels, **labels} + new_labels: Dict[str, int] = {} + addr = base_addr + for line in section.lines: + if line.label: + new_labels[line.label] = addr + if line.line_type == LineType.INSTRUCTION and line.instruction: + addr += riscv.instruction_size(line.instruction, known) + if new_labels == labels: + break + labels = new_labels + return labels + def _calculate_x86_text_labels_iterative( self, - section: "SectionData", + section: SectionData, base_addr: int, max_iterations: int = 5, ) -> Dict[str, int]: @@ -554,30 +594,8 @@ def _estimate_fixed_instr_size( # Unknown symbol - conservatively estimate as lui + ori return base_size * 2 - # RISC-V pseudo-instructions that might expand - if self.isa in ("riscv64", "riscv"): - # li with large immediate: lui + addi = 8 bytes - if mnemonic == "li": - operands = parts[1] if len(parts) > 1 else "" - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 2: - try: - imm_str = ops[1].strip() - if imm_str.startswith('0x'): - imm = int(imm_str, 16) - else: - imm = int(imm_str) - if not (-2048 <= imm <= 2047): - return base_size * 2 - except ValueError: - pass - # la always expands to lui + addi - if mnemonic == "la": - return base_size * 2 - # call is typically a single jal instruction (4 bytes) - # Keystone handles near calls as jal, not auipc+jalr - if mnemonic == "call": - return base_size # Single jal instruction + if self._is_riscv: + return riscv.instruction_size(instr, data_labels or {}) # ARM64 pseudo-instructions that expand to multiple instructions if self.isa in ("arm64", "aarch64"): @@ -677,7 +695,7 @@ def _estimate_x86_instr_size(self, instr: str) -> int: def _assemble_section( self, - section: "SectionData", + section: SectionData, base_addr: int, labels: Dict[str, int], ) -> Tuple[bytes, Dict[str, int], List[str], List[Tuple[int, int]]]: @@ -688,8 +706,6 @@ def _assemble_section( (assembled_bytes, label_addresses, errors, debug_lines) where debug_lines is a list of (address, source_line_number) tuples. """ - from .directives import SectionData - code = bytearray() label_addrs: Dict[str, int] = {} errors: List[str] = [] @@ -710,9 +726,16 @@ def _assemble_section( instr = line.instruction - # Expand pseudo-instructions for RISC-V - if self.isa in ("riscv64", "riscv"): - instr = self._expand_riscv_pseudo(instr, current_addr, labels) + # RISC-V: built-in encoder handles instructions and pseudo-instructions + if self._is_riscv: + try: + encoding = riscv.encode(instr, current_addr, labels) + except riscv.RISCVEncodeError as e: + errors.append(f"Line {line.line_number}: {e} - {line.instruction}") + continue + code.extend(encoding) + current_addr += len(encoding) + continue # Expand pseudo-instructions for MIPS and normalize syntax if self.isa in ("mips32", "mips"): @@ -730,10 +753,6 @@ def _assemble_section( # Assemble instruction try: - # For RISC-V, disable compressed instructions for predictable 4-byte encoding - if self.isa in ("riscv64", "riscv"): - instr = f".option norvc\n{instr}" - encoding, count = self._ks.asm(instr, current_addr) if encoding is None or count == 0: errors.append(f"Line {line.line_number}: Failed to assemble: {line.instruction}") @@ -745,179 +764,17 @@ def _assemble_section( except keystone.KsError as e: errors.append(f"Line {line.line_number}: {e} - {line.instruction}") - return bytes(code), label_addrs, errors, debug_lines + # Pass 1 must have predicted every label's address, or branch offsets + # computed from those predictions would be wrong + if not errors and self.isa != "x86_64": + for name, addr in label_addrs.items(): + if labels.get(name, addr) != addr: + errors.append( + f"Internal error: label '{name}' moved from 0x{labels[name]:x} " + f"to 0x{addr:x} between passes; please report this bug" + ) - def _expand_riscv_pseudo( - self, - instr: str, - addr: int, - labels: Dict[str, int], - ) -> str: - """ - Expand RISC-V pseudo-instructions. - - Keystone handles most, but some need help. - """ - parts = instr.split(None, 1) - if not parts: - return instr - - mnemonic = parts[0].lower() - operands = parts[1] if len(parts) > 1 else "" - - # Handle 'li' (load immediate) - Keystone may not handle large values - if mnemonic == "li": - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 2: - reg = ops[0] - try: - # Try to parse the immediate - imm_str = ops[1].strip() - if imm_str in labels: - imm = labels[imm_str] - elif imm_str.startswith('0x'): - imm = int(imm_str, 16) - else: - imm = int(imm_str) - - # If small enough, let Keystone handle it - if -2048 <= imm <= 2047: - return f"addi {reg}, x0, {imm}" - - # Large immediate: use lui + addi - upper = (imm + 0x800) >> 12 - lower = imm - (upper << 12) - if lower < -2048: - lower += 4096 - upper -= 1 - - return f"lui {reg}, {upper}; addi {reg}, {reg}, {lower}" - - except (ValueError, KeyError): - pass # Let Keystone try - - # Handle 'la' (load address) - use PC-relative auipc + addi - # This works correctly for all addresses in RV64, unlike lui which sign-extends - if mnemonic == "la": - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 2: - reg = ops[0] - symbol = ops[1].strip() - if symbol in labels: - target = labels[symbol] - # Calculate PC-relative offset - # auipc is at addr, so offset = target - addr - offset = target - addr - # Split into upper 20 bits and lower 12 bits - # Add 0x800 to round properly when lower bits are negative - upper = ((offset + 0x800) >> 12) & 0xFFFFF - lower = offset - (upper << 12) - # Ensure lower is in signed 12-bit range - if lower > 2047: - lower -= 4096 - upper = (upper + 1) & 0xFFFFF - elif lower < -2048: - lower += 4096 - upper = (upper - 1) & 0xFFFFF - return f"auipc {reg}, {upper}; addi {reg}, {reg}, {lower}" - - # Handle 'call' pseudo - if mnemonic == "call": - symbol = operands.strip() - if symbol in labels: - target = labels[symbol] - offset = target - addr - if -1048576 <= offset <= 1048575: - return f"jal ra, {offset}" - # For far calls, use auipc + jalr - upper = ((offset + 0x800) >> 12) & 0xFFFFF - lower = offset - (upper << 12) - return f"auipc ra, {upper}; jalr ra, ra, {lower}" - - # Handle 'j' pseudo (jump without link) - if mnemonic == "j": - symbol = operands.strip() - if symbol in labels: - target = labels[symbol] - offset = target - addr - return f"jal x0, {offset}" - - # Handle 'jal' with symbol (calculate offset) - if mnemonic == "jal": - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 2: - rd = ops[0] - symbol = ops[1].strip() - if symbol in labels: - target = labels[symbol] - offset = target - addr - return f"jal {rd}, {offset}" - - # Handle branch instructions with symbols - branch_ops = ["beq", "bne", "blt", "bge", "bltu", "bgeu"] - if mnemonic in branch_ops: - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 3: - rs1, rs2, symbol = ops[0], ops[1], ops[2].strip() - if symbol in labels: - target = labels[symbol] - offset = target - addr - return f"{mnemonic} {rs1}, {rs2}, {offset}" - - # Handle beqz/bnez pseudo-instructions (branch if zero/not zero) - if mnemonic in ("beqz", "bnez"): - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 2: - rs = ops[0] - symbol = ops[1].strip() - if symbol in labels: - target = labels[symbol] - offset = target - addr - real_mnemonic = "beq" if mnemonic == "beqz" else "bne" - return f"{real_mnemonic} {rs}, x0, {offset}" - - # Handle pseudo-branches that swap operands - # ble rs1, rs2, label → bge rs2, rs1, label - # bgt rs1, rs2, label → blt rs2, rs1, label - # bleu rs1, rs2, label → bgeu rs2, rs1, label - # bgtu rs1, rs2, label → bltu rs2, rs1, label - swap_branches = { - "ble": "bge", - "bgt": "blt", - "bleu": "bgeu", - "bgtu": "bltu", - } - if mnemonic in swap_branches: - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 3: - rs1, rs2, symbol = ops[0], ops[1], ops[2].strip() - if symbol in labels: - target = labels[symbol] - offset = target - addr - real_mnemonic = swap_branches[mnemonic] - return f"{real_mnemonic} {rs2}, {rs1}, {offset}" - - # Handle blez/bgtz/bltz/bgez (single register branches) - single_reg_branches = { - "blez": ("bge", "x0"), # blez rs, label → bge x0, rs, label - "bgtz": ("blt", "x0"), # bgtz rs, label → blt x0, rs, label - "bltz": ("blt", None), # bltz rs, label → blt rs, x0, label - "bgez": ("bge", None), # bgez rs, label → bge rs, x0, label - } - if mnemonic in single_reg_branches: - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 2: - rs, symbol = ops[0], ops[1].strip() - if symbol in labels: - target = labels[symbol] - offset = target - addr - real_mnemonic, first_reg = single_reg_branches[mnemonic] - if first_reg is not None: - return f"{real_mnemonic} {first_reg}, {rs}, {offset}" - else: - return f"{real_mnemonic} {rs}, x0, {offset}" - - return instr + return bytes(code), label_addrs, errors, debug_lines def _normalize_mips_syntax(self, instr: str) -> str: """ @@ -1337,6 +1194,11 @@ def assemble_instruction(self, instr: str, address: int = 0) -> bytes: Raises: ValueError: If assembly fails. """ + if self._is_riscv: + try: + return riscv.encode(instr, address, {}) + except riscv.RISCVEncodeError as e: + raise ValueError(f"Assembly error: {e}") try: encoding, count = self._ks.asm(instr, address) if encoding is None or count == 0: diff --git a/mapachespim/toolchain/cli.py b/mapachespim/toolchain/cli.py index b830202..0c1dfb1 100644 --- a/mapachespim/toolchain/cli.py +++ b/mapachespim/toolchain/cli.py @@ -16,6 +16,8 @@ import sys from pathlib import Path +from .. import __version__ + def main(args: list[str] | None = None) -> int: """Main entry point for mapachespim-as CLI.""" @@ -80,7 +82,7 @@ def main(args: list[str] | None = None) -> int: parser.add_argument( "--version", action="version", - version="%(prog)s 0.1.0", + version=f"%(prog)s {__version__}", ) parsed = parser.parse_args(args) diff --git a/mapachespim/toolchain/riscv.py b/mapachespim/toolchain/riscv.py new file mode 100644 index 0000000..cd16b92 --- /dev/null +++ b/mapachespim/toolchain/riscv.py @@ -0,0 +1,630 @@ +""" +Pure-Python RISC-V (RV64IM) instruction encoder. + +The Keystone release on PyPI has no RISC-V support, so MapacheSPIM encodes +RISC-V itself. RISC-V's six instruction formats are regular enough that a +table-driven encoder is short, and doing it here means the most commonly +taught ISA needs no native assembler library at all. + +Supported: + - RV64I base integer instructions (including the *W word variants) + - M extension (multiply/divide) + - ecall, ebreak, fence, and the Zicsr instructions + - Common pseudo-instructions: nop, li, la, lla, mv, not, neg, negw, + sext.w, seqz, snez, sltz, sgtz, beqz, bnez, blez, bgez, bltz, bgtz, + bgt, ble, bgtu, bleu, j, jr, ret, call, tail, csrr, csrw, csrs, csrc + - Operand expressions: integers (dec/hex/bin/octal), character + literals, symbols, symbol +/- constant, %hi(), %lo(), + %pcrel_hi(), %pcrel_lo() + +Compressed (C extension) instructions are never emitted, so every +instruction is 4 bytes, matching what the simulator's disassembler shows. +""" + +from __future__ import annotations + +import re +from typing import Dict, List, Optional, Tuple + +INSTR_SIZE = 4 + + +class RISCVEncodeError(ValueError): + """Raised when an instruction cannot be encoded.""" + + +# --------------------------------------------------------------------------- +# Registers +# --------------------------------------------------------------------------- + +_ABI_NAMES = [ + "zero", "ra", "sp", "gp", "tp", "t0", "t1", "t2", + "s0", "s1", "a0", "a1", "a2", "a3", "a4", "a5", + "a6", "a7", "s2", "s3", "s4", "s5", "s6", "s7", + "s8", "s9", "s10", "s11", "t3", "t4", "t5", "t6", +] # fmt: skip + +REGISTERS: Dict[str, int] = {name: i for i, name in enumerate(_ABI_NAMES)} +REGISTERS.update({f"x{i}": i for i in range(32)}) +REGISTERS["fp"] = 8 + + +def parse_register(text: str) -> int: + name = text.strip().lower() + if name not in REGISTERS: + raise RISCVEncodeError(f"unknown register '{text.strip()}'") + return REGISTERS[name] + + +# --------------------------------------------------------------------------- +# CSRs (a small set of names; any CSR can also be given by number) +# --------------------------------------------------------------------------- + +CSR_NAMES: Dict[str, int] = { + "fflags": 0x001, "frm": 0x002, "fcsr": 0x003, + "cycle": 0xC00, "time": 0xC01, "instret": 0xC02, + "mstatus": 0x300, "misa": 0x301, "mie": 0x304, "mtvec": 0x305, + "mscratch": 0x340, "mepc": 0x341, "mcause": 0x342, "mtval": 0x343, "mip": 0x344, + "mhartid": 0xF14, +} # fmt: skip + + +# --------------------------------------------------------------------------- +# Expressions +# --------------------------------------------------------------------------- + +_RELOC_RE = re.compile(r"^%(hi|lo|pcrel_hi|pcrel_lo)\((.*)\)$", re.IGNORECASE) +_TERM_RE = re.compile(r"\s*([+-]?)\s*([^+\-\s][^+\-]*)") + + +def _parse_int(text: str) -> Optional[int]: + """Parse an integer or character literal, or return None.""" + t = text.strip() + if len(t) >= 3 and t[0] == "'" and t[-1] == "'": + body = t[1:-1] + escapes = {"\\n": 10, "\\t": 9, "\\r": 13, "\\0": 0, "\\\\": 92, "\\'": 39} + if body in escapes: + return escapes[body] + if len(body) == 1: + return ord(body) + return None + try: + return int(t, 0) + except ValueError: + pass + # int(..., 0) rejects leading zeros like "010"; accept them as decimal + if re.fullmatch(r"[+-]?\d+", t): + return int(t, 10) + return None + + +class Context: + """Address and symbol information needed to encode one instruction.""" + + def __init__(self, address: int, labels: Dict[str, int], strict: bool = True): + self.address = address + self.labels = labels + # When not strict (the sizing pass), unknown symbols evaluate to 0 + self.strict = strict + + def symbol(self, name: str) -> int: + if name in self.labels: + return self.labels[name] + if self.strict: + raise RISCVEncodeError(f"undefined symbol '{name}'") + return 0 + + def eval(self, text: str) -> int: + """Evaluate an operand expression such as 'label+8' or '%lo(msg)'.""" + text = text.strip() + if not text: + raise RISCVEncodeError("missing operand") + + m = _RELOC_RE.match(text) + if m: + kind, inner = m.group(1).lower(), m.group(2) + value = self.eval(inner) + if kind == "hi": + return ((value + 0x800) >> 12) & 0xFFFFF + if kind == "lo": + return _sign_extend(value & 0xFFF, 12) + if kind == "pcrel_hi": + offset = value - self.address + return ((offset + 0x800) >> 12) & 0xFFFFF + # %pcrel_lo(label) names the auipc instruction; the low part is + # relative to that instruction's pc and the auipc's target. + raise RISCVEncodeError("%pcrel_lo is not supported; use la or lla instead") + + value = _parse_int(text) + if value is not None: + return value + + total = 0 + pos = 0 + matched = False + for m2 in _TERM_RE.finditer(text): + if m2.start() != pos and text[pos : m2.start()].strip(): + break + sign, term = m2.group(1), m2.group(2).strip() + v = _parse_int(term) + if v is None: + if not re.fullmatch(r"[A-Za-z_.$][\w.$]*", term): + raise RISCVEncodeError(f"cannot parse expression '{text}'") + v = self.symbol(term) + total += -v if sign == "-" else v + pos = m2.end() + matched = True + if not matched or text[pos:].strip(): + raise RISCVEncodeError(f"cannot parse expression '{text}'") + return total + + +def _sign_extend(value: int, bits: int) -> int: + value &= (1 << bits) - 1 + return value - (1 << bits) if value & (1 << (bits - 1)) else value + + +def _check_signed(value: int, bits: int, what: str) -> int: + lo, hi = -(1 << (bits - 1)), (1 << (bits - 1)) - 1 + if not lo <= value <= hi: + raise RISCVEncodeError(f"{what} {value} out of range ({lo} to {hi})") + return value + + +def _check_unsigned(value: int, bits: int, what: str) -> int: + if not 0 <= value < (1 << bits): + raise RISCVEncodeError(f"{what} {value} out of range (0 to {(1 << bits) - 1})") + return value + + +# --------------------------------------------------------------------------- +# Instruction formats +# --------------------------------------------------------------------------- + + +def _r(opcode: int, f3: int, f7: int, rd: int, rs1: int, rs2: int) -> int: + return (f7 << 25) | (rs2 << 20) | (rs1 << 15) | (f3 << 12) | (rd << 7) | opcode + + +def _i(opcode: int, f3: int, rd: int, rs1: int, imm: int) -> int: + imm = _check_signed(imm, 12, "immediate") + return ((imm & 0xFFF) << 20) | (rs1 << 15) | (f3 << 12) | (rd << 7) | opcode + + +def _s(opcode: int, f3: int, rs1: int, rs2: int, imm: int) -> int: + imm = _check_signed(imm, 12, "offset") & 0xFFF + return ( + ((imm >> 5) << 25) | (rs2 << 20) | (rs1 << 15) | (f3 << 12) | ((imm & 0x1F) << 7) | opcode + ) + + +def _b(f3: int, rs1: int, rs2: int, offset: int) -> int: + if offset % 2: + raise RISCVEncodeError(f"branch offset {offset} is not a multiple of 2") + offset = _check_signed(offset, 13, "branch offset") & 0x1FFF + return ( + (((offset >> 12) & 1) << 31) + | (((offset >> 5) & 0x3F) << 25) + | (rs2 << 20) + | (rs1 << 15) + | (f3 << 12) + | (((offset >> 1) & 0xF) << 8) + | (((offset >> 11) & 1) << 7) + | 0x63 + ) + + +def _u(opcode: int, rd: int, imm20: int) -> int: + # Accept both unsigned (0..0xFFFFF) and negative 20-bit forms + if -(1 << 19) <= imm20 < 0: + imm20 &= 0xFFFFF + _check_unsigned(imm20, 20, "upper immediate") + return (imm20 << 12) | (rd << 7) | opcode + + +def _j(rd: int, offset: int) -> int: + if offset % 2: + raise RISCVEncodeError(f"jump offset {offset} is not a multiple of 2") + offset = _check_signed(offset, 21, "jump offset") & 0x1FFFFF + return ( + (((offset >> 20) & 1) << 31) + | (((offset >> 1) & 0x3FF) << 21) + | (((offset >> 11) & 1) << 20) + | (((offset >> 12) & 0xFF) << 12) + | (rd << 7) + | 0x6F + ) + + +# (opcode, funct3, funct7) +R_TYPE: Dict[str, Tuple[int, int, int]] = { + "add": (0x33, 0, 0x00), "sub": (0x33, 0, 0x20), "sll": (0x33, 1, 0x00), + "slt": (0x33, 2, 0x00), "sltu": (0x33, 3, 0x00), "xor": (0x33, 4, 0x00), + "srl": (0x33, 5, 0x00), "sra": (0x33, 5, 0x20), "or": (0x33, 6, 0x00), + "and": (0x33, 7, 0x00), + "addw": (0x3B, 0, 0x00), "subw": (0x3B, 0, 0x20), "sllw": (0x3B, 1, 0x00), + "srlw": (0x3B, 5, 0x00), "sraw": (0x3B, 5, 0x20), + # M extension + "mul": (0x33, 0, 0x01), "mulh": (0x33, 1, 0x01), "mulhsu": (0x33, 2, 0x01), + "mulhu": (0x33, 3, 0x01), "div": (0x33, 4, 0x01), "divu": (0x33, 5, 0x01), + "rem": (0x33, 6, 0x01), "remu": (0x33, 7, 0x01), + "mulw": (0x3B, 0, 0x01), "divw": (0x3B, 4, 0x01), "divuw": (0x3B, 5, 0x01), + "remw": (0x3B, 6, 0x01), "remuw": (0x3B, 7, 0x01), +} # fmt: skip + +# (opcode, funct3) +I_ALU: Dict[str, Tuple[int, int]] = { + "addi": (0x13, 0), "slti": (0x13, 2), "sltiu": (0x13, 3), + "xori": (0x13, 4), "ori": (0x13, 6), "andi": (0x13, 7), + "addiw": (0x1B, 0), +} # fmt: skip + +# (opcode, funct3, funct6/7 high bits, shamt bits) +SHIFT_IMM: Dict[str, Tuple[int, int, int, int]] = { + "slli": (0x13, 1, 0x00, 6), "srli": (0x13, 5, 0x00, 6), "srai": (0x13, 5, 0x10, 6), + "slliw": (0x1B, 1, 0x00, 5), "srliw": (0x1B, 5, 0x00, 5), "sraiw": (0x1B, 5, 0x20, 5), +} # fmt: skip + +LOADS: Dict[str, int] = {"lb": 0, "lh": 1, "lw": 2, "ld": 3, "lbu": 4, "lhu": 5, "lwu": 6} +STORES: Dict[str, int] = {"sb": 0, "sh": 1, "sw": 2, "sd": 3} +BRANCHES: Dict[str, int] = {"beq": 0, "bne": 1, "blt": 4, "bge": 5, "bltu": 6, "bgeu": 7} +CSR_OPS: Dict[str, int] = { + "csrrw": 1, "csrrs": 2, "csrrc": 3, "csrrwi": 5, "csrrsi": 6, "csrrci": 7, +} # fmt: skip + +# Pseudo-branches: name -> (real branch, swap operands, zero operand position) +ZERO_BRANCHES: Dict[str, Tuple[str, bool]] = { + # name: (real, register goes second) + "beqz": ("beq", False), "bnez": ("bne", False), + "bltz": ("blt", False), "bgez": ("bge", False), + "blez": ("bge", True), "bgtz": ("blt", True), +} # fmt: skip +SWAP_BRANCHES: Dict[str, str] = {"bgt": "blt", "ble": "bge", "bgtu": "bltu", "bleu": "bgeu"} + + +# --------------------------------------------------------------------------- +# Operand helpers +# --------------------------------------------------------------------------- + + +def split_operands(text: str) -> List[str]: + """Split operands on commas that are not inside parentheses or quotes.""" + ops: List[str] = [] + depth = 0 + quote = False + cur = "" + for ch in text: + if ch == "'" and depth == 0: + quote = not quote + if not quote: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + elif ch == "," and depth == 0: + ops.append(cur.strip()) + cur = "" + continue + cur += ch + if cur.strip() or ops: + ops.append(cur.strip()) + return ops + + +_MEM_RE = re.compile(r"^(.*)\(\s*([A-Za-z0-9]+)\s*\)$") + + +def _parse_mem(text: str, ctx: Context) -> Tuple[int, int]: + """Parse 'imm(reg)' (imm optional) into (offset, reg).""" + m = _MEM_RE.match(text.strip()) + if not m: + raise RISCVEncodeError(f"expected offset(register), got '{text}'") + offset_text = m.group(1).strip() + offset = ctx.eval(offset_text) if offset_text else 0 + return offset, parse_register(m.group(2)) + + +def _is_mem_operand(text: str) -> bool: + m = _MEM_RE.match(text.strip()) + return bool(m) and m.group(2).lower() in REGISTERS + + +def _expect(ops: List[str], n: int, usage: str) -> None: + if len(ops) != n: + raise RISCVEncodeError(f"expected {usage}") + + +def _parse_csr(text: str, ctx: Context) -> int: + name = text.strip().lower() + if name in CSR_NAMES: + return CSR_NAMES[name] + return _check_unsigned(ctx.eval(text), 12, "CSR number") + + +# --------------------------------------------------------------------------- +# li expansion (shared by encoding and sizing) +# --------------------------------------------------------------------------- + + +def _li_sequence(rd: int, value: int) -> List[int]: + """Instruction words that load a 64-bit constant into rd.""" + value = _sign_extend(value, 64) + if -2048 <= value <= 2047: + return [_i(0x13, 0, rd, 0, value)] # addi rd, x0, value + if -(1 << 31) <= value < (1 << 31): + lo = _sign_extend(value & 0xFFF, 12) + hi = ((value - lo) >> 12) & 0xFFFFF + words = [_u(0x37, rd, hi)] # lui + if lo: + words.append(_i(0x1B, 0, rd, rd, lo)) # addiw keeps it 32-bit sign-extended + return words + # 64-bit: build the upper bits recursively, then shift in 12 bits at a time + lo = _sign_extend(value & 0xFFF, 12) + upper = (value - lo) >> 12 + shift = 12 + while upper and not upper & 1 and shift < 60: + upper >>= 1 + shift += 1 + words = _li_sequence(rd, upper) + words.append((((shift & 0x3F) << 20) | (rd << 15) | (1 << 12) | (rd << 7) | 0x13)) # slli + if lo: + words.append(_i(0x13, 0, rd, rd, lo)) + return words + + +def _pcrel_pair(target: int, pc: int) -> Tuple[int, int]: + """Split a pc-relative offset into auipc (hi20) and addi/jalr (lo12) parts.""" + offset = target - pc + lo = _sign_extend(offset & 0xFFF, 12) + hi = ((offset - lo) >> 12) & 0xFFFFF + return hi, lo + + +# --------------------------------------------------------------------------- +# Encoder +# --------------------------------------------------------------------------- + + +def _encode(mnemonic: str, ops: List[str], ctx: Context) -> List[int]: + m = mnemonic + pc = ctx.address + + def target_offset(text: str) -> int: + """Branch/jump operand: a label (pc-relative) or a literal offset.""" + value = _parse_int(text) + if value is not None: + return value + return ctx.eval(text) - pc + + if m in R_TYPE: + _expect(ops, 3, f"{m} rd, rs1, rs2") + opcode, f3, f7 = R_TYPE[m] + return [_r(opcode, f3, f7, parse_register(ops[0]), parse_register(ops[1]), parse_register(ops[2]))] + + if m in I_ALU: + _expect(ops, 3, f"{m} rd, rs1, imm") + opcode, f3 = I_ALU[m] + return [_i(opcode, f3, parse_register(ops[0]), parse_register(ops[1]), ctx.eval(ops[2]))] + + if m in SHIFT_IMM: + _expect(ops, 3, f"{m} rd, rs1, shamt") + opcode, f3, hi, bits = SHIFT_IMM[m] + shamt = _check_unsigned(ctx.eval(ops[2]), bits, "shift amount") + rd, rs1 = parse_register(ops[0]), parse_register(ops[1]) + return [((hi << 26) if bits == 6 else (hi << 25)) | (shamt << 20) | (rs1 << 15) | (f3 << 12) | (rd << 7) | opcode] + + if m in LOADS: + _expect(ops, 2, f"{m} rd, offset(rs1)") + rd = parse_register(ops[0]) + if _is_mem_operand(ops[1]): + offset, rs1 = _parse_mem(ops[1], ctx) + return [_i(0x03, LOADS[m], rd, rs1, offset)] + # 'lw rd, symbol' -> auipc rd, %pcrel_hi(symbol); lw rd, %pcrel_lo(rd) + hi, lo = _pcrel_pair(ctx.eval(ops[1]), pc) + return [_u(0x17, rd, hi), _i(0x03, LOADS[m], rd, rd, lo)] + + if m in STORES: + if len(ops) == 3: + # 'sw rs2, symbol, rt' -> auipc rt, hi; sw rs2, lo(rt) + rs2, rt = parse_register(ops[0]), parse_register(ops[2]) + hi, lo = _pcrel_pair(ctx.eval(ops[1]), pc) + return [_u(0x17, rt, hi), _s(0x23, STORES[m], rt, rs2, lo)] + _expect(ops, 2, f"{m} rs2, offset(rs1)") + offset, rs1 = _parse_mem(ops[1], ctx) + return [_s(0x23, STORES[m], rs1, parse_register(ops[0]), offset)] + + if m in BRANCHES: + _expect(ops, 3, f"{m} rs1, rs2, label") + return [_b(BRANCHES[m], parse_register(ops[0]), parse_register(ops[1]), target_offset(ops[2]))] + + if m in SWAP_BRANCHES: + _expect(ops, 3, f"{m} rs1, rs2, label") + real = SWAP_BRANCHES[m] + return [_b(BRANCHES[real], parse_register(ops[1]), parse_register(ops[0]), target_offset(ops[2]))] + + if m in ZERO_BRANCHES: + _expect(ops, 2, f"{m} rs, label") + real, reg_second = ZERO_BRANCHES[m] + rs = parse_register(ops[0]) + rs1, rs2 = (0, rs) if reg_second else (rs, 0) + return [_b(BRANCHES[real], rs1, rs2, target_offset(ops[1]))] + + if m == "lui": + _expect(ops, 2, "lui rd, imm") + return [_u(0x37, parse_register(ops[0]), ctx.eval(ops[1]))] + + if m == "auipc": + _expect(ops, 2, "auipc rd, imm") + return [_u(0x17, parse_register(ops[0]), ctx.eval(ops[1]))] + + if m == "jal": + if len(ops) == 1: + return [_j(1, target_offset(ops[0]))] + _expect(ops, 2, "jal [rd,] label") + return [_j(parse_register(ops[0]), target_offset(ops[1]))] + + if m == "jalr": + if len(ops) == 1: + if _is_mem_operand(ops[0]): + offset, rs1 = _parse_mem(ops[0], ctx) + return [_i(0x67, 0, 1, rs1, offset)] + return [_i(0x67, 0, 1, parse_register(ops[0]), 0)] + if len(ops) == 2: + if _is_mem_operand(ops[1]): + offset, rs1 = _parse_mem(ops[1], ctx) + return [_i(0x67, 0, parse_register(ops[0]), rs1, offset)] + return [_i(0x67, 0, parse_register(ops[0]), parse_register(ops[1]), 0)] + _expect(ops, 3, "jalr rd, rs1, offset") + return [_i(0x67, 0, parse_register(ops[0]), parse_register(ops[1]), ctx.eval(ops[2]))] + + if m in CSR_OPS: + _expect(ops, 3, f"{m} rd, csr, source") + csr = _parse_csr(ops[1], ctx) + rd = parse_register(ops[0]) + if m.endswith("i"): + src = _check_unsigned(ctx.eval(ops[2]), 5, "CSR immediate") + else: + src = parse_register(ops[2]) + return [(csr << 20) | (src << 15) | (CSR_OPS[m] << 12) | (rd << 7) | 0x73] + + # --- no-operand instructions --- + if m in ("ecall", "ebreak", "nop", "ret", "fence", "fence.i"): + if ops and m != "fence": + raise RISCVEncodeError(f"{m} takes no operands") + if m == "ecall": + return [0x00000073] + if m == "ebreak": + return [0x00100073] + if m == "nop": + return [_i(0x13, 0, 0, 0, 0)] + if m == "ret": + return [_i(0x67, 0, 0, 1, 0)] + if m == "fence.i": + return [0x0000100F] + return [0x0FF0000F] # fence iorw, iorw + + # --- pseudo-instructions --- + if m == "li": + _expect(ops, 2, "li rd, imm") + return _li_sequence(parse_register(ops[0]), ctx.eval(ops[1])) + + if m in ("la", "lla"): + _expect(ops, 2, f"{m} rd, symbol") + rd = parse_register(ops[0]) + hi, lo = _pcrel_pair(ctx.eval(ops[1]), pc) + return [_u(0x17, rd, hi), _i(0x13, 0, rd, rd, lo)] + + if m == "mv": + _expect(ops, 2, "mv rd, rs") + return [_i(0x13, 0, parse_register(ops[0]), parse_register(ops[1]), 0)] + + if m == "not": + _expect(ops, 2, "not rd, rs") + return [_i(0x13, 4, parse_register(ops[0]), parse_register(ops[1]), -1)] + + if m in ("neg", "negw"): + _expect(ops, 2, f"{m} rd, rs") + opcode = 0x33 if m == "neg" else 0x3B + return [_r(opcode, 0, 0x20, parse_register(ops[0]), 0, parse_register(ops[1]))] + + if m == "sext.w": + _expect(ops, 2, "sext.w rd, rs") + return [_i(0x1B, 0, parse_register(ops[0]), parse_register(ops[1]), 0)] + + if m == "seqz": + _expect(ops, 2, "seqz rd, rs") + return [_i(0x13, 3, parse_register(ops[0]), parse_register(ops[1]), 1)] + + if m == "snez": + _expect(ops, 2, "snez rd, rs") + return [_r(0x33, 3, 0, parse_register(ops[0]), 0, parse_register(ops[1]))] + + if m == "sltz": + _expect(ops, 2, "sltz rd, rs") + return [_r(0x33, 2, 0, parse_register(ops[0]), parse_register(ops[1]), 0)] + + if m == "sgtz": + _expect(ops, 2, "sgtz rd, rs") + return [_r(0x33, 2, 0, parse_register(ops[0]), 0, parse_register(ops[1]))] + + if m == "j": + _expect(ops, 1, "j label") + return [_j(0, target_offset(ops[0]))] + + if m == "jr": + _expect(ops, 1, "jr rs") + return [_i(0x67, 0, 0, parse_register(ops[0]), 0)] + + if m in ("call", "tail"): + _expect(ops, 1, f"{m} label") + link = 1 if m == "call" else 0 + offset = target_offset(ops[0]) + if -(1 << 20) <= offset < (1 << 20) and m == "call": + return [_j(link, offset)] + # Far call/tail: auipc t1 (or ra), then jalr + scratch = 1 if m == "call" else 6 + hi, lo = _pcrel_pair(pc + offset, pc) + return [_u(0x17, scratch, hi), _i(0x67, 0, link, scratch, lo)] + + if m in ("csrr", "csrw", "csrs", "csrc"): + _expect(ops, 2, f"{m} ...") + if m == "csrr": + return [(_parse_csr(ops[1], ctx) << 20) | (2 << 12) | (parse_register(ops[0]) << 7) | 0x73] + f3 = {"csrw": 1, "csrs": 2, "csrc": 3}[m] + return [(_parse_csr(ops[0], ctx) << 20) | (parse_register(ops[1]) << 15) | (f3 << 12) | 0x73] + + raise RISCVEncodeError(f"unknown instruction '{mnemonic}'") + + +def _split_instruction(text: str) -> Tuple[str, List[str]]: + parts = text.strip().split(None, 1) + if not parts: + raise RISCVEncodeError("empty instruction") + mnemonic = parts[0].lower() + ops = split_operands(parts[1]) if len(parts) > 1 else [] + return mnemonic, ops + + +def encode(text: str, address: int, labels: Dict[str, int]) -> bytes: + """ + Encode one RISC-V instruction (or pseudo-instruction) to machine code. + + Args: + text: Instruction text, e.g. "addi a0, a0, 1" or "la a0, msg". + address: Address the instruction will be placed at. + labels: Symbol table used to resolve labels and constants. + + Returns: + Little-endian machine code (4 bytes per instruction). + + Raises: + RISCVEncodeError: If the instruction is invalid. + """ + mnemonic, ops = _split_instruction(text) + words = _encode(mnemonic, ops, Context(address, labels, strict=True)) + return b"".join(w.to_bytes(4, "little") for w in words) + + +def instruction_size(text: str, labels: Dict[str, int]) -> int: + """ + Size in bytes that ``encode`` will produce, for the label-layout pass. + + Label addresses are not known yet during that pass, so labels resolve to + 0. That is safe because the only size-dependent pseudo-instructions are + li (whose size depends on the constant, and constants from .equ are known + early) and far call/tail (which fall back to 8 bytes for undefined + targets, handled here). + """ + mnemonic, ops = _split_instruction(text) + if mnemonic == "call": + # Near calls are a single jal; programs large enough to need a far + # call do not fit in the simulator's default memory anyway. + return INSTR_SIZE + if mnemonic == "tail": + return 2 * INSTR_SIZE + try: + words = _encode(mnemonic, ops, Context(0, labels, strict=False)) + except RISCVEncodeError: + return INSTR_SIZE # the real error is reported by encode() + return len(words) * INSTR_SIZE + diff --git a/pyproject.toml b/pyproject.toml index 5333c3d..fe832d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,10 +32,17 @@ dependencies = [ "pyelftools>=0.29", "unicorn>=2.0.0", "capstone>=5.0.0", - "keystone-engine>=0.9.2", + # Keystone assembles ARM64/x86-64/MIPS (RISC-V uses a built-in encoder). + # Its PyPI release only ships prebuilt wheels for x86 machines, so it is + # installed automatically only there; elsewhere pip would have to compile + # it from source. Use the "keystone" extra to try that explicitly. + "keystone-engine>=0.9.2; platform_machine == 'x86_64' or platform_machine == 'AMD64' or platform_machine == 'i686' or platform_machine == 'x86'", ] [project.optional-dependencies] +keystone = [ + "keystone-engine>=0.9.2", +] dev = [ "pytest>=7.0", "pytest-cov>=4.0", diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..55b0aa1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,25 @@ +""" +Shared pytest configuration. + +Assembling ARM64, x86-64, and MIPS code needs the optional Keystone Engine, +which has no prebuilt wheels for some platforms (e.g. Apple Silicon). Tests +that fail only because Keystone is missing are reported as skipped there, +while RISC-V tests (which use the built-in encoder) still run everywhere. +""" + +import pytest + +from mapachespim.toolchain.assembler import KEYSTONE_AVAILABLE + +KEYSTONE_MISSING_MARKER = "requires the Keystone Engine" + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + if KEYSTONE_AVAILABLE or not report.failed: + return + if KEYSTONE_MISSING_MARKER in str(report.longrepr): + report.outcome = "skipped" + report.longrepr = (str(item.path), item.location[1] or 0, "Skipped: Keystone not installed") diff --git a/tests/test_assembler_accuracy.py b/tests/test_assembler_accuracy.py index f54a4b8..4a514f0 100644 --- a/tests/test_assembler_accuracy.py +++ b/tests/test_assembler_accuracy.py @@ -17,16 +17,14 @@ from typing import Dict, List, Tuple from mapachespim.toolchain import assemble -from mapachespim.toolchain.assembler import Assembler, KEYSTONE_AVAILABLE +from mapachespim.toolchain.assembler import Assembler from mapachespim.toolchain.directives import DirectiveParser, LineType from mapachespim.toolchain.memory_map import get_layout # Skip all tests if Keystone not available -pytestmark = pytest.mark.skipif( - not KEYSTONE_AVAILABLE, - reason="Keystone engine not available" -) +# Tests needing Keystone (ARM64/x86-64/MIPS) are skipped by conftest.py when +# it is not installed; RISC-V tests use the built-in encoder and always run. # ============================================================================= diff --git a/tests/test_forward_references.py b/tests/test_forward_references.py index 0a81eb8..1d4894b 100644 --- a/tests/test_forward_references.py +++ b/tests/test_forward_references.py @@ -3,14 +3,11 @@ import pytest from mapachespim.toolchain import assemble -from mapachespim.toolchain.assembler import KEYSTONE_AVAILABLE # Skip all tests if Keystone not available -pytestmark = pytest.mark.skipif( - not KEYSTONE_AVAILABLE, - reason="Keystone engine not available" -) +# Tests needing Keystone (ARM64/x86-64/MIPS) are skipped by conftest.py when +# it is not installed; RISC-V tests use the built-in encoder and always run. class TestRISCVForwardReferences: diff --git a/tests/test_isa_specification.py b/tests/test_isa_specification.py index bf464d2..c0d30e7 100644 --- a/tests/test_isa_specification.py +++ b/tests/test_isa_specification.py @@ -5,15 +5,12 @@ from pathlib import Path from mapachespim.toolchain import assemble, assemble_file, AssemblyResult -from mapachespim.toolchain.assembler import KEYSTONE_AVAILABLE from mapachespim.toolchain.directives import DirectiveParser # Skip all tests if Keystone not available -pytestmark = pytest.mark.skipif( - not KEYSTONE_AVAILABLE, - reason="Keystone engine not available" -) +# Tests needing Keystone (ARM64/x86-64/MIPS) are skipped by conftest.py when +# it is not installed; RISC-V tests use the built-in encoder and always run. class TestISADirective: diff --git a/tests/test_riscv_encoder.py b/tests/test_riscv_encoder.py new file mode 100644 index 0000000..4dc2f92 --- /dev/null +++ b/tests/test_riscv_encoder.py @@ -0,0 +1,244 @@ +""" +Tests for the built-in pure-Python RISC-V encoder. + +Encodings are checked by disassembling them with Capstone (an independent +implementation), and li sequences are checked by executing them in Unicorn. +""" + +import unittest + +from capstone import CS_ARCH_RISCV, CS_MODE_RISCV64, Cs + +from mapachespim import ISA, Simulator +from mapachespim.toolchain import Assembler +from mapachespim.toolchain.riscv import RISCVEncodeError, encode, instruction_size + +PC = 0x80000000 +LABELS = {"target": PC + 0x40, "back": PC - 0x20, "msg": 0x80100010, "CONST": 42} + +_cs = Cs(CS_ARCH_RISCV, CS_MODE_RISCV64) + + +def disasm(code: bytes, addr: int = PC): + return [f"{i.mnemonic} {i.op_str}".strip() for i in _cs.disasm(code, addr)] + + +class TestInstructionForms(unittest.TestCase): + """Each (source, expected Capstone text) pair round-trips.""" + + CASES = [ + # R-type + ("add a0, a1, a2", "add a0, a1, a2"), + ("sub t0, t1, t2", "sub t0, t1, t2"), + ("sll s0, s1, s2", "sll s0, s1, s2"), + ("slt a0, a1, a2", "slt a0, a1, a2"), + ("sltu a0, a1, a2", "sltu a0, a1, a2"), + ("xor a0, a1, a2", "xor a0, a1, a2"), + ("srl a0, a1, a2", "srl a0, a1, a2"), + ("sra a0, a1, a2", "sra a0, a1, a2"), + ("or a0, a1, a2", "or a0, a1, a2"), + ("and a0, a1, a2", "and a0, a1, a2"), + ("addw a0, a1, a2", "addw a0, a1, a2"), + ("subw a0, a1, a2", "subw a0, a1, a2"), + ("sllw a0, a1, a2", "sllw a0, a1, a2"), + ("srlw a0, a1, a2", "srlw a0, a1, a2"), + ("sraw a0, a1, a2", "sraw a0, a1, a2"), + ("mul a0, a1, a2", "mul a0, a1, a2"), + ("mulh a0, a1, a2", "mulh a0, a1, a2"), + ("mulhsu a0, a1, a2", "mulhsu a0, a1, a2"), + ("mulhu a0, a1, a2", "mulhu a0, a1, a2"), + ("div a0, a1, a2", "div a0, a1, a2"), + ("divu a0, a1, a2", "divu a0, a1, a2"), + ("rem a0, a1, a2", "rem a0, a1, a2"), + ("remu a0, a1, a2", "remu a0, a1, a2"), + ("mulw a0, a1, a2", "mulw a0, a1, a2"), + ("divw a0, a1, a2", "divw a0, a1, a2"), + ("divuw a0, a1, a2", "divuw a0, a1, a2"), + ("remw a0, a1, a2", "remw a0, a1, a2"), + ("remuw a0, a1, a2", "remuw a0, a1, a2"), + # Register names + ("add x5, x6, x7", "add t0, t1, t2"), + ("add fp, s0, zero", "add s0, s0, zero"), + # I-type + ("addi a0, a1, -2048", "addi a0, a1, -0x800"), + ("addi a0, a1, 2047", "addi a0, a1, 0x7ff"), + ("slti a0, a1, 5", "slti a0, a1, 5"), + ("sltiu a0, a1, 5", "sltiu a0, a1, 5"), + ("xori a0, a1, -1", "not a0, a1"), + ("ori a0, a1, 0xff", "ori a0, a1, 0xff"), + ("andi a0, a1, 15", "andi a0, a1, 0xf"), + ("addiw a0, a1, 3", "addiw a0, a1, 3"), + ("slli a0, a1, 63", "slli a0, a1, 0x3f"), + ("srli a0, a1, 1", "srli a0, a1, 1"), + ("srai a0, a1, 40", "srai a0, a1, 0x28"), + ("slliw a0, a1, 31", "slliw a0, a1, 0x1f"), + ("srliw a0, a1, 2", "srliw a0, a1, 2"), + ("sraiw a0, a1, 2", "sraiw a0, a1, 2"), + ("addi a0, a0, CONST", "addi a0, a0, 0x2a"), + ("addi a0, a0, 'A'", "addi a0, a0, 0x41"), + # Loads / stores + ("lb a0, 0(sp)", "lb a0, 0(sp)"), + ("lh a0, -2(sp)", "lh a0, -2(sp)"), + ("lw a0, 4(a1)", "lw a0, 4(a1)"), + ("ld a0, 8(a1)", "ld a0, 8(a1)"), + ("lbu a0, (a1)", "lbu a0, 0(a1)"), + ("lhu a0, 2(a1)", "lhu a0, 2(a1)"), + ("lwu a0, 4(a1)", "lwu a0, 4(a1)"), + ("sb a0, 0(sp)", "sb a0, 0(sp)"), + ("sh a0, 2(sp)", "sh a0, 2(sp)"), + ("sw a0, -4(sp)", "sw a0, -4(sp)"), + ("sd ra, 2040(sp)", "sd ra, 0x7f8(sp)"), + # Branches and jumps (Capstone prints branch targets as pc-relative offsets) + ("beq a0, a1, target", "beq a0, a1, 0x40"), + ("bne a0, a1, back", "bne a0, a1, -0x20"), + ("blt a0, a1, target", "blt a0, a1, 0x40"), + ("bge a0, a1, target", "bge a0, a1, 0x40"), + ("bltu a0, a1, target", "bltu a0, a1, 0x40"), + ("bgeu a0, a1, target", "bgeu a0, a1, 0x40"), + ("jal ra, target", "jal 0x40"), + ("jal target", "jal 0x40"), + ("jalr ra, t0, 0", "jalr t0"), + ("jalr t0", "jalr t0"), + ("jalr ra, 8(t0)", "jalr ra, t0, 8"), + # Upper immediates + ("lui a0, 0x12345", "lui a0, 0x12345"), + ("lui a0, %hi(msg)", "lui a0, 0x80100"), + ("addi a0, a0, %lo(msg)", "addi a0, a0, 0x10"), + ("auipc a0, 1", "auipc a0, 1"), + # System + ("ecall", "ecall"), + ("ebreak", "ebreak"), + ("fence", "fence"), + ("csrrs a0, cycle, zero", "rdcycle a0"), + ("csrr a0, mhartid", "csrr a0, mhartid"), + # Pseudo-instructions + ("nop", "nop"), + ("mv a0, a1", "mv a0, a1"), + ("not a0, a1", "not a0, a1"), + ("neg a0, a1", "neg a0, a1"), + ("negw a0, a1", "negw a0, a1"), + ("sext.w a0, a1", "sext.w a0, a1"), + ("seqz a0, a1", "seqz a0, a1"), + ("snez a0, a1", "snez a0, a1"), + ("sltz a0, a1", "sltz a0, a1"), + ("sgtz a0, a1", "sgtz a0, a1"), + ("beqz a0, target", "beqz a0, 0x40"), + ("bnez a0, target", "bnez a0, 0x40"), + ("bltz a0, target", "bltz a0, 0x40"), + ("bgez a0, target", "bgez a0, 0x40"), + ("blez a0, target", "blez a0, 0x40"), + ("bgtz a0, target", "bgtz a0, 0x40"), + ("bgt a0, a1, target", "blt a1, a0, 0x40"), + ("ble a0, a1, target", "bge a1, a0, 0x40"), + ("bgtu a0, a1, target", "bltu a1, a0, 0x40"), + ("bleu a0, a1, target", "bgeu a1, a0, 0x40"), + ("j target", "j 0x40"), + ("jr ra", "ret"), + ("ret", "ret"), + ("call target", "jal 0x40"), + ] + + def test_forms(self): + for source, expected in self.CASES: + with self.subTest(source=source): + code = encode(source, PC, LABELS) + self.assertEqual(len(code), 4) + self.assertEqual(disasm(code), [expected]) + self.assertEqual(instruction_size(source, LABELS), 4) + + def test_la_is_pc_relative(self): + code = encode("la a0, msg", PC, LABELS) + self.assertEqual(disasm(code), ["auipc a0, 0x100", "addi a0, a0, 0x10"]) + + def test_load_from_symbol(self): + code = encode("lw a0, msg", PC, LABELS) + self.assertEqual(disasm(code), ["auipc a0, 0x100", "lw a0, 0x10(a0)"]) + + def test_store_to_symbol(self): + code = encode("sw a0, msg, t0", PC, LABELS) + self.assertEqual(disasm(code), ["auipc t0, 0x100", "sw a0, 0x10(t0)"]) + + def test_symbol_arithmetic(self): + code = encode("la a0, msg+4", PC, LABELS) + self.assertEqual(disasm(code), ["auipc a0, 0x100", "addi a0, a0, 0x14"]) + + +class TestErrors(unittest.TestCase): + CASES = [ + ("addi a0, a1, 2048", "out of range"), + ("addi a0, a1", "expected"), + ("add a0, a1, q9", "unknown register"), + ("frob a0", "unknown instruction"), + ("beq a0, a1, nowhere", "undefined symbol"), + ("slli a0, a0, 64", "out of range"), + ("ecall a0", "takes no operands"), + ] + + def test_errors(self): + for source, message in self.CASES: + with self.subTest(source=source): + with self.assertRaises(RISCVEncodeError) as cm: + encode(source, PC, LABELS) + self.assertIn(message, str(cm.exception)) + + def test_assembler_reports_line(self): + result = Assembler("riscv64").assemble(".text\n_start:\n addi a0, a1, 5000\n") + self.assertFalse(result.success) + self.assertIn("Line 3", result.errors[0]) + + +class TestLoadImmediate(unittest.TestCase): + """li must produce the exact 64-bit value, verified by executing it.""" + + VALUES = [ + 0, 1, -1, 2047, -2048, 2048, -2049, 0x7FF, 0x800, 0xFFF, 0x1000, + 0x12345, 0x12345678, 0x7FFFFFFF, -0x80000000, 0x80000000, 0xFFFFFFFF, + 0x100000000, 0x123456789, -0x123456789, 0x7FFFFFFFFFFFFFFF, + -0x8000000000000000, 0xDEADBEEFCAFEBABE, 0x8000000000000000, + 0x0000080000000800, 0x00FF00FF00FF00FF, + ] # fmt: skip + + def test_values(self): + for value in self.VALUES: + with self.subTest(value=hex(value)): + code = encode(f"li t0, {value}", PC, {}) + self.assertEqual(len(code), instruction_size(f"li t0, {value}", {})) + sim = Simulator(ISA.RISCV) + sim.write_mem(PC, code) + sim.set_pc(PC) + for _ in range(len(code) // 4): + sim.step() + self.assertEqual(sim.get_reg(5), value & 0xFFFFFFFFFFFFFFFF) + + def test_small_li_is_one_instruction(self): + self.assertEqual(disasm(encode("li a0, 42", PC, {})), ["addi a0, zero, 0x2a"]) + + +class TestProgram(unittest.TestCase): + def test_forward_references_and_constants(self): + source = """ + .equ BIG, 0x12345678 + .text + .globl _start + _start: + li t0, BIG + j skip + li t1, 99 + skip: + la a0, msg + call func + li a7, 10 + ecall + func: + ret + .data + msg: .asciz "hi" + """ + result = Assembler("riscv64").assemble(source) + self.assertTrue(result.success, result.errors) + # li BIG is lui+addiw, so 'skip' is at +16 + self.assertEqual(result.symbols["skip"] - result.symbols["_start"], 16) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index be0d6cc..e1920e0 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -2,15 +2,13 @@ import pytest from mapachespim.toolchain import assemble, AssemblyResult -from mapachespim.toolchain.assembler import Assembler, KEYSTONE_AVAILABLE +from mapachespim.toolchain.assembler import Assembler from mapachespim.toolchain.directives import DirectiveParser, LineType # Skip all tests if Keystone not available -pytestmark = pytest.mark.skipif( - not KEYSTONE_AVAILABLE, - reason="Keystone engine not available" -) +# Tests needing Keystone (ARM64/x86-64/MIPS) are skipped by conftest.py when +# it is not installed; RISC-V tests use the built-in encoder and always run. class TestDirectiveParser: From 100e17287bbae43ee63740c90f34241bd69640d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 00:18:32 +0000 Subject: [PATCH 03/18] Add built-in MIPS32 encoder so MIPS needs no Keystone MIPS is the ISA SPIM itself simulates, so it should assemble on every platform, including Apple Silicon where Keystone has no wheels. - Add toolchain/mips.py: MIPS32 integer instructions plus the common SPIM/GNU pseudo-instructions (li, la, move, not, neg, abs, b, beqz, blt/bgt/ble/bge and unsigned variants, 3-operand div/rem, mul by a constant, loads/stores from labels). Branches and jumps get a delay-slot nop, matching GNU as in ".set reorder" mode - Output is byte-identical to the committed MIPS example binaries - Remove the old Keystone-based MIPS expansion, including its hard-coded branch-offset workarounds and a sizing bug (li with a value in 32768..65535 was sized as 1 instruction but emitted 2) - Move shared operand parsing into toolchain/expr.py - Sizing passes now use real addresses and ignore range checks on not-yet-resolved labels, so layout is exact - Clearer errors: "add t0, t1, t2" says MIPS registers need a $; RISC-V "add a0, a1, 5" is accepted as addi (as in GNU as) - Only ARM64 and x86-64 still need Keystone Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- mapachespim/toolchain/assembler.py | 253 +++------------- mapachespim/toolchain/expr.py | 143 +++++++++ mapachespim/toolchain/mips.py | 457 +++++++++++++++++++++++++++++ mapachespim/toolchain/riscv.py | 166 +++-------- tests/test_mips_encoder.py | 238 +++++++++++++++ tests/test_riscv_encoder.py | 2 + 6 files changed, 912 insertions(+), 347 deletions(-) create mode 100644 mapachespim/toolchain/expr.py create mode 100644 mapachespim/toolchain/mips.py create mode 100644 tests/test_mips_encoder.py diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index d50dcbe..8eb8692 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -25,8 +25,9 @@ KEYSTONE_ERROR = f"Keystone native library failed: {e}" keystone = None -from . import riscv +from . import mips, riscv from .directives import DirectiveParser, LineType, SectionData +from .expr import EncodeError from .dwarf import DWARFv2Builder from .elf_builder import ELFBuilder, Section, Symbol, STT_FUNC, STT_NOTYPE, STB_GLOBAL, STB_LOCAL from .memory_map import get_layout, MemoryLayout @@ -67,14 +68,11 @@ def success(self) -> bool: # Keystone architecture/mode constants KS_ARCH_ARM64 = 2 -KS_ARCH_MIPS = 3 KS_ARCH_X86 = 4 KS_MODE_LITTLE_ENDIAN = 0 -KS_MODE_BIG_ENDIAN = 0x40000000 KS_MODE_64 = 0x8 KS_MODE_32 = 0x4 -KS_MODE_MIPS32 = 0x4 class Assembler: @@ -136,14 +134,15 @@ class Assembler: "mode": KS_MODE_64, "instr_size": None, }, + # MIPS is encoded by the built-in pure-Python encoder (mips.py) "mips32": { - "arch": KS_ARCH_MIPS, - "mode": KS_MODE_MIPS32 | KS_MODE_BIG_ENDIAN, + "arch": None, + "mode": None, "instr_size": 4, }, "mips": { - "arch": KS_ARCH_MIPS, - "mode": KS_MODE_MIPS32 | KS_MODE_BIG_ENDIAN, + "arch": None, + "mode": None, "instr_size": 4, }, } @@ -169,7 +168,7 @@ def __init__(self, isa: str): self._layout = get_layout(isa_lower) self._ks = None - # RISC-V uses the built-in encoder; other ISAs need Keystone + # RISC-V and MIPS use built-in encoders; ARM64 and x86-64 need Keystone if self._config["arch"] is None: return @@ -178,7 +177,7 @@ def __init__(self, isa: str): if KEYSTONE_ERROR: msg += f"Error: {KEYSTONE_ERROR}\n\n" msg += "Install it with: pip install 'mapachespim[keystone]'\n\n" - msg += "RISC-V programs can be assembled without Keystone." + msg += "RISC-V and MIPS programs can be assembled without Keystone." raise ImportError(msg) try: @@ -190,6 +189,15 @@ def __init__(self, isa: str): def _is_riscv(self) -> bool: return self.isa in ("riscv64", "riscv") + @property + def _builtin_encoder(self): # type: ignore[no-untyped-def] + """The pure-Python encoder module for this ISA, or None for Keystone ISAs.""" + if self._is_riscv: + return riscv + if self.isa in ("mips32", "mips"): + return mips + return None + def assemble( self, source: str, @@ -381,8 +389,8 @@ def _calculate_text_labels( if instr_size is None: return self._calculate_x86_text_labels_iterative(section, base_addr) - if self._is_riscv: - return self._calculate_riscv_text_labels(section, base_addr, data_labels or {}) + if self._builtin_encoder is not None: + return self._calculate_builtin_text_labels(section, base_addr, data_labels or {}) # Fixed-size ISA: single pass with size estimation labels: Dict[str, int] = {} @@ -403,17 +411,18 @@ def _calculate_text_labels( return labels - def _calculate_riscv_text_labels( + def _calculate_builtin_text_labels( self, section: SectionData, base_addr: int, data_labels: Dict[str, int], ) -> Dict[str, int]: """ - Calculate RISC-V label addresses using the encoder's own sizing. + Calculate label addresses using the built-in encoder's own sizing. - Only li's size depends on a value, so this converges in a pass or two; - the loop re-sizes with the labels found so far until they are stable. + Only a few pseudo-instructions (li, la, constant operands) have + value-dependent sizes, so this converges in a pass or two; the loop + re-sizes with the labels found so far until they are stable. """ labels: Dict[str, int] = {} for _ in range(8): @@ -424,7 +433,7 @@ def _calculate_riscv_text_labels( if line.label: new_labels[line.label] = addr if line.line_type == LineType.INSTRUCTION and line.instruction: - addr += riscv.instruction_size(line.instruction, known) + addr += self._builtin_encoder.instruction_size(line.instruction, known, addr) if new_labels == labels: break labels = new_labels @@ -553,49 +562,8 @@ def _estimate_fixed_instr_size( mnemonic = parts[0].lower() - # MIPS pseudo-instructions and branch delay slots - if self.isa in ("mips32", "mips"): - # Pseudo-branches expand to 3 instructions: slt + bne + nop (delay slot) - if mnemonic in ("blt", "bge", "ble", "bgt"): - return base_size * 3 - # Regular branches/jumps include delay slot nop: 2 instructions - if mnemonic in ("j", "jal", "jr", "jalr", "beq", "bne", "beqz", "bnez", - "bgtz", "bgez", "bltz", "blez", "bgezal", "bltzal"): - return base_size * 2 # Branch + delay slot nop - # li with large immediate expands to lui + ori - if mnemonic == "li": - operands = parts[1] if len(parts) > 1 else "" - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 2: - try: - imm_str = ops[1].strip() - if imm_str.startswith('0x'): - imm = int(imm_str, 16) - else: - imm = int(imm_str) - # MIPS immediate is 16-bit signed - if not (-32768 <= imm <= 65535): - return base_size * 2 # lui + ori - except ValueError: - pass - # la (load address) - size depends on target address lower 16 bits - if mnemonic == "la": - operands = parts[1] if len(parts) > 1 else "" - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 2 and data_labels: - symbol = ops[1].strip() - if symbol in data_labels: - target = data_labels[symbol] - # If lower 16 bits are 0, only lui is needed (4 bytes) - # Otherwise lui + ori (8 bytes) - if (target & 0xFFFF) == 0: - return base_size # lui only - return base_size * 2 # lui + ori - # Unknown symbol - conservatively estimate as lui + ori - return base_size * 2 - - if self._is_riscv: - return riscv.instruction_size(instr, data_labels or {}) + if self._builtin_encoder is not None: + return self._builtin_encoder.instruction_size(instr, data_labels or {}) # ARM64 pseudo-instructions that expand to multiple instructions if self.isa in ("arm64", "aarch64"): @@ -726,22 +694,19 @@ def _assemble_section( instr = line.instruction - # RISC-V: built-in encoder handles instructions and pseudo-instructions - if self._is_riscv: + # RISC-V and MIPS: built-in encoders handle instructions and + # pseudo-instructions directly + encoder = self._builtin_encoder + if encoder is not None: try: - encoding = riscv.encode(instr, current_addr, labels) - except riscv.RISCVEncodeError as e: + encoding = encoder.encode(instr, current_addr, labels) + except EncodeError as e: errors.append(f"Line {line.line_number}: {e} - {line.instruction}") continue code.extend(encoding) current_addr += len(encoding) continue - # Expand pseudo-instructions for MIPS and normalize syntax - if self.isa in ("mips32", "mips"): - instr = self._normalize_mips_syntax(instr) - instr = self._expand_mips_pseudo(instr, current_addr, labels) - # Expand pseudo-instructions for ARM64 and normalize syntax if self.isa in ("arm64", "aarch64"): instr = self._normalize_arm64_syntax(instr) @@ -776,150 +741,6 @@ def _assemble_section( return bytes(code), label_addrs, errors, debug_lines - def _normalize_mips_syntax(self, instr: str) -> str: - """ - Normalize MIPS syntax for Keystone compatibility. - - Keystone doesn't recognize $zero - convert to $0. - """ - # Replace $zero with $0 - return instr.replace("$zero", "$0") - - def _expand_mips_pseudo( - self, - instr: str, - addr: int, - labels: Dict[str, int], - ) -> str: - """ - Expand MIPS pseudo-instructions. - - Handles la (load address) and li (load immediate). - """ - parts = instr.split(None, 1) - if not parts: - return instr - - mnemonic = parts[0].lower() - operands = parts[1] if len(parts) > 1 else "" - - # Handle 'li' (load immediate) - if mnemonic == "li": - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 2: - reg = ops[0] - try: - imm_str = ops[1].strip() - if imm_str in labels: - imm = labels[imm_str] - elif imm_str.startswith('0x'): - imm = int(imm_str, 16) - else: - imm = int(imm_str) - - # Small immediate: use addiu with $0 - if -32768 <= imm <= 32767: - return f"addiu {reg}, $0, {imm}" - - # Large immediate: use lui + ori - upper = (imm >> 16) & 0xFFFF - lower = imm & 0xFFFF - if lower == 0: - return f"lui {reg}, {upper}" - return f"lui {reg}, {upper}; ori {reg}, {reg}, {lower}" - - except (ValueError, KeyError): - pass - - # Handle 'la' (load address) - if mnemonic == "la": - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 2: - reg = ops[0] - symbol = ops[1].strip() - if symbol in labels: - target = labels[symbol] - return self._expand_mips_pseudo(f"li {reg}, {target}", addr, labels) - - # Handle 'j' (unconditional jump) - Keystone expects byte address - if mnemonic == "j": - symbol = operands.strip() - if symbol in labels: - target = labels[symbol] - return f"j {target}" - - # Handle 'jal' (jump and link) - Keystone expects byte address - if mnemonic == "jal": - symbol = operands.strip() - if symbol in labels: - target = labels[symbol] - return f"jal {target}" - - # Handle branch instructions with symbols - mips_branches = ["beq", "bne", "blez", "bgtz", "bltz", "bgez"] - if mnemonic in mips_branches: - ops = [o.strip() for o in operands.split(',')] - # beq/bne: rs, rt, offset - if mnemonic in ("beq", "bne") and len(ops) == 3: - rs, rt, symbol = ops[0], ops[1], ops[2].strip() - if symbol in labels: - target = labels[symbol] - # Keystone expects offset from text_base, not from current instruction - offset = target - self._layout.text_base - # Convert $zero to $0 for Keystone - rs = "$0" if rs == "$zero" else rs - rt = "$0" if rt == "$zero" else rt - return f"{mnemonic} {rs}, {rt}, {offset}" - # blez/bgtz/bltz/bgez: rs, offset - elif len(ops) == 2: - rs, symbol = ops[0], ops[1].strip() - if symbol in labels: - target = labels[symbol] - # Keystone expects offset from text_base, not from current instruction - offset = target - self._layout.text_base - rs = "$0" if rs == "$zero" else rs - return f"{mnemonic} {rs}, {offset}" - - # Handle pseudo-branch instructions (blt, bge, ble, bgt) - # These expand to: slt $at, rs, rt; beq/bne $at, $0, offset - pseudo_branches = { - "blt": ("slt", "bne"), # blt rs, rt → slt $at, rs, rt; bne $at, $0 - "bge": ("slt", "beq"), # bge rs, rt → slt $at, rs, rt; beq $at, $0 - "ble": ("slt", "beq", True), # ble rs, rt → slt $at, rt, rs; beq $at, $0 (swap) - "bgt": ("slt", "bne", True), # bgt rs, rt → slt $at, rt, rs; bne $at, $0 (swap) - } - if mnemonic in pseudo_branches: - ops = [o.strip() for o in operands.split(',')] - if len(ops) == 3: - rs, rt, symbol = ops[0], ops[1], ops[2].strip() - if symbol in labels: - target = labels[symbol] - config = pseudo_branches[mnemonic] - slt_op = config[0] - branch_op = config[1] - swap = len(config) > 2 and config[2] - - # Normalize register names - rs = "$0" if rs == "$zero" else rs - rt = "$0" if rt == "$zero" else rt - - # Build the slt instruction - if swap: - slt_instr = f"{slt_op} $1, {rt}, {rs}" # $1 is $at - else: - slt_instr = f"{slt_op} $1, {rs}, {rt}" - - # Keystone expects offset from text_base, not from current instruction. - # When we return "slt; beq", Keystone assembles both at `addr`, but - # the beq is actually 4 bytes after the slt. Keystone miscalculates - # the branch offset by 4 bytes, so we compensate by subtracting 4. - offset = target - self._layout.text_base - 4 - branch_instr = f"{branch_op} $1, $0, {offset}" - - return f"{slt_instr}; {branch_instr}" - - return instr - def _normalize_arm64_syntax(self, instr: str) -> str: """ Normalize ARM64 syntax for Keystone compatibility. @@ -1194,10 +1015,10 @@ def assemble_instruction(self, instr: str, address: int = 0) -> bytes: Raises: ValueError: If assembly fails. """ - if self._is_riscv: + if self._builtin_encoder is not None: try: - return riscv.encode(instr, address, {}) - except riscv.RISCVEncodeError as e: + return self._builtin_encoder.encode(instr, address, {}) + except EncodeError as e: raise ValueError(f"Assembly error: {e}") try: encoding, count = self._ks.asm(instr, address) diff --git a/mapachespim/toolchain/expr.py b/mapachespim/toolchain/expr.py new file mode 100644 index 0000000..55046b2 --- /dev/null +++ b/mapachespim/toolchain/expr.py @@ -0,0 +1,143 @@ +""" +Operand parsing and expression evaluation shared by the built-in encoders. +""" + +from __future__ import annotations + +import re +from typing import Dict, List, Optional, Type + + +class EncodeError(ValueError): + """Raised when an instruction cannot be encoded.""" + + +_RELOC_RE = re.compile(r"^%(hi|lo|pcrel_hi|pcrel_lo)\((.*)\)$", re.IGNORECASE) +_TERM_RE = re.compile(r"\s*([+-]?)\s*([^+\-\s][^+\-]*)") + + +def parse_int(text: str) -> Optional[int]: + """Parse an integer or character literal, or return None.""" + t = text.strip() + if len(t) >= 3 and t[0] == "'" and t[-1] == "'": + body = t[1:-1] + escapes = {"\\n": 10, "\\t": 9, "\\r": 13, "\\0": 0, "\\\\": 92, "\\'": 39} + if body in escapes: + return escapes[body] + if len(body) == 1: + return ord(body) + return None + try: + return int(t, 0) + except ValueError: + pass + # int(..., 0) rejects leading zeros like "010"; accept them as decimal + if re.fullmatch(r"[+-]?\d+", t): + return int(t, 10) + return None + + +def sign_extend(value: int, bits: int) -> int: + """Interpret the low ``bits`` bits of value as a signed integer.""" + value &= (1 << bits) - 1 + return value - (1 << bits) if value & (1 << (bits - 1)) else value + + +class Context: + """Address and symbol information needed to encode one instruction.""" + + def __init__( + self, + address: int, + labels: Dict[str, int], + strict: bool = True, + error: Type[EncodeError] = EncodeError, + lo_bits: int = 12, + ): + self.address = address + self.labels = labels + # When not strict (the sizing pass), unknown symbols evaluate to 0 + self.strict = strict + self.error = error + # Width of the low part for %hi/%lo: 12 on RISC-V, 16 on MIPS + self.lo_bits = lo_bits + + def symbol(self, name: str) -> int: + if name in self.labels: + return self.labels[name] + if self.strict: + raise self.error(f"undefined symbol '{name}'") + return 0 + + def eval(self, text: str) -> int: + """Evaluate an operand expression such as 'label+8' or '%lo(msg)'.""" + text = text.strip() + if not text: + raise self.error("missing operand") + + m = _RELOC_RE.match(text) + if m: + kind, inner = m.group(1).lower(), m.group(2) + value = self.eval(inner) + lo_bits = self.lo_bits + hi_mask = (1 << (32 - lo_bits)) - 1 + if kind == "hi": + # Rounded so that (%hi << lo_bits) + sign-extended %lo == value + return ((value + (1 << (lo_bits - 1))) >> lo_bits) & hi_mask + if kind == "lo": + return sign_extend(value, lo_bits) + if kind == "pcrel_hi": + offset = value - self.address + return ((offset + 0x800) >> 12) & 0xFFFFF + # %pcrel_lo(label) names the auipc instruction; the low part is + # relative to that instruction's pc and the auipc's target. + raise self.error("%pcrel_lo is not supported; use la or lla instead") + + value = parse_int(text) + if value is not None: + return value + + total = 0 + pos = 0 + matched = False + for m2 in _TERM_RE.finditer(text): + if m2.start() != pos and text[pos : m2.start()].strip(): + break + sign, term = m2.group(1), m2.group(2).strip() + v = parse_int(term) + if v is None: + if not re.fullmatch(r"[A-Za-z_.$][\w.$]*", term): + raise self.error(f"cannot parse expression '{text}'") + v = self.symbol(term) + total += -v if sign == "-" else v + pos = m2.end() + matched = True + if not matched or text[pos:].strip(): + raise self.error(f"cannot parse expression '{text}'") + return total + + +def split_operands(text: str) -> List[str]: + """Split operands on commas that are not inside parentheses or quotes.""" + ops: List[str] = [] + depth = 0 + quote = False + cur = "" + for ch in text: + if ch == "'" and depth == 0: + quote = not quote + if not quote: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + elif ch == "," and depth == 0: + ops.append(cur.strip()) + cur = "" + continue + cur += ch + if cur.strip() or ops: + ops.append(cur.strip()) + return ops + + diff --git a/mapachespim/toolchain/mips.py b/mapachespim/toolchain/mips.py new file mode 100644 index 0000000..8bc2197 --- /dev/null +++ b/mapachespim/toolchain/mips.py @@ -0,0 +1,457 @@ +""" +Pure-Python MIPS32 (big-endian) instruction encoder. + +MIPS is the ISA SPIM itself simulates, so it should assemble everywhere +without a native library. Like GNU as in its default ".set reorder" mode, +every branch and jump is followed by a nop in its delay slot, so programs +behave as students expect even though the simulator models delay slots. + +Supported: + - MIPS32 integer instructions: arithmetic/logic, shifts, set-less-than, + mult/div and hi/lo moves, mul/madd/msub/clz/clo, loads and stores + (including unaligned lwl/lwr/swl/swr), branches, jumps, syscall, break + - Pseudo-instructions: nop, move, li, la, not, neg, negu, abs, b, bal, + beqz, bnez, blt, bgt, ble, bge (and unsigned variants), mul with an + immediate, three-operand div/divu/rem/remu, lw/sw/... from a label + - Registers as $0-$31 or by name ($zero, $at, $v0, ..., $s8/$fp, $ra) + - Operand expressions: integers, character literals, symbols, + symbol +/- constant, %hi(), %lo() +""" + +from __future__ import annotations + +import re +from typing import Dict, List, Tuple + +from .expr import Context, EncodeError, parse_int, sign_extend, split_operands + +INSTR_SIZE = 4 +NOP = 0x00000000 +AT = 1 # assembler temporary used by pseudo-instructions + + +class MIPSEncodeError(EncodeError): + """Raised when a MIPS instruction cannot be encoded.""" + + +# --------------------------------------------------------------------------- +# Registers +# --------------------------------------------------------------------------- + +_NAMES = [ + "zero", "at", "v0", "v1", "a0", "a1", "a2", "a3", + "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", + "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", + "t8", "t9", "k0", "k1", "gp", "sp", "fp", "ra", +] # fmt: skip + +REGISTERS: Dict[str, int] = {name: i for i, name in enumerate(_NAMES)} +REGISTERS.update({str(i): i for i in range(32)}) +REGISTERS["s8"] = 30 + + +def parse_register(text: str) -> int: + t = text.strip() + if t.startswith("$") and t[1:].lower() in REGISTERS: + return REGISTERS[t[1:].lower()] + if t.lower() in REGISTERS and not t.isdigit(): + raise MIPSEncodeError(f"unknown register '{t}' (MIPS registers need a $: ${t})") + raise MIPSEncodeError(f"unknown register '{t}' (registers look like $t0 or $8)") + + +def _is_register(text: str) -> bool: + t = text.strip() + return t.startswith("$") and t[1:].lower() in REGISTERS + + +# --------------------------------------------------------------------------- +# Formats +# --------------------------------------------------------------------------- + + +def _r(rs: int, rt: int, rd: int, shamt: int, funct: int, opcode: int = 0) -> int: + return (opcode << 26) | (rs << 21) | (rt << 16) | (rd << 11) | (shamt << 6) | funct + + +def _i(opcode: int, rs: int, rt: int, imm: int) -> int: + return (opcode << 26) | (rs << 21) | (rt << 16) | (imm & 0xFFFF) + + +def _signed16(value: int, what: str = "immediate") -> int: + if not -0x8000 <= value <= 0x7FFF: + raise MIPSEncodeError(f"{what} {value} out of range (-32768 to 32767)") + return value + + +def _unsigned16(value: int, what: str = "immediate") -> int: + if not 0 <= value <= 0xFFFF: + raise MIPSEncodeError(f"{what} {value} out of range (0 to 65535)") + return value + + +# funct codes for opcode 0 (SPECIAL) +ALU3: Dict[str, int] = { + "add": 0x20, "addu": 0x21, "sub": 0x22, "subu": 0x23, + "and": 0x24, "or": 0x25, "xor": 0x26, "nor": 0x27, + "slt": 0x2A, "sltu": 0x2B, + "movn": 0x0B, "movz": 0x0A, +} # fmt: skip +SHIFT_IMM: Dict[str, int] = {"sll": 0x00, "srl": 0x02, "sra": 0x03} +SHIFT_VAR: Dict[str, int] = {"sllv": 0x04, "srlv": 0x06, "srav": 0x07} +MULDIV: Dict[str, int] = {"mult": 0x18, "multu": 0x19, "div": 0x1A, "divu": 0x1B} +HILO_TO: Dict[str, int] = {"mfhi": 0x10, "mflo": 0x12} +HILO_FROM: Dict[str, int] = {"mthi": 0x11, "mtlo": 0x13} +# SPECIAL2 (opcode 0x1C) +SPECIAL2_ACC: Dict[str, int] = {"madd": 0x00, "maddu": 0x01, "msub": 0x04, "msubu": 0x05} + +ALU_IMM: Dict[str, Tuple[int, bool]] = { + # name: (opcode, signed immediate) + "addi": (0x08, True), "addiu": (0x09, True), "slti": (0x0A, True), + "sltiu": (0x0B, True), "andi": (0x0C, False), "ori": (0x0D, False), + "xori": (0x0E, False), +} # fmt: skip +# Register-form mnemonics that take an immediate third operand (like GNU as) +IMM_FORMS: Dict[str, str] = { + "add": "addi", "addu": "addiu", "slt": "slti", "sltu": "sltiu", + "and": "andi", "or": "ori", "xor": "xori", +} # fmt: skip + +LOADS: Dict[str, int] = { + "lb": 0x20, "lh": 0x21, "lwl": 0x22, "lw": 0x23, "lbu": 0x24, "lhu": 0x25, "lwr": 0x26, + "ll": 0x30, +} # fmt: skip +STORES: Dict[str, int] = {"sb": 0x28, "sh": 0x29, "swl": 0x2A, "sw": 0x2B, "swr": 0x2E, "sc": 0x38} + +BRANCH2: Dict[str, int] = {"beq": 0x04, "bne": 0x05} +BRANCH1: Dict[str, Tuple[int, int]] = { + # name: (opcode, rt field) + "blez": (0x06, 0), "bgtz": (0x07, 0), + "bltz": (0x01, 0x00), "bgez": (0x01, 0x01), + "bltzal": (0x01, 0x10), "bgezal": (0x01, 0x11), +} # fmt: skip +# Comparison pseudo-branches: name -> (slt variant, swap operands, branch if $at != 0) +COMPARE_BRANCHES: Dict[str, Tuple[str, bool, bool]] = { + "blt": ("slt", False, True), "bge": ("slt", False, False), + "bgt": ("slt", True, True), "ble": ("slt", True, False), + "bltu": ("sltu", False, True), "bgeu": ("sltu", False, False), + "bgtu": ("sltu", True, True), "bleu": ("sltu", True, False), +} # fmt: skip + +_MEM_RE = re.compile(r"^(.*)\(\s*(\$\w+)\s*\)$") + + +def _parse_mem(text: str, ctx: Context) -> Tuple[int, int]: + """Parse 'offset($reg)' into (offset, reg).""" + m = _MEM_RE.match(text.strip()) + if not m: + raise MIPSEncodeError(f"expected offset($register), got '{text}'") + offset_text = m.group(1).strip() + offset = ctx.eval(offset_text) if offset_text else 0 + return _signed16(offset, "offset"), parse_register(m.group(2)) + + +def _expect(ops: List[str], n: int, usage: str) -> None: + if len(ops) != n: + raise MIPSEncodeError(f"expected {usage}") + + +# --------------------------------------------------------------------------- +# Pseudo-instruction helpers +# --------------------------------------------------------------------------- + + +def _li(rt: int, value: int) -> List[int]: + """Load a 32-bit constant (same choices as GNU as).""" + value &= 0xFFFFFFFF + signed = sign_extend(value, 32) + if -0x8000 <= signed <= 0x7FFF: + return [_i(0x09, 0, rt, signed)] # addiu rt, $zero, value + if value <= 0xFFFF: + return [_i(0x0D, 0, rt, value)] # ori rt, $zero, value + words = [_i(0x0F, 0, rt, value >> 16)] # lui + if value & 0xFFFF: + words.append(_i(0x0D, rt, rt, value & 0xFFFF)) # ori + return words + + +def _la(rt: int, address: int) -> List[int]: + """Load an address: lui, plus ori when the low half is non-zero.""" + address &= 0xFFFFFFFF + words = [_i(0x0F, 0, rt, address >> 16)] + if address & 0xFFFF: + words.append(_i(0x0D, rt, rt, address & 0xFFFF)) + return words + + +def _hi_lo(address: int) -> Tuple[int, int]: + """Split an address for lui + signed 16-bit offset.""" + lo = sign_extend(address, 16) + hi = ((address - lo) >> 16) & 0xFFFF + return hi, lo + + +# --------------------------------------------------------------------------- +# Encoder +# --------------------------------------------------------------------------- + + +def _encode(mnemonic: str, ops: List[str], ctx: Context) -> List[int]: + m = mnemonic + pc = ctx.address + + def branch_offset(text: str, at: int) -> int: + """Word offset from the delay slot of a branch at address `at`.""" + value = parse_int(text) + target = value if value is not None else ctx.eval(text) + if not ctx.strict: + return 0 # sizing pass: labels may not be resolved yet + delta = target - (at + 4) + if delta % 4: + raise MIPSEncodeError(f"branch target 0x{target:x} is not word-aligned") + return _signed16(delta >> 2, "branch offset") + + def jump_target(text: str) -> int: + target = ctx.eval(text) + if not ctx.strict: + return 0 + if target % 4: + raise MIPSEncodeError(f"jump target 0x{target:x} is not word-aligned") + if (target & 0xF0000000) != ((pc + 4) & 0xF0000000): + raise MIPSEncodeError(f"jump target 0x{target:x} is outside the current 256MB region") + return (target >> 2) & 0x03FFFFFF + + # --- no-operand --- + if m in ("nop", "syscall", "break", "eret"): + if m == "nop": + return [NOP] + if m == "eret": + return [0x42000018] + code = 0 + if ops: + _expect(ops, 1, f"{m} [code]") + code = ctx.eval(ops[0]) & 0xFFFFF + return [(code << 6) | (0x0C if m == "syscall" else 0x0D)] + + # --- R-type --- + if m in ALU3: + _expect(ops, 3, f"{m} $rd, $rs, $rt") + bare_register_name = ops[2].lower() in REGISTERS and not ops[2].isdigit() + if m in IMM_FORMS and not ops[2].startswith("$") and not bare_register_name: + return _encode(IMM_FORMS[m], ops, ctx) + rd, rs, rt = (parse_register(o) for o in ops) + return [_r(rs, rt, rd, 0, ALU3[m])] + + if m in SHIFT_IMM: + _expect(ops, 3, f"{m} $rd, $rt, shamt") + if _is_register(ops[2]): + return _encode(m + "v", ops, ctx) + shamt = ctx.eval(ops[2]) + if not 0 <= shamt <= 31: + raise MIPSEncodeError(f"shift amount {shamt} out of range (0 to 31)") + return [_r(0, parse_register(ops[1]), parse_register(ops[0]), shamt, SHIFT_IMM[m])] + + if m in SHIFT_VAR: + _expect(ops, 3, f"{m} $rd, $rt, $rs") + rd, rt, rs = (parse_register(o) for o in ops) + return [_r(rs, rt, rd, 0, SHIFT_VAR[m])] + + if m in MULDIV: + if len(ops) == 3 and m in ("div", "divu"): + # Three-operand pseudo form: quotient into rd + rd = parse_register(ops[0]) + return [_r(parse_register(ops[1]), parse_register(ops[2]), 0, 0, MULDIV[m]), _r(0, 0, rd, 0, 0x12)] + _expect(ops, 2, f"{m} $rs, $rt") + return [_r(parse_register(ops[0]), parse_register(ops[1]), 0, 0, MULDIV[m])] + + if m in ("rem", "remu"): + _expect(ops, 3, f"{m} $rd, $rs, $rt") + rd = parse_register(ops[0]) + funct = MULDIV["div" if m == "rem" else "divu"] + return [_r(parse_register(ops[1]), parse_register(ops[2]), 0, 0, funct), _r(0, 0, rd, 0, 0x10)] + + if m in HILO_TO: + _expect(ops, 1, f"{m} $rd") + return [_r(0, 0, parse_register(ops[0]), 0, HILO_TO[m])] + + if m in HILO_FROM: + _expect(ops, 1, f"{m} $rs") + return [_r(parse_register(ops[0]), 0, 0, 0, HILO_FROM[m])] + + if m == "mul": + _expect(ops, 3, "mul $rd, $rs, $rt") + rd, rs = parse_register(ops[0]), parse_register(ops[1]) + if _is_register(ops[2]): + return [_r(rs, parse_register(ops[2]), rd, 0, 0x02, opcode=0x1C)] + # mul by constant: load it into $at first + return _li(AT, ctx.eval(ops[2])) + [_r(rs, AT, rd, 0, 0x02, opcode=0x1C)] + + if m in SPECIAL2_ACC: + _expect(ops, 2, f"{m} $rs, $rt") + return [_r(parse_register(ops[0]), parse_register(ops[1]), 0, 0, SPECIAL2_ACC[m], opcode=0x1C)] + + if m in ("clz", "clo"): + _expect(ops, 2, f"{m} $rd, $rs") + rd, rs = parse_register(ops[0]), parse_register(ops[1]) + return [_r(rs, rd, rd, 0, 0x20 if m == "clz" else 0x21, opcode=0x1C)] + + # --- jumps (each followed by a delay-slot nop) --- + if m == "jr": + _expect(ops, 1, "jr $rs") + return [_r(parse_register(ops[0]), 0, 0, 0, 0x08), NOP] + + if m == "jalr": + if len(ops) == 1: + rd, rs = 31, parse_register(ops[0]) + else: + _expect(ops, 2, "jalr [$rd,] $rs") + rd, rs = parse_register(ops[0]), parse_register(ops[1]) + return [_r(rs, 0, rd, 0, 0x09), NOP] + + if m in ("j", "jal"): + _expect(ops, 1, f"{m} label") + if m == "j" and _is_register(ops[0]): + return _encode("jr", ops, ctx) + opcode = 0x02 if m == "j" else 0x03 + return [(opcode << 26) | jump_target(ops[0]), NOP] + + # --- branches --- + if m in BRANCH2: + _expect(ops, 3, f"{m} $rs, $rt, label") + rs = parse_register(ops[0]) + if _is_register(ops[1]): + return [_i(BRANCH2[m], rs, parse_register(ops[1]), branch_offset(ops[2], pc)), NOP] + # Compare against a constant via $at + words = _li(AT, ctx.eval(ops[1])) + at = pc + 4 * len(words) + return words + [_i(BRANCH2[m], rs, AT, branch_offset(ops[2], at)), NOP] + + if m in BRANCH1: + _expect(ops, 2, f"{m} $rs, label") + opcode, rt = BRANCH1[m] + return [_i(opcode, parse_register(ops[0]), rt, branch_offset(ops[1], pc)), NOP] + + if m in ("beqz", "bnez"): + _expect(ops, 2, f"{m} $rs, label") + opcode = BRANCH2["beq" if m == "beqz" else "bne"] + return [_i(opcode, parse_register(ops[0]), 0, branch_offset(ops[1], pc)), NOP] + + if m in ("b", "bal"): + _expect(ops, 1, f"{m} label") + if m == "b": + return [_i(0x04, 0, 0, branch_offset(ops[0], pc)), NOP] # beq $0, $0 + return [_i(0x01, 0, 0x11, branch_offset(ops[0], pc)), NOP] # bgezal $0 + + if m in COMPARE_BRANCHES: + _expect(ops, 3, f"{m} $rs, $rt, label") + slt, swap, if_set = COMPARE_BRANCHES[m] + rs = parse_register(ops[0]) + words: List[int] = [] + if _is_register(ops[1]): + rt = parse_register(ops[1]) + else: + words = _li(AT, ctx.eval(ops[1])) + rt = AT + a, b = (rt, rs) if swap else (rs, rt) + words.append(_r(a, b, AT, 0, ALU3[slt])) + at = pc + 4 * len(words) + opcode = BRANCH2["bne" if if_set else "beq"] + return words + [_i(opcode, AT, 0, branch_offset(ops[2], at)), NOP] + + # --- immediates --- + if m in ALU_IMM: + _expect(ops, 3, f"{m} $rt, $rs, imm") + opcode, signed = ALU_IMM[m] + rt, rs = parse_register(ops[0]), parse_register(ops[1]) + imm = ctx.eval(ops[2]) + imm = _signed16(imm) if signed else _unsigned16(imm) + return [_i(opcode, rs, rt, imm)] + + if m == "lui": + _expect(ops, 2, "lui $rt, imm") + imm = ctx.eval(ops[1]) + if -0x8000 <= imm < 0: + imm &= 0xFFFF + return [_i(0x0F, 0, parse_register(ops[0]), _unsigned16(imm))] + + # --- loads and stores --- + if m in LOADS or m in STORES: + opcode = LOADS.get(m, STORES.get(m)) + _expect(ops, 2, f"{m} $rt, offset($rs)") + rt = parse_register(ops[0]) + if _MEM_RE.match(ops[1].strip()): + offset, base = _parse_mem(ops[1], ctx) + return [_i(opcode, base, rt, offset)] + # 'lw $t0, label' -> lui $at, %hi(label); lw $t0, %lo(label)($at) + hi, lo = _hi_lo(ctx.eval(ops[1])) + return [_i(0x0F, 0, AT, hi), _i(opcode, AT, rt, lo)] + + # --- data movement pseudo-instructions --- + if m == "move": + _expect(ops, 2, "move $rd, $rs") + return [_r(parse_register(ops[1]), 0, parse_register(ops[0]), 0, ALU3["or"])] + + if m == "li": + _expect(ops, 2, "li $rt, imm") + return _li(parse_register(ops[0]), ctx.eval(ops[1])) + + if m == "la": + _expect(ops, 2, "la $rt, label") + return _la(parse_register(ops[0]), ctx.eval(ops[1])) + + if m == "not": + _expect(ops, 2, "not $rd, $rs") + return [_r(parse_register(ops[1]), 0, parse_register(ops[0]), 0, ALU3["nor"])] + + if m in ("neg", "negu"): + _expect(ops, 2, f"{m} $rd, $rs") + funct = ALU3["sub" if m == "neg" else "subu"] + return [_r(0, parse_register(ops[1]), parse_register(ops[0]), 0, funct)] + + if m == "abs": + _expect(ops, 2, "abs $rd, $rs") + rd, rs = parse_register(ops[0]), parse_register(ops[1]) + # sra $at, rs, 31; xor rd, rs, $at; subu rd, rd, $at + return [_r(0, rs, AT, 31, 0x03), _r(rs, AT, rd, 0, ALU3["xor"]), _r(rd, AT, rd, 0, ALU3["subu"])] + + raise MIPSEncodeError(f"unknown instruction '{mnemonic}'") + + +def _split_instruction(text: str) -> Tuple[str, List[str]]: + parts = text.strip().split(None, 1) + if not parts: + raise MIPSEncodeError("empty instruction") + return parts[0].lower(), split_operands(parts[1]) if len(parts) > 1 else [] + + +def _context(address: int, labels: Dict[str, int], strict: bool) -> Context: + return Context(address, labels, strict=strict, error=MIPSEncodeError, lo_bits=16) + + +def encode(text: str, address: int, labels: Dict[str, int]) -> bytes: + """ + Encode one MIPS instruction (or pseudo-instruction) to big-endian machine code. + + Branches and jumps include their delay-slot nop. + + Raises: + MIPSEncodeError: If the instruction is invalid. + """ + mnemonic, ops = _split_instruction(text) + words = _encode(mnemonic, ops, _context(address, labels, strict=True)) + return b"".join(w.to_bytes(4, "big") for w in words) + + +def instruction_size(text: str, labels: Dict[str, int], address: int = 0) -> int: + """ + Size in bytes that ``encode`` will produce, for the label-layout pass. + + Unknown labels evaluate to 0 here; the only value-dependent sizes are li, + la, and constant operands, and the assembler re-runs layout until label + addresses are stable. + """ + mnemonic, ops = _split_instruction(text) + try: + words = _encode(mnemonic, ops, _context(address, labels, strict=False)) + except MIPSEncodeError: + return INSTR_SIZE # the real error is reported by encode() + return len(words) * INSTR_SIZE diff --git a/mapachespim/toolchain/riscv.py b/mapachespim/toolchain/riscv.py index cd16b92..7e86e9e 100644 --- a/mapachespim/toolchain/riscv.py +++ b/mapachespim/toolchain/riscv.py @@ -24,13 +24,15 @@ from __future__ import annotations import re -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Tuple + +from .expr import Context, EncodeError, parse_int, sign_extend, split_operands INSTR_SIZE = 4 -class RISCVEncodeError(ValueError): - """Raised when an instruction cannot be encoded.""" +class RISCVEncodeError(EncodeError): + """Raised when a RISC-V instruction cannot be encoded.""" # --------------------------------------------------------------------------- @@ -69,101 +71,6 @@ def parse_register(text: str) -> int: } # fmt: skip -# --------------------------------------------------------------------------- -# Expressions -# --------------------------------------------------------------------------- - -_RELOC_RE = re.compile(r"^%(hi|lo|pcrel_hi|pcrel_lo)\((.*)\)$", re.IGNORECASE) -_TERM_RE = re.compile(r"\s*([+-]?)\s*([^+\-\s][^+\-]*)") - - -def _parse_int(text: str) -> Optional[int]: - """Parse an integer or character literal, or return None.""" - t = text.strip() - if len(t) >= 3 and t[0] == "'" and t[-1] == "'": - body = t[1:-1] - escapes = {"\\n": 10, "\\t": 9, "\\r": 13, "\\0": 0, "\\\\": 92, "\\'": 39} - if body in escapes: - return escapes[body] - if len(body) == 1: - return ord(body) - return None - try: - return int(t, 0) - except ValueError: - pass - # int(..., 0) rejects leading zeros like "010"; accept them as decimal - if re.fullmatch(r"[+-]?\d+", t): - return int(t, 10) - return None - - -class Context: - """Address and symbol information needed to encode one instruction.""" - - def __init__(self, address: int, labels: Dict[str, int], strict: bool = True): - self.address = address - self.labels = labels - # When not strict (the sizing pass), unknown symbols evaluate to 0 - self.strict = strict - - def symbol(self, name: str) -> int: - if name in self.labels: - return self.labels[name] - if self.strict: - raise RISCVEncodeError(f"undefined symbol '{name}'") - return 0 - - def eval(self, text: str) -> int: - """Evaluate an operand expression such as 'label+8' or '%lo(msg)'.""" - text = text.strip() - if not text: - raise RISCVEncodeError("missing operand") - - m = _RELOC_RE.match(text) - if m: - kind, inner = m.group(1).lower(), m.group(2) - value = self.eval(inner) - if kind == "hi": - return ((value + 0x800) >> 12) & 0xFFFFF - if kind == "lo": - return _sign_extend(value & 0xFFF, 12) - if kind == "pcrel_hi": - offset = value - self.address - return ((offset + 0x800) >> 12) & 0xFFFFF - # %pcrel_lo(label) names the auipc instruction; the low part is - # relative to that instruction's pc and the auipc's target. - raise RISCVEncodeError("%pcrel_lo is not supported; use la or lla instead") - - value = _parse_int(text) - if value is not None: - return value - - total = 0 - pos = 0 - matched = False - for m2 in _TERM_RE.finditer(text): - if m2.start() != pos and text[pos : m2.start()].strip(): - break - sign, term = m2.group(1), m2.group(2).strip() - v = _parse_int(term) - if v is None: - if not re.fullmatch(r"[A-Za-z_.$][\w.$]*", term): - raise RISCVEncodeError(f"cannot parse expression '{text}'") - v = self.symbol(term) - total += -v if sign == "-" else v - pos = m2.end() - matched = True - if not matched or text[pos:].strip(): - raise RISCVEncodeError(f"cannot parse expression '{text}'") - return total - - -def _sign_extend(value: int, bits: int) -> int: - value &= (1 << bits) - 1 - return value - (1 << bits) if value & (1 << (bits - 1)) else value - - def _check_signed(value: int, bits: int, what: str) -> int: lo, hi = -(1 << (bits - 1)), (1 << (bits - 1)) - 1 if not lo <= value <= hi: @@ -265,6 +172,13 @@ def _j(rd: int, offset: int) -> int: "slliw": (0x1B, 1, 0x00, 5), "srliw": (0x1B, 5, 0x00, 5), "sraiw": (0x1B, 5, 0x20, 5), } # fmt: skip +# Register-form mnemonics that accept an immediate third operand +IMM_FORMS: Dict[str, str] = { + "add": "addi", "and": "andi", "or": "ori", "xor": "xori", "slt": "slti", + "sltu": "sltiu", "sll": "slli", "srl": "srli", "sra": "srai", + "addw": "addiw", "sllw": "slliw", "srlw": "srliw", "sraw": "sraiw", +} # fmt: skip + LOADS: Dict[str, int] = {"lb": 0, "lh": 1, "lw": 2, "ld": 3, "lbu": 4, "lhu": 5, "lwu": 6} STORES: Dict[str, int] = {"sb": 0, "sh": 1, "sw": 2, "sd": 3} BRANCHES: Dict[str, int] = {"beq": 0, "bne": 1, "blt": 4, "bge": 5, "bltu": 6, "bgeu": 7} @@ -287,30 +201,6 @@ def _j(rd: int, offset: int) -> int: # --------------------------------------------------------------------------- -def split_operands(text: str) -> List[str]: - """Split operands on commas that are not inside parentheses or quotes.""" - ops: List[str] = [] - depth = 0 - quote = False - cur = "" - for ch in text: - if ch == "'" and depth == 0: - quote = not quote - if not quote: - if ch == "(": - depth += 1 - elif ch == ")": - depth -= 1 - elif ch == "," and depth == 0: - ops.append(cur.strip()) - cur = "" - continue - cur += ch - if cur.strip() or ops: - ops.append(cur.strip()) - return ops - - _MEM_RE = re.compile(r"^(.*)\(\s*([A-Za-z0-9]+)\s*\)$") @@ -348,25 +238,25 @@ def _parse_csr(text: str, ctx: Context) -> int: def _li_sequence(rd: int, value: int) -> List[int]: """Instruction words that load a 64-bit constant into rd.""" - value = _sign_extend(value, 64) + value = sign_extend(value, 64) if -2048 <= value <= 2047: return [_i(0x13, 0, rd, 0, value)] # addi rd, x0, value if -(1 << 31) <= value < (1 << 31): - lo = _sign_extend(value & 0xFFF, 12) + lo = sign_extend(value & 0xFFF, 12) hi = ((value - lo) >> 12) & 0xFFFFF words = [_u(0x37, rd, hi)] # lui if lo: words.append(_i(0x1B, 0, rd, rd, lo)) # addiw keeps it 32-bit sign-extended return words # 64-bit: build the upper bits recursively, then shift in 12 bits at a time - lo = _sign_extend(value & 0xFFF, 12) + lo = sign_extend(value & 0xFFF, 12) upper = (value - lo) >> 12 shift = 12 while upper and not upper & 1 and shift < 60: upper >>= 1 shift += 1 words = _li_sequence(rd, upper) - words.append((((shift & 0x3F) << 20) | (rd << 15) | (1 << 12) | (rd << 7) | 0x13)) # slli + words.append(((shift & 0x3F) << 20) | (rd << 15) | (1 << 12) | (rd << 7) | 0x13) # slli if lo: words.append(_i(0x13, 0, rd, rd, lo)) return words @@ -375,7 +265,7 @@ def _li_sequence(rd: int, value: int) -> List[int]: def _pcrel_pair(target: int, pc: int) -> Tuple[int, int]: """Split a pc-relative offset into auipc (hi20) and addi/jalr (lo12) parts.""" offset = target - pc - lo = _sign_extend(offset & 0xFFF, 12) + lo = sign_extend(offset & 0xFFF, 12) hi = ((offset - lo) >> 12) & 0xFFFFF return hi, lo @@ -391,13 +281,25 @@ def _encode(mnemonic: str, ops: List[str], ctx: Context) -> List[int]: def target_offset(text: str) -> int: """Branch/jump operand: a label (pc-relative) or a literal offset.""" - value = _parse_int(text) + value = parse_int(text) if value is not None: return value + if not ctx.strict: + ctx.eval(text) # still validate the expression's syntax + return 0 # sizing pass: labels may not be resolved yet return ctx.eval(text) - pc if m in R_TYPE: _expect(ops, 3, f"{m} rd, rs1, rs2") + if m in IMM_FORMS and ops[2].strip().lower() not in REGISTERS: + # Like GNU as, 'add a0, a1, 5' means 'addi a0, a1, 5' + try: + return _encode(IMM_FORMS[m], ops, ctx) + except RISCVEncodeError as e: + # 'add a0, a1, q9' is far more likely a mistyped register + if str(e).startswith("undefined symbol"): + raise RISCVEncodeError(f"unknown register '{ops[2].strip()}'") + raise opcode, f3, f7 = R_TYPE[m] return [_r(opcode, f3, f7, parse_register(ops[0]), parse_register(ops[1]), parse_register(ops[2]))] @@ -601,11 +503,11 @@ def encode(text: str, address: int, labels: Dict[str, int]) -> bytes: RISCVEncodeError: If the instruction is invalid. """ mnemonic, ops = _split_instruction(text) - words = _encode(mnemonic, ops, Context(address, labels, strict=True)) + words = _encode(mnemonic, ops, Context(address, labels, strict=True, error=RISCVEncodeError)) return b"".join(w.to_bytes(4, "little") for w in words) -def instruction_size(text: str, labels: Dict[str, int]) -> int: +def instruction_size(text: str, labels: Dict[str, int], address: int = 0) -> int: """ Size in bytes that ``encode`` will produce, for the label-layout pass. @@ -623,7 +525,9 @@ def instruction_size(text: str, labels: Dict[str, int]) -> int: if mnemonic == "tail": return 2 * INSTR_SIZE try: - words = _encode(mnemonic, ops, Context(0, labels, strict=False)) + words = _encode( + mnemonic, ops, Context(address, labels, strict=False, error=RISCVEncodeError) + ) except RISCVEncodeError: return INSTR_SIZE # the real error is reported by encode() return len(words) * INSTR_SIZE diff --git a/tests/test_mips_encoder.py b/tests/test_mips_encoder.py new file mode 100644 index 0000000..3c6350b --- /dev/null +++ b/tests/test_mips_encoder.py @@ -0,0 +1,238 @@ +""" +Tests for the built-in pure-Python MIPS32 encoder. + +Encodings are checked with Capstone, and pseudo-instructions whose expansion +is easy to get wrong (li, compare-branches, div/rem, abs) are checked by +executing them in Unicorn. +""" + +import itertools +import unittest + +from capstone import CS_ARCH_MIPS, CS_MODE_BIG_ENDIAN, CS_MODE_MIPS32, Cs + +from mapachespim import ISA, Simulator +from mapachespim.toolchain import Assembler +from mapachespim.toolchain.mips import MIPSEncodeError, encode, instruction_size + +PC = 0x00400000 +LABELS = {"target": PC + 0x40, "back": PC - 0x20, "msg": 0x10000024, "CONST": 42} + +_cs = Cs(CS_ARCH_MIPS, CS_MODE_MIPS32 | CS_MODE_BIG_ENDIAN) + + +def disasm(code: bytes, addr: int = PC): + return [f"{i.mnemonic} {i.op_str}".strip() for i in _cs.disasm(code, addr)] + + +class TestInstructionForms(unittest.TestCase): + CASES = [ + # (source, expected Capstone output lines) + ("add $t0, $t1, $t2", ["add $t0, $t1, $t2"]), + ("addu $v0, $a0, $a1", ["addu $v0, $a0, $a1"]), + ("sub $t0, $t1, $t2", ["sub $t0, $t1, $t2"]), + ("subu $t0, $t1, $t2", ["subu $t0, $t1, $t2"]), + ("and $t0, $t1, $t2", ["and $t0, $t1, $t2"]), + ("or $t0, $t1, $t2", ["or $t0, $t1, $t2"]), + ("xor $t0, $t1, $t2", ["xor $t0, $t1, $t2"]), + ("nor $t0, $t1, $t2", ["nor $t0, $t1, $t2"]), + ("slt $t0, $t1, $t2", ["slt $t0, $t1, $t2"]), + ("sltu $t0, $t1, $t2", ["sltu $t0, $t1, $t2"]), + ("add $8, $9, $10", ["add $t0, $t1, $t2"]), + ("add $s8, $fp, $zero", ["add $fp, $fp, $zero"]), + ("sll $t0, $t1, 4", ["sll $t0, $t1, 4"]), + ("srl $t0, $t1, 31", ["srl $t0, $t1, 0x1f"]), + ("sra $t0, $t1, 2", ["sra $t0, $t1, 2"]), + ("sllv $t0, $t1, $t2", ["sllv $t0, $t1, $t2"]), + ("srlv $t0, $t1, $t2", ["srlv $t0, $t1, $t2"]), + ("srav $t0, $t1, $t2", ["srav $t0, $t1, $t2"]), + ("mult $t0, $t1", ["mult $t0, $t1"]), + ("multu $t0, $t1", ["multu $t0, $t1"]), + ("div $t0, $t1", ["div $zero, $t0, $t1"]), + ("divu $t0, $t1", ["divu $zero, $t0, $t1"]), + ("mfhi $t0", ["mfhi $t0"]), + ("mflo $t0", ["mflo $t0"]), + ("mthi $t0", ["mthi $t0"]), + ("mtlo $t0", ["mtlo $t0"]), + ("mul $t0, $t1, $t2", ["mul $t0, $t1, $t2"]), + ("madd $t0, $t1", ["madd $t0, $t1"]), + ("clz $t0, $t1", ["clz $t0, $t1"]), + ("addi $sp, $sp, -4", ["addi $sp, $sp, -4"]), + ("addiu $t0, $zero, 32767", ["addiu $t0, $zero, 0x7fff"]), + ("slti $t0, $t1, 5", ["slti $t0, $t1, 5"]), + ("sltiu $t0, $t1, 5", ["sltiu $t0, $t1, 5"]), + ("andi $t0, $t1, 0xffff", ["andi $t0, $t1, 0xffff"]), + ("ori $t0, $t1, 1", ["ori $t0, $t1, 1"]), + ("xori $t0, $t1, 1", ["xori $t0, $t1, 1"]), + ("lui $t0, 0x1000", ["lui $t0, 0x1000"]), + ("addi $t0, $t0, CONST", ["addi $t0, $t0, 0x2a"]), + ("add $t0, $t1, 5", ["addi $t0, $t1, 5"]), + ("lw $t0, 0($sp)", ["lw $t0, ($sp)"]), + ("lw $t0, ($sp)", ["lw $t0, ($sp)"]), + ("lb $t0, -1($a0)", ["lb $t0, -1($a0)"]), + ("lbu $t0, 1($a0)", ["lbu $t0, 1($a0)"]), + ("lh $t0, 2($a0)", ["lh $t0, 2($a0)"]), + ("lhu $t0, 2($a0)", ["lhu $t0, 2($a0)"]), + ("sw $ra, 4($sp)", ["sw $ra, 4($sp)"]), + ("sb $t0, 0($a0)", ["sb $t0, ($a0)"]), + ("sh $t0, 0($a0)", ["sh $t0, ($a0)"]), + ("lw $t0, msg", ["lui $at, 0x1000", "lw $t0, 0x24($at)"]), + ("syscall", ["syscall"]), + ("break", ["break"]), + ("nop", ["nop"]), + # Branches and jumps carry a delay-slot nop + ("beq $t0, $t1, target", ["beq $t0, $t1, 0x400040", "nop"]), + ("bne $t0, $t1, back", ["bne $t0, $t1, 0x3fffe0", "nop"]), + ("blez $t0, target", ["blez $t0, 0x400040", "nop"]), + ("bgtz $t0, target", ["bgtz $t0, 0x400040", "nop"]), + ("bltz $t0, target", ["bltz $t0, 0x400040", "nop"]), + ("bgez $t0, target", ["bgez $t0, 0x400040", "nop"]), + ("beqz $t0, target", ["beqz $t0, 0x400040", "nop"]), + ("bnez $t0, target", ["bnez $t0, 0x400040", "nop"]), + ("b target", ["b 0x400040", "nop"]), + ("j target", ["j 0x400040", "nop"]), + ("jal target", ["jal 0x400040", "nop"]), + ("jr $ra", ["jr $ra", "nop"]), + ("jalr $t9", ["jalr $t9", "nop"]), + # Pseudo-instructions + ("move $a0, $s0", ["move $a0, $s0"]), + ("li $v0, 10", ["addiu $v0, $zero, 0xa"]), + ("li $t0, -1", ["addiu $t0, $zero, -1"]), + ("li $t0, 0xffff", ["ori $t0, $zero, 0xffff"]), + ("li $t0, 0x10000", ["lui $t0, 1"]), + ("li $t0, 0x12345678", ["lui $t0, 0x1234", "ori $t0, $t0, 0x5678"]), + ("la $a0, msg", ["lui $a0, 0x1000", "ori $a0, $a0, 0x24"]), + ("not $t0, $t1", ["not $t0, $t1"]), + ("neg $t0, $t1", ["neg $t0, $t1"]), + ("blt $t0, $t1, target", ["slt $at, $t0, $t1", "bnez $at, 0x400040", "nop"]), + ("bgt $t0, $t1, target", ["slt $at, $t1, $t0", "bnez $at, 0x400040", "nop"]), + ("div $t0, $t1, $t2", ["div $zero, $t1, $t2", "mflo $t0"]), + ("rem $t0, $t1, $t2", ["div $zero, $t1, $t2", "mfhi $t0"]), + ] + + def test_forms(self): + for source, expected in self.CASES: + with self.subTest(source=source): + code = encode(source, PC, LABELS) + self.assertEqual(disasm(code), expected) + self.assertEqual(instruction_size(source, LABELS, PC), len(code)) + + +class TestErrors(unittest.TestCase): + CASES = [ + ("addi $t0, $t1, 40000", "out of range"), + ("andi $t0, $t1, -1", "out of range"), + ("add $t0, $t1", "expected"), + ("add $t0, $t1, $q9", "unknown register"), + ("add t0, t1, t2", "need a $"), + ("frob $t0", "unknown instruction"), + ("beq $t0, $t1, nowhere", "undefined symbol"), + ("sll $t0, $t1, 32", "out of range"), + ] + + def test_errors(self): + for source, message in self.CASES: + with self.subTest(source=source): + with self.assertRaises(MIPSEncodeError) as cm: + encode(source, PC, LABELS) + self.assertIn(message, str(cm.exception)) + + +def run_snippet(source: str, regs: dict) -> Simulator: + """Assemble `source` at PC, set registers, and run until it falls off the end.""" + sim = Simulator(ISA.MIPS) + code = b"" + labels = {"end": PC + 4 * 64} + for line in source.strip().splitlines(): + code += encode(line.strip(), PC + len(code), labels) + labels["end"] = PC + len(code) + code = b"" + for line in source.strip().splitlines(): + code += encode(line.strip(), PC + len(code), labels) + sim.write_mem(PC, code) + sim.set_pc(PC) + for reg, value in regs.items(): + sim.set_reg(reg, value & 0xFFFFFFFF) + for _ in range(64): + if sim.get_pc() == labels["end"]: + break + sim.step() + return sim + + +class TestExecution(unittest.TestCase): + T0, T1, T2 = 8, 9, 10 + + def test_li_values(self): + for value in [0, 1, -1, 0x7FFF, -0x8000, 0x8000, 0xFFFF, 0x10000, 0x12345678, + 0x7FFFFFFF, -0x80000000, 0xFFFFFFFF, 0xFFFF0000]: # fmt: skip + with self.subTest(value=hex(value)): + sim = run_snippet(f"li $t0, {value}", {}) + self.assertEqual(sim.get_reg(self.T0), value & 0xFFFFFFFF) + + def test_compare_branches(self): + values = [-5, -1, 0, 1, 5, 0x7FFFFFFF, -0x80000000] + checks = { + "blt": lambda a, b: a < b, + "bgt": lambda a, b: a > b, + "ble": lambda a, b: a <= b, + "bge": lambda a, b: a >= b, + "bltu": lambda a, b: (a & 0xFFFFFFFF) < (b & 0xFFFFFFFF), + "bgtu": lambda a, b: (a & 0xFFFFFFFF) > (b & 0xFFFFFFFF), + "bleu": lambda a, b: (a & 0xFFFFFFFF) <= (b & 0xFFFFFFFF), + "bgeu": lambda a, b: (a & 0xFFFFFFFF) >= (b & 0xFFFFFFFF), + } + for op, check in checks.items(): + for a, b in itertools.product(values, repeat=2): + with self.subTest(op=op, a=a, b=b): + # $t2 = 1 if the branch was taken, else 0 + sim = run_snippet( + f""" + li $t2, 1 + {op} $t0, $t1, end + li $t2, 0 + """, + {self.T0: a, self.T1: b}, + ) + self.assertEqual(sim.get_reg(self.T2), int(check(a, b))) + + def test_div_rem_abs(self): + sim = run_snippet( + """ + div $t2, $t0, $t1 + rem $t3, $t0, $t1 + abs $t4, $t0 + """, + {self.T0: -17, self.T1: 5}, + ) + self.assertEqual(sim.get_reg(10), (-3) & 0xFFFFFFFF) + self.assertEqual(sim.get_reg(11), (-2) & 0xFFFFFFFF) + self.assertEqual(sim.get_reg(12), 17) + + +class TestProgram(unittest.TestCase): + def test_forward_references(self): + source = """ + .text + .globl _start + _start: + la $a0, msg + blt $t0, $t1, skip + li $t2, 0x12345678 + skip: + jal func + li $v0, 10 + syscall + func: + jr $ra + .data + msg: .asciiz "hi" + """ + result = Assembler("mips32").assemble(source) + self.assertTrue(result.success, result.errors) + # msg is at 0x10000000, so la is just lui (1) + blt (3) + li (2) + self.assertEqual(result.symbols["skip"] - result.symbols["_start"], 24) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_riscv_encoder.py b/tests/test_riscv_encoder.py index 4dc2f92..0bade9c 100644 --- a/tests/test_riscv_encoder.py +++ b/tests/test_riscv_encoder.py @@ -76,6 +76,8 @@ class TestInstructionForms(unittest.TestCase): ("sraiw a0, a1, 2", "sraiw a0, a1, 2"), ("addi a0, a0, CONST", "addi a0, a0, 0x2a"), ("addi a0, a0, 'A'", "addi a0, a0, 0x41"), + ("add a0, a1, 5", "addi a0, a1, 5"), + ("sll a0, a1, 3", "slli a0, a1, 3"), # Loads / stores ("lb a0, 0(sp)", "lb a0, 0(sp)"), ("lh a0, -2(sp)", "lh a0, -2(sp)"), From 8eb74ab3f76541998c8f8e119a70e54a419baf82 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 00:23:11 +0000 Subject: [PATCH 04/18] Ship examples in the package; load .s files and examples by name After `pip install`, the examples referenced throughout the docs did not exist, and students had to run mapachespim-as separately before every load. - Wheels now include examples/ as mapachespim/examples; the new mapachespim.examples module finds them in wheels and checkouts - load accepts assembly sources (.s/.S/.asm), assembling them with debug info (ISA from .isa or `load prog.s riscv64`), with clear messages for a missing ISA and for assembly errors - load resolves example names (riscv/fibonacci) and the documented examples/... paths even outside the repository - New commands: examples (list bundled programs) and reload (re-assemble after editing; breakpoints on labels follow them) - New flags: --copy-examples [DIR], --isa for sources without .isa; -e accepts sources and example names - Tab completion offers example names; help/quickstart updated - Fix DWARF for big-endian targets: debug info was always written little-endian, so `list` never worked for MIPS. MIPS example binaries are rebuilt (only .debug_info/.debug_line change) - Handle BrokenPipeError when -e output is piped to e.g. head - Friendlier message when Keystone is missing for ARM64/x86-64 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- examples/mips/array_stats/array_stats | Bin 1812 -> 1812 bytes examples/mips/fibonacci/fibonacci | Bin 1088 -> 1088 bytes examples/mips/guess_game/guess_game | Bin 1736 -> 1736 bytes examples/mips/hello_asm/hello_asm | Bin 988 -> 988 bytes examples/mips/matrix_multiply/matrix_mult | Bin 1476 -> 1476 bytes mapachespim/console.py | 400 ++++++++++++++++++---- mapachespim/examples.py | 171 +++++++++ mapachespim/toolchain/assembler.py | 19 +- mapachespim/toolchain/dwarf.py | 31 +- pyproject.toml | 7 + tests/test_loading.py | 220 ++++++++++++ 11 files changed, 767 insertions(+), 81 deletions(-) create mode 100644 mapachespim/examples.py create mode 100644 tests/test_loading.py diff --git a/examples/mips/array_stats/array_stats b/examples/mips/array_stats/array_stats index 78e9615b409299cb4b6358b9071af9d6b1c93cd5..62a941ea10587b3422c849759f676acc2c5e67d6 100644 GIT binary patch delta 59 zcmbQjH-&G*CMF361_nb0CLqbc!U!T696*@y$>dW^vRpvs6tJZ7#) F0{|Sv3RwUE delta 59 zcmbQjH-&G*CMF3(1_lNuAZB1-VFXhQ4h&Bi9T+B`Vv^;W0u*6lPzI8dKQgH^F*t0N IV*bSp03mG(SpWb4 diff --git a/examples/mips/fibonacci/fibonacci b/examples/mips/fibonacci/fibonacci index f33c5ab507b81d028063d20bd843e0b05b1841e4..7d453799f7867fef3f8a30bf53e8b427ff0fe5b5 100644 GIT binary patch delta 81 zcmX@Wae!lk1EVbi1A{&T6Od$JVFZy34q&_`Ei)-UFEKeeQ?HmIH?bfwIU}{WATw7t Yu^6N%8mvWeaxJ4OlLN!%C5&^J0H|OQ-2eap delta 81 zcmX@Wae!lk1EZ}z0|NsS5Hm2aFoG!t2Zk*`oR*oCpO=`NoT*pLkegVLn4FPXT#%Wo cn^?>c4OGX(pa>)<*D|UyF*s~q!Z?Qs0IR_f-2eap diff --git a/examples/mips/guess_game/guess_game b/examples/mips/guess_game/guess_game index a1c0a7303b536a9a666493f1baaad5f98b4e6724..850e225b9b14b156143305555fcf6909417929a2 100644 GIT binary patch delta 82 zcmX@XdxCevY9>1d1_lELCLqbc!U!T696*>+BfT`WxHvvNF*jAOm?1Z@ATc>3wYVTN aS2wX3q$(S%M``j)CN(AphRuS^+nE8^9uj5% delta 82 zcmX@XdxCevY9>1a1_lNuAZB1-VFXhQ4h$NM4h-p~sl~4~|idc_R6i3N$t8L7nu enYp@&#SGa%g-i@eKyva+CN(Aohs}b_+nE8`5)x(r diff --git a/examples/mips/hello_asm/hello_asm b/examples/mips/hello_asm/hello_asm index 5f249496bb6d4b2ea075e87978fc5f1d09f5f312..c56e67aa90155db0601b4e43294f6153060e3ea4 100644 GIT binary patch delta 81 zcmcb^eusSnFQY931A{&T6Od$JVFZy34qzOTk(!f}AD>v9t5?jBn^=&ToRL~wkeRES YSPW9+2iBrE*^5z?$$?>WA!9ZZ0G7EB@&Et; delta 81 zcmcb^eusSnFQctK0|NsS5Hm2aFoG!t2Zj(J&PdJ4$&XJg&ebbs$W1IrOwLFxF38N) cO)O^c1FB E0O_R%?EnA( delta 59 zcmX@YeS~|13X_B(0|NsS5Hm2aFoG!t2L=;H2ZqTuOtM^+KoKSeWgs~@nMs|A!C`X` HlLa#X>+J{a diff --git a/mapachespim/console.py b/mapachespim/console.py index aad3165..2f33e5c 100644 --- a/mapachespim/console.py +++ b/mapachespim/console.py @@ -11,6 +11,7 @@ import cmd import signal import sys +import tempfile from pathlib import Path from types import FrameType from typing import Any, Dict, Generator, List, Optional, Set, Tuple @@ -94,8 +95,12 @@ def get_source_lines( return result -def _parse_dwarf_line_info(elf_path: str) -> SourceInfo: - """Parse DWARF debug info and return SourceInfo object""" +def _parse_dwarf_line_info(elf_path: str, source_dirs: Optional[List[Path]] = None) -> SourceInfo: + """Parse DWARF debug info and return SourceInfo object + + Source files are looked up relative to the current directory, the ELF's + directory, and any extra ``source_dirs``. + """ source_info = SourceInfo() if not ELFTOOLS_AVAILABLE: @@ -148,7 +153,7 @@ def _parse_dwarf_line_info(elf_path: str) -> SourceInfo: # Cache source file content if not already cached if filename not in source_info.source_cache: - _load_source_file(source_info, filename, elf_path) + _load_source_file(source_info, filename, elf_path, source_dirs) prev_state = state @@ -159,13 +164,21 @@ def _parse_dwarf_line_info(elf_path: str) -> SourceInfo: return source_info -def _load_source_file(source_info: SourceInfo, filename: str, elf_path: str) -> None: +def _load_source_file( + source_info: SourceInfo, + filename: str, + elf_path: str, + source_dirs: Optional[List[Path]] = None, +) -> None: """Try to load source file contents into cache""" - # Try to find source file relative to ELF location - elf_dir = Path(elf_path).parent + # Extra directories (e.g. where an assembled .s file lives) come first + search_paths = [] + for directory in source_dirs or []: + search_paths += [directory / filename, directory / Path(filename).name] - # Try multiple search paths - search_paths = [ + # Then relative to the current directory and the ELF location + elf_dir = Path(elf_path).parent + search_paths += [ Path(filename), # Absolute or relative to CWD elf_dir / filename, # Relative to ELF elf_dir / Path(filename).name, # Just filename in ELF dir @@ -199,12 +212,17 @@ class MapacheSPIMConsole(cmd.Cmd): Supports multiple ISAs: RISC-V, ARM64, and x86-64. """ - intro: str = "Welcome to MapacheSPIM. Type help or ? to list commands.\n" + intro: str = ( + "Welcome to MapacheSPIM. Type help or ? to list commands, " + "or quickstart for a tutorial.\n" + ) prompt: str = "(mapachespim) " _verbose: bool sim: Optional[Simulator] loaded_file: Optional[str] + loaded_source: Optional[Path] + _elf_path: Optional[str] breakpoints: Set[int] _interrupted: bool _running: bool @@ -232,7 +250,11 @@ def __init__(self, verbose: bool = False) -> None: pass # readline not available on all platforms self.sim = None - self.loaded_file = None + self.loaded_file = None # what the user loaded (ELF or source file) + self.loaded_source = None # assembly source, when loaded from .s + self._elf_path = None # ELF actually loaded into the simulator + self._isa_override: Optional[str] = None + self._tempdir: Optional[tempfile.TemporaryDirectory] = None self.breakpoints = set() self._interrupted = False self._running = False @@ -295,55 +317,252 @@ def _print_block_end(self) -> None: # --- File Loading --- def do_load(self, arg: str) -> None: - """Load an ELF file + """Load a program (an ELF executable, an assembly file, or an example) Usage: - load + load [isa] + + Loads a program into the simulator. The program counter is set to + the entry point and all breakpoints are cleared. - Loads a compiled ELF executable into the simulator. The ISA is - auto-detected from the ELF file (RISC-V, ARM64, or x86-64). - The program counter is set to the entry point and all - breakpoints are cleared. + - ELF executables are loaded directly; the ISA is auto-detected. + - Assembly source files (.s, .S, .asm) are assembled first, with + debug info so 'list' can show your source. The ISA comes from a + '.isa' directive in the file, or from the optional isa argument + (riscv64, mips32, arm64, x86_64). + - Bundled examples can be loaded by name; see 'examples'. Examples: - load examples/riscv/fibonacci/fibonacci - load examples/arm/test_simple/simple - load examples/x86_64/test_simple/simple + load riscv/hello_asm # A bundled example + load examples/riscv/fibonacci/fibonacci # Same, by path + load myprog.s # Assemble and load + load myprog.s riscv64 # ...choosing the ISA + load myprog # An ELF you built - After loading, use 'status' to see the ISA and entry point. + Tips: + - After editing a .s file, use 'reload' to re-assemble it + - Use 'examples' to list the bundled example programs + - Use Tab to complete file paths and example names """ - if not arg: - self.print_error("Error: Please specify an ELF file to load.") + parts = arg.split() + if not parts: + self.print_error("Error: Please specify a file to load (or see 'examples').") + return + if len(parts) > 2: + self.print_error("Error: Usage: load [isa]") return - filepath = Path(arg) + name = parts[0] + isa = parts[1] if len(parts) > 1 else None + filepath = Path(name).expanduser() if not filepath.exists(): - self.print_error(f'Error: File "{arg}" not found.') + from .examples import find_example + + example = find_example(name) + if example is None: + self.print_error( + f'Error: File "{name}" not found. Type "examples" to see the bundled examples.' + ) + return + filepath = example + + if filepath.is_dir(): + self.print_error(f'Error: "{name}" is a directory.') return + if filepath.suffix in (".s", ".S", ".asm"): + self._assemble_and_load(filepath, isa, display_name=name) + else: + if isa is not None: + self.print_error( + "Error: The ISA argument only applies to assembly files; " + "an ELF file's ISA is detected automatically." + ) + return + self._load_elf_file(str(filepath), display_name=name) + + def _load_elf_file( + self, + elf_path: str, + display_name: str, + source: Optional[Path] = None, + keep_breakpoints: bool = False, + ) -> bool: + """Load an ELF into the simulator and read its debug info""" try: - self.sim.load_elf(str(filepath)) - self.loaded_file = str(filepath) - pc = self.sim.get_pc() - isa_name = self.sim.get_isa_name() - print(f"Loaded {filepath} ({isa_name})", file=self.stdout) - print(f"Entry point: {pc:#018x}", file=self.stdout) - self.breakpoints.clear() - - # Parse DWARF debug information - self.source_info = _parse_dwarf_line_info(str(filepath)) - if self.source_info.has_debug_info: - num_files = len(self.source_info.source_cache) - if num_files > 0: - file_list = ", ".join(self.source_info.source_cache.keys()) - print( - f"Source info: {file_list} ({len(self.source_info.addr_to_line)} address mappings)", - file=self.stdout, - ) - else: - print("Debug info present but source files not found", file=self.stdout) + self.sim.load_elf(elf_path) except Exception as e: self.print_error(f"Error loading ELF file: {e}") + return False + + self.loaded_file = display_name + self.loaded_source = source + self._elf_path = elf_path + self.prev_regs = None + pc = self.sim.get_pc() + isa_name = self.sim.get_isa_name() + print(f"Loaded {display_name} ({isa_name})", file=self.stdout) + print(f"Entry point: {pc:#018x}", file=self.stdout) + if not keep_breakpoints: + self.breakpoints.clear() + + # Parse DWARF debug information + source_dirs = [source.parent] if source is not None else None + self.source_info = _parse_dwarf_line_info(elf_path, source_dirs) + if self.source_info.has_debug_info: + num_files = len(self.source_info.source_cache) + if num_files > 0: + file_list = ", ".join(self.source_info.source_cache.keys()) + print( + f"Source info: {file_list} ({len(self.source_info.addr_to_line)} address mappings)", + file=self.stdout, + ) + else: + print("Debug info present but source files not found", file=self.stdout) + return True + + def _assemble_and_load( + self, + source: Path, + isa: Optional[str], + display_name: str, + keep_breakpoints: bool = False, + ) -> bool: + """Assemble a source file (with debug info) and load the result""" + from .toolchain import assemble_file + + valid_isas = ("riscv64", "arm64", "x86_64", "mips32") + if isa is not None and isa.lower() not in valid_isas: + self.print_error(f'Error: Unknown ISA "{isa}". Use one of: {", ".join(valid_isas)}') + return False + + if self._tempdir is None: + self._tempdir = tempfile.TemporaryDirectory(prefix="mapachespim-") + elf_path = Path(self._tempdir.name) / (source.stem or "program") + + result = assemble_file(source, output_path=elf_path, isa=isa, debug=True) + for warning in result.warnings: + print(f"{source.name}: warning: {warning}", file=self.stdout) + if not result.success: + self._print_block_start() + if any(e.startswith("ISA not specified") for e in result.errors): + print(f"Error: {source.name} does not say which ISA it is for.", file=self.stdout) + print("Add a line like this at the top of the file:", file=self.stdout) + print(" .isa riscv64", file=self.stdout) + print(f"or give the ISA when loading: load {display_name} riscv64", file=self.stdout) + print(f"(ISAs: {', '.join(valid_isas)})", file=self.stdout) + else: + print(f"Error: could not assemble {source.name}:", file=self.stdout) + for error in result.errors: + first, *rest = error.splitlines() + print(f" {source.name}: {first}", file=self.stdout) + for line in rest: + print(f" {line}" if line.strip() else "", file=self.stdout) + self._print_block_end() + return False + + self._isa_override = isa + print(f"Assembled {source.name} ({len(result.elf_bytes)} bytes)", file=self.stdout) + return self._load_elf_file( + str(elf_path), display_name, source=source, keep_breakpoints=keep_breakpoints + ) + + def do_reload(self, arg: str) -> None: + """Reload the current program from disk + + Usage: + reload + + For an assembly file, re-assembles it and loads the result, so you + can edit your .s file and try the new version without retyping the + 'load' command. For an ELF file, reloads it (for example after you + rebuilt it with mapachespim-as). + + Breakpoints set on labels move with their label; breakpoints on + addresses that are no longer labelled are kept as they are. + + Examples: + load myprog.s + run # Find a bug, edit myprog.s... + reload # Re-assemble and load the fixed version + """ + if not self.loaded_file: + self.print_error('Error: No program loaded. Use "load " first.') + return + + # Remember which label each breakpoint was on so it can follow the label + old_symbols = {addr: name for name, addr in self.sim.get_symbols().items()} + by_label = {addr: old_symbols[addr] for addr in self.breakpoints if addr in old_symbols} + + if self.loaded_source is not None: + ok = self._assemble_and_load( + self.loaded_source, + self._isa_override, + self.loaded_file, + keep_breakpoints=True, + ) + else: + ok = self._load_elf_file( + self._elf_path or self.loaded_file, self.loaded_file, keep_breakpoints=True + ) + if not ok: + return + + moved = set() + for addr in list(self.breakpoints): + if addr in by_label: + new_addr = self.sim.lookup_symbol(by_label[addr]) + self.breakpoints.discard(addr) + if new_addr is not None: + moved.add(new_addr) + else: + moved.add(addr) + self.breakpoints = moved + if self.breakpoints: + print(f"Kept {len(self.breakpoints)} breakpoint(s)", file=self.stdout) + + def do_examples(self, arg: str) -> None: + """List the bundled example programs + + Usage: + examples [isa] + + Lists the example programs that come with MapacheSPIM, optionally + only those for one ISA (riscv, mips, arm, x86_64). Load one by name. + + Examples: + examples # List everything + examples riscv # Only RISC-V examples + load riscv/hello_asm # Load one + + Tips: + - To edit the examples, copy them to your own directory with: + mapachespim --copy-examples my-examples + """ + from .examples import ISA_DIRS, list_examples + + wanted = arg.strip().lower() or None + if wanted is not None and wanted not in ISA_DIRS: + self.print_error(f'Error: Unknown ISA "{arg.strip()}". Use one of: {", ".join(ISA_DIRS)}') + return + + examples = [e for e in list_examples() if wanted is None or e.isa == wanted] + if not examples: + self.print_error("No bundled examples found in this installation.") + return + + self._print_block_start() + current = None + for example in examples: + if example.isa != current: + if current is not None: + print(file=self.stdout) + current = example.isa + print(f"{ISA_DIRS[example.isa]}:", file=self.stdout) + print(f" {example.short_name:<24} {example.description}", file=self.stdout) + print(file=self.stdout) + print(f'Load one with, e.g.: load {examples[0].short_name}', file=self.stdout) + self._print_block_end() # --- Execution Control --- @@ -930,7 +1149,7 @@ def do_mem(self, arg: str) -> None: # Look up section try: - with open(self.loaded_file, "rb") as f: + with open(self._elf_path, "rb") as f: elf = ELFFile(f) section = elf.get_section_by_name(addr_or_section) if not section: @@ -1338,7 +1557,7 @@ def do_info(self, arg: str) -> None: return try: - with open(self.loaded_file, "rb") as f: + with open(self._elf_path, "rb") as f: elf = ELFFile(f) self._print_block_start() @@ -1467,6 +1686,8 @@ def do_status(self, arg: str) -> None: """ self._print_block_start() print(f"Loaded file: {self.loaded_file or 'None'}", file=self.stdout) + if self.loaded_source is not None: + print(f"Source: {self.loaded_source}", file=self.stdout) if self.loaded_file: isa_name = self.sim.get_isa_name() pc = self.sim.get_pc() @@ -1640,12 +1861,14 @@ def do_help(self, arg: str) -> None: # Group commands by category categories = { "Loading & Running": [ - ("load", "Load an ELF file"), + ("load", "Load a program (.s file, ELF, or example)"), + ("reload", "Re-assemble/reload the current program"), + ("examples", "List the bundled example programs"), ("run (r)", "Run program until halt or breakpoint"), ("step (s)", "Execute one or more instructions"), ("stepreg (sr)", "Step and show registers"), ("continue (c)", "Continue after breakpoint"), - ("reset", "Reset simulator state"), + ("reset", "Restart the program from the beginning"), ], "Inspection": [ ("regs", "Display all registers"), @@ -1711,6 +1934,14 @@ def complete_load(self, text: str, line: str, begidx: int, endidx: int) -> List[ # Get matching paths matches = glob.glob(pattern) + # Also offer bundled example names like "riscv/fibonacci" + if not use_tilde and text.count("/") <= 1: + from .examples import list_examples + + for example in list_examples(): + if example.short_name.startswith(text) and not Path(example.short_name).exists(): + matches.append(example.short_name) + # Format completions - return full paths that replace `text` completions = [] for match in matches: @@ -1796,15 +2027,17 @@ def do_quickstart(self, arg: str) -> None: print("MapacheSPIM Quick Start Guide", file=self.stdout) print("=" * 60, file=self.stdout) print(file=self.stdout) - print("1. LOAD A PROGRAM (ISA is auto-detected)", file=self.stdout) - print(" load examples/riscv/fibonacci/fibonacci # RISC-V", file=self.stdout) - print(" load examples/arm/fibonacci/fibonacci # ARM64", file=self.stdout) - print(" load examples/x86_64/test_simple/simple # x86-64", file=self.stdout) - print(" (Use Tab to autocomplete file paths!)", file=self.stdout) + print("1. LOAD A PROGRAM", file=self.stdout) + print(" examples - List the bundled examples", file=self.stdout) + print(" load riscv/hello_asm - Load an example by name", file=self.stdout) + print(" load myprog.s - Assemble and load your own program", file=self.stdout) + print(" reload - Re-assemble after editing", file=self.stdout) + print(" (Use Tab to autocomplete file paths and example names!)", file=self.stdout) print(file=self.stdout) print("2. SEE WHERE YOU ARE", file=self.stdout) print(" pc - Show program counter", file=self.stdout) - print(" disasm - Disassemble instructions", file=self.stdout) + print(" list - Show your source code around the PC", file=self.stdout) + print(" disasm - Disassemble instructions at the PC", file=self.stdout) print(" regs - Show all registers", file=self.stdout) print(file=self.stdout) print("3. EXECUTE CODE", file=self.stdout) @@ -1812,6 +2045,7 @@ def do_quickstart(self, arg: str) -> None: print(" step 5 - Execute 5 instructions", file=self.stdout) print(" run - Run until program ends", file=self.stdout) print(" run 100 - Run at most 100 instructions", file=self.stdout) + print(" reset - Start over from the beginning", file=self.stdout) print(file=self.stdout) print("4. SET BREAKPOINTS", file=self.stdout) print(" break - Set breakpoint at address", file=self.stdout) @@ -1821,6 +2055,7 @@ def do_quickstart(self, arg: str) -> None: print(file=self.stdout) print("5. EXAMINE MEMORY", file=self.stdout) print(" mem 0x80000000 - Show memory at address", file=self.stdout) + print(" mem my_label - Show memory at a label", file=self.stdout) print(" mem .data - Show data section", file=self.stdout) print(file=self.stdout) print("6. TIPS FOR DEBUGGING", file=self.stdout) @@ -1837,7 +2072,13 @@ def do_quickstart(self, arg: str) -> None: DEFAULT_MAX_STEPS = 10_000_000 -def _execute(console: MapacheSPIMConsole, path: str, max_steps: int, verbose: bool) -> int: +def _execute( + console: MapacheSPIMConsole, + path: str, + max_steps: int, + verbose: bool, + isa: Optional[str] = None, +) -> int: """Run a program non-interactively and return the process exit status. The status is the program's exit code if it exits normally, 1 if the file @@ -1846,7 +2087,9 @@ def _execute(console: MapacheSPIMConsole, path: str, max_steps: int, verbose: bo """ import io - if not Path(path).exists(): + from .examples import find_example + + if not Path(path).exists() and find_example(path) is None: print(f"Error: File '{path}' not found", file=sys.stderr) return 1 @@ -1855,7 +2098,7 @@ def _execute(console: MapacheSPIMConsole, path: str, max_steps: int, verbose: bo original_stdout = console.stdout if not verbose: console.stdout = captured_output - console.onecmd(f"load {path}") + console.onecmd(f"load {path} {isa}" if isa else f"load {path}") console.stdout = original_stdout if not console.loaded_file: @@ -1910,7 +2153,16 @@ def main() -> None: prog="mapachespim", description="MapacheSPIM - Interactive Multi-ISA Simulator (RISC-V, ARM64, x86-64, MIPS32)", ) - parser.add_argument("file", nargs="?", help="Program to load on startup") + parser.add_argument( + "file", + nargs="?", + help="Program to load: an ELF file, an assembly file (.s), or an example name", + ) + parser.add_argument( + "--isa", + choices=["riscv64", "arm64", "x86_64", "mips32"], + help="ISA of an assembly file without an .isa directive", + ) parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") parser.add_argument( "-v", "--verbose", action="store_true", help="Verbose mode (show extra messages)" @@ -1921,6 +2173,13 @@ def main() -> None: action="store_true", help="Run the program and exit with its exit code (no interactive console)", ) + parser.add_argument( + "--copy-examples", + metavar="DIR", + nargs="?", + const="mapachespim-examples", + help="Copy the bundled example programs to DIR (default: ./mapachespim-examples) and exit", + ) parser.add_argument( "--max-steps", type=int, @@ -1933,6 +2192,18 @@ def main() -> None: if args.max_steps < 1: parser.error("--max-steps must be at least 1") + if args.copy_examples is not None: + from .examples import copy_examples + + try: + dest = copy_examples(Path(args.copy_examples)) + except (FileNotFoundError, FileExistsError) as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + print(f"Copied examples to {dest}/") + print(f"Try: mapachespim {dest}/riscv/hello_asm/hello_asm.s") + sys.exit(0) + # Create console console = MapacheSPIMConsole(verbose=args.verbose) @@ -1941,11 +2212,20 @@ def main() -> None: if not args.file: print("Error: -e/--execute requires a file argument", file=sys.stderr) sys.exit(1) - sys.exit(_execute(console, args.file, args.max_steps, args.verbose)) + try: + status = _execute(console, args.file, args.max_steps, args.verbose, args.isa) + except BrokenPipeError: + # Output was piped into something like `head` that exited early + import os + + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + status = 1 + sys.exit(status) # Interactive mode: start REPL if args.file: - console.onecmd(f"load {args.file}") + console.onecmd(f"load {args.file} {args.isa}" if args.isa else f"load {args.file}") try: console.cmdloop() diff --git a/mapachespim/examples.py b/mapachespim/examples.py new file mode 100644 index 0000000..f7ce591 --- /dev/null +++ b/mapachespim/examples.py @@ -0,0 +1,171 @@ +""" +Access to the bundled example programs. + +Wheels ship the repository's ``examples/`` directory as +``mapachespim/examples``, so examples are available after a plain +``pip install``. In a source checkout (including editable installs) the +repository's ``examples/`` directory is used instead. +""" + +from __future__ import annotations + +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional + +# ISA directory name -> display name, in the order examples are listed +ISA_DIRS = { + "riscv": "RISC-V 64-bit", + "mips": "MIPS32", + "arm": "ARM64", + "x86_64": "x86-64", +} + +SOURCE_SUFFIXES = (".s", ".S", ".asm") + + +@dataclass +class Example: + """One example program.""" + + isa: str + """ISA directory name, e.g. 'riscv'.""" + + name: str + """Example directory name, e.g. 'fibonacci'.""" + + binary: Path + """Prebuilt ELF executable.""" + + source: Optional[Path] + """Assembly source file, if present.""" + + description: str + """One-line description taken from the source file's header comment.""" + + @property + def short_name(self) -> str: + """Name accepted by the console's load command, e.g. 'riscv/fibonacci'.""" + return f"{self.isa}/{self.name}" + + +def examples_dir() -> Optional[Path]: + """Directory containing the bundled examples, or None if unavailable.""" + package_dir = Path(__file__).resolve().parent + for candidate in (package_dir / "examples", package_dir.parent / "examples"): + if (candidate / "riscv").is_dir(): + return candidate + return None + + +def _describe(source: Optional[Path]) -> str: + """Pull a short description from the header comment of a source file.""" + if source is None: + return "" + try: + lines = source.read_text(errors="replace").splitlines()[:6] + except OSError: + return "" + for line in lines: + text = line.strip().lstrip("#/;").strip() + if not text or set(text) <= set("=-*"): + continue + # "hello_asm.s - Your First ..." / "RISC-V 64-bit Assembly: Recursive ..." + for sep in (" - ", ": "): + if sep in text: + text = text.split(sep, 1)[1] + break + return text + return "" + + +def _find_program(directory: Path) -> Optional[Example]: + sources = sorted(p for p in directory.iterdir() if p.suffix in SOURCE_SUFFIXES) + source = sources[0] if sources else None + candidates = [] + if source is not None: + candidates.append(directory / source.stem) + candidates.append(directory / directory.name) + binary = next((c for c in candidates if c.is_file()), None) + if binary is None: + return None + return Example( + isa=directory.parent.name, + name=directory.name, + binary=binary, + source=source, + description=_describe(source), + ) + + +def list_examples() -> List[Example]: + """All bundled examples, grouped by ISA in ISA_DIRS order.""" + root = examples_dir() + if root is None: + return [] + result: List[Example] = [] + for isa in ISA_DIRS: + isa_dir = root / isa + if not isa_dir.is_dir(): + continue + for directory in sorted(p for p in isa_dir.iterdir() if p.is_dir()): + example = _find_program(directory) + if example is not None: + result.append(example) + return result + + +def find_example(name: str) -> Optional[Path]: + """ + Resolve an example name to a file in the bundled examples. + + Accepts short names ('riscv/fibonacci'), paths inside the examples tree + ('riscv/fibonacci/fibonacci', 'riscv/fibonacci/fibonacci.s'), and the + repository-relative paths used in the documentation + ('examples/riscv/fibonacci/fibonacci'). + """ + root = examples_dir() + if root is None: + return None + rel = name.strip().replace("\\", "/").strip("/") + if rel.startswith("examples/"): + rel = rel[len("examples/") :] + if not rel: + return None + + candidate = (root / rel).resolve() + # Only files inside the examples tree are examples + if root.resolve() not in candidate.parents: + return None + if candidate.is_file(): + return candidate + if candidate.is_dir() and candidate.parent.name in ISA_DIRS: + example = _find_program(candidate) + if example is not None: + return example.binary + return None + + +def copy_examples(destination: Path) -> Path: + """ + Copy the bundled examples to ``destination`` so they can be edited. + + Returns the destination path. + + Raises: + FileNotFoundError: If the examples are not available. + FileExistsError: If the destination already exists. + """ + root = examples_dir() + if root is None: + raise FileNotFoundError("The bundled examples are not available in this installation") + destination = Path(destination) + if destination.exists(): + raise FileExistsError(f"{destination} already exists") + shutil.copytree( + root, + destination, + ignore=shutil.ignore_patterns("__pycache__", "*.o", "*.dis"), + ) + return destination diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index 8eb8692..4f18c1f 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -15,9 +15,10 @@ _ = keystone.KS_ARCH_X86 KEYSTONE_AVAILABLE = True KEYSTONE_ERROR = None -except ImportError as e: +except ImportError: + # Simply not installed; the message raised later explains how to add it KEYSTONE_AVAILABLE = False - KEYSTONE_ERROR = str(e) + KEYSTONE_ERROR = None keystone = None except Exception as e: # Keystone installed but native library failed to load @@ -173,11 +174,13 @@ def __init__(self, isa: str): return if not KEYSTONE_AVAILABLE: - msg = f"Assembling {isa} requires the Keystone Engine, which is not installed.\n" if KEYSTONE_ERROR: - msg += f"Error: {KEYSTONE_ERROR}\n\n" - msg += "Install it with: pip install 'mapachespim[keystone]'\n\n" - msg += "RISC-V and MIPS programs can be assembled without Keystone." + msg = f"Assembling {isa} requires the Keystone Engine, which failed to load:\n" + msg += f" {KEYSTONE_ERROR}\n" + else: + msg = f"Assembling {isa} requires the Keystone Engine, which is not installed.\n" + msg += "Install it with: pip install 'mapachespim[keystone]'\n" + msg += "(RISC-V and MIPS programs can be assembled without it.)" raise ImportError(msg) try: @@ -347,7 +350,9 @@ def assemble( min_instr_len = self._config.get("instr_size") or 1 # Build DWARF sections - dwarf = DWARFv2Builder(addr_size=addr_size) + dwarf = DWARFv2Builder( + addr_size=addr_size, big_endian=not self._layout.is_little_endian + ) abbrev = dwarf.build_debug_abbrev() info = dwarf.build_debug_info(source_filename or "source.s", text_start, text_end) line = dwarf.build_debug_line(source_filename or "source.s", debug_lines, min_instr_len) diff --git a/mapachespim/toolchain/dwarf.py b/mapachespim/toolchain/dwarf.py index 73d2f34..213a900 100644 --- a/mapachespim/toolchain/dwarf.py +++ b/mapachespim/toolchain/dwarf.py @@ -96,14 +96,17 @@ class DWARFv2Builder: line = builder.build_debug_line("hello.s", lines) """ - def __init__(self, addr_size: int = 8): + def __init__(self, addr_size: int = 8, big_endian: bool = False): """ Initialize DWARF builder. Args: addr_size: Address size in bytes (8 for 64-bit, 4 for 32-bit). + big_endian: Write multi-byte fields big-endian (e.g. for MIPS). + Debuggers read DWARF in the ELF file's byte order. """ self.addr_size = addr_size + self._endian = ">" if big_endian else "<" def build_debug_abbrev(self) -> bytes: """ @@ -155,15 +158,15 @@ def build_debug_info( die_content.append(0x01) # abbrev number 1 # stmt_list (4 bytes) - die_content.extend(struct.pack(' str: + text = self.out.getvalue() + self.out.truncate(0) + self.out.seek(0) + return text + + def write(self, name: str, text: str) -> Path: + path = self.dir / name + path.write_text(text) + return path + + def test_load_source_and_run(self): + src = self.write("hello.s", HELLO_RISCV) + self.console.onecmd(f"load {src}") + text = self.output() + self.assertIn("Assembled hello.s", text) + self.assertIn("Source info: hello.s", text) + self.console.onecmd("run") + text = self.output() + self.assertIn("hello from source", text) + self.assertIn("exited with code 7", text) + + def test_list_shows_source(self): + src = self.write("hello.s", HELLO_RISCV) + self.console.onecmd(f"load {src}") + self.output() + self.console.onecmd("list") + self.assertIn("la a0, msg", self.output()) + + def test_source_without_isa(self): + src = self.write("noisa.s", HELLO_RISCV.replace(".isa riscv64\n", "")) + self.console.onecmd(f"load {src}") + self.assertIn("does not say which ISA", self.output()) + self.assertIsNone(self.console.loaded_file) + self.console.onecmd(f"load {src} riscv64") + self.assertIn("Assembled", self.output()) + + def test_assembly_errors_are_reported(self): + src = self.write("bad.s", ".isa riscv64\n.text\n_start:\n addi a0, a0\n") + self.console.onecmd(f"load {src}") + text = self.output() + self.assertIn("could not assemble bad.s", text) + self.assertIn("Line 4", text) + + def test_reload_picks_up_edits_and_moves_breakpoints(self): + src = self.write("prog.s", HELLO_RISCV) + self.console.onecmd(f"load {src}") + self.console.onecmd("break done") + old = self.console.sim.lookup_symbol("done") + # Insert two instructions before 'done' so the label moves + src.write_text(HELLO_RISCV.replace("done:", " nop\n nop\ndone:")) + self.console.onecmd("reload") + new = self.console.sim.lookup_symbol("done") + self.assertEqual(new, old + 8) + self.assertEqual(self.console.breakpoints, {new}) + + def test_load_example_by_name(self): + self.console.onecmd("load mips/hello_asm") + self.assertIn("(MIPS)", self.output()) + self.console.onecmd("run") + self.assertIn("Hello, Assembly!", self.output()) + + def test_mips_example_has_source_info(self): + """MIPS DWARF used to be written little-endian and could not be read""" + self.console.onecmd("load mips/hello_asm") + self.assertIn("Source info: hello_asm.s", self.output()) + + def test_examples_command(self): + self.console.onecmd("examples riscv") + text = self.output() + self.assertIn("riscv/fibonacci", text) + self.assertNotIn("mips/", text) + + def test_missing_file_hint(self): + self.console.onecmd("load no/such/file") + self.assertIn('Type "examples"', self.output()) + + +class TestCommandLine(unittest.TestCase): + def test_version(self): + result = run_cli("--version") + self.assertEqual(result.returncode, 0) + self.assertIn("mapachespim", result.stdout) + + def test_execute_source_file_exit_code(self): + with tempfile.TemporaryDirectory() as tmp: + src = Path(tmp) / "hello.s" + src.write_text(HELLO_RISCV) + result = run_cli("-e", str(src)) + self.assertEqual(result.returncode, 7, result.stderr) + self.assertEqual(result.stdout, "hello from source\n") + + def test_execute_with_isa_flag(self): + with tempfile.TemporaryDirectory() as tmp: + src = Path(tmp) / "hello.s" + src.write_text(HELLO_RISCV.replace(".isa riscv64\n", "")) + result = run_cli("-e", "--isa", "riscv64", str(src)) + self.assertEqual(result.returncode, 7, result.stderr) + + def test_execute_example_by_name(self): + result = run_cli("-e", "riscv/hello_asm") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Hello, Assembly!", result.stdout) + + def test_execute_infinite_loop_times_out(self): + with tempfile.TemporaryDirectory() as tmp: + src = Path(tmp) / "loop.s" + src.write_text(".isa riscv64\n.text\n_start:\n j _start\n") + result = run_cli("-e", "--max-steps", "1000", str(src)) + self.assertEqual(result.returncode, 124) + self.assertIn("did not exit within 1000 instructions", result.stderr) + + def test_execute_fault_exit_code(self): + with tempfile.TemporaryDirectory() as tmp: + src = Path(tmp) / "fault.s" + src.write_text(".isa riscv64\n.text\n_start:\n ld a0, 0(zero)\n") + result = run_cli("-e", str(src)) + self.assertEqual(result.returncode, 1) + self.assertIn("Read from unmapped address 0x0", result.stderr) + + def test_copy_examples_flag(self): + with tempfile.TemporaryDirectory() as tmp: + result = subprocess.run( + [sys.executable, "-m", "mapachespim.console", "--copy-examples", "ex"], + cwd=tmp, + capture_output=True, + text=True, + timeout=60, + env={**os.environ, "PYTHONPATH": str(Path(__file__).parent.parent)}, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((Path(tmp) / "ex/mips/hello_asm/hello_asm.s").is_file()) + + +if __name__ == "__main__": + unittest.main() From 6441fd1acb66465720682e3bfb4eaa787a5f1de1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 00:24:11 +0000 Subject: [PATCH 05/18] Fix lint and type errors reported by CI - Undefined SectionData annotations, bare excepts, unused imports and loop variables, a no-op 'or True' condition - Type ISA_CONFIG and byte-order literals so mypy passes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- mapachespim/console.py | 4 +++- mapachespim/toolchain/__init__.py | 2 -- mapachespim/toolchain/assembler.py | 19 ++++++++++--------- mapachespim/toolchain/cli.py | 13 ++++++------- mapachespim/toolchain/directives.py | 4 ++-- mapachespim/toolchain/dwarf.py | 1 - mapachespim/toolchain/elf_builder.py | 2 +- mapachespim/toolchain/expr.py | 6 +++--- mapachespim/toolchain/mips.py | 4 ++-- mapachespim/unicorn_backend.py | 10 ++++------ 10 files changed, 31 insertions(+), 34 deletions(-) diff --git a/mapachespim/console.py b/mapachespim/console.py index 2f33e5c..d3630d2 100644 --- a/mapachespim/console.py +++ b/mapachespim/console.py @@ -16,7 +16,7 @@ from types import FrameType from typing import Any, Dict, Generator, List, Optional, Set, Tuple -from . import Simulator, StepResult +from . import Simulator try: from elftools.elf.elffile import ELFFile @@ -1149,6 +1149,7 @@ def do_mem(self, arg: str) -> None: # Look up section try: + assert self._elf_path is not None # set whenever loaded_file is with open(self._elf_path, "rb") as f: elf = ELFFile(f) section = elf.get_section_by_name(addr_or_section) @@ -1557,6 +1558,7 @@ def do_info(self, arg: str) -> None: return try: + assert self._elf_path is not None # set whenever loaded_file is with open(self._elf_path, "rb") as f: elf = ELFFile(f) diff --git a/mapachespim/toolchain/__init__.py b/mapachespim/toolchain/__init__.py index 46560c1..d9a3c98 100644 --- a/mapachespim/toolchain/__init__.py +++ b/mapachespim/toolchain/__init__.py @@ -131,8 +131,6 @@ def assemble_file( >>> if result.success: ... print(f"Assembled to {result.entry_point:#x}") """ - from .directives import DirectiveParser - source_path = Path(source_path) if not source_path.exists(): diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index 4f18c1f..d9751ed 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -7,7 +7,8 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Dict, List, Optional, Tuple +from types import ModuleType +from typing import Any, Dict, List, Optional, Tuple try: import keystone @@ -28,10 +29,10 @@ from . import mips, riscv from .directives import DirectiveParser, LineType, SectionData -from .expr import EncodeError from .dwarf import DWARFv2Builder -from .elf_builder import ELFBuilder, Section, Symbol, STT_FUNC, STT_NOTYPE, STB_GLOBAL, STB_LOCAL -from .memory_map import get_layout, MemoryLayout +from .elf_builder import STB_GLOBAL, STB_LOCAL, STT_FUNC, STT_NOTYPE, ELFBuilder, Section, Symbol +from .expr import EncodeError +from .memory_map import get_layout @dataclass @@ -98,7 +99,7 @@ class Assembler: """ # ISA configuration mapping - ISA_CONFIG = { + ISA_CONFIG: Dict[str, Dict[str, Any]] = { # RISC-V is encoded by the built-in pure-Python encoder (riscv.py) "riscv64": { "arch": None, @@ -193,7 +194,7 @@ def _is_riscv(self) -> bool: return self.isa in ("riscv64", "riscv") @property - def _builtin_encoder(self): # type: ignore[no-untyped-def] + def _builtin_encoder(self) -> Optional[ModuleType]: """The pure-Python encoder module for this ISA, or None for Keystone ISAs.""" if self._is_riscv: return riscv @@ -469,7 +470,7 @@ def _calculate_x86_text_labels_iterative( labels: Dict[str, int] = {} - for iteration in range(max_iterations): + for _ in range(max_iterations): new_labels: Dict[str, int] = {} current_addr = base_addr @@ -520,7 +521,7 @@ def _get_x86_actual_size( encoding, count = self._ks.asm(converted, addr) if encoding is not None and count > 0: return len(encoding) - except: + except keystone.KsError: pass # Assembly failed - likely unresolved forward reference @@ -539,7 +540,7 @@ def _get_x86_actual_size( encoding, count = self._ks.asm(placeholder_instr, addr) if encoding is not None and count > 0: return len(encoding) - except: + except keystone.KsError: pass # Final fallback: use existing estimation diff --git a/mapachespim/toolchain/cli.py b/mapachespim/toolchain/cli.py index 0c1dfb1..9d0d84c 100644 --- a/mapachespim/toolchain/cli.py +++ b/mapachespim/toolchain/cli.py @@ -128,13 +128,12 @@ def main(args: list[str] | None = None) -> int: return 1 # Success - if parsed.verbose or True: # Always show success message - isa_name = result.isa.upper() if result.isa else "Unknown" - size = len(result.elf_bytes) - symbols = len(result.symbols) - debug_str = ", debug" if parsed.debug else "" - print(f"Assembled: {output_path} ({isa_name}, {size} bytes, {symbols} symbols{debug_str})") - print(f"Entry point: 0x{result.entry_point:08x}") + isa_name = result.isa.upper() if result.isa else "Unknown" + size = len(result.elf_bytes) + symbols = len(result.symbols) + debug_str = ", debug" if parsed.debug else "" + print(f"Assembled: {output_path} ({isa_name}, {size} bytes, {symbols} symbols{debug_str})") + print(f"Entry point: 0x{result.entry_point:08x}") return 0 diff --git a/mapachespim/toolchain/directives.py b/mapachespim/toolchain/directives.py index c875499..d624769 100644 --- a/mapachespim/toolchain/directives.py +++ b/mapachespim/toolchain/directives.py @@ -10,7 +10,7 @@ import re from dataclasses import dataclass, field from enum import Enum, auto -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Literal, Optional, Tuple class LineType(Enum): @@ -114,7 +114,7 @@ def __init__(self) -> None: for name in [".text", ".data", ".rodata", ".bss"]: self.sections[name] = SectionData(name=name) - def _get_endianness(self) -> str: + def _get_endianness(self) -> Literal["little", "big"]: """Return byte order for data directives based on ISA.""" if self.isa in self.BIG_ENDIAN_ISAS: return 'big' diff --git a/mapachespim/toolchain/dwarf.py b/mapachespim/toolchain/dwarf.py index 213a900..1bfd2ac 100644 --- a/mapachespim/toolchain/dwarf.py +++ b/mapachespim/toolchain/dwarf.py @@ -12,7 +12,6 @@ import struct from typing import List, Tuple - # DWARF constants DW_TAG_compile_unit = 0x11 diff --git a/mapachespim/toolchain/elf_builder.py b/mapachespim/toolchain/elf_builder.py index bf7da74..f22dc65 100644 --- a/mapachespim/toolchain/elf_builder.py +++ b/mapachespim/toolchain/elf_builder.py @@ -360,7 +360,7 @@ def _build_elf64(self) -> bytes: # Build program headers phdrs = b"" - for i, section in enumerate(loadable_sections): + for section in loadable_sections: idx = self.sections.index(section) file_offset = section_file_offsets[idx] file_size = 0 if section.sh_type == SHT_NOBITS else len(section.data) diff --git a/mapachespim/toolchain/expr.py b/mapachespim/toolchain/expr.py index 55046b2..0cf1b5d 100644 --- a/mapachespim/toolchain/expr.py +++ b/mapachespim/toolchain/expr.py @@ -93,9 +93,9 @@ def eval(self, text: str) -> int: # relative to that instruction's pc and the auipc's target. raise self.error("%pcrel_lo is not supported; use la or lla instead") - value = parse_int(text) - if value is not None: - return value + literal = parse_int(text) + if literal is not None: + return literal total = 0 pos = 0 diff --git a/mapachespim/toolchain/mips.py b/mapachespim/toolchain/mips.py index 8bc2197..22b6bf8 100644 --- a/mapachespim/toolchain/mips.py +++ b/mapachespim/toolchain/mips.py @@ -345,7 +345,7 @@ def jump_target(text: str) -> int: _expect(ops, 3, f"{m} $rs, $rt, label") slt, swap, if_set = COMPARE_BRANCHES[m] rs = parse_register(ops[0]) - words: List[int] = [] + words = [] if _is_register(ops[1]): rt = parse_register(ops[1]) else: @@ -375,7 +375,7 @@ def jump_target(text: str) -> int: # --- loads and stores --- if m in LOADS or m in STORES: - opcode = LOADS.get(m, STORES.get(m)) + opcode = LOADS[m] if m in LOADS else STORES[m] _expect(ops, 2, f"{m} $rt, offset($rs)") rt = parse_register(ops[0]) if _MEM_RE.match(ops[1].strip()): diff --git a/mapachespim/unicorn_backend.py b/mapachespim/unicorn_backend.py index 13215dc..a549080 100644 --- a/mapachespim/unicorn_backend.py +++ b/mapachespim/unicorn_backend.py @@ -27,7 +27,6 @@ UC_MEM_FETCH_UNMAPPED, UC_MEM_READ_UNMAPPED, UC_MEM_WRITE_UNMAPPED, - UC_MODE_32, UC_MODE_64, UC_MODE_ARM, UC_MODE_BIG_ENDIAN, @@ -466,8 +465,7 @@ def disassemble_one(self, simulator: UnicornSimulator, addr: int) -> str: return f"{instr.mnemonic} {instr.op_str}".strip() # If disassembly failed, show raw bytes - byteorder = "big" if self._isa == ISA.MIPS else "little" - word = int.from_bytes(code[:4], byteorder=byteorder) + word = int.from_bytes(code[:4], byteorder="big" if self._isa == ISA.MIPS else "little") return f".word 0x{word:08x}" def disassemble_with_size(self, simulator: UnicornSimulator, addr: int) -> Tuple[str, int]: @@ -498,8 +496,8 @@ def disassemble_with_size(self, simulator: UnicornSimulator, addr: int) -> Tuple if self._isa == ISA.X86_64: return (f".byte 0x{code[0]:02x}", 1) - byteorder = "big" if self._isa == ISA.MIPS else "little" - return (f".word 0x{int.from_bytes(code[:4], byteorder=byteorder):08x}", 4) + word = int.from_bytes(code[:4], byteorder="big" if self._isa == ISA.MIPS else "little") + return (f".word 0x{word:08x}", 4) class UnicornSimulator: @@ -648,7 +646,7 @@ def _map_new(self, addr: int, size: int) -> None: region in [0x80000000, 0x100000000) is also mapped at its sign-extended alias, backed by the same host memory. """ - if self._isa == ISA.RISCV and 0x80000000 <= addr and addr + size <= 0x100000000: + if self._isa == ISA.RISCV and addr >= 0x80000000 and addr + size <= 0x100000000: buf = ctypes.create_string_buffer(size) self._mem_buffers.append(buf) # keep host memory alive ptr = ctypes.addressof(buf) From 3deabb11cf01f647a3ba7fee6e7c12a9bf4a09f1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 00:24:19 +0000 Subject: [PATCH 06/18] Apply ruff format Formatting only; no functional changes. CI runs ruff format --check. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- mapachespim/console.py | 17 +- mapachespim/memory_map.py | 34 ++-- mapachespim/toolchain/__init__.py | 6 +- mapachespim/toolchain/assembler.py | 262 +++++++++++++++++++-------- mapachespim/toolchain/cli.py | 9 +- mapachespim/toolchain/directives.py | 111 ++++++------ mapachespim/toolchain/dwarf.py | 109 +++++------ mapachespim/toolchain/elf_builder.py | 176 ++++++++++-------- mapachespim/toolchain/expr.py | 2 - mapachespim/toolchain/mips.py | 20 +- mapachespim/toolchain/riscv.py | 42 ++++- mapachespim/unicorn_backend.py | 41 ++++- 12 files changed, 524 insertions(+), 305 deletions(-) diff --git a/mapachespim/console.py b/mapachespim/console.py index d3630d2..92cc4fe 100644 --- a/mapachespim/console.py +++ b/mapachespim/console.py @@ -67,6 +67,7 @@ def get_location(self, addr: int) -> Optional[Tuple[str, int]]: # Binary search for largest address <= addr import bisect + idx = bisect.bisect_right(self._sorted_addrs, addr) - 1 if idx >= 0: @@ -213,8 +214,7 @@ class MapacheSPIMConsole(cmd.Cmd): """ intro: str = ( - "Welcome to MapacheSPIM. Type help or ? to list commands, " - "or quickstart for a tutorial.\n" + "Welcome to MapacheSPIM. Type help or ? to list commands, or quickstart for a tutorial.\n" ) prompt: str = "(mapachespim) " @@ -449,7 +449,9 @@ def _assemble_and_load( print(f"Error: {source.name} does not say which ISA it is for.", file=self.stdout) print("Add a line like this at the top of the file:", file=self.stdout) print(" .isa riscv64", file=self.stdout) - print(f"or give the ISA when loading: load {display_name} riscv64", file=self.stdout) + print( + f"or give the ISA when loading: load {display_name} riscv64", file=self.stdout + ) print(f"(ISAs: {', '.join(valid_isas)})", file=self.stdout) else: print(f"Error: could not assemble {source.name}:", file=self.stdout) @@ -543,7 +545,9 @@ def do_examples(self, arg: str) -> None: wanted = arg.strip().lower() or None if wanted is not None and wanted not in ISA_DIRS: - self.print_error(f'Error: Unknown ISA "{arg.strip()}". Use one of: {", ".join(ISA_DIRS)}') + self.print_error( + f'Error: Unknown ISA "{arg.strip()}". Use one of: {", ".join(ISA_DIRS)}' + ) return examples = [e for e in list_examples() if wanted is None or e.isa == wanted] @@ -561,7 +565,7 @@ def do_examples(self, arg: str) -> None: print(f"{ISA_DIRS[example.isa]}:", file=self.stdout) print(f" {example.short_name:<24} {example.description}", file=self.stdout) print(file=self.stdout) - print(f'Load one with, e.g.: load {examples[0].short_name}', file=self.stdout) + print(f"Load one with, e.g.: load {examples[0].short_name}", file=self.stdout) self._print_block_end() # --- Execution Control --- @@ -847,8 +851,7 @@ def _report_if_exited(self) -> bool: """If the program already exited, say so and return True""" if self.sim.exited: self.print_error( - f"The program has exited (code {self.sim.exit_code}). " - 'Use "reset" to run it again.' + f'The program has exited (code {self.sim.exit_code}). Use "reset" to run it again.' ) return True return False diff --git a/mapachespim/memory_map.py b/mapachespim/memory_map.py index 856e6d6..8089d0d 100644 --- a/mapachespim/memory_map.py +++ b/mapachespim/memory_map.py @@ -54,60 +54,60 @@ def __str__(self) -> str: # These match the linker.ld files in examples/*/ RISCV64_LAYOUT = MemoryLayout( - text_base=0x80000000, # Traditional RISC-V bare-metal + text_base=0x80000000, # Traditional RISC-V bare-metal data_base=0x80100000, rodata_base=0x80080000, bss_base=0x80180000, stack_top=0x83F00000, - stack_size=0x100000, # 1MB stack + stack_size=0x100000, # 1MB stack is_64bit=True, is_little_endian=True, ) ARM64_LAYOUT = MemoryLayout( - text_base=0x10000000, # Unicorn default for ARM64 + text_base=0x10000000, # Unicorn default for ARM64 data_base=0x10100000, rodata_base=0x10080000, bss_base=0x10180000, stack_top=0x13F00000, - stack_size=0x100000, # 1MB stack + stack_size=0x100000, # 1MB stack is_64bit=True, is_little_endian=True, ) X86_64_LAYOUT = MemoryLayout( - text_base=0x400000, # Linux-like layout + text_base=0x400000, # Linux-like layout data_base=0x500000, rodata_base=0x480000, bss_base=0x580000, stack_top=0x7FFFFFFFE000, # Near top of user space - stack_size=0x100000, # 1MB stack + stack_size=0x100000, # 1MB stack is_64bit=True, is_little_endian=True, ) MIPS32_LAYOUT = MemoryLayout( - text_base=0x00400000, # Traditional MIPS user space + text_base=0x00400000, # Traditional MIPS user space data_base=0x10000000, rodata_base=0x00480000, bss_base=0x10080000, - stack_top=0x80000000, # Just below kernel space (kseg0) - stack_size=0x100000, # 1MB stack + stack_top=0x80000000, # Just below kernel space (kseg0) + stack_size=0x100000, # 1MB stack is_64bit=False, - is_little_endian=False, # MIPS is big-endian in MapacheSPIM + is_little_endian=False, # MIPS is big-endian in MapacheSPIM ) # Map of ISA names to layouts MEMORY_LAYOUTS: Dict[str, MemoryLayout] = { "riscv64": RISCV64_LAYOUT, - "riscv": RISCV64_LAYOUT, # Alias + "riscv": RISCV64_LAYOUT, # Alias "arm64": ARM64_LAYOUT, - "aarch64": ARM64_LAYOUT, # Alias + "aarch64": ARM64_LAYOUT, # Alias "x86_64": X86_64_LAYOUT, - "x86-64": X86_64_LAYOUT, # Alias - "x64": X86_64_LAYOUT, # Alias + "x86-64": X86_64_LAYOUT, # Alias + "x64": X86_64_LAYOUT, # Alias "mips32": MIPS32_LAYOUT, - "mips": MIPS32_LAYOUT, # Alias + "mips": MIPS32_LAYOUT, # Alias } @@ -127,7 +127,5 @@ def get_layout(isa: str) -> MemoryLayout: isa_lower = isa.lower().replace("-", "_") if isa_lower not in MEMORY_LAYOUTS: valid = sorted(set(MEMORY_LAYOUTS.keys())) - raise ValueError( - f"Unknown ISA: {isa!r}. Valid options: {', '.join(valid)}" - ) + raise ValueError(f"Unknown ISA: {isa!r}. Valid options: {', '.join(valid)}") return MEMORY_LAYOUTS[isa_lower] diff --git a/mapachespim/toolchain/__init__.py b/mapachespim/toolchain/__init__.py index d9a3c98..9d5a64c 100644 --- a/mapachespim/toolchain/__init__.py +++ b/mapachespim/toolchain/__init__.py @@ -189,10 +189,10 @@ def _detect_isa_from_directive(source: str) -> Optional[str]: for line in source.splitlines(): line = line.strip() # Skip empty lines and comments - if not line or line.startswith('#') or line.startswith('//') or line.startswith(';'): + if not line or line.startswith("#") or line.startswith("//") or line.startswith(";"): continue # Check for .isa directive - if line.lower().startswith('.isa'): + if line.lower().startswith(".isa"): parts = line.split(None, 1) if len(parts) >= 2: isa_value = parts[1].strip().lower().replace("-", "_") @@ -200,7 +200,7 @@ def _detect_isa_from_directive(source: str) -> Optional[str]: return isa_value # Stop after first non-empty, non-comment, non-.isa line # (the .isa directive should be at the top of the file) - if not line.startswith('.'): + if not line.startswith("."): break return None diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index d9751ed..510b84e 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -12,6 +12,7 @@ try: import keystone + # Verify keystone actually works by trying to access a constant _ = keystone.KS_ARCH_X86 KEYSTONE_AVAILABLE = True @@ -313,11 +314,13 @@ def assemble( sect_data = sections[sect_name] if sect_data.data or sect_name == ".text": base = section_bases.get(sect_name, self._layout.text_base) - builder.add_section(Section( - name=sect_name, - data=bytes(sect_data.data), - address=base, - )) + builder.add_section( + Section( + name=sect_name, + data=bytes(sect_data.data), + address=base, + ) + ) # Add symbols for name, addr in all_labels.items(): @@ -329,13 +332,15 @@ def assemble( section_idx = i + 1 break - builder.add_symbol(Symbol( - name=name, - address=addr, - sym_type=STT_FUNC if name == entry_symbol else STT_NOTYPE, - binding=STB_GLOBAL if is_global else STB_LOCAL, - section_index=section_idx, - )) + builder.add_symbol( + Symbol( + name=name, + address=addr, + sym_type=STT_FUNC if name == entry_symbol else STT_NOTYPE, + binding=STB_GLOBAL if is_global else STB_LOCAL, + section_index=section_idx, + ) + ) result.symbols = all_labels result.debug_lines = debug_lines @@ -530,9 +535,31 @@ def _get_x86_actual_size( mnemonic = parts[0].lower() if parts else "" # For branch/call instructions, use placeholder offset - if mnemonic in ('jmp', 'call', 'je', 'jne', 'jz', 'jnz', 'jl', 'jle', - 'jg', 'jge', 'ja', 'jae', 'jb', 'jbe', 'jo', 'jno', - 'js', 'jns', 'jc', 'jnc', 'loop', 'loope', 'loopne'): + if mnemonic in ( + "jmp", + "call", + "je", + "jne", + "jz", + "jnz", + "jl", + "jle", + "jg", + "jge", + "ja", + "jae", + "jb", + "jbe", + "jo", + "jno", + "js", + "jns", + "jc", + "jnc", + "loop", + "loope", + "loopne", + ): # Use a medium-range placeholder to get near (not short) encoding placeholder_addr = addr + 0x100 placeholder_instr = f"{mnemonic} {placeholder_addr}" @@ -577,11 +604,11 @@ def _estimate_fixed_instr_size( # adr with symbol -> movz + movk sequence (typically 2 instrs for 32-bit addr) if mnemonic == "adr": # Check if it's using a symbol (not a numeric offset) - ops = [o.strip() for o in operands.split(',')] + ops = [o.strip() for o in operands.split(",")] if len(ops) == 2: symbol = ops[1].strip() # If it's not a number, it's a symbol that needs expansion - if not symbol.startswith('#') and not symbol.lstrip('-').isdigit(): + if not symbol.startswith("#") and not symbol.lstrip("-").isdigit(): return base_size * 2 # movz + movk # ldr x0, =symbol -> expands to adr -> movz + movk if mnemonic == "ldr" and "=" in operands: @@ -600,68 +627,108 @@ def _estimate_x86_instr_size(self, instr: str) -> int: if not parts: return 1 - mnemonic = parts[0].lower().rstrip('bwlq') + mnemonic = parts[0].lower().rstrip("bwlq") operands = parts[1] if len(parts) > 1 else "" # Single/two-byte instructions - if mnemonic == 'nop': + if mnemonic == "nop": return 1 - if mnemonic == 'ret': + if mnemonic == "ret": return 1 - if mnemonic in ('syscall', 'hlt', 'cld', 'std', 'cdqe', 'cqo'): + if mnemonic in ("syscall", "hlt", "cld", "std", "cdqe", "cqo"): return 2 # Near jumps/calls with label: 5 bytes (1 opcode + 4 offset) # But conditional jumps can be 2 bytes for short jumps, assume 6 for safety - if mnemonic in ('jmp', 'call'): + if mnemonic in ("jmp", "call"): return 5 - if mnemonic in ('je', 'jne', 'jz', 'jnz', 'jl', 'jle', 'jg', 'jge', - 'ja', 'jae', 'jb', 'jbe', 'jo', 'jno', 'js', 'jns', - 'jc', 'jnc', 'loop', 'loope', 'loopne'): + if mnemonic in ( + "je", + "jne", + "jz", + "jnz", + "jl", + "jle", + "jg", + "jge", + "ja", + "jae", + "jb", + "jbe", + "jo", + "jno", + "js", + "jns", + "jc", + "jnc", + "loop", + "loope", + "loopne", + ): return 6 # 0F XX + 4-byte offset # RIP-relative addressing: 7 bytes # Format: symbol(%rip) or [rip + symbol] - if '%rip' in operands or 'rip' in operands.lower(): + if "%rip" in operands or "rip" in operands.lower(): # REX.W (1) + opcode (1-2) + ModR/M (1) + disp32 (4) = 7-8 bytes return 7 # LEA with memory operand - if mnemonic == 'lea': - if '%rip' in operands or 'rip' in operands.lower(): + if mnemonic == "lea": + if "%rip" in operands or "rip" in operands.lower(): return 7 return 4 # Base case # MOV with 64-bit immediate - if mnemonic == 'mov': + if mnemonic == "mov": # Check for 64-bit register destination with immediate - if operands.startswith('$') or operands.startswith('%'): + if operands.startswith("$") or operands.startswith("%"): # AT&T: movq $imm, %reg or Intel mov reg, imm - if any(r in operands for r in ['%rax', '%rbx', '%rcx', '%rdx', - '%rsi', '%rdi', '%rbp', '%rsp', - '%r8', '%r9', '%r10', '%r11', - '%r12', '%r13', '%r14', '%r15', - 'rax', 'rbx', 'rcx', 'rdx']): + if any( + r in operands + for r in [ + "%rax", + "%rbx", + "%rcx", + "%rdx", + "%rsi", + "%rdi", + "%rbp", + "%rsp", + "%r8", + "%r9", + "%r10", + "%r11", + "%r12", + "%r13", + "%r14", + "%r15", + "rax", + "rbx", + "rcx", + "rdx", + ] + ): # Could be 10 bytes for movabs - if '$' in operands: + if "$" in operands: return 10 return 7 # Simple reg-reg or reg-mem: 2-4 bytes return 3 # TEST, CMP with register: 2-3 bytes - if mnemonic in ('test', 'cmp'): - if '(' not in operands and '[' not in operands: + if mnemonic in ("test", "cmp"): + if "(" not in operands and "[" not in operands: return 3 # ADD, SUB, XOR, AND, OR with immediate or register: 3-7 bytes - if mnemonic in ('add', 'sub', 'xor', 'and', 'or'): - if '$' in operands or any(c.isdigit() for c in operands[:5]): + if mnemonic in ("add", "sub", "xor", "and", "or"): + if "$" in operands or any(c.isdigit() for c in operands[:5]): return 7 # Could have 32-bit immediate return 3 # Register-register # Push/pop: 1-2 bytes - if mnemonic in ('push', 'pop'): + if mnemonic in ("push", "pop"): return 2 # Default for unknown instructions @@ -726,7 +793,9 @@ def _assemble_section( try: encoding, count = self._ks.asm(instr, current_addr) if encoding is None or count == 0: - errors.append(f"Line {line.line_number}: Failed to assemble: {line.instruction}") + errors.append( + f"Line {line.line_number}: Failed to assemble: {line.instruction}" + ) continue code.extend(encoding) @@ -755,10 +824,11 @@ def _normalize_arm64_syntax(self, instr: str) -> str: Example: mov x0, #42 -> mov x0, 42 """ import re + # Match # followed by a number (decimal, hex, or negative) # But not at the start of the line (which would be a comment) # Pattern: comma or space, then #, then optional -, then number - return re.sub(r'([\s,])#(-?(?:0x[0-9a-fA-F]+|\d+))', r'\1\2', instr) + return re.sub(r"([\s,])#(-?(?:0x[0-9a-fA-F]+|\d+))", r"\1\2", instr) def _expand_arm64_pseudo( self, @@ -780,7 +850,7 @@ def _expand_arm64_pseudo( # Handle 'adr' with symbol - convert to movz + movk sequence for far addresses if mnemonic == "adr": - ops = [o.strip() for o in operands.split(',')] + ops = [o.strip() for o in operands.split(",")] if len(ops) == 2: reg = ops[0] symbol = ops[1].strip() @@ -837,7 +907,7 @@ def _expand_arm64_pseudo( # Handle cbz/cbnz (compare and branch if zero/not zero) if mnemonic in ("cbz", "cbnz"): - ops = [o.strip() for o in operands.split(',')] + ops = [o.strip() for o in operands.split(",")] if len(ops) == 2: reg, symbol = ops[0], ops[1].strip() if symbol in labels: @@ -846,7 +916,7 @@ def _expand_arm64_pseudo( # Handle tbz/tbnz (test bit and branch if zero/not zero) if mnemonic in ("tbz", "tbnz"): - ops = [o.strip() for o in operands.split(',')] + ops = [o.strip() for o in operands.split(",")] if len(ops) == 3: reg, bit, symbol = ops[0], ops[1], ops[2].strip() if symbol in labels: @@ -879,13 +949,35 @@ def _convert_x86_att_to_intel( operands = parts[1] if len(parts) > 1 else "" # Check if already Intel syntax (no % or $) - is_att_syntax = '%' in instr or '$' in instr + is_att_syntax = "%" in instr or "$" in instr # For Intel-syntax jump/call with symbol, resolve the symbol if not is_att_syntax: - if mnemonic in ('jmp', 'call', 'je', 'jne', 'jz', 'jnz', 'jl', 'jle', - 'jg', 'jge', 'ja', 'jae', 'jb', 'jbe', 'jo', 'jno', - 'js', 'jns', 'jc', 'jnc', 'loop', 'loope', 'loopne'): + if mnemonic in ( + "jmp", + "call", + "je", + "jne", + "jz", + "jnz", + "jl", + "jle", + "jg", + "jge", + "ja", + "jae", + "jb", + "jbe", + "jo", + "jno", + "js", + "jns", + "jc", + "jnc", + "loop", + "loope", + "loopne", + ): symbol = operands.strip() if symbol in labels: return f"{mnemonic} {labels[symbol]}" @@ -893,22 +985,50 @@ def _convert_x86_att_to_intel( # Handle special AT&T mnemonics # movslq = move sign-extend long to quad (Intel: movsxd) - if mnemonic == 'movslq': - mnemonic = 'movsxd' + if mnemonic == "movslq": + mnemonic = "movsxd" # cltq = sign-extend eax to rax (Intel: cdqe) - elif mnemonic == 'cltq': - return 'cdqe' + elif mnemonic == "cltq": + return "cdqe" # cqto = sign-extend rax to rdx:rax (Intel: cqo) - elif mnemonic == 'cqto': - return 'cqo' + elif mnemonic == "cqto": + return "cqo" # Remove size suffixes (b, w, l, q) - if mnemonic.endswith(('b', 'w', 'l', 'q')) and len(mnemonic) > 2: + if mnemonic.endswith(("b", "w", "l", "q")) and len(mnemonic) > 2: base = mnemonic[:-1] - if base in ('mov', 'add', 'sub', 'xor', 'and', 'or', 'cmp', 'test', - 'lea', 'push', 'pop', 'call', 'ret', 'jmp', 'dec', 'inc', - 'neg', 'not', 'mul', 'imul', 'div', 'idiv', 'shl', 'shr', - 'sar', 'sal', 'rol', 'ror', 'rcl', 'rcr'): + if base in ( + "mov", + "add", + "sub", + "xor", + "and", + "or", + "cmp", + "test", + "lea", + "push", + "pop", + "call", + "ret", + "jmp", + "dec", + "inc", + "neg", + "not", + "mul", + "imul", + "div", + "idiv", + "shl", + "shr", + "sar", + "sal", + "rol", + "ror", + "rcl", + "rcr", + ): mnemonic = base # Handle no-operand instructions @@ -920,11 +1040,11 @@ def _convert_x86_att_to_intel( depth = 0 current = "" for c in operands: - if c == '(': + if c == "(": depth += 1 - elif c == ')': + elif c == ")": depth -= 1 - elif c == ',' and depth == 0: + elif c == "," and depth == 0: op_list.append(current.strip()) current = "" continue @@ -938,7 +1058,7 @@ def _convert_x86_att_to_intel( op = op.strip() # Handle RIP-relative: symbol(%rip) -> [rip + address] - rip_match = re.match(r'(\w+)\s*\(\s*%rip\s*\)', op) + rip_match = re.match(r"(\w+)\s*\(\s*%rip\s*\)", op) if rip_match: symbol = rip_match.group(1) if symbol in labels: @@ -952,9 +1072,7 @@ def _convert_x86_att_to_intel( # Handle memory operands with index and scale: offset(base, index, scale) # AT&T: (%r12, %rax, 4) -> Intel: [r12 + rax*4] # AT&T: 16(%rsp, %rax, 8) -> Intel: [rsp + rax*8 + 16] - sib_match = re.match( - r'(-?\d+)?\s*\(\s*%(\w+)\s*,\s*%(\w+)\s*,\s*(\d+)\s*\)', op - ) + sib_match = re.match(r"(-?\d+)?\s*\(\s*%(\w+)\s*,\s*%(\w+)\s*,\s*(\d+)\s*\)", op) if sib_match: offset = sib_match.group(1) base = sib_match.group(2) @@ -968,7 +1086,7 @@ def _convert_x86_att_to_intel( continue # Handle memory operands: (%reg) -> [reg], offset(%reg) -> [reg + offset] - mem_match = re.match(r'(-?\d+)?\s*\(\s*%(\w+)\s*\)', op) + mem_match = re.match(r"(-?\d+)?\s*\(\s*%(\w+)\s*\)", op) if mem_match: offset = mem_match.group(1) reg = mem_match.group(2) @@ -979,7 +1097,7 @@ def _convert_x86_att_to_intel( continue # Handle immediate: $value -> value - if op.startswith('$'): + if op.startswith("$"): value = op[1:] # Check if it's a symbol if value in labels: @@ -989,7 +1107,7 @@ def _convert_x86_att_to_intel( continue # Handle register: %reg -> reg - if op.startswith('%'): + if op.startswith("%"): converted.append(op[1:]) continue @@ -1002,7 +1120,7 @@ def _convert_x86_att_to_intel( converted.append(op) # Reverse operand order for two-operand instructions (AT&T: src, dst -> Intel: dst, src) - if len(converted) == 2 and mnemonic not in ('push', 'pop', 'call', 'jmp', 'syscall'): + if len(converted) == 2 and mnemonic not in ("push", "pop", "call", "jmp", "syscall"): converted = converted[::-1] return f"{mnemonic} {', '.join(converted)}" diff --git a/mapachespim/toolchain/cli.py b/mapachespim/toolchain/cli.py index 9d0d84c..307e04f 100644 --- a/mapachespim/toolchain/cli.py +++ b/mapachespim/toolchain/cli.py @@ -46,7 +46,8 @@ def main(args: list[str] | None = None) -> int: ) parser.add_argument( - "-o", "--output", + "-o", + "--output", type=Path, default=None, help="Output ELF file (default: source with .elf extension)", @@ -68,13 +69,15 @@ def main(args: list[str] | None = None) -> int: ) parser.add_argument( - "-g", "--debug", + "-g", + "--debug", action="store_true", help="Generate DWARF debug information", ) parser.add_argument( - "-v", "--verbose", + "-v", + "--verbose", action="store_true", help="Verbose output", ) diff --git a/mapachespim/toolchain/directives.py b/mapachespim/toolchain/directives.py index d624769..4983b0d 100644 --- a/mapachespim/toolchain/directives.py +++ b/mapachespim/toolchain/directives.py @@ -15,6 +15,7 @@ class LineType(Enum): """Type of assembly source line.""" + EMPTY = auto() COMMENT = auto() LABEL = auto() @@ -84,14 +85,14 @@ class DirectiveParser: """ # Regex patterns - LABEL_PATTERN = re.compile(r'^([a-zA-Z_][a-zA-Z0-9_]*):(.*)$') - DIRECTIVE_PATTERN = re.compile(r'^\s*\.(\w+)\s*(.*)$') + LABEL_PATTERN = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_]*):(.*)$") + DIRECTIVE_PATTERN = re.compile(r"^\s*\.(\w+)\s*(.*)$") # Hash comment: # at start, or # preceded by space and NOT followed by digit/minus (ARM immediate) # This handles both #42 and #-32 as ARM immediates, not comments - HASH_COMMENT = re.compile(r'(^|\s)#(?!-?\d).*$') + HASH_COMMENT = re.compile(r"(^|\s)#(?!-?\d).*$") COMMENT_PATTERNS = [ - re.compile(r'//.*$'), # C++ style - re.compile(r';.*$'), # Semicolon comments + re.compile(r"//.*$"), # C++ style + re.compile(r";.*$"), # Semicolon comments ] # Valid ISA values for .isa directive @@ -117,8 +118,8 @@ def __init__(self) -> None: def _get_endianness(self) -> Literal["little", "big"]: """Return byte order for data directives based on ISA.""" if self.isa in self.BIG_ENDIAN_ISAS: - return 'big' - return 'little' + return "big" + return "little" def parse(self, source: str) -> Dict[str, SectionData]: """ @@ -160,10 +161,10 @@ def _parse_line(self, line_num: int, line: str) -> ParsedLine: original = line # Strip hash comments (# at start or after space, but not ARM immediates like #42) - line = self.HASH_COMMENT.sub(r'\1', line) + line = self.HASH_COMMENT.sub(r"\1", line) # Strip other comment styles for pattern in self.COMMENT_PATTERNS: - line = pattern.sub('', line) + line = pattern.sub("", line) line = line.strip() @@ -189,15 +190,16 @@ def _parse_line(self, line_num: int, line: str) -> ParsedLine: args_str = dir_match.group(2).strip() args = self._parse_args(args_str) if args_str else [] return ParsedLine( - line_num, LineType.DIRECTIVE, original, - label=label, directive=directive, directive_args=args + line_num, + LineType.DIRECTIVE, + original, + label=label, + directive=directive, + directive_args=args, ) # Must be an instruction - return ParsedLine( - line_num, LineType.INSTRUCTION, original, - label=label, instruction=line - ) + return ParsedLine(line_num, LineType.INSTRUCTION, original, label=label, instruction=line) def _parse_args(self, args_str: str) -> List[str]: """Parse comma-separated directive arguments, respecting quotes.""" @@ -211,11 +213,11 @@ def _parse_args(self, args_str: str) -> List[str]: current += char if char == string_char: in_string = False - elif char in '"\'': + elif char in "\"'": in_string = True string_char = char current += char - elif char == ',': + elif char == ",": if current.strip(): args.append(current.strip()) current = "" @@ -258,7 +260,7 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: if directive == "section": if args: sect_name = args[0].strip('"') - if not sect_name.startswith('.'): + if not sect_name.startswith("."): sect_name = f".{sect_name}" if sect_name not in self.sections: self.sections[sect_name] = SectionData(name=sect_name) @@ -307,7 +309,7 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: section.data.extend((value & 0xFFFF).to_bytes(2, endian)) section.current_offset += 2 except Exception: - section.data.extend(b'\x00\x00') + section.data.extend(b"\x00\x00") section.current_offset += 2 return @@ -319,7 +321,7 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: section.data.extend((value & 0xFFFFFFFF).to_bytes(4, endian)) section.current_offset += 4 except Exception: - section.data.extend(b'\x00\x00\x00\x00') + section.data.extend(b"\x00\x00\x00\x00") section.current_offset += 4 return @@ -331,7 +333,7 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: section.data.extend(value.to_bytes(8, endian)) section.current_offset += 8 except Exception: - section.data.extend(b'\x00' * 8) + section.data.extend(b"\x00" * 8) section.current_offset += 8 return @@ -339,14 +341,14 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: if directive == "ascii": for arg in args: s = self._parse_string(arg) - section.data.extend(s.encode('utf-8')) + section.data.extend(s.encode("utf-8")) section.current_offset += len(s) return if directive in ("asciz", "string"): for arg in args: s = self._parse_string(arg) - section.data.extend(s.encode('utf-8')) + section.data.extend(s.encode("utf-8")) section.data.append(0) # Null terminator section.current_offset += len(s) + 1 return @@ -360,7 +362,7 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: alignment = self._evaluate_expr(args[0]) if alignment > 0: padding = (alignment - (section.current_offset % alignment)) % alignment - section.data.extend(b'\x00' * padding) + section.data.extend(b"\x00" * padding) section.current_offset += padding except Exception: pass @@ -372,7 +374,7 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: alignment = self._evaluate_expr(args[0]) if alignment > 0: padding = (alignment - (section.current_offset % alignment)) % alignment - section.data.extend(b'\x00' * padding) + section.data.extend(b"\x00" * padding) section.current_offset += padding except Exception: pass @@ -384,7 +386,7 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: power = self._evaluate_expr(args[0]) alignment = 1 << power padding = (alignment - (section.current_offset % alignment)) % alignment - section.data.extend(b'\x00' * padding) + section.data.extend(b"\x00" * padding) section.current_offset += padding except Exception: pass @@ -420,15 +422,15 @@ def _evaluate_expr(self, expr: str) -> int: return self.constants[expr] # Hex - if expr.startswith('0x') or expr.startswith('0X'): + if expr.startswith("0x") or expr.startswith("0X"): return int(expr, 16) # Binary - if expr.startswith('0b') or expr.startswith('0B'): + if expr.startswith("0b") or expr.startswith("0B"): return int(expr, 2) # Octal - if expr.startswith('0') and len(expr) > 1 and expr[1:].isdigit(): + if expr.startswith("0") and len(expr) > 1 and expr[1:].isdigit(): return int(expr, 8) # Character literal @@ -442,29 +444,29 @@ def _evaluate_expr(self, expr: str) -> int: pass # Simple arithmetic - for op in ['+', '-', '*', '/', '<<', '>>', '|', '&', '^']: + for op in ["+", "-", "*", "/", "<<", ">>", "|", "&", "^"]: if op in expr: parts = expr.rsplit(op, 1) if len(parts) == 2: left = self._evaluate_expr(parts[0]) right = self._evaluate_expr(parts[1]) - if op == '+': + if op == "+": return left + right - if op == '-': + if op == "-": return left - right - if op == '*': + if op == "*": return left * right - if op == '/': + if op == "/": return left // right if right else 0 - if op == '<<': + if op == "<<": return left << right - if op == '>>': + if op == ">>": return left >> right - if op == '|': + if op == "|": return left | right - if op == '&': + if op == "&": return left & right - if op == '^': + if op == "^": return left ^ right raise ValueError(f"Cannot evaluate: {expr}") @@ -475,33 +477,32 @@ def _parse_string(self, s: str) -> str: return "" # Remove quotes - if (s.startswith('"') and s.endswith('"')) or \ - (s.startswith("'") and s.endswith("'")): + if (s.startswith('"') and s.endswith('"')) or (s.startswith("'") and s.endswith("'")): s = s[1:-1] # Process escape sequences result = [] i = 0 while i < len(s): - if s[i] == '\\' and i + 1 < len(s): + if s[i] == "\\" and i + 1 < len(s): next_char = s[i + 1] - if next_char == 'n': - result.append('\n') - elif next_char == 't': - result.append('\t') - elif next_char == 'r': - result.append('\r') - elif next_char == '0': - result.append('\0') - elif next_char == '\\': - result.append('\\') + if next_char == "n": + result.append("\n") + elif next_char == "t": + result.append("\t") + elif next_char == "r": + result.append("\r") + elif next_char == "0": + result.append("\0") + elif next_char == "\\": + result.append("\\") elif next_char == '"': result.append('"') elif next_char == "'": result.append("'") - elif next_char == 'x' and i + 3 < len(s): + elif next_char == "x" and i + 3 < len(s): try: - value = int(s[i+2:i+4], 16) + value = int(s[i + 2 : i + 4], 16) result.append(chr(value)) i += 2 except ValueError: @@ -513,7 +514,7 @@ def _parse_string(self, s: str) -> str: result.append(s[i]) i += 1 - return ''.join(result) + return "".join(result) def get_instructions(self, section_name: str) -> List[Tuple[int, str, Optional[str]]]: """ diff --git a/mapachespim/toolchain/dwarf.py b/mapachespim/toolchain/dwarf.py index 1bfd2ac..5d3a6dc 100644 --- a/mapachespim/toolchain/dwarf.py +++ b/mapachespim/toolchain/dwarf.py @@ -113,23 +113,28 @@ def build_debug_abbrev(self) -> bytes: Returns fixed abbreviation table with one entry for compile_unit. """ - return bytes([ - # Abbreviation 1: DW_TAG_compile_unit - 0x01, # abbrev number - DW_TAG_compile_unit, # tag - 0x00, # no children - - # Attributes - DW_AT_stmt_list, DW_FORM_data4, # stmt_list: offset into .debug_line - DW_AT_low_pc, DW_FORM_addr, # low_pc: start address - DW_AT_high_pc, DW_FORM_addr, # high_pc: end address - DW_AT_name, DW_FORM_string, # name: source filename (inline) - DW_AT_producer, DW_FORM_string, # producer: assembler name (inline) - - 0x00, 0x00, # end of attributes - - 0x00, # end of abbreviations - ]) + return bytes( + [ + # Abbreviation 1: DW_TAG_compile_unit + 0x01, # abbrev number + DW_TAG_compile_unit, # tag + 0x00, # no children + # Attributes + DW_AT_stmt_list, + DW_FORM_data4, # stmt_list: offset into .debug_line + DW_AT_low_pc, + DW_FORM_addr, # low_pc: start address + DW_AT_high_pc, + DW_FORM_addr, # high_pc: end address + DW_AT_name, + DW_FORM_string, # name: source filename (inline) + DW_AT_producer, + DW_FORM_string, # producer: assembler name (inline) + 0x00, + 0x00, # end of attributes + 0x00, # end of abbreviations + ] + ) def build_debug_info( self, @@ -157,22 +162,22 @@ def build_debug_info( die_content.append(0x01) # abbrev number 1 # stmt_list (4 bytes) - die_content.extend(struct.pack(self._endian + 'I', stmt_list_offset)) + die_content.extend(struct.pack(self._endian + "I", stmt_list_offset)) # low_pc and high_pc (addr_size bytes each) if self.addr_size == 8: - die_content.extend(struct.pack(self._endian + 'Q', low_pc)) - die_content.extend(struct.pack(self._endian + 'Q', high_pc)) + die_content.extend(struct.pack(self._endian + "Q", low_pc)) + die_content.extend(struct.pack(self._endian + "Q", high_pc)) else: - die_content.extend(struct.pack(self._endian + 'I', low_pc)) - die_content.extend(struct.pack(self._endian + 'I', high_pc)) + die_content.extend(struct.pack(self._endian + "I", low_pc)) + die_content.extend(struct.pack(self._endian + "I", high_pc)) # name (null-terminated string) - die_content.extend(filename.encode('utf-8')) + die_content.extend(filename.encode("utf-8")) die_content.append(0x00) # producer (null-terminated string) - die_content.extend(producer.encode('utf-8')) + die_content.extend(producer.encode("utf-8")) die_content.append(0x00) # Build CU header @@ -181,10 +186,10 @@ def build_debug_info( unit_length = cu_header_size + len(die_content) header = bytearray() - header.extend(struct.pack(self._endian + 'I', unit_length)) # unit_length - header.extend(struct.pack(self._endian + 'H', 2)) # version = 2 - header.extend(struct.pack(self._endian + 'I', 0)) # abbrev_offset = 0 - header.append(self.addr_size) # address_size + header.extend(struct.pack(self._endian + "I", unit_length)) # unit_length + header.extend(struct.pack(self._endian + "H", 2)) # version = 2 + header.extend(struct.pack(self._endian + "I", 0)) # abbrev_offset = 0 + header.append(self.addr_size) # address_size return bytes(header + die_content) @@ -210,17 +215,17 @@ def build_debug_line( # Build directory table dir_table = bytearray() - dir_table.extend(b'.\x00') # current directory - dir_table.append(0x00) # end of directories + dir_table.extend(b".\x00") # current directory + dir_table.append(0x00) # end of directories # Build file table file_table = bytearray() - file_table.extend(filename.encode('utf-8')) - file_table.append(0x00) # null terminator - file_table.append(0x01) # directory index - file_table.append(0x00) # mtime (ULEB128 0) - file_table.append(0x00) # size (ULEB128 0) - file_table.append(0x00) # end of files + file_table.extend(filename.encode("utf-8")) + file_table.append(0x00) # null terminator + file_table.append(0x01) # directory index + file_table.append(0x00) # mtime (ULEB128 0) + file_table.append(0x00) # size (ULEB128 0) + file_table.append(0x00) # end of files # Build line program program = self._build_line_program(lines, min_instr_length) @@ -229,20 +234,22 @@ def build_debug_line( # header_length includes: min_instr_len(1) + default_is_stmt(1) + line_base(1) + # line_range(1) + opcode_base(1) + std_opcode_lengths(9) + # dir_table + file_table - header_content_len = 1 + 1 + 1 + 1 + 1 + len(std_opcode_lengths) + len(dir_table) + len(file_table) + header_content_len = ( + 1 + 1 + 1 + 1 + 1 + len(std_opcode_lengths) + len(dir_table) + len(file_table) + ) # Build header header = bytearray() # Placeholder for unit_length (filled in later) - header.extend(b'\x00\x00\x00\x00') - header.extend(struct.pack(self._endian + 'H', 2)) # version = 2 - header.extend(struct.pack(self._endian + 'I', header_content_len)) # header_length - header.append(min_instr_length) # minimum_instruction_length - header.append(1) # default_is_stmt - header.append(LINE_BASE & 0xFF) # line_base (signed, -5) - header.append(LINE_RANGE) # line_range (14) - header.append(OPCODE_BASE) # opcode_base (10) - header.extend(std_opcode_lengths) # standard_opcode_lengths + header.extend(b"\x00\x00\x00\x00") + header.extend(struct.pack(self._endian + "H", 2)) # version = 2 + header.extend(struct.pack(self._endian + "I", header_content_len)) # header_length + header.append(min_instr_length) # minimum_instruction_length + header.append(1) # default_is_stmt + header.append(LINE_BASE & 0xFF) # line_base (signed, -5) + header.append(LINE_RANGE) # line_range (14) + header.append(OPCODE_BASE) # opcode_base (10) + header.extend(std_opcode_lengths) # standard_opcode_lengths header.extend(dir_table) header.extend(file_table) @@ -251,7 +258,7 @@ def build_debug_line( # Fill in unit_length (total length minus the 4-byte length field itself) unit_length = len(result) - 4 - struct.pack_into(self._endian + 'I', result, 0, unit_length) + struct.pack_into(self._endian + "I", result, 0, unit_length) return bytes(result) @@ -265,7 +272,7 @@ def _build_line_program( if not lines: # Empty program - just end sequence - program.extend(self._extended_opcode(DW_LNE_end_sequence, b'')) + program.extend(self._extended_opcode(DW_LNE_end_sequence, b"")) return bytes(program) # Sort lines by address @@ -281,9 +288,9 @@ def _build_line_program( if current_addr == 0 or addr < current_addr: # Use extended opcode to set address if self.addr_size == 8: - addr_bytes = struct.pack(self._endian + 'Q', addr) + addr_bytes = struct.pack(self._endian + "Q", addr) else: - addr_bytes = struct.pack(self._endian + 'I', addr) + addr_bytes = struct.pack(self._endian + "I", addr) program.extend(self._extended_opcode(DW_LNE_set_address, addr_bytes)) current_addr = addr @@ -311,7 +318,7 @@ def _build_line_program( current_line = line # End sequence - program.extend(self._extended_opcode(DW_LNE_end_sequence, b'')) + program.extend(self._extended_opcode(DW_LNE_end_sequence, b"")) return bytes(program) diff --git a/mapachespim/toolchain/elf_builder.py b/mapachespim/toolchain/elf_builder.py index f22dc65..f5f1e6e 100644 --- a/mapachespim/toolchain/elf_builder.py +++ b/mapachespim/toolchain/elf_builder.py @@ -212,30 +212,36 @@ def add_debug_sections( debug_line: .debug_line section content. """ # Debug sections: SHT_PROGBITS, no flags (not loadable), address 0 - self.sections.append(Section( - name=".debug_abbrev", - data=debug_abbrev, - address=0, - sh_type=SHT_PROGBITS, - sh_flags=0, - alignment=1, - )) - self.sections.append(Section( - name=".debug_info", - data=debug_info, - address=0, - sh_type=SHT_PROGBITS, - sh_flags=0, - alignment=1, - )) - self.sections.append(Section( - name=".debug_line", - data=debug_line, - address=0, - sh_type=SHT_PROGBITS, - sh_flags=0, - alignment=1, - )) + self.sections.append( + Section( + name=".debug_abbrev", + data=debug_abbrev, + address=0, + sh_type=SHT_PROGBITS, + sh_flags=0, + alignment=1, + ) + ) + self.sections.append( + Section( + name=".debug_info", + data=debug_info, + address=0, + sh_type=SHT_PROGBITS, + sh_flags=0, + alignment=1, + ) + ) + self.sections.append( + Section( + name=".debug_line", + data=debug_line, + address=0, + sh_type=SHT_PROGBITS, + sh_flags=0, + alignment=1, + ) + ) def build(self) -> bytes: """ @@ -297,8 +303,7 @@ def _build_elf64(self) -> bytes: st_other = 0 st_shndx = sym.section_index if sym.section_index else SHN_ABS symtab += struct.pack( - f"{endian}IBBHQQ", - st_name, st_info, st_other, st_shndx, sym.address, sym.size + f"{endian}IBBHQQ", st_name, st_info, st_other, st_shndx, sym.address, sym.size ) # Calculate offsets @@ -343,19 +348,19 @@ def _build_elf64(self) -> bytes: ehdr = bytes(e_ident) + struct.pack( f"{endian}HHIQQQIHHHHHH", - ET_EXEC, # e_type - self.machine, # e_machine - EV_CURRENT, # e_version - self.entry, # e_entry - phdr_offset, # e_phoff - shdr_offset, # e_shoff - 0, # e_flags - ehdr_size, # e_ehsize - phdr_size, # e_phentsize - num_phdrs, # e_phnum - shdr_size, # e_shentsize - num_shdrs, # e_shnum - shstrtab_idx, # e_shstrndx + ET_EXEC, # e_type + self.machine, # e_machine + EV_CURRENT, # e_version + self.entry, # e_entry + phdr_offset, # e_phoff + shdr_offset, # e_shoff + 0, # e_flags + ehdr_size, # e_ehsize + phdr_size, # e_phentsize + num_phdrs, # e_phnum + shdr_size, # e_shentsize + num_shdrs, # e_shnum + shstrtab_idx, # e_shstrndx ) # Build program headers @@ -374,14 +379,14 @@ def _build_elf64(self) -> bytes: phdrs += struct.pack( f"{endian}IIQQQQQQ", - PT_LOAD, # p_type - flags, # p_flags - file_offset, # p_offset - section.address, # p_vaddr - section.address, # p_paddr - file_size, # p_filesz - mem_size, # p_memsz - section.alignment, # p_align + PT_LOAD, # p_type + flags, # p_flags + file_offset, # p_offset + section.address, # p_vaddr + section.address, # p_paddr + file_size, # p_filesz + mem_size, # p_memsz + section.alignment, # p_align ) # Build section data @@ -429,16 +434,16 @@ def _build_elf64(self) -> bytes: shdrs += struct.pack( f"{endian}IIQQQQIIQQ", - sh_name, # sh_name - section.sh_type, # sh_type + sh_name, # sh_name + section.sh_type, # sh_type section.sh_flags, # sh_flags - section.address, # sh_addr - sh_offset, # sh_offset - sh_size, # sh_size - 0, # sh_link - 0, # sh_info - section.alignment, # sh_addralign - 0, # sh_entsize + section.address, # sh_addr + sh_offset, # sh_offset + sh_size, # sh_size + 0, # sh_link + 0, # sh_info + section.alignment, # sh_addralign + 0, # sh_entsize ) # .shstrtab section header @@ -450,7 +455,10 @@ def _build_elf64(self) -> bytes: 0, shstrtab_offset, len(shstrtab), - 0, 0, 1, 0, + 0, + 0, + 1, + 0, ) # .strtab section header @@ -462,7 +470,10 @@ def _build_elf64(self) -> bytes: 0, strtab_offset, len(strtab), - 0, 0, 1, 0, + 0, + 0, + 1, + 0, ) # .symtab section header @@ -475,10 +486,10 @@ def _build_elf64(self) -> bytes: 0, symtab_offset, len(symtab), - strtab_idx, # sh_link = string table index - 1, # sh_info = first global symbol + strtab_idx, # sh_link = string table index + 1, # sh_info = first global symbol 8, - 24, # entry size for 64-bit symbols + 24, # entry size for 64-bit symbols ) return ehdr + phdrs + section_data + shdrs @@ -529,8 +540,7 @@ def _build_elf32(self) -> bytes: st_other = 0 st_shndx = sym.section_index if sym.section_index else SHN_ABS symtab += struct.pack( - f"{endian}IIIBBH", - st_name, sym.address, sym.size, st_info, st_other, st_shndx + f"{endian}IIIBBH", st_name, sym.address, sym.size, st_info, st_other, st_shndx ) # Calculate offsets @@ -663,7 +673,8 @@ def _build_elf32(self) -> bytes: section.address, sh_offset, sh_size, - 0, 0, + 0, + 0, section.alignment, 0, ) @@ -672,28 +683,45 @@ def _build_elf32(self) -> bytes: shdrs += struct.pack( f"{endian}IIIIIIIIII", shstrtab_offsets[".shstrtab"], - SHT_STRTAB, 0, 0, - shstrtab_offset, len(shstrtab), - 0, 0, 1, 0, + SHT_STRTAB, + 0, + 0, + shstrtab_offset, + len(shstrtab), + 0, + 0, + 1, + 0, ) # .strtab shdrs += struct.pack( f"{endian}IIIIIIIIII", shstrtab_offsets[".strtab"], - SHT_STRTAB, 0, 0, - strtab_offset, len(strtab), - 0, 0, 1, 0, + SHT_STRTAB, + 0, + 0, + strtab_offset, + len(strtab), + 0, + 0, + 1, + 0, ) # .symtab shdrs += struct.pack( f"{endian}IIIIIIIIII", shstrtab_offsets[".symtab"], - SHT_SYMTAB, 0, 0, - symtab_offset, len(symtab), - strtab_idx, 1, - 4, 16, + SHT_SYMTAB, + 0, + 0, + symtab_offset, + len(symtab), + strtab_idx, + 1, + 4, + 16, ) return ehdr + phdrs + section_data + shdrs diff --git a/mapachespim/toolchain/expr.py b/mapachespim/toolchain/expr.py index 0cf1b5d..2145095 100644 --- a/mapachespim/toolchain/expr.py +++ b/mapachespim/toolchain/expr.py @@ -139,5 +139,3 @@ def split_operands(text: str) -> List[str]: if cur.strip() or ops: ops.append(cur.strip()) return ops - - diff --git a/mapachespim/toolchain/mips.py b/mapachespim/toolchain/mips.py index 22b6bf8..f6eccfe 100644 --- a/mapachespim/toolchain/mips.py +++ b/mapachespim/toolchain/mips.py @@ -259,7 +259,10 @@ def jump_target(text: str) -> int: if len(ops) == 3 and m in ("div", "divu"): # Three-operand pseudo form: quotient into rd rd = parse_register(ops[0]) - return [_r(parse_register(ops[1]), parse_register(ops[2]), 0, 0, MULDIV[m]), _r(0, 0, rd, 0, 0x12)] + return [ + _r(parse_register(ops[1]), parse_register(ops[2]), 0, 0, MULDIV[m]), + _r(0, 0, rd, 0, 0x12), + ] _expect(ops, 2, f"{m} $rs, $rt") return [_r(parse_register(ops[0]), parse_register(ops[1]), 0, 0, MULDIV[m])] @@ -267,7 +270,10 @@ def jump_target(text: str) -> int: _expect(ops, 3, f"{m} $rd, $rs, $rt") rd = parse_register(ops[0]) funct = MULDIV["div" if m == "rem" else "divu"] - return [_r(parse_register(ops[1]), parse_register(ops[2]), 0, 0, funct), _r(0, 0, rd, 0, 0x10)] + return [ + _r(parse_register(ops[1]), parse_register(ops[2]), 0, 0, funct), + _r(0, 0, rd, 0, 0x10), + ] if m in HILO_TO: _expect(ops, 1, f"{m} $rd") @@ -287,7 +293,9 @@ def jump_target(text: str) -> int: if m in SPECIAL2_ACC: _expect(ops, 2, f"{m} $rs, $rt") - return [_r(parse_register(ops[0]), parse_register(ops[1]), 0, 0, SPECIAL2_ACC[m], opcode=0x1C)] + return [ + _r(parse_register(ops[0]), parse_register(ops[1]), 0, 0, SPECIAL2_ACC[m], opcode=0x1C) + ] if m in ("clz", "clo"): _expect(ops, 2, f"{m} $rd, $rs") @@ -411,7 +419,11 @@ def jump_target(text: str) -> int: _expect(ops, 2, "abs $rd, $rs") rd, rs = parse_register(ops[0]), parse_register(ops[1]) # sra $at, rs, 31; xor rd, rs, $at; subu rd, rd, $at - return [_r(0, rs, AT, 31, 0x03), _r(rs, AT, rd, 0, ALU3["xor"]), _r(rd, AT, rd, 0, ALU3["subu"])] + return [ + _r(0, rs, AT, 31, 0x03), + _r(rs, AT, rd, 0, ALU3["xor"]), + _r(rd, AT, rd, 0, ALU3["subu"]), + ] raise MIPSEncodeError(f"unknown instruction '{mnemonic}'") diff --git a/mapachespim/toolchain/riscv.py b/mapachespim/toolchain/riscv.py index 7e86e9e..025aa8a 100644 --- a/mapachespim/toolchain/riscv.py +++ b/mapachespim/toolchain/riscv.py @@ -301,7 +301,16 @@ def target_offset(text: str) -> int: raise RISCVEncodeError(f"unknown register '{ops[2].strip()}'") raise opcode, f3, f7 = R_TYPE[m] - return [_r(opcode, f3, f7, parse_register(ops[0]), parse_register(ops[1]), parse_register(ops[2]))] + return [ + _r( + opcode, + f3, + f7, + parse_register(ops[0]), + parse_register(ops[1]), + parse_register(ops[2]), + ) + ] if m in I_ALU: _expect(ops, 3, f"{m} rd, rs1, imm") @@ -313,7 +322,14 @@ def target_offset(text: str) -> int: opcode, f3, hi, bits = SHIFT_IMM[m] shamt = _check_unsigned(ctx.eval(ops[2]), bits, "shift amount") rd, rs1 = parse_register(ops[0]), parse_register(ops[1]) - return [((hi << 26) if bits == 6 else (hi << 25)) | (shamt << 20) | (rs1 << 15) | (f3 << 12) | (rd << 7) | opcode] + return [ + ((hi << 26) if bits == 6 else (hi << 25)) + | (shamt << 20) + | (rs1 << 15) + | (f3 << 12) + | (rd << 7) + | opcode + ] if m in LOADS: _expect(ops, 2, f"{m} rd, offset(rs1)") @@ -337,12 +353,21 @@ def target_offset(text: str) -> int: if m in BRANCHES: _expect(ops, 3, f"{m} rs1, rs2, label") - return [_b(BRANCHES[m], parse_register(ops[0]), parse_register(ops[1]), target_offset(ops[2]))] + return [ + _b(BRANCHES[m], parse_register(ops[0]), parse_register(ops[1]), target_offset(ops[2])) + ] if m in SWAP_BRANCHES: _expect(ops, 3, f"{m} rs1, rs2, label") real = SWAP_BRANCHES[m] - return [_b(BRANCHES[real], parse_register(ops[1]), parse_register(ops[0]), target_offset(ops[2]))] + return [ + _b( + BRANCHES[real], + parse_register(ops[1]), + parse_register(ops[0]), + target_offset(ops[2]), + ) + ] if m in ZERO_BRANCHES: _expect(ops, 2, f"{m} rs, label") @@ -471,9 +496,13 @@ def target_offset(text: str) -> int: if m in ("csrr", "csrw", "csrs", "csrc"): _expect(ops, 2, f"{m} ...") if m == "csrr": - return [(_parse_csr(ops[1], ctx) << 20) | (2 << 12) | (parse_register(ops[0]) << 7) | 0x73] + return [ + (_parse_csr(ops[1], ctx) << 20) | (2 << 12) | (parse_register(ops[0]) << 7) | 0x73 + ] f3 = {"csrw": 1, "csrs": 2, "csrc": 3}[m] - return [(_parse_csr(ops[0], ctx) << 20) | (parse_register(ops[1]) << 15) | (f3 << 12) | 0x73] + return [ + (_parse_csr(ops[0], ctx) << 20) | (parse_register(ops[1]) << 15) | (f3 << 12) | 0x73 + ] raise RISCVEncodeError(f"unknown instruction '{mnemonic}'") @@ -531,4 +560,3 @@ def instruction_size(text: str, labels: Dict[str, int], address: int = 0) -> int except RISCVEncodeError: return INSTR_SIZE # the real error is reported by encode() return len(words) * INSTR_SIZE - diff --git a/mapachespim/unicorn_backend.py b/mapachespim/unicorn_backend.py index a549080..03c2738 100644 --- a/mapachespim/unicorn_backend.py +++ b/mapachespim/unicorn_backend.py @@ -377,15 +377,38 @@ class MIPSConfig(ISAConfig): # MIPS register names in standard order ($0-$31) _GPR_NAMES: List[str] = [ - "zero", "at", "v0", "v1", # $0-$3 - "a0", "a1", "a2", "a3", # $4-$7 - "t0", "t1", "t2", "t3", # $8-$11 - "t4", "t5", "t6", "t7", # $12-$15 - "s0", "s1", "s2", "s3", # $16-$19 - "s4", "s5", "s6", "s7", # $20-$23 - "t8", "t9", # $24-$25 - "k0", "k1", # $26-$27 - "gp", "sp", "fp", "ra", # $28-$31 + "zero", + "at", + "v0", + "v1", # $0-$3 + "a0", + "a1", + "a2", + "a3", # $4-$7 + "t0", + "t1", + "t2", + "t3", # $8-$11 + "t4", + "t5", + "t6", + "t7", # $12-$15 + "s0", + "s1", + "s2", + "s3", # $16-$19 + "s4", + "s5", + "s6", + "s7", # $20-$23 + "t8", + "t9", # $24-$25 + "k0", + "k1", # $26-$27 + "gp", + "sp", + "fp", + "ra", # $28-$31 ] def __init__(self) -> None: From 5dd75eeb6639382d6c672230b5f179ca7b814991 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 00:33:10 +0000 Subject: [PATCH 07/18] Update CI, add release workflow, and refresh documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI: - Test on Linux, Windows, and macOS (Apple Silicon, where Keystone is not installed) on Python 3.9 and 3.13, from the built wheel, with a smoke test run outside the repository - Run on all pull requests and on demand (workflow_dispatch) Release: - New release workflow publishes to PyPI with trusted publishing when a GitHub release is published; docs/RELEASING.md has the one-time setup and steps Packaging: - Require Python 3.9+ (3.8 is end-of-life); fix project URLs; add classifiers; absolute README links so the PyPI page renders Console: - The first regs after loading marks registers changed since the program started; ★ falls back to * where the terminal can't show it, and output never crashes on unencodable characters - `.isa riscv64 # comment` is recognized Docs: - README, quick start, console guide, syscall reference, examples guide, and Python API docs rewritten for the new install and workflow; every transcript was checked against real output - Add CHANGELOG.md Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- .github/workflows/ci.yml | 48 ++++- .github/workflows/release.yml | 55 +++++ CHANGELOG.md | 40 ++++ README.md | 270 ++++++++++++++---------- docs/README.md | 22 +- docs/RELEASING.md | 41 ++++ docs/user/console-guide.md | 165 +++++++++------ docs/user/quick-start.md | 338 +++++++++++++----------------- docs/user/syscalls.md | 59 ++++-- examples/README.md | 133 ++++++------ mapachespim/README.md | 76 +++++-- mapachespim/console.py | 27 ++- mapachespim/toolchain/__init__.py | 5 +- pyproject.toml | 24 ++- tests/README.md | 35 ++-- tests/test_loading.py | 14 ++ 16 files changed, 831 insertions(+), 521 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 CHANGELOG.md create mode 100644 docs/RELEASING.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cc0918..a9fe573 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: [main, master] pull_request: - branches: [main, master] + workflow_dispatch: jobs: lint: @@ -14,7 +14,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install linting tools run: pip install ruff mypy @@ -29,10 +29,19 @@ jobs: run: mypy mapachespim/ --ignore-missing-imports test: - runs-on: ubuntu-latest + name: test (${{ matrix.os }}, py${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - python-version: ["3.8", "3.10", "3.12"] + # ubuntu/windows are x86 (Keystone installed automatically); + # macos-14 is Apple Silicon, where Keystone has no wheel, so it + # covers the "no Keystone" install that many students will have. + os: [ubuntu-latest, windows-latest, macos-14] + python-version: ["3.9", "3.13"] + include: + - os: ubuntu-latest + python-version: "3.11" steps: - uses: actions/checkout@v4 @@ -40,8 +49,33 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install package with dev dependencies - run: pip install -e ".[dev]" + - name: Build wheel + run: | + python -m pip install --upgrade pip build + python -m build --wheel --outdir dist + + # Install the built wheel (not an editable checkout) so packaging + # mistakes, like files missing from the wheel, fail CI. + - name: Install wheel + shell: bash + run: pip install dist/*.whl pytest + + - name: Smoke test the installed package outside the repository + shell: bash + working-directory: ${{ runner.temp }} + run: | + set -ex + python -c "import mapachespim, pathlib; p = pathlib.Path(mapachespim.__file__); assert 'site-packages' in str(p), p" + mapachespim --version + mapachespim-as --version + mapachespim -e riscv/hello_asm + mapachespim -e mips/hello_asm + mapachespim -e arm/hello_asm + mapachespim -e x86_64/hello_asm + mapachespim --copy-examples ex + mapachespim -e ex/riscv/fibonacci/fibonacci.s + mapachespim -e ex/mips/fibonacci/fibonacci.s + printf 'load riscv/fibonacci\nbreak fibonacci\nrun\nregs\nlist\nquit\n' | mapachespim - name: Run tests - run: pytest tests/ -v --tb=short + run: python -m pytest tests/ --tb=short diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..0e3ab58 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,55 @@ +name: Release + +# Publishes to PyPI when a GitHub release is published. +# One-time setup is described in docs/RELEASING.md. + +on: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Check the tag matches the package version + env: + TAG: ${{ github.event.release.tag_name }} + run: | + version=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") + if [ "${TAG#v}" != "$version" ]; then + echo "Release tag $TAG does not match pyproject.toml version $version" >&2 + exit 1 + fi + + - name: Build sdist and wheel + run: | + python -m pip install --upgrade build twine + python -m build + python -m twine check --strict dist/* + + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/mapachespim/ + permissions: + id-token: write # PyPI trusted publishing; no API token needed + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6a1dd40 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog + +## Unreleased + +### Installation +- RISC-V and MIPS programs are assembled by new built-in pure-Python encoders, so they assemble on + every platform. (RISC-V could not be assembled at all with the Keystone release on PyPI.) +- Keystone is now only needed for ARM64 and x86-64, and is installed automatically only on x86 + machines where prebuilt wheels exist. Elsewhere it is the optional `keystone` extra. +- The example programs are included in the package, so `load riscv/fibonacci` works after + `pip install`. `mapachespim --copy-examples DIR` copies them somewhere editable. +- Python 3.9 or newer is required. + +### Console +- `load` assembles `.s` files directly (with debug info) and loads bundled examples by name. +- New `reload` command re-assembles after editing, keeping breakpoints on their labels. +- New `examples` command lists the bundled programs. +- `step` now performs syscalls (it used to skip them) and stops at program exit. +- `reset` reloads the program, so it can be run again. +- Errors explain what went wrong and where, e.g. + `Read from unmapped address 0x0 (at PC=0x80000004)` with the source line. +- `disasm` walks x86-64 instructions by their real length and marks the PC. +- `mem`, `disasm`, and `delete` accept labels; `regs decimal` is signed; the first `regs` after + loading marks registers changed since the start; `set show-changes off` works. +- `mapachespim --version`; `-e` returns the program's exit code (124 on hitting `--max-steps`) and + accepts `.s` files and example names. + +### Simulator +- `read_char` returned the second line of input; `print_int` printed negative MIPS values as + large unsigned numbers; unknown syscall numbers are now reported as errors; new SPIM `exit2` (17). +- Address 0 is no longer mapped on ARM64, so null-pointer accesses fault. +- The RISC-V sign-extended address alias now shares memory with the real address. +- Loading a program starts from a fresh machine state and may switch ISA. + +### Assembler +- RISC-V `li` handles all 64-bit constants; `.equ` constants can be used as instruction operands; + `add a0, a1, 5` is accepted as `addi`. +- Debug info for big-endian targets (MIPS) was written in the wrong byte order, so source-level + debugging never worked for MIPS. +- `.isa riscv64 # comment` is recognized. diff --git a/README.md b/README.md index 465acff..3748489 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# MapacheSPIM MapacheSPIM Logo +# MapacheSPIM MapacheSPIM Logo MapacheSPIM is a SPIM-like simulator for assembly programming built on the Unicorn Engine CPU emulator. It provides an interactive, console-based environment for learning assembly language, debugging programs @@ -25,52 +25,59 @@ and why. ### Install -The easiest way to get started with MapacheSPIM is to clone from GitHub: +MapacheSPIM is a pure Python package that runs on Windows, macOS (Intel and Apple Silicon), and +Linux with Python 3.9 or newer. No cross-compiler or other toolchain is needed: it includes its own +assembler. -```git clone https://github.com/mapachespim/MapacheSPIM.git -cd MapacheSPIM -pip install -e . +The easiest way to install it is with [pipx](https://pipx.pypa.io/), which puts the +`mapachespim` command on your path in its own isolated environment: + +```bash +pipx install git+https://github.com/UCSBarchlab/MapacheSPIM.git ``` -**Prerequisites:** -- Python 3.8+ -- RISC-V or ARM toolchain (optional, for compiling your own assembly) +or, with [uv](https://docs.astral.sh/uv/): `uv tool install git+https://github.com/UCSBarchlab/MapacheSPIM.git`. +Plain `pip install git+https://github.com/UCSBarchlab/MapacheSPIM.git` works too (ideally inside a +virtual environment). + +Check that it worked: + +```bash +mapachespim --version +``` -**Dependencies** (automatically installed): -- `unicorn>=2.0.0` - CPU emulator framework -- `capstone>=5.0.0` - Disassembler -- `pyelftools>=0.29` - ELF file parsing +**Dependencies** (installed automatically): +- `unicorn` - CPU emulator framework +- `capstone` - Disassembler +- `pyelftools` - ELF file parsing +- `keystone-engine` - Assembler for ARM64 and x86-64, installed automatically on x86 machines. RISC-V + and MIPS use MapacheSPIM's built-in assembler, so they work everywhere; see + [ISA support](#multi-isa-support) for details. ### Running MapacheSPIM -Run the console by typing: +Start the console by typing: -```mapachespim +```bash +mapachespim ``` -This will start the interactive console where you can load programs, set breakpoints, step through code, and -inspect machine state. Type `help` at the console to see available commands. +Type `examples` to see the example programs that come with MapacheSPIM, `quickstart` for a short +tutorial, or `help` for all commands. ### A Quick Example -Here is an example loading a RISC-V fibonacci program, stepping through the first few instructions, and examining -the machine state: +Here is an example loading the bundled RISC-V Fibonacci program, stopping at a function, and +looking at the source and machine state: ``` $ mapachespim -Welcome to MapacheSPIM. Type help or ? to list commands. +Welcome to MapacheSPIM. Type help or ? to list commands, or quickstart for a tutorial. -(mapachespim) load examples/riscv/fibonacci/fibonacci -Loaded examples/riscv/fibonacci/fibonacci +(mapachespim) load riscv/fibonacci +Loaded riscv/fibonacci (RISCV) Entry point: 0x0000000080000000 - -(mapachespim) info symbols -Symbols (11 total): - 0x80000000 _start - 0x80000034 exit_loop - 0x80000038 fibonacci - 0x80000080 base_case_zero - ... +Source info: fibonacci.s (32 address mappings) (mapachespim) break fibonacci Breakpoint set at fibonacci (0x80000038) @@ -79,59 +86,95 @@ Breakpoint set at fibonacci (0x80000038) Breakpoint hit at 0x0000000080000038 after 6 instructions PC = 0x0000000080000038 -(mapachespim) step -[0x80000038] 0x63040504 beq x10, x0, 0x48 +(mapachespim) list +fibonacci.s: + 84: fibonacci: + 85: # Base case 1: if n == 0, return 0 + 86> beqz a0, base_case_zero # <-- PC: 0x80000038 + 87: + 88: # Base case 2: if n == 1, return 1 + 89: li t0, 1 (mapachespim) step -[0x8000003c] 0x93021000 addi x5, x0, 0x1 +[0x80000038] 0x63040504 beqz a0, 0x48 -(mapachespim) continue -Breakpoint hit at 0x0000000080000038 after 7 instructions -PC = 0x0000000080000038 +(mapachespim) step +[0x8000003c] 0x93021000 addi t0, zero, 1 (mapachespim) regs -(mapachespim) regs +x0 (zero) = 0x0000000000000000 x1 ( ra) = 0x0000000080000018 ★ +x2 ( sp) = 0x0000000080181000 ★ x3 ( gp) = 0x0000000000000000 +x4 ( tp) = 0x0000000000000000 x5 ( t0) = 0x0000000000000001 ★ +... +x10 ( a0) = 0x0000000000000007 ★ x11 ( a1) = 0x0000000000000000 +... + +pc = 0x0000000080000040 + +(mapachespim) mem fib_input 8 -x0 (zero) = 0x0000000000000000 x1 ( ra) = 0x000000008000005c ★ -x2 ( sp) = 0x0000000083efffe8 ★ x3 ( gp) = 0x0000000000000000 -x4 ( tp) = 0x0000000000000000 x5 ( t0) = 0x0000000000000001 ★ -x6 ( t1) = 0x0000000000000000 x7 ( t2) = 0x0000000000000000 -x8 ( s0) = 0x0000000000000000 x9 ( s1) = 0x0000000000000000 -x10 ( a0) = 0x0000000000000006 ★ x11 ( a1) = 0x0000000000001000 -x12 ( a2) = 0x0000000000000000 x13 ( a3) = 0x0000000000000000 -x14 ( a4) = 0x0000000000000000 x15 ( a5) = 0x0000000000000000 -x16 ( a6) = 0x0000000000000000 x17 ( a7) = 0x0000000000000000 -x18 ( s2) = 0x0000000000000000 x19 ( s3) = 0x0000000000000000 -x20 ( s4) = 0x0000000000000000 x21 ( s5) = 0x0000000000000000 -x22 ( s6) = 0x0000000000000000 x23 ( s7) = 0x0000000000000000 -x24 ( s8) = 0x0000000000000000 x25 ( s9) = 0x0000000000000000 -x26 ( s10) = 0x0000000000000000 x27 ( s11) = 0x0000000000000000 -x28 ( t3) = 0x0000000000000000 x29 ( t4) = 0x0000000000000000 -x30 ( t5) = 0x0000000000000000 x31 ( t6) = 0x0000000000000000 - -pc = 0x0000000080000038 - -(mapachespim) mem .text - -0x80000000: 17 01 f0 03 13 01 01 00 97 02 00 00 93 82 82 08 |................| -0x80000010: 03 a5 02 00 ef 00 40 02 97 02 00 00 93 82 c2 07 |......@.........| -0x80000020: 23 a0 a2 00 93 02 10 00 17 13 00 00 13 03 83 fd |#...............| -0x80000030: 23 30 53 00 6f 00 00 00 63 04 05 04 93 02 10 00 |#0S.o...c.......| -0x80000040: 63 04 55 04 13 01 81 fe 23 38 11 00 23 34 81 00 |c.U.....#8..#4..| -0x80000050: 23 30 a1 00 13 05 f5 ff ef f0 1f fe 13 04 05 00 |#0..............| -0x80000060: 03 35 01 00 13 05 e5 ff ef f0 1f fd 33 05 a4 00 |.5..........3...| -0x80000070: 83 30 01 01 03 34 81 00 13 01 81 01 67 80 00 00 |.0...4......g...| -0x80000080: 13 05 00 00 67 80 00 00 13 05 10 00 67 80 00 00 |....g.......g...| +0x80100000: 07 00 00 00 00 00 00 00 |........| (mapachespim) quit Goodbye! ``` -At any point when execution is stopped, you can inspect registers and memory. The full 64-bit value of each +At any point when execution is stopped, you can inspect registers and memory. The full 64-bit value of each register is shown in hex along with its ABI name (like `a0`, `sp`, `ra`). A star (★) appears next to registers -that have changed to help you follow the execution of the program. Memory is shown in bytes, grouped into -4-byte words for easier reading. +that have changed since you last looked, to help you follow the execution of the program. Memory is shown in +bytes, grouped into 4-byte words for easier reading. + +### Writing Your Own Programs + +Write your program in a `.s` file and load it directly; MapacheSPIM assembles it for you (with debug +info, so `list` shows your source): + +```asm +# hello.s +.isa riscv64 # which ISA this file is for + +.data +msg: .asciz "Hello, world!\n" + +.text +.globl _start +_start: + la a0, msg # print_string(msg) + li a7, 4 + ecall + li a7, 10 # exit + ecall +``` + +``` +(mapachespim) load hello.s +Assembled hello.s (1096 bytes) +Loaded hello.s (RISCV) +... +(mapachespim) run +Hello, world! +Program exited with code 0 after 6 instructions +``` + +After editing the file, type `reload` to re-assemble and load the new version; breakpoints on labels +follow their labels. If your file has no `.isa` line, give the ISA when loading: `load hello.s riscv64`. + +To start from the examples, copy them somewhere you can edit them: + +```bash +mapachespim --copy-examples my-examples +``` + +You can also run a program without the interactive console, which is handy for testing and +autograding. The exit status is the program's exit code (1 on a runtime error, 124 if it doesn't finish +within `--max-steps` instructions): + +```bash +mapachespim -e hello.s +``` + +The standalone assembler writes an ELF file if you want one: `mapachespim-as -g hello.s -o hello`. ## Features @@ -166,9 +209,9 @@ li a7, 10 # Syscall 10 = exit ecall ``` -Supported syscalls: `print_int` (1), `print_string` (4), `read_int` (5), `exit` (10), `print_char` (11), `read_char` (12), `exit_code` (93) +Supported syscalls: `print_int` (1), `print_string` (4), `read_int` (5), `exit` (10), `print_char` (11), `read_char` (12), `exit2` (17), `exit_code` (93) -See [Syscall Reference](docs/user/syscalls.md) for complete details. +See [Syscall Reference](https://github.com/UCSBarchlab/MapacheSPIM/blob/main/docs/user/syscalls.md) for complete details. ### Console Commands @@ -176,21 +219,24 @@ Familiar SPIM-like interface: | Command | Alias | Description | |---------|-------|-------------| -| `load ` | | Load an ELF executable | +| `load ` | | Load a `.s` file (assembled for you), an ELF executable, or an example | +| `reload` | | Re-assemble and reload the current program after editing | +| `examples` | | List the bundled example programs | | `step [n]` | `s` | Execute n instructions (default 1) | -| `run [max]` | `r` | Run until halt or max instructions | -| `break ` | `b` | Set breakpoint at address or symbol | +| `run [max]` | `r` | Run until exit, breakpoint, or max instructions | +| `break ` | `b` | Set breakpoint at address or label | | `continue` | `c` | Continue after breakpoint | +| `reset` | | Restart the program from the beginning | | `regs` | | Show all registers | | `pc` | | Show program counter | -| `mem [len]` | | Show memory contents | +| `mem [len]` | | Show memory contents (address, label, or section) | | `disasm [n]` | `d` | Disassemble n instructions | +| `list` | `l` | Show source code around the PC | | `info symbols` | | List all symbols | | `info sections` | | List ELF sections | -| `list` | `l` | Show source code (if debug info) | | `quit` | `q` | Exit simulator | -See [Console Guide](docs/user/console-guide.md) for complete command reference. +See [Console Guide](https://github.com/UCSBarchlab/MapacheSPIM/blob/main/docs/user/console-guide.md) for complete command reference. ## What Makes This Special? @@ -205,21 +251,23 @@ MapacheSPIM is powered by the [Unicorn Engine](https://www.unicorn-engine.org/), ### Multi-ISA Support -MapacheSPIM supports multiple instruction set architectures: +MapacheSPIM supports four instruction set architectures, each with the same console commands, +syscalls, and a matching set of example programs: -``` -MapacheSPIM/ -├── mapachespim/ -│ ├── unicorn_backend.py # Unicorn-based simulator -│ ├── elf_loader.py # ELF parsing with pyelftools -│ └── console.py # Interactive console -├── examples/riscv/ # RISC-V 64-bit examples -├── examples/arm/ # ARM64 (AArch64) examples -├── examples/x86_64/ # x86-64 examples -└── examples/mips/ # MIPS32 examples -``` +| ISA | Examples | Assembling `.s` files | +|-----|----------|-----------------------| +| RISC-V 64-bit (RV64IM) | `examples/riscv/` | Built in, works everywhere | +| MIPS32 (big-endian) | `examples/mips/` | Built in, works everywhere | +| ARM64 (AArch64) | `examples/arm/` | Uses [Keystone](https://www.keystone-engine.org/) | +| x86-64 (AT&T or Intel syntax) | `examples/x86_64/` | Uses [Keystone](https://www.keystone-engine.org/) | + +Keystone is installed automatically on x86 machines (Windows, Intel Macs, x86 Linux). Its PyPI release +has no prebuilt package for ARM machines such as Apple Silicon Macs, so it is not installed there by +default; you can still run and debug ARM64 and x86-64 programs (including all the bundled examples), and +can try `pip install 'mapachespim[keystone]'` to build Keystone from source (this needs CMake and a C++ +compiler). -The same console commands and Python API work across all supported ISAs. Just load an ELF file and MapacheSPIM detects the architecture automatically. +Loading an ELF file detects its ISA automatically. ### Student-Friendly Design @@ -233,32 +281,26 @@ Inspired by SPIM, designed for education: ## Documentation -- [Quick Start Guide](docs/user/quick-start.md) - Get running in 5 minutes -- [Console Guide](docs/user/console-guide.md) - Complete command reference -- [Syscall Reference](docs/user/syscalls.md) - I/O syscalls for programs -- [Examples Guide](examples/README.md) - Learn from example programs +- [Quick Start Guide](https://github.com/UCSBarchlab/MapacheSPIM/blob/main/docs/user/quick-start.md) - Get running in 5 minutes +- [Console Guide](https://github.com/UCSBarchlab/MapacheSPIM/blob/main/docs/user/console-guide.md) - Complete command reference +- [Syscall Reference](https://github.com/UCSBarchlab/MapacheSPIM/blob/main/docs/user/syscalls.md) - I/O syscalls for programs +- [Examples Guide](https://github.com/UCSBarchlab/MapacheSPIM/blob/main/examples/README.md) - Learn from example programs -## Testing +## Development ```bash -# Run all tests (265 tests) -python -m pytest tests/ -v - -# Or run individual test suites -python -m pytest tests/test_simulator.py # Core API tests -python -m pytest tests/test_console_commands.py # Console tests -python -m pytest tests/test_disasm_comprehensive.py # Disassembly tests -python -m pytest tests/test_symbols.py # Symbol tests -python -m pytest tests/test_multi_isa.py # Multi-ISA tests +git clone https://github.com/UCSBarchlab/MapacheSPIM.git +cd MapacheSPIM +pip install -e ".[dev]" + +python -m pytest tests/ # run the test suite +ruff check mapachespim/ # lint +ruff format mapachespim/ # format +mypy mapachespim/ # type check +make -C examples DEBUG=1 # rebuild the example binaries ``` -All tests verify: -- Correct program execution (fibonacci, matrix multiplication) -- Console command functionality -- Symbol table handling -- Disassembly accuracy -- Syscall behavior -- Multi-ISA support (RISC-V and ARM) +See [docs/RELEASING.md](https://github.com/UCSBarchlab/MapacheSPIM/blob/main/docs/RELEASING.md) for how to publish a release. ## License @@ -269,7 +311,7 @@ All tests verify: ## Contact -- Issues: [GitHub Issues](https://github.com/mapachespim/MapacheSPIM/issues) -- Discussions: [GitHub Discussions](https://github.com/mapachespim/MapacheSPIM/discussions) +- Issues: [GitHub Issues](https://github.com/UCSBarchlab/MapacheSPIM/issues) +- Discussions: [GitHub Discussions](https://github.com/UCSBarchlab/MapacheSPIM/discussions) -MapacheSPIM Logo +MapacheSPIM Logo diff --git a/docs/README.md b/docs/README.md index 49fe444..d399663 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,19 +5,13 @@ This directory contains all project documentation organized by audience and purp ## Quick Links ### For Students and Users -- [Quick Start Guide](user/quick-start.md) - Get started in 5 minutes +- [Quick Start Guide](user/quick-start.md) - Install and write your first program in 5 minutes - [Console Guide](user/console-guide.md) - Complete console command reference -- [Examples Guide](../examples/README.md) - Learn from example programs +- [Syscall Reference](user/syscalls.md) - I/O syscalls for each ISA +- [Examples Guide](../examples/README.md) - The bundled example programs and templates -## Documentation Structure - -``` -docs/ -├── README.md (this file) # Documentation index -│ -└── user/ # User documentation - ├── quick-start.md # Getting started guide - └── console-guide.md # Console usage reference - - -``` +### For Developers and Maintainers +- [Python API](../mapachespim/README.md) - Using the simulator and assembler from Python +- [Tests](../tests/README.md) - The test suite +- [Releasing](RELEASING.md) - Publishing a new version to PyPI +- [Changelog](../CHANGELOG.md) diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 0000000..487c43f --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,41 @@ +# Releasing MapacheSPIM + +Releases are published to [PyPI](https://pypi.org/project/mapachespim/) by the +[Release workflow](../.github/workflows/release.yml) when a GitHub release is published. It uses +PyPI *trusted publishing*, so no API token is stored in the repository. + +## One-time setup + +1. Create an account on [pypi.org](https://pypi.org/) (enable two-factor authentication). +2. Add a *pending publisher* for the project, since it does not exist on PyPI yet: go to + **Your account > Publishing > Add a new pending publisher** and enter + - PyPI project name: `mapachespim` + - Owner: `UCSBarchlab` + - Repository name: `MapacheSPIM` + - Workflow name: `release.yml` + - Environment name: `pypi` +3. In the GitHub repository, create an environment named `pypi` + (**Settings > Environments > New environment**). Optionally require a reviewer so every + release needs an approval before it is uploaded. + +After the first release, the pending publisher becomes a normal trusted publisher for the project. + +## Making a release + +1. Make sure CI is green on `main`. +2. Update the version in `pyproject.toml` and `mapachespim/__init__.py` (they must match), and move + the "Unreleased" notes in `CHANGELOG.md` under the new version. Commit and push. +3. If any toolchain change affects code generation, rebuild the example binaries with + `make -C examples DEBUG=1` and commit them. +4. On GitHub, create a release with a tag named `v` (for example `v0.3.0`) and publish it. + The workflow checks the tag matches `pyproject.toml`, builds the sdist and wheel, and uploads + them. + +Once published, students can install with: + +```bash +pipx install mapachespim +``` + +At that point, update the install instructions in `README.md` and `docs/user/quick-start.md` to use +the PyPI name instead of the `git+https://` URL. diff --git a/docs/user/console-guide.md b/docs/user/console-guide.md index 18a19d3..462767f 100644 --- a/docs/user/console-guide.md +++ b/docs/user/console-guide.md @@ -1,6 +1,6 @@ # MapacheSPIM Interactive Console Guide -SPIM-like interactive console for RISC-V programs using the Unicorn Engine. +SPIM-like interactive console for RISC-V, MIPS, ARM64, and x86-64 programs using the Unicorn Engine. ## Quick Start @@ -8,18 +8,40 @@ SPIM-like interactive console for RISC-V programs using the Unicorn Engine. # Launch interactive console mapachespim -# Or load a program on startup -mapachespim examples/riscv/fibonacci/fibonacci +# Or load a program on startup: your own source file, an ELF, or an example +mapachespim myprog.s +mapachespim riscv/fibonacci ``` ## Basic Commands ### File Loading -- `load ` - Load a RISC-V ELF executable +- `load [isa]` - Load a program + - Assembly source (`.s`, `.S`, `.asm`) is assembled for you, with debug info so `list` works. + The ISA comes from an `.isa` directive in the file, or the optional `isa` argument + (`riscv64`, `mips32`, `arm64`, `x86_64`). + - ELF executables are loaded directly and their ISA is detected automatically. + - Bundled examples can be loaded by name. ``` + (mapachespim) load myprog.s + (mapachespim) load myprog.s riscv64 + (mapachespim) load riscv/fibonacci (mapachespim) load examples/riscv/fibonacci/fibonacci ``` +- `reload` - Re-assemble and reload the current program (after you edit it). Breakpoints on + labels move with their labels. + +- `examples [isa]` - List the bundled example programs + ``` + (mapachespim) examples mips + + MIPS32: + mips/array_stats Array Processing with Loops + mips/fibonacci Recursive Fibonacci Calculator + ... + ``` + ### Execution - `step [n]` - Execute 1 or n instructions (alias: `s`) ``` @@ -27,23 +49,31 @@ mapachespim examples/riscv/fibonacci/fibonacci (mapachespim) step 10 # Execute 10 instructions ``` -- `run [max]` - Run until halt or max instructions (alias: `r`) +- `run [max]` - Run until the program exits, hits a breakpoint, or reaches max instructions (alias: `r`) ``` - (mapachespim) run # Run until program halts + (mapachespim) run # Run until program exits (mapachespim) run 1000 # Run max 1000 instructions ``` + If the program crashes, you see why and where: + ``` + Error: Read from unmapped address 0x0 (at PC=0x80000004) + [0x80000004] 0x03350500 ld a0, 0(a0) <_start+4> + prog.s:5: ld a0, 0(a0) + ``` + - `continue` - Continue after breakpoint (alias: `c`) ### State Inspection -- `regs` - Display all 32 registers + PC with ABI names +- `regs [hex|decimal|binary]` - Display all registers + PC with ABI names. A ★ marks registers + that changed since you last displayed them (`regs peek` shows them without resetting that). ``` (mapachespim) regs x0 (zero) = 0x0000000000000000 x1 ( ra) = 0x0000000080000018 ... - x2 ( sp) = 0x0000000083efffe8 x3 ( gp) = 0x0000000000000000 ... + x2 ( sp) = 0x0000000080181000 x3 ( gp) = 0x0000000000000000 ... ... - pc = 0x0000000080000004 + pc = 0x0000000080000004 ``` - `pc` - Show just the program counter @@ -52,9 +82,10 @@ mapachespim examples/riscv/fibonacci/fibonacci pc = 0x0000000080000004 ``` -- `mem [len]` - Display memory contents with ASCII sidebar (default 256 bytes) +- `mem [len]` - Display memory contents with ASCII sidebar (default 256 bytes) ``` (mapachespim) mem 0x80000000 # By address + (mapachespim) mem fib_input 8 # By label (mapachespim) mem 0x80000000 64 # Show 64 bytes (mapachespim) mem .text # By section name (mapachespim) mem .data 128 # Section with length @@ -82,14 +113,16 @@ mapachespim examples/riscv/fibonacci/fibonacci 49: mv t0, a0 50: addi a0, s0, -2 - Tip: Compile with 'as -g' to include debug symbols + Tip: Programs loaded from .s files always have source info ``` ### Breakpoints -- `break ` - Set breakpoint at address (alias: `b`) +- `break ` - Set breakpoint at an address or label (alias: `b`) ``` + (mapachespim) break fibonacci + Breakpoint set at fibonacci (0x80000038) (mapachespim) break 0x80000010 - Breakpoint set at 0x0000000080000010 + Breakpoint set at 0x80000010 ``` - `info breakpoints` - List all breakpoints @@ -105,9 +138,9 @@ mapachespim examples/riscv/fibonacci/fibonacci ``` (mapachespim) info symbols - Symbols (22 total): + Symbols (11 total): 0x80000000 _start - 0x80000030 main + 0x80000034 exit_loop 0x80000038 fibonacci ... ``` @@ -127,7 +160,7 @@ mapachespim examples/riscv/fibonacci/fibonacci Use 'mem
' to view section contents (e.g., mem .data) ``` -- `delete ` - Remove breakpoint at address +- `delete ` - Remove breakpoint at address or label ``` (mapachespim) delete 0x80000010 Breakpoint removed at 0x0000000080000010 @@ -137,7 +170,7 @@ mapachespim examples/riscv/fibonacci/fibonacci ### Utility - `status` - Show simulator status -- `reset` - Reset simulator (keeps program loaded) +- `reset` - Restart the program from the beginning (reloads memory and registers; keeps breakpoints) - `help` - Show all commands - `quit` / `exit` - Exit console (alias: `q`) @@ -145,35 +178,38 @@ mapachespim examples/riscv/fibonacci/fibonacci ``` $ mapachespim -Welcome to MapacheSPIM. Type help or ? to list commands. +Welcome to MapacheSPIM. Type help or ? to list commands, or quickstart for a tutorial. -(mapachespim) load examples/riscv/fibonacci/fibonacci -✓ Loaded examples/riscv/fibonacci/fibonacci +(mapachespim) load riscv/fibonacci +Loaded riscv/fibonacci (RISCV) Entry point: 0x0000000080000000 +Source info: fibonacci.s (32 address mappings) + +(mapachespim) break fibonacci +Breakpoint set at fibonacci (0x80000038) + +(mapachespim) run +Breakpoint hit at 0x0000000080000038 after 6 instructions +PC = 0x0000000080000038 (mapachespim) step -[0x0000000080000000] Executed 1 instruction +[0x80000038] 0x63040504 beqz a0, 0x48 (mapachespim) regs -x0 (zero) = 0x0000000000000000 x1 ( ra) = 0x0000000000000000 ... +x0 (zero) = 0x0000000000000000 x1 ( ra) = 0x0000000080000018 ★ ... -pc = 0x0000000080000004 +pc = 0x000000008000003c -(mapachespim) break 0x80000038 -Breakpoint set at 0x0000000080000038 - -(mapachespim) run -Breakpoint hit at 0x0000000080000038 after 5 instructions -PC = 0x0000000080000038 +(mapachespim) delete fibonacci +Breakpoint removed at 0x0000000080000038 (mapachespim) continue -Executed 95 instruction(s) -PC = 0x00000000800000f0 +Program completed (tohost) after 455 instructions +PC = 0x0000000080000034 -(mapachespim) mem 0x80000000 32 +(mapachespim) mem fib_result 8 -0x80000000: 17 01 f0 03 13 01 01 00 97 02 00 00 93 82 82 08 |................| -0x80000010: 03 a5 02 00 ef 00 40 02 97 02 00 00 93 82 c2 07 |......@.........| +0x80100004: 0d 00 00 00 00 00 00 00 |........| (mapachespim) quit Goodbye! @@ -201,21 +237,29 @@ The console shows both numeric (x0-x31) and ABI names: ## Command Line Options ```bash -# Launch console -mapachespim +mapachespim # Launch the console +mapachespim myprog.s # Load a program on startup +mapachespim myprog.s --isa riscv64 # ...for a .s file without an .isa directive +mapachespim -e myprog.s # Run without the console and exit +mapachespim -e myprog.s --max-steps 1000000 +mapachespim --copy-examples my-dir # Copy the bundled examples somewhere editable +mapachespim --version +``` -# Load file on startup -mapachespim examples/riscv/fibonacci/fibonacci +With `-e`, only the program's output goes to stdout, and the exit status is the program's exit code, +1 if it crashed, or 124 if it did not finish within `--max-steps` instructions (default 10,000,000). +This makes `-e` convenient for testing and autograding: -# Quiet mode (less verbose) -mapachespim -q examples/riscv/fibonacci/fibonacci +```bash +echo 7 | mapachespim -e fib.s > output.txt ``` ## Keyboard Shortcuts - **Ctrl-C** during `run` - Interrupt execution - **Ctrl-D** or `quit` - Exit console -- **Tab** - Command completion +- **Tab** - Complete commands, file paths, example names, and labels (after `break`) +- **Enter** on an empty line - Repeat the last command (handy for `step`) - **Up/Down arrows** - Command history ## Tips @@ -244,10 +288,10 @@ mapachespim -q examples/riscv/fibonacci/fibonacci info sections # List all available sections ``` -4. **Source code viewing**: Requires debug symbols (compile with `-g`) +4. **Source code viewing**: Requires debug info (automatic when you load a `.s` file) ``` list # Show source around current PC - list main # Show source around function + list fibonacci # Show source around function list 25 # Show source around line 25 ``` @@ -256,38 +300,39 @@ mapachespim -q examples/riscv/fibonacci/fibonacci step 10 # Execute 10 instructions at once ``` -## Compiling with Debug Symbols +## Assembling Programs -To enable the `list` command for source code viewing, compile your assembly with debug symbols: +The simplest way is to `load` your `.s` file; MapacheSPIM assembles it with debug info. To produce +an ELF file instead, use the bundled assembler: ```bash -# Assemble with debug symbols -riscv64-unknown-elf-as -march=rv64g -mabi=lp64 -g -o program.o program.s - -# Link as normal -riscv64-unknown-elf-ld -T linker.ld -o program program.o +mapachespim-as -g program.s -o program # .isa directive in the file +mapachespim-as -g program.s -o program --isa mips32 ``` -The `-g` flag adds DWARF debug information that maps machine addresses to source lines. +The `-g` flag adds DWARF debug information that maps machine addresses to source lines, which the +`list` command uses. ELF files from other toolchains (such as GNU `as`/`ld` with `-g`) also load, +as long as they are statically linked executables for a supported ISA. ## Differences from SPIM MapacheSPIM is similar to SPIM but has key differences: -1. **RISC-V instead of MIPS**: Uses RISC-V ISA via Unicorn Engine -2. **ELF files**: Loads compiled ELF binaries (not assembly source) -3. **64-bit**: Full RV64I support by default -4. **Source display**: Optional via `list` command (requires `-g` flag) +1. **Several ISAs**: RISC-V (RV64IM), MIPS32, ARM64, and x86-64, all with the same commands and syscalls +2. **Real executables**: Programs are assembled to ELF files, so the same binaries work with other tools +3. **Delay slots**: On MIPS, the assembler fills each branch and jump delay slot with a `nop` (like + GNU `as` in its default mode), and the simulator executes delay slots as real hardware does +4. **Source display**: `list` shows your source code next to the program counter ## Troubleshooting **Console won't start:** -- Make sure you've installed the package: `pip install -e .` -- Check Python path includes mapachespim package +- See the install instructions in the [Quick Start Guide](quick-start.md) +- You can always run `python3 -m mapachespim.console` -**Can't load ELF file:** -- Verify file is RISC-V ELF: `file examples/riscv/fibonacci/fibonacci` -- Check file path is correct +**Can't load a file:** +- Check the file path is correct; type `examples` to see the bundled programs +- ELF files must be statically linked executables for RISC-V, MIPS, ARM64, or x86-64 **Breakpoint not hit:** - Verify address is correct: `mem ` diff --git a/docs/user/quick-start.md b/docs/user/quick-start.md index 3fb4db2..c77c655 100644 --- a/docs/user/quick-start.md +++ b/docs/user/quick-start.md @@ -1,270 +1,222 @@ # Quick Start Guide -Get started with MapacheSPIM in 5 minutes. This guide will help you install the simulator and debug your first RISC-V program. +Get started with MapacheSPIM in 5 minutes. This guide installs the simulator, runs an example +program, and then has you write and debug your own RISC-V program. -## Prerequisites +## Install -You'll need: -- macOS or Linux (Windows via WSL) -- Python 3.8+ +You need Python 3.9 or newer on Windows, macOS, or Linux. Check with: -Check your Python version: ```bash -python3 --version # Should be 3.8 or higher +python3 --version ``` -## Installation +Install MapacheSPIM with [pipx](https://pipx.pypa.io/) (recommended; it keeps MapacheSPIM in its own +environment and puts the command on your path): ```bash -# Clone the repository -git clone https://github.com/UCSBarchlab/MapacheSPIM.git -cd MapacheSPIM +pipx install git+https://github.com/UCSBarchlab/MapacheSPIM.git +``` -# Create virtual environment (recommended) -python3 -m venv venv -source venv/bin/activate +If you don't have pipx, `pip install git+https://github.com/UCSBarchlab/MapacheSPIM.git` works too, +ideally inside a [virtual environment](https://docs.python.org/3/tutorial/venv.html). -# Install Python package -pip install -e . +Verify the installation: -# Verify installation +```bash mapachespim --version ``` -## Your First Program +You do not need a cross-compiler or any other tools: MapacheSPIM assembles your programs itself. -Let's debug a simple Fibonacci program that's already included. +## Run an Example -### Step 1: Start the Console +Start the console and list the example programs that come with MapacheSPIM: -```bash -mapachespim ``` +$ mapachespim +Welcome to MapacheSPIM. Type help or ? to list commands, or quickstart for a tutorial. -You'll see: -``` -Welcome to MapacheSPIM. Type help or ? to list commands. -(mapachespim) +(mapachespim) examples riscv + +RISC-V 64-bit: + riscv/array_stats Array Processing with Loops + riscv/fibonacci Recursive Fibonacci Calculator + riscv/guess_game Number Guessing Game + riscv/hello_asm Your First RISC-V Assembly Program + riscv/matrix_multiply 3x3 Matrix Multiplication ``` -### Step 2: Load a Program +Load one and look at its source code: ``` -(mapachespim) load examples/riscv/fibonacci/fibonacci -Loaded examples/riscv/fibonacci/fibonacci +(mapachespim) load riscv/hello_asm +Loaded riscv/hello_asm (RISCV) Entry point: 0x0000000080000000 -``` - -### Step 3: See Available Symbols +Source info: hello_asm.s (19 address mappings) +(mapachespim) list +hello_asm.s: + ... + 61> la a0, hello_msg # la = "Load Address" - puts address of hello_msg into a0 # <-- PC: 0x80000000 + 62: li a7, 4 # li = "Load Immediate" - puts the value 4 into a7 + 63: ecall # Make the syscall - this prints the string! ``` -(mapachespim) info symbols -``` - -You'll see function names and their addresses. -### Step 4: Set a Breakpoint +The `>` marks the line the program counter (PC) is on. Step through a few instructions: ``` -(mapachespim) break main -Breakpoint set at main (0x80000030) +(mapachespim) step 3 +[0x80000000] 0x17051000 auipc a0, 0x100 <_start> +[0x80000004] 0x13050500 mv a0, a0 <_start+4> +[0x80000008] 0x93084000 addi a7, zero, 4 <_start+8> ``` -### Step 5: Run to Breakpoint +Each line shows the address, the instruction's bytes, the instruction, and where it is relative to +the nearest label. Notice that the single `la` in the source became two machine instructions +(`auipc` and `addi`, which the disassembler shows as `mv` because its immediate is 0): `la` is a +*pseudo-instruction*. + +Run the rest of the program: ``` (mapachespim) run -Breakpoint hit at main (0x80000030) +Hello, Assembly! +Your lucky number is: 42 +Goodbye! +Program exited with code 0 after 19 instructions ``` -### Step 6: Step Through Code +Use `reset` to start the program over from the beginning. -``` -(mapachespim) step -[0x80000030] 0x13050005 addi x10, x0, 0x5
-Register changes: - x10 ( a0) : 0x0000000000000000 -> 0x0000000000000005 * -``` +## Write Your Own Program -You just saw: -- The address (0x80000030) -- The function name (
) -- The instruction bytes (0x13050005) -- The disassembly (addi x10, x0, 0x5) -- Which register changed +Create a file called `sum.s` in any text editor: -### Step 7: Inspect Registers +```asm +.isa riscv64 # This file is RISC-V (64-bit) -``` -(mapachespim) regs -``` +.data +msg: .asciz "The sum is: " -See all 32 registers with their ABI names and values. +.text +.globl _start +_start: + li t0, 0 # t0 = sum + li t1, 1 # t1 = i + li t2, 10 # t2 = limit +loop: + add t0, t0, t1 # sum += i + addi t1, t1, 1 # i++ + ble t1, t2, loop # repeat while i <= limit -### Step 8: Continue Execution - -``` -(mapachespim) step 5 + la a0, msg # print_string(msg) + li a7, 4 + ecall + mv a0, t0 # print_int(sum) + li a7, 1 + ecall + li a7, 10 # exit + ecall ``` -Step through 5 more instructions to see the program flow. +Load it; MapacheSPIM assembles it for you: -### Step 9: Quit - -``` -(mapachespim) quit -Goodbye! ``` +(mapachespim) load sum.s +Assembled sum.s (1160 bytes) +Loaded sum.s (RISCV) +Entry point: 0x0000000080000000 +Source info: sum.s (14 address mappings) -## Essential Commands - -Here are the commands you'll use most: +(mapachespim) break loop +Breakpoint set at loop (0x8000000c) -| Command | Example | What it does | -|---------|---------|--------------| -| load | load examples/riscv/fibonacci/fibonacci | Load a program | -| break | break main | Set breakpoint at function | -| run | run | Run until breakpoint/halt | -| step | step or step 5 | Execute 1 or N instructions | -| regs | regs | Show all registers | -| mem | mem 0x80000000 64 | Show memory contents | -| disasm | disasm 0x80000000 10 | Disassemble instructions | -| info symbols | info symbols | List all symbols | -| info breakpoints | info breakpoints | List breakpoints | -| delete | delete 0x80000030 | Remove breakpoint | -| clear | clear | Remove all breakpoints | -| pc | pc | Show program counter | -| quit | quit or Ctrl-D | Exit console | - -Tip: Many commands have short aliases: -- step -> s -- break -> b -- run -> r -- continue -> c -- disasm -> d -- quit -> q - -## Debugging Tips - -### 1. Use Symbolic Breakpoints -``` -(mapachespim) break fibonacci (mapachespim) run -``` +Breakpoint hit at 0x000000008000000c after 3 instructions +PC = 0x000000008000000c -Much easier than memorizing addresses. +(mapachespim) continue +Breakpoint hit at 0x000000008000000c after 3 instructions +PC = 0x000000008000000c -### 2. Watch Register Changes -Single-step automatically shows what changed: -``` -(mapachespim) step -[0x80000000] 0x9302a000 addi x5, x0, 0xa <_start> -Register changes: - x5 ( t0) : 0x0000000000000000 -> 0x000000000000000a * +(mapachespim) regs decimal ``` -### 3. Disassemble to Understand Code -``` -(mapachespim) disasm 0x80000000 10 -``` +`regs` shows every register; a ★ marks the ones that changed since you last looked, so you can see +`t0` (the sum) and `t1` (the counter) changing each time around the loop. `regs decimal` shows the +values in decimal. -See the next 10 instructions before executing them. +When you find a bug, edit `sum.s` and type `reload` to re-assemble it. Your breakpoints stay on +their labels. If there's a mistake in the program, the error tells you the line: -### 4. Multi-step for Known-Good Code -``` -(mapachespim) step 10 ``` +(mapachespim) reload -Skip over initialization code quickly. - -### 5. Check Memory Contents -``` -(mapachespim) mem 0x80000000 32 +Error: could not assemble sum.s: + sum.s: Line 13: unknown register 't9' - add t0, t0, t9 ``` -See the raw instruction bytes. - -## Example Debugging Session - -Here's a complete session debugging the Fibonacci program: +To check a program's output without the interactive console, use `-e`: +```bash +$ mapachespim -e sum.s +The sum is: 55 ``` -$ mapachespim -(mapachespim) load examples/riscv/fibonacci/fibonacci -Loaded examples/riscv/fibonacci/fibonacci -(mapachespim) info symbols - 0x80000000 _start - 0x80000030 main - 0x80000038 fibonacci - ... - -(mapachespim) break main -Breakpoint set at main (0x80000030) - -(mapachespim) run -Breakpoint hit at main (0x80000030) - -(mapachespim) step -[0x80000030] 0x13050005 addi x10, x0, 0x5
-Register changes: - x10 ( a0) : 0x0000000000000000 -> 0x0000000000000005 * - -(mapachespim) step -[0x80000034] 0x040000ef jal ra, fibonacci -Register changes: - x1 ( ra) : 0x0000000000000000 -> 0x0000000080000038 * - -(mapachespim) regs -x0 (zero) = 0x0000000000000000 x1 ( ra) = 0x0000000080000038 ... -x10 ( a0) = 0x0000000000000005 ... +## Essential Commands -(mapachespim) continue -Program halted after 42 instructions +| Command | Example | What it does | +|---------|---------|--------------| +| load | `load sum.s` or `load riscv/fibonacci` | Load a program or example | +| reload | `reload` | Re-assemble and reload after editing | +| list | `list` | Show your source code around the PC | +| break | `break loop` | Set a breakpoint at a label (or address) | +| run | `run` | Run until the program exits or hits a breakpoint | +| continue | `continue` | Keep running after a breakpoint | +| step | `step` or `step 5` | Execute 1 or N instructions | +| regs | `regs` or `regs decimal` | Show all registers | +| mem | `mem msg 16` or `mem .data` | Show memory at a label, address, or section | +| disasm | `disasm` or `disasm loop 5` | Disassemble instructions | +| reset | `reset` | Restart the program from the beginning | +| info | `info symbols` / `info breakpoints` | List labels or breakpoints | +| examples | `examples` | List the bundled example programs | +| quit | `quit` or Ctrl-D | Exit the console | + +Many commands have short aliases: `s` (step), `b` (break), `r` (run), `c` (continue), `d` (disasm), +`l` (list), `q` (quit). Pressing Enter on an empty line repeats the last command, which is handy +for stepping. Type `help ` for details on any command. + +## Other ISAs + +Everything above works the same for MIPS, ARM64, and x86-64. Use `.isa mips32`, `.isa arm64`, or +`.isa x86_64` in your file, and try `examples mips`, `examples arm`, and `examples x86_64`. See the +[Syscall Reference](syscalls.md) for the registers each ISA uses for syscalls. + +To get editable copies of all the examples: -(mapachespim) quit -Goodbye! +```bash +mapachespim --copy-examples my-examples ``` -## Next Steps - -Now that you've run your first program: - -1. [Console Guide](console-guide.md) - Learn all commands in detail -2. [Examples](../../examples/README.md) - Explore more example programs -3. Write Your Own - Create and debug your own RISC-V programs - ## Troubleshooting -### Command Not Found: mapachespim +### `mapachespim: command not found` -If the console doesn't start, try: -```bash -# Make sure your virtual environment is activated -source venv/bin/activate +- With pipx, run `pipx ensurepath` and open a new terminal. +- With pip in a virtual environment, make sure it is activated. +- You can always run `python3 -m mapachespim.console` instead. -# Or run from Python package -python3 -m mapachespim.console -``` - -### Import Error +### "Assembling arm64 requires the Keystone Engine" -Make sure you installed the Python package: -```bash -pip3 install -e . -``` +ARM64 and x86-64 programs are assembled with the Keystone library, which is installed automatically +on x86 computers but not on ARM computers such as Apple Silicon Macs (it has no prebuilt package for +them). RISC-V and MIPS always work. You can still load and debug the bundled ARM64 and x86-64 +examples, since they are already assembled. To build Keystone yourself (needs CMake and a C++ +compiler): `pip install 'mapachespim[keystone]'`. ### Still Stuck? -- Check the [full console guide](console-guide.md) -- Look at [example programs](../../examples/README.md) +- Read the [Console Guide](console-guide.md) +- Look at the [example programs](../../examples/README.md) - Open an issue on [GitHub](https://github.com/UCSBarchlab/MapacheSPIM/issues) - -## What's Next? - -After mastering the basics, explore: - -- Advanced Debugging - Watchpoints, conditional breakpoints (future) -- Multiple ISAs - ARM and CHERI support (future) -- Custom Programs - Write and compile your own RISC-V code -- C++ Console - Native performance console (future) diff --git a/docs/user/syscalls.md b/docs/user/syscalls.md index af5608f..f4cb5f8 100644 --- a/docs/user/syscalls.md +++ b/docs/user/syscalls.md @@ -1,33 +1,47 @@ # Syscall Reference -MapacheSPIM supports SPIM-compatible syscalls for simple I/O operations. +MapacheSPIM supports SPIM-compatible syscalls for simple I/O. The syscall numbers are the same on +every ISA; only the registers and the instruction differ. ## Making Syscalls +| ISA | Instruction | Syscall number | Argument | Result | +|-----|-------------|----------------|----------|--------| +| RISC-V | `ecall` | `a7` | `a0` | `a0` | +| MIPS | `syscall` | `$v0` | `$a0` | `$v0` | +| ARM64 | `svc #0` | `x8` | `x0` | `x0` | +| x86-64 | `syscall` | `rax` | `rdi` | `rax` | + +For example, on RISC-V: + ```asm li a7, # Load syscall number into a7 -li a0, # Load arguments into a0-a5 +li a0, # Load the argument into a0 ecall # Execute syscall ``` ## Syscall Table -| Number | Name | Arguments | Return | Description | -|--------|------|-----------|--------|-------------| -| 1 | print_int | a0 = integer | - | Print integer to console | -| 4 | print_string | a0 = address | - | Print null-terminated string | -| 5 | read_int | - | a0 = integer | Read integer from console | -| 10 | exit | - | - | Exit program (code 0) | -| 11 | print_char | a0 = char | - | Print single character | -| 12 | read_char | - | a0 = char | Read single character | -| 93 | exit_code | a0 = code | - | Exit with code | +| Number | Name | Argument | Result | Description | +|--------|------|----------|--------|-------------| +| 1 | print_int | integer | - | Print a signed integer | +| 4 | print_string | address | - | Print a NUL-terminated string | +| 5 | read_int | - | integer | Read a line and parse it as an integer (0 if it isn't one) | +| 10 | exit | - | - | Exit the program with code 0 | +| 11 | print_char | character | - | Print one character | +| 12 | read_char | - | character | Read one character (0 at end of input) | +| 17 | exit2 | exit code | - | Exit the program with the given code (SPIM name) | +| 93 | exit_code | exit code | - | Exit with the given code (Linux RISC-V/ARM64 number) | + +Using any other syscall number stops the program with an error such as +`Unknown syscall number 42 in a7`, which usually means the syscall register wasn't set. ## Examples ### Print String ```asm .data -msg: .string "Hello, World!\n" +msg: .asciz "Hello, World!\n" .text la a0, msg # Load address of string @@ -58,13 +72,24 @@ msg: .string "Hello, World!\n" ### Exit with Code ```asm li a0, 1 # Exit code - li a7, 93 # Syscall 93 = exit_code + li a7, 17 # Syscall 17 = exit2 (93 also works) ecall ``` +### The Same on MIPS +```asm + la $a0, msg # Load address of string + li $v0, 4 # Syscall 4 = print_string + syscall + li $v0, 10 # Syscall 10 = exit + syscall +``` + ## Notes -- Register `a7` holds the syscall number (like MIPS `$v0`) -- Arguments use registers `a0-a5` (like MIPS `$a0-$a3`) -- String addresses must point to null-terminated strings -- Syscalls are educational - they don't invoke the actual OS +- `print_int` treats the register as signed (64-bit on RISC-V/ARM64/x86-64, 32-bit on MIPS), so + `-1` prints as `-1`. +- Input is read a line at a time. `read_char` returns the characters of that line one per call, + including the newline at the end; `read_int` reads a whole line. +- When running with `mapachespim -e`, the program's exit code becomes the process exit status. +- Syscalls are educational - they don't invoke the actual OS. diff --git a/examples/README.md b/examples/README.md index ee842d5..3831050 100644 --- a/examples/README.md +++ b/examples/README.md @@ -28,106 +28,117 @@ Each ISA directory contains 5 progressively challenging programs: # Start MapacheSPIM mapachespim -# Load and run a RISC-V example -(mapachespim) load examples/riscv/fibonacci/fibonacci +# List the examples, then load and run one +(mapachespim) examples +(mapachespim) load riscv/fibonacci (mapachespim) run -# Try an ARM64 example -(mapachespim) load examples/arm/fibonacci/fibonacci +# Try the same program on another ISA +(mapachespim) load arm/fibonacci (mapachespim) run ``` -## Building Examples - -Each ISA directory has a Makefile. You'll need the appropriate cross-compiler: +The examples are included in the installed package, so these names work from any directory. To get +your own editable copy: -### RISC-V ```bash -# macOS -brew install riscv64-elf-gcc - -# Build -cd examples/riscv && make +mapachespim --copy-examples my-examples ``` -### ARM64 -```bash -# macOS -brew install aarch64-elf-gcc +## Building Examples -# Build -cd examples/arm && make -``` +Each directory has the assembly source (`.s`) next to a prebuilt executable. You can load a source +file directly (`load riscv/fibonacci/fibonacci.s`) and MapacheSPIM assembles it for you. To rebuild +the executables with the bundled assembler, from this directory: -### x86-64 ```bash -# macOS -brew install x86_64-elf-binutils x86_64-elf-gcc - -# Build -cd examples/x86_64 && make +make # build all examples +make DEBUG=1 # build with debug info (how the committed binaries are built) +make riscv # build only one ISA: riscv, mips, arm, or x86_64 ``` -### MIPS32 -```bash -# Linux -sudo apt install binutils-mips-linux-gnu gcc-mips-linux-gnu +or assemble a single file: -# Build -cd examples/mips && make +```bash +mapachespim-as -g riscv/hello_asm/hello_asm.s -o hello_asm ``` -## Using the Built-in Assembler +RISC-V and MIPS are assembled by MapacheSPIM's built-in assembler. ARM64 and x86-64 use the Keystone +library, which is installed automatically on x86 computers (see the main README for other machines). -MapacheSPIM includes a built-in assembler that doesn't require external toolchains: - -```bash -# Assemble a RISC-V program -mapachespim-as examples/riscv/hello_asm/hello_asm.s -o hello_asm - -# Then load it -mapachespim -(mapachespim) load hello_asm -(mapachespim) run -``` +The Makefiles inside each ISA directory are an alternative that builds with the GNU cross +toolchains (for example `riscv64-unknown-elf-as`), for anyone who prefers them. ## Creating Your Own Programs +Start each file with an `.isa` line so MapacheSPIM knows which ISA it is for. Each template below +prints a number and exits. + ### RISC-V Template ```assembly +.isa riscv64 .text .globl _start - _start: - # Your code here - li a0, 42 # Return value - li a7, 10 # Exit syscall + li a0, 42 # Value to print + li a7, 1 # print_int + ecall + li a7, 10 # exit ecall ``` -### ARM64 Template +### MIPS Template ```assembly +.isa mips32 .text .globl _start +_start: + li $a0, 42 # Value to print + li $v0, 1 # print_int + syscall + li $v0, 10 # exit + syscall +``` +### ARM64 Template +```assembly +.isa arm64 +.text +.globl _start _start: - // Your code here - mov x0, #42 // Return value - mov x8, #10 // Exit syscall + mov x0, #42 // Value to print + mov x8, #1 // print_int + svc #0 + mov x8, #10 // exit svc #0 ``` -### Syscalls +### x86-64 Template +```assembly +.isa x86_64 +.text +.globl _start +_start: + movq $42, %rdi # Value to print + movq $1, %rax # print_int + syscall + movq $10, %rax # exit + syscall +``` -All ISAs support SPIM-compatible syscalls: +### Syscalls -| # | Name | Arguments | Description | -|---|------|-----------|-------------| -| 1 | print_int | a0 = integer | Print integer | -| 4 | print_string | a0 = address | Print null-terminated string | -| 5 | read_int | | Read integer into v0/a0 | -| 10 | exit | | Exit program | -| 11 | print_char | a0 = char | Print ASCII character | +All ISAs support the same SPIM-compatible syscalls; only the registers differ: + +| # | Name | Description | +|---|------|-------------| +| 1 | print_int | Print integer | +| 4 | print_string | Print null-terminated string | +| 5 | read_int | Read integer | +| 10 | exit | Exit program | +| 11 | print_char | Print ASCII character | +| 12 | read_char | Read a character | +| 17 | exit2 | Exit with a code | See [Syscall Reference](../docs/user/syscalls.md) for complete details. diff --git a/mapachespim/README.md b/mapachespim/README.md index 4378df4..14869ba 100644 --- a/mapachespim/README.md +++ b/mapachespim/README.md @@ -1,17 +1,11 @@ # MapacheSPIM Python API -Python simulator for RISC-V, ARM64, and x86-64 programs, powered by the Unicorn Engine. +Python simulator for RISC-V, MIPS, ARM64, and x86-64 programs, powered by the Unicorn Engine. ## Installation -```bash -pip install -e . -``` - -Dependencies (installed automatically): -- `unicorn>=2.0.0` - CPU emulator framework -- `capstone>=5.0.0` - Disassembler -- `pyelftools>=0.29` - ELF file parsing +See the [main README](../README.md#install). For development, `pip install -e ".[dev]"` from the +repository root. ## Quick Start @@ -38,9 +32,9 @@ all_regs = sim.get_all_regs() # Read memory data = sim.read_mem(0x80000000, 16) -# Run until halt (or max steps) +# Run until the program exits (or max steps); syscalls print to stdout steps = sim.run(max_steps=1000) -print(f"Executed {steps} instructions") +print(f"Executed {steps} instructions, exit code {sim.exit_code}") ``` ## API Reference @@ -49,10 +43,11 @@ print(f"Executed {steps} instructions") #### Initialization - `Simulator(isa=None)` - Create simulator instance - - `isa`: `ISA.RISCV`, `ISA.ARM`, or `ISA.X86_64` (default: auto-detect from ELF) + - `isa`: `ISA.RISCV`, `ISA.ARM`, `ISA.X86_64`, or `ISA.MIPS` (default: take the ISA of each + loaded ELF). A simulator created for one ISA refuses ELF files for another. #### Program Loading -- `load_elf(elf_path)` - Load ELF executable +- `load_elf(elf_path)` - Load ELF executable into a fresh machine state - Automatically detects ISA from ELF headers - Raises: `FileNotFoundError` if file doesn't exist - Raises: `RuntimeError` on invalid ELF @@ -60,12 +55,26 @@ print(f"Executed {steps} instructions") #### Execution Control - `step()` - Execute one instruction - Returns: `StepResult` (OK, HALT, SYSCALL, or ERROR) + - A syscall instruction returns `SYSCALL` without performing it; pass the result to + `check_termination()` to perform it (this is what `run()` and the console do) + - After the program exits, returns `HALT` without executing anything -- `run(max_steps=0)` - Run until halt or max steps - - `max_steps`: Maximum instructions (0 = unlimited) +- `check_termination(result)` - Perform a pending syscall and decide whether to stop + - Returns: `(stop, reason)`, where reason is `"syscall_exit"`, `"halt"`, `"error"`, + `"tohost"`, or `None` + +- `run(max_steps=None)` - Run until the program stops or `max_steps` instructions + - `max_steps`: Maximum instructions (`None` = unlimited) - Returns: Number of instructions executed -- `reset()` - Reset simulator state +- `reset()` - Reload the current ELF and start over + +#### Program Status and I/O +- `exited` / `exit_code` - Whether the program exited via a syscall, and its exit code +- `last_error` - Description of the last execution error, e.g. + `"Read from unmapped address 0x0 (at PC=0x80000004)"` +- `stdout` / `stdin` - Streams used by the I/O syscalls (default: the process's streams). + Assign e.g. `io.StringIO` objects to capture output or provide input #### State Inspection - `get_pc()` - Get program counter (64-bit) @@ -89,20 +98,45 @@ print(f"Executed {steps} instructions") #### Disassembly - `disasm(addr)` - Disassemble instruction at address +- `disasm_with_size(addr)` - Disassemble and return `(text, length_in_bytes)` (x86-64 + instructions vary in length) ### ISA Enum - `ISA.RISCV` - RISC-V 64-bit - `ISA.ARM` - ARM64 (AArch64) - `ISA.X86_64` - x86-64 +- `ISA.MIPS` - MIPS32 (big-endian) ### Helper Functions - `create_simulator(elf_path)` - Create simulator and load ELF in one call - `detect_elf_isa(elf_path)` - Detect ISA from ELF file +### Assembling from Python + +```python +from mapachespim.toolchain import assemble + +result = assemble(source_text, isa="riscv64", debug=True) +if result.success: + open("prog", "wb").write(result.elf_bytes) +else: + print("\n".join(result.errors)) +``` + +### Bundled Examples + +```python +from mapachespim.examples import find_example, list_examples + +for example in list_examples(): + print(example.short_name, example.description) +sim.load_elf(str(find_example("mips/fibonacci"))) +``` + ## Example: Tracing Execution ```python -from mapachespim import Simulator, StepResult +from mapachespim import Simulator sim = Simulator() sim.load_elf("examples/riscv/fibonacci/fibonacci") @@ -113,9 +147,9 @@ for i in range(10): disasm = sim.disasm(pc) print(f"[{i}] 0x{pc:x}: {disasm}") - result = sim.step() - if result == StepResult.HALT: - print("Program halted") + done, reason = sim.check_termination(sim.step()) + if done: + print(f"Stopped: {reason}") break # Show result @@ -134,4 +168,4 @@ Capstone (disassembly) ``` The simulator uses Unicorn Engine for accurate CPU emulation and Capstone for -disassembly. RISC-V 64-bit, ARM64, and x86-64 are supported with the same API. +disassembly. RISC-V 64-bit, MIPS32, ARM64, and x86-64 are supported with the same API. diff --git a/mapachespim/console.py b/mapachespim/console.py index 92cc4fe..f8c739e 100644 --- a/mapachespim/console.py +++ b/mapachespim/console.py @@ -398,7 +398,8 @@ def _load_elf_file( self.loaded_file = display_name self.loaded_source = source self._elf_path = elf_path - self.prev_regs = None + # Registers at the entry point are the baseline for ★ change markers + self.prev_regs = self.sim.get_all_regs() pc = self.sim.get_pc() isa_name = self.sim.get_isa_name() print(f"Loaded {display_name} ({isa_name})", file=self.stdout) @@ -804,8 +805,8 @@ def do_reset(self, arg: str) -> None: Tips: - Use 'clear' to remove breakpoints - - If you edited and re-assembled the program, 'reset' picks up - the new version + - 'reset' restarts the version that was loaded; after editing + a .s file, use 'reload' to re-assemble it """ if not self.loaded_file: self.print_error('Error: No program loaded. Use "load " first.') @@ -815,7 +816,7 @@ def do_reset(self, arg: str) -> None: except Exception as e: self.print_error(f"Error reloading {self.loaded_file}: {e}") return - self.prev_regs = None + self.prev_regs = self.sim.get_all_regs() print( f"Reset {self.loaded_file}. PC = {self.sim.get_pc():#018x}", file=self.stdout, @@ -924,6 +925,15 @@ def _format_reg_value(self, value: int, show_mode: str, leading_zeros_mode: str) return sign + prefix + formatted + def _change_marker(self) -> str: + """Marker for changed registers; '*' where the terminal can't show ★""" + encoding = getattr(self.stdout, "encoding", None) or "utf-8" + try: + "★".encode(encoding) + except (UnicodeEncodeError, LookupError): + return "*" + return "★" + def _is_32bit(self) -> bool: """True if the loaded ISA has 32-bit registers""" from . import ISA @@ -1028,7 +1038,7 @@ def do_regs(self, arg: str) -> None: # Check if this register changed with the last instruction star = ( - " ★ " + f" {self._change_marker()} " if ( self.show_reg_changes and self.prev_regs is not None @@ -2193,6 +2203,13 @@ def main() -> None: help=f"With -e, stop after N instructions (default {DEFAULT_MAX_STEPS:,})", ) + # Some terminals (e.g. legacy Windows code pages) can't show every + # character we print; substitute rather than crash + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is not None: + reconfigure(errors="replace") + args = parser.parse_args() if args.max_steps < 1: parser.error("--max-steps must be at least 1") diff --git a/mapachespim/toolchain/__init__.py b/mapachespim/toolchain/__init__.py index 9d5a64c..e616fe1 100644 --- a/mapachespim/toolchain/__init__.py +++ b/mapachespim/toolchain/__init__.py @@ -193,9 +193,10 @@ def _detect_isa_from_directive(source: str) -> Optional[str]: continue # Check for .isa directive if line.lower().startswith(".isa"): - parts = line.split(None, 1) + parts = line.split() if len(parts) >= 2: - isa_value = parts[1].strip().lower().replace("-", "_") + # First token only, so a trailing comment is ignored + isa_value = parts[1].lower().replace("-", "_") if isa_value in DirectiveParser.VALID_ISAS: return isa_value # Stop after first non-empty, non-comment, non-.isa line diff --git a/pyproject.toml b/pyproject.toml index 0d19bdf..efdbdb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,9 +5,9 @@ build-backend = "hatchling.build" [project] name = "mapachespim" version = "0.2.0" -description = "Educational Multi-ISA Simulator using Unicorn Engine" +description = "SPIM-like educational simulator and assembler for RISC-V, MIPS, ARM64, and x86-64" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" license = {text = "MIT"} authors = [ {name = "MapacheSPIM Contributors"}, @@ -18,12 +18,13 @@ classifiers = [ "Intended Audience :: Education", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", + "Programming Language :: Assembly", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Education", "Topic :: Software Development :: Debuggers", "Topic :: System :: Emulators", @@ -55,10 +56,10 @@ mapachespim = "mapachespim.console:main" mapachespim-as = "mapachespim.toolchain.cli:main" [project.urls] -Homepage = "https://github.com/mapachespim/MapacheSPIM" -Documentation = "https://github.com/mapachespim/MapacheSPIM#readme" -Repository = "https://github.com/mapachespim/MapacheSPIM.git" -Issues = "https://github.com/mapachespim/MapacheSPIM/issues" +Homepage = "https://github.com/UCSBarchlab/MapacheSPIM" +Documentation = "https://github.com/UCSBarchlab/MapacheSPIM#readme" +Repository = "https://github.com/UCSBarchlab/MapacheSPIM.git" +Issues = "https://github.com/UCSBarchlab/MapacheSPIM/issues" [tool.hatch.build.targets.wheel] packages = ["mapachespim"] @@ -72,7 +73,7 @@ include = ["mapachespim", "examples", "tests", "docs", "README.md", "pyproject.t [tool.ruff] line-length = 100 -target-version = "py38" +target-version = "py39" [tool.ruff.lint] select = [ @@ -91,9 +92,10 @@ ignore = [ "SIM108", # use ternary operator (sometimes less readable) "SIM116", # dictionary instead of if statements (sometimes less readable) "SIM118", # dict instead of dict.keys() (explicit is better) - "UP006", # use dict/list instead of Dict/List (need typing module for Python 3.8) - "UP007", # use X | Y instead of Union[X, Y] (need typing module for Python 3.8) - "UP045", # use X | None instead of Optional[X] (need typing module for Python 3.8) + "UP006", # use dict/list instead of Dict/List (typing names kept for consistency) + "UP007", # use X | Y instead of Union[X, Y] (needs Python 3.10) + "UP045", # use X | None instead of Optional[X] (needs Python 3.10) + "UP035", # typing.Dict etc. are deprecated aliases (kept for consistency) "B904", # raise from (valid but would change exception semantics) ] diff --git a/tests/README.md b/tests/README.md index 1547c8e..4a17f9f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,16 +4,25 @@ Comprehensive unit tests for the MapacheSPIM simulator. ## Running Tests -**Run all main test suites:** +**Run the whole suite** (from the repository root): ```bash -# Run each test suite individually (137 tests total) -python3 tests/test_simulator.py # API tests (31 tests) -python3 tests/test_console_working.py # Console tests (38 tests) -python3 tests/test_disasm_comprehensive.py # Disassembly tests (30 tests) -python3 tests/test_symbols.py # Symbol tests (24 tests) -python3 tests/test_program_correctness.py # Correctness tests (10 tests) -python3 tests/test_io_syscalls.py # I/O syscall tests (4 tests) +python -m pytest tests/ +``` + +Tests that assemble ARM64 or x86-64 code need the optional Keystone library; where it is not +installed (e.g. Apple Silicon), they are reported as skipped rather than failed. Everything else, +including all RISC-V and MIPS assembler tests, runs on every platform. + +**Run individual suites:** + +```bash +python -m pytest tests/test_simulator.py # API tests +python -m pytest tests/test_console_working.py # Console tests +python -m pytest tests/test_riscv_encoder.py # Built-in RISC-V assembler +python -m pytest tests/test_mips_encoder.py # Built-in MIPS assembler +python -m pytest tests/test_loading.py # load/reload/examples and the CLI +python -m pytest tests/test_regressions.py # Fixed bugs ``` **Run with verbose output:** @@ -214,17 +223,11 @@ x10 (a0) = 42 (0x2a) - exit code - **test_simple_basic.py** - Basic simulator functionality test - **test_run_completion.py** - Detailed program execution through completion -- **TESTING_NOTES.md** - Important notes about Sail behavior and test strategy ## Test Summary -**Total API Tests:** 31 (all passing) -**Total Console Tests:** 38 (all passing) -**Total Symbol Tests:** 24 (all passing) -**Total Disassembly Tests:** 30 (all passing) -**Total Correctness Tests:** 10 (all passing) -**Total I/O Syscall Tests:** 4 (all passing) -**Total:** 137 tests passing +Run `python -m pytest tests/ -q` for the current count; CI runs the full suite on Linux, Windows, +and macOS (Apple Silicon). **Programs Tested:** - fibonacci (recursive calculation) diff --git a/tests/test_loading.py b/tests/test_loading.py index 025bbf0..e258228 100644 --- a/tests/test_loading.py +++ b/tests/test_loading.py @@ -104,6 +104,20 @@ def test_load_source_and_run(self): self.assertIn("hello from source", text) self.assertIn("exited with code 7", text) + def test_isa_directive_with_comment(self): + src = self.write("c.s", HELLO_RISCV.replace(".isa riscv64", ".isa riscv64 # target")) + self.console.onecmd(f"load {src}") + self.assertIn("Assembled", self.output()) + + def test_first_regs_marks_changes_since_load(self): + self.console.onecmd("load riscv/fibonacci") + self.console.onecmd("step 3") + self.output() + self.console.onecmd("regs") + self.assertIn("★", self.output()) + self.console.onecmd("regs") + self.assertNotIn("★", self.output()) + def test_list_shows_source(self): src = self.write("hello.s", HELLO_RISCV) self.console.onecmd(f"load {src}") From e0dd3fb0a216629d368479ace55067dbcc0c1f4d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 04:17:16 +0000 Subject: [PATCH 08/18] Test built-in assemblers against GNU as and match it byte for byte Add differential tests that generate randomized programs covering every supported RISC-V and MIPS instruction form (all registers, immediates across their full ranges and edges, forward/backward labels) and require byte-identical output from GNU binutils; invalid inputs must be rejected by both. MAPACHESPIM_FUZZ_SCALE scales them up (about 400k instructions compared at 200x). CI installs binutils and runs a 50x job. Changes to match GNU as: - RISC-V: li uses a port of GNU's load_const; call/tail are always auipc+jalr; out-of-range branches are relaxed; %hi range checked - MIPS: la is lui+addiu; compare-branches follow GNU's macro rules (including constant folding); abs, mul by constant, loads/stores with symbols or large offsets; div/rem traps on divide by zero and overflow like SPIM; .set noreorder; jalr/bgezal register checks. Two-operand div/divu keep SPIM semantics (documented exception) Bugs found and fixed: - .align n meant n bytes; GNU uses 2^n on RISC-V/MIPS/ARM - MIPS data was little-endian when the ISA came from --isa - labels in .word/.dword became 0; bad values were silently ignored - "\xNN" in strings emitted two bytes; octal escapes unsupported - custom sections were never written to the ELF - MIPS .half/.word are auto-aligned as GNU does - break codes were encoded in the wrong field Also: an expression evaluator with GNU semantics (precedence, octal, '.', name = expr), data/alignment directives in .text, and clear "Division by zero" messages for MIPS traps. Example binaries rebuilt; all 20 produce identical output. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- .github/workflows/ci.yml | 31 ++ CHANGELOG.md | 17 +- README.md | 9 +- docs/user/console-guide.md | 11 +- examples/mips/array_stats/array_stats | Bin 1812 -> 1824 bytes examples/mips/fibonacci/fibonacci | Bin 1088 -> 1092 bytes examples/mips/guess_game/guess_game | Bin 1736 -> 1740 bytes examples/mips/hello_asm/hello_asm | Bin 988 -> 992 bytes examples/mips/matrix_multiply/matrix_mult | Bin 1476 -> 1480 bytes examples/riscv/fibonacci/fibonacci | Bin 1704 -> 1904 bytes examples/riscv/matrix_multiply/matrix_mult | Bin 2112 -> 2312 bytes examples/x86_64/fibonacci/fibonacci | Bin 1376 -> 1376 bytes examples/x86_64/matrix_multiply/matrix_mult | Bin 1712 -> 1712 bytes mapachespim/toolchain/assembler.py | 154 +++++- mapachespim/toolchain/directives.py | 493 +++++++++++--------- mapachespim/toolchain/expr.py | 294 +++++++++--- mapachespim/toolchain/mips.py | 375 +++++++++++---- mapachespim/toolchain/riscv.py | 148 +++--- mapachespim/unicorn_backend.py | 31 +- tests/README.md | 10 + tests/gnu_reference.py | 168 +++++++ tests/test_differential_data.py | 72 +++ tests/test_differential_mips.py | 369 +++++++++++++++ tests/test_differential_riscv.py | 352 ++++++++++++++ tests/test_mips_encoder.py | 37 +- tests/test_regressions.py | 53 +++ tests/test_riscv_encoder.py | 21 +- 27 files changed, 2192 insertions(+), 453 deletions(-) create mode 100644 tests/gnu_reference.py create mode 100644 tests/test_differential_data.py create mode 100644 tests/test_differential_mips.py create mode 100644 tests/test_differential_riscv.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9fe573..50de417 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,5 +77,36 @@ jobs: mapachespim -e ex/mips/fibonacci/fibonacci.s printf 'load riscv/fibonacci\nbreak fibonacci\nrun\nregs\nlist\nquit\n' | mapachespim + - name: Install GNU binutils for the assembler differential tests + if: runner.os == 'Linux' + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu + - name: Run tests run: python -m pytest tests/ --tb=short + + assembler-fuzz: + # The built-in RISC-V and MIPS assemblers must produce byte-identical + # output to GNU as. This runs the randomized comparison much deeper + # than the regular test job. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install GNU binutils + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu + + - name: Install package + run: pip install -e ".[dev]" + + - name: Compare with GNU as (50x fuzz scale) + env: + MAPACHESPIM_FUZZ_SCALE: "50" + run: python -m pytest tests/test_differential_riscv.py tests/test_differential_mips.py tests/test_differential_data.py --tb=short diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a1dd40..5b510e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,8 +33,21 @@ - Loading a program starts from a fresh machine state and may switch ISA. ### Assembler -- RISC-V `li` handles all 64-bit constants; `.equ` constants can be used as instruction operands; - `add a0, a1, 5` is accepted as `addi`. +- The built-in RISC-V and MIPS assemblers now produce byte-for-byte the same code as GNU `as`, + verified by randomized differential tests against GNU binutils (run in CI). This changed some + pseudo-instruction expansions to GNU's: RISC-V `call`/`tail` are always `auipc`+`jalr`, `li` uses + GNU's algorithm; MIPS `la` is `lui`+`addiu`, and compare-branches, `abs`, `mul`/`div`/`rem` with + constants follow GNU. Out-of-range RISC-V branches are relaxed like GNU does. +- MIPS `div`/`rem`/`divu`/`remu` with three operands now trap on division by zero and overflow + (as SPIM and GNU do), reported as "Division by zero"; `.set noreorder` is supported. +- Fixed: `.align n` meant n bytes instead of 2^n on RISC-V/MIPS/ARM; MIPS data was little-endian + when the ISA came from `--isa`/`load prog.s mips32`; labels in `.word` etc. became 0; `\xNN` in + strings produced two bytes; custom sections were not written to the ELF; invalid values in data + directives were silently ignored; MIPS `.half`/`.word` are now auto-aligned like GNU `as`; `break` + codes were encoded in the wrong field. +- Expressions follow GNU `as` rules (operators, precedence, octal `010`, `'c'` literals); `.`, and + `name = expression` (e.g. `len = . - msg`) are supported. +- RISC-V `.equ` constants can be used as instruction operands; `add a0, a1, 5` is accepted as `addi`. - Debug info for big-endian targets (MIPS) was written in the wrong byte order, so source-level debugging never worked for MIPS. - `.isa riscv64 # comment` is recognized. diff --git a/README.md b/README.md index 3748489..2123a8d 100644 --- a/README.md +++ b/README.md @@ -256,8 +256,8 @@ syscalls, and a matching set of example programs: | ISA | Examples | Assembling `.s` files | |-----|----------|-----------------------| -| RISC-V 64-bit (RV64IM) | `examples/riscv/` | Built in, works everywhere | -| MIPS32 (big-endian) | `examples/mips/` | Built in, works everywhere | +| RISC-V 64-bit (RV64IM) | `examples/riscv/` | Built in, works everywhere\* | +| MIPS32 (big-endian) | `examples/mips/` | Built in, works everywhere\* | | ARM64 (AArch64) | `examples/arm/` | Uses [Keystone](https://www.keystone-engine.org/) | | x86-64 (AT&T or Intel syntax) | `examples/x86_64/` | Uses [Keystone](https://www.keystone-engine.org/) | @@ -267,6 +267,11 @@ default; you can still run and debug ARM64 and x86-64 programs (including all th can try `pip install 'mapachespim[keystone]'` to build Keystone from source (this needs CMake and a C++ compiler). +\* The built-in RISC-V and MIPS assemblers produce byte-for-byte the same machine code as GNU `as` +(the standard assembler used in textbooks and courses), including its pseudo-instruction expansions, +data directives, and branch relaxation. This is checked continuously by randomized differential tests +that assemble hundreds of thousands of instructions with both and compare the results. + Loading an ELF file detects its ISA automatically. ### Student-Friendly Design diff --git a/docs/user/console-guide.md b/docs/user/console-guide.md index 462767f..3776f13 100644 --- a/docs/user/console-guide.md +++ b/docs/user/console-guide.md @@ -321,8 +321,15 @@ MapacheSPIM is similar to SPIM but has key differences: 1. **Several ISAs**: RISC-V (RV64IM), MIPS32, ARM64, and x86-64, all with the same commands and syscalls 2. **Real executables**: Programs are assembled to ELF files, so the same binaries work with other tools 3. **Delay slots**: On MIPS, the assembler fills each branch and jump delay slot with a `nop` (like - GNU `as` in its default mode), and the simulator executes delay slots as real hardware does -4. **Source display**: `list` shows your source code next to the program counter + GNU `as` in its default `.set reorder` mode), and the simulator executes delay slots as real + hardware does. Write `.set noreorder` to fill delay slots yourself +4. **Same code as GNU as**: RISC-V and MIPS programs assemble to exactly what GNU `as` produces, + so disassembly matches what you see in textbooks and `objdump`. One deliberate exception follows + SPIM instead: on MIPS, the two-operand `div $t0, $t1` / `divu $t0, $t1` is the real instruction + (results in HI and LO), whereas GNU `as` treats it as `div $t0, $t0, $t1` +5. **Runtime checks**: MIPS `div`/`rem` with three operands trap on division by zero and on + overflow, as in SPIM, and the console reports "Division by zero" +6. **Source display**: `list` shows your source code next to the program counter ## Troubleshooting diff --git a/examples/mips/array_stats/array_stats b/examples/mips/array_stats/array_stats index 62a941ea10587b3422c849759f676acc2c5e67d6..b6a0318e0f76508f5f87bc1bd1cd5c852c55d1aa 100644 GIT binary patch delta 487 zcmbQjw}5YgGUJYkD$1fC7#J9Vn1L0DS%8?4fq^MyVvI7Q+QbG4NtG4`6%{711dk1e z0D~$A1A`4Gke;|!UJ)qi29s3eU|>RT9eVCFF~D5wJ2K-VzvfX!-Q@PNsI`I9#?N(%yAfF^pEQQ8zk zw1vSAt`}mxLNCy-DCSSrWtzkIVe%=aSUDXX1wA=A1sypBIeC2$DQ6@H0xCKn(ao;R z{}?k}K#m3ikOMOr92hFVJP5x8$e)44uLtsPAo06^{2xgCDGUybz)%7Mka_cfd2Dwn-HL delta 436 zcmZ3$H-&G4GUJ+wD$1fy7#J9Vn1L0DS%8?4fq^MvVvI7Q>cj>KQIi%16%{711dk1e z0D}#u0E6kojq;2p6EBK!2>{hI*susNm`wa=F4`-|z|bqiz@P#YhO3?I%IL&p3|BXK zBBQh*&`=M!p&;Fp_hJ=&$tZ1#p|*v=4z3sIAX5$og4HAC}kpBmXKaatIk!SJ-R(GI#ey}PtMogArQ)gT>*#$^$nw$e9 jpHH3wBpW900g@LcKVefBtpQqX05pRgh(T@wMT-Ccsz*dK diff --git a/examples/mips/fibonacci/fibonacci b/examples/mips/fibonacci/fibonacci index 7d453799f7867fef3f8a30bf53e8b427ff0fe5b5..30d554ed010ad2e9e3eadab25c8d26923e6bf889 100644 GIT binary patch delta 197 zcmX@WafD-nGUJ_zD$1fe7#J8pm=%awfS3`8bSB2gh^lfhF!TaNcmx;(fZ_}`90Clg z6Fb8M!8`_s4pk+V|NlU;6F;Og?wB0G7{I8rc_kwoqoM;yJrDrRVo+yrU|0a+00D?^ z2IQ}qe1XXws6~QVnNer54YN9<$>bCu=`y(oNS06D0wixtz5^sRCbO`pi{=3J6#y|i J)C`b21pu{AB9s6C delta 193 zcmX@Yae!ljGUJVjD$1f;7#J8pm=%awfS3`8G$zK#2=)SbJOT{-Krsdz4gm(!iM?Th zU><`*hpH0G|9>FaiC@wgw@i*<3}BSoyq1xTQPT#b9teQuFsL&)Fw6n-Abc|*e+7&` z`5}|qG_Lc3@U#G?<(LBpoJC0FuR%cL2#NlOF&{mB}0|>Wmqa H4Om0~BU2>3 diff --git a/examples/mips/guess_game/guess_game b/examples/mips/guess_game/guess_game index 850e225b9b14b156143305555fcf6909417929a2..5124a80b7975a9c7b278e9e92080823eec91768a 100644 GIT binary patch delta 397 zcmX@Xdxm#{GGod_6=hK!1_nkTW?%(k79eH>q6HIUWCB%M7+6%8z+yZ$ECLKbz8V@I zUEUrp&mq8|%E7?UE5N{@BFMl1)<5y6u_MrsDKJGqH6lO@A(AZ&r8vYl!^FXcOtxfH z*94k!6DA5Y#TChvM=)`)>dl>umW+%#lMge+G3snqV*bkL830lY1VDc>L?EnA( delta 381 zcmX@ZdxCd@GGoF-6=hKk1_nkTW?%(k79eH>qB#>|WPD9p7+6%8z+yZ$ECLKbz8V@I zUEUrp&mq9jE5N{@BFMl1);sZ}u_MrcDKG^p#G#ZE**nt>i1}J<50H6UgD*ylh diff --git a/examples/mips/hello_asm/hello_asm b/examples/mips/hello_asm/hello_asm index c56e67aa90155db0601b4e43294f6153060e3ea4..091ec27008486a1415daa7576d5905d44224233c 100644 GIT binary patch delta 138 zcmcb^{(yagGGoO=6=l%~1_lNYW(8svAZ7&OGZSNE1zH#wRG7d*JT@!>3@Q^_^%+$r zZp~+mn5@bez^JpijM0`6$l1cA%y?$<4JI`vM#f3Z>WqSuHGrhcXl9vc<`29t@M`iv$McjhyO zOx9!!V3gZj#c0b2qBRp^WF%Bt7#M6=1sGJ8GN{0<7wcv6C%;}8m0Kcs$Pyhe` delta 275 zcmX@XeS~|0GGoj{6=hKq1_nkTW?%(k79eH>q7@TkWJGLO1sF`0GN{lQ)!>G(F4OGgYB*4JPA;2IuxrXsDqse3krVvKC%~P3@7*kb1(m((-jlr0~ zfgu9SgYfNu{2U~{7m(kA#194XZy@pGf&4#6{453sMh*~v@&`tZ$pTF5le<|&fYvWy zQD$5*`3Q?TqsZhpKvHG02&+0{TSi2dMP`5VHd@$bBGR3jhGa CkSN6f diff --git a/examples/riscv/fibonacci/fibonacci b/examples/riscv/fibonacci/fibonacci index 6c34f31dcd6ae8655aa7e37c7ed1039521640da2..626d303857427cd2b8a85cd23f9393772589d24c 100644 GIT binary patch delta 346 zcmZ3%`+;wQhSCNW1~_nFuwYw4GsX=Q8_gMaOx&rj1ri5o z*#TmJfn)(@e39j69PyS?)8vV3^FsYR6bH*^||tv0-v2tN!F4tO868jFY+8)EE^ee`FQs zoB%Qo2&^XyvdJ^*fW=Nj#p1zY7Lzx!iE|n-LDU6-#hk!m9Z<2#$rss_C!4T~XmBt? UR4G7B-~e(#Ob`tM5)g_30E(R_LjV8( diff --git a/examples/riscv/matrix_multiply/matrix_mult b/examples/riscv/matrix_multiply/matrix_mult index 6c1c8bf239ece475eb000bc39e19dad34edd9a2d..8e1986a7201407b548c59964c6d26e080f9f8eea 100644 GIT binary patch delta 316 zcmX>g&>=KIL&<}U0S+7(EErfA92mG6I2ag#qO3ryz&O#%j7foMVv`wT&%~YjS|AA) zAnsuT5?~pS7nf`{WM0n5=&<=Fi#;Qg z0^?*UHanpClcU(|7<(qqWYd>~8_WPP8D^8m>`u%**F+if>i`fX5rQh?r+}Byp)lVXLBN}JtO0b$s5?kC+}d>0IHn)h|P{MWU?Z={^SfM0U&E3yE>!7 zQGF7tx08z>% diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index 510b84e..850ccb3 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field from types import ModuleType -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple try: import keystone @@ -29,13 +29,24 @@ keystone = None from . import mips, riscv -from .directives import DirectiveParser, LineType, SectionData +from .directives import DOT_LABEL_PREFIX, DirectiveParser, LineType, ParsedLine, SectionData from .dwarf import DWARFv2Builder from .elf_builder import STB_GLOBAL, STB_LOCAL, STT_FUNC, STT_NOTYPE, ELFBuilder, Section, Symbol -from .expr import EncodeError +from .expr import EncodeError, ExpressionError, UndefinedSymbol, evaluate from .memory_map import get_layout +def _lookup(values: Dict[str, int]) -> Callable[[str], int]: + """Symbol resolver over a dict, raising UndefinedSymbol for unknown names.""" + + def resolve(name: str) -> int: + if name in values: + return values[name] + raise UndefinedSymbol(name) + + return resolve + + @dataclass class AssemblyResult: """Result of assembly operation.""" @@ -170,6 +181,8 @@ def __init__(self, isa: str): self._config = self.ISA_CONFIG[isa_lower] self._layout = get_layout(isa_lower) self._ks = None + # Replaced for each assemble() call; also used by the layout passes + self._parser = DirectiveParser(isa=isa_lower) # RISC-V and MIPS use built-in encoders; ARM64 and x86-64 need Keystone if self._config["arch"] is None: @@ -190,6 +203,64 @@ def __init__(self, isa: str): except keystone.KsError as e: raise RuntimeError(f"Failed to initialize Keystone for {isa}: {e}") + @staticmethod + def _resolve_constants( + pending: Dict[str, Tuple[str, int]], + constants: Dict[str, int], + labels: Dict[str, int], + ) -> bool: + """Evaluate pending constants whose symbols are now known. + + Resolved entries move from ``pending`` to ``constants``. Returns True + if any were resolved. + """ + resolved_any = False + progress = True + while progress and pending: + progress = False + for name, (expression, _line) in list(pending.items()): + try: + constants[name] = evaluate(expression, _lookup({**constants, **labels})) + except ExpressionError: + continue # still waiting for a symbol (errors reported later) + del pending[name] + progress = resolved_any = True + return resolved_any + + def _text_directive_bytes( + self, line: ParsedLine, addr: int, base: int, labels: Dict[str, int], strict: bool + ) -> bytes: + """Bytes a data or alignment directive in .text produces at ``addr``. + + Alignment padding in RISC-V code is made of nop instructions, as GNU + as does (MIPS's nop is all zeros, the default fill). + """ + values = {**labels, ".": addr} + + def resolve(name: str) -> int: + if name in values: + return values[name] + if strict: + raise UndefinedSymbol(name) + return 0 + + data = self._parser.data_bytes( + line.directive or "", line.directive_args, resolve, offset=addr - base + ) + if data is None: + return b"" + is_align = line.directive in ("align", "balign", "p2align") + explicit_fill = len(line.directive_args) > 1 and line.directive_args[1] + if is_align and not explicit_fill and data and len(data) % 4 == 0 and self._is_riscv: + data = (0x00000013).to_bytes(4, "little") * (len(data) // 4) + return data + + def _line_options(self, line: ParsedLine) -> Dict[str, Any]: + """Per-line encoder options (MIPS: .set reorder / noreorder).""" + if self.isa in ("mips32", "mips"): + return {"reorder": line.reorder} + return {} + @property def _is_riscv(self) -> bool: return self.isa in ("riscv64", "riscv") @@ -225,7 +296,8 @@ def assemble( result = AssemblyResult(isa=self.isa, source_filename=source_filename) # Parse source - parser = DirectiveParser() + parser = DirectiveParser(isa=self.isa) + self._parser = parser sections = parser.parse(source) result.errors.extend(parser.errors) @@ -270,18 +342,33 @@ def assemble( for label_name, offset in sect_data.labels.items(): all_labels[label_name] = base + offset - # Calculate .text section label positions - # Pass all_labels (which now contains data section labels) for pseudo-instruction sizing # .equ/.set constants can be used as instruction operands but are - # not addresses, so they are kept out of the ELF symbol table + # not addresses, so they are kept out of the ELF symbol table. + # Constants that refer to labels (e.g. "len = . - msg") are resolved + # as soon as the labels they use have addresses. constants = dict(parser.constants) + pending = dict(parser.deferred_constants) + self._resolve_constants(pending, constants, all_labels) + # Calculate .text section label positions, using the data labels and + # constants for pseudo-instruction sizing text_section = sections.get(".text") if text_section: text_labels = self._calculate_text_labels( text_section, self._layout.text_base, {**constants, **all_labels} ) all_labels.update(text_labels) + if pending and self._resolve_constants(pending, constants, all_labels): + # Constants that needed code labels may change code size + text_labels = self._calculate_text_labels( + text_section, self._layout.text_base, {**constants, **all_labels} + ) + all_labels.update(text_labels) + for name, (expression, line_number) in pending.items(): + try: + evaluate(expression, _lookup({**constants, **all_labels})) + except ExpressionError as e: + result.errors.append(f"Line {line_number}: {name}: {e}") # Pass 2: Assemble .text section with all labels known debug_lines: List[Tuple[int, int]] = [] @@ -299,17 +386,38 @@ def assemble( if not result.errors: text_section.data = bytearray(code) + # Fill in data values that refer to labels (e.g. ".word array") + symbol_values = {**constants, **all_labels} + for sect_name, sect_data in sections.items(): + if sect_name == ".text": + continue + for fixup in sect_data.fixups: + try: + value = evaluate(fixup.expression, _lookup(symbol_values)) + except ExpressionError as e: + result.errors.append(f"Line {fixup.line_number}: {e}") + continue + mask = (1 << (8 * fixup.size)) - 1 + sect_data.data[fixup.offset : fixup.offset + fixup.size] = (value & mask).to_bytes( + fixup.size, fixup.byteorder + ) + if result.errors: return result + # Internal labels standing in for '.' are not real symbols + all_labels = {k: v for k, v in all_labels.items() if not k.startswith(DOT_LABEL_PREFIX)} + # Build ELF entry_addr = all_labels.get(entry_symbol, self._layout.text_base) result.entry_point = entry_addr builder = ELFBuilder(self.isa, entry=entry_addr) - # Add sections - for sect_name in [".text", ".data", ".rodata", ".bss"]: + # Add sections (standard ones first, then any custom sections) + ordered = [n for n in (".text", ".data", ".rodata", ".bss") if n in sections] + ordered += [n for n in sections if n not in ordered] + for sect_name in ordered: if sect_name in sections: sect_data = sections[sect_name] if sect_data.data or sect_name == ".text": @@ -444,7 +552,16 @@ def _calculate_builtin_text_labels( if line.label: new_labels[line.label] = addr if line.line_type == LineType.INSTRUCTION and line.instruction: - addr += self._builtin_encoder.instruction_size(line.instruction, known, addr) + addr += self._builtin_encoder.instruction_size( + line.instruction, known, addr, **self._line_options(line) + ) + elif line.line_type == LineType.DIRECTIVE: + try: + addr += len( + self._text_directive_bytes(line, addr, base_addr, known, strict=False) + ) + except ExpressionError: + pass # reported when the section is assembled if new_labels == labels: break labels = new_labels @@ -758,6 +875,19 @@ def _assemble_section( if line.label: label_addrs[line.label] = current_addr + # Data and alignment directives in code (built-in encoders only) + if line.line_type == LineType.DIRECTIVE and self._builtin_encoder is not None: + try: + data = self._text_directive_bytes( + line, current_addr, base_addr, labels, strict=True + ) + except ExpressionError as e: + errors.append(f"Line {line.line_number}: .{line.directive}: {e}") + continue + code.extend(data) + current_addr += len(data) + continue + # Skip non-instructions if line.line_type != LineType.INSTRUCTION or not line.instruction: continue @@ -772,7 +902,9 @@ def _assemble_section( encoder = self._builtin_encoder if encoder is not None: try: - encoding = encoder.encode(instr, current_addr, labels) + encoding = encoder.encode( + instr, current_addr, labels, **self._line_options(line) + ) except EncodeError as e: errors.append(f"Line {line.line_number}: {e} - {line.instruction}") continue diff --git a/mapachespim/toolchain/directives.py b/mapachespim/toolchain/directives.py index 4983b0d..576aa8b 100644 --- a/mapachespim/toolchain/directives.py +++ b/mapachespim/toolchain/directives.py @@ -10,7 +10,9 @@ import re from dataclasses import dataclass, field from enum import Enum, auto -from typing import Dict, List, Literal, Optional, Tuple +from typing import Callable, Dict, List, Literal, Optional, Tuple + +from .expr import ExpressionError, UndefinedSymbol, evaluate class LineType(Enum): @@ -46,6 +48,9 @@ class ParsedLine: """Arguments to the directive.""" instruction: Optional[str] = None + + reorder: bool = True + """MIPS: whether the assembler fills delay slots (.set reorder, the default).""" """Instruction mnemonic and operands.""" @@ -68,6 +73,39 @@ class SectionData: current_offset: int = 0 """Current offset within section.""" + fixups: List[Fixup] = field(default_factory=list) + """Data values that refer to labels, filled in once addresses are known.""" + + pending_labels: List[str] = field(default_factory=list) + """Labels defined since the last data or instruction (MIPS auto-alignment + moves these along with the data that follows them).""" + + +@dataclass +class Fixup: + """A data value whose expression needs label addresses.""" + + offset: int + size: int + expression: str + line_number: int + byteorder: Literal["little", "big"] + + +# Data directives: name -> size of each value in bytes +DATA_SIZES: Dict[str, int] = { + "byte": 1, + "half": 2, "short": 2, "2byte": 2, "hword": 2, + "word": 4, "long": 4, "4byte": 4, "int": 4, + "dword": 8, "quad": 8, "8byte": 8, +} # fmt: skip + +# The location counter '.', when it appears as a symbol in an expression +_DOT_RE = re.compile(r"(? None: + # ISAs whose GNU assembler aligns .half/.word/.dword automatically + AUTO_ALIGN_ISAS = {"mips32"} + + def __init__(self, isa: Optional[str] = None) -> None: + """ + Args: + isa: Target ISA, if known (e.g. from the assembler's --isa). It + decides byte order and what .align means; an .isa directive + in the source is used when this is not given. + """ + self.target_isa = isa.lower().replace("-", "_") if isa else None self.sections: Dict[str, SectionData] = {} self.current_section: str = ".text" self.global_symbols: set = set() @@ -110,17 +166,113 @@ def __init__(self) -> None: self.errors: List[str] = [] self.warnings: List[str] = [] self.isa: Optional[str] = None # ISA from .isa directive + # .equ/.set constants that refer to labels: name -> (expression, line) + self.deferred_constants: Dict[str, Tuple[str, int]] = {} + self.reorder = True # MIPS .set reorder / .set noreorder + self._dot_count = 0 # Initialize default sections for name in [".text", ".data", ".rodata", ".bss"]: self.sections[name] = SectionData(name=name) + @property + def _effective_isa(self) -> Optional[str]: + return self.target_isa or self.isa + def _get_endianness(self) -> Literal["little", "big"]: """Return byte order for data directives based on ISA.""" - if self.isa in self.BIG_ENDIAN_ISAS: + if self._effective_isa in self.BIG_ENDIAN_ISAS: return "big" return "little" + def _alignment(self, directive: str, value: int) -> int: + """Alignment in bytes for .align/.balign/.p2align with argument value. + + As in GNU as, .align takes a power of two on RISC-V, MIPS, and ARM, + but a byte count on x86. + """ + if directive == "balign" or (directive == "align" and self._effective_isa == "x86_64"): + return value + return 1 << value + + def _dot_label(self, section: SectionData) -> str: + """Define an internal label at the current location, to stand for '.'.""" + self._dot_count += 1 + name = f"{DOT_LABEL_PREFIX}{self._dot_count}" + section.labels[name] = section.current_offset + return name + + def _constant(self, name: str) -> int: + if name in self.constants: + return self.constants[name] + raise UndefinedSymbol(name) + + def data_bytes( + self, + directive: str, + args: List[str], + resolve: Callable[[str], int], + offset: int = 0, + fixups: Optional[List[Fixup]] = None, + line_number: int = 0, + ) -> Optional[bytes]: + """ + Bytes produced by a data directive (.word, .asciz, .space, .align...). + + Values are evaluated with ``resolve``. If ``fixups`` is given, values + that refer to undefined symbols are emitted as zeros and recorded + there to be filled in later; otherwise UndefinedSymbol propagates. + + Returns: + The bytes, or None if ``directive`` is not a data directive. + + Raises: + ExpressionError: For invalid values. + """ + byteorder = self._get_endianness() + + if directive in DATA_SIZES: + size = DATA_SIZES[directive] + out = bytearray() + for arg in args: + try: + value = evaluate(arg, resolve) + except UndefinedSymbol: + if fixups is None: + raise + fixups.append(Fixup(offset + len(out), size, arg, line_number, byteorder)) + value = 0 + out += (value & ((1 << (8 * size)) - 1)).to_bytes(size, byteorder) + return bytes(out) + + if directive in ("ascii", "asciz", "string"): + out = bytearray() + for arg in args: + out += self._parse_string_bytes(arg) + if directive != "ascii": + out.append(0) + return bytes(out) + + if directive in ("space", "skip", "zero"): + if not args: + raise ExpressionError(f".{directive} needs a size") + size = evaluate(args[0], resolve) + if size < 0: + raise ExpressionError(f".{directive} size must not be negative") + fill = evaluate(args[1], resolve) & 0xFF if len(args) > 1 else 0 + return bytes([fill]) * size + + if directive in ("align", "balign", "p2align"): + if not args: + raise ExpressionError(f".{directive} needs an alignment") + alignment = self._alignment(directive, evaluate(args[0], resolve)) + if alignment <= 0 or alignment & (alignment - 1): + raise ExpressionError(f"alignment {alignment} is not a power of 2") + fill = evaluate(args[1], resolve) & 0xFF if len(args) > 1 and args[1] else 0 + return bytes([fill]) * ((alignment - offset % alignment) % alignment) + + return None + def parse(self, source: str) -> Dict[str, SectionData]: """ Parse assembly source into sections. @@ -137,6 +289,7 @@ def parse(self, source: str) -> Dict[str, SectionData]: try: parsed = self._parse_line(line_num, line) if parsed.line_type not in (LineType.EMPTY, LineType.COMMENT): + parsed.reorder = self.reorder section = self.sections[self.current_section] section.lines.append(parsed) @@ -146,10 +299,13 @@ def parse(self, source: str) -> Dict[str, SectionData]: section.labels[parsed.label] = self.constants[parsed.label] else: section.labels[parsed.label] = section.current_offset + section.pending_labels.append(parsed.label) # Process directive data if parsed.directive: self._process_directive(parsed, section) + elif parsed.line_type == LineType.INSTRUCTION: + section.pending_labels.clear() except Exception as e: self.errors.append(f"Line {line_num}: {e}") @@ -172,6 +328,17 @@ def _parse_line(self, line_num: int, line: str) -> ParsedLine: if not line: return ParsedLine(line_num, LineType.EMPTY, original) + # Symbol assignment: "name = expression" (same as .set name, expression) + assign = self._ASSIGN_PATTERN.match(line) + if assign: + return ParsedLine( + line_num, + LineType.DIRECTIVE, + original, + directive="set", + directive_args=[assign.group(1), assign.group(2).strip()], + ) + # Check for label label = None label_match = self.LABEL_PATTERN.match(line) @@ -278,132 +445,60 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: self.local_symbols.add(arg) return - # Constants - if directive in ("equ", "set"): - if len(args) >= 2: - name = args[0] - try: - value = self._evaluate_expr(args[1]) - self.constants[name] = value - except Exception: - pass + # MIPS assembler options: ".set noreorder", ".set noat", ... + if directive == "set" and len(args) == 1: + option = args[0].lower() + if option in ("reorder", "noreorder"): + self.reorder = option == "reorder" + elif option not in self.KNOWN_SET_OPTIONS: + self.warnings.append(f"Line {parsed.line_number}: Unknown option .set {args[0]}") return - # Data directives - these add bytes to current section - if directive == "byte": - for arg in args: - try: - value = self._evaluate_expr(arg) - section.data.append(value & 0xFF) - section.current_offset += 1 - except Exception: - section.data.append(0) - section.current_offset += 1 - return - - if directive in ("half", "short", "2byte"): - endian = self._get_endianness() - for arg in args: - try: - value = self._evaluate_expr(arg) - section.data.extend((value & 0xFFFF).to_bytes(2, endian)) - section.current_offset += 2 - except Exception: - section.data.extend(b"\x00\x00") - section.current_offset += 2 - return - - if directive in ("word", "long", "4byte"): - endian = self._get_endianness() - for arg in args: - try: - value = self._evaluate_expr(arg) - section.data.extend((value & 0xFFFFFFFF).to_bytes(4, endian)) - section.current_offset += 4 - except Exception: - section.data.extend(b"\x00\x00\x00\x00") - section.current_offset += 4 - return - - if directive in ("dword", "quad", "8byte"): - endian = self._get_endianness() - for arg in args: - try: - value = self._evaluate_expr(arg) - section.data.extend(value.to_bytes(8, endian)) - section.current_offset += 8 - except Exception: - section.data.extend(b"\x00" * 8) - section.current_offset += 8 - return - - # String directives - if directive == "ascii": - for arg in args: - s = self._parse_string(arg) - section.data.extend(s.encode("utf-8")) - section.current_offset += len(s) - return - - if directive in ("asciz", "string"): - for arg in args: - s = self._parse_string(arg) - section.data.extend(s.encode("utf-8")) - section.data.append(0) # Null terminator - section.current_offset += len(s) + 1 - return - - # Alignment - if directive == "align": - if args: - try: - # .align n aligns to 2^n bytes on some platforms - # or to n bytes on others. We use n bytes. - alignment = self._evaluate_expr(args[0]) - if alignment > 0: - padding = (alignment - (section.current_offset % alignment)) % alignment - section.data.extend(b"\x00" * padding) - section.current_offset += padding - except Exception: - pass + # Constants + if directive in ("equ", "set", "equiv"): + if len(args) < 2: + self.errors.append( + f"Line {parsed.line_number}: .{directive} needs a name and a value" + ) + return + self._define_constant(args[0], args[1], parsed.line_number, section) return - if directive == "balign": - if args: - try: - alignment = self._evaluate_expr(args[0]) - if alignment > 0: - padding = (alignment - (section.current_offset % alignment)) % alignment - section.data.extend(b"\x00" * padding) - section.current_offset += padding - except Exception: - pass - return + # Data directives - these add bytes to current section - if directive == "p2align": - if args: - try: - power = self._evaluate_expr(args[0]) - alignment = 1 << power - padding = (alignment - (section.current_offset % alignment)) % alignment - section.data.extend(b"\x00" * padding) - section.current_offset += padding - except Exception: - pass + # Like GNU as for MIPS, align .half/.word/.dword to their size, and + # move labels that were waiting for this data along with it + if ( + directive in DATA_SIZES + and DATA_SIZES[directive] > 1 + and self._effective_isa in self.AUTO_ALIGN_ISAS + ): + size = DATA_SIZES[directive] + padding = (size - section.current_offset % size) % size + if padding: + section.data.extend(b"\x00" * padding) + section.current_offset += padding + for label in section.pending_labels: + section.labels[label] = section.current_offset + + expr_args = [_DOT_RE.sub(lambda m: self._dot_label(section), a) for a in args] + try: + data = self.data_bytes( + directive or "", + expr_args, + self._constant, + offset=section.current_offset, + fixups=section.fixups, + line_number=parsed.line_number, + ) + except ExpressionError as e: + self.errors.append(f"Line {parsed.line_number}: .{directive}: {e}") return - - # Space/skip - if directive in ("space", "skip", "zero"): - if args: - try: - size = self._evaluate_expr(args[0]) - fill = 0 - if len(args) > 1: - fill = self._evaluate_expr(args[1]) & 0xFF - section.data.extend(bytes([fill] * size)) - section.current_offset += size - except Exception: - pass + if data is not None: + section.data.extend(data) + section.current_offset += len(data) + if data: + section.pending_labels.clear() return # Architecture hints (ignored, but don't warn) @@ -413,108 +508,66 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: # Unknown directive self.warnings.append(f"Line {parsed.line_number}: Unknown directive .{directive}") - def _evaluate_expr(self, expr: str) -> int: - """Evaluate a simple expression.""" - expr = expr.strip() - - # Check for known constant - if expr in self.constants: - return self.constants[expr] - - # Hex - if expr.startswith("0x") or expr.startswith("0X"): - return int(expr, 16) - - # Binary - if expr.startswith("0b") or expr.startswith("0B"): - return int(expr, 2) - - # Octal - if expr.startswith("0") and len(expr) > 1 and expr[1:].isdigit(): - return int(expr, 8) - - # Character literal - if expr.startswith("'") and expr.endswith("'"): - return ord(self._parse_string(expr)) - - # Decimal + def _define_constant( + self, name: str, expression: str, line_number: int, section: SectionData + ) -> None: + """Handle .equ/.set/'name = value'.""" + expression = _DOT_RE.sub(lambda m: self._dot_label(section), expression) try: - return int(expr) - except ValueError: - pass - - # Simple arithmetic - for op in ["+", "-", "*", "/", "<<", ">>", "|", "&", "^"]: - if op in expr: - parts = expr.rsplit(op, 1) - if len(parts) == 2: - left = self._evaluate_expr(parts[0]) - right = self._evaluate_expr(parts[1]) - if op == "+": - return left + right - if op == "-": - return left - right - if op == "*": - return left * right - if op == "/": - return left // right if right else 0 - if op == "<<": - return left << right - if op == ">>": - return left >> right - if op == "|": - return left | right - if op == "&": - return left & right - if op == "^": - return left ^ right - - raise ValueError(f"Cannot evaluate: {expr}") - - def _parse_string(self, s: str) -> str: - """Parse a string literal with escape sequences.""" - if not s: - return "" - - # Remove quotes - if (s.startswith('"') and s.endswith('"')) or (s.startswith("'") and s.endswith("'")): - s = s[1:-1] + self.constants[name] = evaluate(expression, self._constant) + except UndefinedSymbol: + # Refers to labels (e.g. "len = . - msg"); resolved after layout + self.deferred_constants[name] = (expression, line_number) + except ExpressionError as e: + self.errors.append(f"Line {line_number}: {name}: {e}") - # Process escape sequences - result = [] + def _evaluate_expr(self, expr: str) -> int: + """Evaluate an expression using the constants defined so far.""" + return evaluate(expr, self._constant) + + _STRING_ESCAPES = { + "n": 10, "t": 9, "r": 13, "\\": 92, '"': 34, "'": 39, + "a": 7, "b": 8, "f": 12, "v": 11, "e": 27, + } # fmt: skip + + def _parse_string_bytes(self, s: str) -> bytes: + """Bytes of a string literal, with GNU as escapes (\\n, \\x41, \\101, ...).""" + s = s.strip() + if len(s) < 2 or s[0] != '"' or s[-1] != '"': + raise ExpressionError(f"expected a quoted string, got {s}") + body = s[1:-1] + out = bytearray() i = 0 - while i < len(s): - if s[i] == "\\" and i + 1 < len(s): - next_char = s[i + 1] - if next_char == "n": - result.append("\n") - elif next_char == "t": - result.append("\t") - elif next_char == "r": - result.append("\r") - elif next_char == "0": - result.append("\0") - elif next_char == "\\": - result.append("\\") - elif next_char == '"': - result.append('"') - elif next_char == "'": - result.append("'") - elif next_char == "x" and i + 3 < len(s): - try: - value = int(s[i + 2 : i + 4], 16) - result.append(chr(value)) - i += 2 - except ValueError: - result.append(next_char) - else: - result.append(next_char) - i += 2 - else: - result.append(s[i]) + while i < len(body): + ch = body[i] + if ch != "\\" or i + 1 >= len(body): + out += ch.encode("utf-8") i += 1 + continue + nxt = body[i + 1] + if nxt in "01234567": + # Up to three octal digits + j = i + 1 + while j < len(body) and j < i + 4 and body[j] in "01234567": + j += 1 + out.append(int(body[i + 1 : j], 8) & 0xFF) + i = j + elif nxt in "xX": + j = i + 2 + while j < len(body) and body[j] in "0123456789abcdefABCDEF": + j += 1 + if j == i + 2: + raise ExpressionError("\\x used with no following hex digits") + out.append(int(body[i + 2 : j], 16) & 0xFF) + i = j + else: + out.append(self._STRING_ESCAPES.get(nxt, ord(nxt) & 0xFF)) + i += 2 + return bytes(out) - return "".join(result) + def _parse_string(self, s: str) -> str: + """Parse a string literal (as text, for compatibility).""" + return self._parse_string_bytes(s).decode("latin-1") def get_instructions(self, section_name: str) -> List[Tuple[int, str, Optional[str]]]: """ diff --git a/mapachespim/toolchain/expr.py b/mapachespim/toolchain/expr.py index 2145095..a57030d 100644 --- a/mapachespim/toolchain/expr.py +++ b/mapachespim/toolchain/expr.py @@ -5,7 +5,7 @@ from __future__ import annotations import re -from typing import Dict, List, Optional, Type +from typing import Callable, Dict, List, Optional, Tuple, Type class EncodeError(ValueError): @@ -13,30 +13,215 @@ class EncodeError(ValueError): _RELOC_RE = re.compile(r"^%(hi|lo|pcrel_hi|pcrel_lo)\((.*)\)$", re.IGNORECASE) -_TERM_RE = re.compile(r"\s*([+-]?)\s*([^+\-\s][^+\-]*)") + +_TOKEN_RE = re.compile( + r"""\s*(?: + (?P0[xX][0-9a-fA-F]+|0[bB][01]+|\d+) + | (?P'(?:\\.|[^\\'])'?) + | (?P%[A-Za-z_]+) + | (?P[A-Za-z_.$][\w.$]*) + | (?P<<|>>|[-+*/%&|^~()!]) + )""", + re.VERBOSE, +) + +_ESCAPES = { + "n": 10, + "t": 9, + "r": 13, + "0": 0, + "\\": 92, + "'": 39, + '"': 34, + "a": 7, + "b": 8, + "f": 12, + "v": 11, +} + + +def _number(text: str) -> int: + """Integer literal with GNU as rules: 0x hex, 0b binary, leading 0 octal.""" + lower = text.lower() + if lower.startswith("0x"): + return int(text[2:], 16) + if lower.startswith("0b"): + return int(text[2:], 2) + if len(text) > 1 and text.startswith("0"): + return int(text, 8) + return int(text, 10) + + +def _char(text: str) -> int: + """Character literal: 'A', 'A (GNU's older form), or an escape like '\\n'.""" + body = text[1:-1] if len(text) > 2 and text.endswith("'") else text[1:] + if body.startswith("\\") and len(body) == 2: + if body[1] not in _ESCAPES: + raise ValueError(f"unknown escape in character literal {text}") + return _ESCAPES[body[1]] + if len(body) != 1: + raise ValueError(f"bad character literal {text}") + return ord(body) def parse_int(text: str) -> Optional[int]: - """Parse an integer or character literal, or return None.""" + """Parse a lone integer or character literal, or return None.""" t = text.strip() - if len(t) >= 3 and t[0] == "'" and t[-1] == "'": - body = t[1:-1] - escapes = {"\\n": 10, "\\t": 9, "\\r": 13, "\\0": 0, "\\\\": 92, "\\'": 39} - if body in escapes: - return escapes[body] - if len(body) == 1: - return ord(body) - return None + sign = 1 + if t[:1] in "+-" and len(t) > 1: + sign = -1 if t[0] == "-" else 1 + t = t[1:].strip() try: - return int(t, 0) + if t.startswith("'"): + return sign * _char(t) + if re.fullmatch(r"0[xX][0-9a-fA-F]+|0[bB][01]+|\d+", t): + return sign * _number(t) except ValueError: - pass - # int(..., 0) rejects leading zeros like "010"; accept them as decimal - if re.fullmatch(r"[+-]?\d+", t): - return int(t, 10) + return None return None +class ExpressionError(ValueError): + """An expression could not be parsed or evaluated.""" + + +class UndefinedSymbol(ExpressionError): + """An expression refers to a symbol that is not (yet) defined.""" + + def __init__(self, name: str): + super().__init__(f"undefined symbol '{name}'") + self.name = name + + +def evaluate( + text: str, + resolve: Callable[[str], int], + function: Optional[Callable[[str, int], int]] = None, +) -> int: + """ + Evaluate an assembler expression with GNU as semantics. + + Supports integer and character literals, symbols (looked up with + ``resolve``, which raises UndefinedSymbol if unknown), parentheses, unary + ``- + ~ !``, and the binary operators in GNU as precedence order: + + highest: * / % << >> + | & ^ ! (a ! b is a | ~b) + lowest: + - + + Functions such as ``%hi(expr)`` are passed to ``function(name, value)``. + Division truncates toward zero, as in C. + + Raises: + ExpressionError: For malformed expressions or division by zero. + """ + tokens: List[Tuple[str, str]] = [] + pos = 0 + text = text.rstrip() + while pos < len(text): + m = _TOKEN_RE.match(text, pos) + if not m or m.end() == pos: + raise ExpressionError(f"cannot parse expression '{text.strip()}'") + kind = m.lastgroup or "" + tokens.append((kind, m.group(kind))) + pos = m.end() + if not tokens: + raise ExpressionError("missing expression") + + index = 0 + + def peek() -> Tuple[Optional[str], Optional[str]]: + return tokens[index] if index < len(tokens) else (None, None) + + def take() -> Tuple[Optional[str], Optional[str]]: + nonlocal index + tok = peek() + index += 1 + return tok + + def expect(value: str) -> None: + kind, tok = take() + if tok != value: + raise ExpressionError(f"expected '{value}' in '{text.strip()}'") + + def primary() -> int: + kind, tok = take() + if tok is None: + raise ExpressionError(f"incomplete expression '{text.strip()}'") + if kind == "num": + return _number(tok) + if kind == "char": + return _char(tok) + if kind == "ident": + return resolve(tok) + if kind == "func": + if function is None: + raise ExpressionError(f"{tok}() is not supported here") + expect("(") + value = additive() + expect(")") + return function(tok[1:].lower(), value) + if tok == "(": + value = additive() + expect(")") + return value + if tok == "-": + return -primary() + if tok == "+": + return primary() + if tok == "~": + return ~primary() + if tok == "!": + return int(primary() == 0) + raise ExpressionError(f"cannot parse expression '{text.strip()}'") + + def multiplicative() -> int: + value = primary() + while peek()[1] in ("*", "/", "%", "<<", ">>"): + op = take()[1] + rhs = primary() + if op == "*": + value *= rhs + elif op in ("/", "%"): + if rhs == 0: + raise ExpressionError("division by zero") + quotient = abs(value) // abs(rhs) * (1 if (value >= 0) == (rhs >= 0) else -1) + value = quotient if op == "/" else value - quotient * rhs + elif op == "<<": + value <<= rhs + else: + value >>= rhs + return value + + def bitwise() -> int: + value = multiplicative() + while peek()[1] in ("|", "&", "^", "!"): + op = take()[1] + rhs = multiplicative() + if op == "|": + value |= rhs + elif op == "&": + value &= rhs + elif op == "^": + value ^= rhs + else: + value |= ~rhs + return value + + def additive() -> int: + value = bitwise() + while peek()[1] in ("+", "-"): + op = take()[1] + rhs = bitwise() + value = value + rhs if op == "+" else value - rhs + return value + + result = additive() + if index != len(tokens): + raise ExpressionError(f"cannot parse expression '{text.strip()}'") + return result + + def sign_extend(value: int, bits: int) -> int: """Interpret the low ``bits`` bits of value as a signed integer.""" value &= (1 << bits) - 1 @@ -53,6 +238,7 @@ def __init__( strict: bool = True, error: Type[EncodeError] = EncodeError, lo_bits: int = 12, + hi_range: Optional[Tuple[int, int]] = None, ): self.address = address self.labels = labels @@ -61,60 +247,48 @@ def __init__( self.error = error # Width of the low part for %hi/%lo: 12 on RISC-V, 16 on MIPS self.lo_bits = lo_bits + # Values %hi accepts (RV64: those lui+addi can build, i.e. signed 32-bit) + self.hi_range = hi_range def symbol(self, name: str) -> int: + if name == ".": + return self.address if name in self.labels: return self.labels[name] if self.strict: - raise self.error(f"undefined symbol '{name}'") + raise UndefinedSymbol(name) return 0 + def _function(self, name: str, value: int) -> int: + lo_bits = self.lo_bits + hi_mask = (1 << (32 - lo_bits)) - 1 + if name == "hi": + if self.hi_range is not None and not self.hi_range[0] <= value <= self.hi_range[1]: + raise ExpressionError( + f"%hi: value 0x{value & 0xFFFFFFFFFFFFFFFF:x} is out of range " + "(on RV64, lui sign-extends; use la to load an address)" + ) + # Rounded so that (%hi << lo_bits) + sign-extended %lo == value + return ((value + (1 << (lo_bits - 1))) >> lo_bits) & hi_mask + if name == "lo": + return sign_extend(value, lo_bits) + if name == "pcrel_hi": + offset = value - self.address + return ((offset + 0x800) >> 12) & 0xFFFFF + if name == "pcrel_lo": + # %pcrel_lo(label) names the auipc instruction, which this + # encoder does not track; la/lla/lw symbol do the same job + raise ExpressionError("%pcrel_lo is not supported; use la or lla instead") + raise ExpressionError(f"unknown function %{name}") + def eval(self, text: str) -> int: """Evaluate an operand expression such as 'label+8' or '%lo(msg)'.""" - text = text.strip() - if not text: + if not text.strip(): raise self.error("missing operand") - - m = _RELOC_RE.match(text) - if m: - kind, inner = m.group(1).lower(), m.group(2) - value = self.eval(inner) - lo_bits = self.lo_bits - hi_mask = (1 << (32 - lo_bits)) - 1 - if kind == "hi": - # Rounded so that (%hi << lo_bits) + sign-extended %lo == value - return ((value + (1 << (lo_bits - 1))) >> lo_bits) & hi_mask - if kind == "lo": - return sign_extend(value, lo_bits) - if kind == "pcrel_hi": - offset = value - self.address - return ((offset + 0x800) >> 12) & 0xFFFFF - # %pcrel_lo(label) names the auipc instruction; the low part is - # relative to that instruction's pc and the auipc's target. - raise self.error("%pcrel_lo is not supported; use la or lla instead") - - literal = parse_int(text) - if literal is not None: - return literal - - total = 0 - pos = 0 - matched = False - for m2 in _TERM_RE.finditer(text): - if m2.start() != pos and text[pos : m2.start()].strip(): - break - sign, term = m2.group(1), m2.group(2).strip() - v = parse_int(term) - if v is None: - if not re.fullmatch(r"[A-Za-z_.$][\w.$]*", term): - raise self.error(f"cannot parse expression '{text}'") - v = self.symbol(term) - total += -v if sign == "-" else v - pos = m2.end() - matched = True - if not matched or text[pos:].strip(): - raise self.error(f"cannot parse expression '{text}'") - return total + try: + return evaluate(text, self.symbol, self._function) + except ExpressionError as e: + raise self.error(str(e)) def split_operands(text: str) -> List[str]: diff --git a/mapachespim/toolchain/mips.py b/mapachespim/toolchain/mips.py index f6eccfe..8f7e220 100644 --- a/mapachespim/toolchain/mips.py +++ b/mapachespim/toolchain/mips.py @@ -23,7 +23,15 @@ import re from typing import Dict, List, Tuple -from .expr import Context, EncodeError, parse_int, sign_extend, split_operands +from .expr import ( + Context, + EncodeError, + ExpressionError, + evaluate, + parse_int, + sign_extend, + split_operands, +) INSTR_SIZE = 4 NOP = 0x00000000 @@ -112,7 +120,8 @@ def _unsigned16(value: int, what: str = "immediate") -> int: } # fmt: skip # Register-form mnemonics that take an immediate third operand (like GNU as) IMM_FORMS: Dict[str, str] = { - "add": "addi", "addu": "addiu", "slt": "slti", "sltu": "sltiu", + "add": "addi", "addu": "addiu", "sub": "addi", "subu": "addiu", + "slt": "slti", "sltu": "sltiu", "and": "andi", "or": "ori", "xor": "xori", } # fmt: skip @@ -174,30 +183,56 @@ def _li(rt: int, value: int) -> List[int]: return words -def _la(rt: int, address: int) -> List[int]: - """Load an address: lui, plus ori when the low half is non-zero.""" - address &= 0xFFFFFFFF - words = [_i(0x0F, 0, rt, address >> 16)] - if address & 0xFFFF: - words.append(_i(0x0D, rt, rt, address & 0xFFFF)) - return words - - def _hi_lo(address: int) -> Tuple[int, int]: - """Split an address for lui + signed 16-bit offset.""" + """Split an address into %hi (for lui) and %lo (a signed 16-bit offset).""" lo = sign_extend(address, 16) hi = ((address - lo) >> 16) & 0xFFFF return hi, lo +def _uses_symbols(text: str) -> bool: + """True if an operand expression refers to any symbol (label or constant).""" + found = False + + def resolve(name: str) -> int: + nonlocal found + found = True + return 0 + + try: + evaluate(text, resolve, lambda name, value: value) + except ExpressionError: + return True + return found + + +def _move(rd: int, rs: int) -> int: + return _r(rs, 0, rd, 0, ALU3["or"]) # move rd, rs = or rd, rs, $zero + + +def _neg(rd: int, rs: int) -> int: + return _r(0, rs, rd, 0, ALU3["sub"]) # neg rd, rs = sub rd, $zero, rs + + +def _break(code1: int = 0, code2: int = 0) -> int: + return (code1 << 16) | (code2 << 6) | 0x0D + + # --------------------------------------------------------------------------- # Encoder # --------------------------------------------------------------------------- -def _encode(mnemonic: str, ops: List[str], ctx: Context) -> List[int]: +def _encode(mnemonic: str, ops: List[str], ctx: Context, reorder: bool = True) -> List[int]: + """Encode one instruction or pseudo-instruction as a list of words. + + With ``reorder`` (GNU as's default ".set reorder"), a nop is placed in + the delay slot of each branch and jump written in the source. With + ".set noreorder", the programmer fills delay slots themselves. + """ m = mnemonic pc = ctx.address + slot = [NOP] if reorder else [] def branch_offset(text: str, at: int) -> int: """Word offset from the delay slot of a branch at address `at`.""" @@ -220,31 +255,53 @@ def jump_target(text: str) -> int: raise MIPSEncodeError(f"jump target 0x{target:x} is outside the current 256MB region") return (target >> 2) & 0x03FFFFFF + def is_reg_or_bare_name(text: str) -> bool: + t = text.strip() + return t.startswith("$") or (t.lower() in REGISTERS and not t.isdigit()) + + def value_of(text: str) -> int: + return ctx.eval(text) & 0xFFFFFFFF + # --- no-operand --- - if m in ("nop", "syscall", "break", "eret"): - if m == "nop": - return [NOP] - if m == "eret": - return [0x42000018] - code = 0 - if ops: - _expect(ops, 1, f"{m} [code]") - code = ctx.eval(ops[0]) & 0xFFFFF - return [(code << 6) | (0x0C if m == "syscall" else 0x0D)] - - # --- R-type --- + if m == "nop": + _expect(ops, 0, "nop") + return [NOP] + if m == "eret": + return [0x42000018] + if m == "syscall": + code = ctx.eval(ops[0]) & 0xFFFFF if ops else 0 + if len(ops) > 1: + raise MIPSEncodeError("expected syscall [code]") + return [(code << 6) | 0x0C] + if m == "break": + if len(ops) > 2: + raise MIPSEncodeError("expected break [code[, code]]") + codes = [ctx.eval(o) & 0x3FF for o in ops] + return [_break(*codes)] + + # --- three-register ALU (also accepting an immediate, like GNU as) --- if m in ALU3: _expect(ops, 3, f"{m} $rd, $rs, $rt") - bare_register_name = ops[2].lower() in REGISTERS and not ops[2].isdigit() - if m in IMM_FORMS and not ops[2].startswith("$") and not bare_register_name: - return _encode(IMM_FORMS[m], ops, ctx) + if m in IMM_FORMS and not is_reg_or_bare_name(ops[2]): + rd, rs = parse_register(ops[0]), parse_register(ops[1]) + imm_op = IMM_FORMS[m] + opcode, is_signed = ALU_IMM[imm_op] + # Constants are 32-bit: 0xffffffff means -1 to a signed field + imm = sign_extend(value_of(ops[2]), 32) if is_signed else value_of(ops[2]) + if m in ("sub", "subu"): + imm = -imm # subtracting is adding the negated value + fits = -0x8000 <= imm <= 0x7FFF if is_signed else 0 <= imm <= 0xFFFF + if fits: + return [_i(opcode, rs, rd, imm)] + # Too large for the immediate field: build it in $at + return _li(AT, ctx.eval(ops[2])) + [_r(rs, AT, rd, 0, ALU3[m])] rd, rs, rt = (parse_register(o) for o in ops) return [_r(rs, rt, rd, 0, ALU3[m])] if m in SHIFT_IMM: _expect(ops, 3, f"{m} $rd, $rt, shamt") if _is_register(ops[2]): - return _encode(m + "v", ops, ctx) + return _encode(m + "v", ops, ctx, reorder) shamt = ctx.eval(ops[2]) if not 0 <= shamt <= 31: raise MIPSEncodeError(f"shift amount {shamt} out of range (0 to 31)") @@ -255,25 +312,43 @@ def jump_target(text: str) -> int: rd, rt, rs = (parse_register(o) for o in ops) return [_r(rs, rt, rd, 0, SHIFT_VAR[m])] - if m in MULDIV: - if len(ops) == 3 and m in ("div", "divu"): - # Three-operand pseudo form: quotient into rd - rd = parse_register(ops[0]) - return [ - _r(parse_register(ops[1]), parse_register(ops[2]), 0, 0, MULDIV[m]), - _r(0, 0, rd, 0, 0x12), - ] - _expect(ops, 2, f"{m} $rs, $rt") + # --- multiply and divide --- + if m in MULDIV and len(ops) == 2: + # The real instruction; results in HI/LO. (SPIM semantics: GNU as + # instead expands 'div $s, $t' to 'div $s, $s, $t'.) return [_r(parse_register(ops[0]), parse_register(ops[1]), 0, 0, MULDIV[m])] - if m in ("rem", "remu"): + if m in ("div", "divu", "rem", "remu"): _expect(ops, 3, f"{m} $rd, $rs, $rt") - rd = parse_register(ops[0]) - funct = MULDIV["div" if m == "rem" else "divu"] - return [ - _r(parse_register(ops[1]), parse_register(ops[2]), 0, 0, funct), - _r(0, 0, rd, 0, 0x10), - ] + rd, rs = parse_register(ops[0]), parse_register(ops[1]) + signed_op = m in ("div", "rem") + funct = MULDIV["div" if signed_op else "divu"] + result = HILO_TO["mflo" if m in ("div", "divu") else "mfhi"] + if _is_register(ops[2]): + rt = parse_register(ops[2]) + if rd == 0 and m in ("div", "divu"): + return [_r(rs, rt, 0, 0, funct)] # 'div $zero, $s, $t' is the real instruction + # Trap on division by zero (break 7), and for signed division on + # overflow of -2^31 / -1 (break 6), as GNU as and SPIM do + words = [_i(0x05, rt, 0, 2), _r(rs, rt, 0, 0, funct), _break(7)] + if signed_op: + words += [ + _i(0x09, 0, AT, -1), # li $at, -1 + _i(0x05, rt, AT, 4), # bne $rt, $at, done + _i(0x0F, 0, AT, 0x8000), # lui $at, 0x8000 (delay slot) + _i(0x05, rs, AT, 2), # bne $rs, $at, done + NOP, + _break(6), + ] + return words + [_r(0, 0, rd, 0, result)] + divisor = sign_extend(value_of(ops[2]), 32) + if divisor == 0: + return [_break(7)] + if divisor == 1 or (divisor == -1 and signed_op): + if m in ("rem", "remu"): + return [_move(rd, 0)] # the remainder is always 0 + return [_move(rd, rs)] if divisor == 1 else [_neg(rd, rs)] + return _li(AT, divisor) + [_r(rs, AT, 0, 0, funct), _r(0, 0, rd, 0, result)] if m in HILO_TO: _expect(ops, 1, f"{m} $rd") @@ -288,8 +363,11 @@ def jump_target(text: str) -> int: rd, rs = parse_register(ops[0]), parse_register(ops[1]) if _is_register(ops[2]): return [_r(rs, parse_register(ops[2]), rd, 0, 0x02, opcode=0x1C)] - # mul by constant: load it into $at first - return _li(AT, ctx.eval(ops[2])) + [_r(rs, AT, rd, 0, 0x02, opcode=0x1C)] + # By a constant: li $at, value; mult $rs, $at; mflo $rd + return _li(AT, ctx.eval(ops[2])) + [ + _r(rs, AT, 0, 0, MULDIV["mult"]), + _r(0, 0, rd, 0, HILO_TO["mflo"]), + ] if m in SPECIAL2_ACC: _expect(ops, 2, f"{m} $rs, $rt") @@ -302,10 +380,10 @@ def jump_target(text: str) -> int: rd, rs = parse_register(ops[0]), parse_register(ops[1]) return [_r(rs, rd, rd, 0, 0x20 if m == "clz" else 0x21, opcode=0x1C)] - # --- jumps (each followed by a delay-slot nop) --- + # --- jumps --- if m == "jr": _expect(ops, 1, "jr $rs") - return [_r(parse_register(ops[0]), 0, 0, 0, 0x08), NOP] + return [_r(parse_register(ops[0]), 0, 0, 0, 0x08)] + slot if m == "jalr": if len(ops) == 1: @@ -313,65 +391,153 @@ def jump_target(text: str) -> int: else: _expect(ops, 2, "jalr [$rd,] $rs") rd, rs = parse_register(ops[0]), parse_register(ops[1]) - return [_r(rs, 0, rd, 0, 0x09), NOP] + if rd == rs: + raise MIPSEncodeError( + "jalr: source and destination must be different registers" + + (" (the destination defaults to $ra)" if len(ops) == 1 else "") + ) + return [_r(rs, 0, rd, 0, 0x09)] + slot if m in ("j", "jal"): _expect(ops, 1, f"{m} label") if m == "j" and _is_register(ops[0]): - return _encode("jr", ops, ctx) + return _encode("jr", ops, ctx, reorder) opcode = 0x02 if m == "j" else 0x03 - return [(opcode << 26) | jump_target(ops[0]), NOP] + return [(opcode << 26) | jump_target(ops[0])] + slot # --- branches --- + def compare_branch(op: str, a: int, b_text: str, target: str) -> List[int]: + """blt/bge/bgt/ble[u] a, b, target, following GNU as's macro rules + (tc-mips.c M_BLT, M_BLTI, ...), including its special cases for 0, + 1, and the extreme values.""" + unsigned = op.endswith("u") + kind = op[:3] # blt, bge, bgt, ble + bne, beq = BRANCH2["bne"], BRANCH2["beq"] + sl = "sltu" if unsigned else "slt" + + if _is_register(b_text): + b = parse_register(b_text) + if kind in ("blt", "bge"): # a < b, a >= b + taken_if_less = kind == "blt" + if b == 0: + if unsigned: + return never() if taken_if_less else always(target) + return [_i(0x01, a, 0 if taken_if_less else 1, off(target))] + slot # bltz/bgez + if a == 0: + if unsigned: + return [_i(bne if taken_if_less else beq, 0, b, off(target))] + slot + return [ + _i(0x07 if taken_if_less else 0x06, b, 0, off(target)) + ] + slot # bgtz/blez + words = [_r(a, b, AT, 0, ALU3[sl])] + opcode = bne if taken_if_less else beq + else: # a > b, a <= b + taken_if_greater = kind == "bgt" + if b == 0: + if unsigned: + return [_i(bne if taken_if_greater else beq, a, 0, off(target))] + slot + return [_i(0x07 if taken_if_greater else 0x06, a, 0, off(target))] + slot + if a == 0: + if unsigned: + return never() if taken_if_greater else always(target) + return [ + _i(0x01, b, 0 if taken_if_greater else 1, off(target)) + ] + slot # bltz/bgez + words = [_r(b, a, AT, 0, ALU3[sl])] + opcode = bne if taken_if_greater else beq + return words + [_i(opcode, AT, 0, offset_after(words, target))] + slot + + # Immediate operand (compared as a 32-bit value) + v = sign_extend(value_of(b_text), 32) + if kind == "bgt": # a > v == a >= v+1 + if (unsigned and (a == 0 or v == -1)) or (not unsigned and v >= 0x7FFFFFFF): + return never() + v, kind = v + 1, "bge" + elif kind == "ble": # a <= v == a < v+1 + if (unsigned and (a == 0 or v == -1)) or (not unsigned and v >= 0x7FFFFFFF): + return always(target) + v, kind = v + 1, "blt" + + if kind == "blt": + if unsigned and v == 0: + return never() + if unsigned and v == 1: + return [_i(beq, a, 0, off(target))] + slot # a < 1 == a == 0 + if not unsigned and v in (0, 1): + return [ + _i(0x01, a, 0, off(target)) if v == 0 else _i(0x06, a, 0, off(target)) + ] + slot + opcode = bne + else: # bge + if unsigned and v == 0 or (not unsigned and v == -0x80000000): + return always(target) + if unsigned and v == 1: + return [_i(bne, a, 0, off(target))] + slot # a >= 1 == a != 0 + if not unsigned and v in (0, 1): + return [ + _i(0x01, a, 1, off(target)) if v == 0 else _i(0x07, a, 0, off(target)) + ] + slot + opcode = beq + # $at = (a < v) + if -0x8000 <= v <= 0x7FFF: + words = [_i(0x0B if unsigned else 0x0A, a, AT, v)] # slti[u] $at, a, v + else: + words = _li(AT, v) + [_r(a, AT, AT, 0, ALU3[sl])] + return words + [_i(opcode, AT, 0, offset_after(words, target))] + slot + + def off(target: str) -> int: + return branch_offset(target, pc) + + def offset_after(words: List[int], target: str) -> int: + return branch_offset(target, pc + 4 * len(words)) + + def always(target: str) -> List[int]: + return [_i(0x04, 0, 0, branch_offset(target, pc))] + slot # b target + + def never() -> List[int]: + return [NOP] # a branch that is never taken (GNU emits a nop) + if m in BRANCH2: _expect(ops, 3, f"{m} $rs, $rt, label") rs = parse_register(ops[0]) if _is_register(ops[1]): - return [_i(BRANCH2[m], rs, parse_register(ops[1]), branch_offset(ops[2], pc)), NOP] - # Compare against a constant via $at - words = _li(AT, ctx.eval(ops[1])) - at = pc + 4 * len(words) - return words + [_i(BRANCH2[m], rs, AT, branch_offset(ops[2], at)), NOP] + return [_i(BRANCH2[m], rs, parse_register(ops[1]), branch_offset(ops[2], pc))] + slot + value = value_of(ops[1]) + if value == 0: + return [_i(BRANCH2[m], rs, 0, branch_offset(ops[2], pc))] + slot + words = _li(AT, value) + return words + [_i(BRANCH2[m], rs, AT, offset_after(words, ops[2]))] + slot if m in BRANCH1: _expect(ops, 2, f"{m} $rs, label") opcode, rt = BRANCH1[m] - return [_i(opcode, parse_register(ops[0]), rt, branch_offset(ops[1], pc)), NOP] + rs = parse_register(ops[0]) + if m in ("bltzal", "bgezal") and rs == 31: + raise MIPSEncodeError(f"{m}: the source register must not be $ra (it is overwritten)") + return [_i(opcode, rs, rt, branch_offset(ops[1], pc))] + slot if m in ("beqz", "bnez"): _expect(ops, 2, f"{m} $rs, label") opcode = BRANCH2["beq" if m == "beqz" else "bne"] - return [_i(opcode, parse_register(ops[0]), 0, branch_offset(ops[1], pc)), NOP] + return [_i(opcode, parse_register(ops[0]), 0, branch_offset(ops[1], pc))] + slot if m in ("b", "bal"): _expect(ops, 1, f"{m} label") if m == "b": - return [_i(0x04, 0, 0, branch_offset(ops[0], pc)), NOP] # beq $0, $0 - return [_i(0x01, 0, 0x11, branch_offset(ops[0], pc)), NOP] # bgezal $0 + return always(ops[0]) + return [_i(0x01, 0, 0x11, branch_offset(ops[0], pc))] + slot # bgezal $0 if m in COMPARE_BRANCHES: _expect(ops, 3, f"{m} $rs, $rt, label") - slt, swap, if_set = COMPARE_BRANCHES[m] - rs = parse_register(ops[0]) - words = [] - if _is_register(ops[1]): - rt = parse_register(ops[1]) - else: - words = _li(AT, ctx.eval(ops[1])) - rt = AT - a, b = (rt, rs) if swap else (rs, rt) - words.append(_r(a, b, AT, 0, ALU3[slt])) - at = pc + 4 * len(words) - opcode = BRANCH2["bne" if if_set else "beq"] - return words + [_i(opcode, AT, 0, branch_offset(ops[2], at)), NOP] + return compare_branch(m, parse_register(ops[0]), ops[1], ops[2]) # --- immediates --- if m in ALU_IMM: _expect(ops, 3, f"{m} $rt, $rs, imm") - opcode, signed = ALU_IMM[m] + opcode, is_signed = ALU_IMM[m] rt, rs = parse_register(ops[0]), parse_register(ops[1]) imm = ctx.eval(ops[2]) - imm = _signed16(imm) if signed else _unsigned16(imm) + imm = _signed16(imm) if is_signed else _unsigned16(imm) return [_i(opcode, rs, rt, imm)] if m == "lui": @@ -386,17 +552,35 @@ def jump_target(text: str) -> int: opcode = LOADS[m] if m in LOADS else STORES[m] _expect(ops, 2, f"{m} $rt, offset($rs)") rt = parse_register(ops[0]) - if _MEM_RE.match(ops[1].strip()): - offset, base = _parse_mem(ops[1], ctx) - return [_i(opcode, base, rt, offset)] - # 'lw $t0, label' -> lui $at, %hi(label); lw $t0, %lo(label)($at) - hi, lo = _hi_lo(ctx.eval(ops[1])) - return [_i(0x0F, 0, AT, hi), _i(opcode, AT, rt, lo)] + operand = ops[1].strip() + match = _MEM_RE.match(operand) + if match: + offset_text, base = match.group(1).strip(), parse_register(match.group(2)) + else: + offset_text, base = operand, 0 + offset = ctx.eval(offset_text) if offset_text else 0 + # A plain offset that fits (or an explicit %lo(...)) is one instruction + if ( + not offset_text + or offset_text.startswith("%") + or (not _uses_symbols(offset_text) and -0x8000 <= offset <= 0x7FFF) + ): + return [_i(opcode, base, rt, _signed16(offset, "offset"))] + # Otherwise: lui tmp, %hi(addr); [addu tmp, tmp, base]; op rt, %lo(addr)(tmp) + # A load can use its own destination as tmp; stores and the partial + # loads lwl/lwr must use $at + partial = m in ("lwl", "lwr") + tmp = rt if (m in LOADS and not partial and rt not in (0, base)) else AT + hi, lo = _hi_lo(offset) + words = [_i(0x0F, 0, tmp, hi)] + if base: + words.append(_r(tmp, base, tmp, 0, ALU3["addu"])) + return words + [_i(opcode, tmp, rt, lo)] # --- data movement pseudo-instructions --- if m == "move": _expect(ops, 2, "move $rd, $rs") - return [_r(parse_register(ops[1]), 0, parse_register(ops[0]), 0, ALU3["or"])] + return [_move(parse_register(ops[0]), parse_register(ops[1]))] if m == "li": _expect(ops, 2, "li $rt, imm") @@ -404,7 +588,11 @@ def jump_target(text: str) -> int: if m == "la": _expect(ops, 2, "la $rt, label") - return _la(parse_register(ops[0]), ctx.eval(ops[1])) + rt = parse_register(ops[0]) + if not _uses_symbols(ops[1]): + return _li(rt, ctx.eval(ops[1])) # a plain number: same as li + hi, lo = _hi_lo(ctx.eval(ops[1])) + return [_i(0x0F, 0, rt, hi), _i(0x09, rt, rt, lo)] # lui; addiu if m == "not": _expect(ops, 2, "not $rd, $rs") @@ -418,12 +606,9 @@ def jump_target(text: str) -> int: if m == "abs": _expect(ops, 2, "abs $rd, $rs") rd, rs = parse_register(ops[0]), parse_register(ops[1]) - # sra $at, rs, 31; xor rd, rs, $at; subu rd, rd, $at - return [ - _r(0, rs, AT, 31, 0x03), - _r(rs, AT, rd, 0, ALU3["xor"]), - _r(rd, AT, rd, 0, ALU3["subu"]), - ] + # bgez $rs, done; move $rd, $rs (delay slot); neg $rd, $rs; done: + delay = NOP if rd == rs else _move(rd, rs) + return [_i(0x01, rs, 1, 2), delay, _neg(rd, rs)] raise MIPSEncodeError(f"unknown instruction '{mnemonic}'") @@ -439,7 +624,7 @@ def _context(address: int, labels: Dict[str, int], strict: bool) -> Context: return Context(address, labels, strict=strict, error=MIPSEncodeError, lo_bits=16) -def encode(text: str, address: int, labels: Dict[str, int]) -> bytes: +def encode(text: str, address: int, labels: Dict[str, int], reorder: bool = True) -> bytes: """ Encode one MIPS instruction (or pseudo-instruction) to big-endian machine code. @@ -449,11 +634,13 @@ def encode(text: str, address: int, labels: Dict[str, int]) -> bytes: MIPSEncodeError: If the instruction is invalid. """ mnemonic, ops = _split_instruction(text) - words = _encode(mnemonic, ops, _context(address, labels, strict=True)) + words = _encode(mnemonic, ops, _context(address, labels, strict=True), reorder) return b"".join(w.to_bytes(4, "big") for w in words) -def instruction_size(text: str, labels: Dict[str, int], address: int = 0) -> int: +def instruction_size( + text: str, labels: Dict[str, int], address: int = 0, reorder: bool = True +) -> int: """ Size in bytes that ``encode`` will produce, for the label-layout pass. @@ -463,7 +650,7 @@ def instruction_size(text: str, labels: Dict[str, int], address: int = 0) -> int """ mnemonic, ops = _split_instruction(text) try: - words = _encode(mnemonic, ops, _context(address, labels, strict=False)) + words = _encode(mnemonic, ops, _context(address, labels, strict=False), reorder) except MIPSEncodeError: return INSTR_SIZE # the real error is reported by encode() return len(words) * INSTR_SIZE diff --git a/mapachespim/toolchain/riscv.py b/mapachespim/toolchain/riscv.py index 025aa8a..1c6a23f 100644 --- a/mapachespim/toolchain/riscv.py +++ b/mapachespim/toolchain/riscv.py @@ -24,12 +24,33 @@ from __future__ import annotations import re -from typing import Dict, List, Tuple - -from .expr import Context, EncodeError, parse_int, sign_extend, split_operands +from typing import Callable, Dict, List, Tuple + +from .expr import ( + Context, + EncodeError, + ExpressionError, + UndefinedSymbol, + evaluate, + parse_int, + sign_extend, + split_operands, +) INSTR_SIZE = 4 +# %hi() operands must be buildable with lui+addi, i.e. signed 32-bit on RV64 +_HI_RANGE = (-(1 << 31), (1 << 31) - 1) + + +def _known(labels: Dict[str, int]) -> Callable[[str], int]: + def resolve(name: str) -> int: + if name in labels: + return labels[name] + raise UndefinedSymbol(name) + + return resolve + class RISCVEncodeError(EncodeError): """Raised when a RISC-V instruction cannot be encoded.""" @@ -237,28 +258,39 @@ def _parse_csr(text: str, ctx: Context) -> int: def _li_sequence(rd: int, value: int) -> List[int]: - """Instruction words that load a 64-bit constant into rd.""" + """Instruction words that load a 64-bit constant into rd (same as GNU as).""" value = sign_extend(value, 64) if -2048 <= value <= 2047: - return [_i(0x13, 0, rd, 0, value)] # addi rd, x0, value - if -(1 << 31) <= value < (1 << 31): - lo = sign_extend(value & 0xFFF, 12) - hi = ((value - lo) >> 12) & 0xFFFFF - words = [_u(0x37, rd, hi)] # lui - if lo: - words.append(_i(0x1B, 0, rd, rd, lo)) # addiw keeps it 32-bit sign-extended + return [_i(0x13, 0, rd, 0, value)] # li is an alias of addi rd, zero, value + return _load_const(rd, value) + + +def _load_const(rd: int, value: int) -> List[int]: + """Port of load_const() from GNU as (binutils gas/config/tc-riscv.c).""" + lower = sign_extend(value, 12) + # GNU computes this in 64-bit arithmetic, which can wrap (e.g. for + # 0x7fffffffffffffff), and later shifts it as a signed int64 + upper = sign_extend(value - lower, 64) + + if not -(1 << 31) <= value < (1 << 31): + # Reduce to a signed 32-bit constant using SLLI and ADDI + shift = 12 + while not (upper >> shift) & 1: + shift += 1 + words = _load_const(rd, upper >> shift) + words.append((shift << 20) | (rd << 15) | (1 << 12) | (rd << 7) | 0x13) # slli + if lower: + words.append(_i(0x13, 0, rd, rd, lower)) # addi return words - # 64-bit: build the upper bits recursively, then shift in 12 bits at a time - lo = sign_extend(value & 0xFFF, 12) - upper = (value - lo) >> 12 - shift = 12 - while upper and not upper & 1 and shift < 60: - upper >>= 1 - shift += 1 - words = _li_sequence(rd, upper) - words.append(((shift & 0x3F) << 20) | (rd << 15) | (1 << 12) | (rd << 7) | 0x13) # slli - if lo: - words.append(_i(0x13, 0, rd, rd, lo)) + + # LUI and/or ADDIW build a sign-extended 32-bit constant + words = [] + hi_reg = 0 + if upper: + words.append(_u(0x37, rd, (upper & 0xFFFFFFFF) >> 12)) # lui + hi_reg = rd + if lower or not hi_reg: + words.append(_i(0x1B, 0, rd, hi_reg, lower)) # addiw return words @@ -285,10 +317,27 @@ def target_offset(text: str) -> int: if value is not None: return value if not ctx.strict: - ctx.eval(text) # still validate the expression's syntax - return 0 # sizing pass: labels may not be resolved yet + # Sizing pass: use labels placed so far; others count as in range + try: + return evaluate(text, _known(ctx.labels), ctx._function) - pc + except UndefinedSymbol: + return 0 + except ExpressionError as e: + raise RISCVEncodeError(str(e)) return ctx.eval(text) - pc + def branch(funct3: int, rs1: int, rs2: int, target: str) -> List[int]: + """A conditional branch, relaxed like GNU as when out of range. + + A target beyond +/-4KB becomes the inverted branch over a jal: + beq a0, a1, far -> bne a0, a1, 8 + jal zero, far + """ + offset = target_offset(target) + if -4096 <= offset < 4096 or parse_int(target) is not None: + return [_b(funct3, rs1, rs2, offset)] + return [_b(funct3 ^ 1, rs1, rs2, 8), _j(0, offset - 4)] + if m in R_TYPE: _expect(ops, 3, f"{m} rd, rs1, rs2") if m in IMM_FORMS and ops[2].strip().lower() not in REGISTERS: @@ -353,28 +402,19 @@ def target_offset(text: str) -> int: if m in BRANCHES: _expect(ops, 3, f"{m} rs1, rs2, label") - return [ - _b(BRANCHES[m], parse_register(ops[0]), parse_register(ops[1]), target_offset(ops[2])) - ] + return branch(BRANCHES[m], parse_register(ops[0]), parse_register(ops[1]), ops[2]) if m in SWAP_BRANCHES: _expect(ops, 3, f"{m} rs1, rs2, label") real = SWAP_BRANCHES[m] - return [ - _b( - BRANCHES[real], - parse_register(ops[1]), - parse_register(ops[0]), - target_offset(ops[2]), - ) - ] + return branch(BRANCHES[real], parse_register(ops[1]), parse_register(ops[0]), ops[2]) if m in ZERO_BRANCHES: _expect(ops, 2, f"{m} rs, label") real, reg_second = ZERO_BRANCHES[m] rs = parse_register(ops[0]) rs1, rs2 = (0, rs) if reg_second else (rs, 0) - return [_b(BRANCHES[real], rs1, rs2, target_offset(ops[1]))] + return branch(BRANCHES[real], rs1, rs2, ops[1]) if m == "lui": _expect(ops, 2, "lui rd, imm") @@ -483,14 +523,18 @@ def target_offset(text: str) -> int: return [_i(0x67, 0, 0, parse_register(ops[0]), 0)] if m in ("call", "tail"): - _expect(ops, 1, f"{m} label") - link = 1 if m == "call" else 0 - offset = target_offset(ops[0]) - if -(1 << 20) <= offset < (1 << 20) and m == "call": - return [_j(link, offset)] - # Far call/tail: auipc t1 (or ra), then jalr - scratch = 1 if m == "call" else 6 - hi, lo = _pcrel_pair(pc + offset, pc) + # As in GNU as, always auipc + jalr so any address is reachable: + # call [rd,] label -> auipc rd, hi; jalr rd, lo(rd) (rd = ra) + # tail label -> auipc t1, hi; jalr zero, lo(t1) + if m == "call" and len(ops) == 2: + link = scratch = parse_register(ops[0]) + target = ops[1] + else: + _expect(ops, 1, f"{m} label") + link = 1 if m == "call" else 0 + scratch = 1 if m == "call" else 6 + target = ops[0] + hi, lo = _pcrel_pair(pc + target_offset(target), pc) return [_u(0x17, scratch, hi), _i(0x67, 0, link, scratch, lo)] if m in ("csrr", "csrw", "csrs", "csrc"): @@ -532,7 +576,11 @@ def encode(text: str, address: int, labels: Dict[str, int]) -> bytes: RISCVEncodeError: If the instruction is invalid. """ mnemonic, ops = _split_instruction(text) - words = _encode(mnemonic, ops, Context(address, labels, strict=True, error=RISCVEncodeError)) + words = _encode( + mnemonic, + ops, + Context(address, labels, strict=True, error=RISCVEncodeError, hi_range=_HI_RANGE), + ) return b"".join(w.to_bytes(4, "little") for w in words) @@ -547,15 +595,11 @@ def instruction_size(text: str, labels: Dict[str, int], address: int = 0) -> int targets, handled here). """ mnemonic, ops = _split_instruction(text) - if mnemonic == "call": - # Near calls are a single jal; programs large enough to need a far - # call do not fit in the simulator's default memory anyway. - return INSTR_SIZE - if mnemonic == "tail": - return 2 * INSTR_SIZE try: words = _encode( - mnemonic, ops, Context(address, labels, strict=False, error=RISCVEncodeError) + mnemonic, + ops, + Context(address, labels, strict=False, error=RISCVEncodeError, hi_range=_HI_RANGE), ) except RISCVEncodeError: return INSTR_SIZE # the real error is reported by encode() diff --git a/mapachespim/unicorn_backend.py b/mapachespim/unicorn_backend.py index 03c2738..e1f5cab 100644 --- a/mapachespim/unicorn_backend.py +++ b/mapachespim/unicorn_backend.py @@ -824,7 +824,7 @@ def step(self) -> StepResult: except UcError as e: # Prefer the specific description recorded by the unmapped-memory # hook (it includes the faulting address) over Unicorn's message. - detail = self._last_error or _describe_uc_error(e) + detail = self._last_error or self._describe_trap(pc) or _describe_uc_error(e) self._last_error = f"{detail} (at PC=0x{pc:x})" # Unicorn may leave PC anywhere after a fault; keep it on the # faulting instruction so the user can inspect it. @@ -841,6 +841,35 @@ def step(self) -> StepResult: return StepResult.OK + def _describe_trap(self, pc: int) -> Optional[str]: + """Explain a trap raised by a breakpoint instruction at ``pc``, if any. + + Assemblers use these for runtime checks: MIPS div/rem expand to + code that executes 'break 7' on division by zero and 'break 6' on + overflow, as SPIM and GNU as do. + """ + try: + word = self.read_mem(pc, 4) + except RuntimeError: + return None + if self._isa == ISA.MIPS: + instr = int.from_bytes(word, "big") + if instr & 0xFC00003F == 0x0000000D: # break + code = (instr >> 16) & 0x3FF + if code == 7: + return "Division by zero" + if code == 6: + return "Integer overflow in division (-2147483648 / -1)" + return f"Break instruction executed (break {code})" + elif self._isa == ISA.RISCV: + if int.from_bytes(word, "little") == 0x00100073: + return "ebreak instruction executed" + elif self._isa == ISA.ARM: + instr = int.from_bytes(word, "little") + if instr & 0xFFE0001F == 0xD4200000: # brk #imm + return f"brk instruction executed (brk #{(instr >> 5) & 0xFFFF})" + return None + def run(self, max_steps: Optional[int] = None) -> int: """ Run until halt, syscall exit, or max_steps reached diff --git a/tests/README.md b/tests/README.md index 4a17f9f..142f601 100644 --- a/tests/README.md +++ b/tests/README.md @@ -14,6 +14,16 @@ Tests that assemble ARM64 or x86-64 code need the optional Keystone library; whe installed (e.g. Apple Silicon), they are reported as skipped rather than failed. Everything else, including all RISC-V and MIPS assembler tests, runs on every platform. +**Assembler differential tests** compare the built-in RISC-V and MIPS assemblers with GNU `as` +byte for byte, using randomized programs. They need the GNU cross binutils and are skipped without +them: + +```bash +sudo apt install binutils-riscv64-linux-gnu binutils-mips-linux-gnu +python -m pytest tests/test_differential_*.py +MAPACHESPIM_FUZZ_SCALE=50 python -m pytest tests/test_differential_*.py # much deeper +``` + **Run individual suites:** ```bash diff --git a/tests/gnu_reference.py b/tests/gnu_reference.py new file mode 100644 index 0000000..e4f5fb9 --- /dev/null +++ b/tests/gnu_reference.py @@ -0,0 +1,168 @@ +""" +Reference assembly with GNU binutils, for differential testing. + +The built-in assemblers promise byte-for-byte the same machine code as GNU as +(with the options below) for every instruction they accept. These helpers +assemble a program with the GNU cross toolchain, link it at MapacheSPIM's +memory layout, and return the .text bytes so tests can compare. + +The GNU tools are optional: tests using them are skipped when they are not +installed. On Debian/Ubuntu: + + sudo apt install binutils-riscv64-linux-gnu binutils-mips-linux-gnu \\ + binutils-aarch64-linux-gnu binutils-x86-64-linux-gnu +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from elftools.elf.elffile import ELFFile + +from mapachespim.memory_map import get_layout + + +@dataclass(frozen=True) +class GnuTarget: + prefix: str + as_flags: Tuple[str, ...] + prologue: str # directives that select MapacheSPIM's conventions + + +TARGETS: Dict[str, GnuTarget] = { + "riscv64": GnuTarget( + "riscv64-linux-gnu-", + ("-march=rv64im", "-mabi=lp64", "-mno-relax"), + # No compressed instructions, no linker relaxation, absolute code + ".option norvc\n.option norelax\n.option nopic\n", + ), + "mips32": GnuTarget( + "mips-linux-gnu-", + ("-march=mips32", "-EB", "-mno-shared", "-O0", "-32"), + # Delay slots filled with nop (reorder mode), no optimization + ".set reorder\n", + ), +} + +# Number of random instances per instruction form; raise for deeper runs: +# MAPACHESPIM_FUZZ_SCALE=20 python -m pytest tests/test_differential_*.py +FUZZ_SCALE = max(1, int(os.environ.get("MAPACHESPIM_FUZZ_SCALE", "1"))) + + +def gnu_available(isa: str) -> bool: + target = TARGETS[isa] + return all(shutil.which(target.prefix + tool) for tool in ("as", "ld")) + + +def _linker_script(isa: str) -> str: + layout = get_layout(isa) + return f""" +SECTIONS {{ + . = 0x{layout.text_base:x}; + .text : {{ *(.text) }} + . = 0x{layout.rodata_base:x}; + .rodata : {{ *(.rodata) }} + . = 0x{layout.data_base:x}; + .data : {{ *(.data) }} + . = 0x{layout.bss_base:x}; + .bss : {{ *(.bss) }} + /* MapacheSPIM places other sections 64KB after .bss */ + . = 0x{layout.bss_base + 0x10000:x}; + .tohost : {{ *(.tohost) }} +}} +""" + + +class GnuError(Exception): + """GNU as or ld rejected the input.""" + + +def gnu_assemble(isa: str, source: str, section: str = ".text") -> bytes: + """Assemble and link `source` with GNU tools; return a section's bytes.""" + target = TARGETS[isa] + with tempfile.TemporaryDirectory() as tmp: + tmpdir = Path(tmp) + (tmpdir / "prog.s").write_text(target.prologue + source + "\n") + (tmpdir / "link.ld").write_text(_linker_script(isa)) + steps = [ + [target.prefix + "as", *target.as_flags, "prog.s", "-o", "prog.o"], + [target.prefix + "ld", "-T", "link.ld", "prog.o", "-o", "prog", "-e", "0"], + ] + for cmd in steps: + result = subprocess.run(cmd, cwd=tmpdir, capture_output=True, text=True) + if result.returncode != 0: + raise GnuError(result.stderr.strip()) + with open(tmpdir / "prog", "rb") as f: + found = ELFFile(f).get_section_by_name(section) + return found.data() if found is not None else b"" + + +def gnu_accepts(isa: str, source: str) -> bool: + try: + gnu_assemble(isa, source) + return True + except GnuError: + return False + + +RISCV_NOP = (0x00000013).to_bytes(4, "little") + + +def trim_padding(gnu: bytes, ours: bytes) -> bytes: + """Drop the padding GNU as adds at the end of a section to reach its + alignment: zeros, or nop instructions in RISC-V code.""" + tail = gnu[len(ours) :] + if len(gnu) > len(ours) and ( + not tail.strip(b"\0") or (len(tail) % 4 == 0 and tail == RISCV_NOP * (len(tail) // 4)) + ): + return gnu[: len(ours)] + return gnu + + +def ours_assemble(isa: str, source: str): # -> AssemblyResult + from mapachespim.toolchain import Assembler + + return Assembler(isa).assemble(source) + + +def ours_text( + isa: str, source: str, section: str = ".text" +) -> Tuple[bytes, List[str], List[Tuple[int, int]]]: + """Assemble with MapacheSPIM; return (section bytes, errors, debug_lines).""" + from io import BytesIO + + result = ours_assemble(isa, source) + if not result.success: + return b"", result.errors, [] + elf = ELFFile(BytesIO(result.elf_bytes)) + found = elf.get_section_by_name(section) + return (found.data() if found is not None else b""), [], result.debug_lines + + +def describe_mismatch(isa: str, source: str, ours: bytes, gnu: bytes, debug_lines) -> str: + """Explain the first differing instruction, with its source line.""" + layout = get_layout(isa) + order = "big" if not layout.is_little_endian else "little" + lines = source.splitlines() + for offset in range(0, max(len(ours), len(gnu)), 4): + a, b = ours[offset : offset + 4], gnu[offset : offset + 4] + if a != b: + addr = layout.text_base + offset + # The source line whose code starts at or before this address + line_no: Optional[int] = None + for line_addr, line in debug_lines: + if line_addr <= addr: + line_no = line + src = lines[line_no - 1].strip() if line_no and line_no <= len(lines) else "?" + fmt = lambda w: f"{int.from_bytes(w, order):08x}" if len(w) == 4 else "(none)" # noqa: E731 + return ( + f"first difference at 0x{addr:x} (line {line_no}: {src!r}): " + f"ours {fmt(a)} vs GNU {fmt(b)}; lengths ours={len(ours)} GNU={len(gnu)}" + ) + return "identical" diff --git a/tests/test_differential_data.py b/tests/test_differential_data.py new file mode 100644 index 0000000..2c85725 --- /dev/null +++ b/tests/test_differential_data.py @@ -0,0 +1,72 @@ +""" +Differential tests for data directives: .data built by MapacheSPIM must be +byte-identical to GNU as, for every ISA with a built-in assembler. + +These caught several bugs: .align meaning n bytes instead of 2**n, MIPS data +in the wrong byte order when the ISA came from --isa, labels in .word +becoming 0, and "\\xff" in strings emitting two bytes. +""" + +import pytest + +from .gnu_reference import gnu_assemble, gnu_available, ours_text, trim_padding + +ISAS = [isa for isa in ("riscv64", "mips32") if gnu_available(isa)] +pytestmark = pytest.mark.skipif(not ISAS, reason="GNU binutils not installed") + +DATA_PROGRAM = r""" +.text +.globl _start +_start: + nop +.data +bytes: .byte 1, 2, 0xff, -1, 'A', 'z', 0b101, 010 +.align 2 +halves: .half 1, -2, 0x1234, 65535 +.align 3 +words: .word 1, -1, 0x12345678, 0xffffffff, 1+2<<3, (1+2)<<3, ~0, 17 % 5, -7 / 2 +table: .word bytes, halves, words, table+4, end - bytes +.align 3 +quads: .dword 0x123456789abcdef0, -2 +strings: .ascii "abc" +len = . - strings + .asciz "with\nescapes\t\\ \"quoted\" \x41\x7f\xff \101\60" + .string "two", "strings" +.balign 8 +space: .space 5 + .space 3, 0xaa +.p2align 4 + .skip 2 +# MIPS aligns this .word (and moves the label) automatically +lens: .word len, SIZE * 2 +.equ SIZE, 12 + .byte 3 +end: .word 0 +""" + + +@pytest.mark.parametrize("isa", ISAS) +def test_data_section_matches_gnu(isa): + ours, errors, _ = ours_text(isa, DATA_PROGRAM, ".data") + assert not errors, errors + gnu = trim_padding(gnu_assemble(isa, DATA_PROGRAM, ".data"), ours) + assert ours == gnu, f"\nours {ours.hex()}\ngnu {gnu.hex()}" + + +@pytest.mark.parametrize("isa", ISAS) +def test_code_alignment_matches_gnu(isa): + source = """ +.text +.globl _start +_start: + nop +.align 4 +aligned: + nop +.balign 8 + nop +""" + ours, errors, _ = ours_text(isa, source) + assert not errors, errors + gnu = trim_padding(gnu_assemble(isa, source), ours) + assert ours == gnu, f"\nours {ours.hex()}\ngnu {gnu.hex()}" diff --git a/tests/test_differential_mips.py b/tests/test_differential_mips.py new file mode 100644 index 0000000..4d31779 --- /dev/null +++ b/tests/test_differential_mips.py @@ -0,0 +1,369 @@ +""" +Differential tests: the built-in MIPS32 assembler against GNU as. + +Like test_differential_riscv.py: random programs per instruction family +(fixed seeds) must assemble to byte-identical .text with GNU as in its default +".set reorder" mode (-O0, non-PIC), which fills branch delay slots with nops. + +One deliberate difference is excluded: SPIM, MARS, and textbooks treat the +two-operand `div $s, $t` / `divu $s, $t` as the real instruction (results in +HI/LO), while GNU as expands it to `div $s, $s, $t`. MapacheSPIM follows SPIM. + +Skipped when mips-linux-gnu binutils are not installed. +Set MAPACHESPIM_FUZZ_SCALE=N to generate N times as many instructions. +""" + +import random + +import pytest + +from mapachespim.toolchain.mips import ( + ALU3, + ALU_IMM, + BRANCH1, + COMPARE_BRANCHES, + LOADS, + SHIFT_IMM, + SHIFT_VAR, + STORES, +) + +from .gnu_reference import ( + FUZZ_SCALE, + describe_mismatch, + gnu_accepts, + gnu_assemble, + gnu_available, + ours_text, + trim_padding, +) + +ISA = "mips32" +pytestmark = pytest.mark.skipif(not gnu_available(ISA), reason="GNU MIPS binutils not installed") + +NAMES = [ + "zero", "at", "v0", "v1", "a0", "a1", "a2", "a3", "t0", "t1", "t2", "t3", "t4", "t5", + "t6", "t7", "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "t8", "t9", "k0", "k1", + "gp", "sp", "fp", "ra", +] # fmt: skip + +# $at is the assembler's temporary; GNU warns when programs use it directly +USABLE = [i for i in range(32) if i != 1] + + +def reg(rng: random.Random) -> str: + n = rng.choice(USABLE) + return f"${NAMES[n]}" if rng.random() < 0.7 else f"${n}" + + +def nonzero_reg(rng: random.Random) -> str: + n = rng.choice([i for i in USABLE if i != 0]) + return f"${NAMES[n]}" if rng.random() < 0.7 else f"${n}" + + +def signed(rng: random.Random, bits: int) -> int: + lo, hi = -(1 << (bits - 1)), (1 << (bits - 1)) - 1 + return rng.choice([lo, hi, 0, -1, 1, lo + 1, hi - 1, rng.randint(lo, hi), rng.randint(lo, hi)]) + + +def unsigned(rng: random.Random, bits: int) -> int: + hi = (1 << bits) - 1 + return rng.choice([0, 1, hi, hi - 1, rng.randint(0, hi), rng.randint(0, hi)]) + + +def fmt_int(rng: random.Random, value: int) -> str: + return hex(value) if value >= 0 and rng.random() < 0.4 else str(value) + + +def any_const(rng: random.Random) -> int: + """A 32-bit constant that may or may not fit an immediate field.""" + bits = rng.choice([4, 15, 16, 17, 31, 32]) + value = rng.randrange(1 << bits) + if rng.random() < 0.2: + value = value << 16 + if rng.random() < 0.4: + value = -value + return max(-(1 << 31), min(value, (1 << 32) - 1)) + + +DATA = """ +.data +d0: .word 1 +d1: .word 2 +d2: .byte 3 +.align 2 +d3: .space 100 +d4: .word 4 +.space 40000 +d5: .word 5 +""" +DATA_LABELS = ["d0", "d1", "d2", "d3", "d4", "d5"] + + +def program(lines, labels=0, rng=None) -> str: + body = list(lines) + if labels and rng is not None: + positions = sorted(rng.sample(range(len(body) + 1), labels)) + for i, pos in enumerate(reversed(positions)): + body.insert(pos, f"L{labels - 1 - i}:") + text = "\n".join(f" {line}" if not line.endswith(":") else line for line in body) + return f".text\n.globl _start\n_start:\n{text}\n{DATA}" + + +def assert_same_chunked(lines, labels, rng, chunk=1500) -> None: + """Check long generated programs in pieces small enough that every branch + stays within MIPS's +/-128KB range (GNU as would relax longer ones).""" + for start in range(0, len(lines), chunk): + assert_same(program(lines[start : start + chunk], labels, rng)) + + +def assert_same(source: str) -> None: + ours, errors, debug_lines = ours_text(ISA, source) + assert not errors, f"MapacheSPIM rejected input GNU accepts: {errors}" + gnu = trim_padding(gnu_assemble(ISA, source), ours) + assert ours == gnu, describe_mismatch(ISA, source, ours, gnu, debug_lines) + + +def n(base: int) -> int: + return base * FUZZ_SCALE + + +# --------------------------------------------------------------------------- +# Real instructions +# --------------------------------------------------------------------------- + + +def test_register_register(): + rng = random.Random(1) + lines = [f"{m} {reg(rng)}, {reg(rng)}, {reg(rng)}" for m in ALU3 for _ in range(n(8))] + lines += [f"{m} {reg(rng)}, {reg(rng)}, {reg(rng)}" for m in SHIFT_VAR for _ in range(n(6))] + lines += [f"mul {reg(rng)}, {reg(rng)}, {reg(rng)}" for _ in range(n(6))] + assert_same(program(lines)) + + +def test_shifts(): + rng = random.Random(2) + lines = [ + f"{m} {reg(rng)}, {reg(rng)}, {rng.choice([0, 1, 31, rng.randrange(32)])}" + for m in SHIFT_IMM + for _ in range(n(8)) + ] + assert_same(program(lines)) + + +def test_immediates(): + rng = random.Random(3) + lines = [] + for m, (_op, is_signed) in ALU_IMM.items(): + for _ in range(n(10)): + value = signed(rng, 16) if is_signed else unsigned(rng, 16) + lines.append(f"{m} {reg(rng)}, {reg(rng)}, {fmt_int(rng, value)}") + for _ in range(n(8)): + lines.append(f"lui {reg(rng)}, {fmt_int(rng, unsigned(rng, 16))}") + assert_same(program(lines)) + + +def test_hi_lo_and_multiply_divide(): + rng = random.Random(4) + lines = [] + for _ in range(n(6)): + lines += [ + f"mult {reg(rng)}, {reg(rng)}", + f"multu {reg(rng)}, {reg(rng)}", + f"div $zero, {reg(rng)}, {reg(rng)}", + f"divu $zero, {reg(rng)}, {reg(rng)}", + f"mfhi {reg(rng)}", + f"mflo {reg(rng)}", + f"mthi {reg(rng)}", + f"mtlo {reg(rng)}", + f"madd {reg(rng)}, {reg(rng)}", + f"maddu {reg(rng)}, {reg(rng)}", + f"msub {reg(rng)}, {reg(rng)}", + f"msubu {reg(rng)}, {reg(rng)}", + f"clz {reg(rng)}, {reg(rng)}", + f"clo {reg(rng)}, {reg(rng)}", + ] + assert_same(program(lines)) + + +def test_loads_and_stores(): + rng = random.Random(5) + lines = [] + for m in list(LOADS) + list(STORES): + for _ in range(n(8)): + offset = signed(rng, 16) + mem = f"({reg(rng)})" if offset == 0 and rng.random() < 0.3 else f"{offset}({reg(rng)})" + lines.append(f"{m} {reg(rng)}, {mem}") + assert_same(program(lines)) + + +def test_branches_and_jumps(): + rng = random.Random(6) + labels = 10 + lines = [] + for _ in range(n(8)): + lines += [ + f"beq {reg(rng)}, {reg(rng)}, L{rng.randrange(labels)}", + f"bne {reg(rng)}, {reg(rng)}, L{rng.randrange(labels)}", + f"beqz {reg(rng)}, L{rng.randrange(labels)}", + f"bnez {reg(rng)}, L{rng.randrange(labels)}", + f"b L{rng.randrange(labels)}", + f"bal L{rng.randrange(labels)}", + f"j L{rng.randrange(labels)}", + f"jal L{rng.randrange(labels)}", + f"jr {reg(rng)}", + "jalr $t0, $t9", + f"jalr {rng.choice(['$t9', '$t0', '$s0', '$v0'])}", + ] + for m in BRANCH1: + rs = reg(rng) if m not in ("bltzal", "bgezal") else rng.choice(["$t0", "$s1", "$a0"]) + lines.append(f"{m} {rs}, L{rng.randrange(labels)}") + rng.shuffle(lines) + assert_same_chunked(lines, labels, rng) + + +def test_system(): + assert_same(program(["syscall", "break", "nop", "syscall 5", "break 7"])) + + +# --------------------------------------------------------------------------- +# Pseudo-instructions +# --------------------------------------------------------------------------- + + +def test_li_and_la(): + rng = random.Random(7) + lines = [] + for value in [0, 1, -1, 0x7FFF, -0x8000, 0x8000, 0xFFFF, 0x10000, 0xFFFF0000, 0x7FFFFFFF, + -0x80000000, 0xFFFFFFFF, 0x12345678] + [any_const(rng) for _ in range(n(40))]: # fmt: skip + lines.append(f"li {nonzero_reg(rng)}, {fmt_int(rng, value)}") + for _ in range(n(10)): + label = rng.choice(DATA_LABELS) + lines.append(f"la {nonzero_reg(rng)}, {label}{rng.choice(['', '+4', '-4', '+1000'])}") + lines.append(f"la {nonzero_reg(rng)}, {fmt_int(rng, any_const(rng))}") + assert_same(program(lines)) + + +def test_register_pseudos(): + rng = random.Random(8) + lines = [] + for _ in range(n(8)): + lines += [ + f"move {reg(rng)}, {reg(rng)}", + f"not {reg(rng)}, {reg(rng)}", + f"neg {reg(rng)}, {reg(rng)}", + f"negu {reg(rng)}, {reg(rng)}", + f"abs {nonzero_reg(rng)}, {nonzero_reg(rng)}", + ] + assert_same(program(lines)) + + +def test_compare_branches(): + rng = random.Random(9) + labels = 10 + lines = [] + for m in COMPARE_BRANCHES: + for _ in range(n(6)): + lines.append(f"{m} {reg(rng)}, {reg(rng)}, L{rng.randrange(labels)}") + lines.append(f"{m} {reg(rng)}, {fmt_int(rng, any_const(rng))}, L{rng.randrange(labels)}") + for value in (0, 1, -1, 0x7FFF, 0x8000, -0x8000): + lines.append(f"{m} {reg(rng)}, {value}, L{rng.randrange(labels)}") + lines.append(f"{m} {reg(rng)}, $zero, L{rng.randrange(labels)}") + lines.append(f"{m} $zero, {reg(rng)}, L{rng.randrange(labels)}") + for m in ("beq", "bne"): + for _ in range(n(6)): + lines.append(f"{m} {reg(rng)}, {fmt_int(rng, any_const(rng))}, L{rng.randrange(labels)}") + rng.shuffle(lines) + assert_same_chunked(lines, labels, rng) + + +def test_register_forms_with_immediates(): + """e.g. 'add $t0, $t1, 5' and constants too large for the immediate field.""" + rng = random.Random(10) + lines = [] + for m in ("add", "addu", "sub", "subu", "and", "or", "xor", "slt", "sltu"): + for _ in range(n(8)): + lines.append(f"{m} {reg(rng)}, {reg(rng)}, {fmt_int(rng, any_const(rng))}") + assert_same(program(lines)) + + +def test_multiply_divide_pseudos(): + rng = random.Random(11) + lines = [] + for m in ("div", "divu", "rem", "remu", "mul"): + for _ in range(n(8)): + lines.append(f"{m} {nonzero_reg(rng)}, {reg(rng)}, {nonzero_reg(rng)}") + for value in (1, 2, 7, -3, 100000): + if m in ("divu", "remu") and value < 0: + continue + lines.append(f"{m} {nonzero_reg(rng)}, {reg(rng)}, {value}") + assert_same(program(lines)) + + +def test_symbol_loads_and_stores(): + rng = random.Random(12) + lines = [] + for _ in range(n(8)): + label = rng.choice(DATA_LABELS) + lines += [ + f"{rng.choice(list(LOADS))} {nonzero_reg(rng)}, {label}", + f"{rng.choice(list(STORES))} {reg(rng)}, {label}", + f"lw {nonzero_reg(rng)}, {label}({reg(rng)})", + f"sw {reg(rng)}, {label}+8({reg(rng)})", + f"lw {nonzero_reg(rng)}, 100000({reg(rng)})", + f"lui {nonzero_reg(rng)}, %hi({label})", + f"addiu {nonzero_reg(rng)}, {reg(rng)}, %lo({label})", + f"lw {nonzero_reg(rng)}, %lo({label})({reg(rng)})", + ] + assert_same(program(lines)) + + +def test_noreorder(): + source = program( + [ + ".set noreorder", + "beq $t0, $t1, L0", + "addiu $t2, $t2, 1", + "jal L0", + "nop", + ".set reorder", + "L0:", + "bne $t0, $t1, L0", + "jr $ra", + ] + ) + assert_same(source) + + +def test_whole_examples(): + from mapachespim.examples import list_examples + + for example in list_examples(): + if example.isa == "mips" and example.source is not None: + assert_same(example.source.read_text().replace(".isa mips32", "")) + + +INVALID = [ + "addi $t0, $t1", + "add $t0, $t1, $t2, $t3", + "sll $t0, $t1, 32", + "frobnicate $t0", + "add $t0, $t1, $q9", + "beq $t0, $t1, undefined_label", + "j undefined_label", + "lw $t0", + "move $t0", + "mfhi", + "jr", + "jalr $t0, $t0", + "jalr $ra", + "bgezal $ra, L0", +] + + +@pytest.mark.parametrize("line", INVALID) +def test_invalid_rejected(line): + source = program([line, "L0:"]) + assert not gnu_accepts(ISA, source), f"test assumption: GNU accepts {line!r}" + _, errors, _ = ours_text(ISA, source) + assert errors, f"MapacheSPIM accepted invalid input {line!r}" diff --git a/tests/test_differential_riscv.py b/tests/test_differential_riscv.py new file mode 100644 index 0000000..f036277 --- /dev/null +++ b/tests/test_differential_riscv.py @@ -0,0 +1,352 @@ +""" +Differential tests: the built-in RISC-V assembler against GNU as. + +Every test generates a random program (fixed seed, so failures reproduce) +covering one family of instructions with random registers, immediates across +their whole range including the boundaries, and forward/backward label +references, then requires byte-identical .text from both assemblers. + +Skipped when riscv64-linux-gnu binutils are not installed. +Set MAPACHESPIM_FUZZ_SCALE=N to generate N times as many instructions. +""" + +import random + +import pytest + +from .gnu_reference import ( + FUZZ_SCALE, + describe_mismatch, + gnu_accepts, + gnu_assemble, + gnu_available, + ours_text, + trim_padding, +) +from mapachespim.toolchain.riscv import ( + BRANCHES, + I_ALU, + LOADS, + R_TYPE, + SHIFT_IMM, + STORES, + SWAP_BRANCHES, + ZERO_BRANCHES, +) + +ISA = "riscv64" +pytestmark = pytest.mark.skipif(not gnu_available(ISA), reason="GNU RISC-V binutils not installed") + +ABI = [ + "zero", "ra", "sp", "gp", "tp", "t0", "t1", "t2", "s0", "s1", "a0", "a1", "a2", "a3", + "a4", "a5", "a6", "a7", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10", "s11", + "t3", "t4", "t5", "t6", +] # fmt: skip + + +def reg(rng: random.Random) -> str: + """A random register, by ABI name or xN.""" + n = rng.randrange(32) + return ABI[n] if rng.random() < 0.7 else f"x{n}" + + +def nonzero_reg(rng: random.Random) -> str: + n = rng.randrange(1, 32) + return ABI[n] if rng.random() < 0.7 else f"x{n}" + + +def signed(rng: random.Random, bits: int) -> int: + """Random signed value, biased toward the edges of the range.""" + lo, hi = -(1 << (bits - 1)), (1 << (bits - 1)) - 1 + return rng.choice([lo, hi, 0, -1, 1, lo + 1, hi - 1, rng.randint(lo, hi), rng.randint(lo, hi)]) + + +def fmt_int(rng: random.Random, value: int) -> str: + """Write an integer in decimal or hex, as students do.""" + if value >= 0 and rng.random() < 0.4: + return hex(value) + return str(value) + + +DATA = """ +.data +d0: .word 1 +d1: .dword 2 +d2: .byte 3 +.align 3 +d3: .space 100 +d4: .word 4 +""" +DATA_LABELS = ["d0", "d1", "d2", "d3", "d4"] + + +def program(lines, labels=0, rng=None) -> str: + """Wrap instruction lines in a program, scattering `labels` labels L0..Ln.""" + body = list(lines) + if labels and rng is not None: + positions = sorted(rng.sample(range(len(body) + 1), labels)) + for i, pos in enumerate(reversed(positions)): + body.insert(pos, f"L{labels - 1 - i}:") + text = "\n".join(f" {line}" if not line.endswith(":") else line for line in body) + return f".text\n.globl _start\n_start:\n{text}\n{DATA}" + + +def assert_same(source: str) -> None: + ours, errors, debug_lines = ours_text(ISA, source) + gnu = trim_padding(gnu_assemble(ISA, source), ours) + assert not errors, f"MapacheSPIM rejected input GNU accepts: {errors}" + assert ours == gnu, describe_mismatch(ISA, source, ours, gnu, debug_lines) + + +def n(base: int) -> int: + return base * FUZZ_SCALE + + +# --------------------------------------------------------------------------- +# Real instructions +# --------------------------------------------------------------------------- + + +def test_register_register(): + rng = random.Random(1) + lines = [f"{m} {reg(rng)}, {reg(rng)}, {reg(rng)}" for m in R_TYPE for _ in range(n(8))] + assert_same(program(lines)) + + +def test_register_immediate(): + rng = random.Random(2) + lines = [ + f"{m} {reg(rng)}, {reg(rng)}, {fmt_int(rng, signed(rng, 12))}" + for m in I_ALU + for _ in range(n(12)) + ] + assert_same(program(lines)) + + +def test_shift_immediate(): + rng = random.Random(3) + lines = [] + for m, (_op, _f3, _hi, bits) in SHIFT_IMM.items(): + for shamt in [0, 1, (1 << bits) - 1] + [rng.randrange(1 << bits) for _ in range(n(6))]: + lines.append(f"{m} {reg(rng)}, {reg(rng)}, {shamt}") + assert_same(program(lines)) + + +def test_loads_and_stores(): + rng = random.Random(4) + lines = [] + for m in list(LOADS) + list(STORES): + for _ in range(n(10)): + offset = signed(rng, 12) + mem = f"({reg(rng)})" if offset == 0 and rng.random() < 0.3 else f"{offset}({reg(rng)})" + lines.append(f"{m} {reg(rng)}, {mem}") + assert_same(program(lines)) + + +def test_branches(): + rng = random.Random(5) + labels = 12 + lines = [] + for m in list(BRANCHES) + list(SWAP_BRANCHES): + for _ in range(n(8)): + lines.append(f"{m} {reg(rng)}, {reg(rng)}, L{rng.randrange(labels)}") + for m in ZERO_BRANCHES: + for _ in range(n(8)): + lines.append(f"{m} {reg(rng)}, L{rng.randrange(labels)}") + rng.shuffle(lines) + assert_same(program(lines, labels, rng)) + + +def test_jumps(): + rng = random.Random(6) + labels = 8 + lines = [] + for _ in range(n(10)): + lines += [ + f"jal L{rng.randrange(labels)}", + f"jal {reg(rng)}, L{rng.randrange(labels)}", + f"j L{rng.randrange(labels)}", + f"jalr {reg(rng)}", + f"jalr {reg(rng)}, {signed(rng, 12)}({reg(rng)})", + f"jalr {reg(rng)}, {reg(rng)}, {signed(rng, 12)}", + f"jr {reg(rng)}", + "ret", + f"call L{rng.randrange(labels)}", + f"tail L{rng.randrange(labels)}", + ] + rng.shuffle(lines) + assert_same(program(lines, labels, rng)) + + +def test_upper_immediates(): + rng = random.Random(7) + lines = [] + for m in ("lui", "auipc"): + for value in [0, 1, 0xFFFFF, 0x80000, 0x7FFFF] + [rng.randrange(1 << 20) for _ in range(n(10))]: + lines.append(f"{m} {reg(rng)}, {hex(value)}") + assert_same(program(lines)) + + +def test_system_and_csr(): + rng = random.Random(8) + lines = ["ecall", "ebreak", "fence", "nop"] + csrs = ["mscratch", "mepc", "mtvec", "mstatus", "0x340"] + for _ in range(n(4)): + csr = rng.choice(csrs) + lines += [ + f"csrrw {reg(rng)}, {csr}, {reg(rng)}", + f"csrrs {reg(rng)}, {csr}, {reg(rng)}", + f"csrrc {reg(rng)}, {csr}, {reg(rng)}", + f"csrrwi {reg(rng)}, {csr}, {rng.randrange(32)}", + f"csrrsi {reg(rng)}, {csr}, {rng.randrange(32)}", + f"csrrci {reg(rng)}, {csr}, {rng.randrange(32)}", + f"csrr {reg(rng)}, {csr}", + f"csrw {csr}, {reg(rng)}", + f"csrs {csr}, {reg(rng)}", + f"csrc {csr}, {reg(rng)}", + f"csrr {reg(rng)}, cycle", + ] + assert_same(program(lines)) + + +# --------------------------------------------------------------------------- +# Pseudo-instructions +# --------------------------------------------------------------------------- + + +def interesting_constants(rng: random.Random, count: int): + """Constants that exercise every path of li's expansion.""" + values = [ + 0, 1, -1, 2047, -2048, 2048, -2049, 0x7FF, 0x800, 0xFFF, 0x1000, 0x12345, + 0x7FFFFFFF, -0x80000000, 0x80000000, 0xFFFFFFFF, 0x100000000, 0x7FFFFFFFFFFFFFFF, + -0x8000000000000000, 0x8000000000000000 - 1, 0x0000080000000800, 0xFFFFFFFF00000000, + 0x00FF00FF00FF00FF, 0xDEADBEEFCAFEBABE, 0x123456789ABCDEF0, + ] # fmt: skip + for _ in range(count): + bits = rng.choice([12, 13, 20, 32, 33, 40, 52, 63, 64]) + value = rng.randrange(1 << bits) + # Also sparse values (few set bits), which take the shift paths + if rng.random() < 0.3: + value = sum(1 << rng.randrange(64) for _ in range(rng.randint(1, 4))) + values.append(value if rng.random() < 0.5 else -value) + return values + + +def test_li(): + rng = random.Random(9) + lines = [] + for value in interesting_constants(rng, n(120)): + # GNU wants 64-bit constants in range for li + if -(1 << 63) <= value < (1 << 64): + lines.append(f"li {nonzero_reg(rng)}, {fmt_int(rng, value)}") + assert_same(program(lines)) + + +def test_register_pseudos(): + rng = random.Random(10) + lines = [] + for m in ("mv", "not", "neg", "negw", "sext.w", "seqz", "snez", "sltz", "sgtz"): + for _ in range(n(8)): + lines.append(f"{m} {reg(rng)}, {reg(rng)}") + assert_same(program(lines)) + + +def test_addresses_and_symbol_access(): + rng = random.Random(11) + lines = [] + for _ in range(n(8)): + label = rng.choice(DATA_LABELS) + offset = rng.choice(["", "+4", "+8", "-4"]) + lines += [ + f"la {nonzero_reg(rng)}, {label}{offset}", + f"lla {nonzero_reg(rng)}, {label}", + f"{rng.choice(list(LOADS))} {nonzero_reg(rng)}, {label}", + f"{rng.choice(list(STORES))} {reg(rng)}, {label}, {nonzero_reg(rng)}", + f"addi {nonzero_reg(rng)}, {reg(rng)}, %lo({label})", + f"lw {nonzero_reg(rng)}, %lo({label})({reg(rng)})", + f"lui {nonzero_reg(rng)}, %hi(SMALL{offset})", + f"addi {nonzero_reg(rng)}, {reg(rng)}, %lo(SMALL{offset})", + ] + source = program(lines).replace(".text", ".equ SMALL, 0x12345678\n.text", 1) + assert_same(source) + + +def test_register_forms_with_immediates(): + """GNU accepts e.g. 'add a0, a1, 5' as 'addi a0, a1, 5'.""" + rng = random.Random(12) + lines = [] + for m in ("add", "and", "or", "xor", "slt", "sltu", "addw"): + for _ in range(n(6)): + lines.append(f"{m} {reg(rng)}, {reg(rng)}, {signed(rng, 12)}") + for m in ("sll", "srl", "sra"): + for _ in range(n(4)): + lines.append(f"{m} {reg(rng)}, {reg(rng)}, {rng.randrange(64)}") + for m in ("sllw", "srlw", "sraw"): + for _ in range(n(4)): + lines.append(f"{m} {reg(rng)}, {reg(rng)}, {rng.randrange(32)}") + assert_same(program(lines)) + + +def test_expressions_and_literals(): + source = program( + [ + "addi a0, a0, 'A'", + "addi a0, a0, '\\n'", + "addi a0, a0, 0b1010", + "addi a0, a0, -0x10", + "li a0, SIZE", + "li a1, SIZE*2" if False else "li a1, SIZE+2", + "addi a2, a2, SIZE-1", + "la a3, d3+16", + "j end", + "nop", + "end:", + "ret", + ] + ).replace(".text", ".equ SIZE, 100\n.text", 1) + assert_same(source) + + +def test_whole_examples(): + """The bundled RISC-V examples assemble identically too.""" + from mapachespim.examples import list_examples + + for example in list_examples(): + if example.isa == "riscv" and example.source is not None: + source = example.source.read_text().replace(".isa riscv64", "") + assert_same(source) + + +# --------------------------------------------------------------------------- +# Invalid input: anything GNU rejects, we reject too +# --------------------------------------------------------------------------- + +INVALID = [ + "addi a0, a1, 2048", + "addi a0, a1, -2049", + "slli a0, a1, 64", + "slliw a0, a1, 32", + "lw a0, 2048(a1)", + "sw a0, -2049(a1)", + "lui a0, 0x100000", + "csrrwi a0, mscratch, 32", + "add a0, a1", + "add a0, a1, a2, a3", + "addi a0, a1, a2", + "lw a0, a1", + "frobnicate a0", + "add a0, a1, x32", + "beq a0, a1, undefined_label", + "ecall a0", + "mv a0", + # lui sign-extends on RV64, so %hi of an address >= 0x80000000 is invalid + "lui a0, %hi(d0)", +] + + +@pytest.mark.parametrize("line", INVALID) +def test_invalid_rejected(line): + source = program([line]) + assert not gnu_accepts(ISA, source), f"test assumption: GNU accepts {line!r}" + _, errors, _ = ours_text(ISA, source) + assert errors, f"MapacheSPIM accepted invalid input {line!r}" diff --git a/tests/test_mips_encoder.py b/tests/test_mips_encoder.py index 3c6350b..eb986c5 100644 --- a/tests/test_mips_encoder.py +++ b/tests/test_mips_encoder.py @@ -76,7 +76,9 @@ class TestInstructionForms(unittest.TestCase): ("sw $ra, 4($sp)", ["sw $ra, 4($sp)"]), ("sb $t0, 0($a0)", ["sb $t0, ($a0)"]), ("sh $t0, 0($a0)", ["sh $t0, ($a0)"]), - ("lw $t0, msg", ["lui $at, 0x1000", "lw $t0, 0x24($at)"]), + # Like GNU as, a load uses its own destination as the temporary + ("lw $t0, msg", ["lui $t0, 0x1000", "lw $t0, 0x24($t0)"]), + ("sw $t0, msg", ["lui $at, 0x1000", "sw $t0, 0x24($at)"]), ("syscall", ["syscall"]), ("break", ["break"]), ("nop", ["nop"]), @@ -101,13 +103,16 @@ class TestInstructionForms(unittest.TestCase): ("li $t0, 0xffff", ["ori $t0, $zero, 0xffff"]), ("li $t0, 0x10000", ["lui $t0, 1"]), ("li $t0, 0x12345678", ["lui $t0, 0x1234", "ori $t0, $t0, 0x5678"]), - ("la $a0, msg", ["lui $a0, 0x1000", "ori $a0, $a0, 0x24"]), + ("la $a0, msg", ["lui $a0, 0x1000", "addiu $a0, $a0, 0x24"]), ("not $t0, $t1", ["not $t0, $t1"]), ("neg $t0, $t1", ["neg $t0, $t1"]), ("blt $t0, $t1, target", ["slt $at, $t0, $t1", "bnez $at, 0x400040", "nop"]), ("bgt $t0, $t1, target", ["slt $at, $t1, $t0", "bnez $at, 0x400040", "nop"]), - ("div $t0, $t1, $t2", ["div $zero, $t1, $t2", "mflo $t0"]), - ("rem $t0, $t1, $t2", ["div $zero, $t1, $t2", "mfhi $t0"]), + ("div $t0, $t1", ["div $zero, $t0, $t1"]), # SPIM: the real instruction + ("div $zero, $t1, $t2", ["div $zero, $t1, $t2"]), + ("div $t0, $t1, 1", ["move $t0, $t1"]), + ("rem $t0, $t1, 1", ["move $t0, $zero"]), + ("mul $t0, $t1, 7", ["addiu $at, $zero, 7", "mult $t1, $at", "mflo $t0"]), ] def test_forms(self): @@ -196,6 +201,26 @@ def test_compare_branches(self): ) self.assertEqual(sim.get_reg(self.T2), int(check(a, b))) + def test_division_traps(self): + """div/rem check for division by zero and overflow, as SPIM does.""" + for source, regs, expected in [ + ("div $t2, $t0, $t1", {8: 7, 9: 0}, "Division by zero"), + ("rem $t2, $t0, $t1", {8: 7, 9: 0}, "Division by zero"), + ("divu $t2, $t0, $t1", {8: 7, 9: 0}, "Division by zero"), + ("div $t2, $t0, $t1", {8: -0x80000000, 9: -1}, "overflow"), + ]: + with self.subTest(source=source, regs=regs): + sim = Simulator(ISA.MIPS) + code = encode(source, PC, {}) + sim.write_mem(PC, code) + sim.set_pc(PC) + for reg, value in regs.items(): + sim.set_reg(reg, value & 0xFFFFFFFF) + for _ in range(len(code) // 4): + if sim.check_termination(sim.step())[0]: + break + self.assertIn(expected, sim.last_error or "") + def test_div_rem_abs(self): sim = run_snippet( """ @@ -230,8 +255,8 @@ def test_forward_references(self): """ result = Assembler("mips32").assemble(source) self.assertTrue(result.success, result.errors) - # msg is at 0x10000000, so la is just lui (1) + blt (3) + li (2) - self.assertEqual(result.symbols["skip"] - result.symbols["_start"], 24) + # la (lui + addiu) + blt (slt, bnez, nop) + li (lui + ori) + self.assertEqual(result.symbols["skip"] - result.symbols["_start"], 28) if __name__ == "__main__": diff --git a/tests/test_regressions.py b/tests/test_regressions.py index a9427ee..7135b7f 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -230,3 +230,56 @@ def test_complete_set_includes_output_spacing(self): if __name__ == "__main__": unittest.main() + + +class TestAssemblerDirectives(unittest.TestCase): + """Directive bugs found by comparing against GNU as.""" + + def assemble(self, source, isa="riscv64"): + from mapachespim.toolchain import assemble + + return assemble(source, isa=isa) + + def section(self, result, name): + from io import BytesIO + + from elftools.elf.elffile import ELFFile + + found = ELFFile(BytesIO(result.elf_bytes)).get_section_by_name(name) + return found.data() if found is not None else None + + def test_undefined_symbol_in_data_is_an_error(self): + """Unknown values in .word used to become 0 silently.""" + result = self.assemble(".text\n_start: nop\n.data\nx: .word no_such_label\n") + self.assertFalse(result.success) + self.assertIn("Line 4", result.errors[0]) + self.assertIn("no_such_label", result.errors[0]) + + def test_bad_directive_values_are_errors(self): + for line in (".space oops", ".align", ".word 1 +", '.ascii "\\x"', ".word 1/0"): + with self.subTest(line=line): + result = self.assemble(f".text\n_start: nop\n.data\n{line}\n") + self.assertFalse(result.success, line) + + def test_label_in_data(self): + result = self.assemble(".text\n_start: nop\n.data\na: .word 7\nptr: .word a\n") + self.assertTrue(result.success, result.errors) + data = self.section(result, ".data") + self.assertEqual(int.from_bytes(data[4:8], "little"), result.symbols["a"]) + + def test_custom_section_contents_are_kept(self): + """Custom sections got addresses but were not written to the ELF.""" + result = self.assemble('.text\n_start: nop\n.section .mydata\nv: .word 0x1234\n') + self.assertTrue(result.success, result.errors) + self.assertEqual(self.section(result, ".mydata"), (0x1234).to_bytes(4, "little")) + + def test_mips_data_is_big_endian_without_isa_directive(self): + result = self.assemble(".text\n_start: nop\n.data\nx: .word 0x11223344\n", "mips32") + self.assertEqual(self.section(result, ".data"), bytes.fromhex("11223344")) + + def test_set_noreorder(self): + result = self.assemble( + ".set noreorder\n.text\n_start:\n b _start\n addiu $t0, $t0, 1\n", "mips32" + ) + self.assertTrue(result.success, result.errors) + self.assertEqual(len(self.section(result, ".text")), 8) # no delay-slot nop added diff --git a/tests/test_riscv_encoder.py b/tests/test_riscv_encoder.py index 0bade9c..355dc93 100644 --- a/tests/test_riscv_encoder.py +++ b/tests/test_riscv_encoder.py @@ -14,7 +14,13 @@ from mapachespim.toolchain.riscv import RISCVEncodeError, encode, instruction_size PC = 0x80000000 -LABELS = {"target": PC + 0x40, "back": PC - 0x20, "msg": 0x80100010, "CONST": 42} +LABELS = { + "target": PC + 0x40, + "back": PC - 0x20, + "msg": 0x80100010, + "CONST": 42, + "SMALL": 0x12345678, +} _cs = Cs(CS_ARCH_RISCV, CS_MODE_RISCV64) @@ -104,7 +110,7 @@ class TestInstructionForms(unittest.TestCase): ("jalr ra, 8(t0)", "jalr ra, t0, 8"), # Upper immediates ("lui a0, 0x12345", "lui a0, 0x12345"), - ("lui a0, %hi(msg)", "lui a0, 0x80100"), + ("lui a0, %hi(SMALL)", "lui a0, 0x12345"), ("addi a0, a0, %lo(msg)", "addi a0, a0, 0x10"), ("auipc a0, 1", "auipc a0, 1"), # System @@ -137,7 +143,6 @@ class TestInstructionForms(unittest.TestCase): ("j target", "j 0x40"), ("jr ra", "ret"), ("ret", "ret"), - ("call target", "jal 0x40"), ] def test_forms(self): @@ -148,6 +153,15 @@ def test_forms(self): self.assertEqual(disasm(code), [expected]) self.assertEqual(instruction_size(source, LABELS), 4) + def test_call_and_tail_use_auipc_jalr(self): + # Like GNU as: always two instructions, so any address is reachable + self.assertEqual(disasm(encode("call target", PC, LABELS)), ["auipc ra, 0", "jalr ra, ra, 0x40"]) + self.assertEqual(disasm(encode("tail target", PC, LABELS)), ["auipc t1, 0", "jalr zero, t1, 0x40"]) + + def test_far_branch_is_relaxed(self): + far = {"far": PC + 0x10000} + self.assertEqual(disasm(encode("beq a0, a1, far", PC, far)), ["bne a0, a1, 8", "j 0xfffc"]) + def test_la_is_pc_relative(self): code = encode("la a0, msg", PC, LABELS) self.assertEqual(disasm(code), ["auipc a0, 0x100", "addi a0, a0, 0x10"]) @@ -174,6 +188,7 @@ class TestErrors(unittest.TestCase): ("beq a0, a1, nowhere", "undefined symbol"), ("slli a0, a0, 64", "out of range"), ("ecall a0", "takes no operands"), + ("lui a0, %hi(msg)", "out of range"), ] def test_errors(self): From 4b58398957ec19c765da68aab696b867ef178ffc Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 04:38:48 +0000 Subject: [PATCH 09/18] Split source lines like GNU as: comments, ';', and local labels - Comment stripping ignored quotes, truncating strings containing '#', ';' or '//'; it now skips string and character literals - ';' separates statements (it was treated as a comment, silently dropping everything after it) - On ARM64, '#' starts a comment only at the beginning of a line, since it marks immediates; '//' and /* ... */ comments work everywhere - Several labels on one line, .L local labels, and GNU numeric local labels (1: ... b 1b / 1f) are supported - Defining the same label twice is an error Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- mapachespim/toolchain/assembler.py | 14 +- mapachespim/toolchain/directives.py | 246 +++++++++++++++++++++++----- tests/test_regressions.py | 47 ++++++ tests/test_toolchain.py | 2 +- 4 files changed, 267 insertions(+), 42 deletions(-) diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index 850ccb3..b726a09 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -29,7 +29,13 @@ keystone = None from . import mips, riscv -from .directives import DOT_LABEL_PREFIX, DirectiveParser, LineType, ParsedLine, SectionData +from .directives import ( + INTERNAL_LABEL_PREFIXES, + DirectiveParser, + LineType, + ParsedLine, + SectionData, +) from .dwarf import DWARFv2Builder from .elf_builder import STB_GLOBAL, STB_LOCAL, STT_FUNC, STT_NOTYPE, ELFBuilder, Section, Symbol from .expr import EncodeError, ExpressionError, UndefinedSymbol, evaluate @@ -405,8 +411,10 @@ def assemble( if result.errors: return result - # Internal labels standing in for '.' are not real symbols - all_labels = {k: v for k, v in all_labels.items() if not k.startswith(DOT_LABEL_PREFIX)} + # Internal labels (standing in for '.' or numeric labels) are not real symbols + all_labels = { + k: v for k, v in all_labels.items() if not k.startswith(INTERNAL_LABEL_PREFIXES) + } # Build ELF entry_addr = all_labels.get(entry_symbol, self._layout.text_base) diff --git a/mapachespim/toolchain/directives.py b/mapachespim/toolchain/directives.py index 576aa8b..8906bb3 100644 --- a/mapachespim/toolchain/directives.py +++ b/mapachespim/toolchain/directives.py @@ -106,6 +106,11 @@ class Fixup: # Prefix of internal labels that stand in for '.' (not emitted as symbols) DOT_LABEL_PREFIX = ".L.dot." +# Prefix of internal names given to numeric local labels like "1:" +LOCAL_LABEL_PREFIX = ".L.local." + +INTERNAL_LABEL_PREFIXES = (DOT_LABEL_PREFIX, LOCAL_LABEL_PREFIX) + class DirectiveParser: """ @@ -123,15 +128,12 @@ class DirectiveParser: """ # Regex patterns - LABEL_PATTERN = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_]*):(.*)$") + # A label: a symbol name (including GNU local labels like .L1 and names + # with $), or a numeric local label like "1:" + LABEL_PATTERN = re.compile(r"^([A-Za-z_.$][\w.$]*|\d+):(.*)$") DIRECTIVE_PATTERN = re.compile(r"^\s*\.(\w+)\s*(.*)$") - # Hash comment: # at start, or # preceded by space and NOT followed by digit/minus (ARM immediate) - # This handles both #42 and #-32 as ARM immediates, not comments - HASH_COMMENT = re.compile(r"(^|\s)#(?!-?\d).*$") - COMMENT_PATTERNS = [ - re.compile(r"//.*$"), # C++ style - re.compile(r";.*$"), # Semicolon comments - ] + # References to numeric local labels: "1b" (backward) and "1f" (forward) + _LOCAL_REF = re.compile(r"(? Dict[str, SectionData]: Returns: Dictionary of section name -> SectionData. """ - lines = source.splitlines() + statements = self._statements(source) + statements = self._rename_local_labels(statements) - for line_num, line in enumerate(lines, 1): + for line_num, text, original in statements: try: - parsed = self._parse_line(line_num, line) - if parsed.line_type not in (LineType.EMPTY, LineType.COMMENT): - parsed.reorder = self.reorder - section = self.sections[self.current_section] - section.lines.append(parsed) - - # Handle labels - if parsed.label: - if parsed.label in self.constants: - section.labels[parsed.label] = self.constants[parsed.label] - else: - section.labels[parsed.label] = section.current_offset - section.pending_labels.append(parsed.label) - - # Process directive data - if parsed.directive: - self._process_directive(parsed, section) - elif parsed.line_type == LineType.INSTRUCTION: - section.pending_labels.clear() - + for parsed in self._parse_statement(line_num, text, original): + self._add_line(parsed) except Exception as e: self.errors.append(f"Line {line_num}: {e}") return self.sections - def _parse_line(self, line_num: int, line: str) -> ParsedLine: - """Parse a single line of assembly.""" - original = line + def _add_line(self, parsed: ParsedLine) -> None: + """Record a parsed line in the current section.""" + parsed.reorder = self.reorder + section = self.sections[self.current_section] + section.lines.append(parsed) + + # Handle labels + if parsed.label: + if parsed.label in section.labels or any( + parsed.label in other.labels for other in self.sections.values() + ): + raise ValueError(f"symbol '{parsed.label}' is already defined") + if parsed.label in self.constants: + section.labels[parsed.label] = self.constants[parsed.label] + else: + section.labels[parsed.label] = section.current_offset + section.pending_labels.append(parsed.label) + + # Process directive data + if parsed.directive: + self._process_directive(parsed, section) + elif parsed.line_type == LineType.INSTRUCTION: + section.pending_labels.clear() - # Strip hash comments (# at start or after space, but not ARM immediates like #42) - line = self.HASH_COMMENT.sub(r"\1", line) - # Strip other comment styles - for pattern in self.COMMENT_PATTERNS: - line = pattern.sub("", line) + # --- Splitting source into statements, as GNU as does --- + def _statements(self, source: str) -> List[Tuple[int, str, str]]: + """Split source into (line_number, statement_text, original_line). + + Removes comments and splits lines on ';', following GNU as's rules + for the target ISA: + + - '#' starts a comment (on ARM64 only at the start of a line, since + '#' marks immediates there) + - '//' starts a comment; '/* ... */' comments may span lines + - ';' separates statements on one line + - none of these count inside "strings" or 'c' character literals + """ + result: List[Tuple[int, str, str]] = [] + in_block = False + for line_num, original in enumerate(source.splitlines(), 1): + self._set_isa_from_line(original) + pieces, in_block = self._split_line(original, in_block) + for piece in pieces: + if piece.strip(): + result.append((line_num, piece.strip(), original)) + return result + + def _set_isa_from_line(self, line: str) -> None: + """Notice '.isa' early, since comment rules depend on the ISA.""" + stripped = line.strip() + if stripped.lower().startswith(".isa") and len(stripped.split()) > 1: + value = stripped.split()[1].lower().replace("-", "_") + if value in self.VALID_ISAS: + self.isa = value + + def _split_line(self, line: str, in_block: bool) -> Tuple[List[str], bool]: + arm = self._effective_isa == "arm64" + pieces: List[str] = [] + current: List[str] = [] + i = 0 + n = len(line) + quote = False + while i < n: + ch = line[i] + if in_block: + if line.startswith("*/", i): + in_block = False + current.append(" ") + i += 2 + else: + i += 1 + continue + if quote: + current.append(ch) + if ch == "\\" and i + 1 < n: + current.append(line[i + 1]) + i += 2 + continue + if ch == '"': + quote = False + i += 1 + continue + if ch == '"': + quote = True + current.append(ch) + i += 1 + continue + if ch == "'": + # Character literal: 'c', '\\n', or GNU's unterminated 'c + j = i + 1 + j += 2 if j < n and line[j] == "\\" else 1 + if j < n and line[j] == "'": + j += 1 + current.append(line[i:j]) + i = j + continue + if line.startswith("/*", i): + in_block = True + i += 2 + continue + if line.startswith("//", i): + break + if ch == "#": + at_line_start = not pieces and not "".join(current).strip() + if not arm or at_line_start: + break + if ch == ";": + pieces.append("".join(current)) + current = [] + i += 1 + continue + current.append(ch) + i += 1 + pieces.append("".join(current)) + return pieces, in_block + + def _rename_local_labels( + self, statements: List[Tuple[int, str, str]] + ) -> List[Tuple[int, str, str]]: + """Give GNU numeric local labels ("1:", used as "1b"/"1f") unique names.""" + # Where each numeric label is defined, in order + definitions: Dict[str, List[int]] = {} + for index, (_, text, _) in enumerate(statements): + for label in self._leading_labels(text)[0]: + if label.isdigit(): + definitions.setdefault(label, []).append(index) + if not definitions and not any(self._LOCAL_REF.search(t) for _, t, _ in statements): + return statements + + def name(label: str, occurrence: int) -> str: + return f"{LOCAL_LABEL_PREFIX}{label}${occurrence}" + + result = [] + seen: Dict[str, int] = {} + for index, (line_num, text, original) in enumerate(statements): + labels, rest = self._leading_labels(text) + new_labels = [] + for label in labels: + if label.isdigit(): + seen[label] = seen.get(label, 0) + 1 + new_labels.append(name(label, seen[label])) + else: + new_labels.append(label) + + def reference(m: re.Match, index: int = index, line_num: int = line_num) -> str: + # "Nb": the nearest definition at or before this statement; + # "Nf": the nearest one after it + label, direction = m.group(1), m.group(2) + places = definitions.get(label, []) + if direction == "b": + candidates = [i for i in places if i <= index] + target = candidates[-1] if candidates else None + else: + candidates = [i for i in places if i > index] + target = candidates[0] if candidates else None + if target is None: + where = "before" if direction == "b" else "after" + self.errors.append( + f"Line {line_num}: no local label '{label}:' {where} '{m.group(0)}'" + ) + return m.group(0) + return name(label, places.index(target) + 1) + + rest = self._LOCAL_REF.sub(reference, rest) + prefix = "".join(f"{lbl}: " for lbl in new_labels) + result.append((line_num, prefix + rest, original)) + return result + + def _leading_labels(self, text: str) -> Tuple[List[str], str]: + """Split 'a: b: rest' into (['a', 'b'], 'rest').""" + labels = [] + rest = text + while True: + if self._ASSIGN_PATTERN.match(rest): + break + m = self.LABEL_PATTERN.match(rest) + if not m: + break + labels.append(m.group(1)) + rest = m.group(2).strip() + return labels, rest + + def _parse_statement(self, line_num: int, text: str, original: str) -> List[ParsedLine]: + """Parse one statement, which may start with several labels.""" + labels, rest = self._leading_labels(text) + lines = [ParsedLine(line_num, LineType.LABEL, original, label=lbl) for lbl in labels[:-1]] + body = (f"{labels[-1]}: " if labels else "") + rest + lines.append(self._parse_line(line_num, body)) + lines[-1].content = original + return lines + + def _parse_line(self, line_num: int, line: str) -> ParsedLine: + """Parse a single statement (comments already removed).""" + original = line line = line.strip() # Empty line diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 7135b7f..cfb7eff 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -283,3 +283,50 @@ def test_set_noreorder(self): ) self.assertTrue(result.success, result.errors) self.assertEqual(len(self.section(result, ".text")), 8) # no delay-slot nop added + + +class TestSourceSyntax(unittest.TestCase): + """Comments, statement separators, and labels follow GNU as.""" + + def parse(self, source, isa="riscv64"): + from mapachespim.toolchain.directives import DirectiveParser + + parser = DirectiveParser(isa=isa) + sections = parser.parse(source) + return parser, sections + + def instructions(self, source, isa="riscv64"): + _, sections = self.parse(source, isa) + return [ln.instruction for ln in sections[".text"].lines if ln.instruction] + + def test_semicolon_separates_statements(self): + """';' used to be treated as a comment, dropping the second instruction.""" + self.assertEqual(self.instructions("li a0, 1; li a1, 2"), ["li a0, 1", "li a1, 2"]) + + def test_comment_characters_inside_strings(self): + parser, sections = self.parse('.data\ns: .asciz "a # b; c // d" # real comment\n') + self.assertEqual(bytes(sections[".data"].data), b"a # b; c // d\x00") + + def test_hash_is_immediate_on_arm(self): + self.assertEqual(self.instructions("mov x0, #1 // one", "arm64"), ["mov x0, #1"]) + self.assertEqual(self.instructions("# a comment\nnop", "arm64"), ["nop"]) + + def test_block_comments(self): + self.assertEqual(self.instructions("nop /* one\ntwo */ nop\nnop"), ["nop", "nop", "nop"]) + + def test_several_labels_and_local_labels(self): + parser, sections = self.parse("a: b: nop\n.Lloop: nop\n") + self.assertEqual(sections[".text"].labels, {"a": 0, "b": 0, ".Lloop": 0}) + + def test_numeric_local_labels(self): + from mapachespim.toolchain import assemble + + result = assemble("_start:\n1: addi a0, a0, 1\n bnez a0, 1b\n j 1f\n nop\n1: ret\n", "riscv64") + self.assertTrue(result.success, result.errors) + + def test_duplicate_label_is_an_error(self): + from mapachespim.toolchain import assemble + + result = assemble("_start: nop\n_start: nop\n", "riscv64") + self.assertFalse(result.success) + self.assertIn("already defined", result.errors[0]) diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index e1920e0..25ced9e 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -47,7 +47,7 @@ def test_parse_inline_hash_comment(self): def test_arm_immediate_not_comment(self): """Test that ARM-style #immediate is not treated as comment.""" - parser = DirectiveParser() + parser = DirectiveParser(isa="arm64") sections = parser.parse("mov x0, #42") lines = sections[".text"].lines assert len(lines) == 1 From cdc1f1eaace7df638ef597f4074b4d81a2feeb64 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 04:47:39 +0000 Subject: [PATCH 10/18] Add built-in ARM64 assembler matching GNU as; fix ARM examples The Keystone-based ARM64 path accepted non-standard code: '#' comments (which GNU as reads as immediates) and 'adr' to data more than 1MB away, silently rewritten into movz/movk. None of the ARM examples assembled with the real ARM toolchain. - Add toolchain/arm64.py: A64 integer instructions (arithmetic, logic with bitmask immediates, moves, shifts/bitfields, multiply/divide, conditional select/compare, loads/stores in all addressing modes, pairs, branches, adr/adrp with :lo12:, svc/brk/nop), choosing the same encodings as GNU as (movz/movn/orr for mov, sub for add of a negative, cmn for cmp of a negative, ldur for unaligned offsets) - 'ldr x0, =value' uses a literal pool laid out like GNU's (dumped at .ltorg/.pool and at the end of .text, deduplicated, 4-byte entries then 8-byte entries) - Differential tests against GNU as for every instruction family (match at 100x scale), plus an exhaustive round-trip test of all logical immediate encodings - Convert the ARM examples to standard syntax: '//' comments and 'ldr xN, =label' for data addresses. Rebuilt; they are identical to GNU's output and behave exactly as before - adr out of range now explains what to use instead - CI installs AArch64 binutils for the differential tests Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- .github/workflows/ci.yml | 8 +- examples/arm/array_stats/array_stats | Bin 2216 -> 2224 bytes examples/arm/array_stats/array_stats.s | 270 +++--- examples/arm/fibonacci/fibonacci | Bin 1392 -> 1400 bytes examples/arm/fibonacci/fibonacci.s | 256 ++--- examples/arm/guess_game/guess_game | Bin 2160 -> 2200 bytes examples/arm/guess_game/guess_game.s | 224 ++--- examples/arm/hello_asm/hello_asm | Bin 1328 -> 1344 bytes examples/arm/hello_asm/hello_asm.s | 274 +++--- examples/arm/matrix_multiply/matrix_mult | Bin 2288 -> 2312 bytes examples/arm/matrix_multiply/matrix_mult.s | 294 +++--- mapachespim/toolchain/arm64.py | 1012 ++++++++++++++++++++ mapachespim/toolchain/assembler.py | 217 ++--- mapachespim/toolchain/directives.py | 5 +- tests/gnu_reference.py | 1 + tests/test_arm64_encoder.py | 128 +++ tests/test_assembler_accuracy.py | 26 +- tests/test_differential_arm64.py | 399 ++++++++ 18 files changed, 2301 insertions(+), 813 deletions(-) create mode 100644 mapachespim/toolchain/arm64.py create mode 100644 tests/test_arm64_encoder.py create mode 100644 tests/test_differential_arm64.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50de417..f9861ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,13 +81,13 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update -qq - sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu + sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu binutils-aarch64-linux-gnu - name: Run tests run: python -m pytest tests/ --tb=short assembler-fuzz: - # The built-in RISC-V and MIPS assemblers must produce byte-identical + # The built-in assemblers must produce byte-identical # output to GNU as. This runs the randomized comparison much deeper # than the regular test job. runs-on: ubuntu-latest @@ -101,7 +101,7 @@ jobs: - name: Install GNU binutils run: | sudo apt-get update -qq - sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu + sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu binutils-aarch64-linux-gnu - name: Install package run: pip install -e ".[dev]" @@ -109,4 +109,4 @@ jobs: - name: Compare with GNU as (50x fuzz scale) env: MAPACHESPIM_FUZZ_SCALE: "50" - run: python -m pytest tests/test_differential_riscv.py tests/test_differential_mips.py tests/test_differential_data.py --tb=short + run: python -m pytest tests/test_differential_*.py --tb=short diff --git a/examples/arm/array_stats/array_stats b/examples/arm/array_stats/array_stats index 58b8d436c64ebbc1fd131b993fda705affdbc084..3ae82f9841ca95c88c5629d1643ee3687807747c 100644 GIT binary patch delta 631 zcmZ`#Jx{_=6uqx4EjWP@3yG;EX(A>}g5t!-q9i(y=+J=y7bA%SQ8x!+Gfvja4`2i* z5*@_B5aXhgUzuQ)b$I1N@! zhL`VtU$uQ;qQQ(%R{?W{HpVG~w5de>871zsax8nOH^~Y4#ZR31I4U|L}aE9`izI)A=I?`27w7N1fvu#~jBP$0d%d98(-|PdH#5 z{1A5Ynx<1Y>BM}*C5EPHPP&$3l1$-9?jq$sa%G80A)KHW4jwwFB^MYJ@YAD)k0#s` z?$$9NtKpjox5nKE%*v|tOfK%o>)4V!*pPMlj7K@i%E**7yTvTkC#))IK@ss@NwEep OZ<_sJmTCj5-i2RPw}R&Y delta 643 zcmdlWxI%D(24luVO>@N^j0|7^qgfza29Pit5NAwmG_F@PzLlSY z8CU*gW|-K((r}4^Y0;++28f(6kQN5=If3GwK-z@SVP^>=!`?!E|lYJOX z>se9Fc)%>W3T8LD`3`L8YQ=D>Wnf2FD~wgG0Vt3Tpecq0hyu{z=#H7p!z3}efT@Ua z$K=OMX#qNN3OYIp3UWY7PC-sy5k$(V$N{kqNDL?il2imEkPr}nv|wX^1a&sgWHx7F zbeMdRRa`QMfk6Nmr*KdLW-&5!OlD*gmpp(b+Xa=qG1(9(`vgsP9#r= min, skip to max check - mov x22, x10 # min = current + b.ge check_max // if current >= min, skip to max check + mov x22, x10 // min = current check_max: - # Check if current > max + // Check if current > max cmp x10, x19 - b.le next_elem # if current <= max, skip - mov x19, x10 # max = current + b.le next_elem // if current <= max, skip + mov x19, x10 // max = current next_elem: - # Move to next element - add x9, x9, #4 # pointer += 4 - sub x20, x20, #1 # counter-- + // Move to next element + add x9, x9, #4 // pointer += 4 + sub x20, x20, #1 // counter-- b stats_loop stats_done: - # Now print all the statistics - # x21 = sum, x22 = min, x19 = max + // Now print all the statistics + // x21 = sum, x22 = min, x19 = max - # Print Sum - adr x0, msg_sum + // Print Sum + ldr x0, =msg_sum mov x8, #4 svc #0 mov x0, x21 mov x8, #1 svc #0 - adr x0, newline + ldr x0, =newline mov x8, #4 svc #0 - # Print Min - adr x0, msg_min + // Print Min + ldr x0, =msg_min mov x8, #4 svc #0 mov x0, x22 mov x8, #1 svc #0 - adr x0, newline + ldr x0, =newline mov x8, #4 svc #0 - # Print Max - adr x0, msg_max + // Print Max + ldr x0, =msg_max mov x8, #4 svc #0 mov x0, x19 mov x8, #1 svc #0 - adr x0, newline + ldr x0, =newline mov x8, #4 svc #0 - # Print Count - adr x0, msg_count + // Print Count + ldr x0, =msg_count mov x8, #4 svc #0 - adr x9, array_len + ldr x9, =array_len ldr w0, [x9] mov x8, #1 svc #0 - adr x0, newline + ldr x0, =newline mov x8, #4 svc #0 - # Restore and return + // Restore and return ldp x21, x22, [sp, #32] ldp x19, x20, [sp, #16] ldp x29, x30, [sp], #48 ret -# ============================================================================ -# SUMMARY OF NEW INSTRUCTIONS -# ============================================================================ -# -# ldr wd, [xn] - Load Word (32-bit): wd = memory[xn] -# ldr xd, [xn] - Load Doubleword (64-bit): xd = memory[xn] -# str wd, [xn] - Store Word: memory[xn] = wd -# stp xa, xb, [sp, #-N]! - Store Pair with pre-decrement (push) -# ldp xa, xb, [sp], #N - Load Pair with post-increment (pop) -# cbz xn, label - Compare and Branch if Zero -# cmp xn, xm - Compare (sets condition flags) -# b.eq, b.ne - Branch if Equal / Not Equal -# b.lt, b.le - Branch if Less Than / Less or Equal -# b.gt, b.ge - Branch if Greater Than / Greater or Equal -# bl label - Branch and Link (function call) -# ret - Return from function -# -# ============================================================================ -# ARM64 vs RISC-V DIFFERENCES -# ============================================================================ -# -# RISC-V ARM64 -# ------ ----- -# lw a0, 0(t0) ldr w0, [x9] (load word) -# sd ra, 0(sp) stp x29, x30, [sp] (store to stack) -# beqz t1, done cbz x20, done (branch if zero) -# bge t5, t3, skip cmp x10, x22 + b.ge (compare + branch) -# jal ra, func bl func (function call) -# ret ret (return) -# -# ============================================================================ -# EXERCISES FOR STUDENTS -# ============================================================================ -# -# 1. Add calculation of the average (sum / count) -# 2. Modify the array values and verify the output changes correctly -# 3. Add a function to find the index of the maximum element -# 4. Print the array in reverse order -# -# ============================================================================ +// ============================================================================ +// SUMMARY OF NEW INSTRUCTIONS +// ============================================================================ +// +// ldr wd, [xn] - Load Word (32-bit): wd = memory[xn] +// ldr xd, [xn] - Load Doubleword (64-bit): xd = memory[xn] +// str wd, [xn] - Store Word: memory[xn] = wd +// stp xa, xb, [sp, #-N]! - Store Pair with pre-decrement (push) +// ldp xa, xb, [sp], #N - Load Pair with post-increment (pop) +// cbz xn, label - Compare and Branch if Zero +// cmp xn, xm - Compare (sets condition flags) +// b.eq, b.ne - Branch if Equal / Not Equal +// b.lt, b.le - Branch if Less Than / Less or Equal +// b.gt, b.ge - Branch if Greater Than / Greater or Equal +// bl label - Branch and Link (function call) +// ret - Return from function +// +// ============================================================================ +// ARM64 vs RISC-V DIFFERENCES +// ============================================================================ +// +// RISC-V ARM64 +// ------ ----- +// lw a0, 0(t0) ldr w0, [x9] (load word) +// sd ra, 0(sp) stp x29, x30, [sp] (store to stack) +// beqz t1, done cbz x20, done (branch if zero) +// bge t5, t3, skip cmp x10, x22 + b.ge (compare + branch) +// jal ra, func bl func (function call) +// ret ret (return) +// +// ============================================================================ +// EXERCISES FOR STUDENTS +// ============================================================================ +// +// 1. Add calculation of the average (sum / count) +// 2. Modify the array values and verify the output changes correctly +// 3. Add a function to find the index of the maximum element +// 4. Print the array in reverse order +// +// ============================================================================ diff --git a/examples/arm/fibonacci/fibonacci b/examples/arm/fibonacci/fibonacci index 4d6ebb19f36134d80b70eb0d23e531ca32e2f7b0..d07d26ba9c962e19b443bcba2d2a2e72a321c85f 100644 GIT binary patch delta 184 zcmeys^@D4I2BXD9O>@Nw3=jaLS)hDIAk7BE4ig&<_*obt6c`+KaxpMWdC1HVG4Vu! z00V;n5Hm2aK{sxL^TLEl4R0m`!G67N2au woB&imk=dNlVe(04dq$7RoGkW?5tBWEB*`N07@bwRsaA1 delta 186 zcmeyt^?_@G2BX15O>@N#1_*%BEKoipkY)p7i;0Z};tUO!7?~D*Qebe{$<4qpr4h=X zc%Wdi1ET?B$K*oBXcip>1)a?o7&RCLEf^RCpvHhH^~s9N;*%AaIanHiGLsFN-6t<# zVx3&UtN_$Hli8fnV)8|1dq#)Jf-Lro0h0rPWcTEWK=Q-ni$D@6&T0=-8OW**00M?7 AcK`qY diff --git a/examples/arm/fibonacci/fibonacci.s b/examples/arm/fibonacci/fibonacci.s index cc07ae1..6824f39 100644 --- a/examples/arm/fibonacci/fibonacci.s +++ b/examples/arm/fibonacci/fibonacci.s @@ -1,29 +1,29 @@ -# ============================================================================ -# ARM64 Assembly: Recursive Fibonacci Calculator -# ============================================================================ -# -# This program demonstrates recursive function calls in ARM64 assembly. -# It calculates the nth Fibonacci number using recursion. -# -# Fibonacci sequence: F(0)=0, F(1)=1, F(n)=F(n-1)+F(n-2) for n>1 -# Example: F(7) = 13 (sequence: 0,1,1,2,3,5,8,13,...) -# -# Learning objectives: -# - Recursive function implementation -# - Stack frame management (push/pop) -# - ARM64 calling convention (x0-x7, x30/LR, sp) -# - Base case and recursive case handling -# - Register save/restore conventions -# ============================================================================ +// ============================================================================ +// ARM64 Assembly: Recursive Fibonacci Calculator +// ============================================================================ +// +// This program demonstrates recursive function calls in ARM64 assembly. +// It calculates the nth Fibonacci number using recursion. +// +// Fibonacci sequence: F(0)=0, F(1)=1, F(n)=F(n-1)+F(n-2) for n>1 +// Example: F(7) = 13 (sequence: 0,1,1,2,3,5,8,13,...) +// +// Learning objectives: +// - Recursive function implementation +// - Stack frame management (push/pop) +// - ARM64 calling convention (x0-x7, x30/LR, sp) +// - Base case and recursive case handling +// - Register save/restore conventions +// ============================================================================ .isa arm64 .section .data - # Input value: calculate Fibonacci(7) + // Input value: calculate Fibonacci(7) fib_input: .word 7 - # Storage for result + // Storage for result fib_result: .word 0 @@ -31,139 +31,139 @@ .globl _start _start: - # Load the input value - adr x1, fib_input - ldr w0, [x1] # x0 = n (input argument) + // Load the input value + ldr x1, =fib_input + ldr w0, [x1] // x0 = n (input argument) - # Call fibonacci function - bl fibonacci # result will be in x0 + // Call fibonacci function + bl fibonacci // result will be in x0 - # Store the result - adr x1, fib_result - str w0, [x1] # save result to memory + // Store the result + ldr x1, =fib_result + str w0, [x1] // save result to memory - # Print result - mov x8, #1 # syscall 1 = print_int + // Print result + mov x8, #1 // syscall 1 = print_int svc #0 - # Print newline + // Print newline mov x0, #10 - mov x8, #11 # syscall 11 = print_char + mov x8, #11 // syscall 11 = print_char svc #0 - # Exit via syscall - mov x8, #10 # syscall 10 = exit + // Exit via syscall + mov x8, #10 // syscall 10 = exit svc #0 -# ============================================================================ -# Function: fibonacci -# Purpose: Recursively calculates the nth Fibonacci number -# -# Mathematical definition: -# F(0) = 0 -# F(1) = 1 -# F(n) = F(n-1) + F(n-2) for n > 1 -# -# ARM64 Calling Convention: -# Arguments: x0 = n (which Fibonacci number to calculate) -# Returns: x0 = F(n) (the nth Fibonacci number) -# Saved: x30/LR (link register), x19 (callee-saved register) -# -# Stack frame layout (when needed): -# sp+0: saved x0 (original n value) -# sp+16: saved x19 and x30/LR -# Total: 32 bytes (16-byte aligned) -# ============================================================================ +// ============================================================================ +// Function: fibonacci +// Purpose: Recursively calculates the nth Fibonacci number +// +// Mathematical definition: +// F(0) = 0 +// F(1) = 1 +// F(n) = F(n-1) + F(n-2) for n > 1 +// +// ARM64 Calling Convention: +// Arguments: x0 = n (which Fibonacci number to calculate) +// Returns: x0 = F(n) (the nth Fibonacci number) +// Saved: x30/LR (link register), x19 (callee-saved register) +// +// Stack frame layout (when needed): +// sp+0: saved x0 (original n value) +// sp+16: saved x19 and x30/LR +// Total: 32 bytes (16-byte aligned) +// ============================================================================ fibonacci: - # Base case 1: if n == 0, return 0 + // Base case 1: if n == 0, return 0 cbz x0, base_case_zero - # Base case 2: if n == 1, return 1 + // Base case 2: if n == 1, return 1 cmp x0, #1 b.eq base_case_one - # Recursive case: n > 1 - # We need to save registers and make recursive calls - # Save x30/LR (return address), x19 (for intermediate result), and x0 (n) - sub sp, sp, #32 # allocate stack frame (32 bytes, 16-byte aligned) - stp x19, x30, [sp, #16] # save x19 and LR - str x0, [sp] # save n + // Recursive case: n > 1 + // We need to save registers and make recursive calls + // Save x30/LR (return address), x19 (for intermediate result), and x0 (n) + sub sp, sp, #32 // allocate stack frame (32 bytes, 16-byte aligned) + stp x19, x30, [sp, #16] // save x19 and LR + str x0, [sp] // save n - # First recursive call: fibonacci(n-1) - sub x0, x0, #1 # x0 = n - 1 - bl fibonacci # call fibonacci(n-1) - mov x19, x0 # x19 = F(n-1) (save result) + // First recursive call: fibonacci(n-1) + sub x0, x0, #1 // x0 = n - 1 + bl fibonacci // call fibonacci(n-1) + mov x19, x0 // x19 = F(n-1) (save result) - # Second recursive call: fibonacci(n-2) - ldr x0, [sp] # restore original n - sub x0, x0, #2 # x0 = n - 2 - bl fibonacci # call fibonacci(n-2) + // Second recursive call: fibonacci(n-2) + ldr x0, [sp] // restore original n + sub x0, x0, #2 // x0 = n - 2 + bl fibonacci // call fibonacci(n-2) - # Add the two results: F(n) = F(n-1) + F(n-2) - add x0, x19, x0 # x0 = F(n-1) + F(n-2) + // Add the two results: F(n) = F(n-1) + F(n-2) + add x0, x19, x0 // x0 = F(n-1) + F(n-2) - # Restore saved registers and return - ldp x19, x30, [sp, #16] # restore x19 and LR - add sp, sp, #32 # deallocate stack frame - ret # return to caller + // Restore saved registers and return + ldp x19, x30, [sp, #16] // restore x19 and LR + add sp, sp, #32 // deallocate stack frame + ret // return to caller base_case_zero: - # F(0) = 0 - mov x0, #0 # return 0 + // F(0) = 0 + mov x0, #0 // return 0 ret base_case_one: - # F(1) = 1 - mov x0, #1 # return 1 + // F(1) = 1 + mov x0, #1 // return 1 ret -# ============================================================================ -# Execution trace for fibonacci(3): -# -# Call fibonacci(3): -# Not base case, make recursive calls -# Call fibonacci(2): -# Not base case, make recursive calls -# Call fibonacci(1): returns 1 -# Call fibonacci(0): returns 0 -# Return 1 + 0 = 1 -# Call fibonacci(1): returns 1 -# Return 1 + 1 = 2 -# -# Therefore: F(3) = 2 -# Similarly: F(7) = 13 -# ============================================================================ - -# ============================================================================ -# Expected Results: -# Input: n = 7 -# Output: F(7) = 13 -# -# Fibonacci sequence reference: -# F(0)=0, F(1)=1, F(2)=1, F(3)=2, F(4)=3, F(5)=5, F(6)=8, F(7)=13 -# ============================================================================ - -# ============================================================================ -# ARM64 vs RISC-V DIFFERENCES -# ============================================================================ -# -# RISC-V ARM64 -# ------ ----- -# beqz a0, label cbz x0, label (branch if zero) -# beq a0, t0, label cmp + b.eq (compare + branch) -# jal ra, func bl func (function call) -# sd ra, 16(sp) stp x19, x30, [sp] (store pair) -# ld ra, 16(sp) ldp x19, x30, [sp] (load pair) -# addi sp, sp, -24 sub sp, sp, #32 (allocate stack) -# ret ret (return) -# -# ============================================================================ -# EXERCISES FOR STUDENTS -# ============================================================================ -# -# 1. Change the input value and verify the result -# 2. Add code to count the total number of function calls -# 3. Implement an iterative (non-recursive) version -# 4. Modify to calculate factorial instead of Fibonacci -# -# ============================================================================ +// ============================================================================ +// Execution trace for fibonacci(3): +// +// Call fibonacci(3): +// Not base case, make recursive calls +// Call fibonacci(2): +// Not base case, make recursive calls +// Call fibonacci(1): returns 1 +// Call fibonacci(0): returns 0 +// Return 1 + 0 = 1 +// Call fibonacci(1): returns 1 +// Return 1 + 1 = 2 +// +// Therefore: F(3) = 2 +// Similarly: F(7) = 13 +// ============================================================================ + +// ============================================================================ +// Expected Results: +// Input: n = 7 +// Output: F(7) = 13 +// +// Fibonacci sequence reference: +// F(0)=0, F(1)=1, F(2)=1, F(3)=2, F(4)=3, F(5)=5, F(6)=8, F(7)=13 +// ============================================================================ + +// ============================================================================ +// ARM64 vs RISC-V DIFFERENCES +// ============================================================================ +// +// RISC-V ARM64 +// ------ ----- +// beqz a0, label cbz x0, label (branch if zero) +// beq a0, t0, label cmp + b.eq (compare + branch) +// jal ra, func bl func (function call) +// sd ra, 16(sp) stp x19, x30, [sp] (store pair) +// ld ra, 16(sp) ldp x19, x30, [sp] (load pair) +// addi sp, sp, -24 sub sp, sp, #32 (allocate stack) +// ret ret (return) +// +// ============================================================================ +// EXERCISES FOR STUDENTS +// ============================================================================ +// +// 1. Change the input value and verify the result +// 2. Add code to count the total number of function calls +// 3. Implement an iterative (non-recursive) version +// 4. Modify to calculate factorial instead of Fibonacci +// +// ============================================================================ diff --git a/examples/arm/guess_game/guess_game b/examples/arm/guess_game/guess_game index a87b860080f344ad7093b94990e1a2d3f645c072..692de3d49a02f42aa5213c092d5b97c51664335c 100644 GIT binary patch delta 531 zcmew$Fhg*H24loTO>;#DMg}l|(JT-y14x(+h(Anh)OTQDXNc%vXt)Fvyz+nz!FE7m zCm^vsfox+&hn*roJqkefWDZ6d3!n`ls~H$V6j%`keE$Dm`~eGs{~pL+fW&VAvM1*- zO4Kg^YCOPGY9!fJn=|m_!1xlAf>CI637?i%r zARq}g=n+)lC6v|z8b0|Vqxj?pj2xSvGqNx;I!u1U6l*0XCl5mkatbPPazIi+0YpJC zPz0y|EC8fI3`G#hz{t3{o!OF!(Pr{NR&mJ^pgTd%hk;5kgOQ;#NMg}l|(JT-y14x(+h>uKc)Q@OjXt>0{wCGa@kOoRzIlzL- zN0)!Vjw;Uy)W-?bZ_MbhQv}FX;DN|b{2)``z{n7Cnt>stK?qgP$N&Gu7x1Bqy#b0P zh@pxdkU-@>U=~@W0d?nmHinM}fMzR5qpD$LU=VST0gEZZJ*6Ovs>p*8;@Qm~7}XgW zMJ7uy$J*)0$-|I>j)IDgoPwN!f`Xi!f(`@&MSu!G0w5YBtO(*VFoHvQ^Ll1WCdmK> z1_5Be!a+Wm#mF#WG9#O~r@2J>Vi4m-w-$%P#10I2_qb^rhX diff --git a/examples/arm/guess_game/guess_game.s b/examples/arm/guess_game/guess_game.s index e9373ff..dda83c9 100644 --- a/examples/arm/guess_game/guess_game.s +++ b/examples/arm/guess_game/guess_game.s @@ -1,26 +1,26 @@ -# ============================================================================ -# guess_game.s - Number Guessing Game -# ============================================================================ -# -# An interactive game that demonstrates: -# 1. Reading user input (read_int syscall) -# 2. Game loop with win condition -# 3. Conditional logic (too high, too low, correct) -# 4. Counting iterations -# -# Try to guess the secret number! The program will tell you if your -# guess is too high or too low. -# -# ============================================================================ +// ============================================================================ +// guess_game.s - Number Guessing Game +// ============================================================================ +// +// An interactive game that demonstrates: +// 1. Reading user input (read_int syscall) +// 2. Game loop with win condition +// 3. Conditional logic (too high, too low, correct) +// 4. Counting iterations +// +// Try to guess the secret number! The program will tell you if your +// guess is too high or too low. +// +// ============================================================================ .isa arm64 .data -# The secret number - change this to play with different values! +// The secret number - change this to play with different values! secret: .word 42 -# Game messages +// Game messages welcome: .asciz "================================\n" title: .asciz " NUMBER GUESSING GAME\n" rules: .asciz "Guess a number between 1 and 100\n" @@ -36,158 +36,158 @@ thanks: .asciz "\nThanks for playing!\n" .text .globl _start -# ============================================================================ -# _start: Program Entry Point -# ============================================================================ +// ============================================================================ +// _start: Program Entry Point +// ============================================================================ _start: - # Print welcome banner - adr x0, welcome + // Print welcome banner + ldr x0, =welcome mov x8, #4 svc #0 - adr x0, title + ldr x0, =title mov x8, #4 svc #0 - adr x0, welcome + ldr x0, =welcome mov x8, #4 svc #0 - adr x0, rules + ldr x0, =rules mov x8, #4 svc #0 - # Load the secret number into x19 (callee-saved register) - adr x9, secret - ldr w19, [x9] # x19 = secret number (use w19 for 32-bit load) + // Load the secret number into x19 (callee-saved register) + ldr x9, =secret + ldr w19, [x9] // x19 = secret number (use w19 for 32-bit load) - # Initialize guess counter - mov x20, #0 # x20 = number of guesses + // Initialize guess counter + mov x20, #0 // x20 = number of guesses -# ============================================================================ -# game_loop: Main game loop -# ============================================================================ +// ============================================================================ +// game_loop: Main game loop +// ============================================================================ game_loop: - # Print prompt - adr x0, prompt + // Print prompt + ldr x0, =prompt mov x8, #4 svc #0 - # Read user's guess - mov x8, #5 # Syscall 5 = read_int - svc #0 # Result is in x0 + // Read user's guess + mov x8, #5 // Syscall 5 = read_int + svc #0 // Result is in x0 - # Increment guess counter + // Increment guess counter add x20, x20, #1 - # Store guess in x9 for comparisons - mov x9, x0 # x9 = user's guess + // Store guess in x9 for comparisons + mov x9, x0 // x9 = user's guess - # Compare guess with secret - cmp x9, x19 # Compare guess with secret - b.eq win # if guess == secret, we win! - b.lt guess_low # if guess < secret, too low + // Compare guess with secret + cmp x9, x19 // Compare guess with secret + b.eq win // if guess == secret, we win! + b.lt guess_low // if guess < secret, too low - # If we get here, guess > secret (too high) - adr x0, too_high + // If we get here, guess > secret (too high) + ldr x0, =too_high mov x8, #4 svc #0 - b game_loop # Try again + b game_loop // Try again guess_low: - adr x0, too_low + ldr x0, =too_low mov x8, #4 svc #0 - b game_loop # Try again + b game_loop // Try again -# ============================================================================ -# win: Player guessed correctly! -# ============================================================================ +// ============================================================================ +// win: Player guessed correctly! +// ============================================================================ win: - # Print victory message - adr x0, correct1 + // Print victory message + ldr x0, =correct1 mov x8, #4 svc #0 - adr x0, correct2 + ldr x0, =correct2 mov x8, #4 svc #0 - # Print number of guesses + // Print number of guesses mov x0, x20 mov x8, #1 svc #0 - # Print "try!" or "tries!" depending on count + // Print "try!" or "tries!" depending on count cmp x20, #1 - b.eq one_guess # if count == 1, use singular + b.eq one_guess // if count == 1, use singular - adr x0, correct3 # "tries!" + ldr x0, =correct3 // "tries!" mov x8, #4 svc #0 b end_game one_guess: - adr x0, one_try # "try!" + ldr x0, =one_try // "try!" mov x8, #4 svc #0 end_game: - # Print thank you message - adr x0, thanks + // Print thank you message + ldr x0, =thanks mov x8, #4 svc #0 - # Exit program + // Exit program mov x8, #10 svc #0 -# ============================================================================ -# SYSCALL REFERENCE -# ============================================================================ -# -# Syscall #1 (print_int): Print integer in x0 -# Syscall #4 (print_string): Print string at address in x0 -# Syscall #5 (read_int): Read integer, result in x0 -# Syscall #10 (exit): Exit program -# -# ============================================================================ -# HOW THE GAME WORKS -# ============================================================================ -# -# 1. The secret number (42 by default) is loaded into register x19 -# 2. A counter (x20) tracks how many guesses the player makes -# 3. Each iteration: -# - Prompt the user for input -# - Read their guess using syscall #5 -# - Compare guess to secret: -# - Equal? Jump to win condition -# - Less? Print "too low" and loop -# - Greater? Print "too high" and loop -# 4. When correct, print the number of attempts and exit -# -# ============================================================================ -# ARM64 vs RISC-V DIFFERENCES -# ============================================================================ -# -# RISC-V ARM64 -# ------ ----- -# beq t1, s0, win cmp x9, x19 + b.eq win (compare + branch) -# blt t1, s0, low cmp x9, x19 + b.lt low (compare + branch) -# j game_loop b game_loop (unconditional branch) -# lw s0, 0(t0) ldr w19, [x9] (load word) -# mv t1, a0 mov x9, x0 (move register) -# addi s1, s1, 1 add x20, x20, #1 (add immediate) -# s0-s11 (saved) x19-x28 (callee-saved) -# -# ============================================================================ -# EXERCISES FOR STUDENTS -# ============================================================================ -# -# 1. Change the secret number and play the game -# 2. Add a maximum number of guesses (e.g., 7 tries) -# 3. Add input validation (check if guess is between 1 and 100) -# 4. Add a "play again?" feature after winning -# 5. Keep track of best score across multiple games -# -# ============================================================================ +// ============================================================================ +// SYSCALL REFERENCE +// ============================================================================ +// +// Syscall #1 (print_int): Print integer in x0 +// Syscall #4 (print_string): Print string at address in x0 +// Syscall #5 (read_int): Read integer, result in x0 +// Syscall #10 (exit): Exit program +// +// ============================================================================ +// HOW THE GAME WORKS +// ============================================================================ +// +// 1. The secret number (42 by default) is loaded into register x19 +// 2. A counter (x20) tracks how many guesses the player makes +// 3. Each iteration: +// - Prompt the user for input +// - Read their guess using syscall #5 +// - Compare guess to secret: +// - Equal? Jump to win condition +// - Less? Print "too low" and loop +// - Greater? Print "too high" and loop +// 4. When correct, print the number of attempts and exit +// +// ============================================================================ +// ARM64 vs RISC-V DIFFERENCES +// ============================================================================ +// +// RISC-V ARM64 +// ------ ----- +// beq t1, s0, win cmp x9, x19 + b.eq win (compare + branch) +// blt t1, s0, low cmp x9, x19 + b.lt low (compare + branch) +// j game_loop b game_loop (unconditional branch) +// lw s0, 0(t0) ldr w19, [x9] (load word) +// mv t1, a0 mov x9, x0 (move register) +// addi s1, s1, 1 add x20, x20, #1 (add immediate) +// s0-s11 (saved) x19-x28 (callee-saved) +// +// ============================================================================ +// EXERCISES FOR STUDENTS +// ============================================================================ +// +// 1. Change the secret number and play the game +// 2. Add a maximum number of guesses (e.g., 7 tries) +// 3. Add input validation (check if guess is between 1 and 100) +// 4. Add a "play again?" feature after winning +// 5. Keep track of best score across multiple games +// +// ============================================================================ diff --git a/examples/arm/hello_asm/hello_asm b/examples/arm/hello_asm/hello_asm index 668f8465461ed93dd5d2b68cf0a2a9882e5f3b6c..d27cb7833ee168cd19b488737e2dd86445097771 100644 GIT binary patch delta 197 zcmdnMb%1Mv1``AGL@hJL39is&aCQAVOJg^99BRMxaz6 svpJ*08ZI#~E&9{}q=6Dw9GD>d zi3cfO>@Nwj0|7^qgfza29Pit5Ian4)E7Cx#Srljh?%$4oG|$sQ#`AjoPvVfWGiNC5jh1Qlv9uglX7zMK(^fGPG){4Ne!SyAbVgS z1k7M$@R+=jRa_9rLy^s%e2`Uq@&{H9$u(#)4N!e&CJVBO3tmB!ojO^OO?+|y8;9f% zG#Q{dK)}c#Fu9OTT+#-}01H5gBVd}5!DsS9wtS$k1li3Q9VUCS+cO@W+zBMFPCm(Q h&m_b=nTx}Yal>R!4*SXR>>QIPa9A*2n0%5$9RTG^VqX9N delta 495 zcmeAW`XD$#gK@(|O>@NxMg}l|(JT-y14x(+h&3iQ>Z>p?G+bg}TJ))rrQs4IkY?g+ zxWojcIT#qGH1I;?CLXy{FANkH28xRS)z4RB`1p#6A>;r*M7{$k1~l{vCy=)1W%!uI z$PkiHRkA|^Dy|@as%{D!!$eJ1hKXPQ{})d{lY0pi`yA}J@^AH_6$hB(uYnxP#K17w zooN|k#pHKP@vJ&JdI~y|gP5&FbQI(ibmSD|=dciz-5s= 3, exit outer loop + b.ge end_outer // if i >= 3, exit outer loop - # Initialize middle loop: j = 0 - mov x10, #0 # x10 = j (column index) + // Initialize middle loop: j = 0 + mov x10, #0 // x10 = j (column index) middle_loop: cmp x10, #3 - b.ge end_middle # if j >= 3, exit middle loop + b.ge end_middle // if j >= 3, exit middle loop - # Initialize accumulator: C[i][j] = 0 - mov x12, #0 # x12 = accumulator for C[i][j] + // Initialize accumulator: C[i][j] = 0 + mov x12, #0 // x12 = accumulator for C[i][j] - # Initialize inner loop: k = 0 - mov x11, #0 # x11 = k + // Initialize inner loop: k = 0 + mov x11, #0 // x11 = k inner_loop: cmp x11, #3 - b.ge end_inner # if k >= 3, exit inner loop + b.ge end_inner // if k >= 3, exit inner loop - # Calculate address of A[i][k] - # Address = base + (i * 3 + k) * 4 + // Calculate address of A[i][k] + // Address = base + (i * 3 + k) * 4 mov x15, #3 - mul x13, x9, x15 # x13 = i * 3 - add x13, x13, x11 # x13 = i * 3 + k - lsl x13, x13, #2 # x13 = (i * 3 + k) * 4 (word offset) - add x13, x19, x13 # x13 = address of A[i][k] - ldr w13, [x13] # x13 = A[i][k] value - - # Calculate address of B[k][j] - # Address = base + (k * 3 + j) * 4 + mul x13, x9, x15 // x13 = i * 3 + add x13, x13, x11 // x13 = i * 3 + k + lsl x13, x13, #2 // x13 = (i * 3 + k) * 4 (word offset) + add x13, x19, x13 // x13 = address of A[i][k] + ldr w13, [x13] // x13 = A[i][k] value + + // Calculate address of B[k][j] + // Address = base + (k * 3 + j) * 4 mov x15, #3 - mul x14, x11, x15 # x14 = k * 3 - add x14, x14, x10 # x14 = k * 3 + j - lsl x14, x14, #2 # x14 = (k * 3 + j) * 4 (word offset) - add x14, x20, x14 # x14 = address of B[k][j] - ldr w14, [x14] # x14 = B[k][j] value - - # Multiply and accumulate: C[i][j] += A[i][k] * B[k][j] - mul x15, x13, x14 # x15 = A[i][k] * B[k][j] - add x12, x12, x15 # accumulator += A[i][k] * B[k][j] - - # Increment inner loop counter - add x11, x11, #1 # k++ + mul x14, x11, x15 // x14 = k * 3 + add x14, x14, x10 // x14 = k * 3 + j + lsl x14, x14, #2 // x14 = (k * 3 + j) * 4 (word offset) + add x14, x20, x14 // x14 = address of B[k][j] + ldr w14, [x14] // x14 = B[k][j] value + + // Multiply and accumulate: C[i][j] += A[i][k] * B[k][j] + mul x15, x13, x14 // x15 = A[i][k] * B[k][j] + add x12, x12, x15 // accumulator += A[i][k] * B[k][j] + + // Increment inner loop counter + add x11, x11, #1 // k++ b inner_loop end_inner: - # Store result: C[i][j] = accumulator - # Address = base + (i * 3 + j) * 4 + // Store result: C[i][j] = accumulator + // Address = base + (i * 3 + j) * 4 mov x15, #3 - mul x13, x9, x15 # x13 = i * 3 - add x13, x13, x10 # x13 = i * 3 + j - lsl x13, x13, #2 # x13 = (i * 3 + j) * 4 (word offset) - add x13, x21, x13 # x13 = address of C[i][j] - str w12, [x13] # C[i][j] = accumulator - - # Increment middle loop counter - add x10, x10, #1 # j++ + mul x13, x9, x15 // x13 = i * 3 + add x13, x13, x10 // x13 = i * 3 + j + lsl x13, x13, #2 // x13 = (i * 3 + j) * 4 (word offset) + add x13, x21, x13 // x13 = address of C[i][j] + str w12, [x13] // C[i][j] = accumulator + + // Increment middle loop counter + add x10, x10, #1 // j++ b middle_loop end_middle: - # Increment outer loop counter - add x9, x9, #1 # i++ + // Increment outer loop counter + add x9, x9, #1 // i++ b outer_loop end_outer: - # Restore callee-saved registers + // Restore callee-saved registers ldp x21, x22, [sp, #32] ldp x19, x20, [sp, #16] ldp x29, x30, [sp], #48 - # Return to caller + // Return to caller ret -# ============================================================================ -# Function: print_matrix -# Purpose: Prints the result matrix C -# ============================================================================ +// ============================================================================ +// Function: print_matrix +// Purpose: Prints the result matrix C +// ============================================================================ print_matrix: stp x29, x30, [sp, #-32]! stp x19, x20, [sp, #16] mov x29, sp - adr x19, matrix_c # x19 = matrix C address - mov x20, #0 # x20 = element counter (0-8) + ldr x19, =matrix_c // x19 = matrix C address + mov x20, #0 // x20 = element counter (0-8) print_row: cmp x20, #9 b.ge print_done - # Print row indent - adr x0, msg_row + // Print row indent + ldr x0, =msg_row mov x8, #4 svc #0 - # Print 3 elements - mov x9, #0 # column counter + // Print 3 elements + mov x9, #0 // column counter print_col: cmp x9, #3 b.ge print_newline - # Load and print element + // Load and print element ldr w0, [x19, x20, lsl #2] mov x8, #1 svc #0 - # Print space - adr x0, space + // Print space + ldr x0, =space mov x8, #4 svc #0 @@ -220,7 +220,7 @@ print_col: b print_col print_newline: - adr x0, newline + ldr x0, =newline mov x8, #4 svc #0 b print_row @@ -230,33 +230,33 @@ print_done: ldp x29, x30, [sp], #32 ret -# ============================================================================ -# Expected Result Matrix C: -# C = A × B = -# [ 30 24 18 ] (1*9+2*6+3*3=30, 1*8+2*5+3*2=24, 1*7+2*4+3*1=18) -# [ 84 69 54 ] (4*9+5*6+6*3=84, 4*8+5*5+6*2=69, 4*7+5*4+6*1=54) -# [138 114 90 ] (7*9+8*6+9*3=138, 7*8+8*5+9*2=114, 7*7+8*4+9*1=90) -# ============================================================================ - -# ============================================================================ -# ARM64 vs RISC-V DIFFERENCES -# ============================================================================ -# -# RISC-V ARM64 -# ------ ----- -# mul t4, t0, t6 mul x13, x9, x15 (multiply) -# slli t4, t4, 2 lsl x13, x13, #2 (shift left) -# lw t4, 0(t4) ldr w13, [x13] (load word) -# sw t3, 0(t4) str w12, [x13] (store word) -# bge t0, t6, end cmp + b.ge (compare + branch) -# -# ============================================================================ -# EXERCISES FOR STUDENTS -# ============================================================================ -# -# 1. Modify the matrices A and B and verify the output -# 2. Add code to print matrices A and B before multiplication -# 3. Implement matrix addition instead of multiplication -# 4. Extend to handle 4x4 matrices -# -# ============================================================================ +// ============================================================================ +// Expected Result Matrix C: +// C = A × B = +// [ 30 24 18 ] (1*9+2*6+3*3=30, 1*8+2*5+3*2=24, 1*7+2*4+3*1=18) +// [ 84 69 54 ] (4*9+5*6+6*3=84, 4*8+5*5+6*2=69, 4*7+5*4+6*1=54) +// [138 114 90 ] (7*9+8*6+9*3=138, 7*8+8*5+9*2=114, 7*7+8*4+9*1=90) +// ============================================================================ + +// ============================================================================ +// ARM64 vs RISC-V DIFFERENCES +// ============================================================================ +// +// RISC-V ARM64 +// ------ ----- +// mul t4, t0, t6 mul x13, x9, x15 (multiply) +// slli t4, t4, 2 lsl x13, x13, #2 (shift left) +// lw t4, 0(t4) ldr w13, [x13] (load word) +// sw t3, 0(t4) str w12, [x13] (store word) +// bge t0, t6, end cmp + b.ge (compare + branch) +// +// ============================================================================ +// EXERCISES FOR STUDENTS +// ============================================================================ +// +// 1. Modify the matrices A and B and verify the output +// 2. Add code to print matrices A and B before multiplication +// 3. Implement matrix addition instead of multiplication +// 4. Extend to handle 4x4 matrices +// +// ============================================================================ diff --git a/mapachespim/toolchain/arm64.py b/mapachespim/toolchain/arm64.py new file mode 100644 index 0000000..06a9b19 --- /dev/null +++ b/mapachespim/toolchain/arm64.py @@ -0,0 +1,1012 @@ +""" +Pure-Python ARM64 (AArch64) instruction encoder, compatible with GNU as. + +Every A64 instruction is 4 bytes. Where the architecture allows several +encodings, the same choice as GNU as is made (for example ``mov`` of a +constant picks movz, then movn, then a bitmask orr; ``add x0, x1, #-5`` is +``sub x0, x1, #5``; misaligned offsets use ldur/stur). + +Supported (64-bit X and 32-bit W forms): + - Arithmetic: add, adds, sub, subs, neg, negs, cmp, cmn, adc, sbc, + mul, madd, msub, mneg, smull, umull, smulh, umulh, smaddl, umaddl, + sdiv, udiv, with immediates, shifted and extended registers + - Logic: and, ands, orr, eor, bic, bics, orn, eon, tst, mvn, with + bitmask immediates or shifted registers + - Moves: mov (register, sp, or constant), movz, movn, movk + - Shifts and bitfields: lsl, lsr, asr, ror (immediate or register), + sxtb, sxth, sxtw, uxtb, uxth, ubfx, sbfx, ubfiz, sbfiz, bfi, bfxil, + ubfm, sbfm, bfm, extr + - Conditional: csel, csinc, csinv, csneg, cset, csetm, cinc, cinv, + cneg, ccmp, ccmn + - Loads/stores: ldr, str, ldrb, strb, ldrh, strh, ldrsb, ldrsh, ldrsw, + ldur/stur variants, ldp, stp, ldpsw; offset, pre-index, post-index, + register offset, and literal (label) addressing; ``ldr x0, =value`` + through a literal pool + - Branches: b, bl, br, blr, ret, b., cbz, cbnz, tbz, tbnz + - Addresses: adr, adrp, with ``:lo12:symbol`` for add and loads/stores + - System: svc, brk, hlt, nop +""" + +from __future__ import annotations + +import re +from typing import Dict, List, Optional, Tuple, cast + +from .expr import Context, EncodeError, ExpressionError, parse_int, sign_extend + +INSTR_SIZE = 4 + + +class ARM64EncodeError(EncodeError): + """Raised when an ARM64 instruction cannot be encoded.""" + + +# --------------------------------------------------------------------------- +# Registers and operands +# --------------------------------------------------------------------------- + + +class Reg: + """A general-purpose register operand.""" + + __slots__ = ("num", "is64", "kind") + + def __init__(self, num: int, is64: bool, kind: str = "gp"): + self.num = num # 0-31 + self.is64 = is64 + self.kind = kind # "gp", "sp", or "zr" + + def __repr__(self) -> str: + return f"Reg({self.num}, {'x' if self.is64 else 'w'}, {self.kind})" + + +def parse_reg(text: str) -> Optional[Reg]: + """Parse a register name, or return None if it is not one.""" + t = text.strip().lower() + if t in ("sp", "wsp"): + return Reg(31, t == "sp", "sp") + if t in ("xzr", "wzr"): + return Reg(31, t == "xzr", "zr") + if t == "fp": + return Reg(29, True) + if t == "lr": + return Reg(30, True) + m = re.fullmatch(r"([xw])(\d+)", t) + if m and int(m.group(2)) <= 30: + return Reg(int(m.group(2)), m.group(1) == "x") + return None + + +def _reg(text: str, what: str = "register") -> Reg: + r = parse_reg(text) + if r is None: + raise ARM64EncodeError(f"expected a {what}, got '{text.strip()}'") + return r + + +CONDITIONS = { + "eq": 0, "ne": 1, "cs": 2, "hs": 2, "cc": 3, "lo": 3, "mi": 4, "pl": 5, + "vs": 6, "vc": 7, "hi": 8, "ls": 9, "ge": 10, "lt": 11, "gt": 12, "le": 13, + "al": 14, "nv": 15, +} # fmt: skip + +SHIFTS = {"lsl": 0, "lsr": 1, "asr": 2, "ror": 3} +EXTENDS = { + "uxtb": 0, "uxth": 1, "uxtw": 2, "uxtx": 3, "sxtb": 4, "sxth": 5, "sxtw": 6, "sxtx": 7, +} # fmt: skip + + +def split_operands(text: str) -> List[str]: + """Split on commas outside [], (), and quotes.""" + ops: List[str] = [] + depth = 0 + cur = "" + quote = False + for ch in text: + if ch == "'" and depth == 0: + quote = not quote + if not quote: + if ch in "[(": + depth += 1 + elif ch in "])": + depth -= 1 + elif ch == "," and depth == 0: + ops.append(cur.strip()) + cur = "" + continue + cur += ch + if cur.strip() or ops: + ops.append(cur.strip()) + return ops + + +def _same_size(*regs: Reg) -> None: + if len({r.is64 for r in regs}) > 1: + raise ARM64EncodeError("operands must all be X registers or all be W registers") + + +def _mask(bits: int) -> int: + return (1 << bits) - 1 + + +# --------------------------------------------------------------------------- +# Immediates +# --------------------------------------------------------------------------- + + +def encode_bitmask(value: int, bits: int) -> Optional[Tuple[int, int, int]]: + """(N, immr, imms) for a logical immediate, or None if not encodable.""" + value &= _mask(bits) + if value == 0 or value == _mask(bits): + return None + # Smallest repeating element + size = bits + while size > 2: + half = size // 2 + if value & _mask(half) != (value >> half) & _mask(half): + break + size = half + elem = value & _mask(size) + ones = bin(elem).count("1") + pattern = _mask(ones) + for rotation in range(size): + rotated = ((pattern >> rotation) | (pattern << (size - rotation))) & _mask(size) + if rotated == elem: + n = 1 if size == 64 else 0 + imms = ((-size << 1) & 0x3F) | (ones - 1) + return n, rotation, imms + return None + + +def _move_wide(value: int, bits: int) -> Optional[Tuple[int, int, int]]: + """(opc, hw, imm16) for movz (opc 2) or movn (opc 0), or None.""" + value &= _mask(bits) + for opc, candidate in ((2, value), (0, ~value & _mask(bits))): + for hw in range(bits // 16): + if candidate & ~(0xFFFF << (16 * hw)) & _mask(bits) == 0: + return opc, hw, (candidate >> (16 * hw)) & 0xFFFF + return None + + +# --------------------------------------------------------------------------- +# Encoder +# --------------------------------------------------------------------------- + +LOAD_STORE = { + # name: (size, opc, access size in bytes, register width 64?) per register kind + "strb": (0, 0, 1), "ldrb": (0, 1, 1), "ldrsb": (0, None, 1), + "strh": (1, 0, 2), "ldrh": (1, 1, 2), "ldrsh": (1, None, 2), + "ldrsw": (2, 2, 4), + "str": (None, 0, None), "ldr": (None, 1, None), +} # fmt: skip +UNSCALED = { + "sturb": "strb", "ldurb": "ldrb", "ldursb": "ldrsb", "sturh": "strh", "ldurh": "ldrh", + "ldursh": "ldrsh", "ldursw": "ldrsw", "stur": "str", "ldur": "ldr", +} # fmt: skip + + +class Encoder: + """Encodes one instruction at a known address.""" + + def __init__(self, ctx: Context): + self.ctx = ctx + self.pc = ctx.address + + # --- operand helpers --- + + def imm(self, text: str) -> int: + t = text.strip() + if t.startswith("#"): + t = t[1:].strip() + if t.lower().startswith(":lo12:"): + return self.ctx.eval(t[6:]) & 0xFFF + return self.ctx.eval(t) + + def is_imm(self, text: str) -> bool: + t = text.strip() + return t.startswith("#") or (parse_reg(t) is None and not t.startswith("[")) + + def target(self, text: str) -> int: + """Branch/literal target: an address (label or expression).""" + t = text.strip() + if t.startswith("#"): + t = t[1:] + value = self.ctx.eval(t) + return value + + def pc_offset(self, text: str, bits: int, scale: int = 4, what: str = "branch target") -> int: + if not self.ctx.strict: + self.ctx.eval(text.strip().lstrip("#")) + return 0 + offset = self.target(text) - self.pc + if offset % scale: + raise ARM64EncodeError(f"{what} is not aligned to {scale} bytes") + offset //= scale + lo, hi = -(1 << (bits - 1)), (1 << (bits - 1)) - 1 + if not lo <= offset <= hi: + reach = (1 << (bits - 1)) * scale + raise ARM64EncodeError(f"{what} out of range (+/-{reach // 1024}KB)") + return offset & _mask(bits) + + def shift_amount(self, text: str, kinds: Dict[str, int]) -> Tuple[int, int]: + """Parse 'lsl #3' style modifiers: (kind code, amount).""" + parts = text.strip().split(None, 1) + name = parts[0].lower() + if name not in kinds: + raise ARM64EncodeError(f"unexpected '{text.strip()}'") + amount = self.imm(parts[1]) if len(parts) > 1 else 0 + return kinds[name], amount + + # --- instruction families --- + + def add_sub(self, op: int, s: int, ops: List[str]) -> int: + """add/adds/sub/subs rd, rn, operand.""" + if len(ops) not in (3, 4): + raise ARM64EncodeError("expected rd, rn, operand") + rd, rn = _reg(ops[0]), _reg(ops[1]) + sf = int(rd.is64) + if self.is_imm(ops[2]): + _same_size(rd, rn) + value = self.imm(ops[2]) + shift = 0 + if len(ops) == 4: + kind, amount = self.shift_amount(ops[3], {"lsl": 0}) + if amount not in (0, 12): + raise ARM64EncodeError("the shift must be lsl #0 or lsl #12") + shift = amount // 12 + elif value < 0 and ops[2].strip().lstrip("#").strip()[:1] != ":": + # add x0, x1, #-5 -> sub x0, x1, #5 (as GNU as does) + value, op = -value, op ^ 1 + if not shift and value > 0xFFF and value & 0xFFF == 0 and value >> 12 <= 0xFFF: + value, shift = value >> 12, 1 + if not 0 <= value <= 0xFFF: + raise ARM64EncodeError(f"immediate {value} out of range (0 to 4095)") + if rd.kind == "zr" and not s: + raise ARM64EncodeError("the destination of add/sub (immediate) cannot be xzr") + if rn.kind == "zr": + raise ARM64EncodeError("the first source cannot be xzr here; use sp or a register") + return ( + (sf << 31) + | (op << 30) + | (s << 29) + | (0x22 << 23) + | (shift << 22) + | (value << 10) + | (rn.num << 5) + | rd.num + ) + rm = _reg(ops[2]) + uses_sp = rn.kind == "sp" or (rd.kind == "sp" and not s) + extend = None + if len(ops) == 4: + name = ops[3].strip().split()[0].lower() + if name in EXTENDS or (uses_sp and name == "lsl"): + extend = ops[3] + if uses_sp or extend is not None: + # Extended register form (the only one that accepts sp) + if extend is not None: + name = extend.strip().split()[0].lower() + amount = ( + self.imm(extend.strip().split(None, 1)[1]) if len(extend.split()) > 1 else 0 + ) + option = EXTENDS.get(name, 3 if sf else 2) + else: + amount, option = 0, 3 if sf else 2 + if not 0 <= amount <= 4: + raise ARM64EncodeError("extend shift must be 0 to 4") + if rd.kind == "zr" and not s: + raise ARM64EncodeError("invalid use of xzr") + return ( + (sf << 31) + | (op << 30) + | (s << 29) + | (0x59 << 21) + | (rm.num << 16) + | (option << 13) + | (amount << 10) + | (rn.num << 5) + | rd.num + ) + _same_size(rd, rn, rm) + shift, amount = (0, 0) + if len(ops) == 4: + shift, amount = self.shift_amount(ops[3], {"lsl": 0, "lsr": 1, "asr": 2}) + if not 0 <= amount < (64 if sf else 32): + raise ARM64EncodeError(f"shift amount {amount} out of range") + return ( + (sf << 31) + | (op << 30) + | (s << 29) + | (0x0B << 24) + | (shift << 22) + | (rm.num << 16) + | (amount << 10) + | (rn.num << 5) + | rd.num + ) + + def logical(self, opc: int, invert: int, ops: List[str]) -> int: + """and/orr/eor/ands (and bic/orn/eon/bics when invert).""" + if len(ops) not in (3, 4): + raise ARM64EncodeError("expected rd, rn, operand") + rd, rn = _reg(ops[0]), _reg(ops[1]) + _same_size(rd, rn) + sf = int(rd.is64) + bits = 64 if sf else 32 + if self.is_imm(ops[2]): + if len(ops) == 4: + raise ARM64EncodeError("a logical immediate cannot be shifted") + value = self.imm(ops[2]) + if invert: + value = ~value + if not sf and not -(1 << 31) <= value < (1 << 32): + raise ARM64EncodeError(f"immediate {value:#x} out of range for a W register") + enc = encode_bitmask(value, bits) + if enc is None: + raise ARM64EncodeError( + f"immediate {self.imm(ops[2]):#x} is not a valid bitmask immediate" + ) + n, immr, imms = enc + if rd.kind == "zr" and opc != 3: + pass # orr xzr would be odd but GNU treats register 31 as sp here + return ( + (sf << 31) + | (opc << 29) + | (0x24 << 23) + | (n << 22) + | (immr << 16) + | (imms << 10) + | (rn.num << 5) + | rd.num + ) + rm = _reg(ops[2]) + _same_size(rd, rm) + shift, amount = (0, 0) + if len(ops) == 4: + shift, amount = self.shift_amount(ops[3], SHIFTS) + if not 0 <= amount < bits: + raise ARM64EncodeError(f"shift amount {amount} out of range") + for r in (rd, rn, rm): + if r.kind == "sp": + raise ARM64EncodeError("sp cannot be used in this instruction") + return ( + (sf << 31) + | (opc << 29) + | (0x0A << 24) + | (shift << 22) + | (invert << 21) + | (rm.num << 16) + | (amount << 10) + | (rn.num << 5) + | rd.num + ) + + def move_wide(self, opc: int, ops: List[str]) -> int: + """movz/movn/movk rd, #imm{, lsl #shift}.""" + if len(ops) not in (2, 3): + raise ARM64EncodeError("expected rd, #imm{, lsl #n}") + rd = _reg(ops[0]) + sf = int(rd.is64) + value = self.imm(ops[1]) + hw = 0 + if len(ops) == 3: + _, amount = self.shift_amount(ops[2], {"lsl": 0}) + if amount % 16 or amount >= (64 if sf else 32): + raise ARM64EncodeError( + "shift must be lsl #0, #16" + (", #32, or #48" if sf else "") + ) + hw = amount // 16 + if not 0 <= value <= 0xFFFF: + raise ARM64EncodeError(f"immediate {value} out of range (0 to 65535)") + return (sf << 31) | (opc << 29) | (0x25 << 23) | (hw << 21) | (value << 5) | rd.num + + def bitfield(self, opc: int, rd: Reg, rn: Reg, immr: int, imms: int) -> int: + _same_size(rd, rn) + sf = int(rd.is64) + bits = 64 if sf else 32 + if not (0 <= immr < bits and 0 <= imms < bits): + raise ARM64EncodeError("bit position out of range") + return ( + (sf << 31) + | (opc << 29) + | (0x26 << 23) + | (sf << 22) + | (immr << 16) + | (imms << 10) + | (rn.num << 5) + | rd.num + ) + + def data2(self, opcode: int, ops: List[str]) -> int: + """Two-source data processing: udiv, sdiv, lslv, lsrv, asrv, rorv.""" + if len(ops) != 3: + raise ARM64EncodeError("expected rd, rn, rm") + rd, rn, rm = (_reg(o) for o in ops) + _same_size(rd, rn, rm) + return ( + (int(rd.is64) << 31) + | (0xD6 << 21) + | (rm.num << 16) + | (opcode << 10) + | (rn.num << 5) + | rd.num + ) + + def data3(self, op31: int, o0: int, rd: Reg, rn: Reg, rm: Reg, ra: Reg, sf: int) -> int: + return ( + (sf << 31) + | (0x1B << 24) + | (op31 << 21) + | (rm.num << 16) + | (o0 << 15) + | (ra.num << 10) + | (rn.num << 5) + | rd.num + ) + + def cond_select(self, op: int, op2: int, rd: Reg, rn: Reg, rm: Reg, cond: int) -> int: + _same_size(rd, rn, rm) + return ( + (int(rd.is64) << 31) + | (op << 30) + | (0xD4 << 21) + | (rm.num << 16) + | (cond << 12) + | (op2 << 10) + | (rn.num << 5) + | rd.num + ) + + def cond(self, text: str) -> int: + name = text.strip().lower() + if name not in CONDITIONS: + raise ARM64EncodeError(f"unknown condition '{text.strip()}'") + return CONDITIONS[name] + + # --- loads and stores --- + + def parse_address(self, ops: List[str]) -> Tuple[str, Reg, object, bool, bool]: + """Parse [base, ...] addressing: (mode, base, offset, writeback, post). + + mode is "imm" (offset is an int), "reg" (offset is (Reg, option, amount)), + or "literal" (offset is the target expression). + """ + text = ops[0].strip() + if not text.startswith("["): + return "literal", Reg(0, True), text, False, False + writeback = text.endswith("!") + if writeback: + text = text[:-1].strip() + if not text.endswith("]"): + raise ARM64EncodeError(f"expected an address like [x1, #8], got '{ops[0]}'") + inner = split_operands(text[1:-1]) + base = _reg(inner[0], "base register") + if not base.is64: + raise ARM64EncodeError("the base register must be an X register or sp") + if base.kind == "zr": + raise ARM64EncodeError("xzr cannot be a base register") + if len(ops) == 2: # post-index: [xn], #imm + if writeback or len(inner) != 1: + raise ARM64EncodeError("expected [xn], #imm for post-index addressing") + return "imm", base, self.imm(ops[1]), True, True + if len(ops) > 2: + raise ARM64EncodeError("too many operands") + if len(inner) == 1: + return "imm", base, 0, writeback, False + if self.is_imm(inner[1]) and len(inner) == 2: + mode = "lo12" if ":lo12:" in inner[1].lower() else "imm" + return mode, base, self.imm(inner[1]), writeback, False + if writeback: + raise ARM64EncodeError("register offsets cannot use writeback (!)") + index = _reg(inner[1], "index register") + option, amount, has_amount = (3 if index.is64 else None), 0, False + if len(inner) == 3: + parts = inner[2].split(None, 1) + name = parts[0].lower() + if name == "lsl": + option = 3 + elif name in ("uxtw", "sxtw", "sxtx"): + option = EXTENDS[name] + else: + raise ARM64EncodeError(f"unexpected '{inner[2]}'") + if len(parts) > 1: + amount, has_amount = self.imm(parts[1]), True + if option is None: + raise ARM64EncodeError("a W index register needs uxtw or sxtw") + if (option in (2, 6)) == index.is64: + raise ARM64EncodeError("uxtw/sxtw take a W index; lsl/sxtx take an X index") + return "reg", base, (index, option, amount, has_amount), False, False + + def load_store(self, name: str, ops: List[str], unscaled_only: bool = False) -> int: + if len(ops) < 2: + raise ARM64EncodeError(f"expected {name} rt, [address]") + rt = _reg(ops[0]) + if rt.kind == "sp": + raise ARM64EncodeError("sp cannot be loaded or stored directly") + size, opc, access = LOAD_STORE[name] + if name in ("ldr", "str"): + size = 3 if rt.is64 else 2 + access = 8 if rt.is64 else 4 + elif name in ("ldrsb", "ldrsh"): + opc = 2 if rt.is64 else 3 + elif name == "ldrsw": + if not rt.is64: + raise ARM64EncodeError("ldrsw loads into an X register") + elif rt.is64: + raise ARM64EncodeError(f"{name} uses a W register") + mode, base, offset, writeback, post = self.parse_address(ops[1:]) + + if mode == "literal": + if unscaled_only or name not in ("ldr", "ldrsw"): + raise ARM64EncodeError(f"{name} cannot load from a label; use ldr") + opc_lit = {("ldr", False): 0, ("ldr", True): 1, ("ldrsw", True): 2}[(name, rt.is64)] + imm19 = self.pc_offset(str(offset), 19, what="literal address") + return (opc_lit << 30) | (0x18 << 24) | (imm19 << 5) | rt.num + + base_bits = (size << 30) | (0x7 << 27) | (opc << 22) | (base.num << 5) | rt.num + assert access is not None + if mode == "reg": + index, option, amount, has_amount = cast(Tuple[Reg, int, int, bool], offset) + scale = {1: 0, 2: 1, 4: 2, 8: 3}[access] + if has_amount and amount not in (0, scale): + raise ARM64EncodeError(f"the index shift must be #0 or #{scale}") + # S selects shifting by the access size; for bytes it records an explicit #0 + s = int(has_amount and (access == 1 or amount == scale)) + return ( + base_bits | (1 << 21) | (index.num << 16) | (option << 13) | (s << 12) | (2 << 10) + ) + + assert isinstance(offset, int) + if mode == "lo12": + # :lo12:symbol always uses the scaled form, so the address must be aligned + if writeback or offset % access: + raise ARM64EncodeError( + f":lo12: offset {offset:#x} is not a multiple of the access size ({access})" + ) + return base_bits | (1 << 24) | ((offset // access) << 10) + if writeback: + if not -256 <= offset <= 255: + raise ARM64EncodeError(f"offset {offset} out of range (-256 to 255)") + mode_bits = 1 if post else 3 + return base_bits | ((offset & 0x1FF) << 12) | (mode_bits << 10) + if not unscaled_only and offset >= 0 and offset % access == 0 and offset // access <= 0xFFF: + return base_bits | (1 << 24) | ((offset // access) << 10) + if -256 <= offset <= 255: + return base_bits | ((offset & 0x1FF) << 12) # ldur/stur + raise ARM64EncodeError( + f"offset {offset} out of range (unsigned multiple of {access} up to " + f"{0xFFF * access}, or -256 to 255)" + ) + + def load_store_pair(self, name: str, ops: List[str]) -> int: + if len(ops) < 3: + raise ARM64EncodeError(f"expected {name} rt1, rt2, [address]") + rt, rt2 = _reg(ops[0]), _reg(ops[1]) + _same_size(rt, rt2) + load = 1 if name.startswith("ld") else 0 + if name == "ldpsw": + if not rt.is64: + raise ARM64EncodeError("ldpsw loads into X registers") + opc, access = 1, 4 + else: + opc, access = (2, 8) if rt.is64 else (0, 4) + mode, base, offset, writeback, post = self.parse_address(ops[2:]) + if mode not in ("imm", "lo12"): + raise ARM64EncodeError(f"{name} needs an address like [xn, #imm]") + assert isinstance(offset, int) + if offset % access or not -64 <= offset // access <= 63: + raise ARM64EncodeError( + f"offset {offset} must be a multiple of {access} from {-64 * access} to {63 * access}" + ) + variant = 1 if post else (3 if writeback else 2) + return ( + (opc << 30) + | (0x5 << 27) + | (variant << 23) + | (load << 22) + | (((offset // access) & 0x7F) << 15) + | (rt2.num << 10) + | (base.num << 5) + | rt.num + ) + + # --- main dispatch --- + + def encode(self, mnemonic: str, ops: List[str]) -> int: # noqa: C901 + m = mnemonic + + # Branches + if m in ("b", "bl"): + self.expect(ops, 1, f"{m} label") + return ((0x25 if m == "bl" else 0x05) << 26) | self.pc_offset(ops[0], 26) + if m.startswith("b.") and m[2:] in CONDITIONS: + self.expect(ops, 1, f"{m} label") + return (0x54 << 24) | (self.pc_offset(ops[0], 19) << 5) | CONDITIONS[m[2:]] + if len(m) >= 3 and m[0] == "b" and m[1:] in CONDITIONS: # 'beq' shorthand + return self.encode("b." + m[1:], ops) + if m in ("cbz", "cbnz"): + self.expect(ops, 2, f"{m} rt, label") + rt = _reg(ops[0]) + return ( + (int(rt.is64) << 31) + | (0x1A << 25) + | ((m == "cbnz") << 24) + | (self.pc_offset(ops[1], 19) << 5) + | rt.num + ) + if m in ("tbz", "tbnz"): + self.expect(ops, 3, f"{m} rt, #bit, label") + rt = _reg(ops[0]) + bit = self.imm(ops[1]) + if not 0 <= bit < (64 if rt.is64 else 32): + raise ARM64EncodeError(f"bit number {bit} out of range") + return ( + ((bit >> 5) << 31) + | (0x1B << 25) + | ((m == "tbnz") << 24) + | ((bit & 0x1F) << 19) + | (self.pc_offset(ops[2], 14) << 5) + | rt.num + ) + if m in ("br", "blr"): + self.expect(ops, 1, f"{m} xn") + rn = _reg(ops[0]) + return (0xD61F0000 if m == "br" else 0xD63F0000) | (rn.num << 5) + if m == "ret": + rn = _reg(ops[0]) if ops else Reg(30, True) + return 0xD65F0000 | (rn.num << 5) + + # System + if m in ("svc", "brk", "hlt", "hvc"): + value = self.imm(ops[0]) if ops else 0 + if not 0 <= value <= 0xFFFF: + raise ARM64EncodeError(f"immediate {value} out of range (0 to 65535)") + base = {"svc": 0xD4000001, "hvc": 0xD4000002, "brk": 0xD4200000, "hlt": 0xD4400000}[m] + return base | (value << 5) + if m == "nop": + self.expect(ops, 0, "nop") + return 0xD503201F + + # Addresses + if m in ("adr", "adrp"): + self.expect(ops, 2, f"{m} xd, label") + rd = _reg(ops[0]) + if not self.ctx.strict: + return 0 + target = self.target(ops[1]) + if m == "adr": + offset = target - self.pc + if not -(1 << 20) <= offset < (1 << 20): + raise ARM64EncodeError( + "adr target out of range (+/-1MB); for data use " + f"'ldr {ops[0]}, ={ops[1].strip()}' or adrp + add :lo12:" + ) + op = 0 + else: + offset = (target >> 12) - (self.pc >> 12) + if not -(1 << 20) <= offset < (1 << 20): + raise ARM64EncodeError("adrp target out of range (+/-4GB)") + op = 1 + offset &= _mask(21) + return (op << 31) | ((offset & 3) << 29) | (0x10 << 24) | ((offset >> 2) << 5) | rd.num + + # Arithmetic + if m in ("add", "adds", "sub", "subs"): + return self.add_sub(int(m.startswith("sub")), int(m.endswith("s")), ops) + if m in ("cmp", "cmn"): + if len(ops) not in (2, 3): + raise ARM64EncodeError(f"expected {m} rn, operand") + rn = _reg(ops[0]) + zr = "xzr" if rn.is64 else "wzr" + return self.add_sub(int(m == "cmp"), 1, [zr, *ops]) + if m in ("neg", "negs"): + if len(ops) not in (2, 3): + raise ARM64EncodeError(f"expected {m} rd, rm") + rd = _reg(ops[0]) + zr = "xzr" if rd.is64 else "wzr" + if parse_reg(ops[1]) is None: + raise ARM64EncodeError(f"{m} needs a register operand") + return self.add_sub(1, int(m == "negs"), [ops[0], zr, *ops[1:]]) + if m in ("adc", "adcs", "sbc", "sbcs"): + self.expect(ops, 3, f"{m} rd, rn, rm") + rd, rn, rm = (_reg(o) for o in ops) + _same_size(rd, rn, rm) + op = int(m.startswith("sbc")) + s = int(m.endswith("s")) + return ( + (int(rd.is64) << 31) + | (op << 30) + | (s << 29) + | (0xD0 << 21) + | (rm.num << 16) + | (rn.num << 5) + | rd.num + ) + + # Logic + logic = { + "and": (0, 0), "orr": (1, 0), "eor": (2, 0), "ands": (3, 0), + "bic": (0, 1), "orn": (1, 1), "eon": (2, 1), "bics": (3, 1), + } # fmt: skip + if m in logic: + opc, invert = logic[m] + return self.logical(opc, invert, ops) + if m == "tst": + if len(ops) not in (2, 3): + raise ARM64EncodeError("expected tst rn, operand") + zr = "xzr" if _reg(ops[0]).is64 else "wzr" + return self.logical(3, 0, [zr, *ops]) + if m == "mvn": + if len(ops) not in (2, 3): + raise ARM64EncodeError("expected mvn rd, rm") + zr = "xzr" if _reg(ops[0]).is64 else "wzr" + return self.logical(1, 1, [ops[0], zr, *ops[1:]]) + + # Moves + if m == "mov": + self.expect(ops, 2, "mov rd, operand") + rd = _reg(ops[0]) + src = parse_reg(ops[1]) + if src is not None: + _same_size(rd, src) + if rd.kind == "sp" or src.kind == "sp": + return self.add_sub(0, 0, [ops[0], ops[1], "#0"]) + zr = "xzr" if rd.is64 else "wzr" + return self.logical(1, 0, [ops[0], zr, ops[1]]) + bits = 64 if rd.is64 else 32 + value = self.imm(ops[1]) + if not rd.is64 and not -(1 << 31) <= value < (1 << 32): + raise ARM64EncodeError(f"immediate {value:#x} out of range for a W register") + wide = _move_wide(value, bits) + if wide is not None and rd.kind != "sp": + opc, hw, imm16 = wide + return ( + (int(rd.is64) << 31) + | (opc << 29) + | (0x25 << 23) + | (hw << 21) + | (imm16 << 5) + | rd.num + ) + enc = encode_bitmask(value, bits) + if enc is not None: + zr = "xzr" if rd.is64 else "wzr" + return self.logical(1, 0, [ops[0], zr, f"#{value}"]) + raise ARM64EncodeError( + f"{value:#x} cannot be loaded with one mov; use 'ldr {ops[0]}, ={ops[1].lstrip('#')}' " + "or movz followed by movk" + ) + if m in ("movz", "movn", "movk"): + return self.move_wide({"movn": 0, "movz": 2, "movk": 3}[m], ops) + + # Multiply and divide + if m in ("mul", "mneg", "madd", "msub"): + n = 3 if m in ("mul", "mneg") else 4 + self.expect(ops, n, f"{m} rd, rn, rm" + (", ra" if n == 4 else "")) + rd, rn, rm = (_reg(o) for o in ops[:3]) + ra = _reg(ops[3]) if n == 4 else Reg(31, rd.is64, "zr") + _same_size(rd, rn, rm, ra) + return self.data3(0, int(m in ("msub", "mneg")), rd, rn, rm, ra, int(rd.is64)) + if m in ("smull", "umull", "smaddl", "umaddl", "smnegl", "umnegl", "smsubl", "umsubl"): + n = 3 if m.endswith("ull") or m.endswith("negl") else 4 + self.expect(ops, n, f"{m} xd, wn, wm" + (", xa" if n == 4 else "")) + rd, rn, rm = (_reg(o) for o in ops[:3]) + ra = _reg(ops[3]) if n == 4 else Reg(31, True, "zr") + if not (rd.is64 and ra.is64 and not rn.is64 and not rm.is64): + raise ARM64EncodeError(f"{m} takes an X destination and W sources") + op31 = 5 if m.startswith("u") else 1 + o0 = int("sub" in m or "neg" in m) + return self.data3(op31, o0, rd, rn, rm, ra, 1) + if m in ("smulh", "umulh"): + self.expect(ops, 3, f"{m} xd, xn, xm") + rd, rn, rm = (_reg(o) for o in ops) + if not (rd.is64 and rn.is64 and rm.is64): + raise ARM64EncodeError(f"{m} takes X registers") + return self.data3(6 if m == "umulh" else 2, 0, rd, rn, rm, Reg(31, True, "zr"), 1) + if m in ("udiv", "sdiv"): + return self.data2(2 if m == "udiv" else 3, ops) + + # Shifts + if m in ("lsl", "lsr", "asr", "ror"): + self.expect(ops, 3, f"{m} rd, rn, amount") + if not self.is_imm(ops[2]): + return self.data2({"lsl": 8, "lsr": 9, "asr": 10, "ror": 11}[m], ops) + rd, rn = _reg(ops[0]), _reg(ops[1]) + bits = 64 if rd.is64 else 32 + amount = self.imm(ops[2]) + if not 0 <= amount < bits: + raise ARM64EncodeError(f"shift amount {amount} out of range (0 to {bits - 1})") + if m == "lsl": + return self.bitfield(2, rd, rn, (-amount) % bits, bits - 1 - amount) + if m == "lsr": + return self.bitfield(2, rd, rn, amount, bits - 1) + if m == "asr": + return self.bitfield(0, rd, rn, amount, bits - 1) + return self.extr(rd, rn, rn, amount) + if m in ("lslv", "lsrv", "asrv", "rorv"): + return self.data2({"lslv": 8, "lsrv": 9, "asrv": 10, "rorv": 11}[m], ops) + if m == "extr": + self.expect(ops, 4, "extr rd, rn, rm, #lsb") + rd, rn, rm = (_reg(o) for o in ops[:3]) + return self.extr(rd, rn, rm, self.imm(ops[3])) + + # Bitfield aliases + if m in ("sxtb", "sxth", "sxtw", "uxtb", "uxth"): + self.expect(ops, 2, f"{m} rd, wn") + rd, rn = _reg(ops[0]), _reg(ops[1]) + if rn.is64: + raise ARM64EncodeError(f"{m} takes a W source register") + width = {"b": 8, "h": 16, "w": 32}[m[-1]] + if m.startswith("u"): + if rd.is64: + raise ARM64EncodeError(f"{m} takes a W destination register") + return self.bitfield(2, rd, rn, 0, width - 1) + if m == "sxtw" and not rd.is64: + raise ARM64EncodeError("sxtw takes an X destination register") + return self.bitfield(0, rd, Reg(rn.num, rd.is64), 0, width - 1) + if m in ("ubfx", "sbfx", "ubfiz", "sbfiz", "bfi", "bfxil"): + self.expect(ops, 4, f"{m} rd, rn, #lsb, #width") + rd, rn = _reg(ops[0]), _reg(ops[1]) + bits = 64 if rd.is64 else 32 + lsb, width = self.imm(ops[2]), self.imm(ops[3]) + if not (0 <= lsb < bits and 1 <= width <= bits - lsb): + raise ARM64EncodeError("bit field out of range") + opc = {"s": 0, "b": 1, "u": 2}[m[0]] + if m in ("ubfx", "sbfx", "bfxil"): + return self.bitfield(opc, rd, rn, lsb, lsb + width - 1) + return self.bitfield(opc, rd, rn, (-lsb) % bits, width - 1) + if m in ("ubfm", "sbfm", "bfm"): + self.expect(ops, 4, f"{m} rd, rn, #immr, #imms") + rd, rn = _reg(ops[0]), _reg(ops[1]) + return self.bitfield( + {"sbfm": 0, "bfm": 1, "ubfm": 2}[m], rd, rn, self.imm(ops[2]), self.imm(ops[3]) + ) + + # Conditional select + csel = {"csel": (0, 0), "csinc": (0, 1), "csinv": (1, 0), "csneg": (1, 1)} + if m in csel: + self.expect(ops, 4, f"{m} rd, rn, rm, cond") + rd, rn, rm = (_reg(o) for o in ops[:3]) + return self.cond_select(*csel[m], rd, rn, rm, self.cond(ops[3])) + if m in ("cset", "csetm"): + self.expect(ops, 2, f"{m} rd, cond") + rd = _reg(ops[0]) + zero = Reg(31, rd.is64, "zr") + op = 1 if m == "csetm" else 0 + return self.cond_select(op, 0 if op else 1, rd, zero, zero, self.inverted(ops[1])) + if m in ("cinc", "cinv", "cneg"): + self.expect(ops, 3, f"{m} rd, rn, cond") + rd, rn = _reg(ops[0]), _reg(ops[1]) + op, op2 = {"cinc": (0, 1), "cinv": (1, 0), "cneg": (1, 1)}[m] + return self.cond_select(op, op2, rd, rn, rn, self.inverted(ops[2])) + if m in ("ccmp", "ccmn"): + self.expect(ops, 4, f"{m} rn, operand, #nzcv, cond") + rn = _reg(ops[0]) + nzcv = self.imm(ops[2]) + if not 0 <= nzcv <= 15: + raise ARM64EncodeError("nzcv must be 0 to 15") + op = 1 if m == "ccmp" else 0 + head = (int(rn.is64) << 31) | (op << 30) | (1 << 29) | (0xD2 << 21) + if self.is_imm(ops[1]): + value = self.imm(ops[1]) + if not 0 <= value <= 31: + raise ARM64EncodeError("immediate must be 0 to 31") + return ( + head + | (value << 16) + | (self.cond(ops[3]) << 12) + | (1 << 11) + | (rn.num << 5) + | nzcv + ) + rm = _reg(ops[1]) + _same_size(rn, rm) + return head | (rm.num << 16) | (self.cond(ops[3]) << 12) | (rn.num << 5) | nzcv + + # Loads and stores + if m in LOAD_STORE: + return self.load_store(m, ops) + if m in UNSCALED: + return self.load_store(UNSCALED[m], ops, unscaled_only=True) + if m in ("ldp", "stp", "ldpsw"): + return self.load_store_pair(m, ops) + + raise ARM64EncodeError(f"unknown instruction '{mnemonic}'") + + def extr(self, rd: Reg, rn: Reg, rm: Reg, lsb: int) -> int: + _same_size(rd, rn, rm) + sf = int(rd.is64) + if not 0 <= lsb < (64 if sf else 32): + raise ARM64EncodeError("shift amount out of range") + return ( + (sf << 31) + | (0x27 << 23) + | (sf << 22) + | (rm.num << 16) + | (lsb << 10) + | (rn.num << 5) + | rd.num + ) + + def inverted(self, text: str) -> int: + cond = self.cond(text) + if cond >= 14: + raise ARM64EncodeError("al and nv cannot be used here") + return cond ^ 1 + + @staticmethod + def expect(ops: List[str], n: int, usage: str) -> None: + if len(ops) != n: + raise ARM64EncodeError(f"expected {usage}") + + +def _split_instruction(text: str) -> Tuple[str, List[str]]: + parts = text.strip().split(None, 1) + if not parts: + raise ARM64EncodeError("empty instruction") + return parts[0].lower(), split_operands(parts[1]) if len(parts) > 1 else [] + + +def _context(address: int, labels: Dict[str, int], strict: bool) -> Context: + return Context(address, labels, strict=strict, error=ARM64EncodeError) + + +def encode(text: str, address: int, labels: Dict[str, int]) -> bytes: + """ + Encode one ARM64 instruction to little-endian machine code (4 bytes). + + Raises: + ARM64EncodeError: If the instruction is invalid. + """ + mnemonic, ops = _split_instruction(text) + encoder = Encoder(_context(address, labels, strict=True)) + try: + word = encoder.encode(mnemonic, ops) + except ExpressionError as e: + raise ARM64EncodeError(str(e)) + return word.to_bytes(4, "little") + + +def instruction_size(text: str, labels: Dict[str, int], address: int = 0) -> int: + """Every A64 instruction is 4 bytes.""" + return INSTR_SIZE + + +# --------------------------------------------------------------------------- +# Literal pools (ldr x0, =value) +# --------------------------------------------------------------------------- + +_LITERAL_LOAD = re.compile(r"^\s*(ldr|ldrsw)\s+([^,]+),\s*=\s*(.+?)\s*$", re.IGNORECASE) + + +def literal_load(text: str) -> Optional[Tuple[str, str, str]]: + """If text is 'ldr rt, =expr', return (mnemonic, rt, expr).""" + m = _LITERAL_LOAD.match(text) + if not m: + return None + return m.group(1).lower(), m.group(2).strip(), m.group(3) + + +def literal_size(mnemonic: str, rt: str) -> int: + """Pool entry size for 'ldr rt, =expr' (8 for X registers, 4 for W).""" + reg = parse_reg(rt) + if reg is None: + raise ARM64EncodeError(f"expected a register, got '{rt}'") + return 8 if reg.is64 and mnemonic == "ldr" else 4 + + +def constant_value(expression: str) -> Optional[int]: + """The value of a constant pool expression, or None if it uses symbols.""" + return parse_int(expression.lstrip("#")) + + +__all__ = [ + "ARM64EncodeError", + "encode", + "encode_bitmask", + "instruction_size", + "literal_load", + "literal_size", + "constant_value", + "sign_extend", +] diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index b726a09..d206f0a 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -28,9 +28,10 @@ KEYSTONE_ERROR = f"Keystone native library failed: {e}" keystone = None -from . import mips, riscv +from . import arm64, mips, riscv from .directives import ( INTERNAL_LABEL_PREFIXES, + POOL_LABEL_PREFIX, DirectiveParser, LineType, ParsedLine, @@ -87,7 +88,6 @@ def success(self) -> bool: # Keystone architecture/mode constants -KS_ARCH_ARM64 = 2 KS_ARCH_X86 = 4 KS_MODE_LITTLE_ENDIAN = 0 @@ -129,14 +129,15 @@ class Assembler: "mode": None, "instr_size": 4, }, + # ARM64 is encoded by the built-in pure-Python encoder (arm64.py) "arm64": { - "arch": KS_ARCH_ARM64, - "mode": KS_MODE_LITTLE_ENDIAN, + "arch": None, + "mode": None, "instr_size": 4, }, "aarch64": { - "arch": KS_ARCH_ARM64, - "mode": KS_MODE_LITTLE_ENDIAN, + "arch": None, + "mode": None, "instr_size": 4, }, "x86_64": { @@ -209,6 +210,73 @@ def __init__(self, isa: str): except keystone.KsError as e: raise RuntimeError(f"Failed to initialize Keystone for {isa}: {e}") + @staticmethod + def _add_literal_pools(section: SectionData) -> None: + """Turn ARM64 'ldr x0, =value' into loads from a literal pool. + + As in GNU as, each load reads a pool entry placed at the next + .ltorg/.pool directive or at the end of the section. Within a pool, + equal values share one entry; 4-byte entries come first, then 8-byte + entries, each group aligned to its size. + """ + lines: List[ParsedLine] = [] + pool: Dict[Tuple[object, ...], Tuple[str, str, int]] = {} # key -> (label, expr, size) + counter = 0 + + def dump(line_number: int) -> None: + for size in (4, 8): + entries = [(label, expr) for label, expr, sz in pool.values() if sz == size] + if not entries: + continue + lines.append( + ParsedLine( + line_number, + LineType.DIRECTIVE, + "", + directive="balign", + directive_args=[str(size)], + ) + ) + for label, expr in entries: + lines.append( + ParsedLine( + line_number, + LineType.DIRECTIVE, + "", + label=label, + directive="word" if size == 4 else "dword", + directive_args=[expr], + ) + ) + pool.clear() + + last_line = 0 + for line in section.lines: + last_line = line.line_number + if line.line_type == LineType.INSTRUCTION and line.instruction: + found = arm64.literal_load(line.instruction) + if found is not None: + mnemonic, rt, expr = found + try: + size = arm64.literal_size(mnemonic, rt) + except EncodeError as e: + raise EncodeError(f"Line {line.line_number}: {e}") + value = arm64.constant_value(expr) + key: Tuple[object, ...] = ( + ("value", value & ((1 << (8 * size)) - 1), size) + if value is not None + else ("expr", expr.replace(" ", ""), size) + ) + if key not in pool: + counter += 1 + pool[key] = (f"{POOL_LABEL_PREFIX}{counter}", expr, size) + line.instruction = f"{mnemonic} {rt}, {pool[key][0]}" + lines.append(line) + if line.line_type == LineType.DIRECTIVE and line.directive in ("ltorg", "pool"): + dump(line.line_number) + dump(last_line) + section.lines = lines + @staticmethod def _resolve_constants( pending: Dict[str, Tuple[str, int]], @@ -278,6 +346,8 @@ def _builtin_encoder(self) -> Optional[ModuleType]: return riscv if self.isa in ("mips32", "mips"): return mips + if self.isa in ("arm64", "aarch64"): + return arm64 return None def assemble( @@ -305,6 +375,11 @@ def assemble( parser = DirectiveParser(isa=self.isa) self._parser = parser sections = parser.parse(source) + if self.isa in ("arm64", "aarch64") and ".text" in sections: + try: + self._add_literal_pools(sections[".text"]) + except EncodeError as e: + result.errors.append(str(e)) result.errors.extend(parser.errors) result.warnings.extend(parser.warnings) @@ -723,22 +798,6 @@ def _estimate_fixed_instr_size( if self._builtin_encoder is not None: return self._builtin_encoder.instruction_size(instr, data_labels or {}) - # ARM64 pseudo-instructions that expand to multiple instructions - if self.isa in ("arm64", "aarch64"): - operands = parts[1] if len(parts) > 1 else "" - # adr with symbol -> movz + movk sequence (typically 2 instrs for 32-bit addr) - if mnemonic == "adr": - # Check if it's using a symbol (not a numeric offset) - ops = [o.strip() for o in operands.split(",")] - if len(ops) == 2: - symbol = ops[1].strip() - # If it's not a number, it's a symbol that needs expansion - if not symbol.startswith("#") and not symbol.lstrip("-").isdigit(): - return base_size * 2 # movz + movk - # ldr x0, =symbol -> expands to adr -> movz + movk - if mnemonic == "ldr" and "=" in operands: - return base_size * 2 - return base_size def _estimate_x86_instr_size(self, instr: str) -> int: @@ -920,11 +979,6 @@ def _assemble_section( current_addr += len(encoding) continue - # Expand pseudo-instructions for ARM64 and normalize syntax - if self.isa in ("arm64", "aarch64"): - instr = self._normalize_arm64_syntax(instr) - instr = self._expand_arm64_pseudo(instr, current_addr, labels) - # Convert AT&T to Intel syntax for x86-64 if self.isa == "x86_64": instr = self._convert_x86_att_to_intel(instr, labels) @@ -956,115 +1010,6 @@ def _assemble_section( return bytes(code), label_addrs, errors, debug_lines - def _normalize_arm64_syntax(self, instr: str) -> str: - """ - Normalize ARM64 syntax for Keystone compatibility. - - Keystone doesn't accept # prefix on immediates, so we strip them. - Example: mov x0, #42 -> mov x0, 42 - """ - import re - - # Match # followed by a number (decimal, hex, or negative) - # But not at the start of the line (which would be a comment) - # Pattern: comma or space, then #, then optional -, then number - return re.sub(r"([\s,])#(-?(?:0x[0-9a-fA-F]+|\d+))", r"\1\2", instr) - - def _expand_arm64_pseudo( - self, - instr: str, - addr: int, - labels: Dict[str, int], - ) -> str: - """ - Expand ARM64 pseudo-instructions. - - Handles adr with far symbols by using adrp + add sequence. - """ - parts = instr.split(None, 1) - if not parts: - return instr - - mnemonic = parts[0].lower() - operands = parts[1] if len(parts) > 1 else "" - - # Handle 'adr' with symbol - convert to movz + movk sequence for far addresses - if mnemonic == "adr": - ops = [o.strip() for o in operands.split(",")] - if len(ops) == 2: - reg = ops[0] - symbol = ops[1].strip() - if symbol in labels: - target = labels[symbol] - # Use movz/movk sequence for 64-bit address - # movz loads the lowest 16 bits, movk adds higher bits - b0 = target & 0xFFFF - b1 = (target >> 16) & 0xFFFF - b2 = (target >> 32) & 0xFFFF - b3 = (target >> 48) & 0xFFFF - - instrs = [f"movz {reg}, {b0}"] - if b1: - instrs.append(f"movk {reg}, {b1}, lsl 16") - if b2: - instrs.append(f"movk {reg}, {b2}, lsl 32") - if b3: - instrs.append(f"movk {reg}, {b3}, lsl 48") - - return "; ".join(instrs) - - # Handle 'ldr' with = syntax (ldr x0, =symbol) - if mnemonic == "ldr" and "=" in operands: - ops = operands.split(",") - if len(ops) == 2: - reg = ops[0].strip() - symbol = ops[1].strip().lstrip("=") - if symbol in labels: - target = labels[symbol] - return self._expand_arm64_pseudo(f"adr {reg}, {symbol}", addr, labels) - - # Handle 'b' (unconditional branch) with symbol - # Note: Keystone expects absolute target address for ARM64 branches - if mnemonic == "b": - symbol = operands.strip() - if symbol in labels: - target = labels[symbol] - return f"b 0x{target:x}" - - # Handle 'bl' (branch and link) with symbol - if mnemonic == "bl": - symbol = operands.strip() - if symbol in labels: - target = labels[symbol] - return f"bl 0x{target:x}" - - # Handle conditional branches (b.eq, b.ne, etc.) - if mnemonic.startswith("b."): - symbol = operands.strip() - if symbol in labels: - target = labels[symbol] - return f"{mnemonic} 0x{target:x}" - - # Handle cbz/cbnz (compare and branch if zero/not zero) - if mnemonic in ("cbz", "cbnz"): - ops = [o.strip() for o in operands.split(",")] - if len(ops) == 2: - reg, symbol = ops[0], ops[1].strip() - if symbol in labels: - target = labels[symbol] - return f"{mnemonic} {reg}, 0x{target:x}" - - # Handle tbz/tbnz (test bit and branch if zero/not zero) - if mnemonic in ("tbz", "tbnz"): - ops = [o.strip() for o in operands.split(",")] - if len(ops) == 3: - reg, bit, symbol = ops[0], ops[1], ops[2].strip() - if symbol in labels: - target = labels[symbol] - return f"{mnemonic} {reg}, {bit}, 0x{target:x}" - - return instr - def _convert_x86_att_to_intel( self, instr: str, diff --git a/mapachespim/toolchain/directives.py b/mapachespim/toolchain/directives.py index 8906bb3..7781a99 100644 --- a/mapachespim/toolchain/directives.py +++ b/mapachespim/toolchain/directives.py @@ -109,7 +109,10 @@ class Fixup: # Prefix of internal names given to numeric local labels like "1:" LOCAL_LABEL_PREFIX = ".L.local." -INTERNAL_LABEL_PREFIXES = (DOT_LABEL_PREFIX, LOCAL_LABEL_PREFIX) +# Prefix of labels for ARM64 literal pool entries (ldr x0, =value) +POOL_LABEL_PREFIX = ".L.pool." + +INTERNAL_LABEL_PREFIXES = (DOT_LABEL_PREFIX, LOCAL_LABEL_PREFIX, POOL_LABEL_PREFIX) class DirectiveParser: diff --git a/tests/gnu_reference.py b/tests/gnu_reference.py index e4f5fb9..166edc4 100644 --- a/tests/gnu_reference.py +++ b/tests/gnu_reference.py @@ -42,6 +42,7 @@ class GnuTarget: # No compressed instructions, no linker relaxation, absolute code ".option norvc\n.option norelax\n.option nopic\n", ), + "arm64": GnuTarget("aarch64-linux-gnu-", (), ""), "mips32": GnuTarget( "mips-linux-gnu-", ("-march=mips32", "-EB", "-mno-shared", "-O0", "-32"), diff --git a/tests/test_arm64_encoder.py b/tests/test_arm64_encoder.py new file mode 100644 index 0000000..e5a96f9 --- /dev/null +++ b/tests/test_arm64_encoder.py @@ -0,0 +1,128 @@ +""" +Unit tests for the built-in ARM64 encoder that complement the GNU +differential tests (test_differential_arm64.py), which need binutils. +""" + +import unittest + +from capstone import CS_ARCH_ARM64, CS_MODE_ARM, Cs + +from mapachespim.toolchain import assemble +from mapachespim.toolchain.arm64 import ARM64EncodeError, encode, encode_bitmask + +PC = 0x10000000 +_cs = Cs(CS_ARCH_ARM64, CS_MODE_ARM) + + +def decode_bitmask(n: int, immr: int, imms: int, bits: int): + """Reference DecodeBitMasks from the Arm Architecture Reference Manual.""" + combined = (n << 6) | (~imms & 0x3F) + if combined == 0: + return None + length = combined.bit_length() - 1 + size = 1 << length + if size > bits or size < 2: + return None + levels = size - 1 + s, r = imms & levels, immr & levels + if s == levels: + return None + elem = (1 << (s + 1)) - 1 + elem = ((elem >> r) | (elem << (size - r))) & ((1 << size) - 1) + value = 0 + for i in range(bits // size): + value |= elem << (i * size) + return value + + +class TestBitmaskImmediates(unittest.TestCase): + def test_every_encodable_value_round_trips(self): + """Decode every (N, immr, imms) and check we choose that encoding back.""" + for bits in (32, 64): + seen = 0 + for n in (0, 1) if bits == 64 else (0,): + for immr in range(64): + for imms in range(64): + value = decode_bitmask(n, immr, imms, bits) + if value is None or (bits == 32 and immr >= 32): + continue + enc = encode_bitmask(value, bits) + self.assertIsNotNone(enc, hex(value)) + self.assertEqual(decode_bitmask(*enc, bits), value) + seen += 1 + # There are 5334 distinct 64-bit and 1302 32-bit logical immediates + self.assertGreater(seen, 1000) + + def test_unencodable_values(self): + for value in (0, 0xFFFFFFFFFFFFFFFF, 5, 0x123456789): + self.assertIsNone(encode_bitmask(value, 64)) + + +class TestExamplesOfUse(unittest.TestCase): + def disasm(self, text): + code = encode(text, PC, {"label": PC + 0x40}) + return [f"{i.mnemonic} {i.op_str}" for i in _cs.disasm(code, PC)] + + def test_common_student_instructions(self): + cases = { + "stp x29, x30, [sp, #-16]!": "stp x29, x30, [sp, #-0x10]!", + "mov x29, sp": "mov x29, sp", + "mov x0, #42": "mov x0, #0x2a", + "add x0, x0, #1": "add x0, x0, #1", + "cmp x0, #10": "cmp x0, #0xa", + "b.lt label": "b.lt #0x10000040", + "ldr w1, [x0, x2, lsl #2]": "ldr w1, [x0, x2, lsl #2]", + "svc #0": "svc #0", + } + for source, expected in cases.items(): + with self.subTest(source=source): + self.assertEqual(self.disasm(source), [expected]) + + def test_helpful_errors(self): + for source, message in { + "mov x0, #0x123456789": "ldr x0, =0x123456789", + "add x0, x1, w2": "X registers", + "adr x0, far": "out of range", + "ldr x0, [x1, #3]!x": "expected", + }.items(): + with self.subTest(source=source): + with self.assertRaises(ARM64EncodeError) as cm: + encode(source, PC, {"far": PC + 0x200000}) + self.assertIn(message, str(cm.exception)) + + def test_hash_comment_hint(self): + """'#' after code is not a comment in ARM assembly (use //).""" + result = assemble(".text\n_start:\n mov x0, #1 # one\n", isa="arm64") + self.assertFalse(result.success) + + +class TestLiteralPool(unittest.TestCase): + def test_ldr_equals_loads_the_value(self): + from mapachespim import ISA, Simulator + + result = assemble( + ".text\n_start:\n ldr x0, =0x123456789abcdef0\n ldr w1, =7\n" + " ldr x2, =value\n mov x8, #10\n svc #0\n.data\nvalue: .word 5\n", + isa="arm64", + ) + self.assertTrue(result.success, result.errors) + import os + import tempfile + + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(result.elf_bytes) + try: + sim = Simulator(ISA.ARM) + sim.load_elf(f.name) + for _ in range(5): + if sim.check_termination(sim.step())[0]: + break + self.assertEqual(sim.get_reg(0), 0x123456789ABCDEF0) + self.assertEqual(sim.get_reg(1), 7) + self.assertEqual(sim.get_reg(2), result.symbols["value"]) + finally: + os.unlink(f.name) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_assembler_accuracy.py b/tests/test_assembler_accuracy.py index 4a514f0..073f0d7 100644 --- a/tests/test_assembler_accuracy.py +++ b/tests/test_assembler_accuracy.py @@ -220,14 +220,14 @@ def test_arm64_mov_immediate(self): assert not errors, f"Label mismatches:\n" + "\n".join(errors) def test_arm64_adr_with_symbol(self): - """ARM64 adr with symbol may expand to movz+movk sequence.""" + """ARM64 ldr =symbol (literal pool load) is one instruction.""" source = """ .data message: .asciz "Hello" .text start: - adr x0, message + ldr x0, =message after_adr: nop """ @@ -251,7 +251,7 @@ def test_arm64_branch_instructions(self): assert not errors, f"Label mismatches:\n" + "\n".join(errors) def test_arm64_multiple_adr(self): - """ARM64 multiple adr instructions - tests cumulative error.""" + """ARM64 multiple literal loads - tests cumulative error.""" source = """ .data msg1: .asciz "Hello" @@ -259,9 +259,9 @@ def test_arm64_multiple_adr(self): .text start: - adr x0, msg1 + ldr x0, =msg1 after_adr1: - adr x1, msg2 + ldr x1, =msg2 after_adr2: bl helper after_bl: @@ -729,23 +729,23 @@ def test_mips_pseudo_branch_sizes(self): f"'{branch}' should expand to 12 bytes (slt+branch+nop), got {size}" def test_arm64_adr_expansion(self): - """Test ARM64 adr with symbol expands correctly.""" + """Test ARM64 ldr =symbol assembles to a single instruction.""" source = """ .data msg: .asciz "Hello" .text start: - adr x0, msg + ldr x0, =msg end: nop """ result = assemble(source, isa="arm64") assert result.success, f"Assembly failed: {result.errors}" - # adr with far symbol should expand to movz + movk(s) + # A literal load is one instruction; the address lives in a literal pool size = result.symbols["end"] - result.symbols["start"] - assert size >= 4, f"adr should be at least 4 bytes, got {size}" + assert size == 4, f"ldr = should be 4 bytes, got {size}" errors = verify_label_accuracy(source, "arm64") assert not errors, f"Label mismatches:\n" + "\n".join(errors) @@ -903,7 +903,7 @@ def test_arm64_branch_after_adr(self): .text _start: - adr x0, msg + ldr x0, =msg bl done nop nop @@ -927,8 +927,8 @@ def test_arm64_multiple_adr_before_branch(self): .text _start: - adr x0, msg1 - adr x1, msg2 + ldr x0, =msg1 + ldr x1, =msg2 bl target nop target: @@ -1038,7 +1038,7 @@ def test_generate_size_report(self): .text start: nop l1: mov x0, 5 - l2: adr x1, msg + l2: ldr x1, =msg l3: bl start l4: cbz x0, start end: nop diff --git a/tests/test_differential_arm64.py b/tests/test_differential_arm64.py new file mode 100644 index 0000000..f5fee7e --- /dev/null +++ b/tests/test_differential_arm64.py @@ -0,0 +1,399 @@ +""" +Differential tests: the built-in ARM64 assembler against GNU as. + +Random programs per instruction family (fixed seeds) must assemble to +byte-identical .text, including the literal pool used by 'ldr x0, =value'. + +Skipped when aarch64-linux-gnu binutils are not installed. +Set MAPACHESPIM_FUZZ_SCALE=N to generate N times as many instructions. +""" + +import random + +import pytest + +from mapachespim.toolchain.arm64 import CONDITIONS, encode_bitmask + +from .gnu_reference import ( + FUZZ_SCALE, + describe_mismatch, + gnu_accepts, + gnu_assemble, + gnu_available, + ours_text, + trim_padding, +) + +ISA = "arm64" +pytestmark = pytest.mark.skipif(not gnu_available(ISA), reason="GNU AArch64 binutils not installed") + + +def xreg(rng, zr=True): + n = rng.randrange(32 if zr else 31) + if n == 31: + return "xzr" + return rng.choice([f"x{n}"] * 4 + (["fp"] if n == 29 else []) + (["lr"] if n == 30 else [])) + + +def wreg(rng, zr=True): + n = rng.randrange(32 if zr else 31) + return "wzr" if n == 31 else f"w{n}" + + +def reg(rng, is64, zr=True): + return xreg(rng, zr) if is64 else wreg(rng, zr) + + +def sp_or_reg(rng, is64): + if rng.random() < 0.2: + return "sp" if is64 else "wsp" + return reg(rng, is64, zr=False) + + +def imm(rng, lo, hi): + return rng.choice([lo, hi, 0 if lo <= 0 <= hi else lo, rng.randint(lo, hi), rng.randint(lo, hi)]) + + +def bitmask(rng, bits): + """A random valid logical immediate.""" + while True: + size = rng.choice([s for s in (2, 4, 8, 16, 32, 64) if s <= bits]) + ones = rng.randint(1, size - 1) + rot = rng.randrange(size) + elem = ((1 << ones) - 1) + elem = ((elem >> rot) | (elem << (size - rot))) & ((1 << size) - 1) + value = 0 + for i in range(bits // size): + value |= elem << (i * size) + if encode_bitmask(value, bits) is not None: + return value + + +DATA = """ +.data +d0: .word 1 +.align 3 +d1: .quad 2 +.byte 9 +d3: .byte 3 +""" + + +def program(lines, labels=0, rng=None) -> str: + body = list(lines) + if labels and rng is not None: + positions = sorted(rng.sample(range(len(body) + 1), labels)) + for i, pos in enumerate(reversed(positions)): + body.insert(pos, f"L{labels - 1 - i}:") + text = "\n".join(f" {line}" if not line.endswith(":") else line for line in body) + return f".text\n.globl _start\n_start:\n{text}\n{DATA}" + + +def assert_same_chunked(lines, labels, rng, chunk=1500) -> None: + """Keep generated programs small enough for tbz's +/-32KB range.""" + for start in range(0, len(lines), chunk): + assert_same(program(lines[start : start + chunk], labels, rng)) + + +def assert_same(source: str) -> None: + ours, errors, debug_lines = ours_text(ISA, source) + assert not errors, f"MapacheSPIM rejected input: {errors[:3]}" + gnu = trim_padding(gnu_assemble(ISA, source), ours) + assert ours == gnu, describe_mismatch(ISA, source, ours, gnu, debug_lines) + + +def n(base): + return base * FUZZ_SCALE + + +def test_add_sub(): + rng = random.Random(1) + lines = [] + for m in ("add", "adds", "sub", "subs"): + for _ in range(n(8)): + is64 = rng.random() < 0.6 + s = m.endswith("s") + rd = reg(rng, is64) if s else sp_or_reg(rng, is64) + rn = sp_or_reg(rng, is64) + lines.append(f"{m} {rd}, {rn}, #{imm(rng, 0, 4095)}") + lines.append(f"{m} {rd}, {rn}, #{imm(rng, 1, 4095)}, lsl #12") + lines.append(f"{m} {rd}, {rn}, #{rng.choice([-5, -4095, 0x5000, 4096])}") + rn = reg(rng, is64, zr=False) + rd = reg(rng, is64) + shift = rng.choice(["", f", lsl #{rng.randrange(32)}", f", lsr #{rng.randrange(32)}", f", asr #{rng.randrange(32)}"]) + lines.append(f"{m} {rd}, {rn}, {reg(rng, is64)}{shift}") + ext = rng.choice(["uxtb", "uxth", "uxtw", "sxtb", "sxth", "sxtw"]) + lines.append(f"{m} {reg(rng, is64, False) if s else sp_or_reg(rng, is64)}, {sp_or_reg(rng, is64)}, {reg(rng, False)}, {ext} #{rng.randrange(5)}") + if is64: + # In the extended-register form, register 31 as destination is sp + rd_ext = reg(rng, True, zr=False) if s else sp_or_reg(rng, True) + lines.append(f"{m} {rd_ext}, {sp_or_reg(rng, True)}, {xreg(rng)}, {rng.choice(['uxtx', 'sxtx', 'lsl'])} #{rng.randrange(5)}") + for m in ("cmp", "cmn"): + for _ in range(n(6)): + is64 = rng.random() < 0.6 + lines.append(f"{m} {sp_or_reg(rng, is64)}, #{imm(rng, 0, 4095)}") + lines.append(f"{m} {reg(rng, is64, False)}, {reg(rng, is64)}") + lines.append(f"{m} {reg(rng, is64, False)}, #-{rng.randint(1, 4095)}") + for m in ("neg", "negs"): + for _ in range(n(4)): + is64 = rng.random() < 0.6 + lines.append(f"{m} {reg(rng, is64)}, {reg(rng, is64)}") + lines.append(f"{m} {reg(rng, is64)}, {reg(rng, is64)}, lsl #{rng.randrange(32)}") + for m in ("adc", "adcs", "sbc", "sbcs"): + for _ in range(n(3)): + is64 = rng.random() < 0.5 + lines.append(f"{m} {reg(rng, is64)}, {reg(rng, is64)}, {reg(rng, is64)}") + assert_same(program(lines)) + + +def test_logical(): + rng = random.Random(2) + lines = [] + for m in ("and", "orr", "eor", "ands", "bic", "orn", "eon", "bics"): + for _ in range(n(8)): + is64 = rng.random() < 0.6 + bits = 64 if is64 else 32 + shift = rng.choice(["", f", lsl #{rng.randrange(bits)}", f", lsr #{rng.randrange(bits)}", + f", asr #{rng.randrange(bits)}", f", ror #{rng.randrange(bits)}"]) + lines.append(f"{m} {reg(rng, is64)}, {reg(rng, is64)}, {reg(rng, is64)}{shift}") + if m in ("and", "orr", "eor", "ands"): + rd = sp_or_reg(rng, is64) if m != "ands" else reg(rng, is64) + lines.append(f"{m} {rd}, {reg(rng, is64)}, #{hex(bitmask(rng, bits))}") + for _ in range(n(6)): + is64 = rng.random() < 0.6 + bits = 64 if is64 else 32 + lines.append(f"tst {reg(rng, is64)}, #{hex(bitmask(rng, bits))}") + lines.append(f"tst {reg(rng, is64)}, {reg(rng, is64)}") + lines.append(f"mvn {reg(rng, is64)}, {reg(rng, is64)}") + assert_same(program(lines)) + + +def test_moves(): + rng = random.Random(3) + lines = [] + for _ in range(n(12)): + is64 = rng.random() < 0.6 + bits = 64 if is64 else 32 + rd = reg(rng, is64, zr=False) + lines.append(f"mov {rd}, {reg(rng, is64)}") + lines.append(f"mov {sp_or_reg(rng, is64)}, {'sp' if is64 else 'wsp'}") + hw = rng.randrange(bits // 16) + lines.append(f"mov {rd}, #{hex(rng.randrange(1 << 16) << (16 * hw))}") + lines.append(f"mov {rd}, #{-rng.randrange(1, 1 << 16)}") + lines.append(f"mov {rd}, #{hex(bitmask(rng, bits))}") + for m in ("movz", "movn", "movk"): + lines.append(f"{m} {rd}, #{rng.randrange(1 << 16)}, lsl #{16 * rng.randrange(bits // 16)}") + lines.append(f"{m} {rd}, #{rng.randrange(1 << 16)}") + assert_same(program(lines)) + + +def test_multiply_divide_shift(): + rng = random.Random(4) + lines = [] + for _ in range(n(8)): + is64 = rng.random() < 0.6 + bits = 64 if is64 else 32 + r = lambda: reg(rng, is64) # noqa: E731 + lines += [ + f"mul {r()}, {r()}, {r()}", + f"mneg {r()}, {r()}, {r()}", + f"madd {r()}, {r()}, {r()}, {r()}", + f"msub {r()}, {r()}, {r()}, {r()}", + f"sdiv {r()}, {r()}, {r()}", + f"udiv {r()}, {r()}, {r()}", + f"lsl {r()}, {r()}, #{rng.randrange(bits)}", + f"lsr {r()}, {r()}, #{rng.randrange(bits)}", + f"asr {r()}, {r()}, #{rng.randrange(bits)}", + f"ror {r()}, {r()}, #{rng.randrange(bits)}", + f"lsl {r()}, {r()}, {r()}", + f"lsr {r()}, {r()}, {r()}", + f"asr {r()}, {r()}, {r()}", + f"ror {r()}, {r()}, {r()}", + f"smull {xreg(rng)}, {wreg(rng)}, {wreg(rng)}", + f"umull {xreg(rng)}, {wreg(rng)}, {wreg(rng)}", + f"smaddl {xreg(rng)}, {wreg(rng)}, {wreg(rng)}, {xreg(rng)}", + f"umaddl {xreg(rng)}, {wreg(rng)}, {wreg(rng)}, {xreg(rng)}", + f"smulh {xreg(rng)}, {xreg(rng)}, {xreg(rng)}", + f"umulh {xreg(rng)}, {xreg(rng)}, {xreg(rng)}", + ] + assert_same(program(lines)) + + +def test_bitfield_and_extend(): + rng = random.Random(5) + lines = [] + for _ in range(n(8)): + is64 = rng.random() < 0.6 + bits = 64 if is64 else 32 + lsb = rng.randrange(bits) + width = rng.randint(1, bits - lsb) + r = lambda: reg(rng, is64) # noqa: E731 + for m in ("ubfx", "sbfx", "ubfiz", "sbfiz", "bfi", "bfxil"): + lines.append(f"{m} {r()}, {r()}, #{lsb}, #{width}") + lines += [ + f"sxtb {r()}, {wreg(rng)}", + f"sxth {r()}, {wreg(rng)}", + f"sxtw {xreg(rng)}, {wreg(rng)}", + f"uxtb {wreg(rng)}, {wreg(rng)}", + f"uxth {wreg(rng)}, {wreg(rng)}", + f"extr {r()}, {r()}, {r()}, #{rng.randrange(bits)}", + f"ubfm {r()}, {r()}, #{rng.randrange(bits)}, #{rng.randrange(bits)}", + ] + assert_same(program(lines)) + + +def test_conditional(): + rng = random.Random(6) + conds = [c for c in CONDITIONS if c not in ("al", "nv")] + lines = [] + for _ in range(n(8)): + is64 = rng.random() < 0.6 + r = lambda: reg(rng, is64) # noqa: E731 + c = rng.choice(conds) + lines += [ + f"csel {r()}, {r()}, {r()}, {c}", + f"csinc {r()}, {r()}, {r()}, {c}", + f"csinv {r()}, {r()}, {r()}, {c}", + f"csneg {r()}, {r()}, {r()}, {c}", + f"cset {r()}, {c}", + f"csetm {r()}, {c}", + f"cinc {r()}, {r()}, {c}", + f"cinv {r()}, {r()}, {c}", + f"cneg {r()}, {r()}, {c}", + f"ccmp {r()}, {r()}, #{rng.randrange(16)}, {c}", + f"ccmn {r()}, #{rng.randrange(32)}, #{rng.randrange(16)}, {c}", + ] + assert_same(program(lines)) + + +def test_loads_and_stores(): + rng = random.Random(7) + lines = [] + forms = [("ldr", True, 8), ("str", True, 8), ("ldr", False, 4), ("str", False, 4), + ("ldrb", False, 1), ("strb", False, 1), ("ldrh", False, 2), ("strh", False, 2), + ("ldrsb", True, 1), ("ldrsb", False, 1), ("ldrsh", True, 2), ("ldrsh", False, 2), + ("ldrsw", True, 4)] # fmt: skip + for m, is64, size in forms: + for _ in range(n(4)): + rt = reg(rng, is64) + base = rng.choice(["sp", xreg(rng, zr=False)]) + scaled = rng.randrange(4096) * size + lines += [ + f"{m} {rt}, [{base}]", + f"{m} {rt}, [{base}, #{scaled}]", + f"{m} {rt}, [{base}, #{rng.randint(-256, 255)}]", + f"{m} {rt}, [{base}, #{rng.randint(-256, 255)}]!", + f"{m} {rt}, [{base}], #{rng.randint(-256, 255)}", + f"{m} {rt}, [{base}, {xreg(rng)}]", + f"{m} {rt}, [{base}, {xreg(rng)}, lsl #{rng.choice([0, {1: 0, 2: 1, 4: 2, 8: 3}[size]])}]", + f"{m} {rt}, [{base}, {wreg(rng)}, {rng.choice(['uxtw', 'sxtw'])}]", + f"{m} {rt}, [{base}, {wreg(rng)}, sxtw #{ {1: 0, 2: 1, 4: 2, 8: 3}[size]}]", + ] + for m in ("ldur", "stur", "ldurb", "sturb", "ldurh", "sturh", "ldursw", "ldursb", "ldursh"): + for _ in range(n(3)): + is64 = m in ("ldursw",) or (m in ("ldur", "stur") and rng.random() < 0.5) + lines.append(f"{m} {reg(rng, is64)}, [{xreg(rng, zr=False)}, #{rng.randint(-256, 255)}]") + for m in ("ldp", "stp"): + for _ in range(n(6)): + is64 = rng.random() < 0.6 + size = 8 if is64 else 4 + a, b = rng.sample(range(31), 2) + rt = (f"x{a}", f"x{b}") if is64 else (f"w{a}", f"w{b}") + off = rng.randint(-64, 63) * size + base = rng.choice(["sp", xreg(rng, zr=False)]) + lines += [ + f"{m} {rt[0]}, {rt[1]}, [{base}]", + f"{m} {rt[0]}, {rt[1]}, [{base}, #{off}]", + f"{m} {rt[0]}, {rt[1]}, [{base}, #{off}]!", + f"{m} {rt[0]}, {rt[1]}, [{base}], #{off}", + ] + lines += ["stp x29, x30, [sp, #-16]!", "ldp x29, x30, [sp], #16"] + assert_same(program(lines)) + + +def test_branches(): + rng = random.Random(8) + labels = 10 + lines = [] + for _ in range(n(8)): + lines += [ + f"b L{rng.randrange(labels)}", + f"bl L{rng.randrange(labels)}", + f"b.{rng.choice(list(CONDITIONS))} L{rng.randrange(labels)}", + f"cbz {reg(rng, rng.random() < 0.5)}, L{rng.randrange(labels)}", + f"cbnz {reg(rng, rng.random() < 0.5)}, L{rng.randrange(labels)}", + f"tbz {xreg(rng)}, #{rng.randrange(64)}, L{rng.randrange(labels)}", + f"tbnz {wreg(rng)}, #{rng.randrange(32)}, L{rng.randrange(labels)}", + f"br {xreg(rng)}", + f"blr {xreg(rng)}", + f"ret {xreg(rng)}", + "ret", + f"adr {xreg(rng)}, L{rng.randrange(labels)}", + f"adrp {xreg(rng)}, L{rng.randrange(labels)}", + ] + rng.shuffle(lines) + assert_same_chunked(lines, labels, rng) + + +def test_literals_and_addresses(): + rng = random.Random(9) + lines = [] + for _ in range(n(6)): + value = rng.choice([rng.randrange(1 << 64), rng.randrange(1 << 16), 0x123456789, 5]) + lines += [ + f"ldr {xreg(rng, zr=False)}, ={hex(value)}", + f"ldr {wreg(rng, zr=False)}, ={rng.randrange(1 << 32)}", + f"ldr {xreg(rng, zr=False)}, ={rng.choice(['d0', 'd1', 'd1+8', '_start'])}", + f"ldrsw {xreg(rng, zr=False)}, ={rng.randrange(1 << 31)}", + f"adrp {xreg(rng)}, {rng.choice(['d0', 'd1'])}", + f"add {xreg(rng, zr=False)}, {xreg(rng, zr=False)}, :lo12:{rng.choice(['d0', 'd1'])}", + f"ldr {xreg(rng)}, [{xreg(rng, zr=False)}, :lo12:d1]", + f"ldr {wreg(rng)}, [{xreg(rng, zr=False)}, #:lo12:d0]", + ] + lines.insert(len(lines) // 2, ".ltorg") + assert_same(program(lines)) + + +def test_system(): + rng = random.Random(10) + lines = ["nop", "svc #0", "brk #1", f"svc #{rng.randrange(1 << 16)}", "hlt #0"] + assert_same(program(lines)) + + +def test_whole_examples(): + from mapachespim.examples import list_examples + + for example in list_examples(): + if example.isa == "arm" and example.source is not None: + assert_same(example.source.read_text().replace(".isa arm64", "")) + + +INVALID = [ + "add x0, x1, #4097", + "add x0, x1, w2", + "add x0, w1, x2", + "mov x0, #0x123456789", + "and x0, x1, #0", + "and x0, x1, #5", + "lsl x0, x1, #64", + "ldr x0, [x1, #4]!x", + "ldr x0, [x1, #1000]!", + "ldr w0, [x1, #32768]", + "ldr x0, [x1, :lo12:d3]", + "ldp x0, x1, [sp, #4]", + "b.xx L0", + "cbz x0", + "frobnicate x0", + "mov x0, x32", + "movz x0, #65536", + "svc #65536", + "adr x0, d0", +] + + +@pytest.mark.parametrize("line", INVALID) +def test_invalid_rejected(line): + source = program([line, "L0:"]) + assert not gnu_accepts(ISA, source), f"test assumption: GNU accepts {line!r}" + _, errors, _ = ours_text(ISA, source) + assert errors, f"MapacheSPIM accepted invalid input {line!r}" From e53c7f4533192305cab9f66a938140ce87ba6964 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 05:17:59 +0000 Subject: [PATCH 11/18] Add built-in x86-64 assembler matching GNU as; remove Keystone All four ISAs are now assembled by pure-Python encoders, so the Keystone dependency (no ARM wheels, no RISC-V support) is gone. x86-64 (toolchain/x86.py): - AT&T and Intel syntax, detected per instruction or set with .att_syntax/.intel_syntax; GNU's choice of encodings (short forms, accumulator forms, xchg special cases, truncated immediates). - GNU's jump relaxation: jumps start short and grow to rel32 only when out of range, seeing labels during a pass the way relax_segment does. Also found by extending the differential tests to ARM64 and x86-64: - Alignment in code is padded like GNU as on each ISA (x86 multi-byte nops with a jump over long runs, ARM64 zeros then nops, RISC-V c.nop); ARM64 aligns an instruction that follows data. - .p2align/.balign max-skip argument (empty arguments were dropped). - .fill directive; .word is 2 bytes on x86. - ISA aliases (mips, x64, aarch64) were not recognized by the directive parser, so e.g. --isa mips produced little-endian data. Tests: GNU differential tests for x86-64 and for alignment/data in code on all ISAs, x86 unit tests; CI installs x86-64 binutils. Docs updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- .github/workflows/ci.yml | 10 +- CHANGELOG.md | 19 +- README.md | 38 +- docs/user/quick-start.md | 8 - examples/README.md | 4 +- mapachespim/toolchain/assembler.py | 820 ++++++----------------- mapachespim/toolchain/directives.py | 91 ++- mapachespim/toolchain/riscv.py | 7 +- mapachespim/toolchain/x86.py | 978 ++++++++++++++++++++++++++++ pyproject.toml | 8 - tests/README.md | 9 +- tests/conftest.py | 25 - tests/gnu_reference.py | 10 +- tests/test_assembler_accuracy.py | 4 - tests/test_differential_align.py | 58 ++ tests/test_differential_data.py | 32 +- tests/test_differential_x86.py | 348 ++++++++++ tests/test_forward_references.py | 4 - tests/test_isa_specification.py | 4 - tests/test_toolchain.py | 6 +- tests/test_x86_encoder.py | 107 +++ 21 files changed, 1824 insertions(+), 766 deletions(-) create mode 100644 mapachespim/toolchain/x86.py delete mode 100644 tests/conftest.py create mode 100644 tests/test_differential_align.py create mode 100644 tests/test_differential_x86.py create mode 100644 tests/test_x86_encoder.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9861ea..a3b06d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,9 +34,7 @@ jobs: strategy: fail-fast: false matrix: - # ubuntu/windows are x86 (Keystone installed automatically); - # macos-14 is Apple Silicon, where Keystone has no wheel, so it - # covers the "no Keystone" install that many students will have. + # macos-14 is Apple Silicon, which many students have. os: [ubuntu-latest, windows-latest, macos-14] python-version: ["3.9", "3.13"] include: @@ -75,13 +73,15 @@ jobs: mapachespim --copy-examples ex mapachespim -e ex/riscv/fibonacci/fibonacci.s mapachespim -e ex/mips/fibonacci/fibonacci.s + mapachespim -e ex/arm/fibonacci/fibonacci.s + mapachespim -e ex/x86_64/fibonacci/fibonacci.S printf 'load riscv/fibonacci\nbreak fibonacci\nrun\nregs\nlist\nquit\n' | mapachespim - name: Install GNU binutils for the assembler differential tests if: runner.os == 'Linux' run: | sudo apt-get update -qq - sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu binutils-aarch64-linux-gnu + sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu binutils-aarch64-linux-gnu binutils-x86-64-linux-gnu - name: Run tests run: python -m pytest tests/ --tb=short @@ -101,7 +101,7 @@ jobs: - name: Install GNU binutils run: | sudo apt-get update -qq - sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu binutils-aarch64-linux-gnu + sudo apt-get install -y -qq binutils-riscv64-linux-gnu binutils-mips-linux-gnu binutils-aarch64-linux-gnu binutils-x86-64-linux-gnu - name: Install package run: pip install -e ".[dev]" diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b510e8..f19a490 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,9 @@ ## Unreleased ### Installation -- RISC-V and MIPS programs are assembled by new built-in pure-Python encoders, so they assemble on - every platform. (RISC-V could not be assembled at all with the Keystone release on PyPI.) -- Keystone is now only needed for ARM64 and x86-64, and is installed automatically only on x86 - machines where prebuilt wheels exist. Elsewhere it is the optional `keystone` extra. +- All four ISAs are assembled by new built-in pure-Python assemblers, so every ISA assembles on + every platform. The Keystone dependency is gone: it had no prebuilt packages for ARM machines + such as Apple Silicon Macs, and its PyPI release could not assemble RISC-V at all. - The example programs are included in the package, so `load riscv/fibonacci` works after `pip install`. `mapachespim --copy-examples DIR` copies them somewhere editable. - Python 3.9 or newer is required. @@ -33,7 +32,7 @@ - Loading a program starts from a fresh machine state and may switch ISA. ### Assembler -- The built-in RISC-V and MIPS assemblers now produce byte-for-byte the same code as GNU `as`, +- The built-in assemblers produce byte-for-byte the same code as GNU `as` for all four ISAs, verified by randomized differential tests against GNU binutils (run in CI). This changed some pseudo-instruction expansions to GNU's: RISC-V `call`/`tail` are always `auipc`+`jalr`, `li` uses GNU's algorithm; MIPS `la` is `lui`+`addiu`, and compare-branches, `abs`, `mul`/`div`/`rem` with @@ -51,3 +50,13 @@ - Debug info for big-endian targets (MIPS) was written in the wrong byte order, so source-level debugging never worked for MIPS. - `.isa riscv64 # comment` is recognized. +- ARM64: `ldr x0, =value` uses a literal pool (`.ltorg`/`.pool`) like GNU `as`; logical immediates, + `mov` aliases, and `//` comments follow GNU. The ARM examples were updated to GNU syntax. +- x86-64: AT&T and Intel syntax (auto-detected, or `.att_syntax`/`.intel_syntax`), GNU's choice of + encodings, and GNU's jump relaxation (short jumps grow to 32-bit only when needed). `.word` is 2 + bytes on x86, as in GNU `as`. +- Alignment in code is padded with nops the way GNU `as` does on each ISA; the `.p2align`/`.balign` + maximum-skip argument works; `.fill` is supported; `;` separates statements; numeric local labels + (`1:` with `1b`/`1f`) work. +- ISA aliases (`mips`, `x64`, `aarch64`) were treated as unknown ISAs by the directive parser, so + e.g. `--isa mips` produced little-endian data. diff --git a/README.md b/README.md index 2123a8d..b3955cd 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,8 @@ mapachespim --version - `unicorn` - CPU emulator framework - `capstone` - Disassembler - `pyelftools` - ELF file parsing -- `keystone-engine` - Assembler for ARM64 and x86-64, installed automatically on x86 machines. RISC-V - and MIPS use MapacheSPIM's built-in assembler, so they work everywhere; see - [ISA support](#multi-isa-support) for details. + +The assembler is built in (pure Python), so there is nothing else to install on any platform. ### Running MapacheSPIM @@ -254,23 +253,22 @@ MapacheSPIM is powered by the [Unicorn Engine](https://www.unicorn-engine.org/), MapacheSPIM supports four instruction set architectures, each with the same console commands, syscalls, and a matching set of example programs: -| ISA | Examples | Assembling `.s` files | -|-----|----------|-----------------------| -| RISC-V 64-bit (RV64IM) | `examples/riscv/` | Built in, works everywhere\* | -| MIPS32 (big-endian) | `examples/mips/` | Built in, works everywhere\* | -| ARM64 (AArch64) | `examples/arm/` | Uses [Keystone](https://www.keystone-engine.org/) | -| x86-64 (AT&T or Intel syntax) | `examples/x86_64/` | Uses [Keystone](https://www.keystone-engine.org/) | - -Keystone is installed automatically on x86 machines (Windows, Intel Macs, x86 Linux). Its PyPI release -has no prebuilt package for ARM machines such as Apple Silicon Macs, so it is not installed there by -default; you can still run and debug ARM64 and x86-64 programs (including all the bundled examples), and -can try `pip install 'mapachespim[keystone]'` to build Keystone from source (this needs CMake and a C++ -compiler). - -\* The built-in RISC-V and MIPS assemblers produce byte-for-byte the same machine code as GNU `as` -(the standard assembler used in textbooks and courses), including its pseudo-instruction expansions, -data directives, and branch relaxation. This is checked continuously by randomized differential tests -that assemble hundreds of thousands of instructions with both and compare the results. +| ISA | Examples | +|-----|----------| +| RISC-V 64-bit (RV64IM) | `examples/riscv/` | +| MIPS32 (big-endian) | `examples/mips/` | +| ARM64 (AArch64) | `examples/arm/` | +| x86-64 (AT&T or Intel syntax) | `examples/x86_64/` | + +All four are assembled by MapacheSPIM's built-in assembler, which works on every platform. For +everything it accepts, it produces byte-for-byte the same machine code as GNU `as` (the standard +assembler used in textbooks and courses), including pseudo-instruction expansions (RISC-V `li`, MIPS +`la`, ARM64 `ldr x0, =value` literal pools), data directives, alignment padding, and branch +relaxation (x86-64 short and long jumps). This is checked continuously by randomized differential +tests that assemble hundreds of thousands of instructions with both and compare the results. + +x86-64 accepts both AT&T syntax (`movq $1, %rax`) and Intel syntax (`mov rax, 1`), detected per +instruction, as well as the `.intel_syntax` / `.att_syntax` directives. Loading an ELF file detects its ISA automatically. diff --git a/docs/user/quick-start.md b/docs/user/quick-start.md index c77c655..1c6d3f9 100644 --- a/docs/user/quick-start.md +++ b/docs/user/quick-start.md @@ -207,14 +207,6 @@ mapachespim --copy-examples my-examples - With pip in a virtual environment, make sure it is activated. - You can always run `python3 -m mapachespim.console` instead. -### "Assembling arm64 requires the Keystone Engine" - -ARM64 and x86-64 programs are assembled with the Keystone library, which is installed automatically -on x86 computers but not on ARM computers such as Apple Silicon Macs (it has no prebuilt package for -them). RISC-V and MIPS always work. You can still load and debug the bundled ARM64 and x86-64 -examples, since they are already assembled. To build Keystone yourself (needs CMake and a C++ -compiler): `pip install 'mapachespim[keystone]'`. - ### Still Stuck? - Read the [Console Guide](console-guide.md) diff --git a/examples/README.md b/examples/README.md index 3831050..06ff6ca 100644 --- a/examples/README.md +++ b/examples/README.md @@ -63,8 +63,8 @@ or assemble a single file: mapachespim-as -g riscv/hello_asm/hello_asm.s -o hello_asm ``` -RISC-V and MIPS are assembled by MapacheSPIM's built-in assembler. ARM64 and x86-64 use the Keystone -library, which is installed automatically on x86 computers (see the main README for other machines). +All four ISAs are assembled by MapacheSPIM's built-in assembler, which produces the same machine +code as GNU `as`. The Makefiles inside each ISA directory are an alternative that builds with the GNU cross toolchains (for example `riscv64-unknown-elf-as`), for anyone who prefers them. diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index d206f0a..49ef010 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -1,5 +1,5 @@ """ -Multi-architecture assembler using Keystone Engine. +Multi-architecture assembler with built-in, GNU as compatible encoders. Assembles source code into machine code for RISC-V, ARM64, x86-64, and MIPS32. """ @@ -8,28 +8,13 @@ from dataclasses import dataclass, field from types import ModuleType -from typing import Any, Callable, Dict, List, Optional, Tuple - -try: - import keystone - - # Verify keystone actually works by trying to access a constant - _ = keystone.KS_ARCH_X86 - KEYSTONE_AVAILABLE = True - KEYSTONE_ERROR = None -except ImportError: - # Simply not installed; the message raised later explains how to add it - KEYSTONE_AVAILABLE = False - KEYSTONE_ERROR = None - keystone = None -except Exception as e: - # Keystone installed but native library failed to load - KEYSTONE_AVAILABLE = False - KEYSTONE_ERROR = f"Keystone native library failed: {e}" - keystone = None - -from . import arm64, mips, riscv +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Set, Tuple + +from . import arm64, mips, riscv, x86 from .directives import ( + ALIGN_DIRECTIVES, + BYTE_DIRECTIVES, + DATA_SIZES, INTERNAL_LABEL_PREFIXES, POOL_LABEL_PREFIX, DirectiveParser, @@ -54,6 +39,39 @@ def resolve(name: str) -> int: return resolve +class _RelaxView(Mapping[str, int]): + """ + Label lookup during one x86 relaxation pass, the way GNU as sees it: + labels already passed have their new address, labels ahead have their + old address shifted by the growth so far (``stretch``). + """ + + def __init__( + self, + data_labels: Dict[str, int], + old: Dict[str, int], + new: Dict[str, int], + stretch: int, + ) -> None: + self.data_labels, self.old, self.new, self.stretch = data_labels, old, new, stretch + + def __contains__(self, name: object) -> bool: + return name in self.new or name in self.old or name in self.data_labels + + def __iter__(self) -> Iterator[str]: + return iter({**self.data_labels, **self.old, **self.new}) + + def __len__(self) -> int: + return len({**self.data_labels, **self.old, **self.new}) + + def __getitem__(self, name: str) -> int: + if name in self.new: + return self.new[name] + if name in self.old: + return self.old[name] + self.stretch + return self.data_labels[name] + + @dataclass class AssemblyResult: """Result of assembly operation.""" @@ -87,20 +105,13 @@ def success(self) -> bool: return len(self.errors) == 0 and len(self.elf_bytes) > 0 -# Keystone architecture/mode constants -KS_ARCH_X86 = 4 - -KS_MODE_LITTLE_ENDIAN = 0 -KS_MODE_64 = 0x8 -KS_MODE_32 = 0x4 - - class Assembler: """ Multi-architecture assembler. - Uses Keystone Engine for instruction encoding and a custom - directive parser for GNU-as compatible source files. + Each ISA has a pure-Python encoder (riscv.py, mips.py, arm64.py, + x86.py) that produces the same bytes as GNU as, plus a shared directive + parser for GNU-as compatible source files. Example: >>> asm = Assembler("riscv64") @@ -118,52 +129,31 @@ class Assembler: # ISA configuration mapping ISA_CONFIG: Dict[str, Dict[str, Any]] = { - # RISC-V is encoded by the built-in pure-Python encoder (riscv.py) "riscv64": { - "arch": None, - "mode": None, "instr_size": 4, }, "riscv": { - "arch": None, - "mode": None, "instr_size": 4, }, - # ARM64 is encoded by the built-in pure-Python encoder (arm64.py) "arm64": { - "arch": None, - "mode": None, "instr_size": 4, }, "aarch64": { - "arch": None, - "mode": None, "instr_size": 4, }, "x86_64": { - "arch": KS_ARCH_X86, - "mode": KS_MODE_64, "instr_size": None, # Variable length }, "x86-64": { - "arch": KS_ARCH_X86, - "mode": KS_MODE_64, "instr_size": None, }, "x64": { - "arch": KS_ARCH_X86, - "mode": KS_MODE_64, "instr_size": None, }, - # MIPS is encoded by the built-in pure-Python encoder (mips.py) "mips32": { - "arch": None, - "mode": None, "instr_size": 4, }, "mips": { - "arch": None, - "mode": None, "instr_size": 4, }, } @@ -177,7 +167,6 @@ def __init__(self, isa: str): Raises: ValueError: If ISA is not supported. - ImportError: If Keystone is not available. """ isa_lower = isa.lower().replace("-", "_") if isa_lower not in self.ISA_CONFIG: @@ -187,28 +176,11 @@ def __init__(self, isa: str): self.isa = isa_lower self._config = self.ISA_CONFIG[isa_lower] self._layout = get_layout(isa_lower) - self._ks = None # Replaced for each assemble() call; also used by the layout passes self._parser = DirectiveParser(isa=isa_lower) - - # RISC-V and MIPS use built-in encoders; ARM64 and x86-64 need Keystone - if self._config["arch"] is None: - return - - if not KEYSTONE_AVAILABLE: - if KEYSTONE_ERROR: - msg = f"Assembling {isa} requires the Keystone Engine, which failed to load:\n" - msg += f" {KEYSTONE_ERROR}\n" - else: - msg = f"Assembling {isa} requires the Keystone Engine, which is not installed.\n" - msg += "Install it with: pip install 'mapachespim[keystone]'\n" - msg += "(RISC-V and MIPS programs can be assembled without it.)" - raise ImportError(msg) - - try: - self._ks = keystone.Ks(self._config["arch"], self._config["mode"]) - except keystone.KsError as e: - raise RuntimeError(f"Failed to initialize Keystone for {isa}: {e}") + self._long_jumps: Set[int] = set() # x86 jumps that need rel32 + self._align_after_data: Set[int] = set() # x86 code alignment after data + self._absolute_symbols: Set[str] = set() # names of .equ constants @staticmethod def _add_literal_pools(section: SectionData) -> None: @@ -234,7 +206,7 @@ def dump(line_number: int) -> None: LineType.DIRECTIVE, "", directive="balign", - directive_args=[str(size)], + directive_args=[str(size), "0"], # zeros, not nops ) ) for label, expr in entries: @@ -301,13 +273,33 @@ def _resolve_constants( progress = resolved_any = True return resolved_any + @staticmethod + def _alignments_after_data(section: Optional[SectionData]) -> Set[int]: + """Alignment directives in code whose last preceding item is data. + + GNU as pads x86 code with a one-byte nop first in that case, since + the data might be an incomplete instruction. + """ + found: Set[int] = set() + after_data = False + for line in section.lines if section else []: + if line.line_type == LineType.INSTRUCTION and line.instruction: + after_data = False + elif line.line_type == LineType.DIRECTIVE: + if line.directive in ALIGN_DIRECTIVES: + if after_data: + found.add(id(line)) + elif Assembler._emits_data(line): + after_data = True + return found + def _text_directive_bytes( self, line: ParsedLine, addr: int, base: int, labels: Dict[str, int], strict: bool ) -> bytes: """Bytes a data or alignment directive in .text produces at ``addr``. - Alignment padding in RISC-V code is made of nop instructions, as GNU - as does (MIPS's nop is all zeros, the default fill). + Alignment padding in code is made of nop instructions, as GNU as + does. """ values = {**labels, ".": addr} @@ -323,32 +315,82 @@ def resolve(name: str) -> int: ) if data is None: return b"" - is_align = line.directive in ("align", "balign", "p2align") - explicit_fill = len(line.directive_args) > 1 and line.directive_args[1] - if is_align and not explicit_fill and data and len(data) % 4 == 0 and self._is_riscv: - data = (0x00000013).to_bytes(4, "little") * (len(data) // 4) + is_align = line.directive in ALIGN_DIRECTIVES + args = line.directive_args + explicit_fill = len(args) > 1 and bool(args[1]) + if self._builtin_encoder is x86 and explicit_fill: + # GNU as treats a fill of 0x90 (nop) like no fill: best nops + explicit_fill = evaluate(args[1], resolve) & 0xFF != 0x90 + if not is_align or explicit_fill or not data: + return data + # Pad code with nops, as GNU as does (MIPS's nop is all zeros) + if self._is_riscv: + # As GNU as's riscv_make_nops: a zero byte if odd, at most one + # 2-byte c.nop, then 4-byte nops + head = bytes(len(data) % 2) + (b"\x01\x00" if len(data) % 4 >= 2 else b"") + data = head + (0x00000013).to_bytes(4, "little") * (len(data) // 4) + elif self._builtin_encoder is arm64: + misaligned = len(data) % 4 + data = bytes(misaligned) + (0xD503201F).to_bytes(4, "little") * (len(data) // 4) + elif self._builtin_encoder is x86: + data = x86.nop_padding(len(data), id(line) in self._align_after_data) return data def _line_options(self, line: ParsedLine) -> Dict[str, Any]: - """Per-line encoder options (MIPS: .set reorder / noreorder).""" + """Per-line encoder options (MIPS: .set reorder / noreorder; x86-64: + syntax, which symbols are constants, and jump size).""" if self.isa in ("mips32", "mips"): return {"reorder": line.reorder} + if self._builtin_encoder is x86: + return { + "syntax": line.syntax, + "absolute_symbols": self._absolute_symbols, + "long_jump": id(line) in self._long_jumps, + } return {} + def _instruction_padding(self, addr: int, after_data: bool) -> int: + """Zero bytes GNU as puts before an instruction at ``addr``. + + ARM64 aligns an instruction to 4 bytes when it directly follows data + (GNU as does this when switching from data to code; an alignment + directive in between counts as code). + """ + return -addr % 4 if after_data and self._builtin_encoder is arm64 else 0 + + @staticmethod + def _emits_data(line: ParsedLine) -> bool: + """Whether ``line`` is a data directive such as .byte or .fill.""" + directive = line.directive or "" + return line.line_type == LineType.DIRECTIVE and ( + directive in DATA_SIZES or directive in BYTE_DIRECTIVES or directive == "value" + ) + + def _is_real_alignment(self, line: ParsedLine, labels: Dict[str, int]) -> bool: + """Whether ``line`` aligns to more than 1 byte (even if it inserts + no padding), which GNU as treats like code.""" + if line.line_type != LineType.DIRECTIVE or line.directive not in ALIGN_DIRECTIVES: + return False + try: + value = evaluate(line.directive_args[0], _lookup(labels)) + except (ExpressionError, IndexError): + return False + return self._parser.alignment(line.directive or "", value) > 1 + @property def _is_riscv(self) -> bool: return self.isa in ("riscv64", "riscv") @property - def _builtin_encoder(self) -> Optional[ModuleType]: - """The pure-Python encoder module for this ISA, or None for Keystone ISAs.""" + def _builtin_encoder(self) -> ModuleType: + """The pure-Python encoder module for this ISA.""" if self._is_riscv: return riscv if self.isa in ("mips32", "mips"): return mips if self.isa in ("arm64", "aarch64"): return arm64 - return None + return x86 def assemble( self, @@ -374,7 +416,10 @@ def assemble( # Parse source parser = DirectiveParser(isa=self.isa) self._parser = parser + self._long_jumps = set() sections = parser.parse(source) + self._absolute_symbols = set(parser.constants) | set(parser.deferred_constants) + self._align_after_data = self._alignments_after_data(sections.get(".text")) if self.isa in ("arm64", "aarch64") and ".text" in sections: try: self._add_literal_pools(sections[".text"]) @@ -572,8 +617,8 @@ def _calculate_text_labels( Pass 1 of two-pass assembly: determine where each label will be based on instruction count. - For fixed-size ISAs (RISC-V, ARM64, MIPS): 4 bytes per instruction. - For variable-size ISAs (x86-64): use iterative assembly until stable. + Sizes come from the encoder itself, so pseudo-instructions and + x86-64's variable-length jumps are laid out exactly as encoded. Args: section: The text section to process. @@ -583,35 +628,7 @@ def _calculate_text_labels( Returns: Dictionary of label name -> address. """ - from .directives import LineType - - instr_size = self._config.get("instr_size", 4) - - # Variable-length ISA (x86-64) needs iterative approach - if instr_size is None: - return self._calculate_x86_text_labels_iterative(section, base_addr) - - if self._builtin_encoder is not None: - return self._calculate_builtin_text_labels(section, base_addr, data_labels or {}) - - # Fixed-size ISA: single pass with size estimation - labels: Dict[str, int] = {} - current_addr = base_addr - - for line in section.lines: - # Record label position - if line.label: - labels[line.label] = current_addr - - # Skip non-instructions - if line.line_type != LineType.INSTRUCTION or not line.instruction: - continue - - # Check for pseudo-instructions that expand to multiple instructions - size = self._estimate_fixed_instr_size(line.instruction, instr_size, data_labels) - current_addr += size - - return labels + return self._calculate_builtin_text_labels(section, base_addr, data_labels or {}) def _calculate_builtin_text_labels( self, @@ -627,17 +644,44 @@ def _calculate_builtin_text_labels( re-sizes with the labels found so far until they are stable. """ labels: Dict[str, int] = {} - for _ in range(8): + for _ in range(100): known = {**data_labels, **labels} new_labels: Dict[str, int] = {} addr = base_addr + grew = False + stretch = 0 # growth so far this pass, as in GNU's relax_segment + after_data = False # see _instruction_padding for line in section.lines: if line.label: new_labels[line.label] = addr + if line.label in labels: + stretch = addr - labels[line.label] if line.line_type == LineType.INSTRUCTION and line.instruction: - addr += self._builtin_encoder.instruction_size( + addr += self._instruction_padding(addr, after_data) + after_data = False + # x86 jumps start short (rel8) and grow to rel32 once, for + # good, when the target is out of range (GNU's relaxation) + if ( + self._builtin_encoder is x86 + and id(line) not in self._long_jumps + and x86.is_relaxable(line.instruction) + and not x86.short_jump_fits( + line.instruction, + addr, + _RelaxView(data_labels, labels, new_labels, stretch), + ) + ): + self._long_jumps.add(id(line)) + grew = True + grown = True + else: + grown = False + size = self._builtin_encoder.instruction_size( line.instruction, known, addr, **self._line_options(line) ) + if grown: + stretch += size - 2 + addr += size elif line.line_type == LineType.DIRECTIVE: try: addr += len( @@ -645,279 +689,15 @@ def _calculate_builtin_text_labels( ) except ExpressionError: pass # reported when the section is assembled - if new_labels == labels: + if self._emits_data(line): + after_data = True + elif self._is_real_alignment(line, known): + after_data = False + if new_labels == labels and not grew: break labels = new_labels return labels - def _calculate_x86_text_labels_iterative( - self, - section: SectionData, - base_addr: int, - max_iterations: int = 5, - ) -> Dict[str, int]: - """ - Calculate x86-64 label addresses using iterative refinement. - - For variable-length instruction sets, instruction sizes can depend on - label addresses (e.g., short vs near jumps). This method iterates - until label addresses stabilize. - - Args: - section: The text section to process. - base_addr: Base address for the section. - max_iterations: Maximum iterations before giving up. - - Returns: - Dictionary of label name -> address. - """ - from .directives import LineType - - labels: Dict[str, int] = {} - - for _ in range(max_iterations): - new_labels: Dict[str, int] = {} - current_addr = base_addr - - for line in section.lines: - # Record label at current position - if line.label: - new_labels[line.label] = current_addr - - # Skip non-instructions - if line.line_type != LineType.INSTRUCTION or not line.instruction: - continue - - # Get actual instruction size by assembling - size = self._get_x86_actual_size(line.instruction, current_addr, labels) - current_addr += size - - # Check for convergence - if new_labels == labels: - return new_labels - - labels = new_labels - - # Return best effort if max iterations reached - return labels - - def _get_x86_actual_size( - self, - instr: str, - addr: int, - labels: Dict[str, int], - ) -> int: - """ - Get actual x86-64 instruction size by assembling it. - - Args: - instr: Instruction text (AT&T or Intel syntax). - addr: Current address for PC-relative calculations. - labels: Currently known label addresses. - - Returns: - Instruction size in bytes. - """ - # Convert to Intel syntax, resolving known labels - converted = self._convert_x86_att_to_intel(instr, labels) - - # Try to assemble - try: - encoding, count = self._ks.asm(converted, addr) - if encoding is not None and count > 0: - return len(encoding) - except keystone.KsError: - pass - - # Assembly failed - likely unresolved forward reference - # Try with a placeholder to get encoding size - parts = converted.split(None, 1) - mnemonic = parts[0].lower() if parts else "" - - # For branch/call instructions, use placeholder offset - if mnemonic in ( - "jmp", - "call", - "je", - "jne", - "jz", - "jnz", - "jl", - "jle", - "jg", - "jge", - "ja", - "jae", - "jb", - "jbe", - "jo", - "jno", - "js", - "jns", - "jc", - "jnc", - "loop", - "loope", - "loopne", - ): - # Use a medium-range placeholder to get near (not short) encoding - placeholder_addr = addr + 0x100 - placeholder_instr = f"{mnemonic} {placeholder_addr}" - try: - encoding, count = self._ks.asm(placeholder_instr, addr) - if encoding is not None and count > 0: - return len(encoding) - except keystone.KsError: - pass - - # Final fallback: use existing estimation - return self._estimate_x86_instr_size(instr) - - def _estimate_fixed_instr_size( - self, - instr: str, - base_size: int, - data_labels: Optional[Dict[str, int]] = None, - ) -> int: - """ - Estimate instruction size for fixed-size ISAs. - - Accounts for pseudo-instructions that expand to multiple instructions. - - Args: - instr: The instruction text. - base_size: Base instruction size (typically 4 bytes). - data_labels: Optional dict of data section labels for address lookups. - """ - parts = instr.split(None, 1) - if not parts: - return base_size - - mnemonic = parts[0].lower() - - if self._builtin_encoder is not None: - return self._builtin_encoder.instruction_size(instr, data_labels or {}) - - return base_size - - def _estimate_x86_instr_size(self, instr: str) -> int: - """ - Estimate the size of an x86-64 instruction. - - This is a conservative estimate for pass 1. - Pass 2 will use actual assembled sizes. - """ - parts = instr.split(None, 1) - if not parts: - return 1 - - mnemonic = parts[0].lower().rstrip("bwlq") - operands = parts[1] if len(parts) > 1 else "" - - # Single/two-byte instructions - if mnemonic == "nop": - return 1 - if mnemonic == "ret": - return 1 - if mnemonic in ("syscall", "hlt", "cld", "std", "cdqe", "cqo"): - return 2 - - # Near jumps/calls with label: 5 bytes (1 opcode + 4 offset) - # But conditional jumps can be 2 bytes for short jumps, assume 6 for safety - if mnemonic in ("jmp", "call"): - return 5 - if mnemonic in ( - "je", - "jne", - "jz", - "jnz", - "jl", - "jle", - "jg", - "jge", - "ja", - "jae", - "jb", - "jbe", - "jo", - "jno", - "js", - "jns", - "jc", - "jnc", - "loop", - "loope", - "loopne", - ): - return 6 # 0F XX + 4-byte offset - - # RIP-relative addressing: 7 bytes - # Format: symbol(%rip) or [rip + symbol] - if "%rip" in operands or "rip" in operands.lower(): - # REX.W (1) + opcode (1-2) + ModR/M (1) + disp32 (4) = 7-8 bytes - return 7 - - # LEA with memory operand - if mnemonic == "lea": - if "%rip" in operands or "rip" in operands.lower(): - return 7 - return 4 # Base case - - # MOV with 64-bit immediate - if mnemonic == "mov": - # Check for 64-bit register destination with immediate - if operands.startswith("$") or operands.startswith("%"): - # AT&T: movq $imm, %reg or Intel mov reg, imm - if any( - r in operands - for r in [ - "%rax", - "%rbx", - "%rcx", - "%rdx", - "%rsi", - "%rdi", - "%rbp", - "%rsp", - "%r8", - "%r9", - "%r10", - "%r11", - "%r12", - "%r13", - "%r14", - "%r15", - "rax", - "rbx", - "rcx", - "rdx", - ] - ): - # Could be 10 bytes for movabs - if "$" in operands: - return 10 - return 7 - # Simple reg-reg or reg-mem: 2-4 bytes - return 3 - - # TEST, CMP with register: 2-3 bytes - if mnemonic in ("test", "cmp"): - if "(" not in operands and "[" not in operands: - return 3 - - # ADD, SUB, XOR, AND, OR with immediate or register: 3-7 bytes - if mnemonic in ("add", "sub", "xor", "and", "or"): - if "$" in operands or any(c.isdigit() for c in operands[:5]): - return 7 # Could have 32-bit immediate - return 3 # Register-register - - # Push/pop: 1-2 bytes - if mnemonic in ("push", "pop"): - return 2 - - # Default for unknown instructions - return 5 - def _assemble_section( self, section: SectionData, @@ -936,14 +716,15 @@ def _assemble_section( errors: List[str] = [] debug_lines: List[Tuple[int, int]] = [] current_addr = base_addr + after_data = False # see _instruction_padding for line in section.lines: # Record label position if line.label: label_addrs[line.label] = current_addr - # Data and alignment directives in code (built-in encoders only) - if line.line_type == LineType.DIRECTIVE and self._builtin_encoder is not None: + # Data and alignment directives in code + if line.line_type == LineType.DIRECTIVE: try: data = self._text_directive_bytes( line, current_addr, base_addr, labels, strict=True @@ -953,54 +734,39 @@ def _assemble_section( continue code.extend(data) current_addr += len(data) + if self._emits_data(line): + after_data = True + elif self._is_real_alignment(line, labels): + after_data = False continue # Skip non-instructions if line.line_type != LineType.INSTRUCTION or not line.instruction: continue + padding = self._instruction_padding(current_addr, after_data) + after_data = False + code.extend(bytes(padding)) + current_addr += padding + # Record debug line info before assembling debug_lines.append((current_addr, line.line_number)) instr = line.instruction - # RISC-V and MIPS: built-in encoders handle instructions and - # pseudo-instructions directly - encoder = self._builtin_encoder - if encoder is not None: - try: - encoding = encoder.encode( - instr, current_addr, labels, **self._line_options(line) - ) - except EncodeError as e: - errors.append(f"Line {line.line_number}: {e} - {line.instruction}") - continue - code.extend(encoding) - current_addr += len(encoding) - continue - - # Convert AT&T to Intel syntax for x86-64 - if self.isa == "x86_64": - instr = self._convert_x86_att_to_intel(instr, labels) - - # Assemble instruction try: - encoding, count = self._ks.asm(instr, current_addr) - if encoding is None or count == 0: - errors.append( - f"Line {line.line_number}: Failed to assemble: {line.instruction}" - ) - continue - - code.extend(encoding) - current_addr += len(encoding) - - except keystone.KsError as e: + encoding = self._builtin_encoder.encode( + instr, current_addr, labels, **self._line_options(line) + ) + except EncodeError as e: errors.append(f"Line {line.line_number}: {e} - {line.instruction}") + continue + code.extend(encoding) + current_addr += len(encoding) # Pass 1 must have predicted every label's address, or branch offsets # computed from those predictions would be wrong - if not errors and self.isa != "x86_64": + if not errors: for name, addr in label_addrs.items(): if labels.get(name, addr) != addr: errors.append( @@ -1010,206 +776,6 @@ def _assemble_section( return bytes(code), label_addrs, errors, debug_lines - def _convert_x86_att_to_intel( - self, - instr: str, - labels: Dict[str, int], - ) -> str: - """ - Convert x86 AT&T syntax to Intel syntax. - - Handles common patterns: - - %reg -> reg (remove % prefix) - - $imm -> imm (remove $ prefix) - - op src, dst -> op dst, src (reverse operand order) - - symbol(%rip) -> [rip + symbol] (RIP-relative addressing) - """ - import re - - parts = instr.split(None, 1) - if not parts: - return instr - - mnemonic = parts[0].lower() - operands = parts[1] if len(parts) > 1 else "" - - # Check if already Intel syntax (no % or $) - is_att_syntax = "%" in instr or "$" in instr - - # For Intel-syntax jump/call with symbol, resolve the symbol - if not is_att_syntax: - if mnemonic in ( - "jmp", - "call", - "je", - "jne", - "jz", - "jnz", - "jl", - "jle", - "jg", - "jge", - "ja", - "jae", - "jb", - "jbe", - "jo", - "jno", - "js", - "jns", - "jc", - "jnc", - "loop", - "loope", - "loopne", - ): - symbol = operands.strip() - if symbol in labels: - return f"{mnemonic} {labels[symbol]}" - return instr - - # Handle special AT&T mnemonics - # movslq = move sign-extend long to quad (Intel: movsxd) - if mnemonic == "movslq": - mnemonic = "movsxd" - # cltq = sign-extend eax to rax (Intel: cdqe) - elif mnemonic == "cltq": - return "cdqe" - # cqto = sign-extend rax to rdx:rax (Intel: cqo) - elif mnemonic == "cqto": - return "cqo" - - # Remove size suffixes (b, w, l, q) - if mnemonic.endswith(("b", "w", "l", "q")) and len(mnemonic) > 2: - base = mnemonic[:-1] - if base in ( - "mov", - "add", - "sub", - "xor", - "and", - "or", - "cmp", - "test", - "lea", - "push", - "pop", - "call", - "ret", - "jmp", - "dec", - "inc", - "neg", - "not", - "mul", - "imul", - "div", - "idiv", - "shl", - "shr", - "sar", - "sal", - "rol", - "ror", - "rcl", - "rcr", - ): - mnemonic = base - - # Handle no-operand instructions - if not operands: - return mnemonic - - # Split operands - op_list = [] - depth = 0 - current = "" - for c in operands: - if c == "(": - depth += 1 - elif c == ")": - depth -= 1 - elif c == "," and depth == 0: - op_list.append(current.strip()) - current = "" - continue - current += c - if current.strip(): - op_list.append(current.strip()) - - # Convert each operand - converted = [] - for op in op_list: - op = op.strip() - - # Handle RIP-relative: symbol(%rip) -> [rip + address] - rip_match = re.match(r"(\w+)\s*\(\s*%rip\s*\)", op) - if rip_match: - symbol = rip_match.group(1) - if symbol in labels: - addr = labels[symbol] - converted.append(f"[0x{addr:x}]") - else: - # Symbol not found, use placeholder - converted.append(f"[rip + {symbol}]") - continue - - # Handle memory operands with index and scale: offset(base, index, scale) - # AT&T: (%r12, %rax, 4) -> Intel: [r12 + rax*4] - # AT&T: 16(%rsp, %rax, 8) -> Intel: [rsp + rax*8 + 16] - sib_match = re.match(r"(-?\d+)?\s*\(\s*%(\w+)\s*,\s*%(\w+)\s*,\s*(\d+)\s*\)", op) - if sib_match: - offset = sib_match.group(1) - base = sib_match.group(2) - index = sib_match.group(3) - scale = sib_match.group(4) - intel_op = f"[{base} + {index}*{scale}" - if offset: - intel_op += f" + {offset}" - intel_op += "]" - converted.append(intel_op) - continue - - # Handle memory operands: (%reg) -> [reg], offset(%reg) -> [reg + offset] - mem_match = re.match(r"(-?\d+)?\s*\(\s*%(\w+)\s*\)", op) - if mem_match: - offset = mem_match.group(1) - reg = mem_match.group(2) - if offset: - converted.append(f"[{reg} + {offset}]") - else: - converted.append(f"[{reg}]") - continue - - # Handle immediate: $value -> value - if op.startswith("$"): - value = op[1:] - # Check if it's a symbol - if value in labels: - converted.append(str(labels[value])) - else: - converted.append(value) - continue - - # Handle register: %reg -> reg - if op.startswith("%"): - converted.append(op[1:]) - continue - - # Check if it's a bare symbol (for jump/call targets) - if op in labels: - converted.append(str(labels[op])) - continue - - # Pass through as-is - converted.append(op) - - # Reverse operand order for two-operand instructions (AT&T: src, dst -> Intel: dst, src) - if len(converted) == 2 and mnemonic not in ("push", "pop", "call", "jmp", "syscall"): - converted = converted[::-1] - - return f"{mnemonic} {', '.join(converted)}" - def assemble_instruction(self, instr: str, address: int = 0) -> bytes: """ Assemble a single instruction. @@ -1224,15 +790,7 @@ def assemble_instruction(self, instr: str, address: int = 0) -> bytes: Raises: ValueError: If assembly fails. """ - if self._builtin_encoder is not None: - try: - return self._builtin_encoder.encode(instr, address, {}) - except EncodeError as e: - raise ValueError(f"Assembly error: {e}") try: - encoding, count = self._ks.asm(instr, address) - if encoding is None or count == 0: - raise ValueError(f"Failed to assemble: {instr}") - return bytes(encoding) - except keystone.KsError as e: + return bytes(self._builtin_encoder.encode(instr, address, {})) + except EncodeError as e: raise ValueError(f"Assembly error: {e}") diff --git a/mapachespim/toolchain/directives.py b/mapachespim/toolchain/directives.py index 7781a99..b058ba3 100644 --- a/mapachespim/toolchain/directives.py +++ b/mapachespim/toolchain/directives.py @@ -14,6 +14,20 @@ from .expr import ExpressionError, UndefinedSymbol, evaluate +ALIGN_DIRECTIVES = ("align", "balign", "p2align") + +_ISA_ALIASES = {"riscv": "riscv64", "aarch64": "arm64", "x64": "x86_64", "mips": "mips32"} + + +def canonical_isa(isa: str) -> str: + """The canonical name for an ISA or one of its aliases (e.g. mips -> mips32).""" + isa = isa.lower().replace("-", "_") + return _ISA_ALIASES.get(isa, isa) + + +# Other directives that emit bytes (besides the DATA_SIZES ones) +BYTE_DIRECTIVES = ("ascii", "asciz", "string", "space", "skip", "zero", "fill") + class LineType(Enum): """Type of assembly source line.""" @@ -51,6 +65,9 @@ class ParsedLine: reorder: bool = True """MIPS: whether the assembler fills delay slots (.set reorder, the default).""" + + syntax: str = "auto" + """x86-64: "att", "intel", or "auto" (from .att_syntax / .intel_syntax).""" """Instruction mnemonic and operands.""" @@ -162,7 +179,7 @@ def __init__(self, isa: Optional[str] = None) -> None: decides byte order and what .align means; an .isa directive in the source is used when this is not given. """ - self.target_isa = isa.lower().replace("-", "_") if isa else None + self.target_isa = canonical_isa(isa) if isa else None self.sections: Dict[str, SectionData] = {} self.current_section: str = ".text" self.global_symbols: set = set() @@ -174,6 +191,7 @@ def __init__(self, isa: Optional[str] = None) -> None: # .equ/.set constants that refer to labels: name -> (expression, line) self.deferred_constants: Dict[str, Tuple[str, int]] = {} self.reorder = True # MIPS .set reorder / .set noreorder + self.syntax = "auto" # x86-64 .att_syntax / .intel_syntax self._dot_count = 0 # Initialize default sections @@ -190,7 +208,16 @@ def _get_endianness(self) -> Literal["little", "big"]: return "big" return "little" - def _alignment(self, directive: str, value: int) -> int: + def data_size(self, directive: str) -> Optional[int]: + """Bytes per value for a data directive like .word, or None. + + As in GNU as, .word (and .value) is 2 bytes on x86 but 4 elsewhere. + """ + if self._effective_isa == "x86_64" and directive in ("word", "value"): + return 2 + return DATA_SIZES.get(directive) + + def alignment(self, directive: str, value: int) -> int: """Alignment in bytes for .align/.balign/.p2align with argument value. As in GNU as, .align takes a power of two on RISC-V, MIPS, and ARM, @@ -236,8 +263,8 @@ def data_bytes( """ byteorder = self._get_endianness() - if directive in DATA_SIZES: - size = DATA_SIZES[directive] + size = self.data_size(directive) + if size is not None: out = bytearray() for arg in args: try: @@ -267,14 +294,38 @@ def data_bytes( fill = evaluate(args[1], resolve) & 0xFF if len(args) > 1 else 0 return bytes([fill]) * size - if directive in ("align", "balign", "p2align"): + if directive == "fill": + # .fill repeat, size, value: as in GNU as, size is capped at 8 + # and only the low 4 bytes of each item come from value + if not args: + raise ExpressionError(".fill needs a repeat count") + repeat = evaluate(args[0], resolve) + size = evaluate(args[1], resolve) if len(args) > 1 and args[1] else 1 + value = evaluate(args[2], resolve) & 0xFFFFFFFF if len(args) > 2 else 0 + if repeat < 0 or size < 0: + raise ExpressionError(".fill repeat and size must not be negative") + size = min(size, 8) + if size <= 4: + item = value.to_bytes(4, byteorder) + item = item[-size:] if byteorder == "big" else item[:size] + else: + item = value.to_bytes(4, byteorder) + bytes(size - 4) + return item * repeat if size else b"" + + if directive in ALIGN_DIRECTIVES: if not args: raise ExpressionError(f".{directive} needs an alignment") - alignment = self._alignment(directive, evaluate(args[0], resolve)) + alignment = self.alignment(directive, evaluate(args[0], resolve)) if alignment <= 0 or alignment & (alignment - 1): raise ExpressionError(f"alignment {alignment} is not a power of 2") fill = evaluate(args[1], resolve) & 0xFF if len(args) > 1 and args[1] else 0 - return bytes([fill]) * ((alignment - offset % alignment) % alignment) + padding = (alignment - offset % alignment) % alignment + # The optional third argument is the most padding to insert (0 + # means no limit); when more would be needed, nothing is inserted. + limit = evaluate(args[2], resolve) if len(args) > 2 and args[2] else 0 + if 0 < limit < padding: + padding = 0 + return bytes([fill]) * padding return None @@ -303,6 +354,7 @@ def parse(self, source: str) -> Dict[str, SectionData]: def _add_line(self, parsed: ParsedLine) -> None: """Record a parsed line in the current section.""" parsed.reorder = self.reorder + parsed.syntax = self.syntax section = self.sections[self.current_section] section.lines.append(parsed) @@ -542,7 +594,10 @@ def _parse_line(self, line_num: int, line: str) -> ParsedLine: return ParsedLine(line_num, LineType.INSTRUCTION, original, label=label, instruction=line) def _parse_args(self, args_str: str) -> List[str]: - """Parse comma-separated directive arguments, respecting quotes.""" + """Parse comma-separated directive arguments, respecting quotes. + + Empty arguments are kept, so later arguments keep their position. + """ args = [] current = "" in_string = False @@ -558,15 +613,12 @@ def _parse_args(self, args_str: str) -> List[str]: string_char = char current += char elif char == ",": - if current.strip(): - args.append(current.strip()) + args.append(current.strip()) # may be empty: ".p2align 4,,7" current = "" else: current += char - if current.strip(): - args.append(current.strip()) - + args.append(current.strip()) return args def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: @@ -618,6 +670,11 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: self.local_symbols.add(arg) return + # x86-64 syntax selection + if directive in ("intel_syntax", "att_syntax"): + self.syntax = "intel" if directive == "intel_syntax" else "att" + return + # MIPS assembler options: ".set noreorder", ".set noat", ... if directive == "set" and len(args) == 1: option = args[0].lower() @@ -641,12 +698,8 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: # Like GNU as for MIPS, align .half/.word/.dword to their size, and # move labels that were waiting for this data along with it - if ( - directive in DATA_SIZES - and DATA_SIZES[directive] > 1 - and self._effective_isa in self.AUTO_ALIGN_ISAS - ): - size = DATA_SIZES[directive] + size = self.data_size(directive or "") + if size is not None and size > 1 and self._effective_isa in self.AUTO_ALIGN_ISAS: padding = (size - section.current_offset % size) % size if padding: section.data.extend(b"\x00" * padding) diff --git a/mapachespim/toolchain/riscv.py b/mapachespim/toolchain/riscv.py index 1c6a23f..f917af6 100644 --- a/mapachespim/toolchain/riscv.py +++ b/mapachespim/toolchain/riscv.py @@ -1,10 +1,9 @@ """ Pure-Python RISC-V (RV64IM) instruction encoder. -The Keystone release on PyPI has no RISC-V support, so MapacheSPIM encodes -RISC-V itself. RISC-V's six instruction formats are regular enough that a -table-driven encoder is short, and doing it here means the most commonly -taught ISA needs no native assembler library at all. +RISC-V's six instruction formats are regular enough that a table-driven +encoder is short. Like the other built-in encoders, it matches GNU as byte +for byte, including pseudo-instruction expansions. Supported: - RV64I base integer instructions (including the *W word variants) diff --git a/mapachespim/toolchain/x86.py b/mapachespim/toolchain/x86.py new file mode 100644 index 0000000..f22529d --- /dev/null +++ b/mapachespim/toolchain/x86.py @@ -0,0 +1,978 @@ +""" +Pure-Python x86-64 instruction encoder, compatible with GNU as. + +Accepts AT&T syntax (GNU's default: ``movq $4, %rax``) and Intel syntax +(``mov rax, 4``). Without a ``.att_syntax``/``.intel_syntax`` directive, each +instruction's syntax is recognized from its operands (``%``/``$`` mean AT&T). + +Encodings follow GNU as's choices: accumulator short forms when shorter, +sign-extended 8-bit immediates when a constant fits, 32-bit fields for +symbolic values, ``c7`` vs ``movabs`` for 64-bit moves, and jumps that +start short (rel8) and grow to rel32 only when the target is out of range. + +Supported instructions (with b/w/l/q operand sizes): + mov, movabs, movzx/movz*, movsx/movs*, movsxd/movslq, lea, xchg, + add, or, adc, sbb, and, sub, xor, cmp, test, inc, dec, neg, not, + mul, imul (1, 2, 3 operands), div, idiv, shl/sal, shr, sar, rol, ror, + rcl, rcr, push, pop, jmp, call, ret, j, set, cmov, + cbw/cwde/cdqe, cwd/cdq/cqo (cwtl/cltq/cltd/cqto), leave, nop, hlt, + syscall, int, int3 +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Callable, Dict, List, Mapping, Optional, Set, Tuple, Union + +from .expr import Context, EncodeError, ExpressionError, UndefinedSymbol, evaluate + +# --------------------------------------------------------------------------- +# Registers +# --------------------------------------------------------------------------- + + +class X86EncodeError(EncodeError): + """Raised when an x86-64 instruction cannot be encoded.""" + + +@dataclass(frozen=True) +class Reg: + num: int # 0-15 (ah/ch/dh/bh use 4-7 with high8) + size: int # bytes: 1, 2, 4, 8 + high8: bool = False # ah, ch, dh, bh (cannot be used with REX) + needs_rex: bool = False # spl, bpl, sil, dil + rip: bool = False # %rip, only valid as a memory base + + +_REGS: Dict[str, Reg] = {} +for _i, _name in enumerate(["ax", "cx", "dx", "bx", "sp", "bp", "si", "di"]): + _REGS["r" + _name] = Reg(_i, 8) + _REGS["e" + _name] = Reg(_i, 4) + _REGS[_name] = Reg(_i, 2) +for _i in range(8, 16): + _REGS[f"r{_i}"] = Reg(_i, 8) + _REGS[f"r{_i}d"] = Reg(_i, 4) + _REGS[f"r{_i}w"] = Reg(_i, 2) + _REGS[f"r{_i}b"] = Reg(_i, 1) + _REGS[f"r{_i}l"] = Reg(_i, 1) +for _i, _name in enumerate(["al", "cl", "dl", "bl"]): + _REGS[_name] = Reg(_i, 1) +for _i, _name in enumerate(["ah", "ch", "dh", "bh"]): + _REGS[_name] = Reg(_i + 4, 1, high8=True) +for _i, _name in enumerate(["spl", "bpl", "sil", "dil"]): + _REGS[_name] = Reg(_i + 4, 1, needs_rex=True) +_REGS["rip"] = Reg(0, 8, rip=True) + +SIZE_SUFFIX = {"b": 1, "w": 2, "l": 4, "q": 8} +PTR_SIZES = {"byte": 1, "word": 2, "dword": 4, "qword": 8} + +CONDITIONS = { + "o": 0, "no": 1, "b": 2, "c": 2, "nae": 2, "ae": 3, "nb": 3, "nc": 3, + "e": 4, "z": 4, "ne": 5, "nz": 5, "be": 6, "na": 6, "a": 7, "nbe": 7, + "s": 8, "ns": 9, "p": 10, "pe": 10, "np": 11, "po": 11, + "l": 12, "nge": 12, "ge": 13, "nl": 13, "le": 14, "ng": 14, "g": 15, "nle": 15, +} # fmt: skip + +ALU_OPS = {"add": 0, "or": 1, "adc": 2, "sbb": 3, "and": 4, "sub": 5, "xor": 6, "cmp": 7} +SHIFT_OPS = {"rol": 0, "ror": 1, "rcl": 2, "rcr": 3, "shl": 4, "sal": 4, "shr": 5, "sar": 7} +UNARY_OPS = {"not": 2, "neg": 3, "mul": 4, "div": 6, "idiv": 7} + + +def _fits8(v: int) -> bool: + return -128 <= v <= 127 + + +def _fits32(v: int) -> bool: + return -(1 << 31) <= v < (1 << 31) + + +# --------------------------------------------------------------------------- +# Operands +# --------------------------------------------------------------------------- + + +@dataclass +class Imm: + expr: str + + +@dataclass +class Mem: + disp: str # expression text ("" for none) + base: Optional[Reg] = None + index: Optional[Reg] = None + scale: int = 1 + size: Optional[int] = None # from Intel "qword ptr" + + +@dataclass +class Target: + """A jump/call destination written as a label or address.""" + + expr: str + + +Operand = Union[Reg, Imm, Mem, Target] + + +def _split(text: str) -> List[str]: + """Split operands on commas outside (), [], and quotes.""" + ops: List[str] = [] + depth = 0 + cur = "" + quote = False + for ch in text: + if ch == "'" and depth == 0: + quote = not quote + if not quote: + if ch in "([": + depth += 1 + elif ch in ")]": + depth -= 1 + elif ch == "," and depth == 0: + ops.append(cur.strip()) + cur = "" + continue + cur += ch + if cur.strip() or ops: + ops.append(cur.strip()) + return ops + + +def _reg_or_none(text: str) -> Optional[Reg]: + return _REGS.get(text.strip().lower()) + + +def parse_att_operand(text: str, is_branch: bool) -> Tuple[Operand, bool]: + """Parse one AT&T operand; returns (operand, indirect '*').""" + t = text.strip() + indirect = t.startswith("*") + if indirect: + t = t[1:].strip() + if t.startswith("%"): + reg = _reg_or_none(t[1:]) + if reg is None or reg.rip: + raise X86EncodeError(f"unknown register '{t}'") + return reg, indirect + if t.startswith("$"): + return Imm(t[1:].strip()), indirect + m = re.fullmatch(r"(.*?)\(\s*(%\w+)?\s*(?:,\s*(%\w+)\s*(?:,\s*([^)]*))?)?\)", t) + if m: + base = _reg_or_none(m.group(2)[1:]) if m.group(2) else None + index = _reg_or_none(m.group(3)[1:]) if m.group(3) else None + if m.group(2) and base is None: + raise X86EncodeError(f"unknown register '{m.group(2)}'") + if m.group(3) and index is None: + raise X86EncodeError(f"unknown register '{m.group(3)}'") + scale = int(m.group(4).strip(), 0) if m.group(4) and m.group(4).strip() else 1 + return Mem(m.group(1).strip(), base, index, scale), indirect + if is_branch and not indirect: + return Target(t), False + return Mem(t), indirect + + +def parse_intel_operand(text: str, is_branch: bool, absolute: Callable[[str], bool]) -> Operand: + """Parse one Intel-syntax operand.""" + t = text.strip() + size = None + m = re.match(r"^(byte|word|dword|qword)\s+ptr\s+(.*)$", t, re.IGNORECASE) + if m: + size = PTR_SIZES[m.group(1).lower()] + t = m.group(2).strip() + reg = _reg_or_none(t.lstrip("%")) + if reg is not None and not reg.rip: + return reg + if t.lower().startswith("offset "): + return Imm(t[7:].strip()) + if "[" in t: + m = re.fullmatch(r"(.*?)\[(.*)\]\s*(.*)", t) + if not m: + raise X86EncodeError(f"cannot parse memory operand '{text}'") + disp_parts = [p for p in (m.group(1).strip(), m.group(3).strip()) if p] + base = index = None + scale = 1 + terms = re.findall(r"([+-]?)\s*([^+-]+)", m.group(2)) + for sign, term in terms: + term = term.strip() + if "*" in term: + a, b = (x.strip() for x in term.split("*", 1)) + r = _reg_or_none(a) or _reg_or_none(b) + if r is not None: + index = r + scale = int(b if _reg_or_none(a) else a, 0) + continue + r = _reg_or_none(term) + if r is not None and sign != "-": + if base is None: + base = r + elif index is None: + index = r + else: + raise X86EncodeError(f"too many registers in '{text}'") + continue + disp_parts.append(f"{sign}{term}" if sign or not disp_parts else f"+{term}") + disp = "".join(p if p[:1] in "+-" or i == 0 else "+" + p for i, p in enumerate(disp_parts)) + return Mem(disp, base, index, scale, size) + if is_branch: + return Target(t) + # A bare symbol is a memory reference in GNU's Intel syntax; numbers + # and absolute constants are immediates + if absolute(t): + if size is not None: + raise X86EncodeError(f"'{text}' is not a memory operand") + return Imm(t) + return Mem(t, size=size) + + +# --------------------------------------------------------------------------- +# Encoder +# --------------------------------------------------------------------------- + + +@dataclass +class _Parts: + """Pieces of one instruction; assembled at the end so RIP-relative + displacements can be computed from the instruction's full length.""" + + prefix: bytes = b"" + rex: int = 0 # W R X B bits + force_rex: bool = False + forbid_rex: bool = False + opcode: bytes = b"" + modrm: bytes = b"" + disp: bytes = b"" + rip_target: Optional[int] = None + imm: bytes = b"" + + +class Encoder: + def __init__(self, ctx: Context, absolute_symbols: Set[str], long_jump: bool): + self.ctx = ctx + self.pc = ctx.address + self.absolute_symbols = absolute_symbols + self.long_jump = long_jump + + # --- values --- + + def value(self, expr: str) -> int: + return self.ctx.eval(expr) + + def is_symbolic(self, expr: str) -> bool: + """Does the expression refer to a label (relocatable), not only constants?""" + found = False + + def resolve(name: str) -> int: + nonlocal found + if name not in self.absolute_symbols: + found = True + return 0 + + try: + evaluate(expr, resolve) + except ExpressionError: + return True + return found + + def is_absolute(self, text: str) -> bool: + return not self.is_symbolic(text) + + # --- ModRM --- + + def modrm(self, parts: _Parts, reg_field: int, rm: Operand) -> None: + """Fill in ModRM/SIB/displacement for reg_field and an r/m operand.""" + if reg_field >= 8: + parts.rex |= 4 # REX.R + if isinstance(rm, Reg): + if rm.num >= 8: + parts.rex |= 1 # REX.B + parts.modrm = bytes([0xC0 | ((reg_field & 7) << 3) | (rm.num & 7)]) + return + if not isinstance(rm, Mem): + raise X86EncodeError("expected a register or memory operand") + disp_value = self.value(rm.disp) if rm.disp else 0 + symbolic = bool(rm.disp) and self.is_symbolic(rm.disp) + base, index = rm.base, rm.index + r = (reg_field & 7) << 3 + if base is not None and base.rip: + if index is not None: + raise X86EncodeError("rip-relative addressing cannot use an index register") + parts.modrm = bytes([r | 0x05]) + parts.rip_target = disp_value + parts.disp = b"\0\0\0\0" + return + for reg in (base, index): + if reg is not None and reg.size not in (8, 4): + raise X86EncodeError("address registers must be 64-bit (or 32-bit)") + if (base is not None and base.size == 4) or (index is not None and index.size == 4): + parts.prefix += b"\x67" + if index is not None: + if index.num == 4: + raise X86EncodeError("%rsp cannot be an index register") + if rm.scale not in (1, 2, 4, 8): + raise X86EncodeError(f"scale must be 1, 2, 4, or 8, not {rm.scale}") + if index.num >= 8: + parts.rex |= 2 # REX.X + if base is not None and base.num >= 8: + parts.rex |= 1 + scale_bits = {1: 0, 2: 1, 4: 2, 8: 3}.get(rm.scale, 0) + + if base is None: + # disp32 with SIB: no base (index or absolute address) + idx = index.num & 7 if index is not None else 4 + parts.modrm = bytes([r | 0x04, (scale_bits << 6) | (idx << 3) | 5]) + parts.disp = (disp_value & 0xFFFFFFFF).to_bytes(4, "little") + if not symbolic and not _fits32(disp_value) and disp_value >= (1 << 32): + raise X86EncodeError(f"address {disp_value:#x} does not fit in 32 bits") + return + + if symbolic or not _fits8(disp_value): + mod, disp = 0x80, (disp_value & 0xFFFFFFFF).to_bytes(4, "little") + elif disp_value == 0 and (base.num & 7) != 5: + mod, disp = 0x00, b"" + else: + mod, disp = 0x40, (disp_value & 0xFF).to_bytes(1, "little") + if index is not None or (base.num & 7) == 4: + idx = index.num & 7 if index is not None else 4 + parts.modrm = bytes([mod | r | 0x04, (scale_bits << 6) | (idx << 3) | (base.num & 7)]) + else: + parts.modrm = bytes([mod | r | (base.num & 7)]) + parts.disp = disp + + def finish( + self, parts: _Parts, size: Optional[int], regs: List[Reg], default64: bool = False + ) -> bytes: + """Assemble prefixes, REX, opcode, ModRM, displacement, immediate.""" + prefix = parts.prefix + if size == 2: + prefix = b"\x66" + prefix + rex = parts.rex + if size == 8 and not default64: + rex |= 8 + force = parts.force_rex or any(r.needs_rex for r in regs) + if any(r.high8 for r in regs) and (rex or force): + raise X86EncodeError("ah, bh, ch, and dh cannot be used with REX registers") + rex_bytes = bytes([0x40 | rex]) if rex or force else b"" + code = prefix + rex_bytes + parts.opcode + parts.modrm + parts.disp + parts.imm + if parts.rip_target is not None: + offset = len(prefix) + len(rex_bytes) + len(parts.opcode) + len(parts.modrm) + rel = parts.rip_target - (self.pc + len(code)) + if self.ctx.strict and not _fits32(rel): + raise X86EncodeError("rip-relative target out of range") + code = code[:offset] + (rel & 0xFFFFFFFF).to_bytes(4, "little") + code[offset + 4 :] + return code + + def imm_bytes(self, expr: str, size: int) -> bytes: + # GNU as truncates an oversized immediate (with a warning) rather + # than rejecting it, so do the same. + value = self.value(expr) + return (value & ((1 << (8 * size)) - 1)).to_bytes(size, "little") + + # --- operand size --- + + @staticmethod + def operand_size(ops: List[Operand], suffix: Optional[int], mnemonic: str) -> int: + sizes = {op.size for op in ops if isinstance(op, Reg)} + sizes |= {op.size for op in ops if isinstance(op, Mem) and op.size} + if suffix: + if sizes and sizes != {suffix}: + raise X86EncodeError(f"operand size does not match the '{mnemonic}' suffix") + return suffix + if len(sizes) > 1: + raise X86EncodeError("operand sizes do not match") + if not sizes: + raise X86EncodeError( + f"cannot tell the operand size of '{mnemonic}'; add a size suffix " + "(b/w/l/q, e.g. movq) or 'qword ptr'" + ) + return sizes.pop() + + # --- instruction families --- + + def alu(self, op: int, dst: Operand, src: Operand, size: int) -> bytes: + regs = [o for o in (dst, src) if isinstance(o, Reg)] + parts = _Parts() + wide = 0 if size == 1 else 1 + if isinstance(src, Imm): + symbolic = self.is_symbolic(src.expr) + value = self.value(src.expr) + acc = isinstance(dst, Reg) and dst.num == 0 and not dst.high8 + if size == 1: + if acc: + parts.opcode = bytes([op * 8 + 4]) + parts.imm = self.imm_bytes(src.expr, 1) + return self.finish(parts, size, regs) + parts.opcode = b"\x80" + self.modrm(parts, op, dst) + parts.imm = self.imm_bytes(src.expr, 1) + return self.finish(parts, size, regs) + if not symbolic and _fits8(value): + parts.opcode = b"\x83" + self.modrm(parts, op, dst) + parts.imm = (value & 0xFF).to_bytes(1, "little") + return self.finish(parts, size, regs) + imm_size = 2 if size == 2 else 4 + if size == 8 and not symbolic and not _fits32(value): + raise X86EncodeError( + f"immediate {value:#x} does not fit in 32 bits (sign-extended)" + ) + if acc: + parts.opcode = bytes([op * 8 + 5]) + else: + parts.opcode = b"\x81" + self.modrm(parts, op, dst) + parts.imm = ( + self.imm_bytes(src.expr, imm_size) + if size != 8 + else (value & 0xFFFFFFFF).to_bytes(4, "little") + ) + return self.finish(parts, size, regs) + if isinstance(src, Reg): + parts.opcode = bytes([op * 8 + wide]) # r/m, reg + self.modrm(parts, src.num, dst) + return self.finish(parts, size, regs) + if isinstance(dst, Reg) and isinstance(src, Mem): + parts.opcode = bytes([op * 8 + 2 + wide]) # reg, r/m + self.modrm(parts, dst.num, src) + return self.finish(parts, size, regs) + raise X86EncodeError("invalid operands (memory to memory is not allowed)") + + def mov(self, dst: Operand, src: Operand, size: int, movabs: bool = False) -> bytes: + regs = [o for o in (dst, src) if isinstance(o, Reg)] + parts = _Parts() + wide = 0 if size == 1 else 1 + if isinstance(src, Imm): + value = self.value(src.expr) + symbolic = self.is_symbolic(src.expr) + if isinstance(dst, Reg): + if size == 8: + if movabs or (not symbolic and not _fits32(value)): + parts.opcode = bytes([0xB8 + (dst.num & 7)]) + parts.rex |= 1 if dst.num >= 8 else 0 + parts.imm = (value & ((1 << 64) - 1)).to_bytes(8, "little") + return self.finish(parts, size, regs) + parts.opcode = b"\xc7" + self.modrm(parts, 0, dst) + parts.imm = (value & 0xFFFFFFFF).to_bytes(4, "little") + return self.finish(parts, size, regs) + base = 0xB0 if size == 1 else 0xB8 + parts.opcode = bytes([base + (dst.num & 7)]) + parts.rex |= 1 if dst.num >= 8 else 0 + parts.imm = self.imm_bytes(src.expr, size) + return self.finish(parts, size, regs) + if movabs: + raise X86EncodeError("movabs takes a register destination") + parts.opcode = b"\xc6" if size == 1 else b"\xc7" + self.modrm(parts, 0, dst) + if size == 8: + if not symbolic and not _fits32(value): + raise X86EncodeError( + f"immediate {value:#x} does not fit in 32 bits (sign-extended)" + ) + parts.imm = (value & 0xFFFFFFFF).to_bytes(4, "little") + else: + parts.imm = self.imm_bytes(src.expr, size) + return self.finish(parts, size, regs) + if isinstance(src, Reg): + parts.opcode = bytes([0x88 + wide]) + self.modrm(parts, src.num, dst) + return self.finish(parts, size, regs) + if isinstance(dst, Reg) and isinstance(src, Mem): + parts.opcode = bytes([0x8A + wide]) + self.modrm(parts, dst.num, src) + return self.finish(parts, size, regs) + raise X86EncodeError("invalid operands for mov") + + def unary(self, ext: int, op: Operand, size: int, opcode8: int, opcode: int) -> bytes: + parts = _Parts() + parts.opcode = bytes([opcode8 if size == 1 else opcode]) + self.modrm(parts, ext, op) + return self.finish(parts, size, [op] if isinstance(op, Reg) else []) + + def shift(self, ext: int, dst: Operand, count: Optional[Operand], size: int) -> bytes: + regs = [dst] if isinstance(dst, Reg) else [] + parts = _Parts() + b8 = size == 1 + if count is None or ( + isinstance(count, Imm) + and not self.is_symbolic(count.expr) + and self.value(count.expr) == 1 + ): + parts.opcode = b"\xd0" if b8 else b"\xd1" + elif isinstance(count, Reg): + if count.num != 1 or count.size != 1: + raise X86EncodeError("the shift count register must be %cl") + parts.opcode = b"\xd2" if b8 else b"\xd3" + elif isinstance(count, Imm): + parts.opcode = b"\xc0" if b8 else b"\xc1" + parts.imm = self.imm_bytes(count.expr, 1) + else: + raise X86EncodeError("the shift count must be an immediate or %cl") + self.modrm(parts, ext, dst) + return self.finish(parts, size, regs) + + def test(self, a: Operand, b: Operand, size: int) -> bytes: + # test is symmetric; normalize to (r/m, reg-or-imm) + if isinstance(a, Imm): + a, b = b, a + if isinstance(a, Reg) and isinstance(b, Mem): + a, b = b, a + regs = [o for o in (a, b) if isinstance(o, Reg)] + parts = _Parts() + if isinstance(b, Imm): + imm_size = min(size, 4) + if isinstance(a, Reg) and a.num == 0 and not a.high8: + parts.opcode = b"\xa8" if size == 1 else b"\xa9" + else: + parts.opcode = b"\xf6" if size == 1 else b"\xf7" + self.modrm(parts, 0, a) + value = self.value(b.expr) + parts.imm = (value & ((1 << (8 * imm_size)) - 1)).to_bytes(imm_size, "little") + return self.finish(parts, size, regs) + if isinstance(b, Reg): + parts.opcode = b"\x84" if size == 1 else b"\x85" + self.modrm(parts, b.num, a) + return self.finish(parts, size, regs) + raise X86EncodeError("invalid operands for test") + + def jump(self, kind: str, cond: int, target: Target) -> bytes: + """jmp/jcc/call to a label. jmp and jcc use rel8 unless the layout + pass decided this jump needs rel32 (GNU as's relaxation).""" + dest = self.value(target.expr) + if kind == "call": + return b"\xe8" + ((dest - (self.pc + 5)) & 0xFFFFFFFF).to_bytes(4, "little") + if not self.long_jump: + rel8 = dest - (self.pc + 2) + if self.ctx.strict and not _fits8(rel8): + raise X86EncodeError("internal error: short jump out of range; please report this") + op = 0xEB if kind == "jmp" else 0x70 + cond + return bytes([op, rel8 & 0xFF]) + if kind == "jmp": + return b"\xe9" + ((dest - (self.pc + 5)) & 0xFFFFFFFF).to_bytes(4, "little") + return bytes([0x0F, 0x80 + cond]) + ((dest - (self.pc + 6)) & 0xFFFFFFFF).to_bytes( + 4, "little" + ) + + +def _raise(name: str) -> int: + raise UndefinedSymbol(name) + + +# --------------------------------------------------------------------------- +# Instruction dispatch +# --------------------------------------------------------------------------- + +_SPECIAL = { + "movzbw": ("movzx", 1, 2), "movzbl": ("movzx", 1, 4), "movzbq": ("movzx", 1, 8), + "movzwl": ("movzx", 2, 4), "movzwq": ("movzx", 2, 8), + "movsbw": ("movsx", 1, 2), "movsbl": ("movsx", 1, 4), "movsbq": ("movsx", 1, 8), + "movswl": ("movsx", 2, 4), "movswq": ("movsx", 2, 8), "movslq": ("movsxd", 4, 8), +} # fmt: skip + +_NO_OPERANDS = { + "ret": b"\xc3", "leave": b"\xc9", "nop": b"\x90", "hlt": b"\xf4", "syscall": b"\x0f\x05", + "int3": b"\xcc", "cbw": b"\x66\x98", "cbtw": b"\x66\x98", "cwde": b"\x98", "cwtl": b"\x98", + "cdqe": b"\x48\x98", "cltq": b"\x48\x98", "cwd": b"\x66\x99", "cwtd": b"\x66\x99", + "cdq": b"\x99", "cltd": b"\x99", "cqo": b"\x48\x99", "cqto": b"\x48\x99", "ud2": b"\x0f\x0b", +} # fmt: skip + +_BASE_MNEMONICS = set(ALU_OPS) | set(SHIFT_OPS) | set(UNARY_OPS) | { + "mov", "movabs", "lea", "test", "inc", "dec", "imul", "push", "pop", "xchg", "jmp", + "call", "ret", "movzx", "movsx", "movsxd", +} # fmt: skip + + +def _split_mnemonic(m: str) -> Tuple[str, Optional[int]]: + """Split an AT&T mnemonic into (base, size from suffix).""" + if m in _BASE_MNEMONICS or m in _NO_OPERANDS or m in _SPECIAL: + return m, None + if m[:-1] in _BASE_MNEMONICS and m[-1] in SIZE_SUFFIX: + return m[:-1], SIZE_SUFFIX[m[-1]] + for prefix in ("set", "cmov"): + if m.startswith(prefix): + rest = m[len(prefix) :] + if rest in CONDITIONS: + return m, None + if rest[:-1] in CONDITIONS and rest[-1] in SIZE_SUFFIX: + return m[:-1], SIZE_SUFFIX[rest[-1]] + return m, None + + +def _looks_att(ops_text: str) -> bool: + return "%" in ops_text or "$" in ops_text or ops_text.lstrip().startswith("*") + + +def encode_instruction( + text: str, + ctx: Context, + syntax: str, + absolute_symbols: Set[str], + long_jump: bool, +) -> bytes: + parts = text.strip().split(None, 1) + if not parts: + raise X86EncodeError("empty instruction") + mnemonic = parts[0].lower() + ops_text = parts[1] if len(parts) > 1 else "" + enc = Encoder(ctx, absolute_symbols, long_jump) + + if mnemonic in _NO_OPERANDS and not ops_text: + return _NO_OPERANDS[mnemonic] + + att = syntax == "att" or (syntax == "auto" and (_looks_att(ops_text) or not ops_text)) + if syntax == "auto" and not att and _split_mnemonic(mnemonic)[1] is not None: + # A size suffix ("imulq d0") only exists in AT&T syntax. + att = True + base, suffix = _split_mnemonic(mnemonic) if att else (mnemonic, None) + is_branch = base in ("jmp", "call") or (base.startswith("j") and base[1:] in CONDITIONS) + raw = _split(ops_text) if ops_text else [] + operands: List[Operand] = [] + indirect = False + if att: + for r in raw: + op, ind = parse_att_operand(r, is_branch) + operands.append(op) + indirect = indirect or ind + operands.reverse() # AT&T is source, destination + else: + operands = [parse_intel_operand(r, is_branch, enc.is_absolute) for r in raw] + if is_branch and operands and not isinstance(operands[0], Target): + indirect = True + + try: + return _dispatch(enc, base, suffix, operands, indirect, mnemonic) + except ExpressionError as e: + raise X86EncodeError(str(e)) + + +def _dispatch( # noqa: C901 + enc: Encoder, + m: str, + suffix: Optional[int], + ops: List[Operand], + indirect: bool, + original: str, +) -> bytes: + def expect(n: int) -> None: + if len(ops) != n: + raise X86EncodeError(f"'{original}' takes {n} operand{'s' if n != 1 else ''}") + + regs = [o for o in ops if isinstance(o, Reg)] + + if m in _NO_OPERANDS: + if m == "ret" and len(ops) == 1 and isinstance(ops[0], Imm): + return b"\xc2" + enc.imm_bytes(ops[0].expr, 2) + expect(0) + return _NO_OPERANDS[m] + + if m == "int": + expect(1) + if not isinstance(ops[0], Imm): + raise X86EncodeError("int takes an immediate") + return b"\xcd" + enc.imm_bytes(ops[0].expr, 1) + + # Jumps and calls + if m in ("jmp", "call") or (m.startswith("j") and m[1:] in CONDITIONS): + expect(1) + target = ops[0] + if indirect or isinstance(target, (Reg, Mem)): + if m not in ("jmp", "call"): + raise X86EncodeError("conditional jumps cannot be indirect") + if isinstance(target, Reg) and target.size != 8: + raise X86EncodeError(f"{m} through a register needs a 64-bit register") + parts = _Parts(opcode=b"\xff") + enc.modrm(parts, 4 if m == "jmp" else 2, target) + return enc.finish(parts, 8, regs, default64=True) + if not isinstance(target, Target): + raise X86EncodeError(f"{m} needs a label") + kind = m if m in ("jmp", "call") else "jcc" + return enc.jump(kind, CONDITIONS.get(m[1:], 0), target) + + if m.startswith("set") and m[3:] in CONDITIONS: + expect(1) + size = ( + Encoder.operand_size(ops, suffix or 1, original) if not isinstance(ops[0], Mem) else 1 + ) + if size != 1: + raise X86EncodeError("set takes an 8-bit operand") + parts = _Parts(opcode=bytes([0x0F, 0x90 + CONDITIONS[m[3:]]])) + enc.modrm(parts, 0, ops[0]) + return enc.finish(parts, 1, regs) + + if m.startswith("cmov") and m[4:] in CONDITIONS: + expect(2) + dst, src = ops + if not isinstance(dst, Reg): + raise X86EncodeError("cmov needs a register destination") + size = Encoder.operand_size(ops, suffix, original) + parts = _Parts(opcode=bytes([0x0F, 0x40 + CONDITIONS[m[4:]]])) + enc.modrm(parts, dst.num, src) + return enc.finish(parts, size, regs) + + if m in ALU_OPS: + expect(2) + dst, src = ops + size = Encoder.operand_size(ops, suffix, original) + return enc.alu(ALU_OPS[m], dst, src, size) + + if m in ("mov", "movabs"): + expect(2) + dst, src = ops + size = Encoder.operand_size(ops, suffix, original) + return enc.mov(dst, src, size, movabs=m == "movabs") + + if m == "lea": + expect(2) + dst, src = ops + if not isinstance(dst, Reg) or not isinstance(src, Mem): + raise X86EncodeError("lea takes a memory operand and a register") + parts = _Parts(opcode=b"\x8d") + enc.modrm(parts, dst.num, src) + return enc.finish(parts, suffix or dst.size, regs) + + if m == "test": + expect(2) + return enc.test(ops[0], ops[1], Encoder.operand_size(ops, suffix, original)) + + if m in ("inc", "dec"): + expect(1) + size = Encoder.operand_size(ops, suffix, original) + return enc.unary(0 if m == "inc" else 1, ops[0], size, 0xFE, 0xFF) + + if m in UNARY_OPS: + expect(1) + size = Encoder.operand_size(ops, suffix, original) + return enc.unary(UNARY_OPS[m], ops[0], size, 0xF6, 0xF7) + + if m == "imul": + if len(ops) == 1: + size = Encoder.operand_size(ops, suffix, original) + return enc.unary(5, ops[0], size, 0xF6, 0xF7) + if len(ops) == 2 and isinstance(ops[1], Imm): + ops = [ops[0], ops[0], ops[1]] # imul $imm, %reg == imul $imm, %reg, %reg + if len(ops) == 2: + dst, src = ops + if not isinstance(dst, Reg): + raise X86EncodeError("imul needs a register destination") + size = Encoder.operand_size(ops, suffix, original) + parts = _Parts(opcode=b"\x0f\xaf") + enc.modrm(parts, dst.num, src) + return enc.finish(parts, size, regs) + if len(ops) == 3: + dst, src, imm = ops + if not isinstance(dst, Reg) or not isinstance(imm, Imm): + raise X86EncodeError("expected imul $imm, source, register") + size = Encoder.operand_size([dst, src], suffix, original) + value = enc.value(imm.expr) + parts = _Parts() + if not enc.is_symbolic(imm.expr) and _fits8(value): + parts.opcode = b"\x6b" + parts.imm = (value & 0xFF).to_bytes(1, "little") + else: + parts.opcode = b"\x69" + parts.imm = (value & ((1 << (16 if size == 2 else 32)) - 1)).to_bytes( + 2 if size == 2 else 4, "little" + ) + enc.modrm(parts, dst.num, src) + return enc.finish(parts, size, [r for r in (dst, src) if isinstance(r, Reg)]) + raise X86EncodeError("imul takes 1, 2, or 3 operands") + + if m in SHIFT_OPS: + if len(ops) == 1: + dst, count = ops[0], None + else: + expect(2) + dst, count = ops[0], ops[1] + size = Encoder.operand_size([dst], suffix, original) + return enc.shift(SHIFT_OPS[m], dst, count, size) + + if m in ("push", "pop"): + expect(1) + op = ops[0] + if isinstance(op, Reg): + if op.size not in (8, 2): + raise X86EncodeError(f"{m} takes a 64-bit register") + parts = _Parts(opcode=bytes([(0x50 if m == "push" else 0x58) + (op.num & 7)])) + parts.rex |= 1 if op.num >= 8 else 0 + return enc.finish(parts, op.size, [op], default64=True) + if isinstance(op, Imm): + if m == "pop": + raise X86EncodeError("pop needs a register or memory destination") + value = enc.value(op.expr) + if not enc.is_symbolic(op.expr) and _fits8(value): + return b"\x6a" + (value & 0xFF).to_bytes(1, "little") + return b"\x68" + (value & 0xFFFFFFFF).to_bytes(4, "little") + size = suffix or 8 + parts = _Parts(opcode=b"\xff" if m == "push" else b"\x8f") + enc.modrm(parts, 6 if m == "push" else 0, op) + return enc.finish(parts, size, [], default64=True) + + if m == "xchg": + expect(2) + a, b = ops + size = Encoder.operand_size(ops, suffix, original) + both = isinstance(a, Reg) and isinstance(b, Reg) + if both and size == 8 and a.num == 0 and b.num == 0: + return b"\x90" # GNU drops REX.W: it is just nop + # 90 would be nop, which (unlike xchg) leaves the top of rax alone. + same32 = both and size == 4 and a.num == 0 and b.num == 0 + if both and size != 1 and not same32 and (a.num == 0 or b.num == 0): + other = b if a.num == 0 else a + parts = _Parts(opcode=bytes([0x90 + (other.num & 7)])) + parts.rex |= 1 if other.num >= 8 else 0 + return enc.finish(parts, size, regs) + if isinstance(b, Reg): + a, b = b, a + if not isinstance(a, Reg): + raise X86EncodeError("xchg needs a register operand") + parts = _Parts(opcode=b"\x86" if size == 1 else b"\x87") + enc.modrm(parts, a.num, b) + return enc.finish(parts, size, regs) + + if m in ("movzx", "movsx", "movsxd") or m in _SPECIAL: + expect(2) + dst, src = ops + if m in _SPECIAL: + kind, src_size, dst_size = _SPECIAL[m] + else: + kind = m + if not isinstance(dst, Reg): + raise X86EncodeError(f"{m} needs a register destination") + dst_size = dst.size + src_size = ( + src.size + if isinstance(src, (Reg, Mem)) and src.size + else (4 if m == "movsxd" else (suffix or 0)) + ) + if not src_size: + raise X86EncodeError(f"cannot tell the source size of '{m}'; use e.g. 'byte ptr'") + if not isinstance(dst, Reg) or dst.size != dst_size: + raise X86EncodeError(f"{original} needs a {8 * dst_size}-bit register destination") + if isinstance(src, Reg) and src.size != src_size: + raise X86EncodeError(f"{original} needs a {8 * src_size}-bit source") + parts = _Parts() + if kind == "movsxd": + parts.opcode = b"\x63" + else: + base = 0xB6 if kind == "movzx" else 0xBE + parts.opcode = bytes([0x0F, base + (1 if src_size == 2 else 0)]) + enc.modrm(parts, dst.num, src) + return enc.finish(parts, dst_size, regs) + + raise X86EncodeError(f"unknown instruction '{original}'") + + +# --------------------------------------------------------------------------- +# Module interface used by the assembler +# --------------------------------------------------------------------------- + + +def _context(address: int, labels: Dict[str, int], strict: bool) -> Context: + return Context(address, labels, strict=strict, error=X86EncodeError) + + +def is_relaxable(text: str) -> bool: + """True for jmp/jcc to a label, whose size depends on the distance.""" + parts = text.strip().split(None, 1) + if len(parts) != 2: + return False + m = parts[0].lower() + op = parts[1].strip() + if op.startswith(("*", "%", "[")) or _reg_or_none(op) is not None: + return False + return m == "jmp" or (m.startswith("j") and m[1:] in CONDITIONS) + + +def encode( + text: str, + address: int, + labels: Dict[str, int], + syntax: str = "auto", + absolute_symbols: Optional[Set[str]] = None, + long_jump: bool = False, +) -> bytes: + """ + Encode one x86-64 instruction. + + Args: + syntax: "att", "intel", or "auto" (decided per instruction). + absolute_symbols: Names that are constants (.equ), not addresses. + long_jump: Use the rel32 form for jmp/jcc (decided by layout). + + Raises: + X86EncodeError: If the instruction is invalid. + """ + return encode_instruction( + text, _context(address, labels, True), syntax, absolute_symbols or set(), long_jump + ) + + +def instruction_size( + text: str, + labels: Dict[str, int], + address: int = 0, + syntax: str = "auto", + absolute_symbols: Optional[Set[str]] = None, + long_jump: bool = False, +) -> int: + """Size in bytes, for the layout pass (unknown labels count as 0).""" + try: + return len( + encode_instruction( + text, _context(address, labels, False), syntax, absolute_symbols or set(), long_jump + ) + ) + except (X86EncodeError, ExpressionError): + return 1 # the real error is reported by encode() + + +# The nops GNU as pads code with, by length +_NOPS = [bytes.fromhex(h) for h in ( + "90", + "6690", + "0f1f00", + "0f1f4000", + "0f1f440000", + "660f1f440000", + "0f1f8000000000", + "0f1f840000000000", + "660f1f840000000000", + "662e0f1f840000000000", + "66662e0f1f840000000000", +)] # fmt: skip + + +def nop_padding(count: int, after_data: bool) -> bytes: + """Alignment padding of ``count`` bytes, as GNU as generates it. + + After data (which might be part of an instruction) it starts with a + one-byte nop. More than 7 of the longest nops are jumped over. + """ + out = bytearray() + if after_data and count: + out.append(0x90) + count -= 1 + if count // len(_NOPS) > 7: + if count - 2 < 128: + count -= 2 + out += bytes([0xEB, count]) + else: + count -= 5 + out += b"\xe9" + count.to_bytes(4, "little") + longest = _NOPS[-1] + out += longest * (count // len(longest)) + if count % len(longest): + out += _NOPS[count % len(longest) - 1] + return bytes(out) + + +def short_jump_fits( + text: str, address: int, labels: Mapping[str, int], syntax: str = "auto" +) -> bool: + """Whether a relaxable jump at ``address`` reaches its target with rel8.""" + parts = text.strip().split(None, 1) + try: + dest = evaluate(parts[1].strip(), lambda n: labels[n] if n in labels else _raise(n)) + except ExpressionError: + return True # unknown yet: stay short + return _fits8(dest - (address + 2)) diff --git a/pyproject.toml b/pyproject.toml index efdbdb2..06497c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,17 +33,9 @@ dependencies = [ "pyelftools>=0.29", "unicorn>=2.0.0", "capstone>=5.0.0", - # Keystone assembles ARM64/x86-64/MIPS (RISC-V uses a built-in encoder). - # Its PyPI release only ships prebuilt wheels for x86 machines, so it is - # installed automatically only there; elsewhere pip would have to compile - # it from source. Use the "keystone" extra to try that explicitly. - "keystone-engine>=0.9.2; platform_machine == 'x86_64' or platform_machine == 'AMD64' or platform_machine == 'i686' or platform_machine == 'x86'", ] [project.optional-dependencies] -keystone = [ - "keystone-engine>=0.9.2", -] dev = [ "pytest>=7.0", "pytest-cov>=4.0", diff --git a/tests/README.md b/tests/README.md index 142f601..034c2c5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -10,16 +10,13 @@ Comprehensive unit tests for the MapacheSPIM simulator. python -m pytest tests/ ``` -Tests that assemble ARM64 or x86-64 code need the optional Keystone library; where it is not -installed (e.g. Apple Silicon), they are reported as skipped rather than failed. Everything else, -including all RISC-V and MIPS assembler tests, runs on every platform. - -**Assembler differential tests** compare the built-in RISC-V and MIPS assemblers with GNU `as` +**Assembler differential tests** compare the built-in assemblers (all four ISAs) with GNU `as` byte for byte, using randomized programs. They need the GNU cross binutils and are skipped without them: ```bash -sudo apt install binutils-riscv64-linux-gnu binutils-mips-linux-gnu +sudo apt install binutils-riscv64-linux-gnu binutils-mips-linux-gnu \ + binutils-aarch64-linux-gnu binutils-x86-64-linux-gnu python -m pytest tests/test_differential_*.py MAPACHESPIM_FUZZ_SCALE=50 python -m pytest tests/test_differential_*.py # much deeper ``` diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 55b0aa1..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Shared pytest configuration. - -Assembling ARM64, x86-64, and MIPS code needs the optional Keystone Engine, -which has no prebuilt wheels for some platforms (e.g. Apple Silicon). Tests -that fail only because Keystone is missing are reported as skipped there, -while RISC-V tests (which use the built-in encoder) still run everywhere. -""" - -import pytest - -from mapachespim.toolchain.assembler import KEYSTONE_AVAILABLE - -KEYSTONE_MISSING_MARKER = "requires the Keystone Engine" - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - report = outcome.get_result() - if KEYSTONE_AVAILABLE or not report.failed: - return - if KEYSTONE_MISSING_MARKER in str(report.longrepr): - report.outcome = "skipped" - report.longrepr = (str(item.path), item.location[1] or 0, "Skipped: Keystone not installed") diff --git a/tests/gnu_reference.py b/tests/gnu_reference.py index 166edc4..9fcfb78 100644 --- a/tests/gnu_reference.py +++ b/tests/gnu_reference.py @@ -43,6 +43,7 @@ class GnuTarget: ".option norvc\n.option norelax\n.option nopic\n", ), "arm64": GnuTarget("aarch64-linux-gnu-", (), ""), + "x86_64": GnuTarget("x86_64-linux-gnu-", ("--64",), ""), "mips32": GnuTarget( "mips-linux-gnu-", ("-march=mips32", "-EB", "-mno-shared", "-O0", "-32"), @@ -117,11 +118,12 @@ def gnu_accepts(isa: str, source: str) -> bool: def trim_padding(gnu: bytes, ours: bytes) -> bytes: """Drop the padding GNU as adds at the end of a section to reach its - alignment: zeros, or nop instructions in RISC-V code.""" + alignment: zeros, or nop instructions in RISC-V code (a zero byte if + odd, a 2-byte c.nop, then 4-byte nops, as GNU's riscv_make_nops).""" tail = gnu[len(ours) :] - if len(gnu) > len(ours) and ( - not tail.strip(b"\0") or (len(tail) % 4 == 0 and tail == RISCV_NOP * (len(tail) // 4)) - ): + riscv_nops = bytes(len(tail) % 2) + (b"\x01\x00" if len(tail) % 4 >= 2 else b"") + riscv_nops += RISCV_NOP * (len(tail) // 4) + if len(gnu) > len(ours) and (not tail.strip(b"\0") or tail == riscv_nops): return gnu[: len(ours)] return gnu diff --git a/tests/test_assembler_accuracy.py b/tests/test_assembler_accuracy.py index 073f0d7..cea380f 100644 --- a/tests/test_assembler_accuracy.py +++ b/tests/test_assembler_accuracy.py @@ -22,10 +22,6 @@ from mapachespim.toolchain.memory_map import get_layout -# Skip all tests if Keystone not available -# Tests needing Keystone (ARM64/x86-64/MIPS) are skipped by conftest.py when -# it is not installed; RISC-V tests use the built-in encoder and always run. - # ============================================================================= # HELPER FUNCTIONS diff --git a/tests/test_differential_align.py b/tests/test_differential_align.py new file mode 100644 index 0000000..6c52153 --- /dev/null +++ b/tests/test_differential_align.py @@ -0,0 +1,58 @@ +""" +Differential tests for alignment and data in code: random mixes of +instructions, data directives, and .p2align/.balign (with fill and +max-skip arguments) must assemble to the same bytes as GNU as. + +GNU's padding rules differ by ISA: x86-64 pads with multi-byte nops (and a +jump over long runs, and a one-byte nop first after data), ARM64 pads with +zeros up to 4 bytes then nops and aligns an instruction that follows data, +RISC-V pads with nops, and MIPS with zeros. +""" + +import random + +import pytest + +from .gnu_reference import FUZZ_SCALE, gnu_assemble, gnu_available, ours_text, trim_padding + +ISAS = [isa for isa in ("x86_64", "arm64", "riscv64", "mips32") if gnu_available(isa)] +pytestmark = pytest.mark.skipif(not ISAS, reason="GNU binutils not installed") + +INSTRUCTION = {"x86_64": "ret", "arm64": "nop", "riscv64": "nop", "mips32": "nop"} + + +def random_program(isa: str, rng: random.Random) -> str: + # GNU as leaves RISC-V instructions misaligned after odd-sized data, so + # RISC-V programs only use data in whole words + unit = 4 if isa == "riscv64" else 1 + lines = ["_start:"] + for _ in range(rng.randint(1, 8)): + kind = rng.random() + if kind < 0.35: + lines.append(INSTRUCTION[isa]) + elif kind < 0.6: + values = [str(rng.randint(0, 255)) for _ in range(rng.randint(1, 5) * unit)] + lines.append(".byte " + ", ".join(values)) + elif kind < 0.7: + lines.append(f".fill {rng.randint(0, 200 // unit) * unit}, 1, 0xcc") + else: + log2 = rng.randint(0, 8) + directive = rng.choice(["p2align", "balign"]) + value = log2 if directive == "p2align" else 1 << log2 + extra = rng.choice( + ["", ", 0x90", ", 0", f",, {rng.randint(0, 300)}", f", 0x90, {rng.randint(0, 300)}"] + ) + lines.append(f".{directive} {value}{extra}") + lines.append(INSTRUCTION[isa]) + return "\n".join(lines) + "\n" + + +@pytest.mark.parametrize("isa", ISAS) +def test_alignment_and_data_in_code(isa): + rng = random.Random(1) + for _ in range(25 * FUZZ_SCALE): + source = random_program(isa, rng) + ours, errors, _ = ours_text(isa, source) + assert not errors, (source, errors) + gnu = trim_padding(gnu_assemble(isa, source), ours) + assert ours == gnu, f"\n{source}\nours {ours.hex()}\ngnu {gnu.hex()}" diff --git a/tests/test_differential_data.py b/tests/test_differential_data.py index 2c85725..52f6602 100644 --- a/tests/test_differential_data.py +++ b/tests/test_differential_data.py @@ -11,7 +11,7 @@ from .gnu_reference import gnu_assemble, gnu_available, ours_text, trim_padding -ISAS = [isa for isa in ("riscv64", "mips32") if gnu_available(isa)] +ISAS = [isa for isa in ("riscv64", "mips32", "arm64", "x86_64") if gnu_available(isa)] pytestmark = pytest.mark.skipif(not ISAS, reason="GNU binutils not installed") DATA_PROGRAM = r""" @@ -21,13 +21,13 @@ nop .data bytes: .byte 1, 2, 0xff, -1, 'A', 'z', 0b101, 010 -.align 2 -halves: .half 1, -2, 0x1234, 65535 -.align 3 +{align4} +halves: .short 1, -2, 0x1234, 65535 +{align8} words: .word 1, -1, 0x12345678, 0xffffffff, 1+2<<3, (1+2)<<3, ~0, 17 % 5, -7 / 2 -table: .word bytes, halves, words, table+4, end - bytes -.align 3 -quads: .dword 0x123456789abcdef0, -2 +table: .long bytes, halves, words, table+4, end - bytes +{align8} +quads: .quad 0x123456789abcdef0, -2 strings: .ascii "abc" len = . - strings .asciz "with\nescapes\t\\ \"quoted\" \x41\x7f\xff \101\60" @@ -37,19 +37,27 @@ .space 3, 0xaa .p2align 4 .skip 2 -# MIPS aligns this .word (and moves the label) automatically -lens: .word len, SIZE * 2 +fills: .fill 3, 1, 0x1ff + .fill 2, 3, 0x123456 + .fill 1, 8, -1 + .fill 1, 7, 0x11 +# MIPS aligns this .long (and moves the label) automatically +lens: .long len, SIZE * 2 .equ SIZE, 12 .byte 3 -end: .word 0 +end: .long 0 """ @pytest.mark.parametrize("isa", ISAS) def test_data_section_matches_gnu(isa): - ours, errors, _ = ours_text(isa, DATA_PROGRAM, ".data") + # .align n means n bytes on x86 but 2**n bytes elsewhere + x86 = isa == "x86_64" + source = DATA_PROGRAM.replace("{align4}", ".align 4" if x86 else ".align 2") + source = source.replace("{align8}", ".align 8" if x86 else ".align 3") + ours, errors, _ = ours_text(isa, source, ".data") assert not errors, errors - gnu = trim_padding(gnu_assemble(isa, DATA_PROGRAM, ".data"), ours) + gnu = trim_padding(gnu_assemble(isa, source, ".data"), ours) assert ours == gnu, f"\nours {ours.hex()}\ngnu {gnu.hex()}" diff --git a/tests/test_differential_x86.py b/tests/test_differential_x86.py new file mode 100644 index 0000000..61663ad --- /dev/null +++ b/tests/test_differential_x86.py @@ -0,0 +1,348 @@ +""" +Differential tests: the built-in x86-64 assembler against GNU as. + +Random programs per instruction family (fixed seeds) must assemble to +byte-identical .text, in AT&T syntax and in Intel syntax +(.intel_syntax noprefix), including GNU's choice of short/long jumps. + +Skipped when x86_64-linux-gnu binutils are not installed. +Set MAPACHESPIM_FUZZ_SCALE=N to generate N times as many instructions. +""" + +import random + +import pytest + +from mapachespim.toolchain.x86 import CONDITIONS + +from .gnu_reference import ( + FUZZ_SCALE, + describe_mismatch, + gnu_accepts, + gnu_assemble, + gnu_available, + ours_text, + trim_padding, +) + +ISA = "x86_64" +pytestmark = pytest.mark.skipif(not gnu_available(ISA), reason="GNU x86-64 binutils not installed") + +R64 = ["rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi"] + [f"r{i}" for i in range(8, 16)] +R32 = ["eax", "ecx", "edx", "ebx", "esp", "ebp", "esi", "edi"] + [f"r{i}d" for i in range(8, 16)] +R16 = ["ax", "cx", "dx", "bx", "sp", "bp", "si", "di"] + [f"r{i}w" for i in range(8, 16)] +R8 = ["al", "cl", "dl", "bl", "spl", "bpl", "sil", "dil"] + [f"r{i}b" for i in range(8, 16)] +REGS = {8: R64, 4: R32, 2: R16, 1: R8} +SUFFIX = {8: "q", 4: "l", 2: "w", 1: "b"} +PTR = {8: "qword", 4: "dword", 2: "word", 1: "byte"} +DATA_LABELS = ["d0", "d1", "d2"] + + +def imm_for(rng, size, symbolic_ok=True): + bits = 8 * min(size, 4) + choice = rng.random() + if symbolic_ok and size >= 4 and choice < 0.1: + return rng.choice(DATA_LABELS) + lo, hi = -(1 << (bits - 1)), (1 << (bits - 1)) - 1 + return rng.choice([0, 1, -1, 127, -128, 128, -129, lo, hi, rng.randint(lo, hi), rng.randint(-200, 200)]) + + +def att_mem(rng): + """A random AT&T memory operand.""" + kind = rng.random() + base = rng.choice(R64) + index = rng.choice([r for r in R64 if r != "rsp"]) + scale = rng.choice([1, 2, 4, 8]) + disp = rng.choice(["", "0", "8", "-8", "127", "-128", "128", "4096", "-70000", rng.choice(DATA_LABELS)]) + if kind < 0.1: + return f"{rng.choice(DATA_LABELS)}(%rip)" + if kind < 0.15: + return rng.choice(DATA_LABELS) + if kind < 0.2: + return f"{disp or '0'}(,%{index},{scale})" + if kind < 0.6: + return f"{disp}(%{base})" + return f"{disp}(%{base},%{index},{scale})" + + +def intel_mem(rng, size): + kind = rng.random() + base = rng.choice(R64) + index = rng.choice([r for r in R64 if r != "rsp"]) + scale = rng.choice([1, 2, 4, 8]) + disp = rng.choice(["", "+8", "-8", "+128", "-4096", f"+{rng.choice(DATA_LABELS)}"]) + if kind < 0.15: + inner = f"rip+{rng.choice(DATA_LABELS)}" + elif kind < 0.6: + inner = f"{base}{disp}" + else: + inner = f"{base}+{index}*{scale}{disp}" + return f"{PTR[size]} ptr [{inner}]" + + +DATA = """ +.data +d0: .quad 1 +d1: .quad 2 +d2: .quad 3 +""" + + +def program(lines, labels=0, rng=None, intel=False): + body = list(lines) + if labels and rng is not None: + positions = sorted(rng.sample(range(len(body) + 1), labels)) + for i, pos in enumerate(reversed(positions)): + body.insert(pos, f"L{labels - 1 - i}:") + text = "\n".join(f" {line}" if not line.endswith(":") else line for line in body) + head = ".intel_syntax noprefix\n" if intel else "" + return f"{head}.text\n.globl _start\n_start:\n{text}\n{DATA}" + + +def assert_same(source): + ours, errors, debug_lines = ours_text(ISA, source) + assert not errors, f"MapacheSPIM rejected input: {errors[:3]}" + gnu = trim_padding(gnu_assemble(ISA, source), ours) + if ours != gnu: + # Report the first differing instruction by disassembling both + from capstone import CS_ARCH_X86, CS_MODE_64, CS_OPT_SYNTAX_ATT, Cs + + cs = Cs(CS_ARCH_X86, CS_MODE_64) + cs.syntax = CS_OPT_SYNTAX_ATT + a = [(i.address, i.bytes.hex(), f"{i.mnemonic} {i.op_str}") for i in cs.disasm(ours, 0x400000)] + b = [(i.address, i.bytes.hex(), f"{i.mnemonic} {i.op_str}") for i in cs.disasm(gnu, 0x400000)] + for x, y in zip(a, b): + if x != y: + line_no = max((ln for addr, ln in debug_lines if addr <= x[0]), default=0) + src = source.splitlines()[line_no - 1].strip() if line_no else "?" + pytest.fail(f"at {x[0]:#x} ({src!r}): ours {x[1]} {x[2]} | GNU {y[1]} {y[2]}") + pytest.fail(describe_mismatch(ISA, source, ours, gnu, debug_lines)) + + +def n(base): + return base * FUZZ_SCALE + + +def test_alu(): + rng = random.Random(1) + lines = [] + for m in ("add", "or", "adc", "sbb", "and", "sub", "xor", "cmp"): + for _ in range(n(6)): + size = rng.choice([8, 4, 2, 1]) + s = SUFFIX[size] + r = lambda: rng.choice(REGS[size]) # noqa: E731 + lines += [ + f"{m}{s} %{r()}, %{r()}", + f"{m}{s} ${imm_for(rng, size)}, %{r()}", + f"{m}{s} ${imm_for(rng, size)}, %{REGS[size][0]}", + f"{m}{s} {att_mem(rng)}, %{r()}", + f"{m}{s} %{r()}, {att_mem(rng)}", + f"{m}{s} ${imm_for(rng, size)}, {att_mem(rng)}", + f"{m} %{r()}, %{r()}", + ] + assert_same(program(lines)) + + +def test_mov(): + rng = random.Random(2) + lines = [] + for _ in range(n(10)): + size = rng.choice([8, 4, 2, 1]) + s = SUFFIX[size] + r = lambda: rng.choice(REGS[size]) # noqa: E731 + lines += [ + f"mov{s} %{r()}, %{r()}", + f"mov{s} ${imm_for(rng, size)}, %{r()}", + f"mov{s} {att_mem(rng)}, %{r()}", + f"mov{s} %{r()}, {att_mem(rng)}", + f"mov{s} ${imm_for(rng, size)}, {att_mem(rng)}", + ] + big = rng.choice([0x80000000, 0x123456789, -0x80000001, 0xFFFFFFFF, 1 << 63]) + lines += [ + f"movq ${big}, %{rng.choice(R64)}", + f"movabsq ${rng.randrange(1 << 64)}, %{rng.choice(R64)}", + f"movq ${rng.choice(DATA_LABELS)}, %{rng.choice(R64)}", + f"movl ${rng.choice(DATA_LABELS)}, %{rng.choice(R32)}", + f"mov %{rng.choice(['ah', 'bh', 'ch', 'dh'])}, %{rng.choice(['al', 'bl', 'cl', 'dl'])}", + f"leaq {att_mem(rng)}, %{rng.choice(R64)}", + f"leal {att_mem(rng)}, %{rng.choice(R32)}", + ] + assert_same(program(lines)) + + +def test_unary_and_multiply(): + rng = random.Random(3) + lines = [] + for _ in range(n(6)): + size = rng.choice([8, 4, 2, 1]) + s = SUFFIX[size] + r = lambda: rng.choice(REGS[size]) # noqa: E731 + for m in ("inc", "dec", "neg", "not", "mul", "div", "idiv", "imul"): + lines.append(f"{m}{s} %{r()}") + lines.append(f"{m}{s} {att_mem(rng)}") + if size > 1: + lines += [ + f"imul{s} %{r()}, %{r()}", + f"imul{s} {att_mem(rng)}, %{r()}", + f"imul{s} ${imm_for(rng, size, False)}, %{r()}, %{r()}", + f"imul{s} ${imm_for(rng, size, False)}, %{r()}", + f"imul{s} ${imm_for(rng, size, False)}, {att_mem(rng)}, %{r()}", + ] + lines += [ + f"test{s} %{r()}, %{r()}", + f"test{s} ${imm_for(rng, size, False)}, %{r()}", + f"test{s} ${imm_for(rng, size, False)}, %{REGS[size][0]}", + f"test{s} %{r()}, {att_mem(rng)}", + f"test{s} ${imm_for(rng, size, False)}, {att_mem(rng)}", + ] + assert_same(program(lines)) + + +def test_shifts(): + rng = random.Random(4) + lines = [] + for m in ("rol", "ror", "rcl", "rcr", "shl", "sal", "shr", "sar"): + for _ in range(n(4)): + size = rng.choice([8, 4, 2, 1]) + s = SUFFIX[size] + r = rng.choice(REGS[size]) + lines += [ + f"{m}{s} %{r}", + f"{m}{s} $1, %{r}", + f"{m}{s} ${rng.randrange(2, 64)}, %{r}", + f"{m}{s} %cl, %{r}", + f"{m}{s} ${rng.randrange(64)}, {att_mem(rng)}", + f"{m}{s} %cl, {att_mem(rng)}", + ] + assert_same(program(lines)) + + +def test_stack_exchange_extend(): + rng = random.Random(5) + lines = [] + for _ in range(n(6)): + lines += [ + f"push %{rng.choice(R64)}", + f"pop %{rng.choice(R64)}", + f"pushq ${imm_for(rng, 4, False)}", + f"pushq {att_mem(rng)}", + f"popq {att_mem(rng)}", + f"pushw %{rng.choice(R16)}", + f"xchg %{rng.choice(R64)}, %{rng.choice(R64)}", + f"xchg %{rng.choice(R32)}, %{rng.choice(R32)}", + f"xchg %{rng.choice(R8)}, %{rng.choice(R8)}", + f"xchgq %{rng.choice(R64)}, {att_mem(rng)}", + f"movzbl %{rng.choice(R8)}, %{rng.choice(R32)}", + f"movzbq {att_mem(rng)}, %{rng.choice(R64)}", + f"movzwl %{rng.choice(R16)}, %{rng.choice(R32)}", + f"movzwq %{rng.choice(R16)}, %{rng.choice(R64)}", + f"movsbl %{rng.choice(R8)}, %{rng.choice(R32)}", + f"movsbq %{rng.choice(R8)}, %{rng.choice(R64)}", + f"movswl {att_mem(rng)}, %{rng.choice(R32)}", + f"movswq %{rng.choice(R16)}, %{rng.choice(R64)}", + f"movslq %{rng.choice(R32)}, %{rng.choice(R64)}", + f"movslq {att_mem(rng)}, %{rng.choice(R64)}", + ] + assert_same(program(lines)) + + +def test_conditions(): + rng = random.Random(6) + lines = [] + for cc in CONDITIONS: + lines += [ + f"set{cc} %{rng.choice(R8)}", + f"set{cc} {att_mem(rng)}", + f"cmov{cc} %{rng.choice(R64)}, %{rng.choice(R64)}", + f"cmov{cc}l {att_mem(rng)}, %{rng.choice(R32)}", + ] + assert_same(program(lines)) + + +def test_jumps_and_relaxation(): + rng = random.Random(7) + labels = 12 + lines = [] + conds = list(CONDITIONS) + for _ in range(n(10)): + lines += [ + f"jmp L{rng.randrange(labels)}", + f"j{rng.choice(conds)} L{rng.randrange(labels)}", + f"call L{rng.randrange(labels)}", + f"jmp *%{rng.choice(R64)}", + f"call *%{rng.choice(R64)}", + f"jmp *{att_mem(rng)}", + f"call *{att_mem(rng)}", + ] + # Filler so some jumps need rel32 and others fit in rel8 + lines += [f"movq $0x12345678, %{rng.choice(R64)}"] * rng.choice([0, 1, 5, 20]) + rng.shuffle(lines) + assert_same(program(lines, labels, rng)) + + +def test_misc(): + lines = ["ret", "ret $8", "leave", "nop", "hlt", "syscall", "int3", "int $0x80", + "cbtw", "cwtl", "cltq", "cwtd", "cltd", "cqto", "cqo", "cdq", "cdqe", "cwde", + "retq", "callq *%rax", "jmpq *%rax"] # fmt: skip + assert_same(program(lines)) + + +def test_intel_syntax(): + rng = random.Random(8) + lines = [] + for _ in range(n(10)): + size = rng.choice([8, 4, 2, 1]) + r = lambda: rng.choice(REGS[size]) # noqa: E731 + m = rng.choice(["add", "sub", "and", "or", "xor", "cmp", "mov"]) + imm = imm_for(rng, size, False) + lines += [ + f"{m} {r()}, {r()}", + f"{m} {r()}, {imm}", + f"{m} {r()}, {intel_mem(rng, size)}", + f"{m} {intel_mem(rng, size)}, {r()}", + f"{m} {intel_mem(rng, size)}, {imm}", + f"lea {rng.choice(R64)}, [{rng.choice(R64)}+{rng.choice(['rcx', 'rdx'])}*4+16]", + f"lea {rng.choice(R64)}, [rip+{rng.choice(DATA_LABELS)}]", + f"push {rng.choice(R64)}", + f"pop {rng.choice(R64)}", + f"inc {intel_mem(rng, size)}", + f"imul {rng.choice(R64)}, {rng.choice(R64)}, 10", + f"shl {r()}, {rng.randrange(1, 8)}", + f"movzx {rng.choice(R32)}, byte ptr [{rng.choice(R64)}]", + f"movsxd {rng.choice(R64)}, {rng.choice(R32)}", + f"mov {rng.choice(R64)}, offset {rng.choice(DATA_LABELS)}", + ] + lines += ["jmp L0", "je L0", "call L0", "jmp rax", "call qword ptr [rax]", "L0:", "ret"] + assert_same(program(lines, intel=True)) + + +def test_whole_examples(): + from mapachespim.examples import list_examples + + for example in list_examples(): + if example.isa == "x86_64" and example.source is not None: + assert_same(example.source.read_text().replace(".isa x86_64", "")) + + +INVALID = [ + "movq %rax", + "addq %rax, %ebx", + "movb %ah, %sil", + "movq (%rax), (%rbx)", + "movq $0x123456789, (%rax)", + "addq $0x123456789, %rax", + "movl (%rax,%rsp,2), %eax", + "movl (%rax,%rbx,3), %eax", + "frobnicate %rax", + "jmp", + "shll %eax, %ebx", + "pushl %eax", +] + + +@pytest.mark.parametrize("line", INVALID) +def test_invalid_rejected(line): + source = program([line]) + assert not gnu_accepts(ISA, source), f"test assumption: GNU accepts {line!r}" + _, errors, _ = ours_text(ISA, source) + assert errors, f"MapacheSPIM accepted invalid input {line!r}" diff --git a/tests/test_forward_references.py b/tests/test_forward_references.py index 1d4894b..ebca0a8 100644 --- a/tests/test_forward_references.py +++ b/tests/test_forward_references.py @@ -5,10 +5,6 @@ from mapachespim.toolchain import assemble -# Skip all tests if Keystone not available -# Tests needing Keystone (ARM64/x86-64/MIPS) are skipped by conftest.py when -# it is not installed; RISC-V tests use the built-in encoder and always run. - class TestRISCVForwardReferences: """Tests for forward references in RISC-V assembly.""" diff --git a/tests/test_isa_specification.py b/tests/test_isa_specification.py index c0d30e7..f5cf5b4 100644 --- a/tests/test_isa_specification.py +++ b/tests/test_isa_specification.py @@ -8,10 +8,6 @@ from mapachespim.toolchain.directives import DirectiveParser -# Skip all tests if Keystone not available -# Tests needing Keystone (ARM64/x86-64/MIPS) are skipped by conftest.py when -# it is not installed; RISC-V tests use the built-in encoder and always run. - class TestISADirective: """Tests for .isa directive in source files.""" diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index 25ced9e..01b1dd3 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -1,4 +1,4 @@ -"""Tests for the MapacheSPIM toolchain (Keystone-based assembler).""" +"""Tests for the MapacheSPIM toolchain (built-in assemblers).""" import pytest from mapachespim.toolchain import assemble, AssemblyResult @@ -6,10 +6,6 @@ from mapachespim.toolchain.directives import DirectiveParser, LineType -# Skip all tests if Keystone not available -# Tests needing Keystone (ARM64/x86-64/MIPS) are skipped by conftest.py when -# it is not installed; RISC-V tests use the built-in encoder and always run. - class TestDirectiveParser: """Tests for the directive parser.""" diff --git a/tests/test_x86_encoder.py b/tests/test_x86_encoder.py new file mode 100644 index 0000000..306b0a3 --- /dev/null +++ b/tests/test_x86_encoder.py @@ -0,0 +1,107 @@ +""" +Unit tests for the built-in x86-64 encoder that complement the GNU +differential tests (test_differential_x86.py), which need binutils. +""" + +import unittest + +from mapachespim.toolchain import assemble +from mapachespim.toolchain.x86 import X86EncodeError, encode + +PC = 0x400000 + +# Encodings produced by GNU as (x86_64-linux-gnu-as --64) +GNU_ENCODINGS = [ + ("movq $60, %rax", "48c7c03c000000"), + ("movl $42, %edi", "bf2a000000"), + ("syscall", "0f05"), + ("addq %rbx, %rax", "4801d8"), + ("leaq 8(%rsp,%rcx,4), %rdx", "488d548c08"), + ("movb %sil, (%r12)", "41883424"), + ("pushq %r15", "4157"), + ("popq %rbp", "5d"), + ("imulq $1000, %rcx, %rdx", "4869d1e8030000"), + ("shrl $3, %eax", "c1e803"), + ("cmpb $0x7f, %al", "3c7f"), + ("movabsq $0x123456789abcdef0, %r10", "49baf0debc9a78563412"), + ("movzbl (%rdi), %eax", "0fb607"), + ("movslq %ecx, %rax", "4863c1"), + ("sete %al", "0f94c0"), + ("cmovlq %rbx, %rax", "480f4cc3"), + ("ret", "c3"), + # GNU special cases + ("xchg %rax, %rax", "90"), + ("xchg %eax, %eax", "87c0"), + ("sbbb $-129, %bpl", "4080dd7f"), # truncated to 8 bits, like GNU +] + + +class TestKnownEncodings(unittest.TestCase): + def test_matches_gnu(self): + for text, expected in GNU_ENCODINGS: + with self.subTest(text): + self.assertEqual(encode(text, PC, {}).hex(), expected) + + def test_intel_syntax_matches_att(self): + pairs = [ + ("mov rax, 60", "movq $60, %rax"), + ("add rax, rbx", "addq %rbx, %rax"), + ("lea rdx, [rsp + rcx*4 + 8]", "leaq 8(%rsp,%rcx,4), %rdx"), + ("mov byte ptr [r12], sil", "movb %sil, (%r12)"), + ("imul rdx, rcx, 1000", "imulq $1000, %rcx, %rdx"), + ] + for intel, att in pairs: + with self.subTest(intel): + self.assertEqual(encode(intel, PC, {}), encode(att, PC, {})) + + def test_suffix_with_bare_symbol_operand_is_att(self): + # No '%' or '$', but the size suffix only exists in AT&T syntax; + # a bare symbol is an absolute address, as in GNU as + self.assertEqual(encode("imulq value", PC, {"value": 0x500000}).hex(), "48f72c2500005000") + + +class TestErrors(unittest.TestCase): + def test_ambiguous_size_is_rejected(self): + with self.assertRaisesRegex(X86EncodeError, "operand size"): + encode("inc (%rax)", PC, {}) + + def test_invalid_combinations(self): + for text in ("movq (%rax), (%rbx)", "addq %rax, %ebx", "movb %ah, %sil"): + with self.subTest(text), self.assertRaises(X86EncodeError): + encode(text, PC, {}) + + +def _text(source: str) -> bytes: + result = assemble(source, isa="x86_64") + assert result.success, result.errors + return result.elf_bytes + + +class TestJumpRelaxation(unittest.TestCase): + """Jumps start short (rel8) and only grow to rel32 when out of range.""" + + def _program(self, pad: int) -> str: + return f""" + .globl _start + _start: + jmp target + .fill {pad}, 1, 0x90 + target: + ret + """ + + def test_boundary(self): + from io import BytesIO + + from elftools.elf.elffile import ELFFile + + def text(pad): + elf = ELFFile(BytesIO(_text(self._program(pad)))) + return elf.get_section_by_name(".text").data() + + self.assertEqual(text(127)[:2].hex(), "eb7f") # furthest short jump + self.assertEqual(text(128)[:5].hex(), "e980000000") # needs rel32 + + +if __name__ == "__main__": + unittest.main() From e5ef4eb56b156589c151d8bc67c2a058be1481b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 05:28:06 +0000 Subject: [PATCH 12/18] Keep ARM64 literal-pool test within GNU's pool size at large fuzz scales Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- tests/test_differential_arm64.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_differential_arm64.py b/tests/test_differential_arm64.py index f5fee7e..e090530 100644 --- a/tests/test_differential_arm64.py +++ b/tests/test_differential_arm64.py @@ -338,7 +338,9 @@ def test_branches(): def test_literals_and_addresses(): rng = random.Random(9) lines = [] - for _ in range(n(6)): + for i in range(n(6)): + if i % 100 == 99: + lines.append(".ltorg") # GNU as allows at most 1024 entries per pool value = rng.choice([rng.randrange(1 << 64), rng.randrange(1 << 16), 0x123456789, 5]) lines += [ f"ldr {xreg(rng, zr=False)}, ={hex(value)}", From 9c45ff107c4a817d8044e9adc09780b5611921ca Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 05:45:33 +0000 Subject: [PATCH 13/18] Ignore coverage output Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GPT2SED9nsbBHaDXjUEa41 --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 7aaa86f..c3556c6 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,8 @@ dist/ # Temporary/debug files *.log + +# Coverage data +.coverage +.coverage.* +htmlcov/ From 887ea990fae79fe09265f38ede9f5c144331be66 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 08:20:32 +0000 Subject: [PATCH 14/18] Describe each ISA once in a registry that every layer uses ISA-specific facts were spread across if/elif chains in the simulator, disassembler, ELF loader and writer, assembler, directive parser, console, CLIs, and examples. They now live in one place: - isa.py: ISASpec per ISA (names/aliases, memory layout, register file, syscall ABI and instruction, instruction sizes, ELF machine/flags, GNU as dialect, trap descriptions) and lookup helpers - engines.py: the Unicorn and Capstone constants for each ISA - toolchain/targets.py: a Target per ISA that drives its encoder (options, code padding, ARM64 literal pools, x86 jump relaxation) The simulator is split into focused modules: syscalls.py (table-driven SPIM syscalls with a ProgramIO object), disassembler.py, symbols.py (bisect-based SymbolTable), and debug_info.py (moved out of the console). Syscall instructions are detected before execution instead of with a per-instruction hook. run_until() with StopReason replaces three separate run loops in the simulator and console. The ELF writer builds 32- and 64-bit files from one code path, and the loader exposes section headers so the console no longer opens ELF files itself. Bugs fixed along the way: - ARM64 registers 29 and 30 (fp, lr) read Unicorn's V0/V1 vector registers, since Unicorn does not number x29/x30 after x28 - ARM64 read_int/read_char crashed: x0 was treated as a read-only zero register - Loading a 32-bit ARM or RISC-V ELF ran it as a 64-bit program - The ELF symbol table listed locals after globals (readelf warns) and put every symbol, including data labels, in .text; the example binaries are rebuilt with the fixed writer - DWARF line entries with an out-of-range file index discarded all debug info Tests: a conformance suite that every registered ISA must pass (syscalls, every register mapping, register API, default memory map), registry consistency tests, SyscallHandler tests with a fake machine, SymbolTable, Disassembler, run_until, ELF symbol-table validity, loader checks, and a check that the bundled binaries match the assembler. The guess_game tests now use the real syscall path instead of reimplementing it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D764BuGwyCPv1b2tstwGAS --- examples/arm/array_stats/array_stats | Bin 2224 -> 2224 bytes examples/arm/fibonacci/fibonacci | Bin 1400 -> 1400 bytes examples/arm/guess_game/guess_game | Bin 2200 -> 2200 bytes examples/arm/hello_asm/hello_asm | Bin 1344 -> 1344 bytes examples/arm/matrix_multiply/matrix_mult | Bin 2312 -> 2312 bytes examples/mips/array_stats/array_stats | Bin 1824 -> 1824 bytes examples/mips/fibonacci/fibonacci | Bin 1092 -> 1092 bytes examples/mips/guess_game/guess_game | Bin 1740 -> 1740 bytes examples/mips/hello_asm/hello_asm | Bin 992 -> 992 bytes examples/mips/matrix_multiply/matrix_mult | Bin 1480 -> 1480 bytes examples/riscv/array_stats/array_stats | Bin 2208 -> 2208 bytes examples/riscv/fibonacci/fibonacci | Bin 1904 -> 1904 bytes examples/riscv/guess_game/guess_game | Bin 2152 -> 2152 bytes examples/riscv/hello_asm/hello_asm | Bin 1328 -> 1328 bytes examples/riscv/matrix_multiply/matrix_mult | Bin 2312 -> 2312 bytes examples/x86_64/array_stats/array_stats | Bin 2184 -> 2184 bytes examples/x86_64/fibonacci/fibonacci | Bin 1376 -> 1376 bytes examples/x86_64/guess_game/guess_game | Bin 2144 -> 2144 bytes examples/x86_64/hello_asm/hello_asm | Bin 1344 -> 1344 bytes examples/x86_64/matrix_multiply/matrix_mult | Bin 1712 -> 1712 bytes mapachespim/__init__.py | 21 +- mapachespim/console.py | 453 ++---- mapachespim/debug_info.py | 163 +++ mapachespim/disassembler.py | 47 + mapachespim/elf_loader.py | 252 ++-- mapachespim/engines.py | 99 ++ mapachespim/examples.py | 11 +- mapachespim/isa.py | 359 +++++ mapachespim/memory_map.py | 28 +- mapachespim/symbols.py | 55 + mapachespim/syscalls.py | 227 +++ mapachespim/toolchain/__init__.py | 74 +- mapachespim/toolchain/assembler.py | 303 +--- mapachespim/toolchain/cli.py | 11 +- mapachespim/toolchain/directives.py | 96 +- mapachespim/toolchain/elf_builder.py | 666 +++------ mapachespim/toolchain/memory_map.py | 26 - mapachespim/toolchain/targets.py | 275 ++++ mapachespim/unicorn_backend.py | 1420 +++++-------------- tests/test_assembler_accuracy.py | 2 +- tests/test_debug_info.py | 13 +- tests/test_e2e_all_isas.py | 244 +--- tests/test_elf_output.py | 126 ++ tests/test_isa_conformance.py | 256 ++++ tests/test_isa_registry.py | 116 ++ tests/test_mips_backend.py | 17 +- tests/test_simulator.py | 6 +- tests/test_simulator_parts.py | 116 ++ tests/test_syscalls.py | 140 ++ tests/test_x86_backend.py | 19 +- 50 files changed, 2868 insertions(+), 2773 deletions(-) create mode 100644 mapachespim/debug_info.py create mode 100644 mapachespim/disassembler.py create mode 100644 mapachespim/engines.py create mode 100644 mapachespim/isa.py create mode 100644 mapachespim/symbols.py create mode 100644 mapachespim/syscalls.py delete mode 100644 mapachespim/toolchain/memory_map.py create mode 100644 mapachespim/toolchain/targets.py create mode 100644 tests/test_elf_output.py create mode 100644 tests/test_isa_conformance.py create mode 100644 tests/test_isa_registry.py create mode 100644 tests/test_simulator_parts.py create mode 100644 tests/test_syscalls.py diff --git a/examples/arm/array_stats/array_stats b/examples/arm/array_stats/array_stats index 3ae82f9841ca95c88c5629d1643ee3687807747c..f8e7b2330333766a82dbd4a56af26e13621f5926 100644 GIT binary patch delta 320 zcmdlWxIu7(Gs|QzmT18ut_p8Ffa%R00}r?2eX(a-)9jA%BrBr zN<(G&SwXT!XtLT+S$nXo9h$5yR5qVgT$qW$15MU@@iEP$PB0@ke2UsZs Ig9MZY07}gr761SM delta 334 zcmdlWxIu7(GYeaMaY`^aOd=rXaDbIDFi1dY E09J?`l>h($ diff --git a/examples/arm/fibonacci/fibonacci b/examples/arm/fibonacci/fibonacci index d07d26ba9c962e19b443bcba2d2a2e72a321c85f..4fb8fb00e397ca3cbc239ad7f8f6d55997842bf3 100644 GIT binary patch delta 135 zcmeyt^@D3e9Mj}fCUv&>;*!LolF5e5vh0iuAiy+vJ(IXF69Wr_fB;wkN{CJV$RsYQ z0Om0=6fiKL$m&m4WESVK1w)0PJ-U Az5oCK delta 134 zcmeyt^@D3e920ANaYOfdk2eud diff --git a/examples/arm/guess_game/guess_game b/examples/arm/guess_game/guess_game index 692de3d49a02f42aa5213c092d5b97c51664335c..bb404521aa4566eebb888a16df1d7686b1e1b193 100644 GIT binary patch delta 315 zcmbOsI74tl9LwZX7Hzip;*!LolF5N=lI)BOAiy+vJ&U*`69Wr_fB;wkNdlToykFL66}l&Aiy|zJ&U*`BLfSAfB;wkNJZvs%QD%8rJF6jxFoTtWb#BNDRxE%5MY}8o<&@ei9v-yKmaTNC4|8=(`0#8 zkZcZ`tU6TIA1vF5CTj|nt!EV#Wnf@r&|qLdG23_YL7=_}uqsA|5H#78$sd8T`B2$4 zXtI@)71_ikTcEO6&}9222LfehLuG%U$u6DT2$Wq9m9=3+b=BU<8`;DePfVW3Dn9uv boAu-cOstzZ*xxXUhyXQkfHg5NNI+=-g+V4< delta 331 zcmeAW>JZvs%fc34T#{H+GTDZuXYwQ_Np?mC5MZ4Ao<&@ekwJw)KmaTNC4|8=<79bO zkZcZ`tU6TIA1vF5CTj|nt!EXTyoF7K(Pi>Ppt=x)H(jWlh3j+BnU><}o z4dm+}@s)vmBP6~ykZ*y+HwN;!g+iCBS@P1_qFYi+~!yKnTbOhK=l}o! delta 325 zcmZ3$w}5YhItyETaY)H(jWlh3j+BnU><}o z4dm+}@s)vmBP6~ykZ*y+HwNiIReixAc1BpL{!GVzj%om1OI1eZw0aGyfAgj{k3#^=* PAG4Y;a)JVc1&9RzHTxT= diff --git a/examples/mips/fibonacci/fibonacci b/examples/mips/fibonacci/fibonacci index 30d554ed010ad2e9e3eadab25c8d26923e6bf889..e2b4b2448bf8496593ec061741aedc32f7b6fd5b 100644 GIT binary patch delta 136 zcmX@YafD-o4%1{0CY8yWOfr)nF$u>qGB^Ms1TX?AZXj;~5?>0)UxUQg5P))kG!u|% l1f*F&LQuXf%rqee2FA$`nbanKU}D|O&TPWS2~q~a0swq3NSzbkY)t3xPdeaNCe6kV{icR7#M^g^0Gj_ o16TmUR|oPIAo0zB{53HCDQ+04{V5{r~^~ diff --git a/examples/mips/guess_game/guess_game b/examples/mips/guess_game/guess_game index 5124a80b7975a9c7b278e9e92080823eec91768a..441dd53e46d8a36f869dc419c272dd39a826b462 100644 GIT binary patch delta 322 zcmX@Zdxm!d6U$^y7Hzip;*!LolF5avBK3>{3~;~%WN`rbEKm^;UjWEgL*k18`SwVB z86ZCqiLVUgmm=}Cf&3{*d=nslGlE~wU<(vDh7e$I1M+Vo@%@4PM@al|Apa#2KaRnH zAppu}1kxEm{v0HJ5s<$HiC@OxzzFjoBZCYBLjzDi1W7?BPy-kUZBArOV&oJ8azGjc E0Q|!kZvX%Q delta 322 zcmX@Zdxm!d6AN2>aYIiL6PCoFGF%@&W+Y CRv1G7 diff --git a/examples/mips/hello_asm/hello_asm b/examples/mips/hello_asm/hello_asm index 091ec27008486a1415daa7576d5905d44224233c..361d2bfe89a0868e5e68a22ec7103ca7e4d4dd87 100644 GIT binary patch delta 72 zcmaFB{(yahIg=q10|NuM00V;%kOlz|UlhpK1oI$#Ss-5<#^0R5l+Vb?3gof?u>b%8 C!Ub~x delta 72 zcmaFB{(yahIg=qH0|NuM00V;%kOlz|UlhpK1oI$#Ss-5<#^0R5l+VZsQpN(r0ssL1 B1#18R diff --git a/examples/mips/matrix_multiply/matrix_mult b/examples/mips/matrix_multiply/matrix_mult index d247e041965e76a0838771981eff7d0fc8f7f24c..8f4b0e5203d0cb62f928fe034bc49ca437560f22 100644 GIT binary patch delta 214 zcmX@XeS&*~3G?JIX0OS*%u17)n1t&Y85|fi7$ATVNC^V@F-UwFAin^KuL0zDAn}cX z{5wc|TLuS^F<<~VdUfh3bFvP003|35zYVr delta 213 zcmX@XeS&*~3G?JuX6ebg%z=}cn1t&Y1sEUzNHYRiTtHd{Bm(6N18EN=z6^r{h{wPn z1W~UH#nAkUWvzRb)f}~i0SO5Ug(h-~h diff --git a/examples/riscv/array_stats/array_stats b/examples/riscv/array_stats/array_stats index 3ec85e6e813a8524c31a9ae38820f0545372cdba..1841e7d80afbef8a0e2656e838f10e7b5068b0d2 100644 GIT binary patch delta 320 zcmZ1=xIl1&G0S8tmTVct$qRwH!og}785|fG8o*2_ku>=t zP&OCLV`S(-ldYJ{$R;k?43)irCfhsNkWE~1I#l)vn(QK=xDW#)1H2Put_p9Ffa%-00}r?2eTL_pJx#V%BrBr zN<(G;gJq4-WVNBP`m7+mc4)G;P}z7^ac4#b4>VbCpsWxB$UO`VAOZ%0zzjwP0W|Tr z$qRwDWI|$P~3z8RXhkP4m7WUVX`g@ hw`2@dqybH3_T)eo@y#1p)-Z{HoWlWD#=sx}r2&KX7Uci{ diff --git a/examples/riscv/guess_game/guess_game b/examples/riscv/guess_game/guess_game index 11d2d1b1a8b6b0cd794a6d24f6beee1c31c11908..34293a57c5a763537fede9de1ac75d6a50c4bb20 100644 GIT binary patch delta 315 zcmaDM@IqjN9?N7?7Hzip;*!LolF5o}lI)BOAiy-ao<&@eiGhVdpaCoZB?Q1U)8zGF zS$i~D8K~@iuxu%stTt4ZpA}^CW;9t_sH{C$_9mLFKU6lKRa}^f;U$`E+~kcwT^V3C zj0`;t3=Lo=lqj0~5GY#)m3@IG+b~&>O`N9_C@aJO_Im>`7~Cfpu&{2V zS$i~D8K~@iuxu%stTt4ZpA}^CW;9t_sH{C$_9mLFKU6lKRa}^n;U$`E++;yEamiGO z8ioJ{h6XSbO5{)82sE=2D!Twpws!JER&kzosO%4*>|{kY_sIn;teZ2~-Y|-Q!h-`W J%fKK3r2%`{BzFJ+ diff --git a/examples/riscv/hello_asm/hello_asm b/examples/riscv/hello_asm/hello_asm index 2ab5d6ec5afce223987fba97b9f1f368f10bdd9f..8158e810581fec509031ca1a799618077c09c189 100644 GIT binary patch delta 71 zcmdnMwSj9x9TOwdJZvs&NA7LC0eW`KO?`mgdwdcKNrl1FD^+eDw$l!q`=O|00K;t&$Eb2GBKzy z2sD5NpoB1(W}5sTD9a3FOEf?wz?2r4WS*?gDk{prz{p_2z%_D8Bp1Z$p=}*Cv&g~NY+9{9-yi3m~6-JZvs&N6uti}+-7mgvbvO!DlE3?RTb`8C>>U_ddge)2+Aamg;I>;W{{DU&aEV*mgE diff --git a/examples/x86_64/array_stats/array_stats b/examples/x86_64/array_stats/array_stats index b03c60c05b9c7571d039200bd04011422a5d96e7..9c4534985d9d45151aca4f82e46f8f7bec63d8ef 100644 GIT binary patch delta 310 zcmeAW>=4`_%`#buC7dn3xFoTtWU?ch0y`rE2rx~a&mu0##Gt?s0OrF8K`?`9@_DeV z4w|eoRQ5ku)&fn|7%HpJ3NqOZP1YGI8_y~#%D}+LAk5$Z7J(7rlNSQ@C4sq&4C!dH zxsxvfWh&ssI20 delta 300 zcmeAW>=4`_&B7L6T#{H+GFgfxVzLvPJUb%;2ry2b&mu0#$e_Rw0OrF8K`?`H@_DeV z4w|eoRQ5ku)&fn|7%HpJ3NqOZP1YGI8_y~_*@R7mF=%ojP+c5El);9<0nCCCnUfa+ zWlNy4wP>>SlP>~gyP&dr&}64fW@HoRo(GjZ%iu8CksH{I&_7R$FI8?TtRaBILfsw(N!2v7+ zBQho*1nMgSa~T5kyU)N0-Fw#h!9W-Bmfu~B*4lU7yyI~ BAdUb4 delta 318 zcmaDL@IYXL8Vg%|aYsH{I&_7R$FI8?TtRa8`n0pw~2un3Gu xojeh!FCWZhWN1Z`t(<%iC|e7ay^SW@KKUc7_+|w*9VQWw!H@u8V32^C4FEvQAO!#b diff --git a/examples/x86_64/hello_asm/hello_asm b/examples/x86_64/hello_asm/hello_asm index 21a8fc210d7e93a475980645d6c9c8e0894f281b..75c782e7387ce25794334544963bb5a04463ec21 100644 GIT binary patch delta 71 zcmX@Wb%1L_9}^?fN&^6k CH3%^P delta 71 zcmX@Wb%1L_9}^?vN&^6j CF$gID diff --git a/examples/x86_64/matrix_multiply/matrix_mult b/examples/x86_64/matrix_multiply/matrix_mult index f6dd73483b097faad2013c99e735e9249946ee66..87a6755c252d37d769270fbe5bbd96d2cdc1a960 100644 GIT binary patch delta 231 zcmdnMyMcGZ1E$GL%wCgkGAT`7#3(a4ft5{|kwKZkfdLr^PJYNNE-3>RVq|bclhv3k z$Ra-Z0W*hSJ(`T|WJeb9L=ULGnP{>>Kyd*ECLkFARskcTq2elN;weyZ4>a-I$s3u) UHy5zzFo|#gm4Gk^NI)nC03!qze*gdg delta 251 zcmdnMyMcGZ1E$G_%+ix@G6hav#3(a)0}D&N00Scf14951BLEkW2@+R96BmYxd!UKS z0L6vC<~RUZ2%rpRGBPMLI3NUpEMus+1%t!nZf0IdJ1E}~O`X@|jm+Ybp-|a+G}-vc c51GX|v!JpwHz%@KGl_sa!vPjzV32^)09;QMc>n+a diff --git a/mapachespim/__init__.py b/mapachespim/__init__.py index 6ba2303..1a606ca 100644 --- a/mapachespim/__init__.py +++ b/mapachespim/__init__.py @@ -2,24 +2,33 @@ MapacheSPIM - Educational Multi-ISA Simulator using Unicorn Engine A Python-based interactive simulator inspired by SPIM, supporting multiple ISAs -(RISC-V, ARM, x86-64) using the Unicorn CPU emulator framework. +(RISC-V, MIPS, ARM64, x86-64) using the Unicorn CPU emulator framework. """ -from .unicorn_backend import ISA, StepResult, UnicornSimulator, create_simulator, detect_elf_isa +from .isa import ISA, ISA_SPECS, ISASpec, get_spec +from .unicorn_backend import ( + RunResult, + StepResult, + StopReason, + UnicornSimulator, + create_simulator, + detect_elf_isa, +) # Primary public API - use this name in new code Simulator = UnicornSimulator -# Deprecated aliases for backward compatibility -SailSimulator = UnicornSimulator # Deprecated: use Simulator instead - __version__ = "0.2.0" __all__ = [ "Simulator", "UnicornSimulator", - "SailSimulator", "StepResult", + "StopReason", + "RunResult", "ISA", + "ISASpec", + "ISA_SPECS", + "get_spec", "create_simulator", "detect_elf_isa", ] diff --git a/mapachespim/console.py b/mapachespim/console.py index f8c739e..03e0b9a 100644 --- a/mapachespim/console.py +++ b/mapachespim/console.py @@ -14,188 +14,11 @@ import tempfile from pathlib import Path from types import FrameType -from typing import Any, Dict, Generator, List, Optional, Set, Tuple +from typing import Any, Dict, Generator, List, Optional, Set -from . import Simulator - -try: - from elftools.elf.elffile import ELFFile - - ELFTOOLS_AVAILABLE = True -except ImportError: - ELFTOOLS_AVAILABLE = False - - -class SourceInfo: - """Cached source code information from DWARF debug info""" - - addr_to_line: Dict[int, Tuple[str, int]] - source_cache: Dict[str, List[str]] - has_debug_info: bool - _sorted_addrs: List[int] # Cached sorted address list for binary search - - def __init__(self) -> None: - self.addr_to_line = {} # address -> (filename, line_number) - self.source_cache = {} # filename -> list of source lines - self.has_debug_info = False - self._sorted_addrs = [] - - def _build_sorted_addrs(self) -> None: - """Build sorted address list for efficient lookup""" - if not self._sorted_addrs and self.addr_to_line: - self._sorted_addrs = sorted(self.addr_to_line.keys()) - - def get_location(self, addr: int) -> Optional[Tuple[str, int]]: - """Get source location for an address. - - Returns (filename, line_num) or None. - - For addresses within multi-instruction pseudo-ops (like 'la' which expands - to auipc+addi), finds the nearest address <= the query address. This is - standard debugger behavior - showing the source line that started the - current instruction sequence. - """ - # Try exact match first (fast path) - if addr in self.addr_to_line: - return self.addr_to_line[addr] - - # Build sorted address list if needed - self._build_sorted_addrs() - - if not self._sorted_addrs: - return None - - # Binary search for largest address <= addr - import bisect - - idx = bisect.bisect_right(self._sorted_addrs, addr) - 1 - - if idx >= 0: - nearest_addr = self._sorted_addrs[idx] - return self.addr_to_line[nearest_addr] - - return None - - def get_source_lines( - self, filename: str, start_line: int, count: int = 10 - ) -> Optional[List[Tuple[int, str]]]: - """Get source lines from cached file. Returns list of (line_num, text)""" - if filename not in self.source_cache: - return None - - lines = self.source_cache[filename] - result: List[Tuple[int, str]] = [] - - # Adjust to 0-indexed - start_idx = max(0, start_line - 1) - end_idx = min(len(lines), start_idx + count) - - for i in range(start_idx, end_idx): - result.append((i + 1, lines[i])) - - return result - - -def _parse_dwarf_line_info(elf_path: str, source_dirs: Optional[List[Path]] = None) -> SourceInfo: - """Parse DWARF debug info and return SourceInfo object - - Source files are looked up relative to the current directory, the ELF's - directory, and any extra ``source_dirs``. - """ - source_info = SourceInfo() - - if not ELFTOOLS_AVAILABLE: - return source_info - - try: - with open(elf_path, "rb") as f: - elf = ELFFile(f) - - if not elf.has_dwarf_info(): - return source_info - - dwarf_info = elf.get_dwarf_info() - source_info.has_debug_info = True - - # Parse line programs from all compilation units - for CU in dwarf_info.iter_CUs(): - line_program = dwarf_info.line_program_for_CU(CU) - if not line_program: - continue - - # Get file entry table - file_entries = line_program["file_entry"] - - # Version-specific delta for file indexing - if line_program["version"] < 5: - delta = 1 - else: - delta = 0 - - # Iterate through line program entries - prev_state = None - for entry in line_program.get_entries(): - if entry.state is None: - continue - - state = entry.state - if not state.end_sequence: - # Map this address to source location - if state.file > 0 and state.file <= len(file_entries) + delta: - file_entry = file_entries[state.file - delta] - filename = ( - file_entry.name.decode("utf-8") - if isinstance(file_entry.name, bytes) - else file_entry.name - ) - - # Store mapping - source_info.addr_to_line[state.address] = (filename, state.line) - - # Cache source file content if not already cached - if filename not in source_info.source_cache: - _load_source_file(source_info, filename, elf_path, source_dirs) - - prev_state = state - - return source_info - - except Exception: - # If DWARF parsing fails, just return empty source info - return source_info - - -def _load_source_file( - source_info: SourceInfo, - filename: str, - elf_path: str, - source_dirs: Optional[List[Path]] = None, -) -> None: - """Try to load source file contents into cache""" - # Extra directories (e.g. where an assembled .s file lives) come first - search_paths = [] - for directory in source_dirs or []: - search_paths += [directory / filename, directory / Path(filename).name] - - # Then relative to the current directory and the ELF location - elf_dir = Path(elf_path).parent - search_paths += [ - Path(filename), # Absolute or relative to CWD - elf_dir / filename, # Relative to ELF - elf_dir / Path(filename).name, # Just filename in ELF dir - ] - - for path in search_paths: - try: - if path.exists() and path.is_file(): - with open(path) as f: - source_info.source_cache[filename] = f.read().splitlines() - return - except Exception: - continue - - # If we couldn't find the file, store empty list - source_info.source_cache[filename] = [] +from . import Simulator, StopReason +from .debug_info import SourceInfo, parse_line_info +from .isa import find_spec, isa_names def _chunk_list(lst: List[Any], n: int) -> Generator[List[Any], None, None]: @@ -409,7 +232,7 @@ def _load_elf_file( # Parse DWARF debug information source_dirs = [source.parent] if source is not None else None - self.source_info = _parse_dwarf_line_info(elf_path, source_dirs) + self.source_info = parse_line_info(elf_path, source_dirs) if self.source_info.has_debug_info: num_files = len(self.source_info.source_cache) if num_files > 0: @@ -432,8 +255,8 @@ def _assemble_and_load( """Assemble a source file (with debug info) and load the result""" from .toolchain import assemble_file - valid_isas = ("riscv64", "arm64", "x86_64", "mips32") - if isa is not None and isa.lower() not in valid_isas: + valid_isas = isa_names() + if isa is not None and find_spec(isa) is None: self.print_error(f'Error: Unknown ISA "{isa}". Use one of: {", ".join(valid_isas)}') return False @@ -546,10 +369,13 @@ def do_examples(self, arg: str) -> None: wanted = arg.strip().lower() or None if wanted is not None and wanted not in ISA_DIRS: - self.print_error( - f'Error: Unknown ISA "{arg.strip()}". Use one of: {", ".join(ISA_DIRS)}' - ) - return + spec = find_spec(wanted) + if spec is None: + self.print_error( + f'Error: Unknown ISA "{arg.strip()}". Use one of: {", ".join(ISA_DIRS)}' + ) + return + wanted = spec.examples_dir examples = [e for e in list_examples() if wanted is None or e.isa == wanted] if not examples: @@ -632,7 +458,7 @@ def do_step(self, arg: str) -> None: result = self.sim.step() should_terminate, reason = self.sim.check_termination(result) - if should_terminate: + if should_terminate and reason is not None: self._report_stop(reason, pc) break @@ -718,47 +544,36 @@ def do_run(self, arg: str) -> None: # Run with breakpoint/interrupt checking self._running = True self._interrupted = False - steps_executed = 0 - stopped = False - try: - while max_steps == 0 or steps_executed < max_steps: - # Check for interrupt - if self._interrupted: - self._interrupted = False - self._end_program_output() - print(f"Interrupted after {steps_executed} instructions", file=self.stdout) - stopped = True - break - - # Check for breakpoint (but skip on first iteration to allow continuing from a breakpoint) - pc = self.sim.get_pc() - if steps_executed > 0 and pc in self.breakpoints: - self._end_program_output() - print( - f"Breakpoint hit at {pc:#018x} after {steps_executed} instructions", - file=self.stdout, - ) - stopped = True - break - - result = self.sim.step() - steps_executed += 1 - - should_terminate, reason = self.sim.check_termination(result) - if should_terminate: - self._report_stop(reason, pc, steps_executed) - stopped = True - break + result = self.sim.run_until(max_steps or None, stop_before=self._stop_before) finally: self._running = False self._end_program_output() - if max_steps > 0 and steps_executed >= max_steps and not stopped: - print(f"Executed {steps_executed} instructions (max limit reached)", file=self.stdout) - if steps_executed > 0 and not self.sim.exited: + if result.reason == StopReason.INTERRUPTED: + self._interrupted = False + print(f"Interrupted after {result.steps} instructions", file=self.stdout) + elif result.reason == StopReason.BREAKPOINT: + print( + f"Breakpoint hit at {result.pc:#018x} after {result.steps} instructions", + file=self.stdout, + ) + elif result.reason is not None: + self._report_stop(result.reason, result.pc, result.steps) + else: + print(f"Executed {result.steps} instructions (max limit reached)", file=self.stdout) + self._end_program_output() + if result.steps > 0 and not self.sim.exited: print(f"PC = {self.sim.get_pc():#018x}", file=self.stdout) + def _stop_before(self, pc: int) -> Optional[StopReason]: + """Stop a run at a breakpoint or when the user pressed Ctrl-C""" + if self._interrupted: + return StopReason.INTERRUPTED + if pc in self.breakpoints: + return StopReason.BREAKPOINT + return None + def do_continue(self, arg: str) -> None: """Continue execution after hitting a breakpoint @@ -833,13 +648,8 @@ def _format_instruction(self, pc: int) -> str: except Exception: text, instr_hex = "", "????????" - sym, offset = self.sim.addr_to_symbol(pc) - if sym and offset == 0: - suffix = f" <{sym}>" - elif sym: - suffix = f" <{sym}+{offset}>" - else: - suffix = "" + symbol = self.sim.symbols.describe(pc) + suffix = f" {symbol}" if symbol else "" return f"[{pc:#010x}] 0x{instr_hex} {text}{suffix}" def _end_program_output(self) -> None: @@ -857,17 +667,17 @@ def _report_if_exited(self) -> bool: return True return False - def _report_stop(self, reason: Optional[str], pc: int, steps: Optional[int] = None) -> None: + def _report_stop(self, reason: StopReason, pc: int, steps: Optional[int] = None) -> None: """Print why execution stopped (exit, error, ...)""" self._end_program_output() after = f" after {steps} instructions" if steps is not None else "" - if reason == "syscall_exit": + if reason == StopReason.EXIT: print(f"Program exited with code {self.sim.exit_code}{after}", file=self.stdout) - elif reason == "halt": + elif reason == StopReason.HALT: print(f"Program halted{after}", file=self.stdout) - elif reason == "tohost": + elif reason == StopReason.TOHOST: print(f"Program completed (tohost){after}", file=self.stdout) - elif reason == "error": + elif reason == StopReason.ERROR: detail = self.sim.last_error or "Execution error" print(f"Error: {detail}", file=self.stdout) print(f" {self._format_instruction(pc)}", file=self.stdout) @@ -890,7 +700,7 @@ def _format_reg_value(self, value: int, show_mode: str, leading_zeros_mode: str) elif show_mode == "decimal": # Signed decimal, since that is how students think of values # like -1 (max 20 digits for 64-bit) - bits = 32 if self._is_32bit() else 64 + bits = self.sim.spec.word_bits if self.sim.get_isa() is not None else 64 if value & (1 << (bits - 1)): value -= 1 << bits sign = "-" if value < 0 else "" @@ -934,12 +744,6 @@ def _change_marker(self) -> str: return "*" return "★" - def _is_32bit(self) -> bool: - """True if the loaded ISA has 32-bit registers""" - from . import ISA - - return self.sim is not None and self.sim.get_isa() == ISA.MIPS - def do_regs(self, arg: str) -> None: """Display all registers @@ -1017,12 +821,7 @@ def do_regs(self, arg: str) -> None: # Format registers in 2 columns (or 1 if binary is too wide) cols = 1 if show_mode == "binary" else 2 num_regs = self.sim.get_register_count() - isa = self.sim.get_isa() - - # Determine register display format based on ISA - from . import ISA - - use_x_prefix = isa in (ISA.RISCV, ISA.ARM) + prefix = self.sim.spec.registers.number_prefix if self.sim.get_isa() is not None else None reg_lines = [] for i in range(0, num_regs, cols): @@ -1049,13 +848,11 @@ def do_regs(self, arg: str) -> None: ) # Format register name based on ISA - if use_x_prefix: - line_parts.append( - f"x{reg_num:<2} ({abi_name:>4}) = {formatted_value:<{value_width}}{star}" - ) - else: - # x86-64: just show the register name (no x prefix) - line_parts.append(f"{abi_name:>3} = {formatted_value:<{value_width}}{star}") + if prefix: # e.g. "x5 (t0)" + label = f"{prefix}{reg_num:<2} ({abi_name:>4})" + else: # just the name, e.g. "rax" + label = f"{abi_name:>3}" + line_parts.append(f"{label} = {formatted_value:<{value_width}}{star}") reg_lines.append(" ".join(line_parts)) for line in reg_lines: @@ -1150,44 +947,24 @@ def do_mem(self, arg: str) -> None: # Check if it's a section name (starts with .) if addr_or_section.startswith("."): - if not ELFTOOLS_AVAILABLE: - self.print_error( - "Error: pyelftools not available. Install with: pip install pyelftools" - ) - return - if not self.loaded_file: self.print_error("Error: No program loaded.") return - # Look up section - try: - assert self._elf_path is not None # set whenever loaded_file is - with open(self._elf_path, "rb") as f: - elf = ELFFile(f) - section = elf.get_section_by_name(addr_or_section) - if not section: - self.print_error( - f'Error: Section "{addr_or_section}" not found. Use "info sections" to see available sections.' - ) - return - - addr = section["sh_addr"] - section_size = section["sh_size"] - - if addr == 0: - self.print_error( - f'Error: Section "{addr_or_section}" is not loaded in memory (address is 0).' - ) - return - - # Limit length to section size if not specified - if len(parts) == 1: # No length given - length = min(length, section_size) - - except Exception as e: - self.print_error(f"Error reading section: {e}") + section = self.sim.find_section(addr_or_section) + if section is None: + self.print_error( + f'Error: Section "{addr_or_section}" not found. Use "info sections" to see available sections.' + ) return + if section.address == 0: + self.print_error( + f'Error: Section "{addr_or_section}" is not loaded in memory (address is 0).' + ) + return + addr = section.address + if len(parts) == 1: # No length given: at most the whole section + length = min(length, section.size) else: # Parse as address or symbol name parsed = self._parse_address(addr_or_section) @@ -1349,10 +1126,6 @@ def do_list(self, arg: str) -> None: self.print_error('Error: No program loaded. Use "load " first.') return - if not ELFTOOLS_AVAILABLE: - self.print_error("Error: pyelftools not available.") - return - if not self.source_info.has_debug_info: self._print_block_start() print("No source information available.", file=self.stdout) @@ -1531,14 +1304,8 @@ def do_info(self, arg: str) -> None: self._print_block_start() print("Breakpoints:", file=self.stdout) for i, addr in enumerate(sorted(self.breakpoints), 1): - # Try to show symbol name if available - sym, offset = self.sim.addr_to_symbol(addr) - if sym and offset == 0: - print(f" {i}. {addr:#010x} <{sym}>", file=self.stdout) - elif sym: - print(f" {i}. {addr:#010x} <{sym}+{offset}>", file=self.stdout) - else: - print(f" {i}. {addr:#010x}", file=self.stdout) + symbol = self.sim.symbols.describe(addr) + print(f" {i}. {addr:#010x} {symbol}".rstrip(), file=self.stdout) self._print_block_end() elif arg == "symbols" or arg == "sym": if not self.loaded_file: @@ -1564,53 +1331,25 @@ def do_info(self, arg: str) -> None: print("No program loaded.", file=self.stdout) return - if not ELFTOOLS_AVAILABLE: - self.print_error( - "Error: pyelftools not available. Install with: pip install pyelftools" - ) - return - - try: - assert self._elf_path is not None # set whenever loaded_file is - with open(self._elf_path, "rb") as f: - elf = ELFFile(f) - - self._print_block_start() - print("ELF Sections:", file=self.stdout) - print(f"{'Name':<20} {'Address':>18} {'Size':>12} {'Flags'}", file=self.stdout) - print("-" * 70, file=self.stdout) - - for section in elf.iter_sections(): - name = section.name - addr = section["sh_addr"] - size = section["sh_size"] - flags = section["sh_flags"] - - # Decode flags - flag_str = "" - if flags & 0x1: # SHF_WRITE - flag_str += "W" - if flags & 0x2: # SHF_ALLOC - flag_str += "A" - if flags & 0x4: # SHF_EXECINSTR - flag_str += "X" - - # Only show allocated sections (those loaded in memory) - if addr > 0: - print( - f"{name:<20} {addr:#18x} {size:>12} {flag_str}", file=self.stdout - ) - - print(file=self.stdout) - print("Flags: W=Write, A=Alloc, X=Execute", file=self.stdout) + self._print_block_start() + print("ELF Sections:", file=self.stdout) + print(f"{'Name':<20} {'Address':>18} {'Size':>12} {'Flags'}", file=self.stdout) + print("-" * 70, file=self.stdout) + # Only sections loaded in memory are of interest + for section in self.sim.get_sections(): + if section.address > 0: print( - "Use 'mem
' to view section contents (e.g., mem .data)", + f"{section.name:<20} {section.address:#18x} {section.size:>12} " + f"{section.flag_letters}", file=self.stdout, ) - self._print_block_end() - - except Exception as e: - self.print_error(f"Error reading ELF sections: {e}") + print(file=self.stdout) + print("Flags: W=Write, A=Alloc, X=Execute", file=self.stdout) + print( + "Use 'mem
' to view section contents (e.g., mem .data)", + file=self.stdout, + ) + self._print_block_end() else: self.print_error("Usage: info [breakpoints|symbols|sections]") @@ -2122,33 +1861,23 @@ def _execute( return 1 sim = console.sim - steps = 0 - pc = sim.get_pc() - reason: Optional[str] = None - while steps < max_steps: - pc = sim.get_pc() - result = sim.step() - steps += 1 - done, reason = sim.check_termination(result) - if done: - break - else: - reason = None + result = sim.run_until(max_steps) + reason = result.reason # Diagnostics go to stderr so stdout contains only the program's output if sim.output_needs_newline: sys.stdout.flush() print(file=sys.stderr) if verbose: - print(f"Program completed in {steps} steps", file=sys.stderr) + print(f"Program completed in {result.steps} steps", file=sys.stderr) - if reason == "syscall_exit": + if reason == StopReason.EXIT: return sim.exit_code or 0 - if reason in ("halt", "tohost"): + if reason in (StopReason.HALT, StopReason.TOHOST): return 0 - if reason == "error": + if reason == StopReason.ERROR: print(f"Error: {sim.last_error or 'execution error'}", file=sys.stderr) - print(f" {console._format_instruction(pc)}", file=sys.stderr) + print(f" {console._format_instruction(result.pc)}", file=sys.stderr) return 1 print( f"Error: program did not exit within {max_steps} instructions " @@ -2175,7 +1904,7 @@ def main() -> None: ) parser.add_argument( "--isa", - choices=["riscv64", "arm64", "x86_64", "mips32"], + choices=isa_names(), help="ISA of an assembly file without an .isa directive", ) parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") diff --git a/mapachespim/debug_info.py b/mapachespim/debug_info.py new file mode 100644 index 0000000..440b3a9 --- /dev/null +++ b/mapachespim/debug_info.py @@ -0,0 +1,163 @@ +""" +Source-level debug information: which source line each address came from. + +Reads the DWARF line table that ``mapachespim-as -g`` (and GNU tools with +``-g``) put in an ELF file, and caches the source files it refers to. +""" + +from __future__ import annotations + +import bisect +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from elftools.elf.elffile import ELFFile + + +class SourceInfo: + """Cached source code information from DWARF debug info""" + + addr_to_line: Dict[int, Tuple[str, int]] + source_cache: Dict[str, List[str]] + has_debug_info: bool + _sorted_addrs: List[int] # Cached sorted address list for binary search + + def __init__(self) -> None: + self.addr_to_line = {} # address -> (filename, line_number) + self.source_cache = {} # filename -> list of source lines + self.has_debug_info = False + self._sorted_addrs = [] + + def _build_sorted_addrs(self) -> None: + """Build sorted address list for efficient lookup""" + if not self._sorted_addrs and self.addr_to_line: + self._sorted_addrs = sorted(self.addr_to_line.keys()) + + def get_location(self, addr: int) -> Optional[Tuple[str, int]]: + """Get source location for an address. + + Returns (filename, line_num) or None. + + For addresses within multi-instruction pseudo-ops (like 'la' which expands + to auipc+addi), finds the nearest address <= the query address. This is + standard debugger behavior - showing the source line that started the + current instruction sequence. + """ + # Try exact match first (fast path) + if addr in self.addr_to_line: + return self.addr_to_line[addr] + + # Build sorted address list if needed + self._build_sorted_addrs() + + if not self._sorted_addrs: + return None + + # Binary search for largest address <= addr + idx = bisect.bisect_right(self._sorted_addrs, addr) - 1 + + if idx >= 0: + nearest_addr = self._sorted_addrs[idx] + return self.addr_to_line[nearest_addr] + + return None + + def get_source_lines( + self, filename: str, start_line: int, count: int = 10 + ) -> Optional[List[Tuple[int, str]]]: + """Get source lines from cached file. Returns list of (line_num, text)""" + if filename not in self.source_cache: + return None + + lines = self.source_cache[filename] + result: List[Tuple[int, str]] = [] + + # Adjust to 0-indexed + start_idx = max(0, start_line - 1) + end_idx = min(len(lines), start_idx + count) + + for i in range(start_idx, end_idx): + result.append((i + 1, lines[i])) + + return result + + +def parse_line_info(elf_path: str, source_dirs: Optional[List[Path]] = None) -> SourceInfo: + """Parse DWARF debug info and return SourceInfo object + + Source files are looked up relative to the current directory, the ELF's + directory, and any extra ``source_dirs``. + """ + source_info = SourceInfo() + + try: + with open(elf_path, "rb") as f: + elf = ELFFile(f) + + if not elf.has_dwarf_info(): + return source_info + + dwarf_info = elf.get_dwarf_info() + source_info.has_debug_info = True + + # Parse line programs from all compilation units + for CU in dwarf_info.iter_CUs(): + line_program = dwarf_info.line_program_for_CU(CU) + if not line_program: + continue + + file_entries = line_program["file_entry"] + # File numbers start at 1 before DWARF 5 and at 0 from DWARF 5 + first_file = 1 if line_program["version"] < 5 else 0 + + for entry in line_program.get_entries(): + state = entry.state + if state is None or state.end_sequence: + continue + index = state.file - first_file + if not 0 <= index < len(file_entries): + continue + name = file_entries[index].name + filename = name.decode("utf-8") if isinstance(name, bytes) else name + source_info.addr_to_line[state.address] = (filename, state.line) + if filename not in source_info.source_cache: + _load_source_file(source_info, filename, elf_path, source_dirs) + + return source_info + + except Exception: + # If DWARF parsing fails, just return empty source info + return source_info + + +def _load_source_file( + source_info: SourceInfo, + filename: str, + elf_path: str, + source_dirs: Optional[List[Path]] = None, +) -> None: + """Try to load source file contents into cache""" + # Extra directories (e.g. where an assembled .s file lives) come first + search_paths = [] + for directory in source_dirs or []: + search_paths += [directory / filename, directory / Path(filename).name] + + # Then relative to the current directory and the ELF location + elf_dir = Path(elf_path).parent + search_paths += [ + Path(filename), # Absolute or relative to CWD + elf_dir / filename, # Relative to ELF + elf_dir / Path(filename).name, # Just filename in ELF dir + ] + + for path in search_paths: + try: + if path.exists() and path.is_file(): + with open(path) as f: + source_info.source_cache[filename] = f.read().splitlines() + return + except Exception: + continue + + # If we couldn't find the file, store empty list + source_info.source_cache[filename] = [] diff --git a/mapachespim/disassembler.py b/mapachespim/disassembler.py new file mode 100644 index 0000000..f064d1a --- /dev/null +++ b/mapachespim/disassembler.py @@ -0,0 +1,47 @@ +"""Capstone-based disassembly for every registered ISA.""" + +from __future__ import annotations + +from typing import Tuple + +from capstone import Cs + +from .engines import engine_for +from .isa import ISASpec + + +class Disassembler: + """Disassembles one instruction at a time from raw bytes.""" + + def __init__(self, spec: ISASpec) -> None: + engine = engine_for(spec) + self._spec = spec + self._cs = Cs(engine.cs_arch, engine.cs_mode) + self._cs.detail = False # We don't need detailed instruction info + + @property + def max_size(self) -> int: + """Bytes needed to be sure of decoding any one instruction.""" + return self._spec.max_instruction_size + + def disassemble(self, code: bytes, addr: int) -> Tuple[str, int]: + """ + Disassemble the instruction at the start of ``code`` (located at ``addr``). + + Returns: + (text, size). Bytes that do not decode are shown as ``.word`` (for + fixed-width ISAs) or ``.byte`` (for variable-length ones), with the + size of that unit, so callers walking a range always advance. + """ + spec = self._spec + if len(code) < spec.min_instruction_size: + return ("", spec.min_instruction_size) + + for instr in self._cs.disasm(code, addr, count=1): + return (f"{instr.mnemonic} {instr.op_str}".strip(), instr.size) + + if spec.fixed_width: + size = spec.min_instruction_size + word = int.from_bytes(code[:size], spec.byteorder) + return (f".word 0x{word:0{2 * size}x}", size) + return (f".byte 0x{code[0]:02x}", 1) diff --git a/mapachespim/elf_loader.py b/mapachespim/elf_loader.py index 3fc2f93..5a0ec89 100644 --- a/mapachespim/elf_loader.py +++ b/mapachespim/elf_loader.py @@ -1,19 +1,15 @@ """ -Pure Python ELF file loader using pyelftools +ELF executable loading (using pyelftools). -Replaces the C++ ELFIO-based loader with a clean Python implementation. -Supports RISC-V, ARM64, and x86-64 ELF binaries. +Reads what the simulator needs from an ELF file: its ISA, entry point, +loadable segments, sections, and symbols. """ from __future__ import annotations -from dataclasses import dataclass -from enum import IntEnum +from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Dict, List - -if TYPE_CHECKING: - from elftools.elf.elffile import ELFFile as ELFFileType +from typing import Dict, List try: from elftools.elf.elffile import ELFFile @@ -23,28 +19,13 @@ f"pyelftools not installed. Install with: pip install pyelftools\nOriginal error: {e}" ) +from .isa import ISA, ISASpec, isa_names, spec_for_elf -class ISA(IntEnum): - """ISA types - matches the C++ enum""" - - RISCV = 0 - ARM = 1 - X86_64 = 2 - MIPS = 3 - UNKNOWN = -1 - +__all__ = ["ELFInfo", "ELFSection", "ELFSegment", "ISA", "load_elf_file"] -class Architecture(IntEnum): - """Architecture variants""" - - RV32 = 0 - RV64 = 1 - ARM32 = 2 - ARM64 = 3 - X86_64 = 4 - MIPS32 = 5 - MIPS64 = 6 - UNKNOWN = -1 +SHF_WRITE = 0x1 +SHF_ALLOC = 0x2 +SHF_EXECINSTR = 0x4 @dataclass @@ -58,142 +39,84 @@ class ELFSegment: data: bytes # Segment data +@dataclass(frozen=True) +class ELFSection: + """An ELF section header""" + + name: str + address: int + size: int + flags: int + + @property + def loaded(self) -> bool: + """True if the section occupies memory when the program runs.""" + return bool(self.flags & SHF_ALLOC) and self.address != 0 + + @property + def flag_letters(self) -> str: + """Flags as in readelf: W (write), A (alloc), X (execute).""" + return "".join( + letter + for bit, letter in ((SHF_WRITE, "W"), (SHF_ALLOC, "A"), (SHF_EXECINSTR, "X")) + if self.flags & bit + ) + + @dataclass class ELFInfo: """Parsed ELF file information""" - isa: ISA - architecture: Architecture + spec: ISASpec entry: int segments: List[ELFSegment] symbols: Dict[str, int] + sections: List[ELFSection] = field(default_factory=list) + @property + def isa(self) -> ISA: + return self.spec.isa -def detect_isa_from_elf(elf: ELFFileType) -> ISA: - """ - Detect ISA from ELF file header - - Args: - elf: ELFFile object - - Returns: - ISA: Detected ISA type - """ - machine = elf.header["e_machine"] - - if machine == "EM_RISCV": - return ISA.RISCV - elif machine == "EM_AARCH64": - return ISA.ARM - elif machine == "EM_ARM": - return ISA.ARM # 32-bit ARM (we'll treat as ARM for now) - elif machine == "EM_X86_64": - return ISA.X86_64 - elif machine == "EM_MIPS": - return ISA.MIPS - else: - return ISA.UNKNOWN - - -def detect_architecture(elf: ELFFileType, isa: ISA) -> Architecture: - """ - Detect specific architecture variant - - Args: - elf: ELFFile object - isa: ISA type - - Returns: - Architecture: Specific architecture - """ - elf_class = elf.header["e_ident"]["EI_CLASS"] - - if isa == ISA.RISCV: - if elf_class == "ELFCLASS64": - return Architecture.RV64 - elif elf_class == "ELFCLASS32": - return Architecture.RV32 - elif isa == ISA.ARM: - if elf_class == "ELFCLASS64": - return Architecture.ARM64 - elif elf_class == "ELFCLASS32": - return Architecture.ARM32 - elif isa == ISA.X86_64: - # x86-64 is always 64-bit - return Architecture.X86_64 - elif isa == ISA.MIPS: - if elf_class == "ELFCLASS64": - return Architecture.MIPS64 - elif elf_class == "ELFCLASS32": - return Architecture.MIPS32 - - return Architecture.UNKNOWN - - -def extract_loadable_segments(elf: ELFFileType) -> List[ELFSegment]: - """ - Extract PT_LOAD segments from ELF file - - Args: - elf: ELFFile object - - Returns: - List[ELFSegment]: List of loadable segments - """ - segments = [] - - for segment in elf.iter_segments(): - if segment["p_type"] == "PT_LOAD": - seg = ELFSegment( - vaddr=segment["p_vaddr"], - paddr=segment["p_paddr"], - filesz=segment["p_filesz"], - memsz=segment["p_memsz"], - data=segment.data(), - ) - segments.append(seg) - - return segments +def _segments(elf: ELFFile) -> List[ELFSegment]: + return [ + ELFSegment( + vaddr=segment["p_vaddr"], + paddr=segment["p_paddr"], + filesz=segment["p_filesz"], + memsz=segment["p_memsz"], + data=segment.data(), + ) + for segment in elf.iter_segments() + if segment["p_type"] == "PT_LOAD" + ] -def parse_symbol_table(elf: ELFFileType) -> Dict[str, int]: - """ - Parse symbol table from ELF file - Only includes STT_FUNC, STT_OBJECT, STT_COMMON, and STT_NOTYPE symbols - that are not SHN_UNDEF. +def _sections(elf: ELFFile) -> List[ELFSection]: + return [ + ELFSection(s.name, s["sh_addr"], s["sh_size"], s["sh_flags"]) + for s in elf.iter_sections() + if s.name + ] - Args: - elf: ELFFile object - Returns: - Dict[str, int]: Dictionary mapping symbol names to addresses - """ +def _symbols(elf: ELFFile) -> Dict[str, int]: + """Named, defined symbols of type FUNC, OBJECT, COMMON, or NOTYPE.""" symbols = {} - - # Iterate through all sections looking for symbol tables for section in elf.iter_sections(): if not isinstance(section, SymbolTableSection): continue - for symbol in section.iter_symbols(): - # Skip undefined symbols - if symbol["st_shndx"] == "SHN_UNDEF": + if symbol["st_shndx"] == "SHN_UNDEF" or not symbol.name: continue - - # Only include specific symbol types - symbol_type = symbol["st_info"]["type"] - if symbol_type not in ("STT_FUNC", "STT_OBJECT", "STT_COMMON", "STT_NOTYPE"): - continue - - # Skip symbols with empty names - name = symbol.name - if not name: + if symbol["st_info"]["type"] not in ( + "STT_FUNC", + "STT_OBJECT", + "STT_COMMON", + "STT_NOTYPE", + ): continue - - # Add to symbol table - symbols[name] = symbol["st_value"] - + symbols[symbol.name] = symbol["st_value"] return symbols @@ -201,44 +124,31 @@ def load_elf_file(path: str) -> ELFInfo: """ Load and parse an ELF file - Args: - path: Path to ELF file - - Returns: - ELFInfo: Parsed ELF information - Raises: FileNotFoundError: If file doesn't exist - RuntimeError: If file is not a valid ELF + RuntimeError: If the file is not a valid ELF for a supported ISA """ elf_path = Path(path) - if not elf_path.exists(): raise FileNotFoundError(f"ELF file not found: {path}") try: with open(elf_path, "rb") as f: elf = ELFFile(f) - - # Detect ISA and architecture - isa = detect_isa_from_elf(elf) - if isa == ISA.UNKNOWN: - raise RuntimeError(f"Unsupported ELF machine type: {elf.header['e_machine']}") - - architecture = detect_architecture(elf, isa) - - # Get entry point - entry = elf.header["e_entry"] - - # Extract loadable segments - segments = extract_loadable_segments(elf) - - # Parse symbol table - symbols = parse_symbol_table(elf) - + machine = elf.header["e_machine"] + is_64bit = elf.elfclass == 64 + spec = spec_for_elf(machine, is_64bit) + if spec is None: + raise RuntimeError( + f"Unsupported ELF machine type: {machine} ({elf.elfclass}-bit); " + f"supported ISAs: {', '.join(isa_names())}" + ) return ELFInfo( - isa=isa, architecture=architecture, entry=entry, segments=segments, symbols=symbols + spec=spec, + entry=elf.header["e_entry"], + segments=_segments(elf), + symbols=_symbols(elf), + sections=_sections(elf), ) - except Exception as e: raise RuntimeError(f"Failed to parse ELF file {path}: {e}") diff --git a/mapachespim/engines.py b/mapachespim/engines.py new file mode 100644 index 0000000..6d143f5 --- /dev/null +++ b/mapachespim/engines.py @@ -0,0 +1,99 @@ +""" +Unicorn (emulation) and Capstone (disassembly) settings for each ISA. + +These are the only emulator-specific facts about an ISA; everything else is +in :mod:`mapachespim.isa`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, Tuple + +try: + import unicorn + from unicorn import arm64_const, mips_const, riscv_const, x86_const +except ImportError as e: + raise ImportError( + f"Unicorn Engine not installed. Install with: pip install unicorn\nOriginal error: {e}" + ) + +try: + import capstone +except ImportError as e: + raise ImportError( + f"Capstone not installed. Install with: pip install capstone\nOriginal error: {e}" + ) + +from .isa import ISA, ISASpec, get_spec + + +@dataclass(frozen=True) +class EngineBinding: + """How to emulate and disassemble one ISA.""" + + uc_arch: int + uc_mode: int + pc_reg: int + sp_reg: int + gpr_regs: Tuple[int, ...] + """Unicorn register constant for each general-purpose register, by number. + + Looked up by name: Unicorn's constants are not always consecutive.""" + + cs_arch: int + cs_mode: int + + +ENGINES: Dict[ISA, EngineBinding] = { + ISA.RISCV: EngineBinding( + uc_arch=unicorn.UC_ARCH_RISCV, + uc_mode=unicorn.UC_MODE_RISCV64, + pc_reg=riscv_const.UC_RISCV_REG_PC, + sp_reg=riscv_const.UC_RISCV_REG_SP, + gpr_regs=tuple(getattr(riscv_const, f"UC_RISCV_REG_X{n}") for n in range(32)), + cs_arch=capstone.CS_ARCH_RISCV, + cs_mode=capstone.CS_MODE_RISCV64, + ), + ISA.MIPS: EngineBinding( + uc_arch=unicorn.UC_ARCH_MIPS, + uc_mode=unicorn.UC_MODE_MIPS32 | unicorn.UC_MODE_BIG_ENDIAN, + pc_reg=mips_const.UC_MIPS_REG_PC, + sp_reg=mips_const.UC_MIPS_REG_SP, + gpr_regs=tuple(getattr(mips_const, f"UC_MIPS_REG_{n}") for n in range(32)), + cs_arch=capstone.CS_ARCH_MIPS, + cs_mode=capstone.CS_MODE_MIPS32 | capstone.CS_MODE_BIG_ENDIAN, + ), + ISA.ARM: EngineBinding( + uc_arch=unicorn.UC_ARCH_ARM64, + uc_mode=unicorn.UC_MODE_ARM, + pc_reg=arm64_const.UC_ARM64_REG_PC, + sp_reg=arm64_const.UC_ARM64_REG_SP, + # Unicorn numbers x0-x28 consecutively but x29 (fp) and x30 (lr) + # separately, so look each one up by name + gpr_regs=tuple(getattr(arm64_const, f"UC_ARM64_REG_X{n}") for n in range(31)) + + (arm64_const.UC_ARM64_REG_SP,), + cs_arch=capstone.CS_ARCH_ARM64, + cs_mode=capstone.CS_MODE_ARM, + ), + ISA.X86_64: EngineBinding( + uc_arch=unicorn.UC_ARCH_X86, + uc_mode=unicorn.UC_MODE_64, + pc_reg=x86_const.UC_X86_REG_RIP, + sp_reg=x86_const.UC_X86_REG_RSP, + gpr_regs=tuple( + getattr(x86_const, f"UC_X86_REG_{name.upper()}") + for name in get_spec(ISA.X86_64).registers.names + ), + cs_arch=capstone.CS_ARCH_X86, + cs_mode=capstone.CS_MODE_64, + ), +} + + +def engine_for(spec: ISASpec) -> EngineBinding: + """The engine binding for an ISA.""" + try: + return ENGINES[spec.isa] + except KeyError: + raise ValueError(f"No emulator binding for {spec.display_name}") diff --git a/mapachespim/examples.py b/mapachespim/examples.py index f7ce591..e73e2a4 100644 --- a/mapachespim/examples.py +++ b/mapachespim/examples.py @@ -14,13 +14,10 @@ from pathlib import Path from typing import List, Optional +from .isa import ISA_SPECS + # ISA directory name -> display name, in the order examples are listed -ISA_DIRS = { - "riscv": "RISC-V 64-bit", - "mips": "MIPS32", - "arm": "ARM64", - "x86_64": "x86-64", -} +ISA_DIRS = {spec.examples_dir: spec.display_name for spec in ISA_SPECS} SOURCE_SUFFIXES = (".s", ".S", ".asm") @@ -54,7 +51,7 @@ def examples_dir() -> Optional[Path]: """Directory containing the bundled examples, or None if unavailable.""" package_dir = Path(__file__).resolve().parent for candidate in (package_dir / "examples", package_dir.parent / "examples"): - if (candidate / "riscv").is_dir(): + if (candidate / ISA_SPECS[0].examples_dir).is_dir(): return candidate return None diff --git a/mapachespim/isa.py b/mapachespim/isa.py new file mode 100644 index 0000000..edf1a44 --- /dev/null +++ b/mapachespim/isa.py @@ -0,0 +1,359 @@ +""" +The ISA registry: everything MapacheSPIM knows about each instruction set. + +Each supported ISA is described once, by an :class:`ISASpec`, and every +other part of the package (simulator, disassembler, ELF loader and writer, +assembler, console, and command-line tools) reads its ISA-specific facts +from here instead of branching on the ISA itself. + +This module is pure Python with no emulator dependencies, so the assembler +can use it without loading Unicorn or Capstone. The Unicorn and Capstone +constants for each ISA live in :mod:`mapachespim.engines`, and each ISA's +instruction encoder is registered in :mod:`mapachespim.toolchain.targets`. + +Adding an ISA means adding an :class:`ISA` member and an :class:`ISASpec` +here, an engine binding, and an assembler target; see +``docs/dev/adding-an-isa.md``. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import IntEnum +from typing import Callable, Dict, Literal, Optional, Tuple, Union + +from .memory_map import ARM64_LAYOUT, MIPS32_LAYOUT, RISCV64_LAYOUT, X86_64_LAYOUT, MemoryLayout + + +class ISA(IntEnum): + """ISA types supported by the simulator""" + + RISCV = 0 + ARM = 1 + X86_64 = 2 + MIPS = 3 + UNKNOWN = -1 + + +@dataclass(frozen=True) +class RegisterFile: + """The general-purpose registers, numbered as the simulator API uses them.""" + + names: Tuple[str, ...] + """Conventional (ABI) name of each register, indexed by number.""" + + zero_register: Optional[int] = None + """Number of the hardwired-zero register, which cannot be written.""" + + number_prefix: Optional[str] = None + """Prefix for numbered display, e.g. "x" shows "x5 (t0)"; None shows only the name.""" + + @property + def count(self) -> int: + return len(self.names) + + def is_writable(self, n: int) -> bool: + return 0 <= n < self.count and n != self.zero_register + + +@dataclass(frozen=True) +class SyscallABI: + """Registers used by the SPIM-style syscall interface (register numbers).""" + + number: int + """Register holding the syscall number.""" + + arg0: int + """Register holding the first argument.""" + + result: int + """Register that receives the result.""" + + +@dataclass(frozen=True) +class InstructionPattern: + """A fixed instruction encoding: matches when ``word & mask == value``. + + ``word`` is the first ``size`` bytes at the instruction address, read in + the ISA's byte order. + """ + + size: int + mask: int + value: int + + def matches(self, word: int) -> bool: + return word & self.mask == self.value + + +@dataclass(frozen=True) +class AsmDialect: + """How GNU as behaves for this ISA, where it differs between ISAs.""" + + word_size: int = 4 + """Bytes emitted by ``.word`` (x86 uses 2, like ``.value``).""" + + align_in_bytes: bool = False + """``.align N`` means N bytes (x86) rather than 2**N bytes.""" + + auto_align_data: bool = False + """``.half``/``.word``/``.dword`` are aligned to their size (MIPS).""" + + hash_comments_only_at_line_start: bool = False + """'#' starts a comment only at the start of a line (ARM64, where '#' marks immediates).""" + + +def _no_trap(word: int) -> Optional[str]: + return None + + +@dataclass(frozen=True) +class ISASpec: + """Everything ISA-specific that is not tied to a particular emulator or encoder.""" + + isa: ISA + name: str + """Canonical name, as used by ``--isa`` and ``.isa`` (e.g. "riscv64").""" + + display_name: str + """Human-readable name (e.g. "RISC-V 64-bit").""" + + aliases: Tuple[str, ...] + """Other accepted names (e.g. "riscv").""" + + examples_dir: str + """Directory of the bundled examples for this ISA (e.g. "riscv").""" + + layout: MemoryLayout + registers: RegisterFile + syscall_abi: SyscallABI + + syscall_instruction: InstructionPattern + """The instruction that performs a syscall (ecall, syscall, svc).""" + + min_instruction_size: int + max_instruction_size: int + + elf_machine: int + """ELF e_machine value.""" + + elf_machine_name: str + """pyelftools' name for ``elf_machine`` (e.g. "EM_RISCV").""" + + elf_flags: int = 0 + """ELF e_flags written by the assembler.""" + + asm: AsmDialect = field(default_factory=AsmDialect) + + sign_extends_32bit_addresses: bool = False + """Addresses built from 32-bit immediates are sign-extended to 64 bits (RV64's + lui), so the simulator mirrors [0x80000000, 2**32) at 0xFFFFFFFF80000000.""" + + describe_trap: Callable[[int], Optional[str]] = _no_trap + """Explain the trap raised by the instruction word, or None if it is not a trap.""" + + @property + def word_bits(self) -> int: + """Width of a general-purpose register.""" + return 64 if self.layout.is_64bit else 32 + + @property + def byteorder(self) -> Literal["little", "big"]: + return "little" if self.layout.is_little_endian else "big" + + @property + def fixed_width(self) -> bool: + """True if every instruction has the same size.""" + return self.min_instruction_size == self.max_instruction_size + + def signed(self, value: int) -> int: + """Interpret a register value as a signed integer of the register width.""" + bits = self.word_bits + value &= (1 << bits) - 1 + return value - (1 << bits) if value & (1 << (bits - 1)) else value + + +# --- Trap descriptions (for runtime-check instructions that stop the program) --- + + +def _mips_trap(word: int) -> Optional[str]: + """MIPS div/rem expand to code that runs 'break 7' on division by zero and + 'break 6' on overflow, as SPIM and GNU as do.""" + if word & 0xFC00003F != 0x0000000D: # break + return None + code = (word >> 16) & 0x3FF + if code == 7: + return "Division by zero" + if code == 6: + return "Integer overflow in division (-2147483648 / -1)" + return f"Break instruction executed (break {code})" + + +def _riscv_trap(word: int) -> Optional[str]: + return "ebreak instruction executed" if word == 0x00100073 else None + + +def _arm64_trap(word: int) -> Optional[str]: + if word & 0xFFE0001F == 0xD4200000: # brk #imm + return f"brk instruction executed (brk #{(word >> 5) & 0xFFFF})" + return None + + +# --- The registry --- + +_RISCV_REGISTERS = ( + "zero", "ra", "sp", "gp", "tp", "t0", "t1", "t2", + "s0", "s1", "a0", "a1", "a2", "a3", "a4", "a5", + "a6", "a7", "s2", "s3", "s4", "s5", "s6", "s7", + "s8", "s9", "s10", "s11", "t3", "t4", "t5", "t6", +) # fmt: skip + +_MIPS_REGISTERS = ( + "zero", "at", "v0", "v1", "a0", "a1", "a2", "a3", + "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", + "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", + "t8", "t9", "k0", "k1", "gp", "sp", "fp", "ra", +) # fmt: skip + +_X86_64_REGISTERS = ( + "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", +) # fmt: skip + +RISCV64 = ISASpec( + isa=ISA.RISCV, + name="riscv64", + display_name="RISC-V 64-bit", + aliases=("riscv",), + examples_dir="riscv", + layout=RISCV64_LAYOUT, + registers=RegisterFile( + names=_RISCV_REGISTERS, + zero_register=0, + number_prefix="x", + ), + syscall_abi=SyscallABI(number=17, arg0=10, result=10), # a7, a0, a0 + syscall_instruction=InstructionPattern(4, 0xFFFFFFFF, 0x00000073), # ecall + min_instruction_size=4, + max_instruction_size=4, + elf_machine=243, + elf_machine_name="EM_RISCV", + describe_trap=_riscv_trap, + sign_extends_32bit_addresses=True, +) + +MIPS32 = ISASpec( + isa=ISA.MIPS, + name="mips32", + display_name="MIPS32", + aliases=("mips",), + examples_dir="mips", + layout=MIPS32_LAYOUT, + registers=RegisterFile( + names=_MIPS_REGISTERS, + zero_register=0, + ), + syscall_abi=SyscallABI(number=2, arg0=4, result=2), # $v0, $a0, $v0 + syscall_instruction=InstructionPattern(4, 0xFFFFFFFF, 0x0000000C), # syscall + min_instruction_size=4, + max_instruction_size=4, + elf_machine=8, + elf_machine_name="EM_MIPS", + elf_flags=0x50001000, # MIPS32R2, O32 ABI + asm=AsmDialect(auto_align_data=True), + describe_trap=_mips_trap, +) + +ARM64 = ISASpec( + isa=ISA.ARM, + name="arm64", + display_name="ARM64", + aliases=("aarch64",), + examples_dir="arm", + layout=ARM64_LAYOUT, + # Register 31 is the stack pointer (in the encoding it is sp or xzr + # depending on the instruction; the simulator API exposes sp). + registers=RegisterFile( + names=tuple(f"x{n}" for n in range(31)) + ("sp",), + number_prefix="x", + ), + syscall_abi=SyscallABI(number=8, arg0=0, result=0), # x8, x0, x0 + syscall_instruction=InstructionPattern(4, 0xFFE0001F, 0xD4000001), # svc #imm + min_instruction_size=4, + max_instruction_size=4, + elf_machine=183, + elf_machine_name="EM_AARCH64", + asm=AsmDialect(hash_comments_only_at_line_start=True), + describe_trap=_arm64_trap, +) + +X86_64 = ISASpec( + isa=ISA.X86_64, + name="x86_64", + display_name="x86-64", + aliases=("x64",), + examples_dir="x86_64", + layout=X86_64_LAYOUT, + registers=RegisterFile( + names=_X86_64_REGISTERS, + ), + syscall_abi=SyscallABI(number=0, arg0=7, result=0), # rax, rdi, rax + syscall_instruction=InstructionPattern(2, 0xFFFF, 0x050F), # syscall (0f 05) + min_instruction_size=1, + max_instruction_size=15, + elf_machine=62, + elf_machine_name="EM_X86_64", + asm=AsmDialect(word_size=2, align_in_bytes=True), +) + +# In the order ISAs are listed to users +ISA_SPECS: Tuple[ISASpec, ...] = (RISCV64, MIPS32, ARM64, X86_64) + +_BY_ISA: Dict[ISA, ISASpec] = {spec.isa: spec for spec in ISA_SPECS} +_BY_NAME: Dict[str, ISASpec] = {} +for _spec in ISA_SPECS: + for _name in (_spec.name, *_spec.aliases): + _BY_NAME[_name] = _spec + + +def _normalize(name: str) -> str: + return name.strip().lower().replace("-", "_") + + +def isa_names() -> Tuple[str, ...]: + """Canonical names of all supported ISAs (e.g. for ``--isa`` choices).""" + return tuple(spec.name for spec in ISA_SPECS) + + +def find_spec(isa: Union[str, ISA, ISASpec]) -> Optional[ISASpec]: + """The spec for an ISA enum member, canonical name, or alias; None if unknown.""" + if isinstance(isa, ISASpec): + return isa + if isinstance(isa, ISA): + return _BY_ISA.get(isa) + return _BY_NAME.get(_normalize(isa)) + + +def get_spec(isa: Union[str, ISA, ISASpec]) -> ISASpec: + """Like :func:`find_spec`, but raises ValueError for an unknown ISA.""" + spec = find_spec(isa) + if spec is None: + raise ValueError(f"Unknown ISA: {isa!r}. Valid options: {', '.join(isa_names())}") + return spec + + +def canonical_isa(name: str) -> str: + """The canonical name for an ISA or one of its aliases (e.g. mips -> mips32). + + Unknown names are returned normalized but otherwise unchanged. + """ + spec = find_spec(name) + return spec.name if spec is not None else _normalize(name) + + +def spec_for_elf(machine_name: str, is_64bit: bool) -> Optional[ISASpec]: + """The spec matching an ELF header's machine and class, or None.""" + for spec in ISA_SPECS: + if spec.elf_machine_name == machine_name and spec.layout.is_64bit == is_64bit: + return spec + return None diff --git a/mapachespim/memory_map.py b/mapachespim/memory_map.py index 8089d0d..cd34420 100644 --- a/mapachespim/memory_map.py +++ b/mapachespim/memory_map.py @@ -3,14 +3,13 @@ Each ISA has different conventional memory layouts for bare-metal programs. The assembler places sections at these addresses and the simulator uses the -same values for its default memory map and initial stack pointer, so this -module is the single source of truth for both. +same values for its default memory map and initial stack pointer. Each +layout is attached to its ISA in :mod:`mapachespim.isa`. """ from __future__ import annotations from dataclasses import dataclass -from typing import Dict @dataclass(frozen=True) @@ -97,26 +96,13 @@ def __str__(self) -> str: is_little_endian=False, # MIPS is big-endian in MapacheSPIM ) -# Map of ISA names to layouts -MEMORY_LAYOUTS: Dict[str, MemoryLayout] = { - "riscv64": RISCV64_LAYOUT, - "riscv": RISCV64_LAYOUT, # Alias - "arm64": ARM64_LAYOUT, - "aarch64": ARM64_LAYOUT, # Alias - "x86_64": X86_64_LAYOUT, - "x86-64": X86_64_LAYOUT, # Alias - "x64": X86_64_LAYOUT, # Alias - "mips32": MIPS32_LAYOUT, - "mips": MIPS32_LAYOUT, # Alias -} - def get_layout(isa: str) -> MemoryLayout: """ Get the memory layout for an ISA. Args: - isa: ISA name (e.g., "riscv64", "arm64", "x86_64", "mips32") + isa: ISA name or alias (e.g., "riscv64", "arm64", "x86_64", "mips32") Returns: MemoryLayout for the specified ISA. @@ -124,8 +110,6 @@ def get_layout(isa: str) -> MemoryLayout: Raises: ValueError: If ISA is not recognized. """ - isa_lower = isa.lower().replace("-", "_") - if isa_lower not in MEMORY_LAYOUTS: - valid = sorted(set(MEMORY_LAYOUTS.keys())) - raise ValueError(f"Unknown ISA: {isa!r}. Valid options: {', '.join(valid)}") - return MEMORY_LAYOUTS[isa_lower] + from .isa import get_spec # the registry imports this module + + return get_spec(isa).layout diff --git a/mapachespim/symbols.py b/mapachespim/symbols.py new file mode 100644 index 0000000..1ed5c73 --- /dev/null +++ b/mapachespim/symbols.py @@ -0,0 +1,55 @@ +"""Symbol table lookups: by name, and from an address to the nearest symbol.""" + +from __future__ import annotations + +import bisect +from typing import Dict, List, Optional, Tuple + +# Addresses further than this past a symbol are not described relative to it +MAX_SYMBOL_DISTANCE = 0x100000 + + +class SymbolTable: + """Program symbols (name -> address) with fast reverse lookup.""" + + def __init__(self, symbols: Optional[Dict[str, int]] = None) -> None: + self._by_name: Dict[str, int] = dict(symbols or {}) + # One name per address; when several symbols share an address the + # last one defined is used + by_addr = {addr: name for name, addr in self._by_name.items()} + self._addrs: List[int] = sorted(by_addr) + self._names: List[str] = [by_addr[a] for a in self._addrs] + + def __len__(self) -> int: + return len(self._by_name) + + def __contains__(self, name: object) -> bool: + return name in self._by_name + + def as_dict(self) -> Dict[str, int]: + """A copy of the table as name -> address.""" + return dict(self._by_name) + + def lookup(self, name: str) -> Optional[int]: + """Address of a symbol, or None.""" + return self._by_name.get(name) + + def nearest(self, addr: int) -> Tuple[Optional[str], Optional[int]]: + """(symbol, offset) for the closest symbol at or before ``addr``. + + Returns (None, None) when there is none within MAX_SYMBOL_DISTANCE. + """ + index = bisect.bisect_right(self._addrs, addr) - 1 + if index < 0: + return (None, None) + offset = addr - self._addrs[index] + if offset > MAX_SYMBOL_DISTANCE: + return (None, None) + return (self._names[index], offset) + + def describe(self, addr: int) -> str: + """```` or ```` for an address, or "" if none.""" + name, offset = self.nearest(addr) + if name is None: + return "" + return f"<{name}>" if offset == 0 else f"<{name}+{offset}>" diff --git a/mapachespim/syscalls.py b/mapachespim/syscalls.py new file mode 100644 index 0000000..6c7fcd2 --- /dev/null +++ b/mapachespim/syscalls.py @@ -0,0 +1,227 @@ +""" +SPIM-compatible syscalls. + +A program requests a service by putting a syscall number in its ISA's syscall +register and executing the syscall instruction (see +:class:`mapachespim.isa.SyscallABI`). The simulator stops at that instruction +and hands the request to :class:`SyscallHandler`, which works with any ISA: +it only reads and writes registers and memory through the small +:class:`Machine` interface. + +To add a syscall, add its number to :class:`Syscall` and a handler method to +``SyscallHandler._handlers``. +""" + +from __future__ import annotations + +import sys +from dataclasses import dataclass +from enum import IntEnum +from typing import Callable, Dict, Optional, Protocol, TextIO + +from .isa import ISASpec + + +class Machine(Protocol): + """What a syscall needs from the simulator.""" + + @property + def spec(self) -> ISASpec: ... + + def get_reg(self, reg_num: int) -> int: ... + + def set_reg(self, reg_num: int, value: int) -> None: ... + + def read_mem(self, addr: int, length: int) -> bytes: ... + + +class Syscall(IntEnum): + """Supported syscall numbers (SPIM's, plus Linux's exit on RISC-V).""" + + PRINT_INT = 1 + PRINT_STRING = 4 + READ_INT = 5 + EXIT = 10 + PRINT_CHAR = 11 + READ_CHAR = 12 + EXIT2 = 17 + EXIT_CODE = 93 + + +@dataclass(frozen=True) +class SyscallResult: + """Outcome of one syscall: the program continues unless it exited or failed.""" + + exit_code: Optional[int] = None + """Set if the program exited.""" + + error: Optional[str] = None + """Set if the syscall could not be performed.""" + + @property + def exited(self) -> bool: + return self.exit_code is not None + + +CONTINUE = SyscallResult() + + +class ProgramIO: + """The program's console: output from print syscalls and input for read syscalls. + + ``stdout`` and ``stdin`` default to the process streams (looked up at call + time, so pytest's capsys and similar redirection work) and can be replaced, + e.g. with StringIO objects for autograding. + """ + + stdout: Optional[TextIO] + stdin: Optional[TextIO] + + def __init__(self) -> None: + self.stdout = None + self.stdin = None + self.output_needs_newline = False + """True if output so far did not end with a newline (consoles use this + to keep their own messages off the program's output line).""" + self._line_buffer = "" + + def clear_input(self) -> None: + """Forget any partially consumed input line.""" + self._line_buffer = "" + + def write(self, text: str) -> None: + if not text: + return + stream = self.stdout if self.stdout is not None else sys.stdout + stream.write(text) + stream.flush() + self.output_needs_newline = not text.endswith("\n") + + def read_line(self) -> Optional[str]: + """Read one line of input, without the newline (None on EOF).""" + # Make sure any prompt the program printed is visible first + (self.stdout if self.stdout is not None else sys.stdout).flush() + # Pressing Enter echoes a newline, so the console is at a line start + self.output_needs_newline = False + if self.stdin is not None: + line = self.stdin.readline() + return None if line == "" else line.rstrip("\r\n") + try: + # input() gives line editing on interactive terminals + return input() + except EOFError: + return None + + def read_int(self) -> int: + """Read an integer from the next line (0 if it is not a number).""" + # A partially consumed line (from read_char) is used first + line = self._line_buffer if self._line_buffer.strip() else self.read_line() + self._line_buffer = "" + try: + return int((line or "").strip(), 0) + except ValueError: + return 0 + + def read_char(self) -> int: + """Read one character (0 on EOF); "ab" yields a, b, then newline.""" + if not self._line_buffer: + line = self.read_line() + self._line_buffer = "" if line is None else line + "\n" + if not self._line_buffer: + return 0 + char, self._line_buffer = self._line_buffer[0], self._line_buffer[1:] + return ord(char) & 0xFF + + +def read_c_string(machine: Machine, addr: int, limit: int = 65536) -> str: + """Read a NUL-terminated string from simulated memory.""" + data = bytearray() + while len(data) < limit: + # Read a chunk, shrinking near the end of a mapped region + chunk = b"" + size = 256 - (addr % 256) + while size > 0: + try: + chunk = machine.read_mem(addr, size) + break + except RuntimeError: + size //= 2 + if not chunk: + if not data: + raise RuntimeError(f"print_string: address 0x{addr:x} is not mapped") + break + nul = chunk.find(0) + if nul >= 0: + data += chunk[:nul] + break + data += chunk + addr += len(chunk) + return data.decode("utf-8", errors="replace") + + +class SyscallHandler: + """Performs syscalls for a :class:`Machine`, using a :class:`ProgramIO`.""" + + def __init__(self, io: ProgramIO) -> None: + self.io = io + self._handlers: Dict[int, Callable[[Machine], SyscallResult]] = { + Syscall.PRINT_INT: self._print_int, + Syscall.PRINT_STRING: self._print_string, + Syscall.READ_INT: self._read_int, + Syscall.EXIT: self._exit, + Syscall.PRINT_CHAR: self._print_char, + Syscall.READ_CHAR: self._read_char, + Syscall.EXIT2: self._exit_with_code, + Syscall.EXIT_CODE: self._exit_with_code, + } + + def handle(self, machine: Machine) -> SyscallResult: + """Perform the syscall the machine's registers request.""" + abi = machine.spec.syscall_abi + number = machine.get_reg(abi.number) + handler = self._handlers.get(number) + if handler is None: + supported = ", ".join(str(int(n)) for n in Syscall) + reg_name = machine.spec.registers.names[abi.number] + return SyscallResult( + error=f"Unknown syscall number {machine.spec.signed(number)} in {reg_name} " + f"(supported: {supported})" + ) + return handler(machine) + + @staticmethod + def _arg0(machine: Machine) -> int: + return machine.get_reg(machine.spec.syscall_abi.arg0) + + @staticmethod + def _set_result(machine: Machine, value: int) -> None: + machine.set_reg(machine.spec.syscall_abi.result, value & 0xFFFFFFFFFFFFFFFF) + + def _print_int(self, machine: Machine) -> SyscallResult: + self.io.write(str(machine.spec.signed(self._arg0(machine)))) + return CONTINUE + + def _print_string(self, machine: Machine) -> SyscallResult: + try: + self.io.write(read_c_string(machine, self._arg0(machine))) + except RuntimeError as e: + return SyscallResult(error=str(e)) + return CONTINUE + + def _print_char(self, machine: Machine) -> SyscallResult: + self.io.write(chr(self._arg0(machine) & 0xFF)) + return CONTINUE + + def _read_int(self, machine: Machine) -> SyscallResult: + self._set_result(machine, self.io.read_int()) + return CONTINUE + + def _read_char(self, machine: Machine) -> SyscallResult: + self._set_result(machine, self.io.read_char()) + return CONTINUE + + def _exit(self, machine: Machine) -> SyscallResult: + return SyscallResult(exit_code=0) + + def _exit_with_code(self, machine: Machine) -> SyscallResult: + return SyscallResult(exit_code=machine.spec.signed(self._arg0(machine)) & 0xFF) diff --git a/mapachespim/toolchain/__init__.py b/mapachespim/toolchain/__init__.py index e616fe1..8453169 100644 --- a/mapachespim/toolchain/__init__.py +++ b/mapachespim/toolchain/__init__.py @@ -9,13 +9,14 @@ from __future__ import annotations -from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Optional, Union -from .assembler import Assembler +from ..isa import isa_names +from ..memory_map import MemoryLayout +from .assembler import Assembler, AssemblyResult +from .directives import find_isa_directive from .elf_builder import ELFBuilder -from .memory_map import MEMORY_LAYOUTS, MemoryLayout __all__ = [ "assemble", @@ -24,38 +25,9 @@ "Assembler", "ELFBuilder", "MemoryLayout", - "MEMORY_LAYOUTS", ] -@dataclass -class AssemblyResult: - """Result of an assembly operation.""" - - elf_bytes: bytes - """The assembled ELF file as bytes.""" - - symbols: Dict[str, int] = field(default_factory=dict) - """Map of symbol names to addresses.""" - - errors: List[str] = field(default_factory=list) - """List of error messages (empty if successful).""" - - warnings: List[str] = field(default_factory=list) - """List of warning messages.""" - - isa: str = "" - """The ISA used for assembly.""" - - entry_point: int = 0 - """Entry point address.""" - - @property - def success(self) -> bool: - """Return True if assembly succeeded (no errors).""" - return len(self.errors) == 0 and len(self.elf_bytes) > 0 - - def assemble( source: str, isa: str, @@ -150,7 +122,7 @@ def assemble_file( # If ISA not specified via flag, check for .isa directive in source if isa is None: - isa = _detect_isa_from_directive(source) + isa = find_isa_directive(source) if isa is None: return AssemblyResult( elf_bytes=b"", @@ -177,39 +149,9 @@ def assemble_file( return result -def _detect_isa_from_directive(source: str) -> Optional[str]: - """ - Look for .isa directive in source code. - - Returns the ISA if found, None otherwise. - """ - from .directives import DirectiveParser - - # Quick scan for .isa directive without full parse - for line in source.splitlines(): - line = line.strip() - # Skip empty lines and comments - if not line or line.startswith("#") or line.startswith("//") or line.startswith(";"): - continue - # Check for .isa directive - if line.lower().startswith(".isa"): - parts = line.split() - if len(parts) >= 2: - # First token only, so a trailing comment is ignored - isa_value = parts[1].lower().replace("-", "_") - if isa_value in DirectiveParser.VALID_ISAS: - return isa_value - # Stop after first non-empty, non-comment, non-.isa line - # (the .isa directive should be at the top of the file) - if not line.startswith("."): - break - - return None - - def _isa_not_specified_error() -> str: """Generate helpful error message when ISA is not specified.""" - return """ISA not specified + return f"""ISA not specified Please specify the target architecture using one of: @@ -219,4 +161,4 @@ def _isa_not_specified_error() -> str: 2. File directive (add at start of file): .isa riscv64 -Supported ISAs: arm64, mips32, riscv64, x86_64""" +Supported ISAs: {", ".join(sorted(isa_names()))}""" diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index 49ef010..6258338 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -7,25 +7,22 @@ from __future__ import annotations from dataclasses import dataclass, field -from types import ModuleType -from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Set, Tuple +from typing import Callable, Dict, Iterator, List, Mapping, Optional, Tuple -from . import arm64, mips, riscv, x86 +from ..isa import get_spec from .directives import ( ALIGN_DIRECTIVES, - BYTE_DIRECTIVES, - DATA_SIZES, INTERNAL_LABEL_PREFIXES, - POOL_LABEL_PREFIX, DirectiveParser, LineType, ParsedLine, SectionData, + emits_data, ) from .dwarf import DWARFv2Builder from .elf_builder import STB_GLOBAL, STB_LOCAL, STT_FUNC, STT_NOTYPE, ELFBuilder, Section, Symbol from .expr import EncodeError, ExpressionError, UndefinedSymbol, evaluate -from .memory_map import get_layout +from .targets import Target, target_for def _lookup(values: Dict[str, int]) -> Callable[[str], int]: @@ -110,8 +107,10 @@ class Assembler: Multi-architecture assembler. Each ISA has a pure-Python encoder (riscv.py, mips.py, arm64.py, - x86.py) that produces the same bytes as GNU as, plus a shared directive - parser for GNU-as compatible source files. + x86.py) that produces the same bytes as GNU as, driven through a + :class:`~mapachespim.toolchain.targets.Target`, plus a shared directive + parser for GNU-as compatible source files. This class is the + ISA-independent part: layout, label resolution, and ELF output. Example: >>> asm = Assembler("riscv64") @@ -127,37 +126,6 @@ class Assembler: True """ - # ISA configuration mapping - ISA_CONFIG: Dict[str, Dict[str, Any]] = { - "riscv64": { - "instr_size": 4, - }, - "riscv": { - "instr_size": 4, - }, - "arm64": { - "instr_size": 4, - }, - "aarch64": { - "instr_size": 4, - }, - "x86_64": { - "instr_size": None, # Variable length - }, - "x86-64": { - "instr_size": None, - }, - "x64": { - "instr_size": None, - }, - "mips32": { - "instr_size": 4, - }, - "mips": { - "instr_size": 4, - }, - } - def __init__(self, isa: str): """ Initialize assembler for a specific ISA. @@ -168,86 +136,12 @@ def __init__(self, isa: str): Raises: ValueError: If ISA is not supported. """ - isa_lower = isa.lower().replace("-", "_") - if isa_lower not in self.ISA_CONFIG: - valid = sorted(set(k for k in self.ISA_CONFIG.keys() if "_" not in k)) - raise ValueError(f"Unknown ISA: {isa!r}. Valid: {', '.join(valid)}") - - self.isa = isa_lower - self._config = self.ISA_CONFIG[isa_lower] - self._layout = get_layout(isa_lower) + self.spec = get_spec(isa) + self.isa = self.spec.name + self._layout = self.spec.layout # Replaced for each assemble() call; also used by the layout passes - self._parser = DirectiveParser(isa=isa_lower) - self._long_jumps: Set[int] = set() # x86 jumps that need rel32 - self._align_after_data: Set[int] = set() # x86 code alignment after data - self._absolute_symbols: Set[str] = set() # names of .equ constants - - @staticmethod - def _add_literal_pools(section: SectionData) -> None: - """Turn ARM64 'ldr x0, =value' into loads from a literal pool. - - As in GNU as, each load reads a pool entry placed at the next - .ltorg/.pool directive or at the end of the section. Within a pool, - equal values share one entry; 4-byte entries come first, then 8-byte - entries, each group aligned to its size. - """ - lines: List[ParsedLine] = [] - pool: Dict[Tuple[object, ...], Tuple[str, str, int]] = {} # key -> (label, expr, size) - counter = 0 - - def dump(line_number: int) -> None: - for size in (4, 8): - entries = [(label, expr) for label, expr, sz in pool.values() if sz == size] - if not entries: - continue - lines.append( - ParsedLine( - line_number, - LineType.DIRECTIVE, - "", - directive="balign", - directive_args=[str(size), "0"], # zeros, not nops - ) - ) - for label, expr in entries: - lines.append( - ParsedLine( - line_number, - LineType.DIRECTIVE, - "", - label=label, - directive="word" if size == 4 else "dword", - directive_args=[expr], - ) - ) - pool.clear() - - last_line = 0 - for line in section.lines: - last_line = line.line_number - if line.line_type == LineType.INSTRUCTION and line.instruction: - found = arm64.literal_load(line.instruction) - if found is not None: - mnemonic, rt, expr = found - try: - size = arm64.literal_size(mnemonic, rt) - except EncodeError as e: - raise EncodeError(f"Line {line.line_number}: {e}") - value = arm64.constant_value(expr) - key: Tuple[object, ...] = ( - ("value", value & ((1 << (8 * size)) - 1), size) - if value is not None - else ("expr", expr.replace(" ", ""), size) - ) - if key not in pool: - counter += 1 - pool[key] = (f"{POOL_LABEL_PREFIX}{counter}", expr, size) - line.instruction = f"{mnemonic} {rt}, {pool[key][0]}" - lines.append(line) - if line.line_type == LineType.DIRECTIVE and line.directive in ("ltorg", "pool"): - dump(line.line_number) - dump(last_line) - section.lines = lines + self._parser = DirectiveParser(isa=self.isa) + self._target: Target = target_for(self.isa, self._parser) @staticmethod def _resolve_constants( @@ -273,26 +167,6 @@ def _resolve_constants( progress = resolved_any = True return resolved_any - @staticmethod - def _alignments_after_data(section: Optional[SectionData]) -> Set[int]: - """Alignment directives in code whose last preceding item is data. - - GNU as pads x86 code with a one-byte nop first in that case, since - the data might be an incomplete instruction. - """ - found: Set[int] = set() - after_data = False - for line in section.lines if section else []: - if line.line_type == LineType.INSTRUCTION and line.instruction: - after_data = False - elif line.line_type == LineType.DIRECTIVE: - if line.directive in ALIGN_DIRECTIVES: - if after_data: - found.add(id(line)) - elif Assembler._emits_data(line): - after_data = True - return found - def _text_directive_bytes( self, line: ParsedLine, addr: int, base: int, labels: Dict[str, int], strict: bool ) -> bytes: @@ -318,53 +192,11 @@ def resolve(name: str) -> int: is_align = line.directive in ALIGN_DIRECTIVES args = line.directive_args explicit_fill = len(args) > 1 and bool(args[1]) - if self._builtin_encoder is x86 and explicit_fill: - # GNU as treats a fill of 0x90 (nop) like no fill: best nops - explicit_fill = evaluate(args[1], resolve) & 0xFF != 0x90 + if explicit_fill: + explicit_fill = self._target.is_explicit_fill(evaluate(args[1], resolve)) if not is_align or explicit_fill or not data: return data - # Pad code with nops, as GNU as does (MIPS's nop is all zeros) - if self._is_riscv: - # As GNU as's riscv_make_nops: a zero byte if odd, at most one - # 2-byte c.nop, then 4-byte nops - head = bytes(len(data) % 2) + (b"\x01\x00" if len(data) % 4 >= 2 else b"") - data = head + (0x00000013).to_bytes(4, "little") * (len(data) // 4) - elif self._builtin_encoder is arm64: - misaligned = len(data) % 4 - data = bytes(misaligned) + (0xD503201F).to_bytes(4, "little") * (len(data) // 4) - elif self._builtin_encoder is x86: - data = x86.nop_padding(len(data), id(line) in self._align_after_data) - return data - - def _line_options(self, line: ParsedLine) -> Dict[str, Any]: - """Per-line encoder options (MIPS: .set reorder / noreorder; x86-64: - syntax, which symbols are constants, and jump size).""" - if self.isa in ("mips32", "mips"): - return {"reorder": line.reorder} - if self._builtin_encoder is x86: - return { - "syntax": line.syntax, - "absolute_symbols": self._absolute_symbols, - "long_jump": id(line) in self._long_jumps, - } - return {} - - def _instruction_padding(self, addr: int, after_data: bool) -> int: - """Zero bytes GNU as puts before an instruction at ``addr``. - - ARM64 aligns an instruction to 4 bytes when it directly follows data - (GNU as does this when switching from data to code; an alignment - directive in between counts as code). - """ - return -addr % 4 if after_data and self._builtin_encoder is arm64 else 0 - - @staticmethod - def _emits_data(line: ParsedLine) -> bool: - """Whether ``line`` is a data directive such as .byte or .fill.""" - directive = line.directive or "" - return line.line_type == LineType.DIRECTIVE and ( - directive in DATA_SIZES or directive in BYTE_DIRECTIVES or directive == "value" - ) + return self._target.code_padding(len(data), line) def _is_real_alignment(self, line: ParsedLine, labels: Dict[str, int]) -> bool: """Whether ``line`` aligns to more than 1 byte (even if it inserts @@ -377,21 +209,6 @@ def _is_real_alignment(self, line: ParsedLine, labels: Dict[str, int]) -> bool: return False return self._parser.alignment(line.directive or "", value) > 1 - @property - def _is_riscv(self) -> bool: - return self.isa in ("riscv64", "riscv") - - @property - def _builtin_encoder(self) -> ModuleType: - """The pure-Python encoder module for this ISA.""" - if self._is_riscv: - return riscv - if self.isa in ("mips32", "mips"): - return mips - if self.isa in ("arm64", "aarch64"): - return arm64 - return x86 - def assemble( self, source: str, @@ -416,15 +233,12 @@ def assemble( # Parse source parser = DirectiveParser(isa=self.isa) self._parser = parser - self._long_jumps = set() + self._target = target_for(self.isa, parser) sections = parser.parse(source) - self._absolute_symbols = set(parser.constants) | set(parser.deferred_constants) - self._align_after_data = self._alignments_after_data(sections.get(".text")) - if self.isa in ("arm64", "aarch64") and ".text" in sections: - try: - self._add_literal_pools(sections[".text"]) - except EncodeError as e: - result.errors.append(str(e)) + try: + self._target.prepare(sections) + except EncodeError as e: + result.errors.append(str(e)) result.errors.extend(parser.errors) result.warnings.extend(parser.warnings) @@ -545,36 +359,26 @@ def assemble( # Add sections (standard ones first, then any custom sections) ordered = [n for n in (".text", ".data", ".rodata", ".bss") if n in sections] ordered += [n for n in sections if n not in ordered] + section_index: Dict[str, int] = {} # ELF section header index for sect_name in ordered: - if sect_name in sections: - sect_data = sections[sect_name] - if sect_data.data or sect_name == ".text": - base = section_bases.get(sect_name, self._layout.text_base) - builder.add_section( - Section( - name=sect_name, - data=bytes(sect_data.data), - address=base, - ) - ) + sect_data = sections[sect_name] + if sect_data.data or sect_name == ".text": + base = section_bases.get(sect_name, self._layout.text_base) + builder.add_section( + Section(name=sect_name, data=bytes(sect_data.data), address=base) + ) + section_index[sect_name] = len(builder.sections) - # Add symbols + # Add symbols, each in the section that defines it for name, addr in all_labels.items(): - is_global = name in parser.global_symbols - # Determine section index (simplified) - section_idx = 1 # Assume .text is section 1 - for i, (sn, _) in enumerate(sections.items()): - if sn == ".text": - section_idx = i + 1 - break - + defined_in = next((n for n, d in sections.items() if name in d.labels), "") builder.add_symbol( Symbol( name=name, address=addr, sym_type=STT_FUNC if name == entry_symbol else STT_NOTYPE, - binding=STB_GLOBAL if is_global else STB_LOCAL, - section_index=section_idx, + binding=STB_GLOBAL if name in parser.global_symbols else STB_LOCAL, + section_index=section_index.get(defined_in, 0), ) ) @@ -589,7 +393,7 @@ def assemble( # Determine address size and instruction length addr_size = 8 if self._layout.is_64bit else 4 - min_instr_len = self._config.get("instr_size") or 1 + min_instr_len = self.spec.min_instruction_size # Build DWARF sections dwarf = DWARFv2Builder( @@ -657,28 +461,13 @@ def _calculate_builtin_text_labels( if line.label in labels: stretch = addr - labels[line.label] if line.line_type == LineType.INSTRUCTION and line.instruction: - addr += self._instruction_padding(addr, after_data) + addr += self._target.padding_before_instruction(addr, after_data) after_data = False - # x86 jumps start short (rel8) and grow to rel32 once, for - # good, when the target is out of range (GNU's relaxation) - if ( - self._builtin_encoder is x86 - and id(line) not in self._long_jumps - and x86.is_relaxable(line.instruction) - and not x86.short_jump_fits( - line.instruction, - addr, - _RelaxView(data_labels, labels, new_labels, stretch), - ) - ): - self._long_jumps.add(id(line)) - grew = True - grown = True - else: - grown = False - size = self._builtin_encoder.instruction_size( - line.instruction, known, addr, **self._line_options(line) + grown = self._target.can_relax and self._target.relax( + line, addr, _RelaxView(data_labels, labels, new_labels, stretch) ) + grew = grew or grown + size = self._target.size(line, addr, known) if grown: stretch += size - 2 addr += size @@ -689,7 +478,7 @@ def _calculate_builtin_text_labels( ) except ExpressionError: pass # reported when the section is assembled - if self._emits_data(line): + if emits_data(line): after_data = True elif self._is_real_alignment(line, known): after_data = False @@ -734,7 +523,7 @@ def _assemble_section( continue code.extend(data) current_addr += len(data) - if self._emits_data(line): + if emits_data(line): after_data = True elif self._is_real_alignment(line, labels): after_data = False @@ -744,7 +533,7 @@ def _assemble_section( if line.line_type != LineType.INSTRUCTION or not line.instruction: continue - padding = self._instruction_padding(current_addr, after_data) + padding = self._target.padding_before_instruction(current_addr, after_data) after_data = False code.extend(bytes(padding)) current_addr += padding @@ -752,12 +541,8 @@ def _assemble_section( # Record debug line info before assembling debug_lines.append((current_addr, line.line_number)) - instr = line.instruction - try: - encoding = self._builtin_encoder.encode( - instr, current_addr, labels, **self._line_options(line) - ) + encoding = self._target.encode(line, current_addr, labels) except EncodeError as e: errors.append(f"Line {line.line_number}: {e} - {line.instruction}") continue @@ -791,6 +576,6 @@ def assemble_instruction(self, instr: str, address: int = 0) -> bytes: ValueError: If assembly fails. """ try: - return bytes(self._builtin_encoder.encode(instr, address, {})) + return bytes(self._target.encoder.encode(instr, address, {})) except EncodeError as e: raise ValueError(f"Assembly error: {e}") diff --git a/mapachespim/toolchain/cli.py b/mapachespim/toolchain/cli.py index 307e04f..e7b59d9 100644 --- a/mapachespim/toolchain/cli.py +++ b/mapachespim/toolchain/cli.py @@ -17,25 +17,24 @@ from pathlib import Path from .. import __version__ +from ..isa import ISA_SPECS, isa_names def main(args: list[str] | None = None) -> int: """Main entry point for mapachespim-as CLI.""" + supported = "\n".join(f" {spec.name:<8} - {spec.display_name}" for spec in ISA_SPECS) parser = argparse.ArgumentParser( prog="mapachespim-as", description="Assemble source files to ELF executables", formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" + epilog=f""" Examples: mapachespim-as program.s -o program.elf --isa riscv64 mapachespim-as hello.s --isa arm64 -g # With debug info mapachespim-as test.s # Auto-detect ISA from source Supported ISAs: - riscv64 - RISC-V 64-bit - arm64 - ARM AArch64 - x86_64 - Intel/AMD 64-bit - mips32 - MIPS 32-bit (big-endian) +{supported} """, ) @@ -56,7 +55,7 @@ def main(args: list[str] | None = None) -> int: parser.add_argument( "--isa", type=str, - choices=["riscv64", "arm64", "x86_64", "mips32"], + choices=isa_names(), default=None, help="Target ISA (auto-detected if not specified)", ) diff --git a/mapachespim/toolchain/directives.py b/mapachespim/toolchain/directives.py index b058ba3..6918886 100644 --- a/mapachespim/toolchain/directives.py +++ b/mapachespim/toolchain/directives.py @@ -12,17 +12,12 @@ from enum import Enum, auto from typing import Callable, Dict, List, Literal, Optional, Tuple +from ..isa import AsmDialect, canonical_isa, find_spec, isa_names from .expr import ExpressionError, UndefinedSymbol, evaluate -ALIGN_DIRECTIVES = ("align", "balign", "p2align") - -_ISA_ALIASES = {"riscv": "riscv64", "aarch64": "arm64", "x64": "x86_64", "mips": "mips32"} - +__all__ = ["DirectiveParser", "ParsedLine", "SectionData", "canonical_isa", "find_isa_directive"] -def canonical_isa(isa: str) -> str: - """The canonical name for an ISA or one of its aliases (e.g. mips -> mips32).""" - isa = isa.lower().replace("-", "_") - return _ISA_ALIASES.get(isa, isa) +ALIGN_DIRECTIVES = ("align", "balign", "p2align") # Other directives that emit bytes (besides the DATA_SIZES ones) @@ -62,13 +57,13 @@ class ParsedLine: """Arguments to the directive.""" instruction: Optional[str] = None + """Instruction mnemonic and operands.""" reorder: bool = True """MIPS: whether the assembler fills delay slots (.set reorder, the default).""" syntax: str = "auto" """x86-64: "att", "intel", or "auto" (from .att_syntax / .intel_syntax).""" - """Instruction mnemonic and operands.""" @dataclass @@ -163,14 +158,8 @@ class DirectiveParser: "mips1", "mips2", "volatile", "novolatile", "move", "nomove", } # fmt: skip - # Valid ISA values for .isa directive - VALID_ISAS = {"riscv64", "arm64", "x86_64", "mips32"} - - # Big-endian ISAs (default is little-endian) - BIG_ENDIAN_ISAS = {"mips32"} - - # ISAs whose GNU assembler aligns .half/.word/.dword automatically - AUTO_ALIGN_ISAS = {"mips32"} + # Valid ISA values for .isa directive (aliases are accepted too) + VALID_ISAS = frozenset(isa_names()) def __init__(self, isa: Optional[str] = None) -> None: """ @@ -202,19 +191,24 @@ def __init__(self, isa: Optional[str] = None) -> None: def _effective_isa(self) -> Optional[str]: return self.target_isa or self.isa + @property + def _dialect(self) -> AsmDialect: + """GNU as behaviors of the target ISA (the defaults if it is not known yet).""" + spec = find_spec(self._effective_isa or "") + return spec.asm if spec is not None else AsmDialect() + def _get_endianness(self) -> Literal["little", "big"]: """Return byte order for data directives based on ISA.""" - if self._effective_isa in self.BIG_ENDIAN_ISAS: - return "big" - return "little" + spec = find_spec(self._effective_isa or "") + return spec.byteorder if spec is not None else "little" def data_size(self, directive: str) -> Optional[int]: """Bytes per value for a data directive like .word, or None. As in GNU as, .word (and .value) is 2 bytes on x86 but 4 elsewhere. """ - if self._effective_isa == "x86_64" and directive in ("word", "value"): - return 2 + if directive in ("word", "value"): + return self._dialect.word_size return DATA_SIZES.get(directive) def alignment(self, directive: str, value: int) -> int: @@ -223,7 +217,7 @@ def alignment(self, directive: str, value: int) -> int: As in GNU as, .align takes a power of two on RISC-V, MIPS, and ARM, but a byte count on x86. """ - if directive == "balign" or (directive == "align" and self._effective_isa == "x86_64"): + if directive == "balign" or (directive == "align" and self._dialect.align_in_bytes): return value return 1 << value @@ -402,14 +396,12 @@ def _statements(self, source: str) -> List[Tuple[int, str, str]]: def _set_isa_from_line(self, line: str) -> None: """Notice '.isa' early, since comment rules depend on the ISA.""" - stripped = line.strip() - if stripped.lower().startswith(".isa") and len(stripped.split()) > 1: - value = stripped.split()[1].lower().replace("-", "_") - if value in self.VALID_ISAS: - self.isa = value + value = _isa_directive_value(line) + if value is not None: + self.isa = value def _split_line(self, line: str, in_block: bool) -> Tuple[List[str], bool]: - arm = self._effective_isa == "arm64" + hash_only_at_start = self._dialect.hash_comments_only_at_line_start pieces: List[str] = [] current: List[str] = [] i = 0 @@ -457,7 +449,7 @@ def _split_line(self, line: str, in_block: bool) -> Tuple[List[str], bool]: break if ch == "#": at_line_start = not pieces and not "".join(current).strip() - if not arm or at_line_start: + if not hash_only_at_start or at_line_start: break if ch == ";": pieces.append("".join(current)) @@ -634,14 +626,14 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: f"(one of: {', '.join(sorted(self.VALID_ISAS))})" ) return - isa_value = args[0].lower().replace("-", "_") - if isa_value not in self.VALID_ISAS: + spec = find_spec(args[0]) + if spec is None: self.errors.append( f"Line {parsed.line_number}: Invalid ISA '{args[0]}'. " f"Valid options: {', '.join(sorted(self.VALID_ISAS))}" ) return - self.isa = isa_value + self.isa = spec.name return # Section directives @@ -699,7 +691,7 @@ def _process_directive(self, parsed: ParsedLine, section: SectionData) -> None: # Like GNU as for MIPS, align .half/.word/.dword to their size, and # move labels that were waiting for this data along with it size = self.data_size(directive or "") - if size is not None and size > 1 and self._effective_isa in self.AUTO_ALIGN_ISAS: + if size is not None and size > 1 and self._dialect.auto_align_data: padding = (size - section.current_offset % size) % size if padding: section.data.extend(b"\x00" * padding) @@ -813,3 +805,39 @@ def get_instructions(self, section_name: str) -> List[Tuple[int, str, Optional[s instructions.append((line.line_number, line.instruction, line.label)) return instructions + + +def emits_data(line: ParsedLine) -> bool: + """Whether ``line`` is a data directive such as .byte or .fill.""" + directive = line.directive or "" + return line.line_type == LineType.DIRECTIVE and ( + directive in DATA_SIZES or directive in BYTE_DIRECTIVES or directive == "value" + ) + + +def _isa_directive_value(line: str) -> Optional[str]: + """The canonical ISA named by an '.isa' directive on this line, if valid.""" + parts = line.split() + if len(parts) < 2 or parts[0].lower() != ".isa": + return None + # First token only, so a trailing comment is ignored + spec = find_spec(parts[1]) + return spec.name if spec is not None else None + + +def find_isa_directive(source: str) -> Optional[str]: + """The ISA named by an '.isa' directive at the top of ``source``, if any. + + Only the leading comments and directives are searched: the directive + belongs at the start of the file. + """ + for line in source.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith(("#", "//", ";")): + continue + value = _isa_directive_value(stripped) + if value is not None: + return value + if not stripped.startswith("."): + break + return None diff --git a/mapachespim/toolchain/elf_builder.py b/mapachespim/toolchain/elf_builder.py index f5f1e6e..4f31ae3 100644 --- a/mapachespim/toolchain/elf_builder.py +++ b/mapachespim/toolchain/elf_builder.py @@ -9,9 +9,9 @@ import struct from dataclasses import dataclass, field -from typing import Dict, List, Optional +from typing import Dict, List, Tuple -from .memory_map import MemoryLayout, get_layout +from ..isa import ISASpec, get_spec # ELF constants (from pyelftools, but defined here to avoid deep imports) # ELF identification @@ -29,12 +29,6 @@ # ELF type ET_EXEC = 2 -# Machine types -EM_MIPS = 8 -EM_X86_64 = 62 -EM_AARCH64 = 183 -EM_RISCV = 243 - # Program header types PT_NULL = 0 PT_LOAD = 1 @@ -69,19 +63,6 @@ SHN_UNDEF = 0 SHN_ABS = 0xFFF1 -# Map ISA names to ELF machine types -ISA_TO_MACHINE = { - "riscv64": EM_RISCV, - "riscv": EM_RISCV, - "arm64": EM_AARCH64, - "aarch64": EM_AARCH64, - "x86_64": EM_X86_64, - "x86-64": EM_X86_64, - "x64": EM_X86_64, - "mips32": EM_MIPS, - "mips": EM_MIPS, -} - @dataclass class Section: @@ -141,6 +122,53 @@ class Symbol: """Index of section containing symbol.""" +@dataclass(frozen=True) +class _ElfClass: + """Sizes and struct layouts that differ between 32- and 64-bit ELF files.""" + + elf_class: int + ehdr: str # after e_ident + phdr: str # fields in the class's own order; see _program_header + shdr: str + sym: str + word: int # alignment of tables in the file, and the size of an address + + @property + def ehdr_size(self) -> int: + return 16 + struct.calcsize("<" + self.ehdr) + + @property + def phdr_size(self) -> int: + return struct.calcsize("<" + self.phdr) + + @property + def shdr_size(self) -> int: + return struct.calcsize("<" + self.shdr) + + @property + def sym_size(self) -> int: + return struct.calcsize("<" + self.sym) + + +_ELF64 = _ElfClass(ELFCLASS64, "HHIQQQIHHHHHH", "IIQQQQQQ", "IIQQQQIIQQ", "IBBHQQ", 8) +_ELF32 = _ElfClass(ELFCLASS32, "HHIIIIIHHHHHH", "IIIIIIII", "IIIIIIIIII", "IIIBBH", 4) + + +def _string_table(names: List[str]) -> Tuple[bytes, Dict[str, int]]: + """An ELF string table and each name's offset in it.""" + table = b"\x00" + offsets: Dict[str, int] = {"": 0} + for name in names: + if name not in offsets: + offsets[name] = len(table) + table += name.encode("ascii") + b"\x00" + return table, offsets + + +def _align(value: int, alignment: int) -> int: + return (value + alignment - 1) & ~(alignment - 1) + + @dataclass class ELFBuilder: """ @@ -165,26 +193,24 @@ class ELFBuilder: symbols: List[Symbol] = field(default_factory=list) """List of symbols.""" - _layout: Optional[MemoryLayout] = field(default=None, init=False) - """Memory layout for the ISA.""" - - def __post_init__(self) -> None: - self._layout = get_layout(self.isa) + @property + def spec(self) -> ISASpec: + return get_spec(self.isa) @property def is_64bit(self) -> bool: """Return True if this is a 64-bit ELF.""" - return self._layout.is_64bit if self._layout else True + return self.spec.layout.is_64bit @property def is_little_endian(self) -> bool: """Return True if this is a little-endian ELF.""" - return self._layout.is_little_endian if self._layout else True + return self.spec.layout.is_little_endian @property def machine(self) -> int: """Return the ELF machine type.""" - return ISA_TO_MACHINE.get(self.isa.lower(), EM_RISCV) + return self.spec.elf_machine def add_section(self, section: Section) -> None: """Add a section to the ELF.""" @@ -205,523 +231,163 @@ def add_debug_sections( Debug sections are non-loadable (no SHF_ALLOC flag), so they only get section headers, not program headers. - - Args: - debug_abbrev: .debug_abbrev section content. - debug_info: .debug_info section content. - debug_line: .debug_line section content. """ - # Debug sections: SHT_PROGBITS, no flags (not loadable), address 0 - self.sections.append( - Section( - name=".debug_abbrev", - data=debug_abbrev, - address=0, - sh_type=SHT_PROGBITS, - sh_flags=0, - alignment=1, - ) - ) - self.sections.append( - Section( - name=".debug_info", - data=debug_info, - address=0, - sh_type=SHT_PROGBITS, - sh_flags=0, - alignment=1, + for name, data in ( + (".debug_abbrev", debug_abbrev), + (".debug_info", debug_info), + (".debug_line", debug_line), + ): + self.sections.append( + Section( + name=name, data=data, address=0, sh_type=SHT_PROGBITS, sh_flags=0, alignment=1 + ) ) - ) - self.sections.append( - Section( - name=".debug_line", - data=debug_line, - address=0, - sh_type=SHT_PROGBITS, - sh_flags=0, - alignment=1, - ) - ) + + def _program_header(self, fmt: _ElfClass, endian: str, section: Section, offset: int) -> bytes: + file_size = 0 if section.sh_type == SHT_NOBITS else len(section.data) + mem_size = len(section.data) + flags = PF_R + if section.sh_flags & SHF_WRITE: + flags |= PF_W + if section.sh_flags & SHF_EXECINSTR: + flags |= PF_X + address, align = section.address, section.alignment + if fmt is _ELF64: # p_flags comes second in 64-bit program headers + fields = (PT_LOAD, flags, offset, address, address, file_size, mem_size, align) + else: + fields = (PT_LOAD, offset, address, address, file_size, mem_size, flags, align) + return struct.pack(endian + fmt.phdr, *fields) + + @staticmethod + def _symbol_entry(fmt: _ElfClass, endian: str, name: int, sym: Symbol) -> bytes: + info = (sym.binding << 4) | sym.sym_type + shndx = sym.section_index if sym.section_index else SHN_ABS + if fmt is _ELF64: + return struct.pack(endian + fmt.sym, name, info, 0, shndx, sym.address, sym.size) + return struct.pack(endian + fmt.sym, name, sym.address, sym.size, info, 0, shndx) def build(self) -> bytes: """ Build the complete ELF file. + Layout: ELF header, program headers, section contents, then the + .shstrtab, .strtab, and .symtab tables, then the section headers. + Returns: The ELF file as bytes. """ - if self.is_64bit: - return self._build_elf64() - else: - return self._build_elf32() - - def _build_elf64(self) -> bytes: - """Build a 64-bit ELF file.""" - # Endianness + fmt = _ELF64 if self.is_64bit else _ELF32 endian = "<" if self.is_little_endian else ">" + word = fmt.word - # ELF header size - ehdr_size = 64 - phdr_size = 56 - shdr_size = 64 - - # Count program headers (one per loadable section + null) - loadable_sections = [s for s in self.sections if s.sh_flags & SHF_ALLOC] - num_phdrs = len(loadable_sections) + loadable = [s for s in self.sections if s.sh_flags & SHF_ALLOC] - # Count section headers - # 0: null, 1+: sections, then .shstrtab, .strtab, .symtab - num_shdrs = 1 + len(self.sections) + 3 + # Section headers: null, the sections, then .shstrtab, .strtab, .symtab shstrtab_idx = 1 + len(self.sections) strtab_idx = shstrtab_idx + 1 - symtab_idx = strtab_idx + 1 - - # Build section name string table - shstrtab = b"\x00" - shstrtab_offsets: Dict[str, int] = {"": 0} - section_names = [s.name for s in self.sections] + [".shstrtab", ".strtab", ".symtab"] - for name in section_names: - if name not in shstrtab_offsets: - shstrtab_offsets[name] = len(shstrtab) - shstrtab += name.encode("ascii") + b"\x00" - - # Build symbol string table - strtab = b"\x00" - strtab_offsets: Dict[str, int] = {"": 0} - for sym in self.symbols: - if sym.name and sym.name not in strtab_offsets: - strtab_offsets[sym.name] = len(strtab) - strtab += sym.name.encode("ascii") + b"\x00" - - # Build symbol table - symtab = b"" - # First entry is null symbol - symtab += struct.pack(f"{endian}IBBHQQ", 0, 0, 0, 0, 0, 0) - for sym in self.symbols: - st_name = strtab_offsets.get(sym.name, 0) - st_info = (sym.binding << 4) | sym.sym_type - st_other = 0 - st_shndx = sym.section_index if sym.section_index else SHN_ABS - symtab += struct.pack( - f"{endian}IBBHQQ", st_name, st_info, st_other, st_shndx, sym.address, sym.size - ) - - # Calculate offsets - phdr_offset = ehdr_size - section_offset = ehdr_size + (num_phdrs * phdr_size) + num_shdrs = strtab_idx + 2 - # Align section offset - section_offset = (section_offset + 7) & ~7 - - # Calculate section data positions - section_file_offsets: List[int] = [] - current_offset = section_offset + shstrtab, shstrtab_offsets = _string_table( + [s.name for s in self.sections] + [".shstrtab", ".strtab", ".symtab"] + ) + # Local symbols must come before global ones + symbols = sorted(self.symbols, key=lambda sym: sym.binding != STB_LOCAL) + first_global = 1 + sum(sym.binding == STB_LOCAL for sym in symbols) + strtab, strtab_offsets = _string_table([sym.name for sym in symbols if sym.name]) + symtab = bytes(fmt.sym_size) # the null symbol + for sym in symbols: + symtab += self._symbol_entry(fmt, endian, strtab_offsets.get(sym.name, 0), sym) + + # File offsets of everything after the program headers + phdr_offset = fmt.ehdr_size + offset = _align(phdr_offset + len(loadable) * fmt.phdr_size, word) + section_offsets: List[int] = [] for section in self.sections: - # Align to section alignment - align = max(section.alignment, 1) - current_offset = (current_offset + align - 1) & ~(align - 1) - section_file_offsets.append(current_offset) + offset = _align(offset, max(section.alignment, 1)) + section_offsets.append(offset) if section.sh_type != SHT_NOBITS: - current_offset += len(section.data) - - # Add string tables and symbol table - shstrtab_offset = (current_offset + 7) & ~7 - current_offset = shstrtab_offset + len(shstrtab) - - strtab_offset = (current_offset + 7) & ~7 - current_offset = strtab_offset + len(strtab) - - symtab_offset = (current_offset + 7) & ~7 - current_offset = symtab_offset + len(symtab) - - # Section header offset - shdr_offset = (current_offset + 7) & ~7 - - # Build ELF header + offset += len(section.data) + table_offsets: List[int] = [] + for table in (shstrtab, strtab, symtab): + offset = _align(offset, word) + table_offsets.append(offset) + offset += len(table) + shdr_offset = _align(offset, word) + + # ELF header e_ident = bytearray(16) e_ident[0:4] = ELFMAG - e_ident[EI_CLASS] = ELFCLASS64 + e_ident[EI_CLASS] = fmt.elf_class e_ident[EI_DATA] = ELFDATA2LSB if self.is_little_endian else ELFDATA2MSB e_ident[EI_VERSION] = EV_CURRENT - e_ident[EI_OSABI] = 0 - e_ident[EI_ABIVERSION] = 0 - - ehdr = bytes(e_ident) + struct.pack( - f"{endian}HHIQQQIHHHHHH", + out = bytearray(e_ident) + out += struct.pack( + endian + fmt.ehdr, ET_EXEC, # e_type self.machine, # e_machine EV_CURRENT, # e_version self.entry, # e_entry phdr_offset, # e_phoff shdr_offset, # e_shoff - 0, # e_flags - ehdr_size, # e_ehsize - phdr_size, # e_phentsize - num_phdrs, # e_phnum - shdr_size, # e_shentsize + self.spec.elf_flags, # e_flags + fmt.ehdr_size, # e_ehsize + fmt.phdr_size, # e_phentsize + len(loadable), # e_phnum + fmt.shdr_size, # e_shentsize num_shdrs, # e_shnum shstrtab_idx, # e_shstrndx ) - # Build program headers - phdrs = b"" - for section in loadable_sections: - idx = self.sections.index(section) - file_offset = section_file_offsets[idx] - file_size = 0 if section.sh_type == SHT_NOBITS else len(section.data) - mem_size = len(section.data) if section.data else 0 - - flags = PF_R - if section.sh_flags & SHF_WRITE: - flags |= PF_W - if section.sh_flags & SHF_EXECINSTR: - flags |= PF_X - - phdrs += struct.pack( - f"{endian}IIQQQQQQ", - PT_LOAD, # p_type - flags, # p_flags - file_offset, # p_offset - section.address, # p_vaddr - section.address, # p_paddr - file_size, # p_filesz - mem_size, # p_memsz - section.alignment, # p_align - ) + # Program headers, one per loadable section + for section in loadable: + offset = section_offsets[self.sections.index(section)] + out += self._program_header(fmt, endian, section, offset) - # Build section data - section_data = b"" - for i, section in enumerate(self.sections): - # Pad to alignment - target_offset = section_file_offsets[i] - current_len = section_offset + len(section_data) - if current_len < target_offset: - section_data += b"\x00" * (target_offset - current_len) + # Contents + def place(data: bytes, at: int) -> None: + out.extend(bytes(at - len(out))) + out.extend(data) + + for section, at in zip(self.sections, section_offsets): if section.sh_type != SHT_NOBITS: - section_data += section.data - - # Pad and add string tables - current_len = section_offset + len(section_data) - if current_len < shstrtab_offset: - section_data += b"\x00" * (shstrtab_offset - current_len) - section_data += shstrtab - - current_len = section_offset + len(section_data) - if current_len < strtab_offset: - section_data += b"\x00" * (strtab_offset - current_len) - section_data += strtab - - current_len = section_offset + len(section_data) - if current_len < symtab_offset: - section_data += b"\x00" * (symtab_offset - current_len) - section_data += symtab - - # Pad to section header offset - current_len = section_offset + len(section_data) - if current_len < shdr_offset: - section_data += b"\x00" * (shdr_offset - current_len) - - # Build section headers - shdrs = b"" - # Null section header - shdrs += struct.pack(f"{endian}IIQQQQIIQQ", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - - # Regular sections - for i, section in enumerate(self.sections): - sh_name = shstrtab_offsets.get(section.name, 0) - sh_offset = section_file_offsets[i] - sh_size = len(section.data) if section.data else 0 - - shdrs += struct.pack( - f"{endian}IIQQQQIIQQ", - sh_name, # sh_name + place(section.data, at) + for table, at in zip((shstrtab, strtab, symtab), table_offsets): + place(table, at) + place(b"", shdr_offset) + + # Section headers + def shdr(*fields: int) -> None: + out.extend(struct.pack(endian + fmt.shdr, *fields)) + + shdr(0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + for section, at in zip(self.sections, section_offsets): + shdr( + shstrtab_offsets.get(section.name, 0), # sh_name section.sh_type, # sh_type section.sh_flags, # sh_flags section.address, # sh_addr - sh_offset, # sh_offset - sh_size, # sh_size + at, # sh_offset + len(section.data), # sh_size 0, # sh_link 0, # sh_info section.alignment, # sh_addralign 0, # sh_entsize ) - - # .shstrtab section header - shdrs += struct.pack( - f"{endian}IIQQQQIIQQ", - shstrtab_offsets[".shstrtab"], - SHT_STRTAB, - 0, - 0, - shstrtab_offset, - len(shstrtab), - 0, - 0, - 1, - 0, - ) - - # .strtab section header - shdrs += struct.pack( - f"{endian}IIQQQQIIQQ", - shstrtab_offsets[".strtab"], - SHT_STRTAB, - 0, - 0, - strtab_offset, - len(strtab), - 0, - 0, - 1, - 0, - ) - - # .symtab section header - num_symbols = 1 + len(self.symbols) - shdrs += struct.pack( - f"{endian}IIQQQQIIQQ", - shstrtab_offsets[".symtab"], - SHT_SYMTAB, - 0, - 0, - symtab_offset, - len(symtab), - strtab_idx, # sh_link = string table index - 1, # sh_info = first global symbol - 8, - 24, # entry size for 64-bit symbols - ) - - return ehdr + phdrs + section_data + shdrs - - def _build_elf32(self) -> bytes: - """Build a 32-bit ELF file (for MIPS32).""" - endian = "<" if self.is_little_endian else ">" - - # ELF header size - ehdr_size = 52 - phdr_size = 32 - shdr_size = 40 - - # Count program headers - loadable_sections = [s for s in self.sections if s.sh_flags & SHF_ALLOC] - num_phdrs = len(loadable_sections) - - # Count section headers - num_shdrs = 1 + len(self.sections) + 3 - shstrtab_idx = 1 + len(self.sections) - strtab_idx = shstrtab_idx + 1 - symtab_idx = strtab_idx + 1 - - # Build section name string table - shstrtab = b"\x00" - shstrtab_offsets: Dict[str, int] = {"": 0} - section_names = [s.name for s in self.sections] + [".shstrtab", ".strtab", ".symtab"] - for name in section_names: - if name not in shstrtab_offsets: - shstrtab_offsets[name] = len(shstrtab) - shstrtab += name.encode("ascii") + b"\x00" - - # Build symbol string table - strtab = b"\x00" - strtab_offsets: Dict[str, int] = {"": 0} - for sym in self.symbols: - if sym.name and sym.name not in strtab_offsets: - strtab_offsets[sym.name] = len(strtab) - strtab += sym.name.encode("ascii") + b"\x00" - - # Build symbol table (32-bit format) - symtab = b"" - # First entry is null symbol - symtab += struct.pack(f"{endian}IIIBBH", 0, 0, 0, 0, 0, 0) - for sym in self.symbols: - st_name = strtab_offsets.get(sym.name, 0) - st_info = (sym.binding << 4) | sym.sym_type - st_other = 0 - st_shndx = sym.section_index if sym.section_index else SHN_ABS - symtab += struct.pack( - f"{endian}IIIBBH", st_name, sym.address, sym.size, st_info, st_other, st_shndx - ) - - # Calculate offsets - phdr_offset = ehdr_size - section_offset = ehdr_size + (num_phdrs * phdr_size) - section_offset = (section_offset + 3) & ~3 - - # Calculate section data positions - section_file_offsets: List[int] = [] - current_offset = section_offset - for section in self.sections: - align = max(section.alignment, 1) - current_offset = (current_offset + align - 1) & ~(align - 1) - section_file_offsets.append(current_offset) - if section.sh_type != SHT_NOBITS: - current_offset += len(section.data) - - # Add string tables and symbol table - shstrtab_offset = (current_offset + 3) & ~3 - current_offset = shstrtab_offset + len(shstrtab) - - strtab_offset = (current_offset + 3) & ~3 - current_offset = strtab_offset + len(strtab) - - symtab_offset = (current_offset + 3) & ~3 - current_offset = symtab_offset + len(symtab) - - shdr_offset = (current_offset + 3) & ~3 - - # Build ELF header - e_ident = bytearray(16) - e_ident[0:4] = ELFMAG - e_ident[EI_CLASS] = ELFCLASS32 - e_ident[EI_DATA] = ELFDATA2LSB if self.is_little_endian else ELFDATA2MSB - e_ident[EI_VERSION] = EV_CURRENT - - # For MIPS, set flags (e.g., MIPS32 ABI) - e_flags = 0 - if self.machine == EM_MIPS: - e_flags = 0x50001000 # MIPS32R2, O32 ABI - - ehdr = bytes(e_ident) + struct.pack( - f"{endian}HHIIIIIHHHHHH", - ET_EXEC, - self.machine, - EV_CURRENT, - self.entry, - phdr_offset, - shdr_offset, - e_flags, - ehdr_size, - phdr_size, - num_phdrs, - shdr_size, - num_shdrs, - shstrtab_idx, - ) - - # Build program headers - phdrs = b"" - for section in loadable_sections: - idx = self.sections.index(section) - file_offset = section_file_offsets[idx] - file_size = 0 if section.sh_type == SHT_NOBITS else len(section.data) - mem_size = len(section.data) if section.data else 0 - - flags = PF_R - if section.sh_flags & SHF_WRITE: - flags |= PF_W - if section.sh_flags & SHF_EXECINSTR: - flags |= PF_X - - phdrs += struct.pack( - f"{endian}IIIIIIII", - PT_LOAD, - file_offset, - section.address, - section.address, - file_size, - mem_size, - flags, - section.alignment, - ) - - # Build section data - section_data = b"" - for i, section in enumerate(self.sections): - target_offset = section_file_offsets[i] - current_len = section_offset + len(section_data) - if current_len < target_offset: - section_data += b"\x00" * (target_offset - current_len) - if section.sh_type != SHT_NOBITS: - section_data += section.data - - # Pad and add tables - current_len = section_offset + len(section_data) - if current_len < shstrtab_offset: - section_data += b"\x00" * (shstrtab_offset - current_len) - section_data += shstrtab - - current_len = section_offset + len(section_data) - if current_len < strtab_offset: - section_data += b"\x00" * (strtab_offset - current_len) - section_data += strtab - - current_len = section_offset + len(section_data) - if current_len < symtab_offset: - section_data += b"\x00" * (symtab_offset - current_len) - section_data += symtab - - current_len = section_offset + len(section_data) - if current_len < shdr_offset: - section_data += b"\x00" * (shdr_offset - current_len) - - # Build section headers - shdrs = b"" - # Null section header - shdrs += struct.pack(f"{endian}IIIIIIIIII", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - - for i, section in enumerate(self.sections): - sh_name = shstrtab_offsets.get(section.name, 0) - sh_offset = section_file_offsets[i] - sh_size = len(section.data) if section.data else 0 - - shdrs += struct.pack( - f"{endian}IIIIIIIIII", - sh_name, - section.sh_type, - section.sh_flags, - section.address, - sh_offset, - sh_size, - 0, - 0, - section.alignment, - 0, - ) - - # .shstrtab - shdrs += struct.pack( - f"{endian}IIIIIIIIII", - shstrtab_offsets[".shstrtab"], - SHT_STRTAB, - 0, - 0, - shstrtab_offset, - len(shstrtab), - 0, - 0, - 1, - 0, - ) - - # .strtab - shdrs += struct.pack( - f"{endian}IIIIIIIIII", - shstrtab_offsets[".strtab"], - SHT_STRTAB, - 0, - 0, - strtab_offset, - len(strtab), - 0, - 0, - 1, - 0, + shstrtab_at, strtab_at, symtab_at = table_offsets + shdr( + shstrtab_offsets[".shstrtab"], SHT_STRTAB, 0, 0, shstrtab_at, len(shstrtab), 0, 0, 1, 0 ) - - # .symtab - shdrs += struct.pack( - f"{endian}IIIIIIIIII", + shdr(shstrtab_offsets[".strtab"], SHT_STRTAB, 0, 0, strtab_at, len(strtab), 0, 0, 1, 0) + shdr( shstrtab_offsets[".symtab"], SHT_SYMTAB, 0, 0, - symtab_offset, + symtab_at, len(symtab), - strtab_idx, - 1, - 4, - 16, + strtab_idx, # sh_link: the string table + first_global, # sh_info: index of the first non-local symbol + word, # sh_addralign + fmt.sym_size, # sh_entsize ) - - return ehdr + phdrs + section_data + shdrs + return bytes(out) diff --git a/mapachespim/toolchain/memory_map.py b/mapachespim/toolchain/memory_map.py deleted file mode 100644 index 3e5c3da..0000000 --- a/mapachespim/toolchain/memory_map.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Re-export of :mod:`mapachespim.memory_map`. - -The memory layout is shared by the assembler and the simulator, so it lives -at the package top level. This module is kept for backward compatibility. -""" - -from ..memory_map import ( - ARM64_LAYOUT, - MEMORY_LAYOUTS, - MIPS32_LAYOUT, - RISCV64_LAYOUT, - X86_64_LAYOUT, - MemoryLayout, - get_layout, -) - -__all__ = [ - "ARM64_LAYOUT", - "MEMORY_LAYOUTS", - "MIPS32_LAYOUT", - "RISCV64_LAYOUT", - "X86_64_LAYOUT", - "MemoryLayout", - "get_layout", -] diff --git a/mapachespim/toolchain/targets.py b/mapachespim/toolchain/targets.py new file mode 100644 index 0000000..e2f5d37 --- /dev/null +++ b/mapachespim/toolchain/targets.py @@ -0,0 +1,275 @@ +""" +Assembler targets: how the ISA-independent assembler drives each ISA's encoder. + +Every encoder module (riscv, mips, arm64, x86) provides the same two +functions:: + + encode(text, address, labels, **options) -> bytes + instruction_size(text, labels, address, **options) -> int + +A :class:`Target` wraps an encoder and adds the ISA's GNU as behaviors that +the assembler needs: per-line encoder options, the nops used to pad code, +and hooks for ARM64 literal pools and x86 jump relaxation. A new Target is +made for each assembly, so targets may keep per-program state. + +To add an ISA, write its encoder module and a Target subclass here, and add +it to ``TARGETS``. +""" + +from __future__ import annotations + +from types import ModuleType +from typing import Any, Dict, List, Mapping, Set, Tuple, Type + +from ..isa import ISASpec, get_spec +from . import arm64, mips, riscv, x86 +from .directives import ( + ALIGN_DIRECTIVES, + POOL_LABEL_PREFIX, + DirectiveParser, + LineType, + ParsedLine, + SectionData, + emits_data, +) +from .expr import EncodeError + + +class Target: + """Default behavior: a fixed-width ISA whose code padding is zero bytes.""" + + isa_name: str + encoder: ModuleType + + def __init__(self, parser: DirectiveParser) -> None: + self.parser = parser + + @property + def spec(self) -> ISASpec: + return get_spec(self.isa_name) + + def prepare(self, sections: Dict[str, SectionData]) -> None: + """Rewrite the parsed program before layout (e.g. add literal pools). + + Raises: + EncodeError: For an error in the program. + """ + + def options(self, line: ParsedLine) -> Dict[str, Any]: + """Extra keyword arguments for the encoder for this line.""" + return {} + + def encode(self, line: ParsedLine, address: int, labels: Mapping[str, int]) -> bytes: + return self.encoder.encode(line.instruction, address, labels, **self.options(line)) + + def size(self, line: ParsedLine, address: int, labels: Mapping[str, int]) -> int: + return self.encoder.instruction_size( + line.instruction, labels, address, **self.options(line) + ) + + def padding_before_instruction(self, address: int, after_data: bool) -> int: + """Zero bytes GNU as puts before an instruction at ``address``.""" + return 0 + + def is_explicit_fill(self, fill: int) -> bool: + """Whether an alignment fill byte replaces the default code padding.""" + return True + + def code_padding(self, count: int, line: ParsedLine) -> bytes: + """``count`` bytes of padding for an alignment directive in code.""" + return bytes(count) + + can_relax = False + """True if instruction sizes can grow during layout (see :meth:`relax`).""" + + def relax(self, line: ParsedLine, address: int, labels: Mapping[str, int]) -> bool: + """Grow the instruction at ``line`` if it cannot reach its target. + + Returns True if it grew. + """ + return False + + +class RISCVTarget(Target): + isa_name = "riscv64" + encoder = riscv + + def code_padding(self, count: int, line: ParsedLine) -> bytes: + # As GNU as's riscv_make_nops: a zero byte if odd, at most one + # 2-byte c.nop, then 4-byte nops + head = bytes(count % 2) + (b"\x01\x00" if count % 4 >= 2 else b"") + return head + (0x00000013).to_bytes(4, "little") * (count // 4) + + +class MIPSTarget(Target): + isa_name = "mips32" + encoder = mips + # The MIPS nop is all zeros, so the default padding is right + + def options(self, line: ParsedLine) -> Dict[str, Any]: + return {"reorder": line.reorder} # .set reorder / noreorder + + +class ARM64Target(Target): + isa_name = "arm64" + encoder = arm64 + + def prepare(self, sections: Dict[str, SectionData]) -> None: + if ".text" in sections: + add_literal_pools(sections[".text"]) + + def padding_before_instruction(self, address: int, after_data: bool) -> int: + # GNU as aligns an instruction to 4 bytes when it directly follows + # data (when switching from data to code; an alignment directive in + # between counts as code) + return -address % 4 if after_data else 0 + + def code_padding(self, count: int, line: ParsedLine) -> bytes: + return bytes(count % 4) + (0xD503201F).to_bytes(4, "little") * (count // 4) + + +class X86Target(Target): + isa_name = "x86_64" + encoder = x86 + can_relax = True + + def __init__(self, parser: DirectiveParser) -> None: + super().__init__(parser) + self._absolute_symbols: Set[str] = set() # names of .equ constants + self._long_jumps: Set[int] = set() # jumps that need rel32 + self._align_after_data: Set[int] = set() # see _alignments_after_data + + def prepare(self, sections: Dict[str, SectionData]) -> None: + self._absolute_symbols = set(self.parser.constants) | set(self.parser.deferred_constants) + self._align_after_data = _alignments_after_data(sections.get(".text")) + + def options(self, line: ParsedLine) -> Dict[str, Any]: + return { + "syntax": line.syntax, + "absolute_symbols": self._absolute_symbols, + "long_jump": id(line) in self._long_jumps, + } + + def is_explicit_fill(self, fill: int) -> bool: + # GNU as treats a fill of 0x90 (nop) like no fill: best nops + return fill & 0xFF != 0x90 + + def code_padding(self, count: int, line: ParsedLine) -> bytes: + return x86.nop_padding(count, id(line) in self._align_after_data) + + def relax(self, line: ParsedLine, address: int, labels: Mapping[str, int]) -> bool: + # Jumps start short (rel8) and grow to rel32 once, for good, when the + # target is out of range (GNU's relaxation) + text = line.instruction or "" + if ( + id(line) in self._long_jumps + or not x86.is_relaxable(text) + or x86.short_jump_fits(text, address, labels) + ): + return False + self._long_jumps.add(id(line)) + return True + + +def _alignments_after_data(section: Any) -> Set[int]: + """Alignment directives in code whose last preceding item is data. + + GNU as pads x86 code with a one-byte nop first in that case, since the + data might be an incomplete instruction. + """ + found: Set[int] = set() + after_data = False + for line in section.lines if section else []: + if line.line_type == LineType.INSTRUCTION and line.instruction: + after_data = False + elif line.line_type == LineType.DIRECTIVE: + if line.directive in ALIGN_DIRECTIVES: + if after_data: + found.add(id(line)) + elif emits_data(line): + after_data = True + return found + + +def add_literal_pools(section: SectionData) -> None: + """Turn ARM64 'ldr x0, =value' into loads from a literal pool. + + As in GNU as, each load reads a pool entry placed at the next + .ltorg/.pool directive or at the end of the section. Within a pool, + equal values share one entry; 4-byte entries come first, then 8-byte + entries, each group aligned to its size. + + Raises: + EncodeError: If a literal load is invalid. + """ + lines: List[ParsedLine] = [] + pool: Dict[Tuple[object, ...], Tuple[str, str, int]] = {} # key -> (label, expr, size) + counter = 0 + + def dump(line_number: int) -> None: + for size in (4, 8): + entries = [(label, expr) for label, expr, sz in pool.values() if sz == size] + if not entries: + continue + lines.append( + ParsedLine( + line_number, + LineType.DIRECTIVE, + "", + directive="balign", + directive_args=[str(size), "0"], # zeros, not nops + ) + ) + for label, expr in entries: + lines.append( + ParsedLine( + line_number, + LineType.DIRECTIVE, + "", + label=label, + directive="word" if size == 4 else "dword", + directive_args=[expr], + ) + ) + pool.clear() + + last_line = 0 + for line in section.lines: + last_line = line.line_number + if line.line_type == LineType.INSTRUCTION and line.instruction: + found = arm64.literal_load(line.instruction) + if found is not None: + mnemonic, rt, expr = found + try: + size = arm64.literal_size(mnemonic, rt) + except EncodeError as e: + raise EncodeError(f"Line {line.line_number}: {e}") + value = arm64.constant_value(expr) + key: Tuple[object, ...] = ( + ("value", value & ((1 << (8 * size)) - 1), size) + if value is not None + else ("expr", expr.replace(" ", ""), size) + ) + if key not in pool: + counter += 1 + pool[key] = (f"{POOL_LABEL_PREFIX}{counter}", expr, size) + line.instruction = f"{mnemonic} {rt}, {pool[key][0]}" + lines.append(line) + if line.line_type == LineType.DIRECTIVE and line.directive in ("ltorg", "pool"): + dump(line.line_number) + dump(last_line) + section.lines = lines + + +TARGETS: Dict[str, Type[Target]] = { + cls.isa_name: cls for cls in (RISCVTarget, MIPSTarget, ARM64Target, X86Target) +} + + +def target_for(isa: str, parser: DirectiveParser) -> Target: + """A new Target for an ISA (name or alias).""" + spec = get_spec(isa) + try: + return TARGETS[spec.name](parser) + except KeyError: + raise ValueError(f"No assembler for {spec.display_name}") diff --git a/mapachespim/unicorn_backend.py b/mapachespim/unicorn_backend.py index e1f5cab..928cba8 100644 --- a/mapachespim/unicorn_backend.py +++ b/mapachespim/unicorn_backend.py @@ -1,99 +1,51 @@ """ -Python bindings for Unicorn Engine-based simulator +The simulator: runs programs for any registered ISA on the Unicorn Engine. -Provides a Pythonic interface to the Unicorn CPU emulator for RISC-V, ARM64, -x86-64, and MIPS32. +ISA-specific facts come from :mod:`mapachespim.isa` (registers, syscall ABI, +memory layout, ...) and :mod:`mapachespim.engines` (Unicorn and Capstone +constants), so nothing here branches on the ISA. Syscalls are performed by +:mod:`mapachespim.syscalls`. """ from __future__ import annotations import ctypes -import struct -import sys -from enum import IntEnum -from typing import TYPE_CHECKING, Dict, List, Optional, TextIO, Tuple, Union - -if TYPE_CHECKING: - from unicorn import Uc +from dataclasses import dataclass +from enum import Enum, IntEnum +from typing import Callable, Dict, List, Optional, TextIO, Tuple, Union try: from unicorn import ( - UC_ARCH_ARM64, - UC_ARCH_MIPS, - UC_ARCH_RISCV, - UC_ARCH_X86, - UC_HOOK_CODE, UC_HOOK_MEM_UNMAPPED, UC_MEM_FETCH_UNMAPPED, UC_MEM_READ_UNMAPPED, UC_MEM_WRITE_UNMAPPED, - UC_MODE_64, - UC_MODE_ARM, - UC_MODE_BIG_ENDIAN, - UC_MODE_MIPS32, - UC_MODE_RISCV64, UC_PROT_ALL, Uc, UcError, ) - from unicorn.arm64_const import UC_ARM64_REG_PC, UC_ARM64_REG_SP, UC_ARM64_REG_X0 - from unicorn.mips_const import UC_MIPS_REG_0, UC_MIPS_REG_PC, UC_MIPS_REG_SP - from unicorn.riscv_const import UC_RISCV_REG_PC, UC_RISCV_REG_SP, UC_RISCV_REG_X0 - from unicorn.x86_const import ( - UC_X86_REG_R8, - UC_X86_REG_R9, - UC_X86_REG_R10, - UC_X86_REG_R11, - UC_X86_REG_R12, - UC_X86_REG_R13, - UC_X86_REG_R14, - UC_X86_REG_R15, - UC_X86_REG_RAX, - UC_X86_REG_RBP, - UC_X86_REG_RBX, - UC_X86_REG_RCX, - UC_X86_REG_RDI, - UC_X86_REG_RDX, - UC_X86_REG_RIP, - UC_X86_REG_RSI, - UC_X86_REG_RSP, - ) except ImportError as e: raise ImportError( f"Unicorn Engine not installed. Install with: pip install unicorn\nOriginal error: {e}" ) -try: - from capstone import ( - CS_ARCH_ARM64, - CS_ARCH_MIPS, - CS_ARCH_RISCV, - CS_ARCH_X86, - CS_MODE_ARM, - CS_MODE_BIG_ENDIAN, - CS_MODE_MIPS32, - CS_MODE_RISCV64, - Cs, - ) - from capstone import CS_MODE_64 as CS_MODE_X86_64 -except ImportError as e: - raise ImportError( - f"Capstone not installed. Install with: pip install capstone\nOriginal error: {e}" - ) - -from .elf_loader import ISA as ELF_ISA -from .elf_loader import ELFSegment, load_elf_file -from .memory_map import MemoryLayout, get_layout - - -class ISA(IntEnum): - """ISA types supported by the simulator""" - - RISCV = 0 - ARM = 1 - X86_64 = 2 - MIPS = 3 - UNKNOWN = -1 +from .disassembler import Disassembler +from .elf_loader import ELFSection, ELFSegment, load_elf_file +from .engines import EngineBinding, engine_for +from .isa import ISA, ISASpec, get_spec +from .memory_map import MemoryLayout +from .symbols import SymbolTable +from .syscalls import ProgramIO, SyscallHandler + +__all__ = [ + "ISA", + "RunResult", + "StepResult", + "StopReason", + "UnicornSimulator", + "create_simulator", + "detect_elf_isa", +] class StepResult(IntEnum): @@ -106,51 +58,51 @@ class StepResult(IntEnum): ERROR = -1 -# RISC-V memory layout constants (kept for backward compatibility; see memory_map) -STACK_TOP = 0x83F00000 -STACK_SIZE = 0x100000 # 1MB stack -STACK_ADDR = STACK_TOP - STACK_SIZE # Bottom of stack +class StopReason(str, Enum): + """Why execution stopped. Compares equal to its string value (e.g. "halt").""" -PAGE_SIZE = 0x1000 + EXIT = "syscall_exit" + """The program made an exit syscall (see ``exit_code``).""" -_UNMAPPED_ACCESS_KINDS = { - UC_MEM_READ_UNMAPPED: "Read from", - UC_MEM_WRITE_UNMAPPED: "Write to", - UC_MEM_FETCH_UNMAPPED: "Instruction fetch from", -} + HALT = "halt" + """The program had already exited.""" + + ERROR = "error" + """An instruction or syscall failed (see ``last_error``).""" + + TOHOST = "tohost" + """The program wrote to the RISC-V HTIF ``tohost`` symbol.""" + + BREAKPOINT = "breakpoint" + INTERRUPTED = "interrupted" + + +@dataclass(frozen=True) +class RunResult: + """Outcome of :meth:`UnicornSimulator.run_until`.""" + + steps: int + """Instructions executed.""" + + reason: Optional[StopReason] + """Why execution stopped, or None if the step limit was reached.""" + + pc: int + """Address of the last instruction executed (or attempted).""" + + +PAGE_SIZE = 0x1000 # Size of the region mapped at the start of each ISA's text/data areas so that # small hand-written programs work without an ELF file. DEFAULT_REGION_SIZE = 0x400000 -_ISA_LAYOUT_NAMES = { - 0: "riscv64", # ISA.RISCV - 1: "arm64", # ISA.ARM - 2: "x86_64", # ISA.X86_64 - 3: "mips32", # ISA.MIPS +_UNMAPPED_ACCESS_KINDS = { + UC_MEM_READ_UNMAPPED: "Read from", + UC_MEM_WRITE_UNMAPPED: "Write to", + UC_MEM_FETCH_UNMAPPED: "Instruction fetch from", } -# SPIM-compatible syscall numbers -SYSCALL_PRINT_INT = 1 -SYSCALL_PRINT_STRING = 4 -SYSCALL_READ_INT = 5 -SYSCALL_EXIT = 10 -SYSCALL_PRINT_CHAR = 11 -SYSCALL_READ_CHAR = 12 -SYSCALL_EXIT2 = 17 -SYSCALL_EXIT_CODE = 93 - -SUPPORTED_SYSCALLS = ( - SYSCALL_PRINT_INT, - SYSCALL_PRINT_STRING, - SYSCALL_READ_INT, - SYSCALL_EXIT, - SYSCALL_PRINT_CHAR, - SYSCALL_READ_CHAR, - SYSCALL_EXIT2, - SYSCALL_EXIT_CODE, -) - def _describe_uc_error(e: UcError) -> str: """Turn a Unicorn error into a message a student can act on""" @@ -166,366 +118,9 @@ def _describe_uc_error(e: UcError) -> str: return text -class ISAConfig: - """Base class for ISA-specific configuration""" - - arch: int - mode: int - - def __init__(self, arch: int, mode: int) -> None: - self.arch = arch - self.mode = mode - - def get_pc_reg(self) -> int: - """Get the program counter register constant""" - raise NotImplementedError - - def get_sp_reg(self) -> int: - """Get the stack pointer register constant""" - raise NotImplementedError - - def get_gpr_reg(self, n: int) -> int: - """Get general purpose register constant for register n (0-31)""" - raise NotImplementedError - - def get_reg_name(self, n: int) -> str: - """Get register name for register n""" - raise NotImplementedError - - def detect_syscall(self, uc: Uc, pc: int) -> bool: - """Detect if instruction at PC is a syscall""" - raise NotImplementedError - - -class RISCVConfig(ISAConfig): - """RISC-V 64-bit configuration""" - - def __init__(self) -> None: - super().__init__(UC_ARCH_RISCV, UC_MODE_RISCV64) - - def get_pc_reg(self) -> int: - return UC_RISCV_REG_PC - - def get_sp_reg(self) -> int: - return UC_RISCV_REG_SP - - def get_gpr_reg(self, n: int) -> int: - """Get RISC-V register constant for x0-x31""" - if not 0 <= n <= 31: - raise ValueError(f"Register number must be 0-31, got {n}") - return UC_RISCV_REG_X0 + n - - def get_reg_name(self, n: int) -> str: - """Get RISC-V register ABI name""" - abi_names = [ - "zero", - "ra", - "sp", - "gp", - "tp", - "t0", - "t1", - "t2", - "s0", - "s1", - "a0", - "a1", - "a2", - "a3", - "a4", - "a5", - "a6", - "a7", - "s2", - "s3", - "s4", - "s5", - "s6", - "s7", - "s8", - "s9", - "s10", - "s11", - "t3", - "t4", - "t5", - "t6", - ] - return abi_names[n] if 0 <= n <= 31 else f"x{n}" - - def detect_syscall(self, uc: Uc, pc: int) -> bool: - """Check if instruction at PC is 'ecall' (0x00000073)""" - try: - instr_bytes = uc.mem_read(pc, 4) - instr = struct.unpack(" None: - super().__init__(UC_ARCH_ARM64, UC_MODE_ARM) - - def get_pc_reg(self) -> int: - return UC_ARM64_REG_PC - - def get_sp_reg(self) -> int: - return UC_ARM64_REG_SP - - def get_gpr_reg(self, n: int) -> int: - """Get ARM64 register constant for x0-x30, sp""" - if not 0 <= n <= 31: - raise ValueError(f"Register number must be 0-31, got {n}") - if n == 31: - return UC_ARM64_REG_SP - return UC_ARM64_REG_X0 + n - - def get_reg_name(self, n: int) -> str: - """Get ARM64 register name""" - if n == 31: - return "sp" - return f"x{n}" if 0 <= n <= 30 else f"?{n}" - - def detect_syscall(self, uc: Uc, pc: int) -> bool: - """Check if instruction at PC is 'svc #0' or similar""" - try: - instr_bytes = uc.mem_read(pc, 4) - instr = struct.unpack(" None: - super().__init__(UC_ARCH_X86, UC_MODE_64) - - def get_pc_reg(self) -> int: - return UC_X86_REG_RIP - - def get_sp_reg(self) -> int: - return UC_X86_REG_RSP - - def get_gpr_reg(self, n: int) -> int: - """Get x86-64 register constant for register n (0-15)""" - if not 0 <= n <= 15: - raise ValueError(f"Register number must be 0-15 for x86-64, got {n}") - return self._GPR_REGS[n] - - def get_reg_name(self, n: int) -> str: - """Get x86-64 register name""" - if 0 <= n <= 15: - return self._GPR_NAMES[n] - return f"?{n}" - - def detect_syscall(self, uc: Uc, pc: int) -> bool: - """Check if instruction at PC is 'syscall' (0x0F 0x05)""" - try: - instr_bytes = uc.mem_read(pc, 2) - return instr_bytes[0] == 0x0F and instr_bytes[1] == 0x05 - except Exception: - return False - - -class MIPSConfig(ISAConfig): - """MIPS32 configuration""" - - # MIPS register names in standard order ($0-$31) - _GPR_NAMES: List[str] = [ - "zero", - "at", - "v0", - "v1", # $0-$3 - "a0", - "a1", - "a2", - "a3", # $4-$7 - "t0", - "t1", - "t2", - "t3", # $8-$11 - "t4", - "t5", - "t6", - "t7", # $12-$15 - "s0", - "s1", - "s2", - "s3", # $16-$19 - "s4", - "s5", - "s6", - "s7", # $20-$23 - "t8", - "t9", # $24-$25 - "k0", - "k1", # $26-$27 - "gp", - "sp", - "fp", - "ra", # $28-$31 - ] - - def __init__(self) -> None: - # MIPS32 big-endian mode - super().__init__(UC_ARCH_MIPS, UC_MODE_MIPS32 | UC_MODE_BIG_ENDIAN) - - def get_pc_reg(self) -> int: - return UC_MIPS_REG_PC - - def get_sp_reg(self) -> int: - return UC_MIPS_REG_SP - - def get_gpr_reg(self, n: int) -> int: - """Get MIPS register constant for $0-$31""" - if not 0 <= n <= 31: - raise ValueError(f"Register number must be 0-31, got {n}") - return UC_MIPS_REG_0 + n - - def get_reg_name(self, n: int) -> str: - """Get MIPS register name""" - if 0 <= n <= 31: - return self._GPR_NAMES[n] - return f"?{n}" - - def detect_syscall(self, uc: Uc, pc: int) -> bool: - """Check if instruction at PC is 'syscall' (0x0000000C)""" - try: - instr_bytes = uc.mem_read(pc, 4) - # MIPS is big-endian in our configuration - instr = struct.unpack(">I", instr_bytes)[0] - return instr == 0x0000000C # syscall instruction - except Exception: - return False - - -class Disassembler: - """Capstone-based disassembler for RISC-V, ARM64, x86-64, and MIPS""" - - _cs: Cs - _isa: ISA - - def __init__(self, isa: ISA) -> None: - if isa == ISA.RISCV: - self._cs = Cs(CS_ARCH_RISCV, CS_MODE_RISCV64) - elif isa == ISA.ARM: - self._cs = Cs(CS_ARCH_ARM64, CS_MODE_ARM) - elif isa == ISA.X86_64: - self._cs = Cs(CS_ARCH_X86, CS_MODE_X86_64) - elif isa == ISA.MIPS: - self._cs = Cs(CS_ARCH_MIPS, CS_MODE_MIPS32 | CS_MODE_BIG_ENDIAN) - else: - raise ValueError(f"Unsupported ISA for disassembly: {isa}") - self._isa = isa - - self._cs.detail = False # We don't need detailed instruction info - - def disassemble_one(self, simulator: UnicornSimulator, addr: int) -> str: - """ - Disassemble single instruction at address - - Args: - simulator: UnicornSimulator instance (to read memory) - addr: Address of instruction - - Returns: - str: Disassembled instruction string - """ - try: - # Read bytes - x86 instructions can be up to 15 bytes, RISC-V/ARM are 4 - read_size = 15 if self._isa == ISA.X86_64 else 4 - code = simulator.read_mem(addr, read_size) - except Exception: - return "" - - # Try to disassemble - for instr in self._cs.disasm(code, addr, count=1): - return f"{instr.mnemonic} {instr.op_str}".strip() - - # If disassembly failed, show raw bytes - word = int.from_bytes(code[:4], byteorder="big" if self._isa == ISA.MIPS else "little") - return f".word 0x{word:08x}" - - def disassemble_with_size(self, simulator: UnicornSimulator, addr: int) -> Tuple[str, int]: - """ - Disassemble one instruction and report its length in bytes. - - x86-64 instructions are variable length, so callers that walk a range - of instructions must use the returned size to find the next one. - - Returns: - (text, size) - size is 4 for undecodable fixed-width instructions - and 1 for undecodable x86 bytes. - """ - read_size = 15 if self._isa == ISA.X86_64 else 4 - code = b"" - # Near the end of a mapped region fewer than 15 bytes may be readable - while read_size > 0: - try: - code = simulator.read_mem(addr, read_size) - break - except Exception: - read_size -= 1 - if not code: - return ("", 4) - - for instr in self._cs.disasm(code, addr, count=1): - return (f"{instr.mnemonic} {instr.op_str}".strip(), instr.size) - - if self._isa == ISA.X86_64: - return (f".byte 0x{code[0]:02x}", 1) - word = int.from_bytes(code[:4], byteorder="big" if self._isa == ISA.MIPS else "little") - return (f".word 0x{word:08x}", 4) - - class UnicornSimulator: """ - Unicorn Engine-based CPU emulator for RISC-V, ARM64, x86-64, and MIPS32 + Unicorn Engine-based CPU emulator for every ISA in :mod:`mapachespim.isa` Provides step-by-step execution, register/memory access, and state inspection. @@ -534,25 +129,18 @@ class UnicornSimulator: and similar redirection work) and can be replaced, e.g. for autograding. """ - stdout: Optional[TextIO] - stdin: Optional[TextIO] - - _isa: Optional[ISA] - _config: Optional[ISAConfig] + _spec: Optional[ISASpec] + _engine: Optional[EngineBinding] _uc: Optional[Uc] - _symbols: Dict[str, int] - _addr_to_symbol: Dict[int, Tuple[str, int]] _disasm: Optional[Disassembler] + _symbols: SymbolTable + _sections: List[ELFSection] _entry_point: Optional[int] - _pending_syscall: bool _last_error: Optional[str] _elf_path: Optional[str] _explicit_isa: bool _mem_buffers: List[ctypes.Array] - _exited: bool _exit_code: Optional[int] - _output_needs_newline: bool - _input_buffer: str _tohost_addr: Optional[int] def __init__(self, isa: Optional[ISA] = None, config_file: Optional[str] = None) -> None: @@ -560,116 +148,69 @@ def __init__(self, isa: Optional[ISA] = None, config_file: Optional[str] = None) Initialize the simulator Args: - isa (ISA, optional): ISA to use (ISA.RISCV, ISA.ARM, ISA.X86_64, ISA.MIPS). - If None, will be auto-detected from ELF file during load_elf() - config_file (str, optional): Not used by Unicorn backend (for compatibility) + isa (ISA, optional): ISA to use (e.g. ISA.RISCV). If None, it is + detected from each ELF file passed to load_elf(). + config_file (str, optional): Not used (kept for compatibility) """ - self._isa = isa self._explicit_isa = isa is not None - self._config = None + self._spec = None + self._engine = None self._uc = None - self._symbols = {} - self._addr_to_symbol = {} self._disasm = None + self._symbols = SymbolTable() + self._sections = [] self._entry_point = None - self._pending_syscall = False self._last_error = None self._elf_path = None self._mem_buffers = [] - self._exited = False self._exit_code = None - self._output_needs_newline = False - self._input_buffer = "" self._tohost_addr = None - self.stdout = None - self.stdin = None + self._io = ProgramIO() + self._syscalls = SyscallHandler(self._io) - # If ISA is specified, initialize Unicorn now if isa is not None: - self._init_unicorn(isa) - - def _init_unicorn(self, isa: ISA) -> None: - """Initialize Unicorn engine with ISA-specific configuration""" - if isa == ISA.RISCV: - self._config = RISCVConfig() - elif isa == ISA.ARM: - self._config = ARM64Config() - elif isa == ISA.X86_64: - self._config = X86_64Config() - elif isa == ISA.MIPS: - self._config = MIPSConfig() - else: - raise ValueError(f"Unsupported ISA: {isa}") + self._init_machine(get_spec(isa)) + + # --- Machine setup --- - # Create Unicorn instance + def _init_machine(self, spec: ISASpec) -> None: + """Create a fresh machine for ``spec`` with the default memory map""" + engine = engine_for(spec) try: - self._uc = Uc(self._config.arch, self._config.mode) + uc = Uc(engine.uc_arch, engine.uc_mode) except UcError as e: raise RuntimeError(f"Failed to initialize Unicorn: {e}") - self._isa = isa + self._spec, self._engine, self._uc = spec, engine, uc + self._disasm = Disassembler(spec) self._mem_buffers = [] - self._exited = False self._exit_code = None self._last_error = None - self._input_buffer = "" - self._pending_syscall = False - self._symbols = {} - self._addr_to_symbol = {} + self._symbols = SymbolTable() + self._sections = [] self._tohost_addr = None + self._io.clear_input() - # Create disassembler - self._disasm = Disassembler(isa) - - # Map a default memory region for tests that don't load ELF files - # This ensures basic operations like write_mem work even without load_elf + uc.hook_add(UC_HOOK_MEM_UNMAPPED, self._on_unmapped_access) self._map_default_memory() - # Install syscall detection hook - self._install_syscall_hook() - - # Install unmapped memory handler - self._install_unmapped_handler() - - def _install_syscall_hook(self) -> None: - """Install code hook to detect syscalls""" - - def syscall_hook(uc: Uc, address: int, size: int, user_data: Optional[object]) -> None: - # Check if this is a syscall instruction - if self._config.detect_syscall(uc, address): - self._pending_syscall = True - uc.emu_stop() - - self._uc.hook_add(UC_HOOK_CODE, syscall_hook) - - def _install_unmapped_handler(self) -> None: - """Install hook to handle unmapped memory accesses""" - - def unmapped_handler( - uc: Uc, access: int, address: int, size: int, value: int, user_data: Optional[object] - ) -> bool: - kind = _UNMAPPED_ACCESS_KINDS.get(access, "access") - self._last_error = f"{kind} unmapped address 0x{address:x}" - return False # Don't handle it, let it error - - self._uc.hook_add(UC_HOOK_MEM_UNMAPPED, unmapped_handler) - - @property - def layout(self) -> MemoryLayout: - """Memory layout for the current ISA""" - if self._isa is None: - raise RuntimeError("Simulator ISA not set") - return get_layout(_ISA_LAYOUT_NAMES[int(self._isa)]) + def _on_unmapped_access( + self, uc: Uc, access: int, address: int, size: int, value: int, user_data: object + ) -> bool: + kind = _UNMAPPED_ACCESS_KINDS.get(access, "access") + self._last_error = f"{kind} unmapped address 0x{address:x}" + return False # Don't handle it, let it error def _map_new(self, addr: int, size: int) -> None: """Map a fresh, page-aligned region that does not overlap existing maps. On RV64, ``lui`` sign-extends, so an address like 0x80100000 built with lui+addi becomes 0xFFFFFFFF80100000. Real hardware with a 32-bit - physical address space treats these as the same location, so any - region in [0x80000000, 0x100000000) is also mapped at its sign-extended - alias, backed by the same host memory. + physical address space treats these as the same location, so for such + ISAs any region in [0x80000000, 0x100000000) is also mapped at its + sign-extended alias, backed by the same host memory. """ - if self._isa == ISA.RISCV and addr >= 0x80000000 and addr + size <= 0x100000000: + mirror = self.spec.sign_extends_32bit_addresses + if mirror and addr >= 0x80000000 and addr + size <= 0x100000000: buf = ctypes.create_string_buffer(size) self._mem_buffers.append(buf) # keep host memory alive ptr = ctypes.addressof(buf) @@ -709,118 +250,103 @@ def _map_default_memory(self) -> None: self._ensure_mapped(layout.text_base, DEFAULT_REGION_SIZE) if not layout.text_base <= layout.data_base < layout.text_base + DEFAULT_REGION_SIZE: self._ensure_mapped(layout.data_base, DEFAULT_REGION_SIZE) - stack_bottom = layout.stack_top - layout.stack_size - stack_span = layout.stack_size - if self._isa == ISA.RISCV: - # RISC-V has historically mapped one page above stack_top too - stack_span += PAGE_SIZE - self._ensure_mapped(stack_bottom, stack_span) + self._setup_stack() # Start at the conventional code address rather than 0 self.set_pc(layout.text_base) - def _map_memory_for_segments(self, segments: List[ELFSegment]) -> None: - """Map memory regions for ELF segments and copy in their contents""" + def _load_segments(self, segments: List[ELFSegment]) -> None: + """Map memory for ELF segments and copy in their contents""" for segment in segments: if segment.memsz == 0: continue self._ensure_mapped(segment.vaddr, segment.memsz) - if segment.data: self._uc.mem_write(segment.vaddr, segment.data) - # Zero-fill BSS (if memsz > filesz) if segment.memsz > segment.filesz: bss_size = segment.memsz - segment.filesz - bss_addr = segment.vaddr + segment.filesz - self._uc.mem_write(bss_addr, b"\x00" * bss_size) + self._uc.mem_write(segment.vaddr + segment.filesz, b"\x00" * bss_size) def _setup_stack(self) -> None: """Map stack memory and point the stack pointer at its top""" layout = self.layout self._ensure_mapped(layout.stack_top - layout.stack_size, layout.stack_size) - self._uc.reg_write(self._config.get_sp_reg(), layout.stack_top - 8) + self._uc.reg_write(self._engine.sp_reg, layout.stack_top - 8) def load_elf(self, elf_path: str) -> bool: """ Load an ELF file into simulator memory Args: - elf_path (str): Path to RISC-V or ARM ELF executable + elf_path (str): Path to an ELF executable for any supported ISA Returns: bool: True if successful """ - # Parse ELF file elf_info = load_elf_file(elf_path) - # Convert ELF ISA enum to our ISA enum - if elf_info.isa == ELF_ISA.RISCV: - detected_isa = ISA.RISCV - elif elf_info.isa == ELF_ISA.ARM: - detected_isa = ISA.ARM - elif elf_info.isa == ELF_ISA.X86_64: - detected_isa = ISA.X86_64 - elif elf_info.isa == ELF_ISA.MIPS: - detected_isa = ISA.MIPS - else: - raise RuntimeError(f"Unknown ISA in ELF file: {elf_path}") - # A simulator created for a specific ISA only accepts that ISA; one # created without an ISA adopts whatever each loaded ELF uses. - if self._explicit_isa and self._isa != detected_isa: + if self._explicit_isa and self._spec is not elf_info.spec: raise RuntimeError( - f"ELF ISA ({detected_isa.name}) doesn't match simulator ISA ({self._isa.name})" + f"ELF ISA ({elf_info.spec.isa.name}) doesn't match simulator ISA " + f"({self.get_isa_name()})" ) # Start from a fresh machine so nothing from a previous program # (memory, registers, exit state) leaks into this one. - self._init_unicorn(detected_isa) + self._init_machine(elf_info.spec) self._elf_path = elf_path - - # Map memory for ELF segments - self._map_memory_for_segments(elf_info.segments) - - # Setup stack + self._load_segments(elf_info.segments) self._setup_stack() - - # Set program counter to entry point self._entry_point = elf_info.entry self.set_pc(elf_info.entry) + self._symbols = SymbolTable(elf_info.symbols) + self._sections = elf_info.sections + self._tohost_addr = self._symbols.lookup("tohost") + return True - # Store symbol table - self._symbols = elf_info.symbols + # --- Execution --- - # Build reverse mapping (addr -> symbol) - self._addr_to_symbol = {addr: name for name, addr in self._symbols.items()} - self._tohost_addr = self._symbols.get("tohost") + def _read_word(self, addr: int, size: int) -> Optional[int]: + """The ``size`` bytes at ``addr`` as an integer in the ISA's byte order""" + try: + return int.from_bytes(self._uc.mem_read(addr, size), self.spec.byteorder) + except UcError: + return None - return True + def _is_syscall(self, pc: int) -> bool: + pattern = self.spec.syscall_instruction + word = self._read_word(pc, pattern.size) + return word is not None and pattern.matches(word) def step(self) -> StepResult: """ Execute one instruction + A syscall instruction is not executed by the CPU: step() moves the PC + past it and returns SYSCALL, and check_termination() performs it. + Returns: - StepResult: Result code (OK, HALT, WAITING, SYSCALL, or ERROR) + StepResult: Result code (OK, HALT, SYSCALL, or ERROR) """ if self._uc is None: return StepResult.ERROR # Once the program has exited there is nothing left to execute - if self._exited: + if self.exited: return StepResult.HALT - self._pending_syscall = False self._last_error = None pc = self.get_pc() + if self._is_syscall(pc): + self.set_pc(pc + self.spec.syscall_instruction.size) + return StepResult.SYSCALL + try: - # Execute one instruction - # Use a large end address - the count=1 ensures we stop after one instruction. - # Note: MIPS has a Unicorn quirk where pc+4 as end address doesn't advance PC. - # Using a larger offset (0x10000) works for all ISAs. - end_addr = pc + 0x10000 - # The hook will stop execution if a syscall is detected - self._uc.emu_start(pc, end_addr, count=1) + # count=1 executes one instruction. The end address must be past + # it (MIPS does not advance with pc+4, so use a larger offset). + self._uc.emu_start(pc, pc + 0x10000, count=1) except UcError as e: # Prefer the specific description recorded by the unmapped-memory # hook (it includes the faulting address) over Unicorn's message. @@ -831,261 +357,247 @@ def step(self) -> StepResult: self.set_pc(pc) return StepResult.ERROR - # Check if we hit a syscall - if self._pending_syscall: - # Advance PC past the syscall instruction - # x86-64 syscall is 2 bytes, RISC-V ecall/ARM svc are 4 bytes - syscall_size = 2 if self._isa == ISA.X86_64 else 4 - self.set_pc(pc + syscall_size) - return StepResult.SYSCALL - return StepResult.OK def _describe_trap(self, pc: int) -> Optional[str]: - """Explain a trap raised by a breakpoint instruction at ``pc``, if any. + """Explain a trap raised by a breakpoint-style instruction at ``pc``, if any.""" + word = self._read_word(pc, self.spec.min_instruction_size) + return None if word is None else self.spec.describe_trap(word) - Assemblers use these for runtime checks: MIPS div/rem expand to - code that executes 'break 7' on division by zero and 'break 6' on - overflow, as SPIM and GNU as do. + def check_termination(self, step_result: StepResult) -> Tuple[bool, Optional[StopReason]]: """ - try: - word = self.read_mem(pc, 4) - except RuntimeError: - return None - if self._isa == ISA.MIPS: - instr = int.from_bytes(word, "big") - if instr & 0xFC00003F == 0x0000000D: # break - code = (instr >> 16) & 0x3FF - if code == 7: - return "Division by zero" - if code == 6: - return "Integer overflow in division (-2147483648 / -1)" - return f"Break instruction executed (break {code})" - elif self._isa == ISA.RISCV: - if int.from_bytes(word, "little") == 0x00100073: - return "ebreak instruction executed" - elif self._isa == ISA.ARM: - instr = int.from_bytes(word, "little") - if instr & 0xFFE0001F == 0xD4200000: # brk #imm - return f"brk instruction executed (brk #{(instr >> 5) & 0xFFFF})" - return None + Check if program should terminate based on step result - def run(self, max_steps: Optional[int] = None) -> int: + Performs the syscall if ``step_result`` is SYSCALL. + + Returns: + tuple: (should_terminate, reason) where reason is a StopReason + (EXIT, HALT, ERROR, or TOHOST) or None """ - Run until halt, syscall exit, or max_steps reached + if step_result == StepResult.SYSCALL: + outcome = self._syscalls.handle(self) + if outcome.error is not None: + self._last_error = outcome.error + return (True, StopReason.ERROR) + if outcome.exit_code is not None: + self._exit_code = outcome.exit_code + return (True, StopReason.EXIT) + + if step_result == StepResult.HALT: + return (True, StopReason.HALT) + + if step_result == StepResult.ERROR: + return (True, StopReason.ERROR) + + # Check for tohost write (HTIF mechanism) + if self._tohost_addr is not None and self._read_word(self._tohost_addr, 8): + return (True, StopReason.TOHOST) + + return (False, None) + + def run_until( + self, + max_steps: Optional[int] = None, + stop_before: Optional[Callable[[int], Optional[StopReason]]] = None, + ) -> RunResult: + """ + Run until the program stops, ``stop_before`` says to stop, or the step limit. Args: - max_steps (int, optional): Maximum number of instructions to execute + max_steps: Maximum number of instructions to execute (None for no limit) + stop_before: Called with the PC before each instruction except the + first (so a run can resume from a breakpoint); returning a + StopReason (e.g. BREAKPOINT) stops the run there. + """ + steps = 0 + pc = self.get_pc() + while max_steps is None or steps < max_steps: + pc = self.get_pc() + if steps > 0 and stop_before is not None: + reason = stop_before(pc) + if reason is not None: + return RunResult(steps, reason, pc) + result = self.step() + steps += 1 + done, reason = self.check_termination(result) + if done: + return RunResult(steps, reason, pc) + return RunResult(steps, None, pc) + + def run(self, max_steps: Optional[int] = None) -> int: + """ + Run until halt, syscall exit, or max_steps reached Returns: int: Number of instructions executed """ - steps_executed = 0 + return self.run_until(max_steps).steps - while max_steps is None or steps_executed < max_steps: - result = self.step() - steps_executed += 1 + def reset(self) -> None: + """Reset the simulator to its initial state - # Check for termination - should_terminate, reason = self.check_termination(result) - if should_terminate: - break + If an ELF file was loaded it is reloaded from disk, restoring memory, + registers, and the entry point, so the program can be run again. + """ + if self._elf_path is not None: + self.load_elf(self._elf_path) + elif self._spec is not None: + self._init_machine(self._spec) + + # --- ISA --- - return steps_executed + @property + def spec(self) -> ISASpec: + """Specification of the current ISA""" + if self._spec is None: + raise RuntimeError("Simulator ISA not set") + return self._spec + + @property + def layout(self) -> MemoryLayout: + """Memory layout for the current ISA""" + return self.spec.layout + + def get_isa(self) -> Optional[ISA]: + """The current ISA, or None if not set yet""" + return self._spec.isa if self._spec is not None else None + + def get_isa_name(self) -> str: + """Name of the current ISA's enum member (e.g. "RISCV"), or "Unknown" """ + return self._spec.isa.name if self._spec is not None else "Unknown" + + # --- Registers --- def get_pc(self) -> int: """Get the program counter""" if self._uc is None: return 0 - return self._uc.reg_read(self._config.get_pc_reg()) + return self._uc.reg_read(self._engine.pc_reg) def set_pc(self, pc: int) -> None: """Set the program counter""" if self._uc is not None: - self._uc.reg_write(self._config.get_pc_reg(), pc) + self._uc.reg_write(self._engine.pc_reg, pc) - def get_reg(self, reg_num: int) -> int: - """ - Get register value + def get_register_count(self) -> int: + """Number of general-purpose registers for the current ISA""" + return self._spec.registers.count if self._spec is not None else 32 - Args: - reg_num (int): Register number (0-31 for RISC-V/ARM, 0-15 for x86-64) + def get_reg_name(self, n: int) -> str: + """The ABI/conventional name for register n (e.g. "ra", "sp", "rax")""" + if self._spec is None: + return f"r{n}" + names = self._spec.registers.names + return names[n] if 0 <= n < len(names) else f"?{n}" - Returns: - int: Register value (64-bit) - """ + def _check_reg(self, reg_num: int) -> int: + count = self.spec.registers.count + if not 0 <= reg_num < count: + raise ValueError(f"Register number must be 0-{count - 1}, got {reg_num}") + return self._engine.gpr_regs[reg_num] + + def get_reg(self, reg_num: int) -> int: + """Get a general-purpose register's value""" if self._uc is None: return 0 - - max_reg = 15 if self._isa == ISA.X86_64 else 31 - if not 0 <= reg_num <= max_reg: - raise ValueError(f"Register number must be 0-{max_reg}, got {reg_num}") - - reg = self._config.get_gpr_reg(reg_num) - return self._uc.reg_read(reg) + return self._uc.reg_read(self._check_reg(reg_num)) def set_reg(self, reg_num: int, value: int) -> None: """ - Set register value + Set a general-purpose register (value is truncated to 64 bits) - Args: - reg_num (int): Register number (1-31 for RISC-V, 0-15 for x86-64) - x0 is read-only on RISC-V - value (int): Value to set (64-bit) + Raises: + ValueError: For an invalid register number or the hardwired zero register """ if self._uc is None: return - - if self._isa == ISA.X86_64: - # x86-64: all 16 registers (0-15) are writable - if not 0 <= reg_num <= 15: - raise ValueError(f"Register number must be 0-15, got {reg_num}") - else: - # RISC-V/ARM: x0 is read-only - if not 1 <= reg_num <= 31: - raise ValueError(f"Register number must be 1-31, got {reg_num}") - - reg = self._config.get_gpr_reg(reg_num) + reg = self._check_reg(reg_num) + if reg_num == self.spec.registers.zero_register: + raise ValueError( + f"Register {reg_num} ({self.get_reg_name(reg_num)}) is always zero " + "and cannot be written" + ) self._uc.reg_write(reg, value & 0xFFFFFFFFFFFFFFFF) - def read_mem(self, addr: int, length: int) -> bytes: - """ - Read memory + def get_all_regs(self) -> List[int]: + """All general-purpose register values, by register number""" + return [self.get_reg(i) for i in range(self.get_register_count())] - Args: - addr (int): Memory address - length (int): Number of bytes to read + # --- Memory --- - Returns: - bytes: Memory contents - """ + def read_mem(self, addr: int, length: int) -> bytes: + """Read memory (raises RuntimeError for unmapped addresses)""" if self._uc is None: raise RuntimeError("Simulator not initialized") - try: - data = self._uc.mem_read(addr, length) - # Unicorn returns bytearray, but API expects bytes - return bytes(data) + return bytes(self._uc.mem_read(addr, length)) except UcError as e: raise RuntimeError(f"Failed to read memory at 0x{addr:x}: {e}") def write_mem(self, addr: int, data: Union[bytes, str]) -> bool: - """ - Write memory - - Args: - addr (int): Memory address - data (bytes): Data to write - - Returns: - bool: True if successful - """ + """Write memory (a str is written as UTF-8); raises RuntimeError on failure""" if self._uc is None: raise RuntimeError("Simulator not initialized") - if isinstance(data, str): data = data.encode("utf-8") - try: self._uc.mem_write(addr, bytes(data)) return True except UcError as e: raise RuntimeError(f"Failed to write memory at 0x{addr:x}: {e}") - def disasm(self, addr: int) -> str: - """ - Disassemble instruction at address - - Args: - addr (int): Address of instruction to disassemble - - Returns: - str: Disassembled instruction string - """ - if self._disasm is None: - raise RuntimeError("Simulator not initialized") - - return self._disasm.disassemble_one(self, addr) + # --- Disassembly --- def disasm_with_size(self, addr: int) -> Tuple[str, int]: """ - Disassemble instruction at address and return its size in bytes + Disassemble the instruction at ``addr`` and return its size in bytes - Returns: - tuple: (instruction text, instruction length in bytes) + Callers that walk a range of instructions must advance by the size, + since x86-64 instructions vary in length. """ if self._disasm is None: raise RuntimeError("Simulator not initialized") + # Near the end of a mapped region fewer bytes may be readable + size = self._disasm.max_size + while size > 0: + try: + return self._disasm.disassemble(self.read_mem(addr, size), addr) + except RuntimeError: + size -= 1 + return self._disasm.disassemble(b"", addr) - return self._disasm.disassemble_with_size(self, addr) - - def get_symbols(self) -> Dict[str, int]: - """ - Get all symbols from the symbol table + def disasm(self, addr: int) -> str: + """Disassemble the instruction at ``addr``""" + return self.disasm_with_size(addr)[0] - Returns: - dict: Dictionary mapping symbol names to addresses - """ - return self._symbols.copy() + # --- Symbols and sections --- - def lookup_symbol(self, name: str) -> Optional[int]: - """ - Look up symbol address by name + @property + def symbols(self) -> SymbolTable: + """The loaded program's symbol table""" + return self._symbols - Args: - name (str): Symbol name to look up + def get_symbols(self) -> Dict[str, int]: + """All symbols as a dictionary mapping names to addresses""" + return self._symbols.as_dict() - Returns: - int: Symbol address, or None if not found - """ - return self._symbols.get(name) + def lookup_symbol(self, name: str) -> Optional[int]: + """Address of a symbol, or None if not found""" + return self._symbols.lookup(name) def addr_to_symbol(self, addr: int) -> Tuple[Optional[str], Optional[int]]: - """ - Convert address to symbol name + offset - - Args: - addr (int): Address to look up + """(symbol_name, offset) for the nearest symbol at or before addr, or (None, None)""" + return self._symbols.nearest(addr) - Returns: - tuple: (symbol_name, offset) if found, or (None, None) if not found - """ - # Maximum distance to consider a symbol match (1MB) - MAX_SYMBOL_DISTANCE = 0x100000 + def get_sections(self) -> List[ELFSection]: + """Sections of the loaded ELF file""" + return list(self._sections) - # Exact match - if addr in self._addr_to_symbol: - return (self._addr_to_symbol[addr], 0) + def find_section(self, name: str) -> Optional[ELFSection]: + """The loaded ELF file's section with this name, or None""" + return next((s for s in self._sections if s.name == name), None) - # Find closest symbol before this address - best_match = None - best_addr = None - - for sym_name, sym_addr in self._symbols.items(): - if sym_addr <= addr: - if best_addr is None or sym_addr > best_addr: - best_addr = sym_addr - best_match = sym_name - - if best_match: - offset = addr - best_addr - # Only return if within reasonable distance - if offset <= MAX_SYMBOL_DISTANCE: - return (best_match, offset) - - return (None, None) - - def reset(self) -> None: - """Reset the simulator to its initial state - - If an ELF file was loaded it is reloaded from disk, restoring memory, - registers, and the entry point, so the program can be run again. - """ - if self._elf_path is not None: - self.load_elf(self._elf_path) - elif self._isa is not None: - # Unicorn doesn't have a native reset, so we reinitialize - self._init_unicorn(self._isa) + # --- Program state and I/O --- @property def last_error(self) -> Optional[str]: @@ -1095,13 +607,31 @@ def last_error(self) -> Optional[str]: @property def exited(self) -> bool: """True once the program has exited via an exit syscall""" - return self._exited + return self._exit_code is not None @property def exit_code(self) -> Optional[int]: """Exit code passed to the exit syscall (None if not exited)""" return self._exit_code + @property + def stdout(self) -> Optional[TextIO]: + """Where program output goes (None for the process's stdout)""" + return self._io.stdout + + @stdout.setter + def stdout(self, stream: Optional[TextIO]) -> None: + self._io.stdout = stream + + @property + def stdin(self) -> Optional[TextIO]: + """Where program input comes from (None for the process's stdin)""" + return self._io.stdin + + @stdin.setter + def stdin(self, stream: Optional[TextIO]) -> None: + self._io.stdin = stream + @property def output_needs_newline(self) -> bool: """True if program output so far did not end with a newline @@ -1109,254 +639,11 @@ def output_needs_newline(self) -> bool: Consoles use this to avoid printing their own messages on the same line as program output. """ - return self._output_needs_newline + return self._io.output_needs_newline @output_needs_newline.setter def output_needs_newline(self, value: bool) -> None: - self._output_needs_newline = value - - def get_all_regs(self) -> List[int]: - """ - Get all register values as a list - - Returns: - list: List of register values (32 for RISC-V/ARM, 16 for x86-64) - """ - num_regs = 16 if self._isa == ISA.X86_64 else 32 - return [self.get_reg(i) for i in range(num_regs)] - - def get_isa(self) -> Optional[ISA]: - """ - Get the current ISA - - Returns: - ISA: The current ISA enum value (ISA.RISCV, ISA.ARM, ISA.X86_64), or None if not set - """ - return self._isa - - def get_isa_name(self) -> str: - """ - Get human-readable name of the current ISA - - Returns: - str: ISA name (e.g., "RISCV", "ARM", "X86_64") or "Unknown" if not set - """ - return self._isa.name if self._isa is not None else "Unknown" - - def get_register_count(self) -> int: - """ - Get the number of general-purpose registers for the current ISA - - Returns: - int: Number of GPRs (32 for RISC-V/ARM, 16 for x86-64) - """ - if self._isa == ISA.X86_64: - return 16 - return 32 - - def get_reg_name(self, n: int) -> str: - """ - Get the ABI/conventional name for register n - - Args: - n: Register number - - Returns: - str: Register name (e.g., "ra", "sp", "x0", "rax") - """ - if self._config is None: - return f"r{n}" - return self._config.get_reg_name(n) - - def _get_syscall_regs(self) -> Tuple[int, int, int]: - """ - Get syscall register mappings for current ISA - - Returns: - tuple: (syscall_num_reg, arg0_reg, result_reg) - """ - if self._isa == ISA.X86_64: - # x86-64 Linux syscall ABI: rax=syscall#, rdi=arg0, return in rax - # Our register indices: rax=0, rdi=7 - return (0, 7, 0) - elif self._isa == ISA.MIPS: - # MIPS SPIM ABI: $v0=syscall# (reg 2), $a0=arg0 (reg 4), return in $v0 - return (2, 4, 2) - elif self._isa == ISA.ARM: - # ARM64: x8=syscall#, x0=arg0, return in x0 - return (8, 0, 0) - else: - # RISC-V: a7=syscall# (x17), a0=arg0 (x10), return in a0 - return (17, 10, 10) - - def _write_output(self, text: str) -> None: - """Write program output (from a print syscall)""" - if not text: - return - stream = self.stdout if self.stdout is not None else sys.stdout - stream.write(text) - stream.flush() - self._output_needs_newline = not text.endswith("\n") - - def _read_input_line(self) -> Optional[str]: - """Read one line of program input, without the newline (None on EOF)""" - # Make sure any prompt the program printed is visible first - out = self.stdout if self.stdout is not None else sys.stdout - out.flush() - # Pressing Enter echoes a newline, so the console is at a line start - self._output_needs_newline = False - if self.stdin is not None: - line = self.stdin.readline() - if line == "": - return None - return line.rstrip("\r\n") - try: - # input() gives line editing on interactive terminals - return input() - except EOFError: - return None - - def _signed(self, value: int) -> int: - """Interpret a register value as a signed integer of the ISA's width""" - bits = 32 if self._isa == ISA.MIPS else 64 - value &= (1 << bits) - 1 - if value & (1 << (bits - 1)): - value -= 1 << bits - return value - - def _read_c_string(self, addr: int, limit: int = 65536) -> str: - """Read a NUL-terminated string from simulated memory""" - data = bytearray() - while len(data) < limit: - # Read a chunk, shrinking near the end of a mapped region - chunk = b"" - size = 256 - (addr % 256) - while size > 0: - try: - chunk = self.read_mem(addr, size) - break - except RuntimeError: - size //= 2 - if not chunk: - if not data: - raise RuntimeError(f"print_string: address 0x{addr:x} is not mapped") - break - nul = chunk.find(0) - if nul >= 0: - data += chunk[:nul] - break - data += chunk - addr += len(chunk) - return data.decode("utf-8", errors="replace") - - def _handle_syscall(self) -> Optional[str]: - """ - Handle SPIM-compatible syscalls - - Returns: - None to continue, "syscall_exit" if the program exited, or "error" - (with ``last_error`` set) if the syscall could not be performed. - """ - syscall_reg, arg_reg, result_reg = self._get_syscall_regs() - syscall_num = self.get_reg(syscall_reg) - - if syscall_num == SYSCALL_PRINT_INT: - self._write_output(str(self._signed(self.get_reg(arg_reg)))) - - elif syscall_num == SYSCALL_PRINT_STRING: - try: - self._write_output(self._read_c_string(self.get_reg(arg_reg))) - except RuntimeError as e: - self._last_error = str(e) - return "error" - - elif syscall_num == SYSCALL_READ_INT: - # A partially consumed line (from read_char) is used first - if self._input_buffer.strip(): - line: Optional[str] = self._input_buffer - else: - line = self._read_input_line() - self._input_buffer = "" - try: - value = int((line or "").strip(), 0) - except ValueError: - value = 0 - self.set_reg(result_reg, value & 0xFFFFFFFFFFFFFFFF) - - elif syscall_num == SYSCALL_READ_CHAR: - # Characters come from a line buffer so "ab" yields a, b, \n - if not self._input_buffer: - line = self._read_input_line() - self._input_buffer = "" if line is None else line + "\n" - if self._input_buffer: - char, self._input_buffer = self._input_buffer[0], self._input_buffer[1:] - self.set_reg(result_reg, ord(char) & 0xFF) - else: - self.set_reg(result_reg, 0) # EOF - - elif syscall_num == SYSCALL_PRINT_CHAR: - self._write_output(chr(self.get_reg(arg_reg) & 0xFF)) - - elif syscall_num == SYSCALL_EXIT: - self._exited = True - self._exit_code = 0 - return "syscall_exit" - - elif syscall_num in (SYSCALL_EXIT2, SYSCALL_EXIT_CODE): - self._exited = True - self._exit_code = self._signed(self.get_reg(arg_reg)) & 0xFF - return "syscall_exit" - - else: - reg_name = self.get_reg_name(syscall_reg) - supported = ", ".join(str(n) for n in SUPPORTED_SYSCALLS) - self._last_error = ( - f"Unknown syscall number {self._signed(syscall_num)} in {reg_name} " - f"(supported: {supported})" - ) - return "error" - - return None - - def check_termination(self, step_result: StepResult) -> Tuple[bool, Optional[str]]: - """ - Check if program should terminate based on step result - - Performs the syscall if ``step_result`` is SYSCALL. - - Args: - step_result: Result from step() - - Returns: - tuple: (should_terminate, reason) where reason is one of - "syscall_exit", "halt", "error", "tohost", or None - """ - if step_result == StepResult.SYSCALL: - reason = self._handle_syscall() - if reason is not None: - return (True, reason) - - if step_result == StepResult.HALT: - return (True, "halt") - - if step_result == StepResult.ERROR: - return (True, "error") - - # Check for tohost write (HTIF mechanism) - if self._tohost_addr is not None: - try: - tohost_bytes = self.read_mem(self._tohost_addr, 8) - if int.from_bytes(tohost_bytes, byteorder="little") != 0: - return (True, "tohost") - except RuntimeError: - pass - - return (False, None) - - def __del__(self) -> None: - """Cleanup when object is destroyed""" - # Unicorn handles its own cleanup - pass + self._io.output_needs_newline = value def __enter__(self) -> UnicornSimulator: """Context manager support""" @@ -1368,63 +655,30 @@ def __exit__( exc_val: Optional[BaseException], exc_tb: Optional[object], ) -> None: - """Context manager cleanup""" - # Unicorn handles its own cleanup - pass + """Context manager cleanup (Unicorn handles its own cleanup)""" def detect_elf_isa(elf_path: str) -> ISA: - """ - Detect the ISA of an ELF file without loading it - - Args: - elf_path (str): Path to ELF file - - Returns: - ISA: ISA type (ISA.RISCV, ISA.ARM, ISA.X86_64, ISA.MIPS, or ISA.UNKNOWN) - """ + """The ISA of an ELF file (ISA.UNKNOWN if it cannot be read or is unsupported)""" try: - elf_info = load_elf_file(elf_path) - if elf_info.isa == ELF_ISA.RISCV: - return ISA.RISCV - elif elf_info.isa == ELF_ISA.ARM: - return ISA.ARM - elif elf_info.isa == ELF_ISA.X86_64: - return ISA.X86_64 - elif elf_info.isa == ELF_ISA.MIPS: - return ISA.MIPS + return load_elf_file(elf_path).isa except Exception: - pass - - return ISA.UNKNOWN + return ISA.UNKNOWN def create_simulator( elf_path: Optional[str] = None, config_file: Optional[str] = None ) -> UnicornSimulator: """ - Factory function to create a simulator with automatic ISA detection - - Args: - elf_path (str, optional): Path to ELF file for ISA auto-detection - config_file (str, optional): Not used by Unicorn backend + Create a simulator, loading ``elf_path`` (whose ISA it adopts) if given - Returns: - UnicornSimulator: Initialized simulator instance + Without an ELF file the simulator is for RISC-V. """ - if elf_path: - # Detect ISA from ELF file - isa = detect_elf_isa(elf_path) - if isa == ISA.UNKNOWN: - raise RuntimeError(f"Could not detect ISA from ELF file: {elf_path}") - - # Create simulator with detected ISA - sim = UnicornSimulator(isa=isa, config_file=config_file) - - # Load the ELF file - sim.load_elf(elf_path) - - return sim - else: - # Default to RISC-V + if not elf_path: return UnicornSimulator(isa=ISA.RISCV, config_file=config_file) + isa = detect_elf_isa(elf_path) + if isa == ISA.UNKNOWN: + raise RuntimeError(f"Could not detect ISA from ELF file: {elf_path}") + sim = UnicornSimulator(isa=isa, config_file=config_file) + sim.load_elf(elf_path) + return sim diff --git a/tests/test_assembler_accuracy.py b/tests/test_assembler_accuracy.py index cea380f..991a61e 100644 --- a/tests/test_assembler_accuracy.py +++ b/tests/test_assembler_accuracy.py @@ -19,7 +19,7 @@ from mapachespim.toolchain import assemble from mapachespim.toolchain.assembler import Assembler from mapachespim.toolchain.directives import DirectiveParser, LineType -from mapachespim.toolchain.memory_map import get_layout +from mapachespim.memory_map import get_layout diff --git a/tests/test_debug_info.py b/tests/test_debug_info.py index 205e973..464661f 100644 --- a/tests/test_debug_info.py +++ b/tests/test_debug_info.py @@ -15,7 +15,8 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) -from mapachespim.console import MapacheSPIMConsole, _parse_dwarf_line_info, SourceInfo +from mapachespim.console import MapacheSPIMConsole +from mapachespim.debug_info import SourceInfo, parse_line_info from mapachespim import create_simulator @@ -24,7 +25,7 @@ class TestDWARFParsing(unittest.TestCase): def test_parse_riscv_debug_info(self): """Test parsing DWARF info from RISC-V ELF with debug symbols""" - info = _parse_dwarf_line_info('examples/riscv/hello_asm/hello_asm') + info = parse_line_info('examples/riscv/hello_asm/hello_asm') self.assertTrue(info.has_debug_info, "Should have debug info") self.assertGreater(len(info.addr_to_line), 0, "Should have address mappings") @@ -34,7 +35,7 @@ def test_parse_riscv_debug_info(self): def test_address_mappings_exist(self): """Test that address mappings point to valid source lines""" - info = _parse_dwarf_line_info('examples/riscv/hello_asm/hello_asm') + info = parse_line_info('examples/riscv/hello_asm/hello_asm') for addr, (filename, line_num) in info.addr_to_line.items(): self.assertIsInstance(addr, int) @@ -48,7 +49,7 @@ class TestSourceLocationMapping(unittest.TestCase): def test_exact_address_lookup(self): """Test that exact address lookup works""" - info = _parse_dwarf_line_info('examples/riscv/hello_asm/hello_asm') + info = parse_line_info('examples/riscv/hello_asm/hello_asm') # Entry point should have a mapping location = info.get_location(0x80000000) @@ -61,7 +62,7 @@ def test_expanded_pseudo_op_mapping(self): The 'la' instruction expands to 'auipc + addi' (2 instructions, 8 bytes). Both 0x80000000 and 0x80000004 should map to line 61. """ - info = _parse_dwarf_line_info('examples/riscv/hello_asm/hello_asm') + info = parse_line_info('examples/riscv/hello_asm/hello_asm') # The first 'la' at 0x80000000 expands to two instructions # Both addresses should map to the same source line @@ -77,7 +78,7 @@ def test_expanded_pseudo_op_mapping(self): def test_all_pcs_have_mapping(self): """Test that stepping through the program, every PC has a source mapping""" - info = _parse_dwarf_line_info('examples/riscv/hello_asm/hello_asm') + info = parse_line_info('examples/riscv/hello_asm/hello_asm') sim = create_simulator('examples/riscv/hello_asm/hello_asm') # Step through first 20 instructions diff --git a/tests/test_e2e_all_isas.py b/tests/test_e2e_all_isas.py index 23575ec..7ed46bd 100644 --- a/tests/test_e2e_all_isas.py +++ b/tests/test_e2e_all_isas.py @@ -19,7 +19,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) -from mapachespim import create_simulator, detect_elf_isa, ISA, StepResult +from mapachespim import create_simulator, detect_elf_isa, ISA, StepResult, StopReason from mapachespim.console import MapacheSPIMConsole @@ -594,211 +594,57 @@ def test_all_examples_complete(self): class TestGuessGameAllISAs(unittest.TestCase): - """Test guess_game interactive program on all ISAs. - - The guess_game is an interactive number guessing game that: - 1. Prints welcome messages - 2. Reads user input (syscall 5 = read_int) - 3. Compares guess to secret number (42) - 4. Exits when correct - - Since it's interactive, we test: - - Program loads and starts correctly - - Required symbols exist - - Program reaches input syscall - - With correct input injected, completes successfully - """ - - GUESS_GAME_EXAMPLES = [ - ("examples/riscv/guess_game/guess_game", "RISCV"), - ("examples/arm/guess_game/guess_game", "ARM"), - ("examples/x86_64/guess_game/guess_game", "X86_64"), - ("examples/mips/guess_game/guess_game", "MIPS"), - ] - - def _get_syscall_regs(self, isa_name: str): - """Get syscall register mappings for ISA.""" - if isa_name == "X86_64": - return (0, 7, 0) # rax=syscall#, rdi=arg0, return in rax - elif isa_name == "MIPS": - return (2, 4, 2) # $v0=syscall#, $a0=arg0, return in $v0 - elif isa_name == "ARM": - return (8, 0, 0) # x8=syscall#, x0=arg0, return in x0 - else: # RISCV - return (17, 10, 10) # a7=syscall#, a0=arg0, return in a0 - - def _run_with_input(self, sim, input_value: int, isa_name: str, max_steps: int = 5000): - """Run program, injecting input when read_int syscall is hit. - - Note: We handle syscalls manually instead of using check_termination() - because check_termination() calls _handle_syscall() which uses input(). - """ - syscall_reg, _, result_reg = self._get_syscall_regs(isa_name) - - for step in range(max_steps): - result = sim.step() - - if result == StepResult.SYSCALL: - syscall_num = sim.get_reg(syscall_reg) - if syscall_num == 5: # read_int - inject our value - # For ARM64, result_reg is 0 (x0) which is writable - if isa_name == "ARM": - sim._uc.reg_write(sim._config.get_gpr_reg(result_reg), input_value) - else: - sim.set_reg(result_reg, input_value) - elif syscall_num == 10 or syscall_num == 93: # exit - return step + 1, "syscall_exit" - # For other syscalls (like print_string), let them pass - # They don't need handling since we're not checking output - - elif result == StepResult.HALT: - return step + 1, "halt" - elif result == StepResult.ERROR: - return step + 1, "error" - - return max_steps, "timeout" - - def test_riscv_guess_game_loads(self): - """RISC-V guess_game loads and has required symbols""" - elf = Path("examples/riscv/guess_game/guess_game") - if not elf.exists(): - self.skipTest(f"Not found: {elf}") - - sim = create_simulator(str(elf)) - symbols = sim.get_symbols() - - # Verify required symbols exist - self.assertIn("secret", symbols, "Should have 'secret' symbol") - self.assertIn("welcome", symbols, "Should have 'welcome' symbol") - self.assertIn("_start", symbols, "Should have '_start' symbol") - - def test_arm_guess_game_loads(self): - """ARM64 guess_game loads and has required symbols""" - elf = Path("examples/arm/guess_game/guess_game") - if not elf.exists(): - self.skipTest(f"Not found: {elf}") - - sim = create_simulator(str(elf)) - symbols = sim.get_symbols() - - self.assertIn("secret", symbols, "Should have 'secret' symbol") - self.assertIn("welcome", symbols, "Should have 'welcome' symbol") - - def test_x86_guess_game_loads(self): - """x86-64 guess_game loads and has required symbols""" - elf = Path("examples/x86_64/guess_game/guess_game") - if not elf.exists(): - self.skipTest(f"Not found: {elf}") - - sim = create_simulator(str(elf)) - symbols = sim.get_symbols() - - self.assertIn("secret", symbols, "Should have 'secret' symbol") - self.assertIn("welcome", symbols, "Should have 'welcome' symbol") - - def test_mips_guess_game_loads(self): - """MIPS guess_game loads and has required symbols""" - elf = Path("examples/mips/guess_game/guess_game") - if not elf.exists(): - self.skipTest(f"Not found: {elf}") - - sim = create_simulator(str(elf)) - symbols = sim.get_symbols() - - self.assertIn("secret", symbols, "Should have 'secret' symbol") - self.assertIn("welcome", symbols, "Should have 'welcome' symbol") - - def test_riscv_guess_game_correct_guess(self): - """RISC-V guess_game completes when correct answer (42) is given""" - elf = Path("examples/riscv/guess_game/guess_game") - if not elf.exists(): - self.skipTest(f"Not found: {elf}") + """Test the interactive guess_game example on all ISAs. - sim = create_simulator(str(elf)) - steps, reason = self._run_with_input(sim, 42, "RISCV") - - self.assertIn(reason, ["exit", "syscall_exit"], - f"RISC-V guess_game should exit cleanly with correct answer, got {reason}") - self.assertLess(steps, 5000, - f"RISC-V guess_game should complete quickly with correct answer") - - def test_arm_guess_game_correct_guess(self): - """ARM64 guess_game completes when correct answer (42) is given""" - elf = Path("examples/arm/guess_game/guess_game") - if not elf.exists(): - self.skipTest(f"Not found: {elf}") - - sim = create_simulator(str(elf)) - steps, reason = self._run_with_input(sim, 42, "ARM") - - self.assertIn(reason, ["exit", "syscall_exit"], - f"ARM64 guess_game should exit cleanly with correct answer, got {reason}") - self.assertLess(steps, 5000, - f"ARM64 guess_game should complete quickly with correct answer") - - def test_x86_guess_game_correct_guess(self): - """x86-64 guess_game completes when correct answer (42) is given""" - elf = Path("examples/x86_64/guess_game/guess_game") - if not elf.exists(): - self.skipTest(f"Not found: {elf}") - - sim = create_simulator(str(elf)) - steps, reason = self._run_with_input(sim, 42, "X86_64") - - self.assertIn(reason, ["exit", "syscall_exit"], - f"x86-64 guess_game should exit cleanly with correct answer, got {reason}") - self.assertLess(steps, 5000, - f"x86-64 guess_game should complete quickly with correct answer") - - def test_mips_guess_game_correct_guess(self): - """MIPS guess_game completes when correct answer (42) is given""" - elf = Path("examples/mips/guess_game/guess_game") - if not elf.exists(): - self.skipTest(f"Not found: {elf}") - - sim = create_simulator(str(elf)) - steps, reason = self._run_with_input(sim, 42, "MIPS") + The program prints a banner, reads guesses with the read_int syscall, + says whether each is too high or too low, and exits when the guess is + the secret number (42). Input comes from ``sim.stdin`` and output goes + to ``sim.stdout``, so the real syscall path is exercised on every ISA. + """ - self.assertIn(reason, ["exit", "syscall_exit"], - f"MIPS guess_game should exit cleanly with correct answer, got {reason}") - self.assertLess(steps, 5000, - f"MIPS guess_game should complete quickly with correct answer") + ISAS = ["riscv", "arm", "x86_64", "mips"] - def test_riscv_guess_game_wrong_then_right(self): - """RISC-V guess_game handles wrong guess then correct""" - elf = Path("examples/riscv/guess_game/guess_game") + def _play(self, isa: str, guesses): + elf = Path(f"examples/{isa}/guess_game/guess_game") if not elf.exists(): self.skipTest(f"Not found: {elf}") - sim = create_simulator(str(elf)) - syscall_reg, _, result_reg = self._get_syscall_regs("RISCV") - - guesses = [10, 42] # Wrong, then correct - guess_idx = 0 - - for step in range(10000): - result = sim.step() - - if result == StepResult.SYSCALL: - syscall_num = sim.get_reg(syscall_reg) - if syscall_num == 5: # read_int - if guess_idx < len(guesses): - sim.set_reg(result_reg, guesses[guess_idx]) - guess_idx += 1 - else: - sim.set_reg(result_reg, 42) # Fallback - elif syscall_num == 10 or syscall_num == 93: # exit - break - # Other syscalls (print_string, etc.) - continue execution - - elif result == StepResult.HALT: - break - elif result == StepResult.ERROR: - self.fail("RISC-V guess_game encountered an error") - else: - self.fail("RISC-V guess_game should complete with two guesses") + sim.stdin = StringIO("".join(f"{g}\n" for g in guesses)) + sim.stdout = StringIO() + result = sim.run_until(20000) + return sim, result, sim.stdout.getvalue() + + def test_guess_game_loads(self): + """guess_game loads and has the symbols it uses""" + for isa in self.ISAS: + with self.subTest(isa=isa): + elf = Path(f"examples/{isa}/guess_game/guess_game") + if not elf.exists(): + self.skipTest(f"Not found: {elf}") + symbols = create_simulator(str(elf)).get_symbols() + self.assertIn("secret", symbols) + self.assertIn("welcome", symbols) - self.assertEqual(guess_idx, 2, "Should have used exactly 2 guesses") + def test_correct_guess(self): + """The game exits cleanly after a correct first guess""" + for isa in self.ISAS: + with self.subTest(isa=isa): + sim, result, output = self._play(isa, [42]) + self.assertEqual(result.reason, StopReason.EXIT, sim.last_error) + self.assertEqual(sim.exit_code, 0) + self.assertIn("NUMBER GUESSING GAME", output) + self.assertIn("CORRECT", output) + self.assertIn("1 try!", output) + + def test_wrong_then_right(self): + """Wrong guesses get hints; the game counts the tries""" + for isa in self.ISAS: + with self.subTest(isa=isa): + sim, result, output = self._play(isa, [10, 90, 42]) + self.assertEqual(result.reason, StopReason.EXIT, sim.last_error) + self.assertIn("Too low", output) + self.assertIn("Too high", output) + self.assertIn("3 tries!", output) class TestComprehensiveISACoverage(unittest.TestCase): diff --git a/tests/test_elf_output.py b/tests/test_elf_output.py new file mode 100644 index 0000000..8eefc25 --- /dev/null +++ b/tests/test_elf_output.py @@ -0,0 +1,126 @@ +"""The assembler's ELF files are well formed, and the loader checks what it loads.""" + +from io import BytesIO +from pathlib import Path + +import pytest +from elftools.elf.elffile import ELFFile + +from mapachespim.elf_loader import load_elf_file +from mapachespim.examples import list_examples +from mapachespim.isa import ISA_SPECS +from mapachespim.toolchain import assemble + +EXAMPLES = list_examples() + + +def _program(spec): + return f""" + .isa {spec.name} + .data + value: .word 1 + table: .word 2, 3 + .text + .globl _start + _start: + nop + helper: + nop + .globl done + done: + nop + """ + + +@pytest.mark.parametrize("spec", ISA_SPECS, ids=lambda spec: spec.name) +def test_symbol_table_is_well_formed(spec): + result = assemble(_program(spec), isa=spec.name) + assert result.success, result.errors + elf = ELFFile(BytesIO(result.elf_bytes)) + assert elf.header["e_machine"] == spec.elf_machine_name + assert elf.header["e_flags"] == spec.elf_flags + assert elf.elfclass == spec.word_bits + assert elf.little_endian == spec.layout.is_little_endian + + symtab = elf.get_section_by_name(".symtab") + symbols = list(symtab.iter_symbols()) + bindings = [s["st_info"]["bind"] for s in symbols] + # Locals first; sh_info is the index of the first global + first_global = bindings.index("STB_GLOBAL") + assert all(b == "STB_LOCAL" for b in bindings[:first_global]) + assert all(b == "STB_GLOBAL" for b in bindings[first_global:]) + assert symtab["sh_info"] == first_global + assert {s.name for s in symbols[first_global:]} == {"_start", "done"} + + # Each symbol points at the section that contains it + for sym in symbols[1:]: + section = elf.get_section(sym["st_shndx"]) + start = section["sh_addr"] + assert start <= sym["st_value"] < start + section["sh_size"], sym.name + expected = ".data" if sym.name in ("value", "table") else ".text" + assert section.name == expected, sym.name + + +@pytest.mark.parametrize("example", EXAMPLES, ids=lambda e: e.short_name) +def test_example_binaries_are_up_to_date(example): + """The bundled binaries are what 'make debug' in examples/ produces. + + If this fails after changing the assembler, rebuild them with + ``make -B debug`` in the examples directory. + """ + source = example.source + result = assemble( + source.read_text(), isa=_isa_of(source), debug=True, source_filename=source.name + ) + assert result.success, result.errors + assert result.elf_bytes == example.binary.read_bytes() + + +def _isa_of(source: Path) -> str: + from mapachespim.toolchain.directives import find_isa_directive + + isa = find_isa_directive(source.read_text()) + assert isa is not None + return isa + + +def _patched(path, offset, data, tmp_path): + raw = bytearray(Path(path).read_bytes()) + raw[offset : offset + len(data)] = data + out = tmp_path / "patched.elf" + out.write_bytes(bytes(raw)) + return str(out) + + +@pytest.fixture +def mips_elf(): + elf = Path("examples/mips/hello_asm/hello_asm") + if not elf.exists(): + pytest.skip("examples not available") + return elf + + +def test_loader_rejects_unknown_machines(mips_elf, tmp_path): + # e_machine is at offset 18; MIPS is big-endian. 40 is 32-bit ARM. + with pytest.raises(RuntimeError, match="Unsupported ELF machine type"): + load_elf_file(_patched(mips_elf, 18, (40).to_bytes(2, "big"), tmp_path)) + + +def test_loader_rejects_wrong_word_size(mips_elf, tmp_path): + # A 32-bit RISC-V file must not be run as RV64 + with pytest.raises(RuntimeError, match="32-bit"): + load_elf_file(_patched(mips_elf, 18, (243).to_bytes(2, "big"), tmp_path)) + + +def test_loader_reports_sections(mips_elf): + info = load_elf_file(str(mips_elf)) + text = next(s for s in info.sections if s.name == ".text") + assert text.loaded and text.flag_letters == "AX" + assert text.address == info.spec.layout.text_base + debug = next(s for s in info.sections if s.name == ".debug_line") + assert not debug.loaded and debug.flag_letters == "" + + +def test_missing_file(): + with pytest.raises(FileNotFoundError): + load_elf_file("no/such/file") diff --git a/tests/test_isa_conformance.py b/tests/test_isa_conformance.py new file mode 100644 index 0000000..1757de9 --- /dev/null +++ b/tests/test_isa_conformance.py @@ -0,0 +1,256 @@ +""" +Conformance tests that every registered ISA must pass. + +Each ISA in ``mapachespim.isa.ISA_SPECS`` supplies, below, a small program +that exercises every syscall and an instruction template that loads a +constant into a register. The tests assemble those with the built-in +assembler and run them on the simulator, so the whole pipeline (registry, +encoder, ELF writer, loader, engine binding, syscalls) is checked the same +way for every ISA. A new ISA fails ``test_every_isa_has_conformance_programs`` +until it adds its entries here. +""" + +from io import StringIO + +import pytest + +from mapachespim import ISA_SPECS, Simulator, StopReason +from mapachespim.toolchain import assemble + +# Prints -5, '!', and "hi\n", echoes an integer and a character read from +# input, then exits with code 3. +SYSCALL_PROGRAMS = { + "riscv64": """ + .isa riscv64 + .data + msg: .asciz "hi\\n" + .text + .globl _start + _start: + li a0, -5 + li a7, 1 + ecall + li a0, 33 + li a7, 11 + ecall + la a0, msg + li a7, 4 + ecall + li a7, 5 + ecall + li a7, 1 + ecall + li a7, 12 + ecall + li a7, 11 + ecall + li a0, 3 + li a7, 17 + ecall + """, + "mips32": """ + .isa mips32 + .data + msg: .asciz "hi\\n" + .text + .globl _start + _start: + li $a0, -5 + li $v0, 1 + syscall + li $a0, 33 + li $v0, 11 + syscall + la $a0, msg + li $v0, 4 + syscall + li $v0, 5 + syscall + move $a0, $v0 + li $v0, 1 + syscall + li $v0, 12 + syscall + move $a0, $v0 + li $v0, 11 + syscall + li $a0, 3 + li $v0, 17 + syscall + """, + "arm64": """ + .isa arm64 + .data + msg: .asciz "hi\\n" + .text + .globl _start + _start: + mov x0, #-5 + mov x8, #1 + svc #0 + mov x0, #33 + mov x8, #11 + svc #0 + ldr x0, =msg + mov x8, #4 + svc #0 + mov x8, #5 + svc #0 + mov x8, #1 + svc #0 + mov x8, #12 + svc #0 + mov x8, #11 + svc #0 + mov x0, #3 + mov x8, #17 + svc #0 + """, + "x86_64": """ + .isa x86_64 + .data + msg: .asciz "hi\\n" + .text + .globl _start + _start: + mov $-5, %rdi + mov $1, %rax + syscall + mov $33, %rdi + mov $11, %rax + syscall + leaq msg(%rip), %rdi + mov $4, %rax + syscall + mov $5, %rax + syscall + mov %rax, %rdi + mov $1, %rax + syscall + mov $12, %rax + syscall + mov %rax, %rdi + mov $11, %rax + syscall + mov $3, %rdi + mov $17, %rax + syscall + """, +} + +# An instruction that sets register ``n`` (named ``name``) to ``value`` +SET_REGISTER = { + "riscv64": lambda n, name, value: f"li x{n}, {value}", + "mips32": lambda n, name, value: f"li ${n}, {value}", + "arm64": lambda n, name, value: f"mov x{n}, #{value}" if n < 31 else f"add sp, sp, #{value}", + "x86_64": lambda n, name, value: f"mov ${value}, %{name}", +} + +SPECS = pytest.mark.parametrize("spec", ISA_SPECS, ids=lambda spec: spec.name) + + +def _load(spec, source): + result = assemble(source, isa=spec.name) + assert result.success, result.errors + sim = Simulator() + elf = _write_temp(result.elf_bytes) + sim.load_elf(elf) + assert sim.spec is spec + return sim + + +def _write_temp(data): + import os + import tempfile + + fd, path = tempfile.mkstemp(suffix=".elf") + with os.fdopen(fd, "wb") as f: + f.write(data) + return path + + +def test_every_isa_has_conformance_programs(): + for spec in ISA_SPECS: + assert spec.name in SYSCALL_PROGRAMS, f"add a syscall program for {spec.name}" + assert spec.name in SET_REGISTER, f"add a register template for {spec.name}" + + +@SPECS +def test_syscalls(spec): + sim = _load(spec, SYSCALL_PROGRAMS[spec.name]) + sim.stdin = StringIO("41\nZ\n") + sim.stdout = StringIO() + result = sim.run_until(1000) + assert result.reason == StopReason.EXIT, sim.last_error + assert sim.exit_code == 3 + assert sim.stdout.getvalue() == "-5!hi\n41Z" + + +@SPECS +def test_every_register_is_mapped(spec): + """Each register number reads the architectural register of that name.""" + regs = spec.registers + lines = [f".isa {spec.name}", ".text", ".globl _start", "_start:"] + expected = {} + for n, name in enumerate(regs.names): + if n == regs.zero_register: + continue + value = 100 + 8 * n # small, positive, and 8-byte aligned for sp + lines.append(SET_REGISTER[spec.name](n, name, value)) + expected[n] = value + sim = _load(spec, "\n".join(lines)) + before = sim.get_all_regs() + result = sim.run_until(len(expected)) + assert result.reason is None, sim.last_error + after = sim.get_all_regs() + for n, value in expected.items(): + if spec.name == "arm64" and n == 31: # sp was incremented + value += before[31] + assert after[n] == value, f"{spec.name} register {n} ({regs.names[n]})" + if regs.zero_register is not None: + assert after[regs.zero_register] == 0 + + +@SPECS +def test_register_api(spec): + sim = Simulator(isa=spec.isa) + assert sim.get_register_count() == spec.registers.count + assert [sim.get_reg_name(n) for n in range(spec.registers.count)] == list(spec.registers.names) + for n in range(spec.registers.count): + if spec.registers.is_writable(n): + sim.set_reg(n, 0x1234 + n) + assert sim.get_reg(n) == 0x1234 + n + else: + with pytest.raises(ValueError): + sim.set_reg(n, 1) + with pytest.raises(ValueError): + sim.get_reg(spec.registers.count) + + +@SPECS +def test_default_machine(spec): + """Without an ELF file, code and data areas and the stack are mapped.""" + sim = Simulator(isa=spec.isa) + layout = spec.layout + assert sim.get_pc() == layout.text_base + for addr in (layout.text_base, layout.data_base, layout.stack_top - 8): + sim.write_mem(addr, b"\x5a") + assert sim.read_mem(addr, 1) == b"\x5a" + with pytest.raises(RuntimeError): + sim.read_mem(0, 4) # null stays unmapped + + +@SPECS +def test_unknown_syscall_is_an_error(spec): + program = SYSCALL_PROGRAMS[spec.name] + sim = _load(spec, program) + sim.set_reg(spec.syscall_abi.number, 999) + # Jump straight to the first syscall instruction + pattern = spec.syscall_instruction + addr = sim.get_pc() + while not pattern.matches(int.from_bytes(sim.read_mem(addr, pattern.size), spec.byteorder)): + addr += sim.disasm_with_size(addr)[1] + sim.set_pc(addr) + result = sim.run_until(1) + assert result.reason == StopReason.ERROR + assert "Unknown syscall number 999" in sim.last_error diff --git a/tests/test_isa_registry.py b/tests/test_isa_registry.py new file mode 100644 index 0000000..1888e5c --- /dev/null +++ b/tests/test_isa_registry.py @@ -0,0 +1,116 @@ +"""The ISA registry is complete and consistent, and every layer uses it.""" + +import pytest + +from mapachespim.engines import ENGINES, engine_for +from mapachespim.examples import ISA_DIRS, examples_dir, list_examples +from mapachespim.isa import ( + ISA, + ISA_SPECS, + canonical_isa, + find_spec, + get_spec, + isa_names, + spec_for_elf, +) +from mapachespim.memory_map import get_layout +from mapachespim.toolchain.directives import DirectiveParser, find_isa_directive +from mapachespim.toolchain.targets import TARGETS + +SPECS = pytest.mark.parametrize("spec", ISA_SPECS, ids=lambda spec: spec.name) + + +def test_every_isa_enum_member_has_a_spec(): + members = {isa for isa in ISA if isa != ISA.UNKNOWN} + assert members == {spec.isa for spec in ISA_SPECS} + + +def test_names_and_aliases_are_unique(): + names = [name for spec in ISA_SPECS for name in (spec.name, *spec.aliases)] + assert len(names) == len(set(names)) + assert len({spec.examples_dir for spec in ISA_SPECS}) == len(ISA_SPECS) + assert len({(spec.elf_machine, spec.word_bits) for spec in ISA_SPECS}) == len(ISA_SPECS) + + +@SPECS +def test_every_layer_supports_the_isa(spec): + """Adding an ISA means adding an engine binding, an assembler target, and examples.""" + engine = engine_for(spec) + assert len(engine.gpr_regs) == spec.registers.count + assert len(set(engine.gpr_regs)) == spec.registers.count + assert spec.name in TARGETS + assert TARGETS[spec.name].isa_name == spec.name + assert spec.examples_dir in ISA_DIRS + if examples_dir() is not None: + assert any(e.isa == spec.examples_dir for e in list_examples()) + + +@SPECS +def test_lookups(spec): + for name in (spec.name, *spec.aliases, spec.name.upper(), spec.name.replace("_", "-")): + assert find_spec(name) is spec + assert canonical_isa(name) == spec.name + assert get_layout(name) is spec.layout + assert get_spec(spec.isa) is spec + assert get_spec(spec) is spec + assert spec_for_elf(spec.elf_machine_name, spec.layout.is_64bit) is spec + assert spec_for_elf(spec.elf_machine_name, not spec.layout.is_64bit) is None + + +@SPECS +def test_spec_facts_are_consistent(spec): + abi = spec.syscall_abi + for reg in (abi.number, abi.arg0, abi.result): + assert 0 <= reg < spec.registers.count + assert spec.registers.is_writable(abi.result) + assert spec.min_instruction_size <= spec.syscall_instruction.size <= spec.max_instruction_size + assert spec.byteorder in ("little", "big") + assert spec.word_bits in (32, 64) + + +def test_unknown_isa(): + assert find_spec("vax") is None + with pytest.raises(ValueError, match="riscv64"): + get_spec("vax") + with pytest.raises(ValueError): + get_layout("vax") + assert canonical_isa("VAX") == "vax" + + +def test_signed(): + riscv, mips = get_spec("riscv64"), get_spec("mips32") + assert riscv.signed(0xFFFFFFFFFFFFFFFF) == -1 + assert riscv.signed(0xFFFFFFFF) == 0xFFFFFFFF + assert mips.signed(0xFFFFFFFF) == -1 + assert mips.signed(0x7FFFFFFF) == 0x7FFFFFFF + + +def test_trap_descriptions(): + assert get_spec("mips32").describe_trap(0x0007000D) == "Division by zero" + assert "overflow" in get_spec("mips32").describe_trap(0x0006000D) + assert get_spec("mips32").describe_trap(0x0000000C) is None # syscall + assert get_spec("riscv64").describe_trap(0x00100073) == "ebreak instruction executed" + assert get_spec("arm64").describe_trap(0xD4200020) == "brk instruction executed (brk #1)" + assert get_spec("x86_64").describe_trap(0xCC) is None + + +@SPECS +def test_isa_directive(spec): + for name in (spec.name, *spec.aliases): + assert find_isa_directive(f"# comment\n.isa {name} // trailing comment\n") == spec.name + source = f"// comment\n.isa {name}\n.text\nnop\n" + assert find_isa_directive(source) == spec.name + parser = DirectiveParser() + parser.parse(source) + assert parser.isa == spec.name + assert parser.errors == [] + + +def test_isa_directive_must_come_first(): + assert find_isa_directive(".text\nnop\n.isa riscv64\n") is None + assert find_isa_directive("") is None + + +def test_cli_choices_come_from_the_registry(): + assert isa_names() == tuple(spec.name for spec in ISA_SPECS) + assert set(ENGINES) == {spec.isa for spec in ISA_SPECS} diff --git a/tests/test_mips_backend.py b/tests/test_mips_backend.py index 84a2f5e..ffa8494 100644 --- a/tests/test_mips_backend.py +++ b/tests/test_mips_backend.py @@ -13,20 +13,16 @@ # Add parent directory to path sys.path.insert(0, str(Path(__file__).parent.parent)) -from mapachespim.unicorn_backend import ISA, MIPSConfig, UnicornSimulator +from mapachespim.isa import get_spec +from mapachespim.unicorn_backend import ISA, UnicornSimulator -class TestMIPSConfig(unittest.TestCase): - """Test MIPS configuration""" - - def test_mips_config_exists(self): - """Test that MIPSConfig class exists""" - config = MIPSConfig() - self.assertIsNotNone(config) +class TestMIPSSpec(unittest.TestCase): + """Test the MIPS ISA spec""" def test_mips_register_names(self): """Test MIPS register names""" - config = MIPSConfig() + names = get_spec(ISA.MIPS).registers.names expected_names = [ "zero", "at", "v0", "v1", # $0-$3 "a0", "a1", "a2", "a3", # $4-$7 @@ -39,7 +35,8 @@ def test_mips_register_names(self): "gp", "sp", "fp", "ra", # $28-$31 ] for i, expected in enumerate(expected_names): - self.assertEqual(config.get_reg_name(i), expected) + self.assertEqual(names[i], expected) + self.assertEqual(UnicornSimulator(isa=ISA.MIPS).get_reg_name(i), expected) class TestMIPSSimulator(unittest.TestCase): diff --git a/tests/test_simulator.py b/tests/test_simulator.py index fbaa83c..a17752b 100755 --- a/tests/test_simulator.py +++ b/tests/test_simulator.py @@ -201,8 +201,8 @@ def test_read_memory_instruction(self): def test_write_and_read_memory(self): """Test writing then reading memory""" - # Write to a safe location (high memory) - addr = 0x83F00000 + # Write to the (mapped, unused) start of the data area + addr = self.sim.layout.data_base test_data = b'\x01\x02\x03\x04\x05\x06\x07\x08' self.sim.write_mem(addr, test_data) @@ -212,7 +212,7 @@ def test_write_and_read_memory(self): def test_write_string_to_memory(self): """Test writing string (auto-converts to bytes)""" - addr = 0x83F00000 + addr = self.sim.layout.data_base test_str = "Hello" self.sim.write_mem(addr, test_str) diff --git a/tests/test_simulator_parts.py b/tests/test_simulator_parts.py new file mode 100644 index 0000000..630b8b4 --- /dev/null +++ b/tests/test_simulator_parts.py @@ -0,0 +1,116 @@ +"""Unit tests for the simulator's building blocks and run_until().""" + +from io import StringIO + +import pytest + +from mapachespim import RunResult, Simulator, StopReason +from mapachespim.disassembler import Disassembler +from mapachespim.isa import ISA_SPECS, get_spec +from mapachespim.symbols import MAX_SYMBOL_DISTANCE, SymbolTable + + +class TestSymbolTable: + def test_lookup(self): + table = SymbolTable({"main": 0x100, "loop": 0x120}) + assert table.lookup("loop") == 0x120 + assert table.lookup("nope") is None + assert "main" in table and len(table) == 2 + assert table.as_dict() == {"main": 0x100, "loop": 0x120} + + def test_nearest(self): + table = SymbolTable({"main": 0x100, "loop": 0x120}) + assert table.nearest(0x100) == ("main", 0) + assert table.nearest(0x11C) == ("main", 0x1C) + assert table.nearest(0x124) == ("loop", 4) + assert table.nearest(0xFF) == (None, None) + assert table.nearest(0x120 + MAX_SYMBOL_DISTANCE + 1) == (None, None) + + def test_same_address_uses_last_defined(self): + table = SymbolTable({".Llocal": 0x100, "_start": 0x100}) + assert table.nearest(0x104) == ("_start", 4) + + def test_describe(self): + table = SymbolTable({"main": 0x100}) + assert table.describe(0x100) == "
" + assert table.describe(0x108) == "" + assert table.describe(0x10) == "" + assert SymbolTable().describe(0) == "" + + +class TestDisassembler: + def test_decodes(self): + riscv = Disassembler(get_spec("riscv64")) + assert riscv.disassemble((0x00000013).to_bytes(4, "little"), 0) == ("nop", 4) + x86 = Disassembler(get_spec("x86_64")) + assert x86.disassemble(b"\x0f\x05\x90", 0) == ("syscall", 2) + + @pytest.mark.parametrize("spec", ISA_SPECS, ids=lambda spec: spec.name) + def test_undecodable_bytes_still_advance(self, spec): + disasm = Disassembler(spec) + text, size = disasm.disassemble(b"\xff" * spec.max_instruction_size, 0x1000) + assert size >= 1 + text, size = disasm.disassemble(b"", 0x1000) + assert text == "" and size == spec.min_instruction_size + + def test_undecodable_fixed_width_shown_as_word(self): + mips = Disassembler(get_spec("mips32")) + assert mips.disassemble(bytes.fromhex("fc000000"), 0) == (".word 0xfc000000", 4) + x86 = Disassembler(get_spec("x86_64")) + assert x86.disassemble(b"\x06", 0) == (".byte 0x06", 1) + + +FIB = "examples/riscv/fibonacci/fibonacci" + + +class TestRunUntil: + def _sim(self): + sim = Simulator() + sim.load_elf(FIB) + return sim + + def test_runs_to_exit(self, capsys): + sim = self._sim() + result = sim.run_until() + assert result.reason == StopReason.TOHOST # fibonacci signals completion via tohost + assert result.steps > 10 + + def test_step_limit(self): + sim = self._sim() + result = sim.run_until(5) + assert result.steps == 5 and result.reason is None + + def test_stop_before_and_resume(self): + sim = self._sim() + target = sim.lookup_symbol("fibonacci") + stop = lambda pc: StopReason.BREAKPOINT if pc == target else None # noqa: E731 + result = sim.run_until(stop_before=stop) + assert result.reason == StopReason.BREAKPOINT + assert result.pc == target == sim.get_pc() + # Resuming does not stop again at the breakpoint it is on + result = sim.run_until(1, stop_before=stop) + assert result.steps == 1 and result.reason is None + + def test_error_reports_faulting_pc(self): + sim = Simulator(isa=get_spec("riscv64").isa) + sim.write_mem(sim.get_pc(), (0).to_bytes(4, "little")) # illegal instruction + result = sim.run_until(10) + assert result.reason == StopReason.ERROR + assert result.pc == sim.get_pc() == get_spec("riscv64").layout.text_base + assert sim.last_error.endswith(f"(at PC={result.pc:#x})") + + def test_stop_reason_compares_to_strings(self): + assert StopReason.EXIT == "syscall_exit" + assert StopReason.ERROR == "error" + + def test_halts_after_exit(self): + sim = Simulator() + sim.load_elf("examples/riscv/hello_asm/hello_asm") + sim.stdout = StringIO() + assert sim.run_until().reason == StopReason.EXIT + assert sim.exited and sim.exit_code == 0 + assert sim.run_until() == RunResult(1, StopReason.HALT, sim.get_pc()) + + def test_run_returns_step_count(self): + sim = self._sim() + assert sim.run(7) == 7 diff --git a/tests/test_syscalls.py b/tests/test_syscalls.py new file mode 100644 index 0000000..40b079a --- /dev/null +++ b/tests/test_syscalls.py @@ -0,0 +1,140 @@ +"""SyscallHandler and ProgramIO, tested without the emulator.""" + +from io import StringIO + +import pytest + +from mapachespim.isa import ISA_SPECS, get_spec +from mapachespim.syscalls import ProgramIO, Syscall, SyscallHandler, read_c_string + + +class FakeMachine: + """Registers and a little memory, with the ISA's register rules.""" + + def __init__(self, spec, memory=None, base=0x1000): + self.spec = spec + self.regs = [0] * spec.registers.count + self.memory = bytes(memory or b"") + self.base = base + + def get_reg(self, n): + return self.regs[n] + + def set_reg(self, n, value): + if not self.spec.registers.is_writable(n): + raise ValueError(n) + self.regs[n] = value & ((1 << self.spec.word_bits) - 1) + + def read_mem(self, addr, length): + start = addr - self.base + if start < 0 or start + length > len(self.memory): + raise RuntimeError("unmapped") + return self.memory[start : start + length] + + def request(self, number, arg0=0): + abi = self.spec.syscall_abi + self.regs[abi.number] = number + self.regs[abi.arg0] = arg0 & ((1 << self.spec.word_bits) - 1) + + +def _handler(stdin=""): + io = ProgramIO() + io.stdout = StringIO() + io.stdin = StringIO(stdin) + return SyscallHandler(io), io + + +SPECS = pytest.mark.parametrize("spec", ISA_SPECS, ids=lambda spec: spec.name) + + +@SPECS +def test_print_int_is_signed_at_register_width(spec): + handler, io = _handler() + machine = FakeMachine(spec) + machine.request(Syscall.PRINT_INT, -7) + assert handler.handle(machine).exit_code is None + assert io.stdout.getvalue() == "-7" + assert io.output_needs_newline + + +@SPECS +def test_print_string_and_char(spec): + handler, io = _handler() + machine = FakeMachine(spec, b"hello\n\x00junk") + machine.request(Syscall.PRINT_STRING, 0x1000) + handler.handle(machine) + machine.request(Syscall.PRINT_CHAR, ord("!") | 0x100) # only the low byte counts + handler.handle(machine) + assert io.stdout.getvalue() == "hello\n!" + + +@SPECS +def test_read_int_and_char(spec): + handler, io = _handler(" 12 \nab\nnope\n") + machine = FakeMachine(spec) + result_reg = spec.syscall_abi.result + machine.request(Syscall.READ_INT) + handler.handle(machine) + assert machine.regs[result_reg] == 12 + for expected in "ab\n": + machine.request(Syscall.READ_CHAR) + handler.handle(machine) + assert machine.regs[result_reg] == ord(expected) + machine.request(Syscall.READ_INT) + handler.handle(machine) + assert machine.regs[result_reg] == 0 # not a number + machine.request(Syscall.READ_CHAR) + handler.handle(machine) + assert machine.regs[result_reg] == 0 # EOF + + +def test_read_int_uses_rest_of_a_partly_read_line(): + handler, io = _handler("x42\n") + machine = FakeMachine(get_spec("riscv64")) + machine.request(Syscall.READ_CHAR) + handler.handle(machine) + machine.request(Syscall.READ_INT) + handler.handle(machine) + assert machine.regs[10] == 42 + + +def test_read_int_accepts_hex_and_negative(): + handler, io = _handler("0x10\n-3\n") + machine = FakeMachine(get_spec("mips32")) + for expected in (16, 0xFFFFFFFD): + machine.request(Syscall.READ_INT) + handler.handle(machine) + assert machine.regs[2] == expected + + +@SPECS +def test_exits(spec): + handler, io = _handler() + machine = FakeMachine(spec) + machine.request(Syscall.EXIT, 5) + assert handler.handle(machine).exit_code == 0 + for number in (Syscall.EXIT2, Syscall.EXIT_CODE): + machine.request(number, 300) + result = handler.handle(machine) + assert result.exited and result.exit_code == 300 & 0xFF + machine.request(Syscall.EXIT2, -1) + assert handler.handle(machine).exit_code == 255 + + +@SPECS +def test_errors(spec): + handler, io = _handler() + machine = FakeMachine(spec) + machine.request(-2) + result = handler.handle(machine) + assert not result.exited + assert "Unknown syscall number -2" in result.error + assert spec.registers.names[spec.syscall_abi.number] in result.error + machine.request(Syscall.PRINT_STRING, 0x9999) + assert "not mapped" in handler.handle(machine).error + + +def test_read_c_string_stops_at_end_of_memory(): + machine = FakeMachine(get_spec("riscv64"), b"abc", base=0x2000) + assert read_c_string(machine, 0x2000) == "abc" + assert read_c_string(machine, 0x2001) == "bc" diff --git a/tests/test_x86_backend.py b/tests/test_x86_backend.py index 1e31ad5..d9ffeb4 100644 --- a/tests/test_x86_backend.py +++ b/tests/test_x86_backend.py @@ -13,28 +13,23 @@ # Add parent directory to path sys.path.insert(0, str(Path(__file__).parent.parent)) -from mapachespim.unicorn_backend import ( - UnicornSimulator, ISA, StepResult, X86_64Config -) +from mapachespim.isa import get_spec +from mapachespim.unicorn_backend import UnicornSimulator, ISA, StepResult -class TestX86Config(unittest.TestCase): - """Test x86-64 configuration""" - - def test_x86_config_exists(self): - """Test that X86_64Config class exists""" - config = X86_64Config() - self.assertIsNotNone(config) +class TestX86Spec(unittest.TestCase): + """Test the x86-64 ISA spec""" def test_x86_register_names(self): """Test x86-64 register names""" - config = X86_64Config() + names = get_spec(ISA.X86_64).registers.names expected_names = [ "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" ] for i, expected in enumerate(expected_names): - self.assertEqual(config.get_reg_name(i), expected) + self.assertEqual(names[i], expected) + self.assertEqual(UnicornSimulator(isa=ISA.X86_64).get_reg_name(i), expected) class TestX86Simulator(unittest.TestCase): From 728c522f86822da9e5f806e91aa139450b86d73d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 08:23:41 +0000 Subject: [PATCH 15/18] Share console argument checks; document the architecture - Console: _require_program() and _parse_positive() replace the "no program loaded" and positive-count checks repeated across commands; disasm's address/count parsing is no longer duplicated per branch - docs/dev/architecture.md: module map, how programs are run and assembled, and a checklist for adding an ISA - tests/README.md rewritten: it described a Sail/C backend and test counts from long ago - Python API docs, changelog, and README updated for the new modules Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D764BuGwyCPv1b2tstwGAS --- CHANGELOG.md | 19 ++ README.md | 5 +- docs/README.md | 1 + docs/dev/architecture.md | 98 ++++++++++ mapachespim/README.md | 18 +- mapachespim/console.py | 120 +++++------- tests/README.md | 387 ++++++--------------------------------- 7 files changed, 241 insertions(+), 407 deletions(-) create mode 100644 docs/dev/architecture.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f19a490..d996742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## Unreleased +### Code structure +- Each ISA is described once, in `mapachespim/isa.py`; the simulator, disassembler, ELF loader and + writer, assembler, console, and command-line tools all read their ISA-specific facts from it. + Emulator constants are in `engines.py` and each ISA's assembler hooks in `toolchain/targets.py`. + See `docs/dev/architecture.md`, which also explains how to add an ISA. +- New `Simulator.run_until()` with `StopReason` and `RunResult`; the console uses it for `run` + and `-e`. New `Simulator.spec`, `symbols`, `get_sections()`, and `find_section()`. +- Removed: `mapachespim.toolchain.memory_map` (use `mapachespim.memory_map`), the `SailSimulator` + alias, the per-ISA `*Config` classes, `STACK_TOP`/`STACK_SIZE`/`STACK_ADDR`, and + `mapachespim.elf_loader.Architecture`. +- Tests that every registered ISA must pass: syscalls, register mapping, ELF output, and more. + +### Fixed +- ARM64 registers x29 (fp) and x30 (lr) showed and changed the wrong (vector) registers. +- ARM64 programs crashed on `read_int` and `read_char`: x0 was treated as a read-only register. +- 32-bit ARM and RISC-V ELF files were run as if they were 64-bit programs; they are now refused. +- ELF symbol tables written by the assembler listed local symbols after global ones and put every + symbol in `.text`. The example binaries are rebuilt. + ### Installation - All four ISAs are assembled by new built-in pure-Python assemblers, so every ISA assembles on every platform. The Keystone dependency is gone: it had no prebuilt packages for ARM machines diff --git a/README.md b/README.md index b3955cd..d3fc58c 100644 --- a/README.md +++ b/README.md @@ -300,10 +300,11 @@ python -m pytest tests/ # run the test suite ruff check mapachespim/ # lint ruff format mapachespim/ # format mypy mapachespim/ # type check -make -C examples DEBUG=1 # rebuild the example binaries +make -B -C examples DEBUG=1 # rebuild the example binaries ``` -See [docs/RELEASING.md](https://github.com/UCSBarchlab/MapacheSPIM/blob/main/docs/RELEASING.md) for how to publish a release. +See [docs/dev/architecture.md](https://github.com/UCSBarchlab/MapacheSPIM/blob/main/docs/dev/architecture.md) +for how the code is organized and how to add an ISA, and [docs/RELEASING.md](https://github.com/UCSBarchlab/MapacheSPIM/blob/main/docs/RELEASING.md) for how to publish a release. ## License diff --git a/docs/README.md b/docs/README.md index d399663..670a273 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,7 @@ This directory contains all project documentation organized by audience and purp ### For Developers and Maintainers - [Python API](../mapachespim/README.md) - Using the simulator and assembler from Python +- [Architecture](dev/architecture.md) - How the code is organized, and how to add an ISA - [Tests](../tests/README.md) - The test suite - [Releasing](RELEASING.md) - Publishing a new version to PyPI - [Changelog](../CHANGELOG.md) diff --git a/docs/dev/architecture.md b/docs/dev/architecture.md new file mode 100644 index 0000000..8f95a9a --- /dev/null +++ b/docs/dev/architecture.md @@ -0,0 +1,98 @@ +# Architecture and Adding an ISA + +MapacheSPIM has two halves that share one description of each instruction set: + +- the **simulator**, which loads ELF executables and runs them on the + [Unicorn Engine](https://www.unicorn-engine.org/), disassembling with + [Capstone](https://www.capstone-engine.org/), and +- the **toolchain**, a pure-Python assembler that produces the same bytes as GNU `as` and writes + ELF files with DWARF line information. + +Nothing outside the files listed under "Per-ISA code" below branches on which ISA is in use. +Everything else asks the ISA registry. + +## Module map + +``` +mapachespim/ + isa.py ISASpec for each ISA: names, memory layout, registers, syscall ABI, + syscall instruction, instruction sizes, ELF machine, GNU as dialect + memory_map.py MemoryLayout (where .text/.data/stack go) for each ISA + engines.py Unicorn and Capstone constants for each ISA (per-ISA code) + + unicorn_backend.py UnicornSimulator: machine setup, step/run_until, registers, memory + syscalls.py SPIM syscalls (table-driven), ProgramIO for program stdin/stdout + disassembler.py Capstone wrapper + symbols.py SymbolTable: name -> address and address -> nearest symbol + elf_loader.py Reads an ELF's ISA, segments, sections, and symbols + debug_info.py DWARF line table -> source lines (for the console's `list`) + examples.py The bundled example programs + + console.py The interactive console and the `mapachespim` command + + toolchain/ + __init__.py assemble(), assemble_file() + assembler.py ISA-independent assembler: layout passes, labels, fixups, ELF output + targets.py A Target per ISA that drives its encoder (per-ISA code) + directives.py GNU as directive parser and source splitting + riscv.py, mips.py, arm64.py, x86.py Instruction encoders (per-ISA code) + expr.py Expression evaluation shared by the encoders + elf_builder.py ELF32/ELF64 writer + dwarf.py DWARF .debug_line writer + cli.py The `mapachespim-as` command +``` + +## How a program runs + +1. `elf_loader.load_elf_file()` reads the ELF header and asks `isa.spec_for_elf()` which + `ISASpec` matches its machine and word size. +2. `UnicornSimulator.load_elf()` creates a Unicorn machine from `engines.engine_for(spec)`, maps the + spec's memory layout, copies in the segments, and points the stack pointer at the layout's stack. +3. `step()` checks whether the next instruction matches `spec.syscall_instruction`. If so it skips + it and returns `StepResult.SYSCALL`; otherwise Unicorn executes one instruction. +4. `check_termination()` performs a pending syscall with `syscalls.SyscallHandler`, which reads the + syscall number and argument from the registers named by `spec.syscall_abi`. +5. `run_until()` repeats steps until a `StopReason` (exit, error, breakpoint, ...) or a step limit. + The console's `run`, `-e` mode, and `Simulator.run()` all use it. + +## How a program is assembled + +1. `DirectiveParser` splits the source into statements and sections, following GNU `as` rules + for the ISA (`spec.asm`: comment characters, `.word` size, `.align` meaning, MIPS data + auto-alignment). +2. `Assembler` asks `targets.target_for(isa, parser)` for a `Target`. The target may rewrite the + program first (ARM64 literal pools), gives the encoder per-line options (MIPS `.set reorder`, + x86 syntax), supplies the nops used to pad code, and can grow instructions during layout (x86 + short/long jumps). +3. The assembler lays out `.text` until label addresses are stable, encodes each instruction with + the target, fills in data that refers to labels, and writes an ELF file with `ELFBuilder` + (using `spec.elf_machine`, `spec.elf_flags`, word size, and byte order). + +## Adding an ISA + +1. **`isa.py`** — add a member to `ISA` and an `ISASpec`, and list it in `ISA_SPECS`. Add a + `MemoryLayout` for it in `memory_map.py`. If the ISA has instructions that deliberately trap + (like MIPS `break` for division by zero), give it a `describe_trap` function. +2. **`engines.py`** — add an `EngineBinding` with the Unicorn architecture, mode, PC and SP + registers, one Unicorn register constant per register in `spec.registers.names` (look them up + by name: Unicorn's constants are not always consecutive), and the Capstone architecture and mode. +3. **Assembler** — write `toolchain/.py` with `encode(text, address, labels, **options)` and + `instruction_size(text, labels, address, **options)`, then add a `Target` subclass in + `toolchain/targets.py` and list it in `TARGETS`. Override only what differs from the defaults + (for example `code_padding` to pad code with that ISA's nop). +4. **Examples** — add `examples//` with at least `hello_asm`, `fibonacci`, + `array_stats`, `matrix_multiply`, and `guess_game`, a `linker.ld`, and a `Makefile` target, then + build the binaries with `make -B debug` in `examples/`. +5. **Tests** — add the ISA's entries to `SYSCALL_PROGRAMS` and `SET_REGISTER` in + `tests/test_isa_conformance.py`. If GNU binutils exist for it, add a + `tests/test_differential_.py` like the existing ones. + +The registry tests (`tests/test_isa_registry.py`) fail until steps 1–4 are consistent, and the +conformance tests then run the new ISA through the whole pipeline: assembling, loading, every +syscall, and every register. Most other cross-ISA tests (`test_e2e_all_isas.py`, +`test_elf_output.py`, `test_syscalls.py`) are parametrized over the registry or the examples and +cover the new ISA automatically. + +## Testing + +See [tests/README.md](../../tests/README.md). diff --git a/mapachespim/README.md b/mapachespim/README.md index 14869ba..28697bc 100644 --- a/mapachespim/README.md +++ b/mapachespim/README.md @@ -60,8 +60,14 @@ print(f"Executed {steps} instructions, exit code {sim.exit_code}") - After the program exits, returns `HALT` without executing anything - `check_termination(result)` - Perform a pending syscall and decide whether to stop - - Returns: `(stop, reason)`, where reason is `"syscall_exit"`, `"halt"`, `"error"`, - `"tohost"`, or `None` + - Returns: `(stop, reason)`, where reason is a `StopReason` (`EXIT`, `HALT`, `ERROR`, or + `TOHOST`) or `None`. `StopReason` values compare equal to the strings `"syscall_exit"`, + `"halt"`, `"error"`, and `"tohost"`. + +- `run_until(max_steps=None, stop_before=None)` - Run until the program stops + - `stop_before(pc)` is called before each instruction except the first and can return a + `StopReason` (e.g. `StopReason.BREAKPOINT`) to stop there + - Returns: `RunResult(steps, reason, pc)`; `reason` is `None` if `max_steps` was reached - `run(max_steps=None)` - Run until the program stops or `max_steps` instructions - `max_steps`: Maximum instructions (`None` = unlimited) @@ -84,8 +90,9 @@ print(f"Executed {steps} instructions, exit code {sim.exit_code}") - `get_all_regs()` - Get all GPRs as list - `get_isa()` - Get current ISA enum - `get_isa_name()` - Get ISA name as string -- `get_register_count()` - Get number of GPRs (32 for RISC-V/ARM, 16 for x86-64) +- `get_register_count()` - Get number of GPRs (32 for RISC-V/MIPS/ARM, 16 for x86-64) - `get_reg_name(n)` - Get ABI name for register n +- `spec` - The current ISA's `ISASpec` (register names, syscall ABI, memory layout, ...) #### Memory Access - `read_mem(addr, length)` - Read memory, returns `bytes` @@ -95,6 +102,8 @@ print(f"Executed {steps} instructions, exit code {sim.exit_code}") - `get_symbols()` - Get all symbols as `{name: address}` dict - `lookup_symbol(name)` - Look up symbol address by name - `addr_to_symbol(addr)` - Convert address to `(name, offset)` tuple +- `symbols` - The `SymbolTable`; `symbols.describe(addr)` gives e.g. `""` +- `get_sections()` / `find_section(name)` - The loaded ELF's sections (name, address, size, flags) #### Disassembly - `disasm(addr)` - Disassemble instruction at address @@ -107,6 +116,9 @@ print(f"Executed {steps} instructions, exit code {sim.exit_code}") - `ISA.X86_64` - x86-64 - `ISA.MIPS` - MIPS32 (big-endian) +Everything MapacheSPIM knows about each ISA is in its `ISASpec` (`mapachespim.ISA_SPECS`, or +`mapachespim.get_spec("mips32")`). See [the architecture guide](../docs/dev/architecture.md). + ### Helper Functions - `create_simulator(elf_path)` - Create simulator and load ELF in one call - `detect_elf_isa(elf_path)` - Detect ISA from ELF file diff --git a/mapachespim/console.py b/mapachespim/console.py index 03e0b9a..86227e5 100644 --- a/mapachespim/console.py +++ b/mapachespim/console.py @@ -312,9 +312,9 @@ def do_reload(self, arg: str) -> None: run # Find a bug, edit myprog.s... reload # Re-assemble and load the fixed version """ - if not self.loaded_file: - self.print_error('Error: No program loaded. Use "load " first.') + if not self._require_program(): return + assert self.loaded_file is not None # Remember which label each breakpoint was on so it can follow the label old_symbols = {addr: name for name, addr in self.sim.get_symbols().items()} @@ -423,20 +423,12 @@ def do_step(self, arg: str) -> None: - After stepping, use 'regs' to see register changes - Set breakpoints before stepping to stop at key locations """ - if not self.loaded_file: - self.print_error('Error: No program loaded. Use "load " first.') + if not self._require_program(): return - n_steps = 1 - if arg: - try: - n_steps = int(arg) - if n_steps <= 0: - self.print_error("Error: Number of steps must be positive.") - return - except ValueError: - self.print_error(f'Error: Invalid number "{arg}".') - return + n_steps = self._parse_positive(arg, "Number of steps", "number", base=10) if arg else 1 + if n_steps is None: + return if self._report_if_exited(): return @@ -523,19 +515,13 @@ def do_run(self, arg: str) -> None: - After running, use 'pc' and 'regs' to inspect state - Use 'continue' to resume after hitting a breakpoint """ - if not self.loaded_file: - self.print_error('Error: No program loaded. Use "load " first.') + if not self._require_program(): return - max_steps = 0 # 0 means unlimited + max_steps: Optional[int] = None # no limit if arg: - try: - max_steps = int(arg) - if max_steps <= 0: - self.print_error("Error: Max steps must be positive.") - return - except ValueError: - self.print_error(f'Error: Invalid number "{arg}".') + max_steps = self._parse_positive(arg, "Max steps", "number", base=10) + if max_steps is None: return if self._report_if_exited(): @@ -545,7 +531,7 @@ def do_run(self, arg: str) -> None: self._running = True self._interrupted = False try: - result = self.sim.run_until(max_steps or None, stop_before=self._stop_before) + result = self.sim.run_until(max_steps, stop_before=self._stop_before) finally: self._running = False @@ -623,8 +609,7 @@ def do_reset(self, arg: str) -> None: - 'reset' restarts the version that was loaded; after editing a .s file, use 'reload' to re-assemble it """ - if not self.loaded_file: - self.print_error('Error: No program loaded. Use "load " first.') + if not self._require_program(): return try: self.sim.reset() @@ -658,6 +643,25 @@ def _end_program_output(self) -> None: print(file=self.stdout) self.sim.output_needs_newline = False + def _require_program(self) -> bool: + """True if a program is loaded; otherwise say how to load one""" + if self.loaded_file: + return True + self.print_error('Error: No program loaded. Use "load " first.') + return False + + def _parse_positive(self, text: str, name: str, noun: str, base: int = 0) -> Optional[int]: + """Parse a positive integer argument, printing an error (and returning None) if invalid""" + try: + value = int(text, base) + except ValueError: + self.print_error(f'Error: Invalid {noun} "{text}".') + return None + if value <= 0: + self.print_error(f"Error: {name} must be positive.") + return None + return value + def _report_if_exited(self) -> bool: """If the program already exited, say so and return True""" if self.sim.exited: @@ -936,14 +940,10 @@ def do_mem(self, arg: str) -> None: # Parse length (default 256 bytes) length = 256 if len(parts) > 1: - try: - length = int(parts[1], 0) - if length <= 0: - self.print_error("Error: Length must be positive.") - return - except ValueError: - self.print_error(f'Error: Invalid length "{parts[1]}".') + parsed_length = self._parse_positive(parts[1], "Length", "length") + if parsed_length is None: return + length = parsed_length # Check if it's a section name (starts with .) if addr_or_section.startswith("."): @@ -1039,44 +1039,19 @@ def do_disasm(self, arg: str) -> None: - Instruction sizes: RISC-V/ARM64/MIPS = 4 bytes, x86-64 = variable - Use 'mem ' to see raw instruction bytes """ - parts = arg.split() if arg else [] - - # Default to current PC if no args - if not parts: - addr = self.sim.get_pc() - count = 10 - elif parts[0].lower() in ("pc", "$"): - # "pc" or "$" means current PC - addr = self.sim.get_pc() - count = 10 - if len(parts) > 1: - try: - count = int(parts[1], 0) - if count <= 0: - self.print_error("Error: Count must be positive.") - return - except ValueError: - self.print_error(f'Error: Invalid count "{parts[1]}".') - return - else: - # Parse address or symbol name - parsed = self._parse_address(parts[0]) - if parsed is None: - self.print_error(f'Error: "{parts[0]}" is not a valid address or known symbol.') + parts = arg.split() + where = parts[0] if parts else "pc" + # "pc" or "$" means the current PC + addr = self.sim.get_pc() if where.lower() in ("pc", "$") else self._parse_address(where) + if addr is None: + self.print_error(f'Error: "{where}" is not a valid address or known symbol.') + return + count = 10 + if len(parts) > 1: + parsed_count = self._parse_positive(parts[1], "Count", "count") + if parsed_count is None: return - addr = parsed - - # Parse count (default 10) - count = 10 - if len(parts) > 1: - try: - count = int(parts[1], 0) - if count <= 0: - self.print_error("Error: Count must be positive.") - return - except ValueError: - self.print_error(f'Error: Invalid count "{parts[1]}".') - return + count = parsed_count # Disassemble instructions (x86-64 instructions vary in length, so # advance by each instruction's actual size) @@ -1122,8 +1097,7 @@ def do_list(self, arg: str) -> None: - Shows 10 lines by default - Current PC is marked with '# <-- PC: 0xXXXXXXXX' """ - if not self.loaded_file: - self.print_error('Error: No program loaded. Use "load " first.') + if not self._require_program(): return if not self.source_info.has_debug_info: diff --git a/tests/README.md b/tests/README.md index 034c2c5..cf21608 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,343 +1,72 @@ # MapacheSPIM Test Suite -Comprehensive unit tests for the MapacheSPIM simulator. +## Running the tests -## Running Tests - -**Run the whole suite** (from the repository root): +From the repository root: ```bash -python -m pytest tests/ +python -m pytest tests/ # everything +python -m pytest tests/test_isa_conformance.py # one file +python -m pytest tests/ -k mips # tests whose names mention mips ``` -**Assembler differential tests** compare the built-in assemblers (all four ISAs) with GNU `as` -byte for byte, using randomized programs. They need the GNU cross binutils and are skipped without -them: +CI runs the suite on Linux, Windows, and macOS (Apple Silicon) against the installed wheel, plus +`ruff check`, `ruff format --check`, and `mypy` on `mapachespim/`. + +### Comparing the assemblers with GNU `as` + +The `test_differential_*.py` tests assemble randomized programs with both the built-in assemblers +and GNU `as` and require byte-identical output. They need the GNU cross binutils and are skipped +without them: ```bash sudo apt install binutils-riscv64-linux-gnu binutils-mips-linux-gnu \ binutils-aarch64-linux-gnu binutils-x86-64-linux-gnu python -m pytest tests/test_differential_*.py -MAPACHESPIM_FUZZ_SCALE=50 python -m pytest tests/test_differential_*.py # much deeper -``` - -**Run individual suites:** - -```bash -python -m pytest tests/test_simulator.py # API tests -python -m pytest tests/test_console_working.py # Console tests -python -m pytest tests/test_riscv_encoder.py # Built-in RISC-V assembler -python -m pytest tests/test_mips_encoder.py # Built-in MIPS assembler -python -m pytest tests/test_loading.py # load/reload/examples and the CLI -python -m pytest tests/test_regressions.py # Fixed bugs -``` - -**Run with verbose output:** - -```bash -python3 tests/test_simulator.py -v -``` - -**Run specific test class:** - -```bash -python3 -m unittest tests.test_simulator.TestFibonacci -``` - -**Run specific test:** - -```bash -python3 -m unittest tests.test_simulator.TestFibonacci.test_fibonacci_single_step -``` - -## Test Coverage - -### TestSimulatorBasic (3 tests) -- ✅ Initialization without config -- ✅ Context manager support -- ✅ PC access after initialization - -### TestFibonacci (5 tests) -Tests using the `fibonacci` example program: -- ✅ ELF loading -- ✅ Entry point verification (0x80000000) -- ✅ Single-step execution -- ✅ Multiple steps (10 instructions) -- ✅ Run with max_steps limit - -### TestMatrixMultiply (4 tests) -Tests using the `matrix_multiply` example program: -- ✅ ELF loading -- ✅ Entry point verification (0x80000000) -- ✅ Single-step execution -- ✅ Run with max_steps limit - -### TestRegisterAccess (9 tests) -- ✅ Get all 32 registers -- ✅ x0 always reads as 0 -- ✅ Read all valid registers (0-31) -- ✅ Invalid register numbers raise ValueError -- ✅ Set register value -- ✅ Cannot set x0 (raises ValueError) -- ✅ 64-bit value overflow handling - -### TestMemoryAccess (4 tests) -- ✅ Read memory at PC -- ✅ Read 4-byte instruction -- ✅ Write and read back memory -- ✅ String to bytes conversion - -### TestPCAccess (3 tests) -- ✅ Get program counter -- ✅ Set program counter -- ✅ PC advances after step - -### TestReset (1 test) -- ✅ Reset clears simulator state - -### TestEdgeCases (3 tests) -- ✅ Loading nonexistent file raises error -- ✅ Stepping before loading program -- ✅ Run with max_steps=0 (unlimited) - -## Test Statistics - -- **Total Tests:** 31 -- **Pass Rate:** 100% -- **Test Time:** ~0.05 seconds -- **Programs Tested:** fibonacci, matrix_multiply -- **API Functions Tested:** All 13 core functions - -## Tested Programs - -### Fibonacci (Recursive) -- **Path:** `examples/riscv/fibonacci/fibonacci` -- **Entry:** 0x80000000 -- **Tests:** Single-step, multi-step, run - -### Matrix Multiplication (3x3) -- **Path:** `examples/riscv/matrix_multiply/matrix_mult` -- **Entry:** 0x80000000 -- **Tests:** Single-step, multi-step, run - -## API Functions Tested - -| Function | Tested | Test Count | -|----------|--------|------------| -| `Simulator()` | ✅ | 3 | -| `load_elf()` | ✅ | 4 | -| `step()` | ✅ | 5 | -| `run()` | ✅ | 4 | -| `get_pc()` | ✅ | 4 | -| `set_pc()` | ✅ | 1 | -| `get_reg()` | ✅ | 4 | -| `set_reg()` | ✅ | 3 | -| `get_all_regs()` | ✅ | 1 | -| `read_mem()` | ✅ | 3 | -| `write_mem()` | ✅ | 2 | -| `reset()` | ✅ | 1 | -| Context manager | ✅ | 1 | - -## Adding New Tests - -To add tests for a new RISC-V program: - -```python -class TestMyProgram(unittest.TestCase): - """Tests using my_program example""" - - def setUp(self): - self.sim = Simulator() - self.elf_path = "examples/my_program/my_program" - - def test_load_program(self): - """Test loading the program""" - result = self.sim.load_elf(self.elf_path) - self.assertTrue(result) - - def test_program_behavior(self): - """Test specific program behavior""" - self.sim.load_elf(self.elf_path) - # Execute and verify - self.sim.run(100) - # Check results - result_reg = self.sim.get_reg(10) # a0 register - self.assertEqual(result_reg, expected_value) +MAPACHESPIM_FUZZ_SCALE=50 python -m pytest tests/test_differential_*.py # much deeper (CI runs this) ``` -## Continuous Testing - -These tests serve as regression tests to ensure: -- API stability across changes -- Correct simulator behavior with different programs -- Error handling works as expected -- Memory safety and bounds checking - -Run tests before committing changes to verify nothing broke! - ---- - -## Console Command Tests - -### test_console_working.py -**Comprehensive console command test suite** - 38 tests covering all console commands - -Status: ✅ **All 38 tests passing** - -```bash -# Run console command tests -python3 tests/test_console_working.py -``` - -#### Coverage - -| Command | Tests | Status | -|---------|-------|--------| -| `load` | 3 tests | ✅ Pass | -| `step` | 5 tests | ✅ Pass | -| `run` | 2 tests | ✅ Pass | -| `break` | 3 tests | ✅ Pass | -| `info breakpoints` | 1 test | ✅ Pass | -| `delete` | 1 test | ✅ Pass | -| `clear` | 1 test | ✅ Pass | -| `continue` | 1 test | ✅ Pass | -| `regs` | 1 test | ✅ Pass | -| `pc` | 1 test | ✅ Pass | -| `mem` | 4 tests | ✅ Pass | -| `info sections` | 3 tests | ✅ Pass | -| `list` (source display) | 7 tests | ✅ Pass | -| `status` | 2 tests | ✅ Pass | -| **Aliases** | 3 tests | ✅ Pass | - -#### Test Program: examples/riscv/test_simple/simple - -A deterministic 9-instruction program with documented cycle-by-cycle behavior. - -**Expected final register values:** -``` -x1 (ra) = 93 (0x5d) - exit syscall number -x5 (t0) = 10 (0x0a) -x6 (t1) = 20 (0x14) -x7 (t2) = 30 (0x1e) -x8 (s0) = 20 (0x14) -x9 (s1) = 40 (0x28) -x10 (a0) = 42 (0x2a) - exit code -``` - -**Documentation:** See `examples/riscv/test_simple/EXPECTED_BEHAVIOR.md` for complete step-by-step behavior documentation. - -#### Additional Test Files - -- **test_simple_basic.py** - Basic simulator functionality test -- **test_run_completion.py** - Detailed program execution through completion - -## Test Summary - -Run `python -m pytest tests/ -q` for the current count; CI runs the full suite on Linux, Windows, -and macOS (Apple Silicon). - -**Programs Tested:** -- fibonacci (recursive calculation) -- matrix_multiply (3x3 matrices) -- test_simple (deterministic 9-instruction program) -- hello_world (I/O syscall test program) - ---- - -## Program Correctness Tests - -### test_program_correctness.py -**Critical validation tests** - 10 tests verifying programs produce correct results - -Status: All 10 tests passing - -```bash -# Run program correctness tests -python3 tests/test_program_correctness.py -``` - -#### Test Classes - -| Test Class | Tests | Status | Description | -|------------|-------|--------|-------------| -| `TestFibonacciCorrectness` | 4 tests | Pass | Verifies Fibonacci(7) = 13 | -| `TestToHostMechanism` | 2 tests | Pass | Verifies HTIF exit mechanism | -| `TestMatrixMultiplyCorrectness` | 2 tests | Pass | Verifies matrix result correct | -| `TestProgramCompletion` | 2 tests | Pass | Verifies completion detection | - -#### Key Validations - -**Fibonacci Correctness:** -- Fibonacci(7) returns 13 (correct result) -- Program completes in ~462 steps (not infinite loop) -- Result written to memory correctly -- tohost mechanism works - -**Matrix Multiply Correctness:** -- Computes correct 3x3 result matrix -- All 9 elements verified individually -- Completes in ~663 steps - -**Expected Results:** -``` -Fibonacci(7) = 13 -Matrix C = [[30, 24, 18], [84, 69, 54], [138, 114, 90]] -``` - -These tests verify the simulator's HTIF (Host-Target Interface) tohost detection -mechanism, which allows programs to signal completion by writing to the tohost -symbol. Without this, programs would run indefinitely in their exit loops. - ---- - -## I/O Syscall Tests - -### test_io_syscalls.py -**SPIM-compatible syscall emulation tests** - 4 tests verifying I/O functionality - -Status: All 4 tests passing - -```bash -# Run I/O syscall tests -python3 tests/test_io_syscalls.py -``` - -#### Coverage - -| Test Class | Tests | Status | Description | -|------------|-------|--------|-------------| -| `TestPrintStringSyscall` | 2 tests | Pass | Syscall 4 (print_string) | -| `TestSyscallDetection` | 1 test | Pass | SYSCALL step result | -| `TestFibonacciStillWorks` | 1 test | Pass | Regression test | - -#### Syscalls Tested - -- **Syscall 4 (print_string)**: Print null-terminated string -- **Syscall 10 (exit)**: Exit program cleanly -- **StepResult.SYSCALL**: Verify ecall detection in C backend - -#### Test Program: examples/riscv/hello_world/hello - -Simple program that prints "Hello, World!\n" and exits. - -**Assembly:** -```asm -_start: - la a0, msg # Load address of string - li a7, 4 # Syscall 4 = print_string - ecall - - li a7, 10 # Syscall 10 = exit - ecall -``` - -**Expected behavior:** -- Prints "Hello, World!\n" to stdout -- Exits cleanly in 6 steps - -#### Implementation Details - -- **ISA-agnostic design**: Syscall detection in C backend (ecall opcode 0x00000073) -- **PC advancement**: C backend advances PC by 4 bytes past ecall (RISC-V specific) -- **Trap bypass**: User-level syscall emulation bypasses Sail's trap mechanism -- **Clean exit**: Programs exit immediately on syscall 10, not hitting step limits +## What is tested where + +Tests that run for **every registered ISA** (parametrized over `mapachespim.isa.ISA_SPECS` or the +bundled examples), so a new ISA is covered as soon as it is registered: + +| File | What it checks | +|------|----------------| +| `test_isa_registry.py` | Every ISA has a consistent spec, engine binding, assembler target, and examples; name/alias/ELF lookups; `.isa` directives | +| `test_isa_conformance.py` | Assembles and runs small programs per ISA: every syscall, every register mapping, the register API, the default memory map | +| `test_syscalls.py` | `SyscallHandler` and `ProgramIO` with a fake machine (no emulator) | +| `test_elf_output.py` | ELF headers and symbol tables are well formed; bundled binaries match the assembler; the loader rejects unsupported files | +| `test_e2e_all_isas.py` | Every example program on every ISA: loading, stepping, results, console, `-e` mode | +| `test_simulator_parts.py` | `SymbolTable`, `Disassembler`, and `run_until()` | + +Other areas: + +| Area | Files | +|------|-------| +| Simulator API | `test_simulator.py`, `test_python_bindings.py`, `test_multi_isa.py`, `test_loading.py`, `test_symbols.py`, `test_program_correctness.py`, `test_io_syscalls.py`, `test_cross_isa_regression.py`, `test_mips_backend.py`, `test_x86_backend.py`, `test_arm_*.py` | +| Console | `test_console_working.py`, `test_console_commands.py`, `test_console_disasm.py`, `test_debug_info.py`, `test_disasm*.py` | +| Assembler | `test_toolchain.py`, `test_isa_specification.py`, `test_forward_references.py`, `test_assembler_accuracy.py`, `test_*_encoder.py`, `test_differential_*.py` (with `gnu_reference.py`) | +| Fixed bugs | `test_regressions.py` | + +## Writing tests + +- Prefer the public API (`Simulator`, `assemble`, the console's `onecmd`) over private attributes, + so tests survive refactoring. +- For behavior every ISA should share, parametrize over `ISA_SPECS` instead of copying a test per + ISA: + + ```python + import pytest + from mapachespim import ISA_SPECS, Simulator + + @pytest.mark.parametrize("spec", ISA_SPECS, ids=lambda spec: spec.name) + def test_stack_pointer_starts_near_stack_top(spec): + sim = Simulator(isa=spec.isa) + ... + ``` + +- To give a program input or capture its output, set `sim.stdin` / `sim.stdout` to `io.StringIO` + objects and use `sim.run_until()`; the real syscalls then run. +- If a change to the assembler changes the bundled example binaries, rebuild them with + `make -B debug` in `examples/` (`test_elf_output.py` checks they are current). From 67972c727c8c20e10b63797f981d22e4b1cf8803 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 16:55:57 +0000 Subject: [PATCH 16/18] Keep conformance-test ELF files in pytest's tmp_path They were written with mkstemp and never deleted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D764BuGwyCPv1b2tstwGAS --- tests/test_isa_conformance.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/tests/test_isa_conformance.py b/tests/test_isa_conformance.py index 1757de9..a7396d9 100644 --- a/tests/test_isa_conformance.py +++ b/tests/test_isa_conformance.py @@ -149,26 +149,17 @@ SPECS = pytest.mark.parametrize("spec", ISA_SPECS, ids=lambda spec: spec.name) -def _load(spec, source): +def _load(spec, source, tmp_path): result = assemble(source, isa=spec.name) assert result.success, result.errors + elf = tmp_path / f"{spec.name}.elf" + elf.write_bytes(result.elf_bytes) sim = Simulator() - elf = _write_temp(result.elf_bytes) - sim.load_elf(elf) + sim.load_elf(str(elf)) assert sim.spec is spec return sim -def _write_temp(data): - import os - import tempfile - - fd, path = tempfile.mkstemp(suffix=".elf") - with os.fdopen(fd, "wb") as f: - f.write(data) - return path - - def test_every_isa_has_conformance_programs(): for spec in ISA_SPECS: assert spec.name in SYSCALL_PROGRAMS, f"add a syscall program for {spec.name}" @@ -176,8 +167,8 @@ def test_every_isa_has_conformance_programs(): @SPECS -def test_syscalls(spec): - sim = _load(spec, SYSCALL_PROGRAMS[spec.name]) +def test_syscalls(spec, tmp_path): + sim = _load(spec, SYSCALL_PROGRAMS[spec.name], tmp_path) sim.stdin = StringIO("41\nZ\n") sim.stdout = StringIO() result = sim.run_until(1000) @@ -187,7 +178,7 @@ def test_syscalls(spec): @SPECS -def test_every_register_is_mapped(spec): +def test_every_register_is_mapped(spec, tmp_path): """Each register number reads the architectural register of that name.""" regs = spec.registers lines = [f".isa {spec.name}", ".text", ".globl _start", "_start:"] @@ -198,7 +189,7 @@ def test_every_register_is_mapped(spec): value = 100 + 8 * n # small, positive, and 8-byte aligned for sp lines.append(SET_REGISTER[spec.name](n, name, value)) expected[n] = value - sim = _load(spec, "\n".join(lines)) + sim = _load(spec, "\n".join(lines), tmp_path) before = sim.get_all_regs() result = sim.run_until(len(expected)) assert result.reason is None, sim.last_error @@ -241,9 +232,9 @@ def test_default_machine(spec): @SPECS -def test_unknown_syscall_is_an_error(spec): +def test_unknown_syscall_is_an_error(spec, tmp_path): program = SYSCALL_PROGRAMS[spec.name] - sim = _load(spec, program) + sim = _load(spec, program, tmp_path) sim.set_reg(spec.syscall_abi.number, 999) # Jump straight to the first syscall instruction pattern = spec.syscall_instruction From 5f1f6f894712ef16979ada282f92e850b8ef85c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 16:58:26 +0000 Subject: [PATCH 17/18] Fix tab completion paths and temp-file tests on Windows - complete_load: glob joins with backslashes on Windows, so completing "examples/" gave "examples\arm/", which no longer extends the typed text. Keep forward slashes unless the user typed backslashes. New test simulates Windows' glob output. - test_isa_specification: the TestAssembleFile tests unlinked NamedTemporaryFile files while they were still open, which Windows refuses. Use pytest's tmp_path instead. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D764BuGwyCPv1b2tstwGAS --- mapachespim/console.py | 6 +- tests/test_console_commands.py | 12 ++++ tests/test_isa_specification.py | 124 ++++++++++++-------------------- 3 files changed, 61 insertions(+), 81 deletions(-) diff --git a/mapachespim/console.py b/mapachespim/console.py index 86227e5..43ea1df 100644 --- a/mapachespim/console.py +++ b/mapachespim/console.py @@ -9,6 +9,7 @@ from __future__ import annotations import cmd +import os import signal import sys import tempfile @@ -1659,8 +1660,11 @@ def complete_load(self, text: str, line: str, begidx: int, endidx: int) -> List[ # No text yet, list current directory pattern = "*" - # Get matching paths + # Get matching paths. On Windows glob joins with backslashes; keep + # the forward slashes the user typed so completions extend their text. matches = glob.glob(pattern) + if os.sep != "/" and "\\" not in text: + matches = [m.replace(os.sep, "/") for m in matches] # Also offer bundled example names like "riscv/fibonacci" if not use_tilde and text.count("/") <= 1: diff --git a/tests/test_console_commands.py b/tests/test_console_commands.py index 6a93110..5805d84 100644 --- a/tests/test_console_commands.py +++ b/tests/test_console_commands.py @@ -580,6 +580,18 @@ def test_complete_load_returns_full_paths(self): for r in result: self.assertTrue(r.startswith('examples/'), f"Expected path to start with 'examples/', got '{r}'") + def test_complete_load_keeps_forward_slashes_on_windows(self): + """On Windows glob returns backslash paths; completions keep the user's '/'""" + import os + from unittest import mock + + with mock.patch.object(os, "sep", "\\"), mock.patch( + "glob.glob", return_value=["examples\\arm", "examples\\riscv"] + ): + result = self.console.complete_load('examples/', 'load examples/', 5, 14) + self.assertIn('examples/arm/', result) + self.assertIn('examples/riscv/', result) + def test_complete_break_with_symbols(self): """Test break command completes symbol names""" self.console.onecmd('load examples/riscv/fibonacci/fibonacci') diff --git a/tests/test_isa_specification.py b/tests/test_isa_specification.py index f5cf5b4..e84086e 100644 --- a/tests/test_isa_specification.py +++ b/tests/test_isa_specification.py @@ -1,7 +1,6 @@ """Tests for ISA specification via --isa flag and .isa directive.""" import pytest -import tempfile from pathlib import Path from mapachespim.toolchain import assemble, assemble_file, AssemblyResult @@ -104,102 +103,67 @@ def test_isa_directive_missing_argument(self): assert "requires an argument" in parser.errors[0] +def _source(tmp_path, text, name="program.s"): + """Write assembly source to a file and return its path.""" + path = tmp_path / name + path.write_text(text) + return path + + class TestAssembleFile: """Tests for assemble_file with ISA specification.""" - def test_explicit_isa_flag(self): + def test_explicit_isa_flag(self, tmp_path): """Test that explicit ISA flag works.""" - with tempfile.NamedTemporaryFile(suffix=".s", delete=False, mode='w') as f: - f.write(".text\n_start: nop\n") - f.flush() - - result = assemble_file(f.name, isa="riscv64") - assert result.success - assert result.isa == "riscv64" - - Path(f.name).unlink() + result = assemble_file(_source(tmp_path, ".text\n_start: nop\n"), isa="riscv64") + assert result.success + assert result.isa == "riscv64" - def test_isa_directive_in_file(self): + def test_isa_directive_in_file(self, tmp_path): """Test that .isa directive in file is detected.""" - with tempfile.NamedTemporaryFile(suffix=".s", delete=False, mode='w') as f: - f.write(".isa arm64\n.text\n_start: nop\n") - f.flush() - - result = assemble_file(f.name) # No explicit ISA - assert result.success - assert result.isa == "arm64" - - Path(f.name).unlink() + result = assemble_file(_source(tmp_path, ".isa arm64\n.text\n_start: nop\n")) + assert result.success + assert result.isa == "arm64" - def test_explicit_isa_overrides_directive(self): + def test_explicit_isa_overrides_directive(self, tmp_path): """Test that --isa flag overrides .isa directive in file.""" - with tempfile.NamedTemporaryFile(suffix=".s", delete=False, mode='w') as f: - # File says arm64, but we'll pass riscv64 - f.write(".isa arm64\n.text\n_start: nop\n") - f.flush() - - # Explicit flag should override - result = assemble_file(f.name, isa="riscv64") - assert result.success - assert result.isa == "riscv64" - - Path(f.name).unlink() + # File says arm64, but the explicit flag should win + path = _source(tmp_path, ".isa arm64\n.text\n_start: nop\n") + result = assemble_file(path, isa="riscv64") + assert result.success + assert result.isa == "riscv64" - def test_missing_isa_produces_error(self): + def test_missing_isa_produces_error(self, tmp_path): """Test that missing ISA produces helpful error.""" - with tempfile.NamedTemporaryFile(suffix=".s", delete=False, mode='w') as f: - f.write(".text\n_start: nop\n") - f.flush() - - result = assemble_file(f.name) # No ISA specified anywhere - assert not result.success - assert len(result.errors) == 1 - assert "ISA not specified" in result.errors[0] - assert "--isa" in result.errors[0] - assert ".isa" in result.errors[0] - - Path(f.name).unlink() - - def test_isa_directive_at_top_of_file(self): + result = assemble_file(_source(tmp_path, ".text\n_start: nop\n")) + assert not result.success + assert len(result.errors) == 1 + assert "ISA not specified" in result.errors[0] + assert "--isa" in result.errors[0] + assert ".isa" in result.errors[0] + + def test_isa_directive_at_top_of_file(self, tmp_path): """Test that .isa directive works when at top (after comments).""" - with tempfile.NamedTemporaryFile(suffix=".s", delete=False, mode='w') as f: - f.write("# This is a comment\n") - f.write("// Another comment\n") - f.write(".isa mips32\n") - f.write(".text\n_start: nop\n") - f.flush() - - result = assemble_file(f.name) - assert result.success - assert result.isa == "mips32" - - Path(f.name).unlink() + source = "# This is a comment\n// Another comment\n.isa mips32\n.text\n_start: nop\n" + result = assemble_file(_source(tmp_path, source)) + assert result.success + assert result.isa == "mips32" - def test_all_isas_with_flag(self): + def test_all_isas_with_flag(self, tmp_path): """Test that all ISAs work with explicit flag.""" for isa in ["riscv64", "arm64", "x86_64", "mips32"]: - with tempfile.NamedTemporaryFile(suffix=".s", delete=False, mode='w') as f: - f.write(".text\n_start: nop\n") - f.flush() - - result = assemble_file(f.name, isa=isa) - assert result.success, f"Failed for ISA: {isa}" - assert result.isa == isa - - Path(f.name).unlink() + path = _source(tmp_path, ".text\n_start: nop\n", f"{isa}.s") + result = assemble_file(path, isa=isa) + assert result.success, f"Failed for ISA: {isa}" + assert result.isa == isa - def test_all_isas_with_directive(self): + def test_all_isas_with_directive(self, tmp_path): """Test that all ISAs work with .isa directive.""" for isa in ["riscv64", "arm64", "x86_64", "mips32"]: - with tempfile.NamedTemporaryFile(suffix=".s", delete=False, mode='w') as f: - f.write(f".isa {isa}\n.text\n_start: nop\n") - f.flush() - - result = assemble_file(f.name) - assert result.success, f"Failed for ISA: {isa}" - assert result.isa == isa - - Path(f.name).unlink() + path = _source(tmp_path, f".isa {isa}\n.text\n_start: nop\n", f"{isa}.s") + result = assemble_file(path) + assert result.success, f"Failed for ISA: {isa}" + assert result.isa == isa class TestDirectAssemble: From fa01a576cdfbe597d01f029728bf7158566a537c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 17:03:58 +0000 Subject: [PATCH 18/18] Size RISC-V branches with GNU's view of label addresses The layout passes sized every instruction with the previous pass's label addresses. GNU's relax_segment gives labels already passed in the current pass their new address and only labels ahead their old one, so a branch could be relaxed to bne+jal here but not by GNU. The 50x differential fuzz run in CI found such a program (test_branches); a model of the two rules matched GNU on 160 of 160 random branch programs with GNU's rule and 146 with the old one. Instruction sizing now uses the same _RelaxView the x86 relaxation uses, with no stretch. New regression test at normal fuzz scale. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D764BuGwyCPv1b2tstwGAS --- mapachespim/toolchain/assembler.py | 14 ++++++++++---- tests/test_differential_riscv.py | 10 ++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/mapachespim/toolchain/assembler.py b/mapachespim/toolchain/assembler.py index 6258338..3c18988 100644 --- a/mapachespim/toolchain/assembler.py +++ b/mapachespim/toolchain/assembler.py @@ -38,9 +38,11 @@ def resolve(name: str) -> int: class _RelaxView(Mapping[str, int]): """ - Label lookup during one x86 relaxation pass, the way GNU as sees it: - labels already passed have their new address, labels ahead have their - old address shifted by the growth so far (``stretch``). + Label lookup during one layout pass, the way GNU as sees it: labels + already passed have their new address, labels ahead have their old + address, shifted by the growth so far (``stretch``) for x86's generic + relaxation and unshifted for instruction sizing (as RISC-V's branch + relaxation does). """ def __init__( @@ -467,7 +469,11 @@ def _calculate_builtin_text_labels( line, addr, _RelaxView(data_labels, labels, new_labels, stretch) ) grew = grew or grown - size = self._target.size(line, addr, known) + # As in GNU's relax_segment, labels already passed this + # pass have their new address, labels ahead their old one + size = self._target.size( + line, addr, _RelaxView(data_labels, labels, new_labels, 0) + ) if grown: stretch += size - 2 addr += size diff --git a/tests/test_differential_riscv.py b/tests/test_differential_riscv.py index f036277..2482420 100644 --- a/tests/test_differential_riscv.py +++ b/tests/test_differential_riscv.py @@ -157,6 +157,16 @@ def test_branches(): assert_same(program(lines, labels, rng)) +def test_branch_relaxation_sees_labels_like_gnu(): + """Deciding whether a branch reaches its label, GNU as uses this layout + pass's address for labels already passed and the previous pass's for + labels ahead. Using the previous pass's address for both relaxes an + extra branch in this program (found by the 50x fuzz run).""" + rng = random.Random(4) + lines = [f"beq a0, a1, L{rng.randrange(4)}" for _ in range(1600)] + assert_same(program(lines, 4, rng)) + + def test_jumps(): rng = random.Random(6) labels = 8