From c89faf22bb2479c7007e41e0fd091f00f5fc5d1e Mon Sep 17 00:00:00 2001 From: buzzer-re <22428720+buzzer-re@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:24:45 +0900 Subject: [PATCH 1/4] Add folder exports with cross-binary link mappings `tocode ` exports every binary in a directory (app folders, macOS .app bundles, .NET publish output) in load order, one isolated process per binary, resumable, and links them using only import/load tables and each platform's loader rules: - links/: pure-Python ELF, PE, Mach-O (fat, chained fixups, bind opcodes, export trie) and .NET metadata readers; loader-rule resolution (@rpath/@loader_path, RPATH/RUNPATH/SONAME, app directory, KnownDLLs/API sets, assembly probing, P/Invoke); symbol binding (two-level namespace and re-exports, ELF versions and global scope, PE ordinals and forwarders, .NET member refs by signature). - folder_links: maps bound imports to the defining function's file and line range, finds importer call sites, writes mappings.json and enriches imports.json/functions.json/exports.json. - folder_annotate: line-stable /* tocode-link: */ source comments. - Master AGENTS.md, project.json, inventory.json, dependency-graph.json. - CLI: directory input, --only/--exclude, --no-managed, --arch, --plan, --links-only; two-bar progress display, nested logs go to tocode.log. Also: IDA records PE calls through the IAT as import calls, project.json and export-manifest.json carry image_base, isolated export processes die with their parent, and the .NET export can skip P/Invoke neighbours. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + AGENTS.md | 7 +- DEVELOPMENT.md | 23 +- README.md | 29 +- src/tocode/apk_native.py | 63 +- src/tocode/backends/dotnet_pe.py | 20 +- src/tocode/backends/ida.py | 31 + src/tocode/cli.py | 98 +++ src/tocode/dotnet.py | 9 +- src/tocode/dotnet_metadata.py | 9 +- src/tocode/exporter.py | 4 + src/tocode/folder.py | 965 ++++++++++++++++++++++++++++++ src/tocode/folder_annotate.py | 152 +++++ src/tocode/folder_docs.py | 184 ++++++ src/tocode/folder_links.py | 669 +++++++++++++++++++++ src/tocode/links/__init__.py | 7 + src/tocode/links/analyze.py | 42 ++ src/tocode/links/bind.py | 498 +++++++++++++++ src/tocode/links/binfmt_dotnet.py | 626 +++++++++++++++++++ src/tocode/links/binfmt_elf.py | 390 ++++++++++++ src/tocode/links/binfmt_macho.py | 585 ++++++++++++++++++ src/tocode/links/binfmt_pe.py | 236 ++++++++ src/tocode/links/extract.py | 85 +++ src/tocode/links/inventory.py | 313 ++++++++++ src/tocode/links/model.py | 158 +++++ src/tocode/links/names.py | 121 ++++ src/tocode/links/resolve.py | 598 ++++++++++++++++++ src/tocode/links/runtime_libs.py | 32 + src/tocode/links/sniff.py | 113 ++++ src/tocode/progress.py | 17 +- tests/link_fixtures.py | 711 ++++++++++++++++++++++ tests/test_cli.py | 68 +++ tests/test_folder.py | 336 +++++++++++ tests/test_links.py | 558 +++++++++++++++++ 34 files changed, 7743 insertions(+), 15 deletions(-) create mode 100644 src/tocode/folder.py create mode 100644 src/tocode/folder_annotate.py create mode 100644 src/tocode/folder_docs.py create mode 100644 src/tocode/folder_links.py create mode 100644 src/tocode/links/__init__.py create mode 100644 src/tocode/links/analyze.py create mode 100644 src/tocode/links/bind.py create mode 100644 src/tocode/links/binfmt_dotnet.py create mode 100644 src/tocode/links/binfmt_elf.py create mode 100644 src/tocode/links/binfmt_macho.py create mode 100644 src/tocode/links/binfmt_pe.py create mode 100644 src/tocode/links/extract.py create mode 100644 src/tocode/links/inventory.py create mode 100644 src/tocode/links/model.py create mode 100644 src/tocode/links/names.py create mode 100644 src/tocode/links/resolve.py create mode 100644 src/tocode/links/runtime_libs.py create mode 100644 src/tocode/links/sniff.py create mode 100644 tests/link_fixtures.py create mode 100644 tests/test_folder.py create mode 100644 tests/test_links.py diff --git a/.gitignore b/.gitignore index a1f37f3..6f63baa 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ __pycache__/ *_decompiler/ /dist /build +*_decompiled/ diff --git a/AGENTS.md b/AGENTS.md index ffc41ac..12a0b63 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,8 @@ This repository contains ToCode, a Python-only binary exporter. ToCode takes one - `src/tocode/backends/dotnet.py` / `dotnet_pe.py`: .NET backend (input discovery for assemblies, apphosts, single-file bundles and NuGet packages; dnlib inventory; pure-Python PE/CLR/bundle parsing and IL token scan; worker-side C#/IL decompilation). `backends/dotnet_libs.py` downloads dnlib and ICSharpCode.Decompiler on demand (ask once, hash-pinned) into a per-user folder. - `src/tocode/dotnet.py`: .NET export pipeline; `dotnet_metadata.py`: .NET JSON documents and the generated `AGENTS.md`. - `src/tocode/apk.py`: APK export pipeline (extraction, decompile pool, resource decoding, metadata); `apk_metadata.py`: manifest parsing and Android JSON documents; `apk_native.py`: extraction and background native export of every `.so`. +- `src/tocode/links/`: pure-Python link facts for folder exports (no backend involved): `sniff.py` file kinds; `binfmt_elf.py`, `binfmt_pe.py`, `binfmt_macho.py` (thin/fat, chained fixups, bind opcodes, export trie), `binfmt_dotnet.py` (dnlib-free metadata tables, signatures, P/Invoke); `inventory.py` folder scan; `resolve.py` loader rules per platform; `bind.py` symbol binding; `names.py` backend-name matching keys. +- `src/tocode/folder.py`: folder export orchestration (plan, job keys and resume state in `.tocode-folder/`, one isolated process per binary via `NativeRunner`, the two-bar progress display); `folder_links.py`: joins link facts to the exports (`mappings.json`, per-file enrichment); `folder_annotate.py`: line-stable `tocode-link:` source comments; `folder_docs.py`: master `AGENTS.md`/JSON. - `src/tocode/exporter.py`: project writer, function rendering, worker-session rendering, generated export `AGENTS.md`. - `src/tocode/metadata.py`: JSON metadata and triage documents. - `src/tocode/cluster.py`: call-graph clustering. @@ -91,7 +93,10 @@ Framework/runtime assemblies and native runtime libraries inside bundles and pac - Keep generated text, environment variables, and user-visible strings branded as ToCode. - Keep status output concise but informative. Progress bars are handled by `tqdm` through `Progress.bar(...)`. - Use `apply_patch` for manual edits. -- Do not commit generated export directories such as `here/`, `ls/`, or `*_decompiler/`. +- Do not commit generated export directories such as `here/`, `ls/`, `*_decompiler/`, or `*_decompiled/`. +- Folder links must come only from import/load tables and loader rules; never add name-similarity or string-based (`dlopen`/`LoadLibrary`) links. `links/names.py` keys only locate a backend's function for a symbol the tables already bound. +- Source annotations must never add or remove lines (`folder_annotate` asserts it); line ranges in the JSON depend on it. +- Folder runs print one overall bar and one current-binary bar; nested export logs go to `tocode.log` only. Keep per-file terminal output to failures. ## Dependencies diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5be96a4..b7664b2 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -74,7 +74,28 @@ uv run tocode /bin/true -o /tmp/tocode-check --backend auto -j 2 Confirm the tree matches the export contract in `AGENTS.md`, `tocode.log` has no tracebacks, and `export-manifest.json` reports no unexpected failures. Never -commit export directories (`*_decompiler/` and similar). +commit export directories (`*_decompiler/`, `*_decompiled/`, and similar). + +For folder mode, build a small folder whose links you know and check that +every expected link lands on the right file and line range: + +- Linux: an executable with `-Wl,-rpath,'$ORIGIN/../lib'`, a library with a + version script and `RUNPATH $ORIGIN`, and one library only reachable by + SONAME. +- Windows (`apt install gcc-mingw-w64`): an `.exe` and DLLs with a `.def` + forwarder, a `NONAME` ordinal export, a delay import (`dlltool -y`), and a + 32-bit binary next to 64-bit DLLs. +- .NET: `dotnet publish` of a console app referencing a class library with + overloads and a `[DllImport]` into a gcc-built `.so`. + +```bash +uv run tocode --plan path/to/folder # facts only, no decompilation +uv run tocode path/to/folder -j 2 +uv run tocode path/to/folder --links-only # must leave files unchanged when run twice +``` + +`tests/link_fixtures.py` builds the same shapes synthetically for the unit +tests. ## Dependencies diff --git a/README.md b/README.md index 463b594..a4c3a17 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ToCode -ToCode exports a binary, IDA database, Android APK, or .NET assembly into a source-like project tree: raw recovered C (Java for APKs, C# and IL for .NET), matching assembly, function summaries, section data, optional IDA database, and metadata that coding agents can read directly. +ToCode exports a binary, IDA database, Android APK, .NET assembly, or a whole application folder into a source-like project tree: raw recovered C (Java for APKs, C# and IL for .NET), matching assembly, function summaries, section data, optional IDA database, and metadata that coding agents can read directly. ## Why @@ -142,6 +142,33 @@ tocode app.apks --no-native -j 4 # bundle, DEX/Android side only ``` +### Folders + +Point `tocode` at a directory (a game folder, a Linux app, a macOS `.app` bundle, a .NET publish folder) and it exports every binary in it, then links them together. + +The links come from the binaries themselves: import and load tables, resolved with each platform's loader rules (`@rpath`/`@loader_path` on macOS, `RPATH`/`RUNPATH` and symbol versions on Linux, the application directory and forwarders on Windows, AssemblyRef/MemberRef signatures and P/Invoke for .NET). ToCode never matches by name similarity. When the executable calls `foo` from `libfoo.so`, the link tells you which file defines it and the file and line range of that function in the library's export. + +```text +Game_decompiled/ + AGENTS.md, CLAUDE.md, project.json, export-manifest.json, tocode.log + inventory.json # every file: kind, arch, hash, export status + dependency-graph.json # which file loads which, and the loader rule used + mappings.json # every import -> defining file, function, line range; its call sites + bin/game/ # one regular export per binary, mirroring the folder + lib/libengine.so/ +``` + +Inside each export, `imports.json` and `functions.json` carry the same links, and calls into other files get a trailing comment in the decompiled source: `/* tocode-link: foo -> lib/libfoo.so/src/raw/app/x.c:23-45 */`. The comments never add lines, so all line numbers stay valid. + +Binaries are exported one at a time, executables first and then the libraries they load. Each runs in its own process, so a crash or out-of-memory kill only fails that file. Re-running the same command skips finished binaries and resumes the rest. Bundled runtimes (VC++/MinGW/libstdc++ copies, the .NET framework) are linked but only decompiled with `--include-framework`. Universal Mach-O files use the main executable's architecture (`--arch` to override). + +```bash +tocode --plan Game/ # list binaries, export order, and links; decompile nothing +tocode Game/ --only 'bin/*' --only '**/libengine.so' # export a subset; the rest stays linked +tocode Game/ --exclude '*.dll' # skip matches +tocode Game/ --links-only # recompute links over existing exports +``` + ### Using ToCode supports Windows, Linux, and macOS with Python 3.10 or newer. diff --git a/src/tocode/apk_native.py b/src/tocode/apk_native.py index a3af729..2dc5843 100644 --- a/src/tocode/apk_native.py +++ b/src/tocode/apk_native.py @@ -105,6 +105,7 @@ class NativeOptions: jobs: int | None = None tree: bool = False entropy: bool = False + restart: bool = False def _rel(path: Path | None, root: Path) -> str | None: @@ -195,6 +196,7 @@ def run_native_export( jobs=options.jobs, tree=options.tree, entropy=options.entropy, + restart=options.restart, origin=describe_origin(lib), ) return ( @@ -220,6 +222,7 @@ def _native_export_child( os.setsid() except OSError: # pragma: no cover - already a session leader pass + watch_parent() progress = Progress(enabled=False) progress.set_log_path(log_path) _redirect_output(log_path) @@ -233,6 +236,33 @@ def _native_export_child( result_queue.put(("ok", result)) +PARENT_POLL_SECONDS = 1.0 + + +def watch_parent(poll_seconds: float = PARENT_POLL_SECONDS) -> None: + """Kill this process group when the parent ToCode process goes away. + + An isolated export leads its own process group (so Ctrl-C can kill it with + its decompiler workers), which also means it outlives a parent killed from + outside (``kill``, a closed terminal, the OOM killer) and keeps holding + gigabytes. Polling the parent pid works on every POSIX system. + """ + if not hasattr(os, "getppid") or not hasattr(os, "killpg"): + return + parent = os.getppid() + + def run() -> None: + while True: + time.sleep(poll_seconds) + if os.getppid() != parent: + try: + os.killpg(os.getpgrp(), signal.SIGKILL) + except OSError: # pragma: no cover - not a group leader + os._exit(1) + + threading.Thread(target=run, name="tocode-parent-watch", daemon=True).start() + + _active_children: set[int] = set() _active_lock = threading.Lock() @@ -363,11 +393,15 @@ def __init__( progress: Progress, *, poll_seconds: float = NATIVE_LOG_TAIL_SECONDS, + echo: bool = True, ) -> None: self.log_path = log_path self.lib = lib self.progress = progress self.poll_seconds = poll_seconds + # False: lines go to the main log file only and no bar is drawn; the + # caller renders progress from ``rendered``/``total``/``last_line``. + self.echo = echo self._stop = threading.Event() self._thread: threading.Thread | None = None self._offset = 0 @@ -426,8 +460,12 @@ def _handle_line(self, line: str) -> None: total = _RENDER_TOTAL_RX.search(line) if total is not None: self.total = int(total.group(1)) - self._open_bar() - self.progress.log(f"native[{self.lib.name}]: {line}") + if self.echo: + self._open_bar() + if self.echo: + self.progress.log(f"native[{self.lib.name}]: {line}") + else: + self.progress.file_log(f"[{self.lib.name}] {line}") def _open_bar(self) -> None: if self._bar is not None or self.total is None: @@ -467,6 +505,9 @@ class NativeRunner: poll_seconds: float = NATIVE_MEMORY_POLL_SECONDS max_wait_seconds: float = NATIVE_MEMORY_MAX_WAIT_SECONDS mirror_logs: bool = True + # Folder exports draw their own progress display: per-library lines and + # the nested logs then go to the log file only. + quiet: bool = False current: NativeLib | None = None waiting: tuple[NativeLib, int] | None = None current_started: float = 0.0 @@ -530,7 +571,7 @@ def _wait_for_memory(self, lib: NativeLib) -> None: break self.waiting = (lib, available) if not logged: - self.progress.log( + (self.progress.file_log if self.quiet else self.progress.log)( f"native: waiting for memory before {lib.name} ({lib.abi}): " f"{available} MB free, need {self.min_free_mb} MB" ) @@ -544,17 +585,23 @@ def _wait_for_memory(self, lib: NativeLib) -> None: ) def _export_one(self, lib: NativeLib) -> None: - out_dir = native_export_dir(self.root, lib) + # Folder exports place each library themselves; APK/.NET use native/. + out_dir = lib.export_dir or native_export_dir(self.root, lib) lib.export_dir = out_dir log_path = out_dir / "tocode.log" child = Progress(enabled=False) child.set_log_path(log_path) - self.progress.log( + say = self.progress.file_log if self.quiet else self.progress.log + say( f"native: {lib.name} ({lib.abi}) export started -> " f"{_rel(out_dir, self.root)}" ) started = time.monotonic() - tail = NestedLogTail(log_path, lib, self.progress) if self.mirror_logs else None + tail = ( + NestedLogTail(log_path, lib, self.progress, echo=not self.quiet) + if self.mirror_logs + else None + ) self.current = lib self.current_started = started self.current_tail = tail @@ -581,14 +628,14 @@ def _export_one(self, lib: NativeLib) -> None: self.current_tail = None lib.seconds = time.monotonic() - started if lib.status == "done": - self.progress.log( + say( f"native: {lib.name} ({lib.abi}) exported with {lib.backend} " f"in {lib.seconds:.1f}s: functions={lib.function_count} " f"failures={lib.failure_count}" ) else: self.errors.append(f"{lib.abi}/{lib.name}: {lib.status}") - self.progress.log(f"native: {lib.name} ({lib.abi}) {lib.status}") + say(f"native: {lib.name} ({lib.abi}) {lib.status}") def extract_native_libs( diff --git a/src/tocode/backends/dotnet_pe.py b/src/tocode/backends/dotnet_pe.py index 0cfe765..0b29066 100644 --- a/src/tocode/backends/dotnet_pe.py +++ b/src/tocode/backends/dotnet_pe.py @@ -104,6 +104,11 @@ class PeInfo: strong_name_rva: int = 0 managed_native_header_rva: int = 0 streams: list[MetadataStream] = field(default_factory=list) + # (rva, size) per IMAGE_DIRECTORY_ENTRY_*; used by the folder link parser. + data_directories: list[tuple[int, int]] = field(default_factory=list) + size_of_headers: int = 0 + characteristics: int = 0 # IMAGE_FILE_* (0x2000 = DLL) + entry_point_rva: int = 0 @property def is_managed(self) -> bool: @@ -139,6 +144,10 @@ def cor_flag_names(self) -> list[str]: return names def rva_to_offset(self, rva: int) -> int | None: + if 0 <= rva < self.size_of_headers and all( + rva < section.virtual_address for section in self.sections + ): + return rva # inside the headers, which are mapped as-is for section in self.sections: span = max(section.virtual_size, section.raw_size) if section.virtual_address <= rva < section.virtual_address + span: @@ -161,7 +170,9 @@ def parse_pe(data: bytes | mmap.mmap) -> PeInfo | None: if data[pe_offset : pe_offset + 4] != b"PE\0\0": return None machine, section_count = struct.unpack_from(" PeInfo | None: pe32_plus=pe32_plus, image_base=image_base, sections=sections, + characteristics=file_characteristics, + entry_point_rva=struct.unpack_from(" _CLR_DIRECTORY: clr_rva, clr_size = struct.unpack_from( diff --git a/src/tocode/backends/ida.py b/src/tocode/backends/ida.py index e68cc83..3964d96 100644 --- a/src/tocode/backends/ida.py +++ b/src/tocode/backends/ida.py @@ -818,8 +818,39 @@ def calls_from( name = self._db.names.get_at(target) if name: imported.add(name) + if self._indirect_import_calls(): + imported.update(self._iat_references(func, imports)) return sorted(edges), sorted(name for name in imported if name) + def _indirect_import_calls(self) -> bool: + # PE code calls imports through the IAT (``call [__imp_X]``); IDA has + # no callee function for that, only an xref to the IAT slot. + cached = getattr(self, "_iat_calls", None) + if cached is None: + fmt = str(self._db.format or "").lower() + cached = self._ida_xref is not None and ("pe" in fmt or "portable" in fmt) + self._iat_calls = cached + return cached + + def _iat_references(self, func: Any, imports: Any) -> set[str]: + xref = self._ida_xref + names: set[str] = set() + ea = int(func.start_ea) + end = int(func.end_ea) + block = xref.xrefblk_t() + while ea < end: + ok = block.first_from(ea, xref.XREF_FAR) + while ok: + target = int(block.to) + if target in imports: + names.add(imports[target].name) + ok = block.next_from() + next_ea = self._ida_bytes.next_head(ea, end) + if next_ea <= ea: + break + ea = next_ea + return names + def data_xrefs(self, addresses: Any) -> dict[int, list[tuple[int, bool]]]: if self._ida_xref is None: return {} diff --git a/src/tocode/cli.py b/src/tocode/cli.py index 5b9bb41..954c9ab 100644 --- a/src/tocode/cli.py +++ b/src/tocode/cli.py @@ -94,6 +94,40 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help=".NET input: also decompile .NET framework/runtime assemblies and native runtime libraries found in bundles and packages.", ) + parser.add_argument( + "--only", + action="append", + default=[], + metavar="GLOB", + help="Folder input: export only binaries whose path (relative to the folder) matches; '**' crosses directories and a pattern without '/' matches the file name. Repeatable. Other binaries are still scanned and linked.", + ) + parser.add_argument( + "--exclude", + action="append", + default=[], + metavar="GLOB", + help="Folder input: do not export binaries matching this glob (same syntax as --only). Repeatable.", + ) + parser.add_argument( + "--no-managed", + action="store_true", + help="Folder input: do not export .NET assemblies (they are still scanned and linked).", + ) + parser.add_argument( + "--arch", + default=None, + help="Folder input: slice to use from universal (fat) Mach-O files, e.g. arm64 or x86_64 (default: the main executable's).", + ) + parser.add_argument( + "--plan", + action="store_true", + help="Folder input: scan, resolve links, print what would be exported in which order, and exit without decompiling.", + ) + parser.add_argument( + "--links-only", + action="store_true", + help="Folder input: skip decompilation and only recompute mappings.json, per-file links, and source comments over the existing exports.", + ) parser.add_argument( "--no-splits", action="store_true", @@ -200,6 +234,10 @@ def main(argv: list[str] | None = None) -> int: if args.binja_view is not None and args.all_views: parser.error("--binja-view and --all-views cannot be combined") if args.backend == "binja": + if args.binary is not None and args.binary.is_dir(): + parser.error( + "--backend binja is not supported for folder input; use ida, r2, or angr" + ) if args.binary is not None and is_apk_input(args.binary): parser.error( "--backend binja is not supported for APK input; native libraries " @@ -225,6 +263,8 @@ def main(argv: list[str] | None = None) -> int: args.out_dir = ( args.out_dir.expanduser().resolve() if args.out_dir is not None else None ) + if binary.is_dir(): + return _run_folder(binary, args=args, progress=progress) if is_apk_input(binary): return _run_apk(binary, args=args, progress=progress, parser=parser) if not args.as_native and binary.is_file() and is_dotnet_input(binary): @@ -368,6 +408,64 @@ def _run_dotnet(binary: Path, *, args: argparse.Namespace, progress: Progress) - return 0 +def _run_folder(folder: Path, *, args: argparse.Namespace, progress: Progress) -> int: + from .folder import FolderOptions, export_folder, format_plan + + def ensure_dotnet() -> bool: + from .backends import dotnet_libs + + try: + dotnet_libs.ensure_libraries( + interactive=sys.stdin.isatty() and sys.stderr.isatty(), + ask=_ask_on_stderr, + log=progress.log, + ) + except ToCodeError as exc: + progress.log(f"tocode: .NET assemblies will not be decompiled: {exc}") + return False + return True + + options = FolderOptions( + out_dir=args.out_dir, + only=list(args.only), + exclude=list(args.exclude), + include_framework=args.include_framework, + native=not args.no_native, + managed=not args.no_managed, + ensure_dotnet=ensure_dotnet, + arch=args.arch, + plan=args.plan, + links_only=args.links_only, + restart=args.restart, + jobs=args.jobs, + native_options=_native_options(args), + ) + try: + summary = export_folder(folder, options=options, progress=progress) + except KeyboardInterrupt: + print("tocode: interrupted; rerun the same command to resume", file=sys.stderr) + return 130 + except ToCodeError as exc: + progress.log(f"tocode: {exc}") + print(f"tocode: {exc}", file=sys.stderr) + return 1 + if summary.plan_only: + print(format_plan(summary)) + return 0 + if not args.quiet: + print(f"Project: {summary.root_dir}") + print( + f"Summary: binaries={summary.binaries} exported={summary.done} " + f"failed={summary.failed} library_refs={summary.edges} " + f"imports={summary.links} bound_in_folder={summary.bound} " + f"mapped_to_functions={summary.mapped}" + ) + for line in summary.errors: + print(f"failed: {line}", file=sys.stderr) + print(f"Exported in {_format_duration(summary.seconds)}") + return 130 if summary.interrupted else 0 + + def _ask_on_stderr(prompt: str) -> str: print(prompt, end="", file=sys.stderr, flush=True) line = sys.stdin.readline() diff --git a/src/tocode/dotnet.py b/src/tocode/dotnet.py index bbe17b1..204a090 100644 --- a/src/tocode/dotnet.py +++ b/src/tocode/dotnet.py @@ -79,6 +79,12 @@ class DotnetExportOptions: native: bool = True include_framework: bool = False native_options: NativeOptions = field(default_factory=NativeOptions) + # Copy and decompile the native libraries next to the assembly that its + # P/Invoke declarations name. Folder exports turn this off: they export + # those libraries themselves and link them. + pinvoke_neighbours: bool = True + # Provenance paragraph for AGENTS.md (set by folder exports). + origin: str | None = None @dataclass @@ -278,7 +284,7 @@ def _collect_natives(context: DotnetContext) -> None: session = _need(context.session) root = _need(context.root) natives = list(found.natives) - if found.kind in {"assembly", "apphost"}: + if found.kind in {"assembly", "apphost"} and context.options.pinvoke_neighbours: known = {native.path.resolve() for native in natives} for module, path in pinvoke_neighbours( found.input.parent, session.pinvoke_modules @@ -1045,6 +1051,7 @@ def _write_final_documents(context: DotnetContext) -> None: native_count=sum( 1 for lib in context.libs if not lib.status.startswith("skipped") ), + origin=context.options.origin, ) + "\n", ) diff --git a/src/tocode/dotnet_metadata.py b/src/tocode/dotnet_metadata.py index 20d08a3..436ea26 100644 --- a/src/tocode/dotnet_metadata.py +++ b/src/tocode/dotnet_metadata.py @@ -811,7 +811,13 @@ def resources_json(session: DotnetSession, root: Path) -> dict[str, Any]: def build_dotnet_agents( - *, title: str, kind: str, assemblies: list[str], native: bool, native_count: int + *, + title: str, + kind: str, + assemblies: list[str], + native: bool, + native_count: int, + origin: str | None = None, ) -> str: shown = ", ".join(f"`{name}`" for name in assemblies[:8]) if len(assemblies) > 8: @@ -822,6 +828,7 @@ def build_dotnet_agents( f"You are working inside a ToCode .NET export of `{title}` ({kind}; assemblies: {shown or 'none'}).", "Treat the recovered C#, the IL, the metadata, and the extracted resources as evidence for reverse engineering.", "", + *(["## Origin", "", origin, ""] if origin else []), "## Mission", "", "Reverse the program, answer user questions, or serve as an oracle/helper to the user regarding this recovered source code.", diff --git a/src/tocode/exporter.py b/src/tocode/exporter.py index 68cfcb5..0f1073c 100644 --- a/src/tocode/exporter.py +++ b/src/tocode/exporter.py @@ -2803,6 +2803,9 @@ def write_project_json(context: ExportContext) -> None: "binary": str(analysis.binary.path), "backend": context.analyzer.backend_name, "decompiler": context.analyzer.decompiler_label, + # Base the backend loaded the image at; addresses in this export are + # relative to it (folder exports join link-time addresses with it). + "image_base": f"0x{analysis.binary.image_base:x}", "root_dir": str(root.resolve()), "src_dir": str(_need(context.raw_dir).resolve()), "raw_src_dir": str(_need(context.raw_dir).resolve()), @@ -2852,6 +2855,7 @@ def write_manifest(context: ExportContext) -> Path: "format": analysis.binary.format_name, "arch": analysis.binary.arch, "bits": analysis.binary.bits, + "image_base": f"0x{analysis.binary.image_base:x}", "entrypoints": [f"0x{item:x}" for item in analysis.binary.entrypoints], "header": str(_need(context.header_path).resolve()), "ida_database": str(context.ida_database) diff --git a/src/tocode/folder.py b/src/tocode/folder.py new file mode 100644 index 0000000..afe0c35 --- /dev/null +++ b/src/tocode/folder.py @@ -0,0 +1,965 @@ +"""Folder exports: every binary of an application folder, plus the links between them. + +``tocode `` scans the folder, reads the import/export and load +tables of every binary (``tocode.links``), schedules one export per binary in +dependency order (executables, then what they load), runs each in an isolated +process, and finally joins the link facts to the exported functions +(``folder_links``): ``mappings.json`` at the top, and per-file enrichment and +source comments inside each export. + +Re-running the same command skips every binary whose export is complete and +unchanged, resumes an interrupted one from its checkpoint, and recomputes the +links. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +import hashlib +import json +import multiprocessing +import os +from pathlib import Path +import posixpath +import re +import shutil +import sys +import time +from typing import Any, Callable + +from .apk_native import ( + NativeExporter, + NativeLib, + NativeOptions, + NativeRunner, + _active_children, + _active_lock, + _redirect_output, + collect_child_result, + native_min_free_mb, + run_native_export, + watch_parent, +) +from .errors import ToCodeError +from .links.analyze import FolderLinks, link_folder +from .links.inventory import JOB_APK, JOB_DOTNET, JOB_NATIVE, Entry, Inventory, scan +from .metadata import write_json +from .naming import clean_path_component +from .parallel import available_memory_mb +from .progress import Progress + +STATE_DIR = ".tocode-folder" +STATE_SCHEMA = 1 +MASTER_SUFFIX = "_decompiled" + +# Job statuses +DONE = "done" +PENDING = "pending" +FILTERED = "filtered" +RUNTIME = "skipped: runtime library (use --include-framework)" +NO_NATIVE = "skipped: --no-native" +NO_MANAGED = "skipped: --no-managed" +NO_DOTNET_LIBS = "skipped: .NET libraries not installed (run tocode --setup-dotnet)" + + +@dataclass(slots=True) +class FolderOptions: + out_dir: Path | None = None + only: list[str] = field(default_factory=list) + exclude: list[str] = field(default_factory=list) + include_framework: bool = False + native: bool = True + managed: bool = True + dotnet_available: bool = True + # Called once when managed binaries are scheduled: makes the .NET libraries + # available (asking the user once), returns False when they are not. + ensure_dotnet: Callable[[], bool] | None = None + arch: str | None = None + plan: bool = False + links_only: bool = False + restart: bool = False + jobs: int | None = None + native_options: NativeOptions = field(default_factory=NativeOptions) + + +@dataclass(slots=True) +class FolderJob: + relpath: str + job: str + path: Path # file the backend opens (a thin slice for fat Mach-O) + export_dir: Path + key: str + status: str = PENDING + backend: str | None = None + decompiler: str | None = None + function_count: int | None = None + failure_count: int | None = None + seconds: float | None = None + reused: bool = False # complete from an earlier run + + def to_json(self, root: Path) -> dict[str, Any]: + return { + "path": self.relpath, + "job": self.job, + "export_dir": _rel(self.export_dir, root), + "status": self.status, + "backend": self.backend, + "decompiler": self.decompiler, + "function_count": self.function_count, + "failure_count": self.failure_count, + "seconds": round(self.seconds, 1) if self.seconds is not None else None, + "key": self.key, + } + + +@dataclass +class FolderSummary: + root_dir: Path + input_dir: Path + binaries: int + jobs: list[FolderJob] + edges: int + links: int + bound: int + mapped: int + plan_only: bool = False + interrupted: bool = False + errors: list[str] = field(default_factory=list) + seconds: float = 0.0 + inventory: Inventory | None = None + facts: FolderLinks | None = None + + @property + def done(self) -> int: + return sum(1 for job in self.jobs if job.status == DONE) + + @property + def failed(self) -> int: + return sum(1 for job in self.jobs if job.status.startswith("failed")) + + +def default_master_dir(input_dir: Path) -> Path: + name = f"{clean_path_component(input_dir.name or 'folder')}{MASTER_SUFFIX}" + default_root = os.environ.get("TOCODE_DEFAULT_OUT_ROOT", "").strip() + if default_root: + return Path(default_root).expanduser().resolve() / name + return input_dir.parent / name + + +# -------------------------------------------------------------------------- +# Globs +# -------------------------------------------------------------------------- + + +def glob_regex(pattern: str) -> re.Pattern[str]: + """``**``-aware glob over POSIX relative paths.""" + text = pattern.strip().lstrip("./") if pattern.startswith("./") else pattern.strip() + out: list[str] = [] + index = 0 + while index < len(text): + char = text[index] + if text.startswith("**/", index): + out.append("(?:.*/)?") + index += 3 + elif text.startswith("**", index): + out.append(".*") + index += 2 + elif char == "*": + out.append("[^/]*") + index += 1 + elif char == "?": + out.append("[^/]") + index += 1 + elif char == "[": + end = text.find("]", index + 1) + if end < 0: + out.append(re.escape(char)) + index += 1 + else: + out.append(text[index : end + 1]) + index = end + 1 + else: + out.append(re.escape(char)) + index += 1 + return re.compile("^" + "".join(out) + "$") + + +def path_matches(relpath: str, patterns: list[str]) -> bool: + for pattern in patterns: + target = relpath if "/" in pattern.strip("/") else posixpath.basename(relpath) + if glob_regex( + pattern.strip("/") if "/" in pattern.strip("/") else pattern + ).match(target): + return True + return False + + +# -------------------------------------------------------------------------- +# Planning +# -------------------------------------------------------------------------- + + +def export_dir_for(master: Path, relpath: str) -> Path: + """One export per binary, mirroring the input layout, named after the file.""" + return master.joinpath(*relpath.split("/")) + + +def job_key(entry: Entry, options: FolderOptions) -> str: + native = options.native_options + payload = { + "schema": STATE_SCHEMA, + "sha256": entry.sha256 or f"size:{entry.size}", + "arch": entry.arch, + "job": entry.job, + "backend": native.backend if entry.job == JOB_NATIVE else None, + "analysis": native.analysis_command if entry.job == JOB_NATIVE else None, + "tree": native.tree, + "entropy": native.entropy, + "include_framework": options.include_framework + if entry.job == JOB_DOTNET + else None, + "native_in_apk": options.native if entry.job == JOB_APK else None, + } + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()[:24] + + +def schedule(inventory: Inventory, links: FolderLinks) -> list[str]: + """Binaries in export order: each chain in load order, then the rest.""" + order: list[str] = [] + seen: set[str] = set() + for root in inventory.roots: + for relpath in links.resolution.chains.get(root, [root]): + if relpath not in seen: + seen.add(relpath) + order.append(relpath) + for root, chain in links.resolution.chains.items(): + for relpath in chain: + if relpath not in seen: + seen.add(relpath) + order.append(relpath) + for entry in inventory.entries: + if entry.job is not None and entry.relpath not in seen: + seen.add(entry.relpath) + order.append(entry.relpath) + return order + + +def plan_jobs( + inventory: Inventory, + links: FolderLinks, + master: Path, + options: FolderOptions, +) -> list[FolderJob]: + jobs: list[FolderJob] = [] + for relpath in schedule(inventory, links): + entry = inventory.entry(relpath) + if entry is None or entry.job is None: + continue + source = inventory.root.joinpath(*relpath.split("/")) + job = FolderJob( + relpath=relpath, + job=entry.job, + path=source, + export_dir=export_dir_for(master, relpath), + key=job_key(entry, options), + ) + if entry.duplicate_of is not None: + job.status = f"duplicate of {entry.duplicate_of}" + job.export_dir = export_dir_for(master, entry.duplicate_of) + elif (options.only and not path_matches(relpath, options.only)) or ( + options.exclude and path_matches(relpath, options.exclude) + ): + job.status = FILTERED + elif entry.runtime and not options.include_framework: + job.status = RUNTIME + elif entry.job == JOB_NATIVE and not options.native: + job.status = NO_NATIVE + elif entry.job == JOB_DOTNET and not options.managed: + job.status = NO_MANAGED + elif entry.job == JOB_DOTNET and not options.dotnet_available: + job.status = NO_DOTNET_LIBS + jobs.append(job) + return jobs + + +# -------------------------------------------------------------------------- +# State +# -------------------------------------------------------------------------- + + +def state_path(master: Path) -> Path: + return master / STATE_DIR / "state.json" + + +def load_state(master: Path) -> dict[str, Any]: + try: + payload = json.loads(state_path(master).read_text(encoding="utf-8")) + except (OSError, ValueError): + return {"schema": STATE_SCHEMA, "jobs": {}} + if not isinstance(payload, dict) or payload.get("schema") != STATE_SCHEMA: + return {"schema": STATE_SCHEMA, "jobs": {}} + payload.setdefault("jobs", {}) + return payload + + +def save_state(master: Path, jobs: list[FolderJob], state: dict[str, Any]) -> None: + recorded = state.setdefault("jobs", {}) + for job in jobs: + if job.status == DONE or job.status.startswith("failed"): + recorded[job.relpath] = { + "key": job.key, + "status": job.status, + "backend": job.backend, + "decompiler": job.decompiler, + "function_count": job.function_count, + "failure_count": job.failure_count, + "seconds": round(job.seconds, 1) if job.seconds is not None else None, + } + write_json(state_path(master), state) + + +def apply_state(jobs: list[FolderJob], state: dict[str, Any]) -> None: + """Mark jobs whose export is complete and still matches as done.""" + recorded = state.get("jobs", {}) + for job in jobs: + if job.status != PENDING: + continue + previous = recorded.get(job.relpath) + if not isinstance(previous, dict) or previous.get("key") != job.key: + continue + if previous.get("status") != DONE: + continue + if not (job.export_dir / "export-manifest.json").is_file(): + continue + job.status = DONE + job.reused = True + job.backend = previous.get("backend") + job.decompiler = previous.get("decompiler") + job.function_count = previous.get("function_count") + job.failure_count = previous.get("failure_count") + job.seconds = previous.get("seconds") + + +# -------------------------------------------------------------------------- +# Running one binary +# -------------------------------------------------------------------------- + + +@dataclass(slots=True) +class JobSettings: + """What a spawned export process needs (picklable).""" + + job: str + native_options: NativeOptions + jobs: int | None + include_framework: bool + native: bool + origin: str + restart: bool + + +def run_folder_job( + lib: NativeLib, out_dir: Path, settings: JobSettings, progress: Progress +) -> tuple[str, str, int, int]: + if settings.job == JOB_NATIVE: + return run_native_export(lib, out_dir, settings.native_options, progress) + if settings.job == JOB_DOTNET: + from .dotnet import DotnetExportOptions, export_dotnet + + summary = export_dotnet( + lib.path, + options=DotnetExportOptions( + out_dir=out_dir, + jobs=settings.jobs, + native=False, + include_framework=settings.include_framework, + native_options=settings.native_options, + pinvoke_neighbours=False, + origin=settings.origin, + ), + progress=progress, + ) + return ( + "dotnet", + "ICSharpCode.Decompiler", + summary.method_count, + len(summary.failed_types), + ) + if settings.job == JOB_APK: + from .apk import ApkExportOptions, export_apk + + apk = export_apk( + lib.path, + options=ApkExportOptions( + out_dir=out_dir, + jobs=settings.jobs, + native=settings.native, + splits=False, + native_options=settings.native_options, + ), + progress=progress, + ) + return ("asc", "droidasc", apk.method_count, len(apk.failed_classes)) + raise ToCodeError(f"unknown job type {settings.job}") + + +def _folder_job_child( + lib: NativeLib, + out_dir: Path, + settings: JobSettings, + log_path: Path, + result_queue: Any, +) -> None: + """Entry point of the isolated per-binary export process.""" + if hasattr(os, "setsid"): + try: + os.setsid() + except OSError: # pragma: no cover - already a session leader + pass + watch_parent() + progress = Progress(enabled=False) + progress.set_log_path(log_path) + _redirect_output(log_path) + try: + result = run_folder_job(lib, out_dir, settings, progress) + except ToCodeError as exc: + result_queue.put(("error", str(exc))) + except BaseException as exc: # pragma: no cover - backend crash isolation + result_queue.put(("error", f"{type(exc).__name__}: {exc}")) + else: + result_queue.put(("ok", result)) + + +def isolated_exporter( + settings_for: Callable[[NativeLib], JobSettings], +) -> NativeExporter: + """Each binary in its own process: a crash or OOM fails only that binary.""" + + def run( + lib: NativeLib, out_dir: Path, progress: Progress + ) -> tuple[str, str, int, int]: + context = multiprocessing.get_context("spawn") + result_queue: Any = context.Queue() + log_path = progress.log_path or out_dir / "tocode.log" + process = context.Process( + target=_folder_job_child, + args=(lib, out_dir, settings_for(lib), log_path, result_queue), + name=f"tocode-folder-{lib.stem}", + ) + process.start() + if process.pid is not None: + with _active_lock: + _active_children.add(process.pid) + return collect_child_result(process, result_queue) + + return run + + +# -------------------------------------------------------------------------- +# Export +# -------------------------------------------------------------------------- + + +def origin_text(input_dir: Path, relpath: str, entry: Entry | None) -> str: + up = "/".join([".."] * len(relpath.split("/"))) + arch = ( + f", {entry.arch} slice of a universal binary" + if entry and len(entry.archs) > 1 + else "" + ) + return ( + f"This is the decompiled `{relpath}` from the folder `{input_dir.name}`{arch}, " + "exported as part of a ToCode folder export. The folder's master metadata is " + f"`{up}`: its `AGENTS.md` explains the layout, `dependency-graph.json` lists the " + "files this one loads and is loaded by (with the loader rule behind each link), " + "and `mappings.json` maps every imported symbol to the file and line range of its " + "definition in the other exports. In this export, `imports.json` (`libraries`, " + "per-import `link`) and `functions.json` (`import_links`, `imported_by`) carry the " + "same links, and calls into other files of the folder are annotated in the " + "decompiled source with `tocode-link:` comments." + ) + + +def export_folder( + input_dir: Path, + *, + options: FolderOptions | None = None, + progress: Progress | None = None, + exporter: NativeExporter | None = None, +) -> FolderSummary: + from . import folder_docs, folder_links + + options = options or FolderOptions() + progress = progress or Progress() + started = time.monotonic() + input_dir = input_dir.resolve() + master = (options.out_dir or default_master_dir(input_dir)).resolve() + if master == input_dir or input_dir in master.parents: + # Inside the input, the scan skips it by name (*_decompiled); refuse + # anything else rather than export our own output on the next run. + if master == input_dir or not master.name.endswith(MASTER_SUFFIX): + raise ToCodeError( + f"output directory {master} is inside the input folder; " + f"pick a different -o or name it *{MASTER_SUFFIX}" + ) + if not options.plan: + progress.set_log_path(master / "tocode.log") + progress.file_log(f"Command: {' '.join(sys.argv)}") + progress.log(f"Scanning {input_dir}") + inventory = scan(input_dir, arch=options.arch, log=progress.file_log) + links = link_folder(inventory) + jobs = plan_jobs(inventory, links, master, options) + code = [entry for entry in inventory.entries if entry.job is not None] + progress.log( + f"Found {len(code)} binaries ({_kind_counts(code)}) in {len(inventory.entries)} files; " + f"{len(links.resolution.edges)} library references, {len(links.links)} imported symbols" + ) + summary = FolderSummary( + root_dir=master, + input_dir=input_dir, + binaries=len(code), + jobs=jobs, + edges=len(links.resolution.edges), + links=len(links.links), + bound=sum(1 for link in links.links if link.status == "bound"), + mapped=0, + plan_only=options.plan, + inventory=inventory, + facts=links, + ) + if options.plan: + summary.seconds = time.monotonic() - started + return summary + + state = ( + {"schema": STATE_SCHEMA, "jobs": {}} if options.restart else load_state(master) + ) + if options.restart: + shutil.rmtree(master / STATE_DIR / "index", ignore_errors=True) + apply_state(jobs, state) + managed_pending = [ + job for job in jobs if job.job == JOB_DOTNET and job.status == PENDING + ] + if managed_pending and not options.links_only and options.ensure_dotnet is not None: + if not options.ensure_dotnet(): + for job in managed_pending: + job.status = NO_DOTNET_LIBS + _prepare_slices(master, inventory, jobs) + folder_docs.write_master_documents( + master, inventory, links, jobs, options, progress=None + ) + + try: + if not options.links_only: + _run_jobs(master, inventory, jobs, options, state, progress, exporter) + except KeyboardInterrupt: + summary.interrupted = True + progress.log("tocode: interrupted; finished exports are kept, rerun to resume") + finally: + save_state(master, jobs, state) + try: + progress.log("Linking exports (mappings.json, per-file links, source comments)") + result = folder_links.link_exports( + master, inventory, links, jobs, progress=progress + ) + summary.mapped = result.mapped + folder_docs.write_master_documents( + master, + inventory, + links, + jobs, + options, + progress=progress, + link_result=result, + ) + except KeyboardInterrupt: + summary.interrupted = True + summary.errors = [ + f"{job.relpath}: {job.status}" + for job in jobs + if job.status.startswith("failed") + ] + summary.seconds = time.monotonic() - started + return summary + + +def _kind_counts(entries: list[Entry]) -> str: + counts: dict[str, int] = {} + for entry in entries: + label = entry.format or entry.detail or entry.kind + counts[label] = counts.get(label, 0) + 1 + return ", ".join(f"{count} {label}" for label, count in sorted(counts.items())) + + +def _prepare_slices(master: Path, inventory: Inventory, jobs: list[FolderJob]) -> None: + """Thin the chosen slice out of universal binaries (backends want one arch).""" + for job in jobs: + entry = inventory.entry(job.relpath) + if entry is None or entry.kind != "macho_fat" or entry.arch is None: + continue + facts = inventory.folder.facts(job.relpath, entry.arch) + if facts is None or facts.slice_size is None: + continue + target = ( + master + / STATE_DIR + / "slices" + / f"{(entry.sha256 or 'x')[:16]}-{entry.arch}" + / posixpath.basename(job.relpath) + ) + if not target.is_file() or target.stat().st_size != facts.slice_size: + target.parent.mkdir(parents=True, exist_ok=True) + with job.path.open("rb") as source, target.open("wb") as sink: + source.seek(facts.slice_offset) + remaining = facts.slice_size + while remaining > 0: + chunk = source.read(min(remaining, 1 << 20)) + if not chunk: + break + sink.write(chunk) + remaining -= len(chunk) + # A stable mtime keeps backend caches keyed on it valid across runs. + stat = job.path.stat() + os.utime(target, (stat.st_atime, stat.st_mtime)) + job.path = target + + +def _run_jobs( + master: Path, + inventory: Inventory, + jobs: list[FolderJob], + options: FolderOptions, + state: dict[str, Any], + progress: Progress, + exporter: NativeExporter | None, +) -> None: + pending = [job for job in jobs if job.status == PENDING] + reused = sum(1 for job in jobs if job.reused) + if reused: + progress.log(f"{reused} binaries already exported and unchanged; skipping them") + if not pending: + return + progress.log( + f"Exporting {len(pending)} binaries one at a time, in load order " + f"(backend: {options.native_options.backend}; per-binary logs in tocode.log)" + ) + by_entry: dict[str, FolderJob] = {} + libs: list[NativeLib] = [] + for job in pending: + entry = inventory.entry(job.relpath) + lib = NativeLib( + abi=(entry.arch if entry and entry.arch else job.job), + name=job.relpath, + entry=job.relpath, + source_apk=inventory.root.name, + size=job.path.stat().st_size, + sha256=(entry.sha256 if entry else "") or "", + path=job.path, + export_dir=job.export_dir, + origin=origin_text(inventory.root, job.relpath, entry), + ) + by_entry[job.relpath] = job + libs.append(lib) + native = options.native_options + native_options = NativeOptions( + backend=native.backend, + analysis_command=native.analysis_command, + idadir=native.idadir, + ida_domain_path=native.ida_domain_path, + purge_cache=native.purge_cache, + jobs=options.jobs, + tree=native.tree, + entropy=native.entropy, + restart=options.restart, + ) + + def settings_for(lib: NativeLib) -> JobSettings: + return JobSettings( + job=by_entry[lib.entry].job, + native_options=native_options, + jobs=options.jobs, + include_framework=options.include_framework, + native=options.native, + origin=lib.origin or "", + restart=options.restart, + ) + + run_one = exporter or isolated_exporter(settings_for) + holder: list[NativeRunner] = [] + + def tracked( + lib: NativeLib, out_dir: Path, child: Progress + ) -> tuple[str, str, int, int]: + job = by_entry[lib.entry] + started = time.monotonic() + try: + result = run_one(lib, out_dir, child) + except BaseException as exc: + # Killed by Ctrl-C: stays pending and resumes from its checkpoint. + if not (holder and holder[0].stop_event.is_set()): + job.status = f"failed: {exc}" + job.seconds = time.monotonic() - started + save_state(master, jobs, state) + raise + job.status = DONE + job.backend, job.decompiler, job.function_count, job.failure_count = result + job.seconds = time.monotonic() - started + save_state(master, jobs, state) + return result + + runner = NativeRunner( + libs=libs, + root=master, + progress=progress, + exporter=tracked, + min_free_mb=native_min_free_mb(), + available_memory=available_memory_mb, + quiet=True, + ) + holder.append(runner) + display = FolderDisplay(progress, pending) + runner.start() + thread = runner.thread + try: + while thread is not None and thread.is_alive(): + thread.join(timeout=DISPLAY_REFRESH_SECONDS) + display.refresh(runner) + except KeyboardInterrupt: + display.close() + runner.stop() + runner.join() + raise + runner.join() + display.refresh(runner) + display.close() + for lib in libs: + job = by_entry[lib.entry] + if ( + job.status == PENDING + and lib.status.startswith("failed") + and not runner.stop_event.is_set() + ): + job.status = lib.status + done = sum(1 for job in pending if job.status == DONE) + failed = sum(1 for job in pending if job.status.startswith("failed")) + progress.log( + f"Exported {done}/{len(pending)} binaries" + + ( + f", {failed} failed (details in tocode.log and project.json)" + if failed + else "" + ) + ) + + +DISPLAY_REFRESH_SECONDS = 0.5 +TEXT_HEARTBEAT_SECONDS = 60.0 +NAME_WIDTH = 36 +LABEL_WIDTH = NAME_WIDTH + 4 +# Same columns for every bar: label, bar, count, extra, time. +BAR_FORMAT = ( + "{desc} {bar:24} {n_fmt}/{total_fmt} {unit_label}{postfix} [{elapsed}<{remaining}]" +) + + +def _short(text: str, width: int = NAME_WIDTH) -> str: + return text if len(text) <= width else "…" + text[-(width - 1) :] + + +def _duration(seconds: float) -> str: + seconds = int(seconds) + if seconds < 60: + return f"{seconds}s" + minutes, seconds = divmod(seconds, 60) + if minutes < 60: + return f"{minutes}m{seconds:02d}s" + hours, minutes = divmod(minutes, 60) + return f"{hours}h{minutes:02d}m" + + +class FolderDisplay: + """Progress for many binaries without flooding the terminal. + + On a terminal: one bar for the folder (binaries done, failures, ETA) and + one for the binary being exported (its phase, then functions rendered). + Nested export logs only go to ``tocode.log``; failures are printed above + the bars. When stderr is not a terminal: one line per finished binary and + a status line every minute. + """ + + def __init__(self, progress: Progress, jobs: list[FolderJob]) -> None: + self.progress = progress + self.jobs = jobs + self.total = len(jobs) + self.started = time.monotonic() + self.reported: set[str] = set() + self.overall = progress.new_bar( + total=self.total, + desc=f"{'binaries':<{LABEL_WIDTH}}", + unit="bin", + position=0, + disable=None, # bars only on a terminal + bar_format=BAR_FORMAT.replace("{unit_label}", "binaries"), + ) + self.tty = self.overall is not None and not getattr( + self.overall, "disable", True + ) + if self.overall is not None and not self.tty: + self.overall.close() + self.overall = None + self.current: Any = None + self.current_name: str | None = None + self.current_total: int | None = None + self.last_text = self.started + + def _finished(self) -> list[FolderJob]: + return [job for job in self.jobs if job.status != PENDING] + + def refresh(self, runner: NativeRunner) -> None: + finished = self._finished() + for job in finished: + if job.relpath in self.reported: + continue + self.reported.add(job.relpath) + self._report(job, len(self.reported)) + if self.tty and self.overall is not None: + failed = sum(1 for job in finished if job.status.startswith("failed")) + self.overall.n = len(finished) + self.overall.set_postfix_str( + f"{failed} failed" if failed else "", refresh=False + ) + self.overall.refresh() + self._refresh_current(runner) + elif time.monotonic() - self.last_text >= TEXT_HEARTBEAT_SECONDS: + self.last_text = time.monotonic() + status = runner.describe_current() + if status: + self.progress.log( + f"[{len(finished)}/{self.total}] " + + status.replace("native: ", "", 1).replace("libraries", "binaries") + ) + + def _report(self, job: FolderJob, number: int) -> None: + took = _duration(job.seconds or 0) + if job.status.startswith("failed"): + reason = job.status.removeprefix("failed: ").splitlines()[0][:160] + self.progress.log(f"✗ {job.relpath}: {reason}") + elif not self.tty: + self.progress.log( + f"[{number}/{self.total}] ✓ {job.relpath}: {job.function_count} functions in {took}" + ) + + def _refresh_current(self, runner: NativeRunner) -> None: + lib = runner.current + waiting = runner.waiting + name = lib.name if lib is not None else (waiting[0].name if waiting else None) + if name != self.current_name: + self._close_current() + self.current_name = name + if name is None: + return + self._open_current(None) + tail = runner.current_tail + if self.current is None: + return + if tail is not None and tail.total and self.current_total is None: + self._close_current() + self._open_current(tail.total) + if self.current_total is not None and tail is not None: + self.current.n = min(tail.rendered, self.current_total) + self.current.refresh() + return + if waiting is not None: + phase = f"waiting for memory ({waiting[1]} MB free, need {runner.min_free_mb} MB)" + elif tail is not None: + phase = tail.last_line + else: + phase = "starting" + self.current.set_postfix_str(_short(phase, 60), refresh=False) + self.current.refresh() + + def _open_current(self, total: int | None) -> None: + label = " ↳ " + _short(self.current_name or "", NAME_WIDTH) + self.current_total = total + if total is None: + fmt = "{desc}{postfix} [{elapsed}]" # " ↳ name, [time]" + else: + label = f"{label:<{LABEL_WIDTH}}" + fmt = BAR_FORMAT.replace("{unit_label}", "functions") + self.current = self.progress.new_bar( + total=total, desc=label, position=1, disable=None, bar_format=fmt + ) + + def _close_current(self) -> None: + if self.current is not None: + self.current.close() + self.current = None + self.current_total = None + + def close(self) -> None: + self._close_current() + if self.overall is not None: + self.overall.close() + self.overall = None + + +def _rel(path: Path | None, root: Path) -> str | None: + if path is None: + return None + try: + return path.relative_to(root).as_posix() + except ValueError: + return str(path) + + +def format_plan(summary: FolderSummary) -> str: + """Human-readable ``--plan`` output.""" + inventory = summary.inventory + facts = summary.facts + lines = [f"Folder: {summary.input_dir}", f"Output: {summary.root_dir}"] + if inventory is not None: + lines.append( + f"Files: {len(inventory.entries)}, binaries: {summary.binaries}, " + f"architecture: {inventory.arch or 'n/a'}" + ) + lines.extend(["", "Export order (status):"]) + for number, job in enumerate(summary.jobs, 1): + entry = inventory.entry(job.relpath) if inventory else None + detail = "" + if entry is not None: + role = entry.role or entry.kind + detail = f" [{entry.format or entry.kind} {role}{', ' + entry.arch if entry.arch else ''}]" + lines.append(f" {number:3}. {job.relpath}{detail} - {job.status}") + if facts is not None: + edges = facts.resolution.edges + statuses = Counter(edge.status for edge in edges) + lines.extend(["", f"Library references: {len(edges)} ({_counts(statuses)})"]) + for edge in edges: + if edge.status in ("missing", "arch_mismatch", "name_match_unreachable"): + lines.append( + f" {edge.source} -> {edge.name}: {edge.status} ({'; '.join(edge.via[-2:])})" + ) + if facts.resolution.cycles: + lines.append(f"Cycles: {len(facts.resolution.cycles)}") + for cycle in facts.resolution.cycles[:10]: + lines.append(" " + " <-> ".join(cycle)) + link_status = Counter(link.status for link in facts.links) + lines.append(f"Imported symbols: {len(facts.links)} ({_counts(link_status)})") + bound = Counter( + (link.importer, link.target) + for link in facts.links + if link.status == "bound" and link.target != link.importer + ) + if bound: + lines.append("Symbols bound between files of the folder (top 20):") + for (importer, target), count in bound.most_common(20): + lines.append(f" {importer} -> {target}: {count}") + lines.append( + f"\nPlanned in {summary.seconds:.1f}s; nothing was decompiled (--plan)." + ) + return "\n".join(lines) + + +def _counts(counter: Counter[str]) -> str: + return ", ".join(f"{key} {value}" for key, value in sorted(counter.items())) diff --git a/src/tocode/folder_annotate.py b/src/tocode/folder_annotate.py new file mode 100644 index 0000000..cf88ef9 --- /dev/null +++ b/src/tocode/folder_annotate.py @@ -0,0 +1,152 @@ +"""Line-stable ``tocode-link:`` comments in decompiled source. + +A call into another file of the folder gets a trailing comment on the line +that uses the imported symbol, pointing at the file and line range of its +definition. Comments are only appended at the end of existing lines, never as +new lines, so every line number in ``functions.json``/``function-index.json`` +stays valid. Previous annotations are stripped first, which makes the pass +idempotent. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import re + +from .metadata import write_text_atomic + +MARK = "tocode-link:" +MAX_PER_LINE = 3 +_TRAILING_RX = re.compile(r" /\* tocode-link: .*? \*/(?= /\* tocode-link: | ?$)") +_METADATA_RX = re.compile(r"^(/\* metadata:)( tocode-link: .*)?$") + + +@dataclass(slots=True) +class Annotation: + source_file: str # master-relative + line_start: int # the calling function's range (1-based, inclusive) + line_end: int + tokens: list[str] + text: str # "symbol -> file:a-b" + + +def strip_annotations(line: str) -> str: + match = _METADATA_RX.match(line) + if match is not None: + return match.group(1) + if MARK not in line: + return line + previous = None + while previous != line: + previous = line + line = _TRAILING_RX.sub("", line) + return line + + +def _code_mask(lines: list[str]) -> list[bool]: + """Per line: True when the line ends outside comments and strings.""" + mask: list[bool] = [] + in_block = False + for line in lines: + index = 0 + in_string: str | None = None + while index < len(line): + char = line[index] + pair = line[index : index + 2] + if in_block: + if pair == "*/": + in_block = False + index += 2 + continue + elif in_string: + if char == "\\": + index += 2 + continue + if char == in_string: + in_string = None + elif pair == "/*": + in_block = True + index += 2 + continue + elif pair == "//": + break + elif char in "\"'": + in_string = char + index += 1 + mask.append(not in_block and in_string is None and not line.endswith("\\")) + return mask + + +def _code_has_token(line: str, pattern: re.Pattern[str]) -> bool: + # Drop string literals and comments before looking for the identifier. + text = re.sub(r'"(?:\\.|[^"\\])*"', '""', line) + text = re.sub(r"'(?:\\.|[^'\\])*'", "''", text) + text = re.sub(r"/\*.*?\*/", " ", text) + text = text.split("//", 1)[0] + return pattern.search(text) is not None + + +def annotate_file(path: Path, annotations: list[Annotation]) -> int: + """Rewrite one source file; returns how many lines carry a link comment.""" + try: + original = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return 0 + lines = original.split("\n") + cleaned = [strip_annotations(line) for line in lines] + mask = _code_mask(cleaned) + notes: dict[int, list[str]] = {} + for item in annotations: + start = max(item.line_start, 1) + end = min(item.line_end, len(cleaned)) + if not item.tokens or start > end: + continue + pattern = re.compile( + r"(? MAX_PER_LINE + else "" + ) + if line.startswith("/* metadata:"): + # Inside an open comment: no "*/" here. + result[number - 1] = f"{line} {MARK} " + "; ".join(shown) + more + else: + suffix = "".join( + f" /* {MARK} {text.replace('*/', '* /')} */" for text in shown + ) + if more: + suffix = suffix[:-3] + more + " */" + result[number - 1] = line + suffix + if len(result) != len(lines): # pragma: no cover - invariant + raise AssertionError("annotation changed the line count") + text = "\n".join(result) + if text != original: + write_text_atomic(path, text) + return len(notes) diff --git a/src/tocode/folder_docs.py b/src/tocode/folder_docs.py new file mode 100644 index 0000000..e2369ac --- /dev/null +++ b/src/tocode/folder_docs.py @@ -0,0 +1,184 @@ +"""Top-level documents of a folder export.""" + +from __future__ import annotations + +from collections import Counter +from pathlib import Path +from typing import Any + +from . import __version__ +from .links.analyze import FolderLinks +from .links.inventory import Inventory +from .metadata import write_json, write_text_atomic +from .progress import Progress + + +def _rel(path: Path, root: Path) -> str: + try: + return path.relative_to(root).as_posix() + except ValueError: + return str(path) + + +def write_master_documents( + master: Path, + inventory: Inventory, + links: FolderLinks, + jobs: list[Any], + options: Any, + *, + progress: Progress | None, + link_result: Any = None, +) -> None: + master.mkdir(parents=True, exist_ok=True) + job_by_path = {job.relpath: job for job in jobs} + entries = [] + for entry in inventory.entries: + row = entry.to_json() + job = job_by_path.get(entry.relpath) + if job is not None: + row["export_dir"] = _rel(job.export_dir, master) + row["status"] = job.status + entries.append(row) + write_json( + master / "inventory.json", + { + "input": str(inventory.root), + "arch": inventory.arch, + "file_count": len(inventory.entries), + "binary_count": sum(1 for entry in inventory.entries if entry.job), + "skipped_dirs": inventory.skipped_dirs, + "files": entries, + }, + ) + edges = [edge.to_json() for edge in links.resolution.edges] + for row in edges: + target = row.get("to") + job = job_by_path.get(target) if target else None + row["to_export_dir"] = _rel(job.export_dir, master) if job else None + loaded_by: dict[str, list[str]] = {} + for edge in links.resolution.edges: + if edge.target and edge.status == "in_folder": + loaded_by.setdefault(edge.target, []).append(edge.source) + write_json( + master / "dependency-graph.json", + { + "schema_version": 1, + "roots": inventory.roots, + "chains": links.resolution.chains, + "export_order": [job.relpath for job in jobs], + "cycles": links.resolution.cycles, + "edge_status": dict( + sorted(Counter(edge.status for edge in links.resolution.edges).items()) + ), + "loaded_by": { + key: sorted(set(value)) for key, value in sorted(loaded_by.items()) + }, + "edges": edges, + }, + ) + statuses = Counter(job.status.split(":", 1)[0] for job in jobs) + project: dict[str, object] = { + "kind": "folder", + "tocode_version": __version__, + "input": str(inventory.root), + "root_dir": str(master), + "arch": inventory.arch, + "options": { + "only": list(options.only), + "exclude": list(options.exclude), + "include_framework": options.include_framework, + "native": options.native, + "managed": options.managed, + "backend": options.native_options.backend, + "tree": options.native_options.tree, + "entropy": options.native_options.entropy, + }, + "binary_count": sum(1 for entry in inventory.entries if entry.job), + "job_status": dict(sorted(statuses.items())), + "binaries": [job.to_json(master) for job in jobs], + "inventory": "inventory.json", + "dependency_graph": "dependency-graph.json", + "mappings": "mappings.json", + "agents": "AGENTS.md", + "log": "tocode.log", + } + write_json(master / "project.json", project) + manifest: dict[str, object] = dict(project) + if link_result is not None: + manifest["links"] = { + "bound_in_folder": link_result.bound, + "mapped_to_function": link_result.mapped, + "annotated_source_lines": link_result.annotated_lines, + } + manifest["address_spaces"] = link_result.address_spaces + write_json(master / "export-manifest.json", manifest) + write_text_atomic( + master / "AGENTS.md", build_folder_agents(inventory, links, jobs, master) + ) + write_text_atomic(master / "CLAUDE.md", "@./AGENTS.md\n") + if progress is not None: + progress.file_log(f"Wrote folder metadata to {master}") + + +def build_folder_agents( + inventory: Inventory, links: FolderLinks, jobs: list[Any], master: Path +) -> str: + done = [job for job in jobs if job.status == "done"] + other = [job for job in jobs if job.status != "done"] + lines = [ + "# AGENTS", + "", + f"You are working inside a ToCode folder export of `{inventory.root.name}`: every binary " + "of the folder decompiled into its own export, plus the links between them.", + "Treat the recovered source, assembly, metadata, and link tables as evidence for reverse engineering.", + "", + "## Mission", + "", + "Reverse the application, answer user questions, or serve as an oracle/helper to the user regarding this recovered code.", + "Write a report only when the user asks for one; otherwise keep findings focused on the question or task at hand.", + "Do not refactor or modify the generated export unless the user explicitly asks for edits.", + "", + "## What a link is", + "", + "Links come only from the binaries' own load and import tables, resolved with the platform " + "loader's rules (Mach-O two-level namespace, `@rpath`/`@loader_path`; ELF `DT_NEEDED`, " + "`RPATH`/`RUNPATH`, symbol versions; PE import/delay-import tables, forwarders; .NET " + "AssemblyRef/MemberRef signatures and P/Invoke). Nothing is matched by similarity. " + "Strings passed to `dlopen`/`LoadLibrary` and Objective-C message sends are not links.", + "", + "## Files", + "", + "- `//`: one export per binary, mirroring the folder (the directory is named after the file). Each has its own `AGENTS.md`.", + "- `project.json` and `export-manifest.json`: inputs, options, and the status of every binary's export.", + "- `inventory.json`: every file of the folder with its kind, architecture(s), SHA-256, export directory, and status (filtered, runtime library, duplicate, failed, ...).", + "- `dependency-graph.json`: which file loads which (`edges`, with the loader rule in `via`), the load chain of each executable, `loaded_by`, export order, and cycles.", + "- `mappings.json`: every imported symbol of every binary: where it binds (`target`, `export`, `target_va`/`target_token`), the defining function's source file and line range in the target's export (`target_function`), and the importer functions that call it (`call_sites`). `address_spaces` records how link-time addresses were matched to each export.", + "- Inside each export: `imports.json` has `libraries` (resolved library edges) and a `link` per import; `functions.json` has `import_links` on callers and `imported_by` on the functions other files call; `exports.json` has `imported_by_count`. Calls are annotated in `src/raw` with `/* tocode-link: symbol -> file:start-end */` (or in the function's `/* metadata:` header when the call line could not be identified). Annotations never add lines, so all line numbers stay valid.", + "- `tocode.log`: the folder run, with each binary's export log mirrored.", + "", + "## Working Style", + "", + "- Start with `dependency-graph.json` to see the architecture, then the main executable's export.", + "- Follow a call across files with `mappings.json` or the `tocode-link` comments instead of searching by name.", + "- A link whose `target_status` is not `decompiled` (filtered, runtime library, failed) still names the file and address; export it with `--only` to read it.", + "- Cite file paths, function names, addresses, and line numbers for every major claim.", + "", + "## Binaries", + "", + ] + for job in done[:200]: + rel = _rel(job.export_dir, master) + lines.append( + f"- `{job.relpath}` -> `{rel}/` ({job.function_count} functions, {job.backend})" + ) + if len(done) > 200: + lines.append(f"- ... {len(done) - 200} more in `project.json`") + if other: + lines.extend(["", "Not exported in this run:", ""]) + for job in other[:100]: + lines.append(f"- `{job.relpath}`: {job.status}") + if len(other) > 100: + lines.append(f"- ... {len(other) - 100} more in `project.json`") + lines.append("") + return "\n".join(lines) diff --git a/src/tocode/folder_links.py b/src/tocode/folder_links.py new file mode 100644 index 0000000..861c2f4 --- /dev/null +++ b/src/tocode/folder_links.py @@ -0,0 +1,669 @@ +"""Join the folder's link facts to the per-binary exports. + +The linker tables give, for every bound import, the defining file and the +export's link-time address (or .NET MethodDef token). Here that address is +moved into the backend's address space (``image_base`` delta, checked against +exported names), mapped to the exported function that contains it, and +written out with file and line ranges: + +- ``mappings.json`` at the top of the folder export; +- ``imports.json`` / ``functions.json`` / ``exports.json`` of each export + (``libraries``, ``link``, ``import_links``, ``imported_by``); +- ``tocode-link:`` comments in the importer's decompiled source. + +Everything is recomputed from scratch on each run, so it is idempotent and +picks up exports that finished since the last run. +""" + +from __future__ import annotations + +from bisect import bisect_right +from collections import Counter +from dataclasses import dataclass, field +import hashlib +import json +from pathlib import Path +from typing import Any + +from .links.analyze import FolderLinks +from .links.bind import LINK_BOUND, SymbolLink +from .links.inventory import Inventory +from .links.names import c_tokens, link_key +from .metadata import write_json +from .progress import Progress + +LABEL_WIDTH = 40 +BAR_FORMAT = ( + "{desc} {bar:24} {n_fmt}/{total_fmt} {unit_label}{postfix} [{elapsed}<{remaining}]" +) + +MAX_CALL_SITES = 16 +MAX_IMPORTED_BY = 32 +MIN_NAME_CONFIRMATIONS = 3 + + +@dataclass(slots=True) +class FunctionRow: + address: int + size: int + name: str + source_file: str | None # master-relative + line_start: int | None + line_end: int | None + asm_file: str | None + asm_line_start: int | None + asm_line_end: int | None + token: int | None = None + + def location(self) -> dict[str, Any]: + return { + "address": f"0x{self.address:x}" if self.token is None else None, + "token": f"0x{self.token:08x}" if self.token is not None else None, + "name": self.name, + "source_file": self.source_file, + "line_start": self.line_start, + "line_end": self.line_end, + "asm_file": self.asm_file, + "asm_line_start": self.asm_line_start, + "asm_line_end": self.asm_line_end, + } + + +@dataclass +class ExportIndex: + relpath: str + export_dir: Path + rows: list[FunctionRow] + starts: list[int] = field(default_factory=list) + by_token: dict[int, FunctionRow] = field(default_factory=dict) + by_key: dict[str, list[FunctionRow]] = field(default_factory=dict) + backend_base: int | None = None + delta: int = 0 + delta_source: str = "none" + confirmed: int = 0 + checked: int = 0 + + def __post_init__(self) -> None: + self.rows.sort(key=lambda row: row.address) + self.starts = [row.address for row in self.rows] + for row in self.rows: + if row.token is not None: + self.by_token[row.token] = row + self.by_key.setdefault(link_key(row.name), []).append(row) + + def containing(self, address: int) -> FunctionRow | None: + index = bisect_right(self.starts, address) - 1 + if index < 0: + return None + row = self.rows[index] + if row.address == address or address < row.address + max(row.size, 1): + return row + return None + + +@dataclass +class LinkResult: + mapped: int = 0 + bound: int = 0 + annotated_lines: int = 0 + mappings_path: Path | None = None + address_spaces: dict[str, Any] = field(default_factory=dict) + + +def _available(job_export_dir: Path) -> bool: + return (job_export_dir / "export-manifest.json").is_file() and ( + job_export_dir / "functions.json" + ).is_file() + + +def _rel_to(master: Path, export_dir: Path, value: Any) -> str | None: + if not value: + return None + path = Path(str(value)) + if not path.is_absolute(): + path = export_dir / path + try: + return path.resolve().relative_to(master.resolve()).as_posix() + except (ValueError, OSError): + return str(value) + + +def _int(value: Any) -> int | None: + if isinstance(value, int): + return value + if isinstance(value, str): + text = value.rsplit("!", 1)[-1] + try: + return int(text, 16) if text.lower().startswith("0x") else int(text) + except ValueError: + return None + return None + + +def load_index( + master: Path, relpath: str, export_dir: Path, cache_dir: Path +) -> ExportIndex | None: + manifest = export_dir / "export-manifest.json" + functions = export_dir / "functions.json" + try: + stamp = f"{functions.stat().st_mtime_ns}:{functions.stat().st_size}:{manifest.stat().st_mtime_ns}" + except OSError: + return None + cache = cache_dir / (hashlib.sha1(relpath.encode()).hexdigest()[:16] + ".json") + rows: list[FunctionRow] | None = None + try: + cached = json.loads(cache.read_text(encoding="utf-8")) + if cached.get("stamp") == stamp: + rows = [FunctionRow(*item) for item in cached["rows"]] + except (OSError, ValueError, TypeError, KeyError): + rows = None + if rows is None: + try: + payload = json.loads(functions.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + rows = [] + for item in payload.get("functions", []): + token = _int(item.get("token")) if item.get("token") else None + address = _int(item.get("address")) + if address is None: + address = token or 0 + rows.append( + FunctionRow( + address=address, + size=int(item.get("size") or 0), + name=str(item.get("name") or ""), + source_file=_rel_to(master, export_dir, item.get("source_file")), + line_start=item.get("source_line_start"), + line_end=item.get("source_line_end"), + asm_file=_rel_to(master, export_dir, item.get("asm_file")), + asm_line_start=item.get("asm_line_start"), + asm_line_end=item.get("asm_line_end"), + token=token, + ) + ) + del payload + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text( + json.dumps( + { + "stamp": stamp, + "rows": [ + [ + row.address, + row.size, + row.name, + row.source_file, + row.line_start, + row.line_end, + row.asm_file, + row.asm_line_start, + row.asm_line_end, + row.token, + ] + for row in rows + ], + } # fmt: skip + ), + encoding="utf-8", + ) + index = ExportIndex(relpath=relpath, export_dir=export_dir, rows=rows) + try: + project = json.loads((export_dir / "project.json").read_text(encoding="utf-8")) + index.backend_base = _int(project.get("image_base")) + except (OSError, ValueError, AttributeError): + index.backend_base = None + return index + + +def choose_delta( + index: ExportIndex, link_base: int, exports: list[tuple[str, int]] +) -> None: + """Pick the link-time -> backend address delta best confirmed by names.""" + candidates: dict[int, str] = {0: "identity"} + if index.backend_base is not None: + candidates.setdefault(index.backend_base - link_base, "image_base") + diffs: Counter[int] = Counter() + for name, address in exports: + for row in index.by_key.get(link_key(name), [])[:4]: + diffs[row.address - address] += 1 + if diffs: + delta, count = diffs.most_common(1)[0] + if count >= MIN_NAME_CONFIRMATIONS: + candidates.setdefault(delta, "export names") + best = (-1, 0, "none") + checked = 0 + for delta, source in candidates.items(): + confirmed = 0 + checked = 0 + for name, address in exports: + rows = index.by_key.get(link_key(name)) + if not rows: + continue + checked += 1 + if any(row.address == address + delta for row in rows): + confirmed += 1 + if confirmed > best[0] or (confirmed == best[0] and source == "image_base"): + best = (confirmed, delta, source) + index.confirmed, index.delta, index.delta_source = best[0], best[1], best[2] + index.checked = checked + if index.confirmed <= 0 and index.backend_base is not None: + index.delta = index.backend_base - link_base + index.delta_source = "image_base (unconfirmed)" + + +def link_exports( + master: Path, + inventory: Inventory, + links: FolderLinks, + jobs: list[Any], + *, + progress: Progress | None = None, +) -> LinkResult: + from .folder_annotate import Annotation, annotate_file + + result = LinkResult() + cache_dir = master / ".tocode-folder" / "index" + job_by_path = {job.relpath: job for job in jobs} + indexes: dict[str, ExportIndex | None] = {} + + def index_for(relpath: str) -> ExportIndex | None: + if relpath in indexes: + return indexes[relpath] + job = job_by_path.get(relpath) + index = None + if job is not None and _available(job.export_dir): + index = load_index(master, relpath, job.export_dir, cache_dir) + facts = inventory.folder.facts(relpath) + if index is not None and facts is not None and facts.format != "dotnet": + exports = [ + (item.name, item.address) + for item in facts.exports + if item.address is not None and item.kind != "absolute" + ] + choose_delta(index, facts.image_base, exports) + result.address_spaces[relpath] = { + "link_base": f"0x{facts.image_base:x}", + "backend_base": f"0x{index.backend_base:x}" + if index.backend_base is not None + else None, + "delta": f"{'-' if index.delta < 0 else ''}0x{abs(index.delta):x}", + "source": index.delta_source, + "confirmed_by_export_names": index.confirmed, + "export_names_found": index.checked, + } + indexes[relpath] = index + return index + + def export_dir(relpath: str | None) -> str | None: + job = job_by_path.get(relpath or "") + if job is None: + return None + return _rel_to(master, master, job.export_dir) + + def target_status(relpath: str) -> str: + job = job_by_path.get(relpath) + if job is None: + return "not_exported" + if job.status == "done": + return "decompiled" if _available(job.export_dir) else "export_missing" + return job.status + + # 1. Each bound link -> the defining function in the target's export. + rows_out: list[dict[str, Any]] = [] + importer_sites: dict[str, list[tuple[SymbolLink, dict[str, Any]]]] = {} + imported_by: dict[str, dict[int, list[dict[str, Any]]]] = {} + for link in links.links: + row = link.to_json() + if link.target is not None: + row["target_export_dir"] = export_dir(link.target) + row["target_status"] = target_status(link.target) + target_row: FunctionRow | None = None + if link.status == LINK_BOUND and link.target is not None: + result.bound += 1 + index = index_for(link.target) + if index is not None: + if link.token is not None: + target_row = index.by_token.get(link.token) + elif link.address is not None: + target_row = index.containing(link.address + index.delta) + if target_row is None: + row["target_status"] = "not_in_a_function" + if target_row is not None: + row["target_function"] = target_row.location() + result.mapped += 1 + rows_out.append(row) + importer_sites.setdefault(link.importer, []).append((link, row)) + if target_row is not None: + imported_by.setdefault(link.target or "", {}).setdefault( + target_row.address, [] + ).append(row) + + # 2. Call sites in the importers, enrichment, comments. + annotated = 0 + bar = ( + progress.new_bar( + total=len(importer_sites), + desc=f"{'linking':<{LABEL_WIDTH}}", + disable=None, + bar_format=BAR_FORMAT.replace("{unit_label}", "binaries"), + ) + if progress is not None + else None + ) + for importer, pairs in importer_sites.items(): + if bar is not None: + bar.update(1) + index = index_for(importer) + job = job_by_path.get(importer) + if index is None or job is None or job.status.startswith("duplicate"): + continue + aliases: dict[int, set[str]] = {} + callers = _call_sites(importer, index, pairs, inventory, aliases) + for link, row in pairs: + sites = callers.get(id(link), []) + if sites: + row["call_site_count"] = len(sites) + row["call_sites"] = [site.location() for site in sites[:MAX_CALL_SITES]] + annotations: list[Annotation] = [] + for link, row in pairs: + target_fn = row.get("target_function") + if link.status != LINK_BOUND or link.target is None: + continue + note = _note(link, row) + for site in callers.get(id(link), []): + if site.source_file and site.line_start and site.line_end: + annotations.append( + Annotation( + source_file=site.source_file, + line_start=site.line_start, + line_end=site.line_end, + tokens=c_tokens( + link.symbol + if link.kind != "pinvoke" + else (link.owner or link.symbol), + link.version, + ) + + sorted(aliases.get(id(link), ())), + text=note, + ) + ) + del target_fn + by_file: dict[str, list[Annotation]] = {} + for item in annotations: + by_file.setdefault(item.source_file, []).append(item) + for relfile, items in by_file.items(): + annotated += annotate_file(master / relfile, items) + _enrich_importer( + master, job.export_dir, importer, pairs, links, export_dir, callers + ) + result.annotated_lines = annotated + if bar is not None: + bar.close() + + for target, by_address in imported_by.items(): + job = job_by_path.get(target) + if job is None or not _available(job.export_dir): + continue + _enrich_target(job.export_dir, by_address, indexes.get(target)) + + path = master / "mappings.json" + statuses = Counter(row["status"] for row in rows_out) + write_json( + path, + { + "schema_version": 1, + "summary": { + "imports": len(rows_out), + "bound_in_folder": result.bound, + "mapped_to_function": result.mapped, + "by_status": dict(sorted(statuses.items())), + "annotated_source_lines": annotated, + }, + "address_spaces": result.address_spaces, + "links": rows_out, + }, + ) + result.mappings_path = path + if progress is not None: + progress.log( + f"Links: {result.bound} imports bound inside the folder, {result.mapped} mapped " + f"to a decompiled function, {annotated} source lines annotated (mappings.json)" + ) + return result + + +def _note(link: SymbolLink, row: dict[str, Any]) -> str: + target = row.get("target_function") or {} + where = row.get("target_export_dir") or link.target + if target.get("source_file") and target.get("line_start"): + return f"{link.symbol} -> {target['source_file']}:{target['line_start']}-{target['line_end']}" + if target: + place = target.get("address") or target.get("token") + return f"{link.symbol} -> {where} {target.get('name')} @ {place} (no source lines in that export)" + return f"{link.symbol} -> {where} ({row.get('target_status', 'not decompiled')})" + + +def _call_sites( + importer, index: ExportIndex, pairs, inventory, aliases: dict[int, set[str]] +) -> dict[int, list[FunctionRow]]: + """Importer functions that call each linked import. + + ``aliases`` collects the identifiers the backend uses for each import + (from slot matches), so the source annotation can find the call line. + """ + facts = inventory.folder.facts(importer) + result: dict[int, list[FunctionRow]] = {} + if facts is not None and facts.format == "dotnet": + for link, _row in pairs: + sites = [ + index.by_token[token] + for token in (link.callers or []) + if token in index.by_token + ] + if sites: + result[id(link)] = sites + return result + by_key: dict[str, list[SymbolLink]] = {} + by_slot: dict[int, SymbolLink] = {} + for link, _row in pairs: + by_key.setdefault(link_key(link.symbol), []).append(link) + if link.slot is not None: + by_slot[link.slot] = link + # The backend may name an import differently (ordinal imports, renamed + # slots); its imports.json address is the same pointer slot the linker + # tables give, which ties the backend's name to the link. + if by_slot: + imports = _load_json(index.export_dir / "imports.json") or {} + for group in imports.get("imports", []): + for item in group.get("functions", []) if isinstance(group, dict) else []: + address = _int(item.get("address")) if isinstance(item, dict) else None + link = ( + by_slot.get(address - index.delta) if address is not None else None + ) + if link is not None: + backend_name = str(item.get("name") or "") + key = link_key(backend_name) + if link not in by_key.setdefault(key, []): + by_key[key].append(link) + aliases.setdefault(id(link), set()).update({backend_name, key}) + try: + payload = json.loads( + (index.export_dir / "functions.json").read_text(encoding="utf-8") + ) + except (OSError, ValueError): + return result + functions = payload.get("functions", []) + stubs: dict[int, str] = {} + for item in functions: + key = link_key(str(item.get("name") or "")) + address = _int(item.get("address")) + if key in by_key and address is not None and (item.get("size") or 0) <= 64: + stubs[address] = key + rows_by_address = {row.address: row for row in index.rows} + for item in functions: + address = _int(item.get("address")) + if address is None or address in stubs: + continue + keys = {link_key(str(name)) for name in item.get("callees_imports") or []} + for callee in item.get("callees") or []: + target = _int(callee) + if target in stubs: + keys.add(stubs[target]) + row = rows_by_address.get(address) + if row is None: + continue + for key in keys: + for link in by_key.get(key, []): + result.setdefault(id(link), []).append(row) + return result + + +def _load_json(path: Path) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return payload if isinstance(payload, dict) else None + + +def _compact_link(row: dict[str, Any]) -> dict[str, Any]: + keys = ( + "symbol", + "library", + "status", + "resolution", + "target", + "target_export_dir", + "target_status", + "export", + "target_va", + "target_token", + "target_function", + ) + return {key: row[key] for key in keys if row.get(key) is not None} + + +def _enrich_importer( + master, + export_dir_path: Path, + importer, + pairs, + links: FolderLinks, + export_dir, + callers, +) -> None: + imports_path = export_dir_path / "imports.json" + payload = _load_json(imports_path) + if payload is not None: + edges = [] + for edge in links.resolution.edges: + if edge.source != importer: + continue + row = edge.to_json() + row["to_export_dir"] = export_dir(edge.target) if edge.target else None + edges.append(row) + payload["libraries"] = edges + by_key: dict[str, list[dict[str, Any]]] = {} + for link, row in pairs: + by_key.setdefault(link_key(link.symbol), []).append(row) + for group in payload.get("imports", []): + for item in group.get("functions", []) if isinstance(group, dict) else []: + if not isinstance(item, dict): + continue + item.pop("link", None) + name = str(item.get("name") or "") + matches = by_key.get(link_key(name)) + if matches is None and "::" not in name and " " in name: + # .NET full names: "RetType Ns.Type::Member(Args)" + member = name.split(" ", 1)[1].split("(", 1)[0] + matches = by_key.get(link_key(member)) + if matches: + item["link"] = ( + _compact_link(matches[0]) + if len(matches) == 1 + else [_compact_link(row) for row in matches[:8]] + ) + write_json(imports_path, payload) + functions_path = export_dir_path / "functions.json" + links_by_row: dict[int, list[dict[str, Any]]] = {} + for link, row in pairs: + for site in callers.get(id(link), []): + target = row.get("target_function") or {} + links_by_row.setdefault(site.address, []).append( + { + "symbol": link.symbol, + "target": link.target, + "status": row.get("target_status") + if link.status == LINK_BOUND + else link.status, + "source_file": target.get("source_file"), + "line_start": target.get("line_start"), + "line_end": target.get("line_end"), + } + ) + payload = _load_json(functions_path) + if payload is None: + return + changed = False + for item in payload.get("functions", []): + if "import_links" in item: + del item["import_links"] + changed = True + address = ( + _int(item.get("address")) + if not item.get("token") + else _int(item.get("token")) + ) + rows = links_by_row.get(address or -1) + if rows: + item["import_links"] = rows + changed = True + if changed: + write_json(functions_path, payload) + + +def _enrich_target( + export_dir_path: Path, + by_address: dict[int, list[dict[str, Any]]], + index: ExportIndex | None, +) -> None: + functions_path = export_dir_path / "functions.json" + payload = _load_json(functions_path) + if payload is None: + return + for item in payload.get("functions", []): + item.pop("imported_by", None) + item.pop("imported_by_count", None) + address = ( + _int(item.get("token")) if item.get("token") else _int(item.get("address")) + ) + rows = by_address.get(address if address is not None else -1) + if not rows: + continue + item["imported_by_count"] = len(rows) + item["imported_by"] = [ + { + "importer": row["importer"], + "symbol": row["symbol"], + "call_site_count": row.get("call_site_count", 0), + "call_sites": row.get("call_sites", [])[:4], + } + for row in rows[:MAX_IMPORTED_BY] + ] + write_json(functions_path, payload) + exports_path = export_dir_path / "exports.json" + exports = _load_json(exports_path) + if exports is None: + return + counts: Counter[str] = Counter() + for rows in by_address.values(): + for row in rows: + counts[link_key(str(row.get("export") or row["symbol"]))] += 1 + for item in exports.get("exports", []): + if isinstance(item, dict): + item.pop("imported_by_count", None) + count = counts.get(link_key(str(item.get("name") or ""))) + if count: + item["imported_by_count"] = count + write_json(exports_path, exports) diff --git a/src/tocode/links/__init__.py b/src/tocode/links/__init__.py new file mode 100644 index 0000000..e9adddf --- /dev/null +++ b/src/tocode/links/__init__.py @@ -0,0 +1,7 @@ +"""Link facts between the binaries of a folder. + +Everything here is pure Python and reads the files directly: the import and +export tables the linker wrote, and the loader rules that consume them. No +decompiler backend is involved, so a whole application folder can be mapped +in seconds and the result does not depend on how a backend names symbols. +""" diff --git a/src/tocode/links/analyze.py b/src/tocode/links/analyze.py new file mode 100644 index 0000000..3fc5114 --- /dev/null +++ b/src/tocode/links/analyze.py @@ -0,0 +1,42 @@ +"""Resolve and bind a scanned folder: the full set of link facts.""" + +from __future__ import annotations + +from dataclasses import dataclass +import posixpath + +from .bind import Binder, SymbolLink +from .inventory import Inventory +from .model import EDGE_IN_FOLDER, FileEdge +from .resolve import Resolution, resolve_folder + +LIB_APPHOST = "apphost" + + +@dataclass +class FolderLinks: + resolution: Resolution + links: list[SymbolLink] + + +def link_folder(inventory: Inventory) -> FolderLinks: + resolution = resolve_folder(inventory.folder, inventory.roots) + for entry in inventory.entries: + target = entry.apphost_for + if target is None or target not in inventory.folder.images: + continue + # The .NET apphost carries the path of the app assembly it starts. + resolution.edges.append( + FileEdge( + source=entry.relpath, + library=-1, + name=posixpath.basename(target), + kind=LIB_APPHOST, + status=EDGE_IN_FOLDER, + target=target, + via=["apphost launcher"], + chain_root=entry.relpath, + ) + ) + links = Binder(inventory.folder, resolution).bind_all() + return FolderLinks(resolution=resolution, links=links) diff --git a/src/tocode/links/bind.py b/src/tocode/links/bind.py new file mode 100644 index 0000000..7502c8b --- /dev/null +++ b/src/tocode/links/bind.py @@ -0,0 +1,498 @@ +"""Symbol binding: which export of which file each import binds to.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import posixpath +from typing import Any + +from .model import ( + EDGE_APISET, + EDGE_IN_FOLDER, + EDGE_SYSTEM, + LIB_REEXPORT, + SPECIAL_FLAT, + SPECIAL_MAIN, + SPECIAL_SELF, + SPECIAL_WEAK, + ExportSym, + FileEdge, + FileFacts, + ImportSym, +) +from .resolve import ( + EDGE_EXTERNAL, + EDGE_NAME_UNREACHABLE, + KNOWN_DLLS, + Folder, + Resolution, +) + +# SymbolLink.status values beyond the FileEdge statuses +LINK_BOUND = "bound" # bound to an export of a file in the folder +LINK_UNRESOLVED = "unresolved_symbol" # the library is here, the symbol is not +LINK_NO_LIBRARY = "no_definer_in_folder" # flat/weak/global lookup found none here +MAX_FOLLOW = 16 + + +@dataclass(slots=True) +class SymbolLink: + importer: str + symbol: str + status: str + library: str | None = None # the library reference as written + kind: str | None = None + resolution: str | None = None # how it bound (two_level, versioned, ...) + target: str | None = None # file relpath of the defining binary + export: str | None = None # export name in the target + address: int | None = None # link-time VA in the target + token: int | None = None # .NET MethodDef token in the target + version: str | None = None + ordinal: int | None = None + via: list[str] = field(default_factory=list) + candidates: list[str] = field(default_factory=list) + callers: list[int] | None = None # .NET MethodDef tokens in the importer + owner: str | None = None # .NET declaring type / managed P/Invoke method + slot: int | None = None # importer's pointer slot (PE IAT, Mach-O bind) + + def to_json(self) -> dict[str, Any]: + row: dict[str, Any] = { + "importer": self.importer, + "symbol": self.symbol, + "status": self.status, + "library": self.library, + } + for key in ( + "kind", + "resolution", + "target", + "export", + "version", + "ordinal", + "owner", + ): + value = getattr(self, key) + if value is not None: + row[key] = value + if self.address is not None: + row["target_va"] = f"0x{self.address:x}" + if self.token is not None: + row["target_token"] = f"0x{self.token:08x}" + if self.via: + row["via"] = self.via + if self.candidates: + row["candidates"] = self.candidates + return row + + +class _Exports: + def __init__(self, facts: FileFacts) -> None: + self.facts = facts + self.by_name: dict[str, list[ExportSym]] = {} + self.by_ordinal: dict[int, ExportSym] = {} + self.by_signature: dict[tuple[str, str | None], ExportSym] = {} + for item in facts.exports: + self.by_name.setdefault(item.name, []).append(item) + if item.ordinal is not None: + self.by_ordinal.setdefault(item.ordinal, item) + if item.signature is not None: + self.by_signature.setdefault((item.name, item.signature), item) + + def named(self, name: str, version: str | None = None) -> ExportSym | None: + rows = self.by_name.get(name) + if not rows: + return None + if version is not None: + for item in rows: + if item.version == version: + return item + unversioned = [item for item in rows if item.version is None] + return unversioned[0] if unversioned else None + defaults = [item for item in rows if item.default_version] + return (defaults or rows)[0] + + +class Binder: + def __init__(self, folder: Folder, resolution: Resolution) -> None: + self.folder = folder + self.resolution = resolution + self._exports: dict[tuple[str, str], _Exports] = {} + self.edges: dict[tuple[str, int], list[FileEdge]] = {} + for edge in resolution.edges: + self.edges.setdefault((edge.source, edge.library), []).append(edge) + + def exports(self, relpath: str, arch: str | None = None) -> _Exports | None: + facts = self.folder.facts(relpath, arch) + if facts is None: + return None + key = (relpath, facts.arch) + cached = self._exports.get(key) + if cached is None: + cached = self._exports[key] = _Exports(facts) + return cached + + def chain_order(self, relpath: str) -> list[str]: + root = self.resolution.loaded_by.get(relpath, relpath) + return self.resolution.chains.get(root, [relpath]) + + def bind_all(self) -> list[SymbolLink]: + links: list[SymbolLink] = [] + for relpath in sorted(self.folder.images): + facts = self.folder.facts(relpath) + if facts is not None: + links.extend(self.bind_file(relpath, facts)) + return links + + def bind_file(self, relpath: str, facts: FileFacts) -> list[SymbolLink]: + rows: list[SymbolLink] = [] + for item in facts.imports: + library = ( + facts.libraries[item.library].name + if item.library is not None and item.library < len(facts.libraries) + else None + ) + if item.kind == "pinvoke": + rows.extend(self._pinvoke(relpath, facts, item, library)) + continue + link = SymbolLink( + importer=relpath, + symbol=item.name, + status=LINK_NO_LIBRARY, + library=library, + kind=item.kind, + version=item.version, + ordinal=item.ordinal, + callers=item.callers, + owner=item.owner, + slot=item.slot, + ) + if facts.format == "macho": + self._macho(link, relpath, facts, item) + elif facts.format == "elf": + self._elf(link, relpath, facts, item) + elif facts.format == "dotnet" and item.signature is not None: + self._member(link, relpath, facts, item) + else: + self._pe(link, relpath, facts, item) + rows.append(link) + return rows + + # -- helpers ------------------------------------------------------------ + def _edge(self, relpath: str, library: int | None) -> FileEdge | None: + if library is None: + return None + edges = self.edges.get((relpath, library)) + return edges[0] if edges else None + + def _apply_edge(self, link: SymbolLink, edge: FileEdge | None) -> str | None: + """Copy a non-folder edge status onto the link; return the target relpath.""" + if edge is None: + link.status = EDGE_EXTERNAL + return None + if edge.status == EDGE_NAME_UNREACHABLE and edge.target is not None: + link.via.append( + f"{edge.target} is reached by name only (not by a loader search path)" + ) + return edge.target + if edge.status != EDGE_IN_FOLDER or edge.target is None: + link.status = edge.status + if edge.target is not None: + link.via.append(f"{edge.status}: {edge.target}") + return None + return edge.target + + def _bound( + self, link: SymbolLink, target: str, export: ExportSym, resolution: str + ) -> None: + link.status = LINK_BOUND + link.resolution = link.resolution or resolution + link.target = target + link.export = export.name + link.address = export.address + link.token = export.token + + # -- Mach-O ------------------------------------------------------------- + def _macho( + self, link: SymbolLink, relpath: str, facts: FileFacts, item: ImportSym + ) -> None: + if item.special in (SPECIAL_FLAT, SPECIAL_WEAK) or ( + item.library is None and item.special is None + ): + self._scope_lookup( + link, relpath, facts, item.name, item.special or SPECIAL_FLAT + ) + return + if item.special == SPECIAL_SELF: + self._macho_find(link, relpath, facts.arch, item.name, "self", 0) + return + if item.special == SPECIAL_MAIN: + root = self.resolution.loaded_by.get(relpath, relpath) + self._macho_find(link, root, facts.arch, item.name, "main_executable", 0) + return + target = self._apply_edge(link, self._edge(relpath, item.library)) + if target is None: + return + if not self._macho_find(link, target, facts.arch, item.name, "two_level", 0): + link.status = LINK_UNRESOLVED + link.target = target + + def _macho_find(self, link, target, arch, name, resolution, depth) -> bool: + if depth > MAX_FOLLOW: + return False + table = self.exports(target, arch) + if table is None: + return False + export = table.named(name) + if export is not None: + if export.reexport_library is not None: + edge = self._edge(target, export.reexport_library) + if edge is None or edge.status != EDGE_IN_FOLDER or edge.target is None: + link.status = edge.status if edge else EDGE_EXTERNAL + link.via.append( + f"re-exported by {target} from {edge.name if edge else '?'}" + ) + link.target = None + return True + link.via.append(f"re-exported by {target}") + return self._macho_find( + link, + edge.target, + arch, + export.reexport_name or name, + "reexport", + depth + 1, + ) + self._bound(link, target, export, resolution) + return True + # Symbols of LC_REEXPORT_DYLIB libraries are visible through this one. + facts = table.facts + for index, lib in enumerate(facts.libraries): + if lib.kind != LIB_REEXPORT: + continue + edge = self._edge(target, index) + if edge is None or edge.status != EDGE_IN_FOLDER or edge.target is None: + continue + mark = len(link.via) + link.via.append(f"re-exported library of {target}") + if self._macho_find(link, edge.target, arch, name, "reexport", depth + 1): + return True + del link.via[mark:] + return False + + def _scope_lookup(self, link, relpath, facts, name, mode) -> None: + """dyld flat/weak lookup and ELF global scope: first definer in load order.""" + definers: list[tuple[str, ExportSym]] = [] + for image in self.chain_order(relpath): + if image == relpath and mode not in (SPECIAL_WEAK,): + continue + table = self.exports(image, facts.arch) + if table is None: + continue + export = table.named(name) + if export is not None and export.reexport_library is None: + definers.append((image, export)) + link.resolution = mode + if not definers: + link.status = LINK_NO_LIBRARY + return + if mode == SPECIAL_WEAK: + strong = [entry for entry in definers if not entry[1].weak] + chosen = (strong or definers)[0] + else: + chosen = definers[0] + self._bound(link, chosen[0], chosen[1], mode) + link.candidates = [image for image, _export in definers if image != chosen[0]] + + # -- ELF ---------------------------------------------------------------- + def _elf( + self, link: SymbolLink, relpath: str, facts: FileFacts, item: ImportSym + ) -> None: + if item.library is not None: + # Versioned reference: only the file named by the version need binds it. + target = self._apply_edge(link, self._edge(relpath, item.library)) + if target is None: + return + table = self.exports(target, facts.arch) + export = table.named(item.name, item.version) if table else None + if export is None: + link.status = LINK_UNRESOLVED + link.target = target + return + self._bound(link, target, export, "versioned") + return + self._scope_lookup(link, relpath, facts, item.name, "global_scope") + if link.status == LINK_BOUND: + outside = [ + edge.name + for edge in self.resolution.edges + if edge.source in self.chain_order(relpath) + and edge.status not in (EDGE_IN_FOLDER, EDGE_NAME_UNREACHABLE) + ] + if outside: + link.via.append( + f"{len(set(outside))} libraries outside the folder are also in the " + "lookup scope and were not inspected" + ) + + # -- PE ----------------------------------------------------------------- + def _pe( + self, link: SymbolLink, relpath: str, facts: FileFacts, item: ImportSym + ) -> None: + target = self._apply_edge(link, self._edge(relpath, item.library)) + if target is None: + return + self._pe_find( + link, + relpath, + target, + facts.arch, + item.name, + item.ordinal, + "name" if item.ordinal is None else "ordinal", + 0, + ) + + def _pe_find( + self, link, importer, target, arch, name, ordinal, resolution, depth + ) -> None: + table = self.exports(target, arch) + export = None + if table is not None: + export = ( + table.by_ordinal.get(ordinal) + if ordinal is not None + else table.named(name) + ) + if export is None: + link.status = LINK_UNRESOLVED + link.target = target + return + if export.forwarder is None: + self._bound(link, target, export, resolution) + return + if depth >= MAX_FOLLOW: + link.status = LINK_UNRESOLVED + link.via.append("forwarder chain too long") + return + module, _, symbol = export.forwarder.rpartition(".") + link.via.append(f"forwarded by {target} to {export.forwarder}") + link.resolution = "forwarder" + dll = module if module.lower().endswith(".dll") else f"{module}.dll" + lower = dll.lower() + if lower.startswith(("api-ms-win-", "ext-ms-win-")): + link.status, link.target = EDGE_APISET, None + return + if lower in KNOWN_DLLS: + link.status, link.target = EDGE_SYSTEM, None + return + next_target = self._pe_module(importer, dll) + if next_target is None: + link.status, link.target = EDGE_EXTERNAL, None + return + if symbol.startswith("#") and symbol[1:].isdigit(): + self._pe_find( + link, + importer, + next_target, + arch, + symbol, + int(symbol[1:]), + "forwarder", + depth + 1, + ) + else: + self._pe_find( + link, importer, next_target, arch, symbol, None, "forwarder", depth + 1 + ) + + def _pe_module(self, importer: str, dll: str) -> str | None: + lower = dll.lower() + for image in self.chain_order(importer): + if posixpath.basename(image).lower() == lower: + return image + root = self.resolution.loaded_by.get(importer, importer) + return self.folder.lookup( + posixpath.join(posixpath.dirname(root), dll), fold_case=True + ) + + # -- .NET ----------------------------------------------------------------- + def _member( + self, link: SymbolLink, relpath: str, facts: FileFacts, item: ImportSym + ) -> None: + target = self._apply_edge(link, self._edge(relpath, item.library)) + if target is None: + return + owner = item.owner or "" + for _ in range(MAX_FOLLOW): + table = self.exports(target) + if table is None: + link.status, link.target = LINK_UNRESOLVED, target + return + if item.kind == "data": + link.status, link.target = LINK_BOUND, target + link.resolution = "field" + return + export = table.by_signature.get((item.name, item.signature)) + if export is not None: + self._bound(link, target, export, link.resolution or "member_ref") + return + forwarded = (table.facts.managed or {}).get("forwarders", {}).get(owner) + if not forwarded: + break + matches = self.folder.by_assembly(forwarded) + link.via.append(f"type forwarded by {target} to {forwarded}") + link.resolution = "type_forwarder" + if not matches: + link.status, link.target = EDGE_EXTERNAL, None + return + target = matches[0] + link.status, link.target = LINK_UNRESOLVED, target + + def _pinvoke(self, relpath, facts, item, library) -> list[SymbolLink]: + edges: list[FileEdge | None] = list( + self.edges.get((relpath, item.library), []) + if item.library is not None + else [] + ) + rows: list[SymbolLink] = [] + for edge in edges or [None]: + link = SymbolLink( + importer=relpath, + symbol=item.name, + status=LINK_NO_LIBRARY, + library=library, + kind="pinvoke", + callers=item.callers, + owner=item.owner, + ) + target = self._apply_edge(link, edge) + if target is not None: + table = self.exports(target) + export = None + if table is not None: + for name in _pinvoke_names( + item.name, item.signature, table.facts.format + ): + if name.startswith("#") and name[1:].isdigit(): + export = table.by_ordinal.get(int(name[1:])) + else: + export = table.named(name) + if export is not None: + break + if export is None: + link.status, link.target = LINK_UNRESOLVED, target + else: + self._bound(link, target, export, "pinvoke") + rows.append(link) + return rows + + +def _pinvoke_names(name: str, charset: str | None, target_format: str) -> list[str]: + """Entry point names the runtime tries, in order (ExactSpelling/CharSet).""" + if target_format == "macho": + return [f"_{name}"] + if target_format != "pe" or charset == "exact": + return [name] + if charset in ("unicode", "auto"): + return [f"{name}W", name] + return [name, f"{name}A"] diff --git a/src/tocode/links/binfmt_dotnet.py b/src/tocode/links/binfmt_dotnet.py new file mode 100644 index 0000000..90c387d --- /dev/null +++ b/src/tocode/links/binfmt_dotnet.py @@ -0,0 +1,626 @@ +""".NET metadata tables read directly (no dnlib, no runtime). + +Produces the binding facts the CLR itself uses: the assembly identity, the +AssemblyRef list, every MemberRef whose declaring type lives in another +assembly (with a canonical signature so it can be matched to the MethodDef it +binds to), the methods that reference each one (from an IL token scan), the +P/Invoke declarations (ImplMap -> ModuleRef), and type forwarders. +""" + +from __future__ import annotations + +import mmap +import struct +from typing import Any + +from ..backends.dotnet_pe import ( + TOK_METHOD, + PeInfo, + method_body, + parse_pe, + scan_il_tokens, +) +from .model import ( + LIB_ASSEMBLY, + LIB_PINVOKE, + ExportSym, + FileFacts, + ImportSym, + LibraryRef, +) + +Buffer = bytes | mmap.mmap + +# Table ids +MODULE, TYPEREF, TYPEDEF, FIELDPTR, FIELD, METHODPTR, METHODDEF, PARAMPTR, PARAM = ( + range(9) +) +INTERFACEIMPL, MEMBERREF = 0x09, 0x0A +MODULEREF, TYPESPEC, IMPLMAP = 0x1A, 0x1B, 0x1C +ASSEMBLY, ASSEMBLYREF, FILE, EXPORTEDTYPE = 0x20, 0x23, 0x26, 0x27 +NESTEDCLASS, GENERICPARAM, METHODSPEC = 0x29, 0x2A, 0x2B +TABLE_COUNT = 0x2D + +# Coded index kinds: (tag bits, tables); None = unused tag value. +CODED: dict[str, tuple[int, tuple[int | None, ...]]] = { + "TypeDefOrRef": (2, (TYPEDEF, TYPEREF, TYPESPEC)), + "HasConstant": (2, (FIELD, PARAM, 0x17)), + "HasCustomAttribute": ( + 5, + ( + METHODDEF, + FIELD, + TYPEREF, + TYPEDEF, + PARAM, + INTERFACEIMPL, + MEMBERREF, + MODULE, + 0x0E, + 0x17, + 0x14, + 0x11, + MODULEREF, + TYPESPEC, + ASSEMBLY, + ASSEMBLYREF, + FILE, + EXPORTEDTYPE, + 0x28, + GENERICPARAM, + 0x2C, + METHODSPEC, + ), + ), # fmt: skip + "HasFieldMarshal": (1, (FIELD, PARAM)), + "HasDeclSecurity": (2, (TYPEDEF, METHODDEF, ASSEMBLY)), + "MemberRefParent": (3, (TYPEDEF, TYPEREF, MODULEREF, METHODDEF, TYPESPEC)), + "HasSemantics": (1, (0x14, 0x17)), + "MethodDefOrRef": (1, (METHODDEF, MEMBERREF)), + "MemberForwarded": (1, (FIELD, METHODDEF)), + "Implementation": (2, (FILE, ASSEMBLYREF, EXPORTEDTYPE)), + "CustomAttributeType": (3, (None, None, METHODDEF, MEMBERREF, None)), + "ResolutionScope": (2, (MODULE, MODULEREF, ASSEMBLYREF, TYPEREF)), + "TypeOrMethodDef": (1, (TYPEDEF, METHODDEF)), +} + +# Column specs: "2"/"4"/"1" fixed, "S" string, "G" guid, "B" blob, +# "T:" table index, "C:" coded index. +SCHEMA: dict[int, tuple[str, ...]] = { + 0x00: ("2", "S", "G", "G", "G"), + 0x01: ("C:ResolutionScope", "S", "S"), + 0x02: ("4", "S", "S", "C:TypeDefOrRef", "T:4", "T:6"), + 0x03: ("T:4",), + 0x04: ("2", "S", "B"), + 0x05: ("T:6",), + 0x06: ("4", "2", "2", "S", "B", "T:8"), + 0x07: ("T:8",), + 0x08: ("2", "2", "S"), + 0x09: ("T:2", "C:TypeDefOrRef"), + 0x0A: ("C:MemberRefParent", "S", "B"), + 0x0B: ("1", "1", "C:HasConstant", "B"), + 0x0C: ("C:HasCustomAttribute", "C:CustomAttributeType", "B"), + 0x0D: ("C:HasFieldMarshal", "B"), + 0x0E: ("2", "C:HasDeclSecurity", "B"), + 0x0F: ("2", "4", "T:2"), + 0x10: ("4", "T:4"), + 0x11: ("B",), + 0x12: ("T:2", "T:20"), + 0x13: ("T:20",), + 0x14: ("2", "S", "C:TypeDefOrRef"), + 0x15: ("T:2", "T:23"), + 0x16: ("T:23",), + 0x17: ("2", "S", "B"), + 0x18: ("2", "T:6", "C:HasSemantics"), + 0x19: ("T:2", "C:MethodDefOrRef", "C:MethodDefOrRef"), + 0x1A: ("S",), + 0x1B: ("B",), + 0x1C: ("2", "C:MemberForwarded", "S", "T:26"), + 0x1D: ("4", "T:4"), + 0x1E: ("4", "4"), + 0x1F: ("4",), + 0x20: ("4", "2", "2", "2", "2", "4", "B", "S", "S"), + 0x21: ("4",), + 0x22: ("4", "4", "4"), + 0x23: ("2", "2", "2", "2", "4", "B", "S", "S", "B"), + 0x24: ("4", "T:35"), + 0x25: ("4", "4", "4", "T:35"), + 0x26: ("4", "S", "B"), + 0x27: ("4", "4", "S", "S", "C:Implementation"), + 0x28: ("4", "4", "S", "C:Implementation"), + 0x29: ("T:2", "T:2"), + 0x2A: ("2", "2", "C:TypeOrMethodDef", "S"), + 0x2B: ("C:MethodDefOrRef", "B"), + 0x2C: ("T:42", "C:TypeDefOrRef"), +} + +PINVOKE_NO_MANGLE = 0x0001 +PINVOKE_CHARSET_MASK = 0x0006 +PINVOKE_CHARSET_ANSI = 0x0002 +PINVOKE_CHARSET_UNICODE = 0x0004 +PINVOKE_CHARSET_AUTO = 0x0006 + +_PRIMITIVES = { + 0x01: "void", 0x02: "bool", 0x03: "char", 0x04: "sbyte", 0x05: "byte", + 0x06: "short", 0x07: "ushort", 0x08: "int", 0x09: "uint", 0x0A: "long", + 0x0B: "ulong", 0x0C: "float", 0x0D: "double", 0x0E: "string", + 0x16: "typedref", 0x18: "nint", 0x19: "nuint", 0x1C: "object", +} # fmt: skip + + +class DotnetMetadataError(ValueError): + pass + + +class Metadata: + """Row access to the ``#~``/``#-`` tables of one managed image.""" + + def __init__(self, data: Buffer, info: PeInfo) -> None: + self.data = data + self.info = info + tables = info.stream("#~") or info.stream("#-") + if tables is None: + raise DotnetMetadataError("no metadata tables stream") + strings = info.stream("#Strings") + blobs = info.stream("#Blob") + self.strings_at = strings.offset if strings else 0 + self.strings_end = strings.offset + strings.size if strings else 0 + self.blobs_at = blobs.offset if blobs else 0 + self.blobs_end = blobs.offset + blobs.size if blobs else 0 + base = tables.offset + heap_sizes = data[base + 6] + valid = struct.unpack_from("> table & 1: + self.rows[table] = struct.unpack_from(" int: + if column in ("1", "2", "4"): + return int(column) + if column == "S": + return self.string_size + if column == "G": + return self.guid_size + if column == "B": + return self.blob_size + kind, _, arg = column.partition(":") + if kind == "T": + return 2 if self.rows[int(arg)] < 0x10000 else 4 + bits, tables = CODED[arg] + largest = max( + (self.rows[item] for item in tables if item is not None), default=0 + ) + return 2 if largest < 1 << (16 - bits) else 4 + + def row(self, table: int, rid: int) -> list[int]: + """Raw column values of row ``rid`` (1-based).""" + if not 1 <= rid <= self.rows[table]: + raise DotnetMetadataError(f"row {rid} out of range for table {table:#x}") + start, columns = self.layout[table] + cursor = start + (rid - 1) * self.row_sizes[table] + values: list[int] = [] + for width, _column in columns: + if width == 1: + values.append(self.data[cursor]) + elif width == 2: + values.append(struct.unpack_from(" str: + start = self.strings_at + index + if not index or start >= self.strings_end: + return "" + end = self.data.find(b"\0", start, self.strings_end) + return bytes(self.data[start : end if end >= 0 else self.strings_end]).decode( + "utf-8", errors="replace" + ) + + def blob(self, index: int) -> bytes: + start = self.blobs_at + index + if start >= self.blobs_end: + return b"" + size, used = _compressed(self.data, start) + return bytes(self.data[start + used : start + used + size]) + + @staticmethod + def decode(kind: str, value: int) -> tuple[int | None, int]: + bits, tables = CODED[kind] + tag = value & ((1 << bits) - 1) + table = tables[tag] if tag < len(tables) else None + return table, value >> bits + + +def _compressed(data: Buffer | bytes, offset: int) -> tuple[int, int]: + first = data[offset] + if first & 0x80 == 0: + return first, 1 + if first & 0xC0 == 0x80: + return ((first & 0x3F) << 8) | data[offset + 1], 2 + return ( + ((first & 0x1F) << 24) + | (data[offset + 1] << 16) + | (data[offset + 2] << 8) + | data[offset + 3], + 4, + ) + + +class _Names: + """Canonical type names (``Namespace.Outer/Inner``) and signatures.""" + + def __init__(self, md: Metadata) -> None: + self.md = md + self.enclosing: dict[int, int] = {} + for rid in range(1, md.rows[NESTEDCLASS] + 1): + nested, enclosing = md.row(NESTEDCLASS, rid) + self.enclosing[nested] = enclosing + self._typedef: dict[int, str] = {} + self._typeref: dict[int, str] = {} + + def typedef(self, rid: int) -> str: + cached = self._typedef.get(rid) + if cached is not None: + return cached + row = self.md.row(TYPEDEF, rid) + name = self.md.string(row[1]) + outer = self.enclosing.get(rid) + if outer is not None and outer != rid: + text = f"{self.typedef(outer)}/{name}" + else: + namespace = self.md.string(row[2]) + text = f"{namespace}.{name}" if namespace else name + self._typedef[rid] = text + return text + + def typeref(self, rid: int, depth: int = 0) -> str: + cached = self._typeref.get(rid) + if cached is not None: + return cached + scope, name_index, namespace_index = self.md.row(TYPEREF, rid) + name = self.md.string(name_index) + table, scope_rid = Metadata.decode("ResolutionScope", scope) + if table == TYPEREF and scope_rid and depth < 32: + text = f"{self.typeref(scope_rid, depth + 1)}/{name}" + else: + namespace = self.md.string(namespace_index) + text = f"{namespace}.{name}" if namespace else name + self._typeref[rid] = text + return text + + def typeref_assembly(self, rid: int, depth: int = 0) -> int | None: + """AssemblyRef rid a TypeRef resolves through (None: this module).""" + scope = self.md.row(TYPEREF, rid)[0] + table, scope_rid = Metadata.decode("ResolutionScope", scope) + if table == ASSEMBLYREF: + return scope_rid + if table == TYPEREF and scope_rid and depth < 32: + return self.typeref_assembly(scope_rid, depth + 1) + return None + + def coded_type(self, value: int, depth: int) -> str: + table, rid = Metadata.decode("TypeDefOrRef", value) + if table == TYPEDEF: + return self.typedef(rid) + if table == TYPEREF: + return self.typeref(rid) + if table == TYPESPEC and depth < 16: + return self.typespec(rid, depth + 1) + return "?" + + def typespec(self, rid: int, depth: int = 0) -> str: + blob = self.md.blob(self.md.row(TYPESPEC, rid)[0]) + text, _ = self.sig_type(blob, 0, depth) + return text + + def sig_type(self, blob: bytes, pos: int, depth: int = 0) -> tuple[str, int]: + if depth > 48 or pos >= len(blob): + return "?", len(blob) + element = blob[pos] + pos += 1 + if element in _PRIMITIVES: + return _PRIMITIVES[element], pos + if element in (0x11, 0x12): # VALUETYPE, CLASS + value, used = _compressed(blob, pos) + return self.coded_type(value, depth), pos + used + if element == 0x0F: # PTR + inner, pos = self.sig_type(blob, pos, depth + 1) + return f"{inner}*", pos + if element == 0x10: # BYREF + inner, pos = self.sig_type(blob, pos, depth + 1) + return f"{inner}&", pos + if element == 0x1D: # SZARRAY + inner, pos = self.sig_type(blob, pos, depth + 1) + return f"{inner}[]", pos + if element == 0x14: # ARRAY + inner, pos = self.sig_type(blob, pos, depth + 1) + rank, used = _compressed(blob, pos) + pos += used + for _ in range(2): # sizes, then lower bounds + count, used = _compressed(blob, pos) + pos += used + for _ in range(count): + _value, used = _compressed(blob, pos) + pos += used + return f"{inner}[{',' * max(rank - 1, 0)}]", pos + if element == 0x15: # GENERICINST + pos += 1 # CLASS or VALUETYPE + value, used = _compressed(blob, pos) + pos += used + generic = self.coded_type(value, depth) + count, used = _compressed(blob, pos) + pos += used + args = [] + for _ in range(count): + arg, pos = self.sig_type(blob, pos, depth + 1) + args.append(arg) + return f"{generic}<{','.join(args)}>", pos + if element in (0x13, 0x1E): # VAR, MVAR + number, used = _compressed(blob, pos) + return f"{'!' if element == 0x13 else '!!'}{number}", pos + used + if element in (0x1F, 0x20): # CMOD_REQD, CMOD_OPT + value, used = _compressed(blob, pos) + modifier = self.coded_type(value, depth) + inner, pos = self.sig_type(blob, pos + used, depth + 1) + keyword = "modreq" if element == 0x1F else "modopt" + return f"{inner} {keyword}({modifier})", pos + if element == 0x1B: # FNPTR + text, pos = self.method_sig(blob, pos, depth + 1) + return f"method {text}", pos + if element in (0x41, 0x45): # SENTINEL, PINNED + return self.sig_type(blob, pos, depth + 1) + return "?", len(blob) + + def method_sig(self, blob: bytes, pos: int = 0, depth: int = 0) -> tuple[str, int]: + if pos >= len(blob): + return "()", pos + convention = blob[pos] + pos += 1 + generic = 0 + if convention & 0x10: + generic, used = _compressed(blob, pos) + pos += used + count, used = _compressed(blob, pos) + pos += used + returns, pos = self.sig_type(blob, pos, depth) + params = [] + for _ in range(min(count, 512)): + if pos < len(blob) and blob[pos] == 0x41: # vararg sentinel + pos += 1 + params.append("...") + param, pos = self.sig_type(blob, pos, depth) + params.append(param) + prefix = "instance " if convention & 0x20 else "" + arity = f"<{generic}>" if generic else "" + return f"{prefix}{returns} {arity}({','.join(params)})", pos + + +def parse_dotnet_links(data: Buffer, path: str = "") -> FileFacts: + info = parse_pe(data) + if info is None or not info.is_managed: + raise DotnetMetadataError("not a managed PE image") + md = Metadata(data, info) + names = _Names(md) + facts = FileFacts( + path=path, + format="dotnet", + # IL (and ReadyToRun, whose machine field is OS-salted) binds by name + # alone; only C++/CLI images carry real native code for one machine. + arch=info.machine if info.mixed_mode else "msil", + bits=64 if info.pe32_plus else 32, + kind="library" if info.characteristics & 0x2000 else "executable", + image_base=info.image_base, + ) + managed: dict[str, Any] = {"forwarders": {}, "mixed_mode": info.mixed_mode} + if md.rows[ASSEMBLY]: + row = md.row(ASSEMBLY, 1) + facts.identity = md.string(row[7]) + managed["version"] = ".".join(str(item) for item in row[1:5]) + if md.rows[MODULE]: + managed["module"] = md.string(md.row(MODULE, 1)[1]) + # Assembly references, in table order (index = rid - 1). + for rid in range(1, md.rows[ASSEMBLYREF] + 1): + row = md.row(ASSEMBLYREF, rid) + facts.libraries.append( + LibraryRef( + md.string(row[6]), + kind=LIB_ASSEMBLY, + version=".".join(str(item) for item in row[0:4]), + ) + ) + # Type forwarders: [TypeForwardedTo] compiles to ExportedType rows whose + # implementation is an AssemblyRef. + for rid in range(1, md.rows[EXPORTEDTYPE] + 1): + _flags, _typedef, name_index, namespace_index, implementation = md.row( + EXPORTEDTYPE, rid + ) + table, target = Metadata.decode("Implementation", implementation) + if table == ASSEMBLYREF and 1 <= target <= len(facts.libraries): + namespace = md.string(namespace_index) + name = md.string(name_index) + full = f"{namespace}.{name}" if namespace else name + managed["forwarders"][full] = facts.libraries[target - 1].name + # Methods defined here (binding targets for other assemblies). + method_owner: dict[int, int] = {} + typedef_count = md.rows[TYPEDEF] + method_count = md.rows[METHODDEF] + for rid in range(1, typedef_count + 1): + first = md.row(TYPEDEF, rid)[5] + last = md.row(TYPEDEF, rid + 1)[5] if rid < typedef_count else method_count + 1 + for method in range(first, min(last, method_count + 1)): + method_owner[method] = rid + for method, type_rid in sorted(method_owner.items()): + row = md.row(METHODDEF, method) + signature, _ = names.method_sig(md.blob(row[4])) + facts.exports.append( + ExportSym( + name=f"{names.typedef(type_rid)}::{md.string(row[3])}", + kind="method", + signature=signature, + token=0x06000000 | method, + ) + ) + # Member references into other assemblies. + ref_index: dict[int, int] = {} # MemberRef rid -> imports index + for rid in range(1, md.rows[MEMBERREF] + 1): + parent, name_index, sig_index = md.row(MEMBERREF, rid) + table, parent_rid = Metadata.decode("MemberRefParent", parent) + typeref_rid = _owner_typeref(md, table, parent_rid) + if typeref_rid is None: + continue + assembly = names.typeref_assembly(typeref_rid) + if assembly is None or not 1 <= assembly <= len(facts.libraries): + continue + owner = names.typeref(typeref_rid) + blob = md.blob(sig_index) + is_field = bool(blob) and blob[0] & 0x0F == 0x06 + if is_field: + field_type, _ = names.sig_type(blob, 1) + signature = field_type + else: + signature, _ = names.method_sig(blob) + ref_index[rid] = len(facts.imports) + facts.imports.append( + ImportSym( + name=f"{owner}::{md.string(name_index)}", + library=assembly - 1, + kind="data" if is_field else "method", + owner=owner, + signature=signature, + callers=[], + ) + ) + # P/Invoke declarations. + module_library: dict[int, int] = {} + pinvokes: list[tuple[int, int]] = [] # (imports index, method rid) + for rid in range(1, md.rows[IMPLMAP] + 1): + flags, forwarded, import_name, module = md.row(IMPLMAP, rid) + table, method = Metadata.decode("MemberForwarded", forwarded) + if table != METHODDEF or not 1 <= module <= md.rows[MODULEREF]: + continue + if module not in module_library: + module_library[module] = len(facts.libraries) + facts.libraries.append( + LibraryRef(md.string(md.row(MODULEREF, module)[0]), kind=LIB_PINVOKE) + ) + owner_rid = method_owner.get(method) + method_name = md.string(md.row(METHODDEF, method)[3]) + owner = ( + f"{names.typedef(owner_rid)}::{method_name}" if owner_rid else method_name + ) + charset = flags & PINVOKE_CHARSET_MASK + facts.imports.append( + ImportSym( + name=md.string(import_name) or method_name, + library=module_library[module], + kind="pinvoke", + owner=owner, + signature=_charset_name(flags, charset), + callers=[0x06000000 | method], + ) + ) + pinvokes.append((len(facts.imports) - 1, method)) + if ref_index: + _scan_callers(data, info, md, ref_index, facts) + managed["entry_point_token"] = info.entry_point_token or None + facts.managed = managed + return facts + + +def _charset_name(flags: int, charset: int) -> str: + if flags & PINVOKE_NO_MANGLE: + return "exact" + return { + PINVOKE_CHARSET_ANSI: "ansi", + PINVOKE_CHARSET_UNICODE: "unicode", + PINVOKE_CHARSET_AUTO: "auto", + }.get(charset, "ansi") # CharSet.None behaves as Ansi + + +def _owner_typeref(md: Metadata, table: int | None, rid: int) -> int | None: + if table == TYPEREF: + return rid + if table == TYPESPEC and rid: + # Generic instantiation: the generic type definition is a TypeRef. + blob = md.blob(md.row(TYPESPEC, rid)[0]) + if len(blob) > 2 and blob[0] == 0x15: + value, _ = _compressed(blob, 2) + kind, target = Metadata.decode("TypeDefOrRef", value) + if kind == TYPEREF: + return target + return None + + +def _scan_callers( + data: Buffer, + info: PeInfo, + md: Metadata, + ref_index: dict[int, int], + facts: FileFacts, +) -> None: + """Fill ``callers`` from an IL scan of every method body.""" + method_specs: dict[int, int] = {} + for rid in range(1, md.rows[METHODSPEC] + 1): + method, _blob = md.row(METHODSPEC, rid) + table, target = Metadata.decode("MethodDefOrRef", method) + if table == MEMBERREF: + method_specs[rid] = target + for method in range(1, md.rows[METHODDEF] + 1): + rva = md.row(METHODDEF, method)[0] + if not rva: + continue + offset = info.rva_to_offset(rva) + if offset is None: + continue + code = method_body(data, offset) + if not code: + continue + token_owner = 0x06000000 | method + seen: set[int] = set() + for kind, token in scan_il_tokens(code): + if kind not in (TOK_METHOD, "field", "token"): + continue + table = token >> 24 + rid = token & 0xFFFFFF + if table == METHODSPEC: + rid = method_specs.get(rid, 0) + table = MEMBERREF + if table != MEMBERREF or rid in seen: + continue + index = ref_index.get(rid) + if index is None: + continue + seen.add(rid) + callers = facts.imports[index].callers + if callers is not None: + callers.append(token_owner) diff --git a/src/tocode/links/binfmt_elf.py b/src/tocode/links/binfmt_elf.py new file mode 100644 index 0000000..5a6f855 --- /dev/null +++ b/src/tocode/links/binfmt_elf.py @@ -0,0 +1,390 @@ +"""ELF dynamic linking facts: DT_NEEDED, SONAME, RPATH/RUNPATH, dynamic symbols. + +Driven by the program headers (PT_DYNAMIC and PT_LOAD), the same view the +runtime loader has, so it works on stripped files without section headers. +""" + +from __future__ import annotations + +import mmap +import struct + +from .model import ExportSym, FileFacts, ImportSym, LibraryRef + +PT_LOAD = 1 +PT_DYNAMIC = 2 +ET_EXEC = 2 +ET_DYN = 3 +DT_NULL = 0 +DT_NEEDED = 1 +DT_HASH = 4 +DT_STRTAB = 5 +DT_SYMTAB = 6 +DT_STRSZ = 10 +DT_SYMENT = 11 +DT_SONAME = 14 +DT_RPATH = 15 +DT_RUNPATH = 29 +DT_FLAGS_1 = 0x6FFFFFFB +DT_GNU_HASH = 0x6FFFFEF5 +DT_VERSYM = 0x6FFFFFF0 +DT_VERDEF = 0x6FFFFFFC +DT_VERDEFNUM = 0x6FFFFFFD +DT_VERNEED = 0x6FFFFFFE +DT_VERNEEDNUM = 0x6FFFFFFF +DF_1_PIE = 0x08000000 +SHT_DYNSYM = 11 +SHN_UNDEF = 0 +STB_GLOBAL = 1 +STB_WEAK = 2 +STB_GNU_UNIQUE = 10 +STT_OBJECT = 1 +STT_FUNC = 2 +STT_TLS = 6 +STT_GNU_IFUNC = 10 +STV_HIDDEN = 2 +STV_INTERNAL = 1 +VER_FLG_BASE = 1 +VER_FLG_WEAK = 2 +EM_ARM = 40 +MAX_SYMBOLS = 2_000_000 + +ELF_MACHINES = { + 0x03: "x86", + 0x08: "mips", + 0x14: "ppc", + 0x15: "ppc64", + 0x28: "arm", + 0x3E: "x86_64", + 0xB7: "arm64", + 0xF3: "riscv", + 0x102: "loongarch", +} + +Buffer = bytes | mmap.mmap + + +class ElfError(ValueError): + pass + + +class _Reader: + def __init__(self, data: Buffer) -> None: + self.data = data + if bytes(data[:4]) != b"\x7fELF": + raise ElfError("not an ELF file") + self.is64 = data[4] == 2 + self.endian = "<" if data[5] == 1 else ">" + if data[4] not in (1, 2) or data[5] not in (1, 2): + raise ElfError("bad ELF class or data encoding") + self.word = "Q" if self.is64 else "I" + self.loads: list[tuple[int, int, int, int]] = [] # vaddr, memsz, offset, filesz + + def unpack(self, fmt: str, offset: int) -> tuple: + if offset < 0: + raise ElfError("negative offset") + return struct.unpack_from(self.endian + fmt, self.data, offset) + + def va_to_offset(self, va: int) -> int | None: + for vaddr, _memsz, offset, filesz in self.loads: + if vaddr <= va < vaddr + filesz: + return offset + (va - vaddr) + return None + + def cstring(self, offset: int, limit: int = 4096) -> str: + if offset < 0 or offset >= len(self.data): + return "" + end = self.data.find(b"\0", offset, min(len(self.data), offset + limit)) + if end < 0: + end = min(len(self.data), offset + limit) + return bytes(self.data[offset:end]).decode("utf-8", errors="replace") + + +def parse_elf(data: Buffer, path: str = "") -> FileFacts: + r = _Reader(data) + if r.is64: + (e_type, e_machine, _ver, e_entry, e_phoff, e_shoff, _flags, _ehsize, + e_phentsize, e_phnum, e_shentsize, e_shnum, _shstrndx) = r.unpack( + "HHIQQQIHHHHHH", 16 + ) # fmt: skip + else: + (e_type, e_machine, _ver, e_entry, e_phoff, e_shoff, _flags, _ehsize, + e_phentsize, e_phnum, e_shentsize, e_shnum, _shstrndx) = r.unpack( + "HHIIIIIHHHHHH", 16 + ) # fmt: skip + facts = FileFacts( + path=path, + format="elf", + arch=ELF_MACHINES.get(e_machine, f"elf-{e_machine:#x}"), + bits=64 if r.is64 else 32, + kind="other", + entry=e_entry or None, + ) + dynamic: tuple[int, int] | None = None + for index in range(min(e_phnum, 4096)): + base = e_phoff + index * e_phentsize + if r.is64: + p_type, _pf, p_offset, p_vaddr, _pa, p_filesz, p_memsz, _al = r.unpack( + "IIQQQQQQ", base + ) + else: + p_type, p_offset, p_vaddr, _pa, p_filesz, p_memsz, _pf, _al = r.unpack( + "IIIIIIII", base + ) + if p_type == PT_LOAD: + r.loads.append((p_vaddr, p_memsz, p_offset, p_filesz)) + elif p_type == PT_DYNAMIC: + dynamic = (p_offset, p_filesz) + if r.loads: + facts.image_base = min(item[0] for item in r.loads) & ~0xFFF + tags = _dynamic_tags(r, dynamic) if dynamic else [] + flags_1 = next((value for tag, value in tags if tag == DT_FLAGS_1), 0) + has_soname = any(tag == DT_SONAME for tag, _value in tags) + if e_type == ET_EXEC or (e_type == ET_DYN and flags_1 & DF_1_PIE): + facts.kind = "executable" + elif e_type == ET_DYN: + # Older toolchains do not set DF_1_PIE: a PIE has an interpreter and no + # SONAME. libc.so.6 has both (it is runnable) and is a library. + interp = _has_interp(r, e_phoff, e_phnum, e_phentsize) + facts.kind = "executable" if interp and not has_soname else "library" + if not tags: + return facts + strtab_va = _tag(tags, DT_STRTAB) + strtab = r.va_to_offset(strtab_va) if strtab_va is not None else None + if strtab is None: + facts.notes.append("no dynamic string table") + return facts + + def dynstr(offset: int) -> str: + return r.cstring(strtab + offset) + + for tag, value in tags: + if tag == DT_NEEDED: + facts.libraries.append(LibraryRef(dynstr(value))) + elif tag == DT_SONAME: + facts.identity = dynstr(value) + elif tag == DT_RPATH: + facts.rpaths.extend(_split_paths(dynstr(value))) + elif tag == DT_RUNPATH: + facts.runpaths.extend(_split_paths(dynstr(value))) + needed_index = {lib.name: index for index, lib in enumerate(facts.libraries)} + # Version tables: index -> (name, file or None, weak) + versions: dict[int, tuple[str, str | None, bool]] = {} + verneed_va = _tag(tags, DT_VERNEED) + if verneed_va is not None: + _read_verneed(r, verneed_va, _tag(tags, DT_VERNEEDNUM) or 0, dynstr, versions) + verdef_va = _tag(tags, DT_VERDEF) + if verdef_va is not None: + _read_verdef(r, verdef_va, _tag(tags, DT_VERDEFNUM) or 0, dynstr, versions) + + symtab_va = _tag(tags, DT_SYMTAB) + symtab = r.va_to_offset(symtab_va) if symtab_va is not None else None + if symtab is None: + return facts + count = _symbol_count(r, tags, e_shoff, e_shnum, e_shentsize, symtab) + versym_va = _tag(tags, DT_VERSYM) + versym = r.va_to_offset(versym_va) if versym_va is not None else None + entsize = 24 if r.is64 else 16 + thumb = e_machine == EM_ARM + for index in range(1, count): + base = symtab + index * entsize + if base + entsize > len(data): + break + if r.is64: + st_name, st_info, st_other, st_shndx, st_value, _size = r.unpack( + "IBBHQQ", base + ) + else: + st_name, st_value, _size, st_info, st_other, st_shndx = r.unpack( + "IIIBBH", base + ) + bind = st_info >> 4 + if bind not in (STB_GLOBAL, STB_WEAK, STB_GNU_UNIQUE) or not st_name: + continue + name = dynstr(st_name) + if not name: + continue + sym_type = st_info & 0xF + kind = _symbol_kind(sym_type) + version: str | None = None + version_file: str | None = None + hidden = False + if versym is not None: + raw = r.unpack("H", versym + index * 2)[0] + hidden = bool(raw & 0x8000) + entry = versions.get(raw & 0x7FFF) + if entry is not None: + version, version_file, _weak = entry + if st_shndx == SHN_UNDEF: + library = needed_index.get(version_file) if version_file else None + facts.imports.append( + ImportSym( + name=name, + library=library, + version=version, + weak=bind == STB_WEAK, + kind=kind, + ) + ) + continue + visibility = st_other & 3 + if visibility in (STV_HIDDEN, STV_INTERNAL): + continue + address = st_value + if thumb and sym_type == STT_FUNC: + address &= ~1 + facts.exports.append( + ExportSym( + name=name, + address=address, + version=version, + default_version=not hidden, + weak=bind == STB_WEAK, + kind=kind, + ) + ) + return facts + + +def _symbol_kind(sym_type: int) -> str | None: + if sym_type in (STT_FUNC, STT_GNU_IFUNC): + return "function" + if sym_type == STT_OBJECT: + return "data" + if sym_type == STT_TLS: + return "tls" + return None + + +def _has_interp(r: _Reader, phoff: int, phnum: int, phentsize: int) -> bool: + for index in range(min(phnum, 4096)): + if r.unpack("I", phoff + index * phentsize)[0] == 3: # PT_INTERP + return True + return False + + +def _tag(tags: list[tuple[int, int]], wanted: int) -> int | None: + return next((value for tag, value in tags if tag == wanted), None) + + +def _split_paths(value: str) -> list[str]: + return [item for item in value.split(":") if item] + + +def _dynamic_tags(r: _Reader, dynamic: tuple[int, int]) -> list[tuple[int, int]]: + offset, size = dynamic + entsize = 16 if r.is64 else 8 + fmt = "qQ" if r.is64 else "iI" + tags: list[tuple[int, int]] = [] + for index in range(size // entsize): + tag, value = r.unpack(fmt, offset + index * entsize) + if tag == DT_NULL: + break + tags.append((tag & 0xFFFFFFFF if tag < 0 else tag, value)) + return tags + + +def _read_verneed(r, va, count, dynstr, versions) -> None: + offset = r.va_to_offset(va) + seen = 0 + while offset is not None and seen < max(count, 1) and seen < 4096: + _vn_version, vn_cnt, vn_file, vn_aux, vn_next = r.unpack("HHIII", offset) + file_name = dynstr(vn_file) + aux = offset + vn_aux + for _ in range(min(vn_cnt, 4096)): + _hash, flags, other, name, next_aux = r.unpack("IHHII", aux) + versions[other] = (dynstr(name), file_name, bool(flags & VER_FLG_WEAK)) + if not next_aux: + break + aux += next_aux + seen += 1 + if not vn_next: + break + offset += vn_next + + +def _read_verdef(r, va, count, dynstr, versions) -> None: + offset = r.va_to_offset(va) + seen = 0 + while offset is not None and seen < max(count, 1) and seen < 4096: + _version, flags, ndx, _cnt, _hash, aux, next_def = r.unpack("HHHHIII", offset) + if not flags & VER_FLG_BASE: + name_offset = r.unpack("I", offset + aux)[0] + versions[ndx] = (dynstr(name_offset), None, False) + seen += 1 + if not next_def: + break + offset += next_def + + +def _symbol_count(r: _Reader, tags, shoff, shnum, shentsize, symtab) -> int: + # Section headers, when present, give the exact size of .dynsym. + if shoff and shnum and shentsize: + for index in range(min(shnum, 65535)): + base = shoff + index * shentsize + try: + if r.is64: + ( + _name, + sh_type, + _f, + _a, + sh_offset, + sh_size, + _l, + _i, + _al, + sh_entsize, + ) = r.unpack("IIQQQQIIQQ", base) + else: + ( + _name, + sh_type, + _f, + _a, + sh_offset, + sh_size, + _l, + _i, + _al, + sh_entsize, + ) = r.unpack("IIIIIIIIII", base) + except struct.error: + break + if sh_type == SHT_DYNSYM and sh_entsize: + return min(sh_size // sh_entsize, MAX_SYMBOLS) + hash_va = _tag(tags, DT_HASH) + if hash_va is not None: + offset = r.va_to_offset(hash_va) + if offset is not None: + return min(r.unpack("II", offset)[1], MAX_SYMBOLS) + gnu_va = _tag(tags, DT_GNU_HASH) + if gnu_va is not None: + offset = r.va_to_offset(gnu_va) + if offset is not None: + return min(_gnu_hash_count(r, offset), MAX_SYMBOLS) + # Last resort: the string table usually follows the symbol table. + strtab = r.va_to_offset(_tag(tags, DT_STRTAB) or 0) + entsize = 24 if r.is64 else 16 + if strtab is not None and strtab > symtab: + return min((strtab - symtab) // entsize, MAX_SYMBOLS) + return 0 + + +def _gnu_hash_count(r: _Reader, offset: int) -> int: + nbuckets, symoffset, bloom_size, _shift = r.unpack("IIII", offset) + word = 8 if r.is64 else 4 + buckets = offset + 16 + bloom_size * word + chains = buckets + nbuckets * 4 + highest = 0 + for index in range(min(nbuckets, 1 << 24)): + highest = max(highest, r.unpack("I", buckets + index * 4)[0]) + if highest < symoffset: + return symoffset + index = highest + while index - symoffset < MAX_SYMBOLS: + value = r.unpack("I", chains + (index - symoffset) * 4)[0] + if value & 1: + break + index += 1 + return index + 1 diff --git a/src/tocode/links/binfmt_macho.py b/src/tocode/links/binfmt_macho.py new file mode 100644 index 0000000..8cd2361 --- /dev/null +++ b/src/tocode/links/binfmt_macho.py @@ -0,0 +1,585 @@ +"""Mach-O linking facts: load commands, two-level imports, the export trie. + +Imports come from whichever table dyld itself reads: chained fixups +(``LC_DYLD_CHAINED_FIXUPS``), the bind opcode streams of +``LC_DYLD_INFO(_ONLY)``, or, for old/simple images, the undefined ``nlist`` +symbols. Each import keeps its two-level library ordinal, so the library it +binds to is a fact recorded by the static linker, not a guess. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import mmap +import struct +import zlib + +from .model import ( + LIB_LAZY, + LIB_NEEDED, + LIB_REEXPORT, + LIB_UPWARD, + LIB_WEAK, + SPECIAL_FLAT, + SPECIAL_MAIN, + SPECIAL_SELF, + SPECIAL_WEAK, + ExportSym, + FileFacts, + ImportSym, + LibraryRef, +) +from .sniff import FAT_MAGIC, FAT_MAGIC_64, KNOWN_CPU_TYPES, MAX_FAT_ARCHS + +MH_MAGIC = 0xFEEDFACE +MH_MAGIC_64 = 0xFEEDFACF +MH_EXECUTE = 0x2 +MH_DYLIB = 0x6 +MH_BUNDLE = 0x8 +MH_DYLINKER = 0x7 +MH_DSYM = 0xA +MH_TWOLEVEL = 0x80 +LC_REQ_DYLD = 0x80000000 +LC_SEGMENT = 0x1 +LC_SYMTAB = 0x2 +LC_LOAD_DYLIB = 0xC +LC_ID_DYLIB = 0xD +LC_SEGMENT_64 = 0x19 +LC_UUID = 0x1B +LC_LOAD_WEAK_DYLIB = 0x18 | LC_REQ_DYLD +LC_RPATH = 0x1C | LC_REQ_DYLD +LC_REEXPORT_DYLIB = 0x1F | LC_REQ_DYLD +LC_LAZY_LOAD_DYLIB = 0x20 +LC_DYLD_INFO = 0x22 +LC_DYLD_INFO_ONLY = 0x22 | LC_REQ_DYLD +LC_LOAD_UPWARD_DYLIB = 0x23 | LC_REQ_DYLD +LC_MAIN = 0x28 | LC_REQ_DYLD +LC_DYLD_EXPORTS_TRIE = 0x33 | LC_REQ_DYLD +LC_DYLD_CHAINED_FIXUPS = 0x34 | LC_REQ_DYLD +DYLIB_COMMANDS = { + LC_LOAD_DYLIB: LIB_NEEDED, + LC_LOAD_WEAK_DYLIB: LIB_WEAK, + LC_REEXPORT_DYLIB: LIB_REEXPORT, + LC_LAZY_LOAD_DYLIB: LIB_LAZY, + LC_LOAD_UPWARD_DYLIB: LIB_UPWARD, +} +FILE_KINDS = { + MH_EXECUTE: "executable", + MH_DYLIB: "library", + MH_BUNDLE: "bundle", + MH_DYLINKER: "library", + MH_DSYM: "debug", + 0x1: "object", +} +EXPORT_SYMBOL_FLAGS_KIND_MASK = 0x03 +EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL = 0x01 +EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE = 0x02 +EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION = 0x04 +EXPORT_SYMBOL_FLAGS_REEXPORT = 0x08 +EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER = 0x10 +BIND_SYMBOL_FLAGS_WEAK_IMPORT = 0x1 +N_STAB = 0xE0 +N_TYPE = 0x0E +N_EXT = 0x01 +N_UNDF = 0x0 +N_SECT = 0xE +N_WEAK_REF = 0x40 +MAX_TRIE_NODES = 4_000_000 +MAX_SYMBOLS = 4_000_000 + +Buffer = bytes | mmap.mmap + + +class MachoError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class FatSlice: + arch: str + cputype: int + cpusubtype: int + offset: int + size: int + + +def cpu_name(cputype: int, cpusubtype: int = 0) -> str: + name = KNOWN_CPU_TYPES.get(cputype, f"cpu-{cputype:#x}") + if name == "arm64" and cpusubtype & 0xFF == 2: + return "arm64e" + return name + + +def fat_slices(data: Buffer) -> list[FatSlice]: + magic, count = struct.unpack_from(">II", data, 0) + if magic not in (FAT_MAGIC, FAT_MAGIC_64) or not 0 < count <= MAX_FAT_ARCHS: + raise MachoError("not a fat Mach-O file") + slices: list[FatSlice] = [] + for index in range(count): + if magic == FAT_MAGIC_64: + cputype, subtype, offset, size, _align, _res = struct.unpack_from( + ">iiQQII", data, 8 + index * 32 + ) + else: + cputype, subtype, offset, size, _align = struct.unpack_from( + ">iiIII", data, 8 + index * 20 + ) + cputype &= 0xFFFFFFFF + if offset + size > len(data): + raise MachoError("fat slice extends past the end of the file") + slices.append( + FatSlice(cpu_name(cputype, subtype), cputype, subtype, offset, size) + ) + return slices + + +def _uleb(data: Buffer, offset: int, end: int) -> tuple[int, int]: + result = 0 + shift = 0 + while offset < end: + byte = data[offset] + offset += 1 + result |= (byte & 0x7F) << shift + if byte < 0x80: + return result, offset + shift += 7 + if shift > 63: + raise MachoError("ULEB128 too long") + raise MachoError("truncated ULEB128") + + +def _sleb(data: Buffer, offset: int, end: int) -> tuple[int, int]: + result = 0 + shift = 0 + byte = 0x80 + while offset < end and byte & 0x80: + byte = data[offset] + offset += 1 + result |= (byte & 0x7F) << shift + shift += 7 + if byte & 0x80: + raise MachoError("truncated SLEB128") + if byte & 0x40: + result -= 1 << shift + return result, offset + + +def _cstring(data: Buffer, offset: int, end: int) -> tuple[str, int]: + stop = data.find(b"\0", offset, end) + if stop < 0: + raise MachoError("unterminated string") + return bytes(data[offset:stop]).decode("utf-8", errors="replace"), stop + 1 + + +def _special(ordinal: int) -> str | None: + return {0: SPECIAL_SELF, -1: SPECIAL_MAIN, -2: SPECIAL_FLAT, -3: SPECIAL_WEAK}.get( + ordinal + ) + + +def parse_macho( + data: Buffer, path: str = "", *, base: int = 0, size: int | None = None +) -> FileFacts: + """Parse one thin Mach-O image starting at ``base`` (a fat slice offset).""" + end_of_image = base + size if size is not None else len(data) + magic = struct.unpack_from(" limit: + break + cmd, cmdsize = struct.unpack_from(" limit: + facts.notes.append(f"bad load command size at {cursor - base:#x}") + break + if cmd in DYLIB_COMMANDS or cmd == LC_ID_DYLIB: + name_offset, _stamp, current, compat = struct.unpack_from( + " str: + return f"{packed >> 16}.{(packed >> 8) & 0xFF}.{packed & 0xFF}" + + +class _Image: + def __init__(self, data, base, end, facts, segments, is64) -> None: + self.data = data + self.base = base + self.end = end + self.facts = facts + self.segments = segments + self.pointer = 8 if is64 else 4 + self._bound_names: set[str] = set() + + def _span(self, offset: int, size: int) -> tuple[int, int]: + start = self.base + offset + stop = start + size + if offset < 0 or stop > self.end: + raise MachoError("link-edit data outside the image") + return start, stop + + def _library(self, ordinal: int) -> tuple[int | None, str | None]: + special = _special(ordinal) if ordinal <= 0 else None + if special is not None: + return None, special + if self.facts.flat_namespace: + return None, SPECIAL_FLAT + index = ordinal - 1 + if 0 <= index < len(self.facts.libraries): + return index, None + return None, None + + # -- chained fixups --------------------------------------------------- + def chained_imports(self, offset: int, size: int) -> None: + start, stop = self._span(offset, size) + (_version, _starts, imports_offset, symbols_offset, count, imports_format, + symbols_format) = struct.unpack_from("<7I", self.data, start) # fmt: skip + symbols_start = start + symbols_offset + if symbols_format == 1: + pool: Buffer = zlib.decompress(bytes(self.data[symbols_start:stop])) + pool_base, pool_end = 0, len(pool) + else: + pool, pool_base, pool_end = self.data, symbols_start, stop + cursor = start + imports_offset + seen: set[tuple[str, int | None, str | None]] = set() + entry_size = {1: 4, 2: 8, 3: 16}.get(imports_format) + if entry_size is None: + raise MachoError(f"unknown chained imports format {imports_format}") + for _ in range(min(count, MAX_SYMBOLS)): + if imports_format == 3: + raw = struct.unpack_from("> 16 & 1) + name_offset = raw >> 32 + ordinal = lib - 0x10000 if lib > 0xFFF0 else lib + else: + raw = struct.unpack_from("> 8 & 1) + name_offset = raw >> 9 + ordinal = lib - 0x100 if lib > 0xF0 else lib + cursor += entry_size + name, _ = _cstring(pool, pool_base + name_offset, pool_end) + library, special = self._library(ordinal) + # The same symbol appears once per distinct addend; one fact is enough. + key = (name, library, special) + if key in seen: + continue + seen.add(key) + self.facts.imports.append( + ImportSym(name=name, library=library, special=special, weak=weak) + ) + + # -- bind opcodes ----------------------------------------------------- + def bind_opcodes(self, info: tuple[int, ...]) -> None: + seen: set[tuple[str, int | None, str | None]] = set() + (_rebase_off, _rebase_size, bind_off, bind_size, weak_off, weak_size, + lazy_off, lazy_size, _export_off, _export_size) = info # fmt: skip + for offset, size, stream in ( + (bind_off, bind_size, "bind"), + (lazy_off, lazy_size, "lazy"), + (weak_off, weak_size, "weak"), + ): + if offset and size: + self._run_bind(offset, size, stream, seen) + + def _run_bind(self, offset: int, size: int, stream: str, seen: set) -> None: + cursor, stop = self._span(offset, size) + bound_names = self._bound_names + data = self.data + ordinal = 0 + name = "" + flags = 0 + segment = 0 + seg_offset = 0 + pointer = self.pointer + + def record() -> None: + if not name: + return + special: str | None + library: int | None + if stream == "weak": + # 0x8 marks this image's own strong definition, not an import; + # a symbol the two-level binds already cover keeps that library. + if flags & 0x8 or name in bound_names: + return + library, special = None, SPECIAL_WEAK + else: + library, special = self._library(ordinal) + slot = None + if segment < len(self.segments): + slot = self.segments[segment][1] + seg_offset + key = (name, library, special) + if key in seen: + return + seen.add(key) + bound_names.add(name) + self.facts.imports.append( + ImportSym( + name=name, + library=library, + special=special, + weak=bool(flags & BIND_SYMBOL_FLAGS_WEAK_IMPORT), + slot=slot, + ) + ) + + while cursor < stop: + byte = data[cursor] + cursor += 1 + opcode = byte & 0xF0 + imm = byte & 0x0F + if opcode == 0x00: # DONE (separates lazy entries) + if stream != "lazy": + break + elif opcode == 0x10: + ordinal = imm + elif opcode == 0x20: + ordinal, cursor = _uleb(data, cursor, stop) + elif opcode == 0x30: + ordinal = 0 if imm == 0 else (imm | 0xF0) - 0x100 + elif opcode == 0x40: + flags = imm + name, cursor = _cstring(data, cursor, stop) + elif opcode == 0x50: + pass # type + elif opcode == 0x60: + _addend, cursor = _sleb(data, cursor, stop) + elif opcode == 0x70: + segment = imm + seg_offset, cursor = _uleb(data, cursor, stop) + elif opcode == 0x80: + delta, cursor = _uleb(data, cursor, stop) + seg_offset += delta + elif opcode == 0x90: + record() + seg_offset += pointer + elif opcode == 0xA0: + record() + delta, cursor = _uleb(data, cursor, stop) + seg_offset += delta + pointer + elif opcode == 0xB0: + record() + seg_offset += imm * pointer + pointer + elif opcode == 0xC0: + count, cursor = _uleb(data, cursor, stop) + skip, cursor = _uleb(data, cursor, stop) + record() + seg_offset += count * (skip + pointer) + elif opcode == 0xD0: # THREADED + if imm == 0: + _size, cursor = _uleb(data, cursor, stop) + else: + raise MachoError(f"unknown bind opcode {byte:#x}") + + # -- exports ---------------------------------------------------------- + def export_trie(self, offset: int, size: int) -> None: + if not size: + return + start, stop = self._span(offset, size) + data = self.data + image_base = self.facts.image_base + stack: list[tuple[int, str]] = [(start, "")] + visited: set[int] = set() + while stack: + node, prefix = stack.pop() + if node in visited or not start <= node < stop: + continue + visited.add(node) + if len(visited) > MAX_TRIE_NODES: + raise MachoError("export trie too large") + terminal, cursor = _uleb(data, node, stop) + children_at = cursor + terminal + if terminal: + flags, cursor = _uleb(data, cursor, stop) + kind_bits = flags & EXPORT_SYMBOL_FLAGS_KIND_MASK + weak = bool(flags & EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION) + if flags & EXPORT_SYMBOL_FLAGS_REEXPORT: + ordinal, cursor = _uleb(data, cursor, stop) + imported, cursor = _cstring(data, cursor, stop) + index = ordinal - 1 + self.facts.exports.append( + ExportSym( + name=prefix, + reexport_library=index + if 0 <= index < len(self.facts.libraries) + else None, + reexport_name=imported or prefix, + weak=weak, + ) + ) + else: + address, cursor = _uleb(data, cursor, stop) + kind = "function" + if flags & EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER: + _resolver, cursor = _uleb(data, cursor, stop) + kind = "resolver" + if kind_bits == EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL: + kind = "tls" + elif kind_bits == EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE: + kind = "absolute" + self.facts.exports.append( + ExportSym( + name=prefix, + address=address + if kind_bits == EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE + else image_base + address, + weak=weak, + kind=None if kind == "function" else kind, + ) + ) + if children_at >= stop: + continue + count = data[children_at] + cursor = children_at + 1 + for _ in range(count): + edge, cursor = _cstring(data, cursor, stop) + child, cursor = _uleb(data, cursor, stop) + stack.append((start + child, prefix + edge)) + + def nlist( + self, + symoff: int, + nsyms: int, + stroff: int, + strsize: int, + *, + imports: bool, + exports: bool, + ) -> None: + if not (imports or exports) or not nsyms: + return + entsize = 16 if self.pointer == 8 else 12 + sym_start, _ = self._span(symoff, min(nsyms, MAX_SYMBOLS) * entsize) + str_start, str_stop = self._span(stroff, strsize) + fmt = "> 8 & 0xFF + ordinal = {0xFE: -2, 0xFF: -1}.get(raw, raw) + library, special = self._library(ordinal) + self.facts.imports.append( + ImportSym( + name=name, + library=library, + special=special, + weak=bool(desc & N_WEAK_REF), + ) + ) + elif kind == N_SECT and exports: + self.facts.exports.append(ExportSym(name=name, address=value)) + + +def parse_macho_file( + data: Buffer, path: str = "", *, arch: str | None = None +) -> list[FileFacts]: + """Every image in a thin or fat file (only ``arch`` when given).""" + magic = struct.unpack_from(">I", data, 0)[0] + if magic in (FAT_MAGIC, FAT_MAGIC_64): + result = [] + for item in fat_slices(data): + if arch is not None and item.arch != arch: + continue + result.append(parse_macho(data, path, base=item.offset, size=item.size)) + return result + return [parse_macho(data, path)] diff --git a/src/tocode/links/binfmt_pe.py b/src/tocode/links/binfmt_pe.py new file mode 100644 index 0000000..60d27ab --- /dev/null +++ b/src/tocode/links/binfmt_pe.py @@ -0,0 +1,236 @@ +"""PE import, delay-import, and export tables (native and managed images).""" + +from __future__ import annotations + +import mmap +import struct + +from ..backends.dotnet_pe import PeInfo, parse_pe +from .model import LIB_DELAY, ExportSym, FileFacts, ImportSym, LibraryRef + +DIR_EXPORT = 0 +DIR_IMPORT = 1 +DIR_DEBUG = 6 +DIR_DELAY_IMPORT = 13 +IMAGE_FILE_DLL = 0x2000 +IMAGE_DEBUG_TYPE_CODEVIEW = 2 +MAX_IMPORT_DESCRIPTORS = 4096 +MAX_THUNKS = 1 << 16 +MAX_EXPORTS = 1 << 20 + +Buffer = bytes | mmap.mmap + + +class PeError(ValueError): + pass + + +def parse_pe_links(data: Buffer, path: str = "") -> FileFacts: + info = parse_pe(data) + if info is None: + raise PeError("not a PE file") + facts = FileFacts( + path=path, + format="pe", + arch=info.machine, + bits=64 if info.pe32_plus else 32, + kind="library" if info.characteristics & IMAGE_FILE_DLL else "executable", + image_base=info.image_base, + entry=info.image_base + info.entry_point_rva if info.entry_point_rva else None, + ) + reader = _Reader(data, info) + _read_imports(reader, facts) + _read_delay_imports(reader, facts) + _read_exports(reader, facts) + facts.uuid = _codeview_id(reader) + return facts + + +class _Reader: + def __init__(self, data: Buffer, info: PeInfo) -> None: + self.data = data + self.info = info + self.pointer = 8 if info.pe32_plus else 4 + + def directory(self, index: int) -> tuple[int, int]: + dirs = self.info.data_directories + return dirs[index] if index < len(dirs) else (0, 0) + + def offset(self, rva: int) -> int | None: + return self.info.rva_to_offset(rva) + + def u32(self, rva: int) -> int: + offset = self.offset(rva) + if offset is None: + raise PeError(f"rva {rva:#x} is not mapped") + return struct.unpack_from(" tuple: + offset = self.offset(rva) + if offset is None: + raise PeError(f"rva {rva:#x} is not mapped") + return struct.unpack_from(fmt, self.data, offset) + + def cstring(self, rva: int, limit: int = 1024) -> str: + offset = self.offset(rva) + if offset is None: + return "" + end = self.data.find(b"\0", offset, min(len(self.data), offset + limit)) + if end < 0: + end = min(len(self.data), offset + limit) + return bytes(self.data[offset:end]).decode("ascii", errors="replace") + + +def _thunks( + r: _Reader, lookup_rva: int, slot_rva: int, library: int +) -> list[ImportSym]: + rows: list[ImportSym] = [] + fmt = " None: + rva, size = r.directory(DIR_IMPORT) + if not rva or not size: + return + for index in range(MAX_IMPORT_DESCRIPTORS): + try: + lookup, _stamp, _chain, name_rva, first_thunk = r.unpack( + " None: + rva, size = r.directory(DIR_DELAY_IMPORT) + if not rva or not size: + return + for index in range(MAX_IMPORT_DESCRIPTORS): + try: + attrs, name_rva, _hmod, iat, int_rva, _bound, _unload, _stamp = r.unpack( + " None: + rva, size = r.directory(DIR_EXPORT) + if not rva or not size: + return + try: + (_chars, _stamp, _major, _minor, name_rva, base, function_count, name_count, + functions_rva, names_rva, ordinals_rva) = r.unpack(" bool: + for section in r.info.sections: + span = max(section.virtual_size, section.raw_size) + if section.virtual_address <= rva < section.virtual_address + span: + return "x" in section.perms + return False + + +def _codeview_id(r: _Reader) -> str | None: + rva, size = r.directory(DIR_DEBUG) + if not rva or not size: + return None + for index in range(min(size // 28, 32)): + try: + entry = r.unpack(" list[FileFacts]: + """Facts for every image in ``path`` (one per fat slice). + + ``kind`` is the sniffed kind. Parse problems are reported as a single + ``FileFacts`` with notes, so one bad file never stops a folder scan. + """ + try: + with path.open("rb") as handle: + with mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as view: + return _read(view, relpath, kind) + except PARSE_ERRORS as exc: + broken = FileFacts( + path=relpath, format=kind, arch="unknown", bits=0, kind="other" + ) + broken.notes.append(f"unparsed: {type(exc).__name__}: {exc}") + return [broken] + + +def _read(view: mmap.mmap, relpath: str, kind: str) -> list[FileFacts]: + if kind == ELF: + return [parse_elf(view, relpath)] + if kind == MACHO: + return [parse_macho(view, relpath)] + if kind == MACHO_FAT: + result = [] + for item in fat_slices(view): + try: + result.append( + parse_macho(view, relpath, base=item.offset, size=item.size) + ) + except PARSE_ERRORS as exc: + broken = FileFacts( + path=relpath, format="macho", arch=item.arch, bits=0, kind="other" + ) + broken.notes.append(f"unparsed slice: {exc}") + result.append(broken) + return result + if kind == PE: + facts = parse_pe_links(view, relpath) + from ..backends.dotnet_pe import parse_pe + + info = parse_pe(view) + if info is not None and info.is_managed: + try: + managed = parse_dotnet_links(view, relpath) + except PARSE_ERRORS as exc: + facts.notes.append(f"managed metadata unreadable: {exc}") + return [facts] + if info.mixed_mode: + # C++/CLI: keep the native import table next to the managed refs. + offset = len(managed.libraries) + managed.libraries.extend(facts.libraries) + for symbol in facts.imports: + if symbol.library is not None: + symbol.library += offset + managed.imports.extend(facts.imports) + managed.exports.extend(facts.exports) + return [managed] + return [facts] + raise ValueError(f"no link parser for {kind}") diff --git a/src/tocode/links/inventory.py b/src/tocode/links/inventory.py new file mode 100644 index 0000000..c105e62 --- /dev/null +++ b/src/tocode/links/inventory.py @@ -0,0 +1,313 @@ +"""Walk a folder and collect the binaries and their link facts.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import hashlib +import os +from pathlib import Path +import plistlib +import posixpath +from typing import Any, Callable + +from .extract import read_facts +from .model import FileFacts +from .resolve import Folder +from .runtime_libs import is_runtime_library +from .sniff import CODE_KINDS, MACHO_FAT, PE, ZIP, classify + +# Job types +JOB_NATIVE = "native" +JOB_DOTNET = "dotnet" # one managed assembly (or a single-file bundle, or a nupkg) +JOB_APK = "apk" + + +@dataclass(slots=True) +class Entry: + relpath: str + size: int + kind: str # sniff kind: elf | pe | macho | macho_fat | zip | java_class | data + detail: str = "" + sha256: str | None = None + archs: list[str] = field(default_factory=list) + arch: str | None = None # slice linked and decompiled + format: str | None = None # elf | pe | macho | dotnet + role: str | None = None # executable | library | bundle | ... + job: str | None = None # JOB_* when the file has code to decompile + runtime: bool = False # bundled runtime/framework library + alias_of: str | None = None # symlink to another file of the folder + duplicate_of: str | None = None # byte-identical to an earlier file + main: bool = False # the bundle's/app's main executable + apphost_for: str | None = None # .NET apphost launcher -> its App.dll + bundle: bool = False # .NET single-file bundle + notes: list[str] = field(default_factory=list) + + def to_json(self) -> dict[str, Any]: + row: dict[str, Any] = { + "path": self.relpath, + "size": self.size, + "kind": self.kind, + } + for key in ( + "detail", "sha256", "archs", "arch", "format", "role", "job", "alias_of", + "duplicate_of", "apphost_for", "notes", + ): # fmt: skip + value = getattr(self, key) + if value: + row[key] = value + for key in ("runtime", "main", "bundle"): + if getattr(self, key): + row[key] = True + return row + + +@dataclass +class Inventory: + root: Path + entries: list[Entry] + folder: Folder + roots: list[str] # chain roots: main executables first, then other executables + arch: str | None + skipped_dirs: list[str] = field(default_factory=list) + + def entry(self, relpath: str) -> Entry | None: + return self._by_path.get(relpath) + + def __post_init__(self) -> None: + self._by_path = {item.relpath: item for item in self.entries} + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def walk( + root: Path, *, exclude_dirs: Callable[[str], bool] | None = None +) -> tuple[list[tuple[str, Path]], dict[str, str], list[str]]: + """Regular files (relpath, path), file symlinks (relpath -> target relpath).""" + files: list[tuple[str, Path]] = [] + aliases: dict[str, str] = {} + skipped: list[str] = [] + real_root = os.path.realpath(root) + for directory, dirnames, filenames in os.walk(root, followlinks=False): + dirnames.sort() + rel_dir = Path(directory).relative_to(root).as_posix() + rel_dir = "" if rel_dir == "." else rel_dir + if exclude_dirs is not None: + kept = [] + for name in dirnames: + relpath = posixpath.join(rel_dir, name) + if exclude_dirs(relpath): + skipped.append(relpath) + else: + kept.append(name) + dirnames[:] = kept + for name in sorted(filenames): + path = Path(directory) / name + relpath = posixpath.join(rel_dir, name) + if path.is_symlink(): + real = os.path.realpath(path) + if real.startswith(real_root + os.sep) and os.path.isfile(real): + aliases[relpath] = Path(real).relative_to(real_root).as_posix() + continue + if path.is_file(): + files.append((relpath, path)) + return files, aliases, skipped + + +def scan( + root: Path, + *, + arch: str | None = None, + log: Callable[[str], None] = lambda _message: None, + hash_files: bool = True, +) -> Inventory: + root = root.resolve() + files, aliases, skipped = walk(root, exclude_dirs=_is_noise_dir) + entries: list[Entry] = [] + images: dict[str, dict[str, FileFacts]] = {} + seen_hashes: dict[str, str] = {} + log(f"Scanning {len(files)} files under {root}") + for relpath, path in files: + try: + size = path.stat().st_size + except OSError: + continue + sniff = classify(path) if size >= 4 else None + kind = sniff.kind if sniff else "data" + entry = Entry( + relpath=relpath, size=size, kind=kind, detail=sniff.detail if sniff else "" + ) + entries.append(entry) + if kind == ZIP: + flavour = entry.detail + if flavour == "apk": + entry.job = JOB_APK + elif flavour in ("nupkg", "snupkg"): + entry.job = JOB_DOTNET + continue + if kind not in CODE_KINDS: + continue + if hash_files: + entry.sha256 = _sha256(path) + first = seen_hashes.setdefault(entry.sha256, relpath) + if first != relpath: + entry.duplicate_of = first + slices = read_facts(path, relpath, kind) + by_arch: dict[str, FileFacts] = {} + for facts in slices: + by_arch.setdefault(facts.arch, facts) + entry.notes.extend(facts.notes) + entry.archs = list(by_arch) + first_facts = slices[0] + entry.format = first_facts.format + entry.role = first_facts.kind + if first_facts.format == "dotnet": + entry.job = JOB_DOTNET + managed = first_facts.managed or {} + if managed.get("entry_point_token"): + entry.role = "executable" + for facts in by_arch.values(): + facts.kind = "executable" + elif first_facts.arch != "unknown": + entry.job = JOB_NATIVE + if kind == PE or kind in ("elf", "macho"): + _dotnet_host(path, entry) + entry.runtime = entry.runtime or is_runtime_library( + posixpath.basename(relpath), managed=first_facts.format == "dotnet" + ) + if entry.job is not None: + images[relpath] = by_arch + _mark_main(root, entries, aliases) + chosen = arch or _main_arch(entries, images) + selected: dict[str, str] = {} + for entry in entries: + if entry.relpath not in images: + continue + archs = entry.archs + if len(archs) == 1: + entry.arch = archs[0] + elif chosen in archs: + entry.arch = chosen + else: + entry.arch = archs[0] + entry.notes.append(f"no {chosen} slice; using {archs[0]}") + selected[entry.relpath] = entry.arch + for entry in entries: + if entry.kind == MACHO_FAT and entry.arch is not None and len(entry.archs) > 1: + entry.detail = f"fat ({', '.join(entry.archs)}); using {entry.arch}" + folder = Folder(root=root, images=images, selected=selected, aliases=aliases) + roots = _chain_roots(entries) + alias_entries = [ + Entry(relpath=alias, size=0, kind="symlink", alias_of=target) + for alias, target in sorted(aliases.items()) + ] + return Inventory( + root=root, + entries=entries + alias_entries, + folder=folder, + roots=roots, + arch=chosen, + skipped_dirs=skipped, + ) + + +def _is_noise_dir(relpath: str) -> bool: + name = posixpath.basename(relpath) + # Our own outputs and VCS metadata are never part of the application. + return ( + name in {".git", ".hg", ".svn", "__pycache__", ".tocode-folder"} + or name.endswith("_decompiled") + or name.endswith("_decompiler") + ) + + +def _dotnet_host(path: Path, entry: Entry) -> None: + from ..backends.dotnet import _apphost_assembly + from ..backends.dotnet_pe import read_bundle + + if read_bundle(path) is not None: + entry.bundle = True + entry.job = JOB_DOTNET + return + assembly = _apphost_assembly(path) + if assembly is not None: + entry.apphost_for = posixpath.join( + posixpath.dirname(entry.relpath), assembly.name + ) + entry.runtime = True # the generic Microsoft launcher + + +def _mark_main(root: Path, entries: list[Entry], aliases: dict[str, str]) -> None: + by_path = {item.relpath: item for item in entries} + found = False + for entry in entries: + if posixpath.basename(entry.relpath) != "Info.plist": + continue + contents = posixpath.dirname(entry.relpath) + if posixpath.basename(contents) != "Contents": + continue + try: + with (root / entry.relpath).open("rb") as handle: + plist = plistlib.load(handle) + except Exception: # noqa: BLE001 - malformed plist: no main executable + continue + executable = ( + plist.get("CFBundleExecutable") if isinstance(plist, dict) else None + ) + if not isinstance(executable, str): + continue + target = posixpath.join(contents, "MacOS", executable) + target = aliases.get(target, target) + main = by_path.get(target) + if main is not None and main.job is not None: + main.main = True + found = True + if found: + return + executables = [ + item for item in entries + if item.role == "executable" and item.job is not None and not item.runtime + ] # fmt: skip + if executables: + depth = min(item.relpath.count("/") for item in executables) + for item in executables: + if item.relpath.count("/") == depth: + item.main = True + + +def _main_arch( + entries: list[Entry], images: dict[str, dict[str, FileFacts]] +) -> str | None: + for wanted_main in (True, False): + for entry in entries: + if entry.main != wanted_main or entry.role != "executable": + continue + archs = [item for item in entry.archs if item != "msil"] + if len(archs) == 1: + return archs[0] + for entry in entries: + archs = [item for item in entry.archs if item != "msil"] + if len(archs) == 1: + return archs[0] + return None + + +def _chain_roots(entries: list[Entry]) -> list[str]: + executables = [ + item for item in entries + if item.job is not None and item.role == "executable" and item.duplicate_of is None + ] # fmt: skip + executables.sort( + key=lambda item: ( + not item.main, + item.relpath.count("/"), + -item.size, + item.relpath, + ) + ) + return [item.relpath for item in executables] diff --git a/src/tocode/links/model.py b/src/tocode/links/model.py new file mode 100644 index 0000000..c10390b --- /dev/null +++ b/src/tocode/links/model.py @@ -0,0 +1,158 @@ +"""Facts parsed from one binary, and the links resolved between binaries.""" + +from __future__ import annotations + +from dataclasses import MISSING, asdict, dataclass, field, fields +from typing import Any + +# Library reference kinds (what the importer asked the loader for). +LIB_NEEDED = "needed" # ELF DT_NEEDED, Mach-O LC_LOAD_DYLIB, PE import descriptor +LIB_WEAK = "weak" # Mach-O LC_LOAD_WEAK_DYLIB (missing at runtime is fine) +LIB_REEXPORT = "reexport" # Mach-O LC_REEXPORT_DYLIB +LIB_UPWARD = "upward" # Mach-O LC_LOAD_UPWARD_DYLIB +LIB_LAZY = "lazy" # Mach-O LC_LAZY_LOAD_DYLIB +LIB_DELAY = "delay" # PE delay-load import descriptor +LIB_ASSEMBLY = "assembly" # .NET AssemblyRef +LIB_PINVOKE = "pinvoke" # .NET ModuleRef used by P/Invoke (ImplMap) + +# Mach-O special library ordinals (two-level namespace lookups). +SPECIAL_SELF = "self" +SPECIAL_MAIN = "main_executable" +SPECIAL_FLAT = "flat_lookup" +SPECIAL_WEAK = "weak_lookup" + + +@dataclass(slots=True) +class LibraryRef: + """One library the binary names, in the order the loader sees them.""" + + name: str # as written: "@rpath/core.dylib", "libfoo.so.1", "KERNEL32.dll" + kind: str = LIB_NEEDED + # Mach-O: compatibility/current version; .NET: assembly version. + version: str | None = None + + +@dataclass(slots=True) +class ImportSym: + name: str # as the linker wrote it (Mach-O keeps the leading "_") + # Index into ``FileFacts.libraries``; None for flat/global lookup (ELF + # without version info, Mach-O flat namespace, special ordinals). + library: int | None = None + special: str | None = None # SPECIAL_* for Mach-O special ordinals + ordinal: int | None = None # PE import by ordinal + version: str | None = None # ELF symbol version ("GLIBC_2.17") + weak: bool = False + slot: int | None = None # link-time VA of the pointer the loader fills + kind: str | None = None # "function" | "data" | "tls" | None (unknown) + # .NET: declaring type of a member reference (or the managed method that + # declares a P/Invoke), its canonical signature, and the MethodDef tokens of + # the importer's methods whose IL references it. + owner: str | None = None + signature: str | None = None + callers: list[int] | None = None + + +@dataclass(slots=True) +class ExportSym: + name: str + address: int | None = None # link-time VA; None for re-exports/forwarders + ordinal: int | None = None # PE export ordinal + # PE forwarder ("OTHER.Func" / "OTHER.#12") or Mach-O re-export. + forwarder: str | None = None + reexport_library: int | None = None # index into ``FileFacts.libraries`` + reexport_name: str | None = None + version: str | None = None # ELF symbol version + default_version: bool = True # ELF "@@" (default) vs "@" (hidden) + weak: bool = False + kind: str | None = None # "function" | "data" | "tls" | "absolute" | "resolver" + signature: str | None = None # .NET canonical method signature + token: int | None = None # .NET MethodDef token + + +@dataclass(slots=True) +class FileFacts: + """Everything the link resolver needs from one binary (or one fat slice).""" + + path: str # POSIX path relative to the input folder + format: str # "elf" | "pe" | "macho" | "dotnet" + arch: str + bits: int + kind: str # "executable" | "library" | "bundle" | "object" | "other" + image_base: int = 0 # link-time base the addresses below are relative to + identity: str | None = None # SONAME / install name / PE export name / assembly + libraries: list[LibraryRef] = field(default_factory=list) + rpaths: list[str] = field(default_factory=list) # ELF DT_RPATH, Mach-O LC_RPATH + runpaths: list[str] = field(default_factory=list) # ELF DT_RUNPATH + imports: list[ImportSym] = field(default_factory=list) + exports: list[ExportSym] = field(default_factory=list) + flat_namespace: bool = False # Mach-O without MH_TWOLEVEL + uuid: str | None = None # Mach-O LC_UUID, ELF build-id, PE RSDS GUID+age + slice_offset: int = 0 # fat slice offset inside the file (0 for thin files) + slice_size: int | None = None + entry: int | None = None + notes: list[str] = field(default_factory=list) # parse warnings + # Managed only: the assembly name/version and the files .NET would probe. + managed: dict[str, Any] | None = None + + def to_json(self) -> dict[str, Any]: + payload = asdict(self) + # Compact: drop defaulted keys from every import/export row. + payload["imports"] = [_compact(item) for item in self.imports] + payload["exports"] = [_compact(item) for item in self.exports] + payload["libraries"] = [_compact(item) for item in self.libraries] + return payload + + @classmethod + def from_json(cls, payload: dict[str, Any]) -> "FileFacts": + data = dict(payload) + data["libraries"] = [LibraryRef(**item) for item in data.get("libraries", [])] + data["imports"] = [ImportSym(**item) for item in data.get("imports", [])] + data["exports"] = [ExportSym(**item) for item in data.get("exports", [])] + known = {item.name for item in fields(cls)} + return cls(**{key: value for key, value in data.items() if key in known}) + + +def _compact(item: Any) -> dict[str, Any]: + row: dict[str, Any] = {} + for entry in fields(item): + value = getattr(item, entry.name) + if entry.default is not MISSING and entry.default == value: + continue + row[entry.name] = value + return row + + +# FileEdge.status values +EDGE_IN_FOLDER = "in_folder" +EDGE_SYSTEM = "system" +EDGE_APISET = "apiset" +EDGE_MISSING = "missing" +EDGE_WEAK_MISSING = "weak_missing" +EDGE_ARCH_MISMATCH = "arch_mismatch" +EDGE_SONAME_UNREACHABLE = "soname_match_unreachable" +EDGE_FRAMEWORK = "framework" # .NET framework assembly not in the folder + + +@dataclass(slots=True) +class FileEdge: + """``source`` loads ``target`` (or tries to): one resolved library reference.""" + + source: str # importer relpath + library: int # index into the importer's ``libraries`` + name: str # the reference as written + kind: str + status: str + target: str | None = None # relpath inside the folder when resolved + via: list[str] = field(default_factory=list) # how the loader got there + chain_root: str | None = None # executable whose load chain produced this + + def to_json(self) -> dict[str, Any]: + return { + "from": self.source, + "library": self.name, + "kind": self.kind, + "status": self.status, + "to": self.target, + "via": self.via, + "chain_root": self.chain_root, + } diff --git a/src/tocode/links/names.py b/src/tocode/links/names.py new file mode 100644 index 0000000..3f3ed0c --- /dev/null +++ b/src/tocode/links/names.py @@ -0,0 +1,121 @@ +"""Match backend function names against linker symbol names. + +Backends decorate names differently: IDA writes ``.printf``/``j_printf`` +thunks and keeps ``name@@VERSION`` on ELF imports, radare2 uses +``sym.imp.printf``/``sym.printf``, PE imports appear as ``__imp_X``, and +Mach-O symbols carry a leading underscore that some backends strip. A link +key removes exactly those decorations so the same symbol compares equal +across the linker's tables and a backend's ``functions.json``. Keys are only +used to find the backend's function for a symbol the linker tables already +bound; they never create links themselves. +""" + +from __future__ import annotations + +import re + +_VERSION_RX = re.compile(r"(?<=[^@])@@?[A-Za-z0-9_.]+$") +_PREFIXES = ( + "sym.imp.", + "sym.", + "imp.", + "reloc.", + "__imp__", + "__imp_", + "_imp__", + "j_", + "loc.", +) + + +def link_key(name: str) -> str: + text = name.strip() + text = _VERSION_RX.sub("", text) + changed = True + while changed: + changed = False + for prefix in _PREFIXES: + if text.startswith(prefix) and len(text) > len(prefix): + text = text[len(prefix) :] + changed = True + if text.startswith(".") and len(text) > 1: # IDA thunk ".printf" + text = text[1:] + changed = True + return text.lstrip("_") or text + + +def c_tokens(name: str, version: str | None = None) -> list[str]: + """Identifiers a decompiler may print for ``name`` in C output.""" + key = link_key(name) + tokens = [name, key, f"_{key}", f"__imp_{key}", f"j_{key}"] + if version: # "foo@@V1.0" printed as foo__V1_0 + tokens.extend([f"{name}@@{version}", f"{name}@{version}"]) + phrases: list[str] = [] + if "::" in name: # .NET Type::Member -> Member, or "new Type" for a constructor + owner, _, member = name.rpartition("::") + type_name = owner.rpartition(".")[2].rpartition("/")[2].split("`")[0] + if member == ".ctor": + phrases.append(f"new {type_name}") + elif member != ".cctor": + tokens.append(member) + seen: dict[str, None] = {} + for token in tokens: + cleaned = re.sub(r"[^0-9A-Za-z_]", "_", token) + if cleaned and not cleaned.isdigit(): + seen.setdefault(cleaned, None) + phrases.extend(itanium_hints(name)) + for phrase in phrases: + seen.setdefault(phrase, None) + return list(seen) + + +_CV = "rVKRO" + + +def itanium_hints(name: str) -> list[str]: + """``Class::method`` as a decompiler prints a mangled C++ symbol. + + Only the simple shapes are handled (nested names without templates or + substitutions); anything else yields nothing rather than a guess. + """ + text = name.lstrip("_") + if not text.startswith("Z") or len(text) < 3: + return [] + pos = 1 + parts: list[str] = [] + special: str | None = None + if text[pos] == "N": + pos += 1 + while pos < len(text) and text[pos] in _CV: + pos += 1 + if text.startswith("St", pos): + parts.append("std") + pos += 2 + while pos < len(text) and text[pos] != "E": + char = text[pos] + if char.isdigit(): + end = pos + while end < len(text) and text[end].isdigit(): + end += 1 + length = int(text[pos:end]) + parts.append(text[end : end + length]) + pos = end + length + elif char in "CD" and pos + 1 < len(text) and text[pos + 1] in "0123459": + special = "~" if char == "D" else "" + pos += 2 + else: + return [] # templates, substitutions, operators: not handled + if pos >= len(text) or not parts: + return [] + elif text[pos].isdigit(): + end = pos + while end < len(text) and text[end].isdigit(): + end += 1 + parts.append(text[end : end + int(text[pos:end])]) + else: + return [] + if special is not None: + parts.append(special + parts[-1]) + if len(parts) == 1: + return [parts[0]] if len(parts[0]) > 3 else [] + return ["::".join(parts[-2:])] diff --git a/src/tocode/links/resolve.py b/src/tocode/links/resolve.py new file mode 100644 index 0000000..18a2f65 --- /dev/null +++ b/src/tocode/links/resolve.py @@ -0,0 +1,598 @@ +"""Loader rules: which file each library reference loads. + +Each executable starts a load chain, walked breadth-first the way the +platform loader would, so context-dependent rules (``@executable_path``, +``@rpath`` stacks inherited from the loading images, ELF ``DT_RPATH`` +inheritance, already-loaded SONAMEs, the process directory for Windows DLLs) +get the context of a real process. Libraries no executable here loads become +their own chain roots. Every edge records how it was resolved in ``via``. +""" + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field +import os +from pathlib import Path +import posixpath + +from .model import ( + EDGE_APISET, + EDGE_ARCH_MISMATCH, + EDGE_FRAMEWORK, + EDGE_IN_FOLDER, + EDGE_MISSING, + EDGE_SYSTEM, + EDGE_WEAK_MISSING, + LIB_ASSEMBLY, + LIB_PINVOKE, + LIB_WEAK, + FileEdge, + FileFacts, +) + +# Not in the folder, and the loader would look in system locations next +# (ld.so cache, the Windows search order, the GAC): could be an OS library or +# a missing one; ToCode cannot tell without the target system. +EDGE_EXTERNAL = "external" +# A file with that name exists in the folder, but no loader rule reaches it +# from this importer (typically loaded through LD_LIBRARY_PATH set by a launch +# script, or SetDllDirectory). Recorded with ``target`` and flagged. +EDGE_NAME_UNREACHABLE = "name_match_unreachable" + +# HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\KnownDLLs (Windows 10/11), +# plus ntdll/kernelbase which every process maps first. +KNOWN_DLLS = frozenset( + name.lower() + for name in ( + "advapi32.dll", "clbcatq.dll", "combase.dll", "comdlg32.dll", "coml2.dll", + "difxapi.dll", "gdi32.dll", "gdiplus.dll", "imagehlp.dll", "imm32.dll", + "kernel32.dll", "msctf.dll", "msvcrt.dll", "normaliz.dll", "nsi.dll", + "ole32.dll", "oleaut32.dll", "psapi.dll", "rpcrt4.dll", "sechost.dll", + "setupapi.dll", "shcore.dll", "shell32.dll", "shlwapi.dll", "user32.dll", + "wldap32.dll", "wow64.dll", "wow64cpu.dll", "wow64win.dll", "ws2_32.dll", + "ntdll.dll", "kernelbase.dll", + ) +) # fmt: skip +MACHO_SYSTEM_PREFIXES = ("/System/", "/usr/lib/", "/Library/Apple/") + + +@dataclass +class Folder: + """The binaries of one input folder, as the resolver sees them.""" + + root: Path + # relpath -> arch -> facts (one entry per fat slice; "msil" for IL images) + images: dict[str, dict[str, FileFacts]] + # Arch each image is linked/decompiled as (the chosen fat slice). + selected: dict[str, str] + aliases: dict[str, str] = field(default_factory=dict) # symlink -> canonical + _exact: dict[str, str] = field(default_factory=dict, init=False) + _folded: dict[str, str] = field(default_factory=dict, init=False) + _by_name: dict[str, list[str]] = field(default_factory=dict, init=False) + _by_soname: dict[str, list[str]] = field(default_factory=dict, init=False) + _by_assembly: dict[str, list[str]] = field(default_factory=dict, init=False) + + def __post_init__(self) -> None: + for relpath in self.images: + self._index(relpath, relpath) + for alias, target in self.aliases.items(): + if target in self.images: + self._index(alias, target) + for relpath in self.images: + facts = self.facts(relpath) + if facts is None or not facts.identity: + continue + if facts.format == "elf": + self._by_soname.setdefault(facts.identity, []).append(relpath) + elif facts.format == "dotnet": + self._by_assembly.setdefault(facts.identity.lower(), []).append(relpath) + + def _index(self, path: str, target: str) -> None: + self._exact.setdefault(path, target) + self._folded.setdefault(path.lower(), target) + self._by_name.setdefault(posixpath.basename(path).lower(), []).append(target) + + def facts(self, relpath: str, arch: str | None = None) -> FileFacts | None: + slices = self.images.get(relpath) + if not slices: + return None + wanted = arch or self.selected.get(relpath) + if wanted in slices: + return slices[wanted] + if "msil" in slices: + return slices["msil"] + return None if arch else next(iter(slices.values())) + + def lookup(self, candidate: str, *, fold_case: bool) -> str | None: + """Folder file at ``candidate`` (relative POSIX path), via symlinks too.""" + path = posixpath.normpath(candidate) + if path.startswith("../") or path == ".." or path.startswith("/"): + return None + found = self._exact.get(path) + if found is None and fold_case: + found = self._folded.get(path.lower()) + if found is not None: + return found + # Directory symlinks (Foo.framework/Versions/Current -> A) are not + # walked; let the filesystem resolve them, but stay inside the folder. + try: + real = os.path.realpath(self.root / path) + root = os.path.realpath(self.root) + except OSError: + return None + if not real.startswith(root + os.sep): + return None + relative = Path(real).relative_to(root).as_posix() + return self._exact.get(relative) or ( + self._folded.get(relative.lower()) if fold_case else None + ) + + def by_name(self, name: str) -> list[str]: + return self._by_name.get(name.lower(), []) + + def by_soname(self, soname: str) -> list[str]: + return self._by_soname.get(soname, []) + + def by_assembly(self, name: str) -> list[str]: + return self._by_assembly.get(name.lower(), []) + + +@dataclass(slots=True) +class _Node: + relpath: str + parent: "_Node | None" + + def lineage(self) -> list[str]: + items: list[str] = [] + node: _Node | None = self + while node is not None: + items.append(node.relpath) + node = node.parent + return items # self first, chain root last + + +@dataclass +class Resolution: + edges: list[FileEdge] = field(default_factory=list) + # chain root -> images in load order (the root first) + chains: dict[str, list[str]] = field(default_factory=dict) + # image -> first chain root that loaded it + loaded_by: dict[str, str] = field(default_factory=dict) + cycles: list[list[str]] = field(default_factory=list) + + def edges_from(self, relpath: str) -> list[FileEdge]: + return [edge for edge in self.edges if edge.source == relpath] + + +def resolve_folder(folder: Folder, roots: list[str]) -> Resolution: + result = Resolution() + edge_index: dict[tuple[str, int, str | None], int] = {} + best: dict[tuple[str, int], int] = {} + pending = [item for item in roots if item in folder.images] + seen_roots: set[str] = set() + main_exe = pending[0] if pending else None + + def add_edge(edge: FileEdge) -> None: + if edge.kind == LIB_PINVOKE: + key3 = (edge.source, edge.library, edge.target) + if key3 not in edge_index: + edge_index[key3] = len(result.edges) + result.edges.append(edge) + return + key = (edge.source, edge.library) + index = best.get(key) + if index is None: + best[key] = len(result.edges) + result.edges.append(edge) + elif ( + result.edges[index].status != EDGE_IN_FOLDER + and edge.status == EDGE_IN_FOLDER + ): + result.edges[index] = edge + + def run_chain(root: str, standalone: bool) -> None: + order: list[str] = [] + loaded: dict[str, _Node] = {} + root_facts = folder.facts(root) + if root_facts is None: + return + context = _Context(folder, root, main_exe if standalone else root, standalone) + queue: deque[_Node] = deque([_Node(root, None)]) + loaded[root] = queue[0] + while queue: + node = queue.popleft() + order.append(node.relpath) + result.loaded_by.setdefault(node.relpath, root) + facts = folder.facts(node.relpath) + if facts is None: + continue + for index, lib in enumerate(facts.libraries): + for edge in context.resolve( + node, facts, index, lib.name, lib.kind, loaded + ): + edge.chain_root = root + add_edge(edge) + target = edge.target + # A name-only match is followed too (flagged on the edge): + # it is what loads once LD_LIBRARY_PATH/SetDllDirectory or a + # launcher makes the directory searchable. + if ( + edge.status not in (EDGE_IN_FOLDER, EDGE_NAME_UNREACHABLE) + or target is None + ): + continue + if target in loaded: + continue + child = _Node(target, node) + loaded[target] = child + queue.append(child) + result.chains[root] = order + + for root in pending: + if root not in seen_roots: + seen_roots.add(root) + run_chain(root, standalone=False) + # Libraries no executable here loads: resolve their own references. + for relpath in sorted(folder.images): + if relpath not in result.loaded_by and relpath not in seen_roots: + seen_roots.add(relpath) + run_chain(relpath, standalone=True) + result.cycles = dependency_cycles(result.edges) + return result + + +def dependency_cycles(edges: list[FileEdge]) -> list[list[str]]: + """Strongly connected groups of files that load each other (Tarjan).""" + graph: dict[str, set[str]] = {} + for edge in edges: + if edge.status == EDGE_IN_FOLDER and edge.target and edge.target != edge.source: + graph.setdefault(edge.source, set()).add(edge.target) + graph.setdefault(edge.target, set()) + index: dict[str, int] = {} + low: dict[str, int] = {} + stack: list[str] = [] + on_stack: set[str] = set() + groups: list[list[str]] = [] + counter = 0 + for start in sorted(graph): + if start in index: + continue + work: list[tuple[str, list[str]]] = [(start, sorted(graph[start]))] + index[start] = low[start] = counter + counter += 1 + stack.append(start) + on_stack.add(start) + while work: + node, children = work[-1] + if children: + child = children.pop() + if child not in index: + index[child] = low[child] = counter + counter += 1 + stack.append(child) + on_stack.add(child) + work.append((child, sorted(graph[child]))) + elif child in on_stack: + low[node] = min(low[node], index[child]) + continue + work.pop() + if work: + parent = work[-1][0] + low[parent] = min(low[parent], low[node]) + if low[node] == index[node]: + group: list[str] = [] + while True: + item = stack.pop() + on_stack.discard(item) + group.append(item) + if item == node: + break + if len(group) > 1: + groups.append(sorted(group)) + return sorted(groups) + + +class _Context: + def __init__( + self, folder: Folder, root: str, executable: str | None, standalone: bool + ) -> None: + self.folder = folder + self.root = root + self.executable = executable or root + self.standalone = standalone + + def resolve(self, node, facts, index, name, kind, loaded) -> list[FileEdge]: + if facts.format == "macho": + edges = [self._macho(node, facts, index, name, kind)] + elif facts.format == "elf": + edges = [self._elf(node, facts, index, name, kind, loaded)] + elif facts.format == "pe" or ( + facts.format == "dotnet" and kind not in (LIB_ASSEMBLY, LIB_PINVOKE) + ): + edges = [self._pe(node, facts, index, name, kind, loaded)] + elif kind == LIB_PINVOKE: + edges = self._pinvoke(node, facts, index, name, kind) + else: + edges = [self._assembly(node, facts, index, name, kind)] + return edges + + # -- shared ----------------------------------------------------------- + def _edge(self, node, index, name, kind, status, target=None, via=None) -> FileEdge: + return FileEdge( + source=node.relpath, + library=index, + name=name, + kind=kind, + status=status, + target=target, + via=via or [], + ) + + def _arch_checked(self, node, facts, index, name, kind, target, via) -> FileEdge: + if self.folder.facts(target, facts.arch) is None and facts.arch != "msil": + have = sorted(self.folder.images.get(target, {})) + return self._edge( + node, index, name, kind, EDGE_ARCH_MISMATCH, target, + via + [f"{target} has {', '.join(have) or 'no'} code, importer is {facts.arch}"], + ) # fmt: skip + return self._edge(node, index, name, kind, EDGE_IN_FOLDER, target, via) + + # -- Mach-O (dyld) ---------------------------------------------------- + def _macho(self, node, facts, index, name, kind) -> FileEdge: + here = posixpath.dirname(node.relpath) + exe_dir = posixpath.dirname(self.executable) + via: list[str] = [] + candidates: list[tuple[str, str]] = [] + if name.startswith("@executable_path/"): + note = "@executable_path" + if self.standalone: + note += f" assumed {exe_dir or '.'} (not loaded by an executable here)" + candidates.append((posixpath.join(exe_dir, name[17:]), note)) + elif name.startswith("@loader_path/"): + candidates.append( + (posixpath.join(here, name[13:]), f"@loader_path={here or '.'}") + ) + elif name.startswith("@rpath/"): + tail = name[7:] + for owner in node.lineage(): + owner_facts = self.folder.facts(owner) + if owner_facts is None: + continue + owner_dir = posixpath.dirname(owner) + for rpath in owner_facts.rpaths: + expanded = _expand_macho_path(rpath, owner_dir, exe_dir) + if expanded is None: + via.append(f"LC_RPATH {rpath} of {owner}: outside the folder") + continue + candidates.append( + (posixpath.join(expanded, tail), f"LC_RPATH {rpath} of {owner}") + ) + elif name.startswith("/"): + status = ( + EDGE_SYSTEM if name.startswith(MACHO_SYSTEM_PREFIXES) else EDGE_EXTERNAL + ) + if kind == LIB_WEAK and status == EDGE_EXTERNAL: + status = EDGE_WEAK_MISSING + return self._edge( + node, index, name, kind, status, via=["absolute install name"] + ) + else: + candidates.append((posixpath.join(here, name), "relative name")) + for candidate, note in candidates: + target = self.folder.lookup(candidate, fold_case=True) + if target is not None: + return self._arch_checked( + node, facts, index, name, kind, target, via + [note] + ) + via.append(f"{note}: no {candidate}") + status = EDGE_WEAK_MISSING if kind == LIB_WEAK else EDGE_MISSING + return self._unreachable( + node, index, name, kind, status, via, posixpath.basename(name) + ) + + def _unreachable(self, node, index, name, kind, status, via, leaf) -> FileEdge: + others = [item for item in self.folder.by_name(leaf) if item != node.relpath] + if others: + return self._edge( + node, index, name, kind, EDGE_NAME_UNREACHABLE, others[0], + via + [f"{others[0]} has this name but no loader rule reaches it"], + ) # fmt: skip + return self._edge(node, index, name, kind, status, via=via) + + # -- ELF (glibc ld.so) ------------------------------------------------ + def _elf(self, node, facts, index, name, kind, loaded) -> FileEdge: + here = posixpath.dirname(node.relpath) + if "/" in name: + if name.startswith("/"): + return self._edge( + node, index, name, kind, EDGE_EXTERNAL, via=["absolute path"] + ) + target = self.folder.lookup(posixpath.join(here, name), fold_case=False) + if target is not None: + return self._arch_checked( + node, facts, index, name, kind, target, ["relative path (cwd)"] + ) + return self._edge( + node, index, name, kind, EDGE_MISSING, via=["relative path (cwd)"] + ) + for relpath in loaded: + other = self.folder.facts(relpath) + if other is not None and other.format == "elf" and other.identity == name: + return self._arch_checked( + node, + facts, + index, + name, + kind, + relpath, + [f"SONAME already loaded ({relpath})"], + ) + via: list[str] = [] + search: list[tuple[str, str, str]] = [] # (path entry, owner relpath, label) + if not facts.runpaths: + for owner in node.lineage(): + owner_facts = self.folder.facts(owner) + if owner_facts is None or ( + owner != node.relpath and owner_facts.runpaths + ): + continue + for entry in owner_facts.rpaths: + search.append((entry, owner, f"DT_RPATH {entry} of {owner}")) + for entry in facts.runpaths: + search.append((entry, node.relpath, f"DT_RUNPATH {entry}")) + for entry, owner, label in search: + expanded = _expand_origin(entry, posixpath.dirname(owner)) + if expanded is None: + via.append(f"{label}: outside the folder") + continue + target = self.folder.lookup(posixpath.join(expanded, name), fold_case=False) + if target is not None: + return self._arch_checked( + node, facts, index, name, kind, target, via + [label] + ) + via.append(f"{label}: not found") + via.append("then the system library path") + matches = [item for item in self.folder.by_soname(name) if item != node.relpath] + if matches: + return self._edge( + node, index, name, kind, EDGE_NAME_UNREACHABLE, matches[0], + via + [f"{matches[0]} has SONAME {name} but no search path reaches it"], + ) # fmt: skip + return self._unreachable(node, index, name, kind, EDGE_EXTERNAL, via, name) + + # -- PE (Windows loader) ---------------------------------------------- + def _pe(self, node, facts, index, name, kind, loaded) -> FileEdge: + lower = name.lower() + if lower.startswith(("api-ms-win-", "ext-ms-win-")): + return self._edge( + node, index, name, kind, EDGE_APISET, via=["API set contract"] + ) + if lower in KNOWN_DLLS: + return self._edge(node, index, name, kind, EDGE_SYSTEM, via=["KnownDLLs"]) + for relpath in loaded: + if posixpath.basename(relpath).lower() == lower: + return self._arch_checked( + node, + facts, + index, + name, + kind, + relpath, + [f"module already loaded ({relpath})"], + ) + app_dir = posixpath.dirname(self.executable) + label = "application directory" + if self.standalone: + app_dir = posixpath.dirname(node.relpath) + label = "DLL directory (not loaded by an executable here)" + target = self.folder.lookup(posixpath.join(app_dir, name), fold_case=True) + if target is not None: + return self._arch_checked( + node, facts, index, name, kind, target, [f"{label} {app_dir or '.'}"] + ) + via = [ + f"{label} {app_dir or '.'}: not found", + "then the system directories and PATH", + ] + return self._unreachable(node, index, name, kind, EDGE_EXTERNAL, via, name) + + # -- .NET --------------------------------------------------------------- + def _assembly(self, node, facts, index, name, kind) -> FileEdge: + here = posixpath.dirname(node.relpath) + app_dir = here if self.standalone else posixpath.dirname(self.executable) + for directory, label in ( + (app_dir, "application base"), + (here, "assembly directory"), + ): + for suffix in (".dll", ".exe"): + target = self.folder.lookup( + posixpath.join(directory, name + suffix), fold_case=True + ) + if target is not None: + return self._edge( + node, index, name, kind, EDGE_IN_FOLDER, target, [label] + ) + from ..backends.dotnet import is_framework_assembly + + if is_framework_assembly(f"{name}.dll"): + return self._edge( + node, index, name, kind, EDGE_FRAMEWORK, via=["shared framework"] + ) + matches = [ + item for item in self.folder.by_assembly(name) if item != node.relpath + ] + if matches: + return self._edge( + node, index, name, kind, EDGE_NAME_UNREACHABLE, matches[0], + [f"{matches[0]} is assembly {name} but outside the probing paths"], + ) # fmt: skip + return self._edge( + node, index, name, kind, EDGE_MISSING, via=["not in the application base"] + ) + + def _pinvoke(self, node, facts, index, name, kind) -> list[FileEdge]: + here = posixpath.dirname(node.relpath) + stem = name + for suffix in (".dll", ".so", ".dylib"): + if stem.lower().endswith(suffix): + stem = stem[: -len(suffix)] + names = [ + name, + f"{stem}.dll", + f"{stem}.so", + f"lib{stem}.so", + f"{stem}.dylib", + f"lib{stem}.dylib", + ] + dirs = [here] + # deps.json native assets of portable apps: runtimes//native/ + runtimes = posixpath.join(here, "runtimes") + for relpath in self.folder.images: + if relpath.startswith(runtimes + "/") and "/native/" in relpath: + directory = posixpath.dirname(relpath) + if directory not in dirs: + dirs.append(directory) + edges: list[FileEdge] = [] + for directory in dirs: + for candidate in dict.fromkeys(names): + target = self.folder.lookup( + posixpath.join(directory, candidate), fold_case=True + ) + if target is None or any(edge.target == target for edge in edges): + continue + label = ( + "NATIVE_DLL_SEARCH_DIRECTORIES" + if directory == here + else "runtimes//native" + ) + edges.append( + self._edge(node, index, name, kind, EDGE_IN_FOLDER, target, [label]) + ) + if edges: + return edges + from ..backends.dotnet import is_framework_native + + status = EDGE_FRAMEWORK if is_framework_native(name) else EDGE_EXTERNAL + return [ + self._edge( + node, index, name, kind, status, via=["not next to the assembly"] + ) + ] + + +def _expand_macho_path(path: str, owner_dir: str, exe_dir: str) -> str | None: + for token, base in (("@loader_path", owner_dir), ("@executable_path", exe_dir)): + if path == token or path.startswith(token + "/"): + return posixpath.normpath( + posixpath.join(base or ".", path[len(token) :].lstrip("/")) + ) + return None # absolute rpaths point outside the folder + + +def _expand_origin(entry: str, owner_dir: str) -> str | None: + for token in ("${ORIGIN}", "$ORIGIN"): + if entry.startswith(token): + rest = entry[len(token) :].lstrip("/") + return posixpath.join(owner_dir, rest) if rest else (owner_dir or ".") + return None diff --git a/src/tocode/links/runtime_libs.py b/src/tocode/links/runtime_libs.py new file mode 100644 index 0000000..af02cd5 --- /dev/null +++ b/src/tocode/links/runtime_libs.py @@ -0,0 +1,32 @@ +"""Recognise bundled runtime/framework libraries by file name. + +These are linked like any other file but only decompiled with +``--include-framework``: a VC++ runtime, the .NET shared framework, or a copy +of libstdc++ is the same code in every application that ships it. +""" + +from __future__ import annotations + +import re + +from ..backends.dotnet import is_framework_assembly, is_framework_native + +_NATIVE_RUNTIME_RX = re.compile( + r"^(" + # Microsoft C/C++ runtimes and API set forwarders + r"msvc[pr]\d+[a-z]*|vcruntime\d+(_\d+)?|ucrtbased?|concrt\d+|vccorlib\d+|" + r"vcomp\d+|mfc\d+[a-z]*|atl\d+|api-ms-win-.*|ext-ms-win-.*|" + # MinGW runtimes + r"libgcc_s_[a-z0-9_]+-\d+|libstdc\+\+-\d+|libwinpthread-\d+|libgomp-\d+|" + # glibc / GCC / LLVM runtime copies on Linux and macOS + r"libc|libm|libdl|librt|libpthread|libresolv|libutil|ld-linux(-[a-z0-9_-]+)?|" + r"libstdc\+\+|libgcc_s|libatomic|libgomp|libc\+\+(abi)?|libunwind" + r")(\.dll|\.so(\.[0-9.]+)?|\.dylib|\.[0-9.]+\.dylib)?$", + re.IGNORECASE, +) + + +def is_runtime_library(name: str, *, managed: bool) -> bool: + if managed: + return is_framework_assembly(name) + return bool(_NATIVE_RUNTIME_RX.match(name)) or is_framework_native(name) diff --git a/src/tocode/links/sniff.py b/src/tocode/links/sniff.py new file mode 100644 index 0000000..46e2460 --- /dev/null +++ b/src/tocode/links/sniff.py @@ -0,0 +1,113 @@ +"""Classify a file from its first bytes without parsing it.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import struct +import zipfile + +ELF_MAGIC = b"\x7fELF" +MACHO_MAGICS = { + b"\xcf\xfa\xed\xfe": 64, # MH_MAGIC_64, little-endian + b"\xce\xfa\xed\xfe": 32, # MH_MAGIC +} +FAT_MAGIC = 0xCAFEBABE +FAT_MAGIC_64 = 0xCAFEBABF +# A Java .class starts with the same 0xCAFEBABE; its next 4 bytes are +# (minor, major) with major >= 45, while a fat header has a small arch count. +MAX_FAT_ARCHS = 30 +KNOWN_CPU_TYPES = { + 7: "x86", + 0x01000007: "x86_64", + 12: "arm", + 0x0100000C: "arm64", + 0x0200000C: "arm64_32", + 18: "ppc", + 0x01000012: "ppc64", +} +HEAD_SIZE = 4096 + +# Kinds +ELF = "elf" +PE = "pe" +MACHO = "macho" +MACHO_FAT = "macho_fat" +JAVA_CLASS = "java_class" +ZIP = "zip" +DATA = "data" +CODE_KINDS = frozenset({ELF, PE, MACHO, MACHO_FAT}) + + +@dataclass(frozen=True, slots=True) +class Sniff: + kind: str + detail: str = "" + + +def classify_bytes(head: bytes) -> Sniff: + if head[:4] == ELF_MAGIC: + return Sniff(ELF) + if head[:4] in MACHO_MAGICS: + return Sniff(MACHO) + if len(head) >= 8: + magic, count = struct.unpack_from(">II", head, 0) + if magic in (FAT_MAGIC, FAT_MAGIC_64): + if 0 < count <= MAX_FAT_ARCHS and _fat_cpu_known(head, magic, count): + return Sniff(MACHO_FAT, f"{count} slices") + if magic == FAT_MAGIC: + return Sniff(JAVA_CLASS) + if head[:2] == b"MZ" and len(head) >= 0x40: + pe_offset = struct.unpack_from(" len(head): + return Sniff(PE, "header beyond sniff window") + return Sniff(DATA, "DOS stub only") + if head[:4] in (b"PK\x03\x04", b"PK\x05\x06"): + return Sniff(ZIP) + return Sniff(DATA) + + +def _fat_cpu_known(head: bytes, magic: int, count: int) -> bool: + size = 32 if magic == FAT_MAGIC_64 else 20 + first = 8 + if len(head) < first + size: + return False + cpu = struct.unpack_from(">I", head, first)[0] + return cpu in KNOWN_CPU_TYPES + + +def read_head(path: Path, size: int = HEAD_SIZE) -> bytes: + try: + with path.open("rb") as handle: + return handle.read(size) + except OSError: + return b"" + + +def classify(path: Path) -> Sniff: + head = read_head(path) + sniff = classify_bytes(head) + if sniff.kind == PE and sniff.detail: + # The PE header sits past the first 4 KiB (rare: large DOS stubs). + from ..backends.dotnet_pe import read_pe_info + + return Sniff(PE) if read_pe_info(path) is not None else Sniff(DATA) + if sniff.kind == ZIP: + return Sniff(ZIP, _zip_flavour(path)) + return sniff + + +def _zip_flavour(path: Path) -> str: + suffix = path.suffix.lower() + if suffix in {".apk", ".nupkg", ".snupkg", ".jar", ".aar"}: + return suffix.lstrip(".") + try: + with zipfile.ZipFile(path) as archive: + names = set(archive.namelist()[:2000]) + except (OSError, zipfile.BadZipFile, ValueError): + return "zip" + if "AndroidManifest.xml" in names: + return "apk" + return "zip" diff --git a/src/tocode/progress.py b/src/tocode/progress.py index cd1da1c..2e31d5c 100644 --- a/src/tocode/progress.py +++ b/src/tocode/progress.py @@ -42,9 +42,24 @@ def set_log_path(self, path: Path) -> None: def log(self, message: str) -> None: if self.enabled: - print(message, file=sys.stderr) + writer = getattr(_tqdm, "write", None) + if callable(writer): + # Prints above any active progress bars instead of through them. + writer(message, file=sys.stderr) + else: + print(message, file=sys.stderr) self.file_log(message) + def new_bar(self, **kwargs: Any) -> Any: + """A tqdm bar the caller updates and closes itself (or a no-op).""" + tqdm_factory: Any = _tqdm + if not self.enabled or tqdm_factory is None: + return None + kwargs.setdefault("dynamic_ncols", True) + kwargs.setdefault("leave", False) + kwargs.setdefault("file", sys.stderr) + return tqdm_factory(**kwargs) + def file_log(self, message: str) -> None: if self.log_path is not None: self.log_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/link_fixtures.py b/tests/link_fixtures.py new file mode 100644 index 0000000..b92177e --- /dev/null +++ b/tests/link_fixtures.py @@ -0,0 +1,711 @@ +"""Synthetic ELF, PE, Mach-O, and .NET images for the folder link tests. + +Each builder writes only the structures the link parsers read, with the same +layout a linker produces, so the tests run without any toolchain. +""" + +from __future__ import annotations + +from pathlib import Path +import struct + +from tocode.links.binfmt_dotnet import SCHEMA + +# -------------------------------------------------------------------------- +# ELF (64-bit little-endian, one PT_LOAD at vaddr 0 mapping the whole file) +# -------------------------------------------------------------------------- + +EM_AARCH64 = 0xB7 +EM_X86_64 = 0x3E + + +def elf( + *, + soname: str | None = None, + needed: list[str] = (), # type: ignore[assignment] + rpath: str | None = None, + runpath: str | None = None, + exports: list[tuple[str, int, str | None]] = (), # type: ignore[assignment] + imports: list[tuple[str, str | None, str | None]] = (), # type: ignore[assignment] + executable: bool = False, + machine: int = EM_AARCH64, +) -> bytes: + """``exports``: (name, address, version); ``imports``: (name, version, file).""" + strtab = bytearray(b"\0") + + def string(value: str) -> int: + offset = len(strtab) + strtab.extend(value.encode() + b"\0") + return offset + + # Version indexes: 2.. for definitions, then requirements. + def_versions = sorted({version for _n, _a, version in exports if version}) + need_versions: dict[str, list[str]] = {} + for _name, version, file in imports: + if version and file: + need_versions.setdefault(file, []) + if version not in need_versions[file]: + need_versions[file].append(version) + index_of: dict[tuple[str | None, str], int] = {} + next_index = 2 + for version in def_versions: + index_of[(None, version)] = next_index + next_index += 1 + for file, versions in need_versions.items(): + for version in versions: + index_of[(file, version)] = next_index + next_index += 1 + + symbols = [(0, 0, 0, 0, 0)] # name, info, shndx, value, versym + for name, address, version in exports: + symbols.append( + ( + string(name), + 0x12, + 1, + address, + index_of.get((None, version), 1) if version else 1, + ) + ) + for name, version, file in imports: + symbols.append( + ( + string(name), + 0x12, + 0, + 0, + index_of[(file, version)] if version and file else 1, + ) + ) + needed_offsets = [string(item) for item in needed] + soname_offset = string(soname) if soname else None + rpath_offset = string(rpath) if rpath else None + runpath_offset = string(runpath) if runpath else None + verdef = bytearray() + for position, version in enumerate(def_versions): + version_name = string(version) + last = position == len(def_versions) - 1 + verdef += struct.pack( + " None: + while (base + len(blob)) % align: + blob.append(0) + layout[key] = base + len(blob) + blob.extend(data) + + dynsym = b"".join( + struct.pack(" None: + self.rva = rva + self.data = bytearray() + + def add(self, payload: bytes, align: int = 4) -> int: + while len(self.data) % align: + self.data.append(0) + offset = self.rva + len(self.data) + self.data.extend(payload) + return offset + + def reserve(self, size: int, align: int = 8) -> int: + return self.add(b"\0" * size, align) + + def put(self, rva: int, payload: bytes) -> None: + start = rva - self.rva + self.data[start : start + len(payload)] = payload + + +def pe( + *, + name: str | None = None, + exports: list[tuple[str | None, int, int | str]] = (), # type: ignore[assignment] + imports: dict[str, list[str | int]] | None = None, + delay: dict[str, list[str]] | None = None, + dll: bool = True, + machine: int = 0x8664, + image_base: int = 0x180000000, +) -> bytes: + """``exports``: (name or None for NONAME, ordinal, text rva or "DLL.Forwarder").""" + plus = machine in (0x8664, 0xAA64) + pointer = 8 if plus else 4 + rdata = _Rdata(0x2000) + directories: dict[int, tuple[int, int]] = {} + + def thunk_tables(names: list[str | int]) -> tuple[int, int]: + entries = [] + for item in names: + if isinstance(item, int): + entries.append((1 << (pointer * 8 - 1)) | item) + else: + entries.append( + rdata.add(struct.pack(" bytes: + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def export_trie(entries: list[tuple[str, int] | tuple[str, int, str]]) -> bytes: + """Root node with one edge per symbol. (name, address) or (name, ordinal, imported) for re-exports.""" + terminals = [] + for entry in entries: + if len(entry) == 3: + name, ordinal, imported = entry # type: ignore[misc] + info = _uleb(0x08) + _uleb(ordinal) + imported.encode() + b"\0" + else: + name, address = entry # type: ignore[misc] + info = _uleb(0) + _uleb(address) + terminals.append((name, _uleb(len(info)) + info + b"\0")) + # Fixed point: child offsets depend on the root size. + root_size = 0 + for _ in range(4): + offsets = [] + cursor = root_size + for _name, node in terminals: + offsets.append(cursor) + cursor += len(node) + root = bytearray(b"\0" + bytes([len(terminals)])) + for (name, _node), offset in zip(terminals, offsets): + root += name.encode() + b"\0" + _uleb(offset) + if len(root) == root_size: + break + root_size = len(root) + return bytes(root) + b"".join(node for _name, node in terminals) + + +def macho( + *, + filetype: str = "dylib", + install_name: str | None = None, + libraries: list[tuple[str, int]] = (), # type: ignore[assignment] + rpaths: list[str] = (), # type: ignore[assignment] + imports: list[tuple[str, int]] = (), # type: ignore[assignment] + exports: list[tuple[str, int] | tuple[str, int, str]] = (), # type: ignore[assignment] + imports_format: int = 1, + bind_opcodes: bool = False, + cputype: int = CPU_ARM64, +) -> bytes: + """``libraries``: (path, load command); ``imports``: (symbol, ordinal) with + ordinal 1-based or special (0 self, -1 main, -2 flat, -3 weak).""" + text_vmaddr = 0x100000000 if filetype == "executable" else 0 + commands: list[bytes] = [] + + def dylib_command(cmd: int, path: str) -> bytes: + name = path.encode() + b"\0" + size = (24 + len(name) + 7) & ~7 + return struct.pack(" bytes: + align = 0x4000 + header = bytearray(struct.pack(">II", 0xCAFEBABE, len(slices))) + offset = align + body = bytearray() + for cputype, data in slices: + header += struct.pack(">iiIII", cputype, 0, offset, len(data), 14) + body += data + b"\0" * ((-len(data)) % align) + offset += len(data) + ((-len(data)) % align) + return bytes(header + b"\0" * (align - len(header)) + body) + + +# -------------------------------------------------------------------------- +# .NET metadata +# -------------------------------------------------------------------------- + + +class Metadata: + """Rows for a #~ stream with small (2-byte) heap and table indexes.""" + + def __init__(self) -> None: + self.strings = bytearray(b"\0") + self.blobs = bytearray(b"\0") + self.tables: dict[int, list[tuple[int, ...]]] = {} + + def string(self, value: str) -> int: + offset = len(self.strings) + self.strings += value.encode() + b"\0" + return offset + + def blob(self, value: bytes) -> int: + offset = len(self.blobs) + self.blobs += bytes([len(value)]) + value + return offset + + def add(self, table: int, *values: int) -> int: + self.tables.setdefault(table, []).append(values) + return len(self.tables[table]) + + def stream(self) -> bytes: + valid = 0 + for table in self.tables: + valid |= 1 << table + out = bytearray(struct.pack(" bytes: + """A PE32+ with ``md`` as its metadata. ``bodies``: MethodDef rid -> IL; + each MethodDef's RVA column is patched to point at its body.""" + body_rva = 0x3000 + code = bytearray() + for rid, il in sorted((bodies or {}).items()): + rva = body_rva + len(code) + row = list(md.tables[6][rid - 1]) + row[0] = rva + md.tables[6][rid - 1] = tuple(row) + code += bytes([(len(il) << 2) | 0x2]) + il + while len(code) % 4: + code.append(0) + strings = bytes(md.strings) + b"\0" * ((-len(md.strings)) % 4) + blobs = bytes(md.blobs) + b"\0" * ((-len(md.blobs)) % 4) + streams = [ + ("#~", md.stream()), + ("#Strings", strings), + ("#Blob", blobs), + ("#GUID", b"\0" * 16), + ] + version = b"v4.0.30319\0\0" + root = bytearray( + b"BSJB" + + struct.pack(" Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(mode) + return path diff --git a/tests/test_cli.py b/tests/test_cli.py index 40e5631..bfe2e86 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -226,3 +226,71 @@ def test_main_dotnet_input_without_libraries_explains_setup( # pytest's stdin is not a terminal, so this must fail fast, not prompt. assert cli.main(["-q", str(binary)]) == 1 assert "tocode --setup-dotnet" in capsys.readouterr().err + + +def test_parser_has_folder_flags() -> None: + args = build_parser().parse_args( + ["--only", "**/a.dylib", "--only", "b*", "--exclude", "x", "--no-managed", + "--arch", "arm64", "--plan", "--links-only", "dir"] + ) # fmt: skip + + assert args.only == ["**/a.dylib", "b*"] and args.exclude == ["x"] + assert args.no_managed and args.arch == "arm64" and args.plan and args.links_only + assert build_parser().parse_args(["dir"]).only == [] + + +def test_main_routes_a_directory_to_the_folder_export(tmp_path, monkeypatch) -> None: + from tocode import cli, folder + + seen = {} + + def fake_export(path, *, options, progress): + seen["path"] = path + seen["options"] = options + return folder.FolderSummary( + root_dir=tmp_path / "out", input_dir=path, binaries=0, jobs=[], + edges=0, links=0, bound=0, mapped=0, + ) # fmt: skip + + monkeypatch.setattr(folder, "export_folder", fake_export) + + code = cli.main( + [ + "--only", + "*.dll", + "--no-native", + "--include-framework", + "-j", + "2", + "-q", + str(tmp_path), + ] + ) + + assert code == 0 + assert seen["path"] == tmp_path.resolve() + options = seen["options"] + assert options.only == ["*.dll"] and options.native is False + assert options.include_framework is True and options.jobs == 2 + assert options.ensure_dotnet is not None + + +def test_main_rejects_binja_backend_for_folder_input(tmp_path) -> None: + from tocode.cli import main + + with pytest.raises(SystemExit) as info: + main(["--backend", "binja", str(tmp_path)]) + + assert info.value.code == 2 + + +def test_main_plans_a_folder_without_writing(tmp_path, capsys) -> None: + from link_fixtures import elf, write + + from tocode.cli import main + + write(tmp_path / "app/app", elf(executable=True)) + + assert main(["--plan", str(tmp_path / "app")]) == 0 + assert "1. app" in capsys.readouterr().out + assert not (tmp_path / "app_decompiled").exists() diff --git a/tests/test_folder.py b/tests/test_folder.py new file mode 100644 index 0000000..1ffe448 --- /dev/null +++ b/tests/test_folder.py @@ -0,0 +1,336 @@ +"""Folder exports: scheduling, resume, filters, linking, annotation, CLI.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from link_fixtures import elf, pe, write +import pytest + +from tocode.apk_native import NativeLib +from tocode.folder import ( + DONE, + FILTERED, + RUNTIME, + FolderOptions, + export_folder, + format_plan, + glob_regex, + path_matches, +) +from tocode.folder_annotate import Annotation, annotate_file, strip_annotations +from tocode.links.inventory import scan +from tocode.progress import Progress + + +def _app(root: Path) -> Path: + write( + root / "bin/app", + elf( + executable=True, + needed=["libfoo.so.1", "libbar.so"], + rpath="$ORIGIN/../lib", + imports=[("foo_add", "FOO_1.0", "libfoo.so.1"), ("bar_mul", None, None)], + ), + ) + write( + root / "lib/libfoo.so.1", + elf(soname="libfoo.so.1", exports=[("foo_add", 0x700, "FOO_1.0")]), + ) + write( + root / "lib/libbar.so", + elf(soname="libbar.so", exports=[("bar_mul", 0x500, None)]), + ) + write(root / "lib/libstdc++.so.6", elf(soname="libstdc++.so.6")) + write(root / "README.txt", b"not a binary\n", 0o644) + return root + + +class FakeExporter: + """Writes the files a real export leaves, from the binary's link facts.""" + + def __init__(self, fail: set[str] | None = None) -> None: + self.calls: list[str] = [] + self.fail = fail or set() + + def __call__( + self, lib: NativeLib, out_dir: Path, progress: Progress + ) -> tuple[str, str, int, int]: + from tocode.errors import ToCodeError + from tocode.links.extract import read_facts + from tocode.links.sniff import classify + + self.calls.append(lib.entry) + if lib.entry in self.fail: + raise ToCodeError("backend crashed") + facts = read_facts(lib.path, lib.entry, classify(lib.path).kind)[0] + out_dir.mkdir(parents=True, exist_ok=True) + source = ["/* Generated by ToCode. */"] + rows: list[dict[str, Any]] = [] + for export in facts.exports: + start = len(source) + 1 + source += [ + "/* metadata:", + " */", + f"int {export.name}(int a) {{", + " return a;", + "}", + ] + rows.append( + self._row(export.address or 0, export.name, start, len(source), []) + ) + if facts.kind == "executable": + start = len(source) + 1 + names = [item.name for item in facts.imports] + calls = [ + f" x = {item.name}__{item.version.replace('.', '_')}(x);" + if item.version + else f" x = {item.name}(x);" + for item in facts.imports + ] + source += [ + "/* metadata:", + " */", + "int main(int x) {", + *calls, + " return x;", + "}", + ] + rows.append(self._row(0x900, "main", start, len(source), names)) + (out_dir / "src/raw").mkdir(parents=True, exist_ok=True) + (out_dir / "src/raw/a.c").write_text("\n".join(source) + "\n") + (out_dir / "functions.json").write_text(json.dumps({"functions": rows})) + (out_dir / "imports.json").write_text( + json.dumps( + { + "imports": [ + { + "dll": "x", + "functions": [ + {"name": item.name, "address": "0x0"} + for item in facts.imports + ], + } + ] + } + ) + ) + (out_dir / "exports.json").write_text( + json.dumps({"exports": [{"name": item.name} for item in facts.exports]}) + ) + (out_dir / "project.json").write_text(json.dumps({"image_base": "0x0"})) + (out_dir / "export-manifest.json").write_text("{}") + return ("fake", "fake", len(rows), 0) + + @staticmethod + def _row( + address: int, name: str, start: int, end: int, imports: list[str] + ) -> dict[str, Any]: + return { + "address": f"0x{address:x}", + "name": name, + "size": 0x10, + "callees": [], + "callees_imports": imports, + "source_file": "src/raw/a.c", + "source_line_start": start, + "source_line_end": end, + "asm_file": "src/raw/a.asm", + "asm_line_start": 1, + "asm_line_end": 2, + } + + +def _export(root: Path, tmp_path: Path, exporter: FakeExporter, **kwargs: Any): + options = FolderOptions(out_dir=tmp_path / "out", **kwargs) + return export_folder( + root, options=options, progress=Progress(enabled=False), exporter=exporter + ) + + +def test_folder_export_links_exports_and_annotates(tmp_path: Path) -> None: + root = _app(tmp_path / "app") + exporter = FakeExporter() + + summary = _export(root, tmp_path, exporter) + + # Executable first, then what it loads; the bundled libstdc++ is skipped. + assert exporter.calls == ["bin/app", "lib/libfoo.so.1", "lib/libbar.so"] + statuses = {job.relpath: job.status for job in summary.jobs} + assert statuses["lib/libstdc++.so.6"] == RUNTIME + out = tmp_path / "out" + mappings = json.loads((out / "mappings.json").read_text()) + by_symbol = {row["symbol"]: row for row in mappings["links"]} + foo = by_symbol["foo_add"] + assert foo["target_function"]["source_file"] == "lib/libfoo.so.1/src/raw/a.c" + assert foo["call_sites"][0]["name"] == "main" + source = (out / "bin/app/src/raw/a.c").read_text() + assert ( + "x = foo_add__FOO_1_0(x); /* tocode-link: foo_add -> lib/libfoo.so.1/src/raw/a.c:2-6 */" + in source + ) + functions = json.loads((out / "lib/libbar.so/functions.json").read_text())[ + "functions" + ] + assert functions[0]["imported_by_count"] == 1 + imports = json.loads((out / "bin/app/imports.json").read_text()) + assert {edge["library"] for edge in imports["libraries"]} == { + "libfoo.so.1", + "libbar.so", + } + for name in ( + "AGENTS.md", + "CLAUDE.md", + "project.json", + "inventory.json", + "dependency-graph.json", + "export-manifest.json", + ): + assert (out / name).is_file() + assert json.loads((out / "project.json").read_text())["kind"] == "folder" + agents = out / "bin/app/AGENTS.md" + assert not agents.exists() # written by the real exporter, not the fake + + +def test_folder_export_resumes_and_relinks_idempotently(tmp_path: Path) -> None: + root = _app(tmp_path / "app") + _export(root, tmp_path, FakeExporter()) + source = tmp_path / "out/bin/app/src/raw/a.c" + first = source.read_text() + + again = FakeExporter() + summary = _export(root, tmp_path, again) + + assert again.calls == [] # unchanged binaries are not exported again + assert all(job.reused for job in summary.jobs if job.status == DONE) + assert source.read_text() == first + + # A changed binary gets a new job key and is exported again. + write( + root / "lib/libbar.so", + elf(soname="libbar.so", exports=[("bar_mul", 0x540, None)]), + ) + changed = FakeExporter() + _export(root, tmp_path, changed) + assert changed.calls == ["lib/libbar.so"] + + +def test_folder_export_filters_and_failures(tmp_path: Path) -> None: + root = _app(tmp_path / "app") + exporter = FakeExporter(fail={"lib/libfoo.so.1"}) + + summary = _export(root, tmp_path, exporter, exclude=["libbar*"]) + + statuses = {job.relpath: job.status for job in summary.jobs} + assert statuses["lib/libbar.so"] == FILTERED + assert statuses["lib/libfoo.so.1"].startswith("failed") + assert statuses["bin/app"] == DONE # one failure does not stop the run + mappings = json.loads((tmp_path / "out/mappings.json").read_text()) + bar = next(row for row in mappings["links"] if row["symbol"] == "bar_mul") + assert bar["target"] == "lib/libbar.so" and bar["target_status"] == FILTERED + + +def test_links_only_does_not_export(tmp_path: Path) -> None: + root = _app(tmp_path / "app") + exporter = FakeExporter() + + _export(root, tmp_path, exporter, links_only=True) + + assert exporter.calls == [] + assert (tmp_path / "out/mappings.json").is_file() + + +def test_plan_writes_nothing(tmp_path: Path) -> None: + root = _app(tmp_path / "app") + + summary = _export(root, tmp_path, FakeExporter(), plan=True) + + assert not (tmp_path / "out").exists() + text = format_plan(summary) + assert "1. bin/app" in text and "lib/libfoo.so.1" in text + + +def test_output_inside_the_input_must_be_a_decompiled_folder(tmp_path: Path) -> None: + from tocode.errors import ToCodeError + + root = _app(tmp_path / "app") + with pytest.raises(ToCodeError, match="inside the input folder"): + export_folder( + root, options=FolderOptions(out_dir=root / "out"), exporter=FakeExporter() + ) + + +def test_scan_skips_our_own_outputs(tmp_path: Path) -> None: + root = _app(tmp_path / "app") + write(root / "app_decompiled/bin/app/x.so", elf(soname="x.so")) + + inventory = scan(root) + + assert all( + not entry.relpath.startswith("app_decompiled") for entry in inventory.entries + ) + + +def test_pe_duplicate_files_export_once(tmp_path: Path) -> None: + root = tmp_path / "w" + data = pe(name="a.dll", exports=[("f", 1, 0x1000)]) + write(root / "a.dll", data) + write(root / "copy/a.dll", data) + exporter = FakeExporter() + + summary = _export(root, tmp_path, exporter) + + assert exporter.calls == ["a.dll"] + duplicate = next(job for job in summary.jobs if job.relpath == "copy/a.dll") + assert duplicate.status == "duplicate of a.dll" + + +# -- globs and annotation ----------------------------------------------------- + + +def test_globs_cross_directories_and_match_basenames() -> None: + assert glob_regex("**/app.dylib").match("Contents/MacOS/app.dylib") + assert glob_regex("**/app.dylib").match("app.dylib") + assert not glob_regex("Contents/*.dylib").match("Contents/MacOS/app.dylib") + assert path_matches("Contents/MacOS/core.dylib", ["core*"]) + assert not path_matches("Contents/MacOS/core.dylib", ["Contents/core*"]) + + +def test_annotation_keeps_line_count_and_skips_comments_and_strings( + tmp_path: Path, +) -> None: + path = tmp_path / "a.c" + original = "\n".join( + [ + "/* metadata:", + " * foo here is a comment", + " */", + "int f(void) {", + ' puts("foo");', + " return foo(1);", + "}", + "", + ] + ) + path.write_text(original) + note = Annotation("a.c", 1, 7, ["foo"], "foo -> lib.so/src/raw/b.c:3-9") + + assert annotate_file(path, [note]) == 1 + lines = path.read_text().split("\n") + assert len(lines) == len(original.split("\n")) + assert ( + lines[5] == " return foo(1); /* tocode-link: foo -> lib.so/src/raw/b.c:3-9 */" + ) + assert lines[4] == ' puts("foo");' and lines[1] == " * foo here is a comment" + + # Idempotent, and a call that cannot be found goes into the metadata line. + assert annotate_file(path, [note]) == 1 + assert path.read_text().count("tocode-link") == 1 + other = Annotation("a.c", 1, 7, ["missing"], "missing -> x") + annotate_file(path, [other]) + text = path.read_text().split("\n") + assert text[0] == "/* metadata: tocode-link: missing -> x" + assert "tocode-link" not in text[5] + assert strip_annotations(text[0]) == "/* metadata:" diff --git a/tests/test_links.py b/tests/test_links.py new file mode 100644 index 0000000..e38b2f7 --- /dev/null +++ b/tests/test_links.py @@ -0,0 +1,558 @@ +"""Link facts: binary parsers, loader rules, and symbol binding.""" + +from __future__ import annotations + +from pathlib import Path + +from link_fixtures import ( + CPU_ARM64, + CPU_X86_64, + EM_X86_64, + LC_LOAD_DYLIB, + LC_LOAD_WEAK_DYLIB, + LC_REEXPORT_DYLIB, + Metadata, + elf, + fat, + macho, + managed, + pe, + write, +) +import pytest + +from tocode.links.analyze import link_folder +from tocode.links.binfmt_dotnet import parse_dotnet_links +from tocode.links.binfmt_elf import parse_elf +from tocode.links.binfmt_macho import parse_macho, parse_macho_file +from tocode.links.binfmt_pe import parse_pe_links +from tocode.links.inventory import scan +from tocode.links.names import c_tokens, itanium_hints, link_key +from tocode.links.sniff import JAVA_CLASS, MACHO_FAT, classify_bytes + + +def _links(root: Path): + inventory = scan(root) + return inventory, link_folder(inventory) + + +def _edge(result, source: str, name: str): + return next( + edge + for edge in result.resolution.edges + if edge.source == source and edge.name == name + ) + + +def _link(result, importer: str, symbol: str): + return next( + link + for link in result.links + if link.importer == importer and link.symbol == symbol + ) + + +# -- sniffing ----------------------------------------------------------------- + + +def test_sniff_tells_fat_mach_o_from_java_class() -> None: + universal = fat([(CPU_ARM64, macho())]) + java = b"\xca\xfe\xba\xbe\x00\x00\x00\x41" + b"\x00" * 32 # class file v65 + + assert classify_bytes(universal).kind == MACHO_FAT + assert classify_bytes(java).kind == JAVA_CLASS + + +# -- parsers ------------------------------------------------------------------ + + +def test_elf_reads_needed_paths_and_versioned_symbols() -> None: + facts = parse_elf( + elf( + soname="libfoo.so.1", + needed=["libbar.so", "libc.so.6"], + runpath="$ORIGIN", + exports=[("foo_add", 0x1000, "FOO_1.0"), ("foo_raw", 0x1010, None)], + imports=[("bar_mul", None, None), ("puts", "GLIBC_2.17", "libc.so.6")], + ) + ) + + assert facts.kind == "library" and facts.identity == "libfoo.so.1" + assert [lib.name for lib in facts.libraries] == ["libbar.so", "libc.so.6"] + assert facts.runpaths == ["$ORIGIN"] and facts.rpaths == [] + imports = {item.name: item for item in facts.imports} + assert imports["bar_mul"].library is None # unversioned: global lookup + assert imports["puts"].version == "GLIBC_2.17" and imports["puts"].library == 1 + exports = {item.name: item for item in facts.exports} + assert ( + exports["foo_add"].address == 0x1000 and exports["foo_add"].version == "FOO_1.0" + ) + + +def test_elf_pie_is_an_executable() -> None: + assert parse_elf(elf(executable=True)).kind == "executable" + + +def test_pe_reads_imports_delay_imports_ordinals_and_forwarders() -> None: + facts = parse_pe_links( + pe( + name="core.dll", + exports=[ + ("core_add", 1, 0x1000), + (None, 42, 0x1010), + ("fwd", 3, "mathx.mathx_mul"), + ], + imports={"KERNEL32.dll": ["Sleep", 7]}, + delay={"lazy.dll": ["lazy_log"]}, + ) + ) + + assert facts.kind == "library" and facts.identity == "core.dll" + assert [(lib.name, lib.kind) for lib in facts.libraries] == [ + ("KERNEL32.dll", "needed"), + ("lazy.dll", "delay"), + ] + by_name = {item.name: item for item in facts.imports} + assert by_name["#7"].ordinal == 7 + # IAT slots are consecutive pointers from the image base. + sleep_slot, ordinal_slot = by_name["Sleep"].slot, by_name["#7"].slot + assert sleep_slot is not None and sleep_slot > 0x180000000 + assert ordinal_slot == sleep_slot + 8 + exports = {item.name: item for item in facts.exports} + assert exports["#42"].ordinal == 42 and exports["#42"].address == 0x180001010 + assert ( + exports["fwd"].forwarder == "mathx.mathx_mul" and exports["fwd"].address is None + ) + + +@pytest.mark.parametrize("imports_format", [1, 2, 3]) +def test_macho_chained_imports_keep_two_level_ordinals(imports_format: int) -> None: + facts = parse_macho( + macho( + install_name="@rpath/a.dylib", + libraries=[ + ("@rpath/b.dylib", LC_LOAD_DYLIB), + ("/usr/lib/libSystem.B.dylib", LC_LOAD_DYLIB), + ], + rpaths=["@loader_path"], + imports=[ + ("_b_fn", 1), + ("_puts", 2), + ("_weak", -3), + ("_main_sym", -1), + ("_self", 0), + ], + exports=[("_a_fn", 0x4000), ("_re", 1, "_b_fn")], + imports_format=imports_format, + ) + ) + + assert facts.identity == "@rpath/a.dylib" and facts.rpaths == ["@loader_path"] + rows = {item.name: (item.library, item.special) for item in facts.imports} + assert rows == { + "_b_fn": (0, None), + "_puts": (1, None), + "_weak": (None, "weak_lookup"), + "_main_sym": (None, "main_executable"), + "_self": (None, "self"), + } + exports = {item.name: item for item in facts.exports} + assert exports["_a_fn"].address == 0x4000 + assert ( + exports["_re"].reexport_library == 0 and exports["_re"].reexport_name == "_b_fn" + ) + + +def test_macho_bind_opcodes_give_slots() -> None: + facts = parse_macho( + macho( + libraries=[("@rpath/b.dylib", LC_LOAD_DYLIB)], + imports=[("_x", 1), ("_y", -1)], + exports=[("_e", 0x10)], + bind_opcodes=True, + ) + ) + + assert [(item.name, item.library, item.special) for item in facts.imports] == [ + ("_x", 0, None), + ("_y", None, "main_executable"), + ] + first_slot = facts.imports[0].slot + assert first_slot is not None and facts.imports[1].slot == first_slot + 8 + assert [(item.name, item.address) for item in facts.exports] == [("_e", 0x10)] + + +def test_macho_fat_file_yields_one_facts_per_slice() -> None: + data = fat( + [ + (CPU_X86_64, macho(cputype=CPU_X86_64, exports=[("_a", 0x10)])), + (CPU_ARM64, macho(exports=[("_a", 0x20)])), + ] + ) + + slices = parse_macho_file(data) + assert [(item.arch, item.exports[0].address) for item in slices] == [ + ("x86_64", 0x10), + ("arm64", 0x20), + ] + assert [item.arch for item in parse_macho_file(data, arch="arm64")] == ["arm64"] + + +def _lib_metadata() -> Metadata: + md = Metadata() + md.add(0x00, 0, md.string("MathLib.dll"), 1, 0, 0) # Module + md.add(0x02, 0, md.string(""), 0, 0, 1, 1) + md.add(0x02, 0x100001, md.string("Calc"), md.string("MathLib"), 0, 1, 1) + int_sig = md.blob(bytes([0x00, 2, 0x08, 0x08, 0x08])) # static int (int,int) + double_sig = md.blob( + bytes([0x00, 2, 0x0D, 0x0D, 0x0D]) + ) # static double (double,double) + md.add(0x06, 0, 0, 0x16, md.string("Add"), int_sig, 1) + md.add(0x06, 0, 0, 0x16, md.string("Add"), double_sig, 1) + md.add(0x20, 0x8004, 1, 0, 0, 0, 0, 0, md.string("MathLib"), 0) # Assembly + return md + + +def _app_metadata() -> Metadata: + md = Metadata() + md.add(0x00, 0, md.string("App.dll"), 1, 0, 0) + md.add(0x23, 1, 0, 0, 0, 0, 0, md.string("MathLib"), 0, 0) # AssemblyRef 1 + scope = (1 << 2) | 2 # ResolutionScope: AssemblyRef #1 + md.add(0x01, scope, md.string("Calc"), md.string("MathLib")) # TypeRef 1 + parent = (1 << 3) | 1 # MemberRefParent: TypeRef #1 + md.add(0x0A, parent, md.string("Add"), md.blob(bytes([0x00, 2, 0x0D, 0x0D, 0x0D]))) + md.add(0x02, 0, md.string(""), 0, 0, 1, 1) + md.add(0x02, 0x100001, md.string("Program"), 0, 0, 1, 1) + md.add(0x06, 0, 0, 0x16, md.string("Main"), md.blob(bytes([0x00, 0, 0x01])), 1) + md.add(0x1A, md.string("nativemath")) # ModuleRef 1 + md.add( + 0x06, + 0, + 0x80, + 0x2016, + md.string("Square"), + md.blob(bytes([0x00, 1, 0x08, 0x08])), + 1, + ) + md.add( + 0x1C, 0, (2 << 1) | 1, md.string("native_square"), 1 + ) # ImplMap -> MethodDef 2 + md.add(0x20, 0x8004, 1, 0, 0, 0, 0, 0, md.string("App"), 0) + return md + + +def test_dotnet_reads_member_refs_with_signatures_callers_and_pinvoke() -> None: + # call MathLib.Calc::Add(double,double) (MemberRef 0x0A000001), ret + body = bytes([0x28]) + (0x0A000001).to_bytes(4, "little") + bytes([0x2A]) + facts = parse_dotnet_links(managed(_app_metadata(), bodies={1: body})) + + assert facts.format == "dotnet" and facts.identity == "App" + assert [(lib.name, lib.kind) for lib in facts.libraries] == [ + ("MathLib", "assembly"), + ("nativemath", "pinvoke"), + ] + member = next(item for item in facts.imports if item.kind == "method") + assert member.name == "MathLib.Calc::Add" + assert member.signature == "double (double,double)" + assert member.callers == [0x06000001] + pinvoke = next(item for item in facts.imports if item.kind == "pinvoke") + assert pinvoke.name == "native_square" and pinvoke.owner == "Program::Square" + + library = parse_dotnet_links(managed(_lib_metadata())) + signatures = {(item.name, item.signature): item.token for item in library.exports} + assert signatures[("MathLib.Calc::Add", "double (double,double)")] == 0x06000002 + + +# -- resolution and binding --------------------------------------------------- + + +def test_elf_rpath_runpath_versions_and_name_only_libraries(tmp_path: Path) -> None: + root = tmp_path / "app" + write( + root / "bin/app", + elf( + executable=True, + needed=["libfoo.so.1", "libbar.so", "libbaz.so"], + rpath="$ORIGIN/../lib", + imports=[ + ("foo_add", "FOO_1.0", "libfoo.so.1"), + ("bar_mul", None, None), + ("baz_hi", None, None), + ], + ), + ) + write( + root / "lib/libfoo.so.1", + elf( + soname="libfoo.so.1", + needed=["libbar.so"], + runpath="$ORIGIN", + exports=[("foo_add", 0x700, "FOO_1.0")], + imports=[("bar_mul", None, None)], + ), + ) + write( + root / "lib/libbar.so", + elf(soname="libbar.so", exports=[("bar_mul", 0x500, None)]), + ) + write( + root / "plugins/libbaz.so", + elf(soname="libbaz.so", exports=[("baz_hi", 0x600, None)]), + ) + + inventory, result = _links(root) + + assert inventory.roots == ["bin/app"] + foo = _edge(result, "bin/app", "libfoo.so.1") + assert (foo.status, foo.target) == ("in_folder", "lib/libfoo.so.1") + assert "DT_RPATH $ORIGIN/../lib of bin/app" in foo.via + bar = _edge(result, "lib/libfoo.so.1", "libbar.so") + assert bar.status == "in_folder" and "SONAME already loaded" in bar.via[0] + baz = _edge(result, "bin/app", "libbaz.so") + assert (baz.status, baz.target) == ("name_match_unreachable", "plugins/libbaz.so") + + versioned = _link(result, "bin/app", "foo_add") + assert (versioned.status, versioned.resolution, versioned.address) == ( + "bound", + "versioned", + 0x700, + ) + scoped = _link(result, "bin/app", "bar_mul") + assert (scoped.target, scoped.resolution) == ("lib/libbar.so", "global_scope") + name_only = _link(result, "bin/app", "baz_hi") + assert name_only.target == "plugins/libbaz.so" and name_only.status == "bound" + + +def test_elf_runpath_is_not_inherited(tmp_path: Path) -> None: + root = tmp_path / "app" + write(root / "app", elf(executable=True, needed=["liba.so"], runpath="$ORIGIN/lib")) + write(root / "lib/liba.so", elf(soname="liba.so", needed=["libb.so"])) + write(root / "lib/libb.so", elf(soname="libb.so")) + + _inventory, result = _links(root) + + assert _edge(result, "app", "liba.so").status == "in_folder" + # DT_RUNPATH of the executable does not apply to liba's own dependencies. + inherited = _edge(result, "lib/liba.so", "libb.so") + assert inherited.status == "name_match_unreachable" + + +def test_macho_rpath_stack_loader_path_reexports_and_weak(tmp_path: Path) -> None: + root = tmp_path / "Game.app/Contents" + write( + root / "Info.plist", + b'CFBundleExecutableGame', + ) + write( + root / "MacOS/Game", + macho( + filetype="executable", + libraries=[ + ("@rpath/core.dylib", LC_LOAD_DYLIB), + ("@rpath/opt.dylib", LC_LOAD_WEAK_DYLIB), + ], + rpaths=["@executable_path/../Frameworks"], + imports=[("_core_fn", 1), ("_umbrella_fn", 1), ("_inline_fn", -3)], + exports=[("_inline_fn", 0x100000400)], + ), + ) + write( + root / "Frameworks/core.dylib", + macho( + install_name="@rpath/core.dylib", + libraries=[ + ("@loader_path/impl.dylib", LC_REEXPORT_DYLIB), + ("@rpath/util.dylib", LC_LOAD_DYLIB), + ], + imports=[("_util_fn", 2)], + exports=[("_core_fn", 0x1000)], + ), + ) + write( + root / "Frameworks/impl.dylib", + macho( + install_name="@loader_path/impl.dylib", exports=[("_umbrella_fn", 0x2000)] + ), + ) + # util.dylib is found through the executable's LC_RPATH (inherited stack). + write( + root / "Frameworks/util.dylib", + macho(install_name="@rpath/util.dylib", exports=[("_util_fn", 0x3000)]), + ) + + inventory, result = _links(tmp_path / "Game.app") + + assert inventory.roots == ["Contents/MacOS/Game"] + assert inventory.entry("Contents/MacOS/Game").main + assert ( + _edge(result, "Contents/MacOS/Game", "@rpath/core.dylib").target + == "Contents/Frameworks/core.dylib" + ) + assert ( + _edge(result, "Contents/MacOS/Game", "@rpath/opt.dylib").status + == "weak_missing" + ) + util = _edge(result, "Contents/Frameworks/core.dylib", "@rpath/util.dylib") + assert util.target == "Contents/Frameworks/util.dylib" + assert any("of Contents/MacOS/Game" in step for step in util.via) + + assert _link(result, "Contents/MacOS/Game", "_core_fn").address == 0x1000 + umbrella = _link(result, "Contents/MacOS/Game", "_umbrella_fn") + assert (umbrella.target, umbrella.resolution) == ( + "Contents/Frameworks/impl.dylib", + "reexport", + ) + weak = _link(result, "Contents/MacOS/Game", "_inline_fn") + assert (weak.target, weak.resolution) == ("Contents/MacOS/Game", "weak_lookup") + + +def test_fat_library_links_the_executable_architecture(tmp_path: Path) -> None: + root = tmp_path / "app" + write( + root / "Game", + macho( + filetype="executable", + libraries=[("@executable_path/lib.dylib", LC_LOAD_DYLIB)], + imports=[("_f", 1)], + ), + ) + write( + root / "lib.dylib", + fat( + [ + (CPU_X86_64, macho(cputype=CPU_X86_64, exports=[("_f", 0x10)])), + (CPU_ARM64, macho(exports=[("_f", 0x20)])), + ] + ), + ) + + inventory, result = _links(root) + + assert inventory.arch == "arm64" and inventory.entry("lib.dylib").arch == "arm64" + assert _link(result, "Game", "_f").address == 0x20 + + +def test_pe_app_directory_forwarders_ordinals_and_arch(tmp_path: Path) -> None: + root = tmp_path / "game" + write( + root / "app.exe", + pe( + dll=False, + image_base=0x140000000, + imports={ + "core.dll": ["core_add", "core_mul2", 42], + "KERNEL32.dll": ["Sleep"], + "api-ms-win-crt-runtime-l1-1-0.dll": ["exit"], + }, + delay={"lazy.dll": ["lazy_log"]}, + ), + ) + write( + root / "core.dll", + pe( + name="core.dll", + exports=[ + ("core_add", 1, 0x1000), + ("core_mul2", 2, "mathx.mathx_mul"), + (None, 42, 0x1010), + ], + ), + ) + write( + root / "mathx.dll", + pe( + name="mathx.dll", image_base=0x190000000, exports=[("mathx_mul", 1, 0x1000)] + ), + ) + write( + root / "plugins/lazy.dll", + pe(name="lazy.dll", exports=[("lazy_log", 1, 0x1000)]), + ) + write( + root / "tool32.exe", + pe( + dll=False, + machine=0x14C, + image_base=0x400000, + imports={"mathx.dll": ["mathx_mul"]}, + ), + ) + + _inventory, result = _links(root) + + assert _edge(result, "app.exe", "core.dll").target == "core.dll" + assert _edge(result, "app.exe", "KERNEL32.dll").status == "system" + assert ( + _edge(result, "app.exe", "api-ms-win-crt-runtime-l1-1-0.dll").status == "apiset" + ) + lazy = _edge(result, "app.exe", "lazy.dll") + assert (lazy.kind, lazy.status) == ("delay", "name_match_unreachable") + assert _edge(result, "tool32.exe", "mathx.dll").status == "arch_mismatch" + + forwarded = _link(result, "app.exe", "core_mul2") + assert (forwarded.target, forwarded.export, forwarded.resolution) == ( + "mathx.dll", + "mathx_mul", + "forwarder", + ) + assert forwarded.address == 0x190001000 + by_ordinal = _link(result, "app.exe", "#42") + assert (by_ordinal.resolution, by_ordinal.address) == ("ordinal", 0x180001010) + + +def test_dotnet_assembly_refs_and_pinvoke_bind(tmp_path: Path) -> None: + root = tmp_path / "dn" + body = bytes([0x28]) + (0x0A000001).to_bytes(4, "little") + bytes([0x2A]) + write(root / "App.dll", managed(_app_metadata(), bodies={1: body}, dll=False)) + write(root / "MathLib.dll", managed(_lib_metadata())) + write( + root / "libnativemath.so", + elf( + soname="libnativemath.so", + machine=EM_X86_64, + exports=[("native_square", 0x500, None)], + ), + ) + + _inventory, result = _links(root) + + assert _edge(result, "App.dll", "MathLib").target == "MathLib.dll" + member = _link(result, "App.dll", "MathLib.Calc::Add") + assert (member.status, member.token) == ("bound", 0x06000002) # the double overload + native = _link(result, "App.dll", "native_square") + assert (native.target, native.address) == ("libnativemath.so", 0x500) + + +def test_cycles_are_reported(tmp_path: Path) -> None: + root = tmp_path / "c" + write(root / "app", elf(executable=True, needed=["liba.so"], rpath="$ORIGIN")) + write(root / "liba.so", elf(soname="liba.so", needed=["libb.so"], rpath="$ORIGIN")) + write(root / "libb.so", elf(soname="libb.so", needed=["liba.so"], rpath="$ORIGIN")) + + _inventory, result = _links(root) + + assert result.resolution.cycles == [["liba.so", "libb.so"]] + + +# -- names -------------------------------------------------------------------- + + +def test_link_keys_strip_backend_decorations() -> None: + for raw in ( + "sym.imp.printf", + "printf@@GLIBC_2.17", + ".printf", + "__imp_printf", + "_printf", + "j_printf", + ): + assert link_key(raw) == "printf" + assert "foo__FOO_1_0" in c_tokens("foo", "FOO_1.0") + assert "new Counter" in c_tokens("MathLib.Counter::.ctor") + + +def test_itanium_hints_cover_simple_shapes_only() -> None: + assert itanium_hints("__ZN1r6StringD2Ev") == ["String::~String"] + assert itanium_hints("__ZNK1r4Mesh4sizeEv") == ["Mesh::size"] + assert itanium_hints("__Z16GetAsyncKeyStatei") == ["GetAsyncKeyState"] + assert itanium_hints("_ZNSt6vectorIiE9push_backEi") == [] From 5ef6ba0a2bf82ec2726704a7e1631e9cb34f8379 Mon Sep 17 00:00:00 2001 From: buzzer-re <22428720+buzzer-re@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:26:30 +0900 Subject: [PATCH 2/4] Release 0.3.0 Co-Authored-By: Claude Opus 5.5 --- src/tocode/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tocode/__init__.py b/src/tocode/__init__.py index 72e9618..0983677 100644 --- a/src/tocode/__init__.py +++ b/src/tocode/__init__.py @@ -7,7 +7,7 @@ __all__ = ["__version__", "export_from_binaryview"] -__version__ = "0.2.1" +__version__ = "0.3.0" def export_from_binaryview( From 5f4e8145e510cab5472fe0865ae0d96bc8f65c22 Mon Sep 17 00:00:00 2001 From: buzzer-re <22428720+buzzer-re@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:43:05 -0300 Subject: [PATCH 3/4] Add headless and MCP Binary Ninja backends - Headless: BinjaHeadlessSession loads the input with the binaryninja API in process (files, folders, APK .so, .NET native code, .bndb input), caches the analysis as a .bndb for re-runs and render workers, and ships it with the export. auto picks it after IDA and falls back when the license refuses headless use. - A core that crashes in create_database is detected once in a child process; the export then renders in process without a .bndb. An empty or incomplete cached database (left by a crashed save) is discarded and rebuilt. - --binja-license / TOCODE_BINJA_LICENSE hands a license file to the core (core_set_license) in every process; its contents are never logged. - MCP: a standard-library MCP client and BinjaMcpSession over Binary Ninja 6's built-in server (--binja-transport, --binja-mcp-url). - Strip Binary Ninja gutter glyphs from every rendered line. - Tests run against a fake on-disk binaryninja API and a fake MCP server. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 69 +- DEVELOPMENT.md | 2 +- README.md | 56 +- pyproject.toml | 4 +- src/tocode/analysis.py | 76 +- src/tocode/apk_native.py | 2 + src/tocode/backends/base.py | 114 ++- src/tocode/backends/binja.py | 32 +- src/tocode/backends/binja_headless.py | 524 ++++++++++++++ src/tocode/backends/binja_mcp.py | 764 ++++++++++++++++++++ src/tocode/backends/mcp_client.py | 282 ++++++++ src/tocode/cli.py | 239 +++++- src/tocode/exporter.py | 89 ++- src/tocode/folder.py | 1 + src/tocode/metadata.py | 25 +- src/tocode/parallel.py | 54 +- tests/conftest.py | 9 + tests/fake_binja/binaryninja/__init__.py | 468 ++++++++++++ tests/fake_binja/binaryninja/typeprinter.py | 18 + tests/test_binja_backend.py | 8 + tests/test_binja_headless.py | 586 +++++++++++++++ tests/test_binja_mcp.py | 736 +++++++++++++++++++ 22 files changed, 4048 insertions(+), 110 deletions(-) create mode 100644 src/tocode/backends/binja_headless.py create mode 100644 src/tocode/backends/binja_mcp.py create mode 100644 src/tocode/backends/mcp_client.py create mode 100644 tests/fake_binja/binaryninja/__init__.py create mode 100644 tests/fake_binja/binaryninja/typeprinter.py create mode 100644 tests/test_binja_headless.py create mode 100644 tests/test_binja_mcp.py diff --git a/AGENTS.md b/AGENTS.md index 12a0b63..6feb38b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ This repository contains ToCode, a Python-only binary exporter. ToCode takes one - `src/tocode/cli.py`: command-line entry point for `tocode`. - `src/tocode/__init__.py`: package version and `export_from_binaryview()`, the in-UI Binary Ninja library entry. - `src/tocode/analysis.py`: backend-neutral binary inventory and call graph normalization. -- `src/tocode/backends/`: IDA Domain, radare2, angr, and Binary Ninja (`binja.py`) session adapters, plus the ASC/droidasc APK backend (`asc.py`: APK set discovery, DEX inventory, worker-side class decompilation). +- `src/tocode/backends/`: IDA Domain, radare2, angr, and Binary Ninja (`binja_headless.py` over the in-process `binaryninja` API, `binja_mcp.py` over Binary Ninja's MCP server, `binja.py` over RPyc or an injected UI view) session adapters, plus the ASC/droidasc APK backend (`asc.py`: APK set discovery, DEX inventory, worker-side class decompilation). - `src/tocode/backends/dotnet.py` / `dotnet_pe.py`: .NET backend (input discovery for assemblies, apphosts, single-file bundles and NuGet packages; dnlib inventory; pure-Python PE/CLR/bundle parsing and IL token scan; worker-side C#/IL decompilation). `backends/dotnet_libs.py` downloads dnlib and ICSharpCode.Decompiler on demand (ask once, hash-pinned) into a per-user folder. - `src/tocode/dotnet.py`: .NET export pipeline; `dotnet_metadata.py`: .NET JSON documents and the generated `AGENTS.md`. - `src/tocode/apk.py`: APK export pipeline (extraction, decompile pool, resource decoding, metadata); `apk_metadata.py`: manifest parsing and Android JSON documents; `apk_native.py`: extraction and background native export of every `.so`. @@ -59,6 +59,10 @@ When the IDA backend is used, the export also contains: - `.i64` or `.idb` +When the headless Binary Ninja backend is used, the export also contains: + +- `.bndb` + When `--tree` is passed, the export also contains: - `src/tree/**/*.c` @@ -73,7 +77,7 @@ APK input (`.apk`, `.apks`, `.xapk`) uses the ASC backend and writes instead: - `data/apk/**` (all other entries verbatim), `data/res/**/*.xml`, `data/resources.json` - `tocode.log`, generated `AGENTS.md` and `CLAUDE.md` -`base.apk` merges sibling `split_*.apk` files unless `--no-splits`; bundles are unpacked and merged. `--backend` selects the native backend for the `.so` exports (`binja` is rejected). +`base.apk` merges sibling `split_*.apk` files unless `--no-splits`; bundles are unpacked and merged. `--backend` selects the native backend for the `.so` exports (`binja` needs the headless API). .NET input (a managed PE, an apphost launcher with its `.dll`, a single-file bundle, or a `.nupkg`; detected by content, `--as-native` opts out) uses the .NET backend and writes instead: @@ -107,23 +111,61 @@ Framework/runtime assemblies and native runtime libraries inside bundles and pac - ASC (`droidasc`, PyPI) is the APK/DEX backend and a core runtime dependency. It pulls androguard. `backends/asc.py` is the only module that imports it; it pre-imports the modules ASC would otherwise stub in `sys.modules` and silences androguard's loguru logging. Class decompilation is not thread-safe, so it runs in spawned worker processes (recycled in rounds; do not use `max_tasks_per_child`, it deadlocks spawn pools on some CPython builds). The big per-method/per-class/per-string JSON documents are streamed row by row (`apk_metadata.write_json_rows`), never built as one object. - APK native libraries are exported by `apk_native.py` on a background thread, each `export_binary` in its own spawned process so a backend OOM-kill or crash only fails that library (`native-libs.json` status). The thread waits for `TOCODE_NATIVE_MIN_FREE_MB` (default 1024) of free memory before starting each library so it does not starve the DEX pool. `TOCODE_WORKER_TMP_DIR` also places the unpacked `.apks` bundle. - The .NET backend needs `pythonnet` (core dependency, pinned per Python version) and a .NET 9+ runtime on the host. Never commit or package DLLs: dnlib 4.5.0 and ICSharpCode.Decompiler 11.1 are downloaded from nuget.org by `backends/dotnet_libs.py` after the user agrees (asked once, remembered; `tocode --setup-dotnet` for non-interactive use). Their `.nupkg` and DLL SHA-256 hashes are pinned in `LIBRARIES` and checked at download time only (normal runs check presence; `--setup-dotnet` re-verifies). To bump a version, update both hashes. ICSharpCode.Decompiler 11 needs System.Reflection.Metadata 9, hence .NET 9+. Decompilation runs in spawned worker processes (recycled per round; a batch that crashes or hangs a worker is retried type by type so only the culprit fails). Tests must not require .NET: use `tests/dotnet_fixtures.py` (synthetic PE/bundles) and fake sessions; runtime-backed tests skip without it. -- Binary Ninja is an opt-in backend (`--backend binja`, never auto-selected). The - `binaryninja` module is supplied by the Binary Ninja install (in-UI) or the - remote VM, so it is not a pip dependency. `rpyc`, the client used for the - headless path via [binja-headless](https://github.com/hugsy/binja-headless), +- Binary Ninja comes after IDA in `auto` (IDA, the headless Binary Ninja API, + radare2, angr). The `binaryninja` module is supplied by the Binary Ninja + install (headless or in-UI) or the remote VM, so it is not a pip dependency. + `rpyc`, the client for [binja-headless](https://github.com/hugsy/binja-headless), is a core runtime dependency so the backend works out of the box. - Pin new dependencies exactly (`==`) and capture them in `uv.lock`. - Keep dependencies minimal and tied to exporting a project. ## Binary Ninja backend -- Headless (no enterprise license): run binja-headless inside a running Binary +Three transports (`--binja-transport auto|headless|mcp|rpyc`); `auto` tries them +in this order: headless when a file is given and the API passes its license +check, then MCP, then RPyc. + +- Headless (`backends/binja_headless.py`, a license that allows headless use): + `BinjaHeadlessSession` subclasses `BinjaSession` over a view this process + loaded, so it takes a file like IDA: folders, APK `.so` and .NET native code + work, and `.bndb` input always routes here (or to MCP/RPyc without the API). + `base.discover_binja_dir` finds the API (`--binja-dir`, `TOCODE_BINJA_DIR`, + `BN_INSTALL_DIR`, `install_api.py`'s `binaryninja.pth`, install paths); + `TOCODE_DISABLE_BINJA_HEADLESS=1` turns it off (tests set it). + `--binja-license` / `TOCODE_BINJA_LICENSE` names a license file that + `import_binaryninja` passes to `core_set_license` before the first core call + (the CLI exports the variable so workers and per-binary processes inherit it); + never log, print, or inspect license contents. The core + refuses a non-headless license on first use, not on import: + `headless_problem()` checks before choosing it, and `create_analyzer` retries + `auto` with binja skipped when opening fails. Analysis runs in `analyze()` and + is saved as `.v.bndb` in `TOCODE_BINJA_CACHE_DIR` (default + `~/.cache/tocode/binja`); bump `_BNDB_CACHE_VERSION` when the build changes. + An unreadable cache is rebuilt. Render workers (`worker_backend = + "binja-headless"`) each open that `.bndb`; the parent closes its view while + they run and `read_bytes` reopens it for `data/*.bin`. The database is + published into the export like the IDA `.i64`. `tests/fake_binja/` is a fake + API on disk so spawned workers can import it. +- MCP (Binary Ninja 6+): `backends/mcp_client.py` is a standard- + library MCP client (streamable HTTP, JSON or SSE responses); `backends/ + binja_mcp.py` maps the `bn_*` tools to the session interface. List tools return + paginated Markdown tables (front matter with `nextOffset`, `\|` escapes) and + decompile/disassembly return fenced text; keep parsing in `mcp_client.py`. The + server has one global active view: the session activates its view, re-checks it + every `ACTIVE_CHECK_EVERY` calls, and restores the user's view on close. Files + opened via `bn_open_item_open` stay open (the UI refuses to discard analysis + without its prompt). Section bytes come from `read_bytes` (`bn_memory_read`), + since the input may be a `.bndb`. `--binja-transport auto|mcp|rpyc`, + `--binja-mcp-url` / `TOCODE_BINJA_MCP_URL` (default `http://127.0.0.1:24642/mcp`). + Never call mutating `bn_*` tools from the exporter. +- RPyc (any license): run binja-headless inside a running Binary Ninja, then `tocode --backend binja --binja-host `. It exports an already **loaded** view (the binary positional is optional). Configure the connection with `--binja-host`/`--binja-port` or `TOCODE_BINJA_HOST` / `TOCODE_BINJA_PORT` (defaults `127.0.0.1:18812`). -- Choosing the view (headless): default is the focused view (the one binja-headless - exposes as `conn.root.bv`). `--list-binja` prints every open view with an index; +- Choosing the view (MCP/RPyc): MCP defaults to the active view, RPyc to the + focused view (the one binja-headless exposes as `conn.root.bv`). + `--list-binja` prints every open view with an index; `--binja-view N` exports that index; `--all-views` exports every open view, each into its own folder under `-o` (or the cwd). Active-context lookup is Qt-main- thread bound and unavailable over RPyc, so the focused view comes from @@ -133,10 +175,11 @@ Framework/runtime assemblies and native runtime libraries inside bundles and pac - In the Binary Ninja UI: call `tocode.export_from_binaryview(bv, out_dir)` from the scripting console (`bv` is the live view; `binaryninja` is imported locally). It runs the same export pipeline as the CLI. -- The backend renders serially against the live view (`parallel_safe = False` and - excluded from `exporter.TIMEOUT_WORKER_BACKENDS`): a `BinaryView` / RPyc - connection cannot be pickled into a worker process. Do not add it to the - spawned-worker path. +- UI, MCP, and RPyc render serially against the live view (`parallel_safe = + False`, and `binja` is excluded from `exporter.TIMEOUT_WORKER_BACKENDS`): a + live `BinaryView` / RPyc connection cannot be pickled into a worker process. + Do not add them to the spawned-worker path; only `binja-headless` workers, + which reopen the `.bndb`, belong there. ## Verification diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index b7664b2..ecd8a59 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -29,7 +29,7 @@ area you are changing: | IDA | IDA Pro 9 with idalib | `ida-domain`/`idapro` come with ToCode; IDA is found via `IDADIR`, `--idadir`, or common install paths. | | radare2 | `r2` plus r2ghidra (`r2pm -ci r2ghidra r2ghidra-sleigh`) | Used by `--backend r2` and as the `auto` fallback. | | angr | `uv sync --extra angr` | Large; tests in `tests/test_angr_backend.py` skip without it. | -| Binary Ninja | Binary Ninja with binja-headless | See the Binary Ninja section of the README. | +| Binary Ninja | A license that allows headless use (the API is found via `--binja-dir`, `TOCODE_BINJA_DIR`, or `install_api.py`'s `.pth`), or Binary Ninja 6+ with its MCP server on (binja-headless for older versions) | See the Binary Ninja section of the README. `tests/test_binja_headless.py` runs against a fake API in `tests/fake_binja/`, `tests/test_binja_mcp.py` against a fake MCP server. | | Android | nothing extra (`droidasc` is a core dependency) | Real APK test: set `TOCODE_TEST_APK=/path/to/app.apk`. | | .NET | a .NET 9+ runtime, then `uv run tocode --setup-dotnet` | The SDK (`dotnet-sdk-10.0`) lets you build test assemblies. Real test: `TOCODE_TEST_DOTNET=/path/to/App.dll`. | diff --git a/README.md b/README.md index a4c3a17..2be8e88 100644 --- a/README.md +++ b/README.md @@ -92,11 +92,12 @@ sample_decompiler/ ### Supported backends -Three backends are supported, selected with `--backend` (default `auto`, which prefers them in this order): +Four backends are supported, selected with `--backend` (default `auto`, which prefers them in this order): 1. **IDA** – uses the ida-domain/idapro Python libraries (best decompilation quality). -2. **radare2** – uses r2pipe with the r2ghidra decompiler. -3. **angr** – a pure-Python fallback with no external tooling, so ToCode still runs when neither IDA nor radare2 is available. It is an optional extra (it is large: ~450 MB of native dependencies), so it is not installed by default. Get it in any of these ways: +2. **Binary Ninja** – uses the Binary Ninja Python API when your license allows headless use. `--backend binja` can also export what you have open in a running Binary Ninja (any license). See [Binary Ninja](#binary-ninja) below. +3. **radare2** – uses r2pipe with the r2ghidra decompiler. +4. **angr** – a pure-Python fallback with no external tooling, so ToCode still runs when neither IDA nor radare2 is available. It is an optional extra (it is large: ~450 MB of native dependencies), so it is not installed by default. Get it in any of these ways: ```bash bash ./install.sh --all # recommended: installs every backend @@ -106,8 +107,6 @@ Three backends are supported, selected with `--backend` (default `auto`, which p The angr export is structurally identical to the others (same files and metadata); its pseudo-C is lower quality than Hex-Rays or r2ghidra. - 4. **Binary Ninja**, is opt-in (`--backend binja`) and never chosen by `auto`, because it drives a running Binary Ninja instead of reading a file. See [Binary Ninja](#binary-ninja) below. - Other disassemblers may be added in the future. ### .NET assemblies @@ -131,7 +130,7 @@ tocode Vendor.Lib.1.0.0.nupkg - `src/raw//.java`: one decompiled Java file per class, folders follow Java packages (no clustering, no summaries). - `AndroidManifest.xml` + `manifest.json`: decoded and parsed manifest (permissions, components with intent filters and exported state, application attributes, split manifests). - `classes.json`, `functions.json`, `function-index.json`, `strings.json`, `imports.json`, `exports.json` (exported components + JNI methods), `reachable.json` (from manifest components), `package-graph.json`, `sections.json`, `triage.json`. -- `lib//*.so`: every native library extracted; `native///`: a complete nested ToCode export per library (all ABIs), produced on a background thread while the DEX side decompiles. `native-libs.json` records the status of each (each library runs in its own process; one failing or being OOM-killed never fails the APK export, and the native thread waits for `TOCODE_NATIVE_MIN_FREE_MB`, default 1024 MB, of free memory before each library). Pass `--no-native` to skip the native decompilation (libraries are still extracted). `--backend` picks the native backend (`auto`/`ida`/`r2`/`angr`; `binja` is not supported for APKs). +- `lib//*.so`: every native library extracted; `native///`: a complete nested ToCode export per library (all ABIs), produced on a background thread while the DEX side decompiles. `native-libs.json` records the status of each (each library runs in its own process; one failing or being OOM-killed never fails the APK export, and the native thread waits for `TOCODE_NATIVE_MIN_FREE_MB`, default 1024 MB, of free memory before each library). Pass `--no-native` to skip the native decompilation (libraries are still extracted). `--backend` picks the native backend (`auto`/`ida`/`binja`/`r2`/`angr`; `binja` needs the headless API here). - `data/apk/**`: every other APK entry verbatim; `data/res/**/*.xml` and `data/resources.json`: decoded binary XML and `resources.arsc`. `base.apk` automatically merges sibling `split_*.apk` files (config and ABI splits) into the same project; `--no-splits` exports it alone. The default output directory is `_decompiler`. @@ -229,9 +228,39 @@ tocode firmwareX.bin.i64 -o firmwareX_decompiled/ ### Binary Ninja -The Binary Ninja backend exports the binary you already have **open** in Binary Ninja, emitting its Pseudo C. It does **not** need a Binary Ninja *headless* license: ToCode connects to the running GUI over a small RPC bridge. +The Binary Ninja backend uses Binary Ninja's own analysis and Pseudo C. ToCode reaches Binary Ninja in one of three ways, tried in this order (`--binja-transport headless`, `mcp` or `rpyc` forces one): + +1. **Headless**: the Binary Ninja Python API runs inside ToCode. Needs a license that allows headless use. +2. **MCP**: Binary Ninja 6's built-in MCP server exports what you have open in the UI. Any license. +3. **RPC bridge**: a small server in Binary Ninja's Python console, for older versions. Any license. + +**Headless.** With a license that allows headless use, Binary Ninja works like the other backends: it takes a file, and `auto` picks it when IDA isn't available. + +```bash +tocode ./program --backend binja -o out/ +tocode ./program.bndb -o out/ # a Binary Ninja database always goes to Binary Ninja +tocode ./firmware/ --backend binja # folders, APK native libraries, and .NET native code too +``` + +ToCode finds the API through `--binja-dir` or `TOCODE_BINJA_DIR` (the install folder or its `python/` folder), `BN_INSTALL_DIR`, the `binaryninja.pth` written by Binary Ninja's `scripts/install_api.py`, or the default install locations. The finished analysis is cached as a `.bndb` (in `~/.cache/tocode/binja`, or `TOCODE_BINJA_CACHE_DIR`), so a re-run and every render worker open it instead of analyzing again; `--purge-cache` rebuilds it. The export includes it as `.bndb`, like IDA's `.i64`. Each worker loads Binary Ninja and the database, so free memory caps the worker count (`TOCODE_BINJA_WORKER_MEMORY_MB`, default 1024 per worker). + +If the license refuses headless use, ToCode moves on: `--backend binja` to a running Binary Ninja, `auto` to radare2 or angr. `TOCODE_DISABLE_BINJA_HEADLESS=1` skips the API altogether. + +**MCP (Binary Ninja 6 and newer).** Turn on the MCP server in Binary Ninja, then: -Since I don't have access to Binary Ninja headless, run this script once in Binary Ninja's Python console to open an RPC server ToCode can connect to: +```bash +tocode --backend binja --list-binja # open views, with an index +tocode --backend binja -o out/ # export the active view +tocode --backend binja --binja-view 1 -o out/ # export one view by index +tocode --backend binja --all-views -o out/ # every open view, one folder each +tocode --backend binja ./program -o out/ # export that file (opened in Binary Ninja if it isn't) +``` + +With a headless license, a file argument is exported headless; add `--binja-transport mcp` to export it through the UI instead. + +ToCode connects to `http://127.0.0.1:24642/mcp` by default; set `--binja-mcp-url` or `TOCODE_BINJA_MCP_URL` to change it. MCP has one active view shared with the UI. ToCode switches it to the view it exports and switches it back when done. A file ToCode opens for you stays open in Binary Ninja, because Binary Ninja only closes analyzed files through its save prompt. Section data (`data/*.bin`) is read from Binary Ninja's view, so exporting a `.bndb` works. + +**RPC bridge (older versions).** If the MCP server doesn't answer, ToCode falls back to the bridge automatically. Run this once in Binary Ninja's Python console to start the bridge: ```python import threading, rpyc, binaryninja @@ -256,16 +285,7 @@ print("ToCode RPC server listening on 127.0.0.1:18812") It exposes the whole Binary Ninja Python VM with no authentication, so keep it bound to `127.0.0.1`. The [binja-headless](https://github.com/hugsy/binja-headless) plugin exposes the same interface if you prefer a packaged option. -Then drive it from ToCode: - -```bash -tocode --backend binja --list-binja # list open views with an index -tocode --backend binja -o out/ # export the focused view -tocode --backend binja --binja-view 1 -o out/ # export a specific view by index -tocode --backend binja --all-views -o out/ # export every open view, one folder each -``` - -Use `--binja-host` / `--binja-port` (or `TOCODE_BINJA_HOST` / `TOCODE_BINJA_PORT`; default `127.0.0.1:18812`) to reach Binary Ninja on another machine. +The same commands work over the bridge (the default view is the focused one). Use `--binja-host` / `--binja-port` (or `TOCODE_BINJA_HOST` / `TOCODE_BINJA_PORT`; default `127.0.0.1:18812`) to reach Binary Ninja on another machine. Already scripting inside Binary Ninja? Skip the server and export the live view directly: diff --git a/pyproject.toml b/pyproject.toml index c353bf5..ae7bf4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,8 +72,8 @@ angr = [ ] # The Binary Ninja backend's headless client (rpyc) is a core runtime dependency # above so `tocode --backend binja` works out of the box. The binaryninja module -# itself is provided by the Binary Ninja install (in-UI) or the remote VM via -# binja-headless, so it is intentionally not a pip dependency. +# itself is provided by the Binary Ninja install (headless or in-UI) or the +# remote VM via binja-headless, so it is intentionally not a pip dependency. # angr is an optional backend dependency, so it may be absent from the type # checking environment. Ignore its missing stubs/imports rather than failing. diff --git a/src/tocode/analysis.py b/src/tocode/analysis.py index 6063575..c7c9f84 100644 --- a/src/tocode/analysis.py +++ b/src/tocode/analysis.py @@ -4,7 +4,14 @@ import time from typing import Any -from .backends.base import BackendRequest, DecompilerSession, choose_backend +from .backends.base import ( + BackendRequest, + DecompilerSession, + choose_backend, + is_binja_database, + probe_binja_headless, +) +from .errors import BackendError from .backends.ida import IdaSession from .backends.r2 import R2Session from .cluster import describe_imports @@ -472,19 +479,49 @@ def create_analyzer( binja_port: int = 18812, binja_bv: Any = None, binja_bn: Any = None, + session: DecompilerSession | None = None, + binja_dir: Path | None = None, ) -> BinaryAnalyzer: - choice = choose_backend( - backend, - input_path=binary, - idadir=idadir, - ida_domain_path=ida_domain_path, - ) - if progress is not None: - progress.log(f"Using {choice.selected.upper()} as backend.") - # Log before constructing the session: opening (and, for a fresh binary, - # loading) the IDA database happens inside the session constructor and can - # take a while, so the user should see activity instead of a silent gap. - progress.log(f"Loading {Path(binary).resolve()}") + skip: frozenset[str] = frozenset() + while True: + choice = choose_backend( + backend, + input_path=binary, + idadir=idadir, + ida_domain_path=ida_domain_path, + binja_dir=binja_dir, + skip=skip, + ) + if progress is not None: + progress.log(f"Using {choice.selected.upper()} as backend.") + # Log before constructing the session: opening (and, for a fresh + # binary, loading) a database happens inside the session constructor + # and can take a while, so the user sees activity instead of a gap. + progress.log(f"Loading {Path(binary).resolve()}") + if choice.selected != "binja" or session is not None or binja_bv is not None: + break + if backend == "binja" and not probe_binja_headless(binja_dir): + break # explicit binja without the local API: binja-headless (RPyc) + from .backends.binja_headless import BinjaHeadlessSession + + try: + headless = BinjaHeadlessSession( + binary, + binja_dir=binja_dir, + purge_cache=purge_cache, + progress=progress, + ) + except BackendError as exc: + if choice.requested != "auto" or is_binja_database(Path(binary)): + raise + if progress is not None: + progress.log( + f"Binary Ninja headless unavailable ({exc}); trying the next backend" + ) + skip = skip | {"binja"} + continue + return BinaryAnalyzer(binary, session=headless, progress=progress) + if choice.selected == "ida": return BinaryAnalyzer( binary, @@ -506,17 +543,22 @@ def create_analyzer( session=AngrSession(binary), progress=progress, ) + if choice.selected == "binja" and session is not None: + # A session built by the caller (Binary Ninja over MCP). + return BinaryAnalyzer(binary, session=session, progress=progress) if choice.selected == "binja": # Imported lazily so non-binja exports never need the binaryninja/rpyc # stack. With an injected view (in the Binary Ninja UI) the session wraps - # it directly; otherwise connect headless via binja-headless (RPyc). + # it directly; otherwise connect to binja-headless over RPyc. from .backends.binja import BinjaSession if binja_bv is not None: - session: DecompilerSession = BinjaSession(bv=binja_bv, bn=binja_bn) + binja_session: DecompilerSession = BinjaSession(bv=binja_bv, bn=binja_bn) else: - session = BinjaSession.connect_headless(binja_host, binja_port, path=binary) - return BinaryAnalyzer(binary, session=session, progress=progress) + binja_session = BinjaSession.connect_headless( + binja_host, binja_port, path=binary + ) + return BinaryAnalyzer(binary, session=binja_session, progress=progress) return BinaryAnalyzer( binary, session=R2Session(binary, analysis_command=analysis_command), diff --git a/src/tocode/apk_native.py b/src/tocode/apk_native.py index 2dc5843..c612920 100644 --- a/src/tocode/apk_native.py +++ b/src/tocode/apk_native.py @@ -106,6 +106,7 @@ class NativeOptions: tree: bool = False entropy: bool = False restart: bool = False + binja_dir: Path | None = None def _rel(path: Path | None, root: Path) -> str | None: @@ -188,6 +189,7 @@ def run_native_export( idadir=options.idadir, ida_domain_path=options.ida_domain_path, purge_cache=options.purge_cache, + binja_dir=options.binja_dir, ) as analyzer: summary = export_binary( analyzer, diff --git a/src/tocode/backends/base.py b/src/tocode/backends/base.py index 0624d4f..466dc8d 100644 --- a/src/tocode/backends/base.py +++ b/src/tocode/backends/base.py @@ -100,19 +100,24 @@ def choose_backend( input_path: Path | None = None, idadir: Path | None = None, ida_domain_path: Path | None = None, + binja_dir: Path | None = None, + skip: frozenset[str] = frozenset(), ) -> BackendChoice: if input_path is not None and is_ida_database(input_path): if requested in {"r2", "angr", "binja"}: raise ToCodeError("IDA database input requires the IDA backend") requested = "ida" + if input_path is not None and is_binja_database(input_path): + if requested in {"r2", "angr", "ida"}: + raise ToCodeError( + "Binary Ninja database (.bndb) input requires the binja backend" + ) + requested = "binja" if requested == "binja": - # binja is satisfied either by a local ``binaryninja`` (running inside the - # Binary Ninja UI) or by an RPyc connection to binja-headless. Neither is - # checked here: the in-UI session is constructed with an injected view, - # and the headless connection's real failure surfaces with a clear - # message at connect time. ``auto`` never picks binja -- it requires a - # running Binary Ninja, so it stays opt-in. + # Satisfied by the local Binary Ninja API (headless, chosen in + # create_analyzer when a file is given), by a view injected from the + # Binary Ninja UI, or by the MCP/RPyc routes the CLI drives itself. return BackendChoice(requested, "binja", "selected by CLI") if requested == "r2": @@ -137,9 +142,18 @@ def choose_backend( raise ToCodeError(f"IDA backend requested but unavailable: {probe.reason}") return BackendChoice(requested, "ida", probe.reason) - # auto: prefer IDA, then radare2, then angr as a last resort. + # auto: prefer IDA, then the Binary Ninja API, then radare2, then angr as a + # last resort. A Binary Ninja license that does not allow headless use is + # only detected when the file is opened; create_analyzer then retries with + # binja in ``skip``. if probe.available: return BackendChoice(requested, "ida", probe.reason) + if "binja" not in skip and probe_binja_headless(binja_dir): + return BackendChoice( + requested, + "binja", + f"IDA unavailable ({probe.reason}); using the Binary Ninja Python API", + ) if probe_r2(): return BackendChoice( requested, "r2", f"IDA unavailable ({probe.reason}); using r2" @@ -169,8 +183,86 @@ def probe_angr() -> bool: def probe_binja() -> bool: # Advisory only: true when running inside (or alongside) a Binary Ninja that - # exposes the ``binaryninja`` module locally. The headless path needs ``rpyc`` - # instead, so binja selection is explicit rather than auto-probed. + # exposes the ``binaryninja`` module locally. + return importlib.util.find_spec("binaryninja") is not None + + +def discover_binja_dir(explicit: Path | None = None) -> Path | None: + """Binary Ninja's ``python`` folder (holding the ``binaryninja`` package).""" + candidates: list[Path] = [] + if explicit is not None: + candidates.append(explicit.expanduser()) + for env_name in ("TOCODE_BINJA_DIR", "BN_INSTALL_DIR", "BINJA_DIR"): + value = os.environ.get(env_name, "").strip() + if value: + candidates.append(Path(value).expanduser()) + # Binary Ninja's own install_api.py writes the path into a .pth file in the + # user site-packages of the Python it was run with; reuse it from any Python. + candidates.extend(_binaryninja_pth_paths()) + home = Path.home() + candidates.extend( + [ + home / "binaryninja", + home / "Applications" / "Binary Ninja.app" / "Contents" / "Resources", + Path("/Applications/Binary Ninja.app/Contents/Resources"), + Path("/opt/binaryninja"), + ] + ) + for env_name in ("ProgramFiles", "LOCALAPPDATA"): + root = os.environ.get(env_name) + if root: + candidates.append(Path(root) / "Vector35" / "BinaryNinja") + for candidate in candidates: + for python_dir in (candidate, candidate / "python"): + if (python_dir / "binaryninja" / "__init__.py").is_file(): + return python_dir.resolve() + return None + + +def _binaryninja_pth_paths() -> list[Path]: + found: list[Path] = [] + roots: list[Path] = [] + for pattern in ( + ".local/lib/python3*/site-packages", + "Library/Python/3*/lib/python/site-packages", + ): + roots.extend(Path.home().glob(pattern)) + appdata = os.environ.get("APPDATA") + if appdata: + roots.extend(Path(appdata).glob("Python/Python3*/site-packages")) + for root in roots: + pth = root / "binaryninja.pth" + try: + lines = pth.read_text(encoding="utf-8").splitlines() + except OSError: + continue + found.extend(Path(line.strip()) for line in lines if line.strip()) + return found + + +def bootstrap_binja(explicit: Path | None = None) -> Path | None: + """Make ``import binaryninja`` work; returns the python folder used.""" + if importlib.util.find_spec("binaryninja") is not None: + return None + python_dir = discover_binja_dir(explicit) + if python_dir is not None: + _prepend(python_dir) + return python_dir + + +def probe_binja_headless(explicit: Path | None = None) -> bool: + """True when the Binary Ninja Python API can be imported (cheap: no import). + + Whether the license allows headless use is only known when a file is + opened; ``auto`` then falls back to the next backend. + """ + if os.environ.get("TOCODE_DISABLE_BINJA_HEADLESS", "").strip() in ( + "1", + "true", + "yes", + ): + return False + bootstrap_binja(explicit) return importlib.util.find_spec("binaryninja") is not None @@ -178,6 +270,10 @@ def is_ida_database(path: Path) -> bool: return path.suffix.lower() in IDA_DATABASE_SUFFIXES +def is_binja_database(path: Path) -> bool: + return path.suffix.lower() == ".bndb" + + @dataclass(slots=True) class IdaRuntime: idapro: Any diff --git a/src/tocode/backends/binja.py b/src/tocode/backends/binja.py index 577940d..1835980 100644 --- a/src/tocode/backends/binja.py +++ b/src/tocode/backends/binja.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +import re from typing import Any from .base import BackendName @@ -24,6 +25,26 @@ ) +_GUTTER_RX = re.compile(r"^[^\x00-\x7f]*\s*") +_GUTTER_GLYPH_RX = re.compile(r"^[^\x00-\x7f]+") + + +def strip_gutter(body: str) -> str: + """Drop Binary Ninja's UI gutter: the indent or marker glyph before the + header line, and marker glyphs (such as an unresolved jump's) before body + lines, which keep their indentation.""" + lines = body.split("\n") + header_seen = False + for index, line in enumerate(lines): + if not header_seen: + if line.strip(): + lines[index] = _GUTTER_RX.sub("", line, count=1) + header_seen = True + continue + lines[index] = _GUTTER_GLYPH_RX.sub("", line, count=1) + return "\n".join(lines) + + # -- headless (binja-headless / RPyc) helpers ------------------------------ # # binja-headless exposes ``conn.root.binaryninja`` (the module), ``conn.root.bv`` @@ -548,6 +569,15 @@ def calls_from( imported.add(name) return sorted(edges), sorted(imported) + def read_bytes(self, address: int, size: int) -> bytes | None: + """Section bytes from the view (the input may be a .bndb, and sections + are addressed by virtual address here, not by file offset).""" + try: + data = self._bv.read(int(address), int(size)) + except Exception: # noqa: BLE001 + return None + return bytes(data) if data else None + def data_xrefs(self, addresses) -> dict[int, list[tuple[int, bool]]]: # Data cross-references ship as an empty stub for v1. return {} @@ -752,7 +782,7 @@ def _render_pseudo_c(self, func: Any, address: int) -> str: proto = self._signature(func) or f"int sub_{address:x}()" return f"{proto}\n{{\n /* Binary Ninja render failed: {exc} */\n}}\n" - text = "\n".join(body).strip("\n") + text = strip_gutter("\n".join(body)).strip("\n") if not text: proto = self._signature(func) or f"int sub_{address:x}()" return f"{proto}\n{{\n}}\n" diff --git a/src/tocode/backends/binja_headless.py b/src/tocode/backends/binja_headless.py new file mode 100644 index 0000000..d8ccb74 --- /dev/null +++ b/src/tocode/backends/binja_headless.py @@ -0,0 +1,524 @@ +"""Binary Ninja headless backend (the ``binaryninja`` Python API, in process). + +Like the IDA backend it takes a file: a binary, or a Binary Ninja database +(``.bndb``). Analysis of a binary is cached as a ``.bndb`` keyed by the file's +SHA-256, so re-runs and parallel workers open the finished analysis instead of +repeating it. A ``.bndb`` given as input is opened as-is and never written. + +The rendering is shared with the UI/RPyc backend (``BinjaSession``); this +session adds what is cheap only when the API is local: section bytes, full C +type definitions, and data cross-references with read/write direction. +""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.util +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import threading +from typing import Any + +from ..errors import BackendError +from ..progress import Progress +from .base import bootstrap_binja, is_binja_database +from .binja import BinjaSession + + +# Bump when a change to how the cached database is built makes old ones stale. +_BNDB_CACHE_VERSION = 1 +MAX_TYPES = 50_000 +MAX_REFS_PER_ADDRESS = 256 +ANALYSIS_POLL_SECONDS = 0.5 +# How long an aborted analysis may take to stop before the view is closed. +ABORT_WAIT_SECONDS = 10.0 +SAVE_PROBE_TIMEOUT_SECONDS = 180.0 + + +def cache_root() -> Path: + explicit = os.environ.get("TOCODE_BINJA_CACHE_DIR", "").strip() + if explicit: + return Path(explicit).expanduser() + xdg = os.environ.get("XDG_CACHE_HOME", "").strip() + base = Path(xdg).expanduser() if xdg else Path.home() / ".cache" + return base / "tocode" / "binja" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def cached_database(binary: Path) -> Path: + return cache_root() / f"{_sha256(binary)}.v{_BNDB_CACHE_VERSION}.bndb" + + +def license_path() -> Path | None: + """The license file named by ``--binja-license`` / ``TOCODE_BINJA_LICENSE``. + + Without one, Binary Ninja reads its own (``license.dat`` in its user folder). + """ + value = os.environ.get("TOCODE_BINJA_LICENSE", "").strip() + return Path(value).expanduser() if value else None + + +def _apply_license(bn: Any) -> None: + """Hand the license file to the core before its first call validates one. + + The contents go straight to ``core_set_license``; they are never logged. + Spawned workers inherit ``TOCODE_BINJA_LICENSE`` and apply it the same way. + """ + path = license_path() + if path is None: + return + try: + data = path.read_text(encoding="utf-8") + except OSError as exc: + raise BackendError( + f"cannot read the Binary Ninja license file {path}: {exc.strerror or exc}" + ) from exc + set_license = getattr(bn, "core_set_license", None) + if not callable(set_license): + raise BackendError( + "this Binary Ninja API cannot take a license file (no core_set_license)" + ) + set_license(data) + + +def import_binaryninja(binja_dir: Path | None = None) -> Any: + bootstrap_binja(binja_dir) + already_imported = "binaryninja" in sys.modules + if importlib.util.find_spec("binaryninja") is None: + raise BackendError( + "the Binary Ninja Python API was not found; pass --binja-dir or set " + "TOCODE_BINJA_DIR to the Binary Ninja install folder" + ) + try: + bn = importlib.import_module("binaryninja") + except Exception as exc: # noqa: BLE001 - core load failures are not ImportError + raise BackendError(f"cannot import the Binary Ninja Python API: {exc}") from exc + if not already_imported: + # The core validates a license on its first call, not on import. + _apply_license(bn) + # Headless Binary Ninja logs analysis warnings to the terminal; failures + # reach ToCode as exceptions, so keep the terminal to ToCode's own output. + disable_log = getattr(bn, "disable_default_log", None) + if callable(disable_log): + try: + disable_log() + except Exception: # noqa: BLE001 + pass + return bn + + +def headless_problem(binja_dir: Path | None = None) -> str | None: + """Why this process cannot use the Binary Ninja API, or None when it can. + + Imports the API and initializes the core, which is where a license that + does not allow headless use is refused (before any file is opened). + """ + try: + bn = import_binaryninja(binja_dir) + except BackendError as exc: + return str(exc) + check = getattr(bn, "core_product", None) + if not callable(check): + return None + try: + check() + except Exception as exc: # noqa: BLE001 + return f"Binary Ninja refused headless use: {exc}" + return None + + +# Run in a child process: a core that cannot write databases may crash in +# BNCreateDatabase instead of returning False. +_SAVE_PROBE = """ +import sys +from pathlib import Path +from tocode.backends.binja_headless import import_binaryninja + +work = Path(sys.argv[2]) +bn = import_binaryninja(Path(sys.argv[1]) if sys.argv[1] else None) +sample = work / "probe.bin" +sample.write_bytes(bytes(64)) +view = bn.load(str(sample), update_analysis=False) +print("saved" if view.create_database(str(work / "probe.bndb")) else "refused") +""" + + +def _save_probe_marker(bn: Any) -> Path: + try: + identity = f"{bn.core_version()}-{bn.core_product()}" + except Exception: # noqa: BLE001 + identity = "unknown" + return cache_root() / f"save-probe-{re.sub(r'[^A-Za-z0-9.]+', '-', identity)}.txt" + + +def can_create_databases(bn: Any, binja_dir: Path | None = None) -> bool: + """Whether this Binary Ninja can write a ``.bndb`` without crashing. + + Some headless builds crash inside ``create_database`` (Binary Ninja 6.0 + Personal on Linux did on every save), which would take ToCode down with + it. A child process tries once per core version and product; the answer + is kept next to the database cache. + """ + marker = _save_probe_marker(bn) + try: + cached = marker.read_text(encoding="utf-8").strip() + except OSError: + cached = "" + if cached in ("ok", "fail"): + return cached == "ok" + with tempfile.TemporaryDirectory(prefix="tocode-binja-probe-") as work: + try: + result = subprocess.run( + [sys.executable, "-c", _SAVE_PROBE, str(binja_dir or ""), work], + capture_output=True, + text=True, + timeout=SAVE_PROBE_TIMEOUT_SECONDS, + ) + works = ( + result.returncode == 0 + and result.stdout.strip().endswith("saved") + and (Path(work) / "probe.bndb").is_file() + ) + except (OSError, subprocess.TimeoutExpired): + works = False + try: + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("ok" if works else "fail", encoding="utf-8") + except OSError: + pass + return works + + +def _peak_rss_mb() -> int | None: + """This process's peak resident memory, in MB (None where unsupported).""" + try: + import resource + except ImportError: # Windows + return None + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # Linux reports KiB, macOS bytes. + return int(peak // (1024 * 1024) if sys.platform == "darwin" else peak // 1024) + + +def _show_analysis_progress(bar: Any, bv: Any) -> None: + try: + status = bv.analysis_progress + count, total = int(status.count), int(status.total) + phase = str(status).split(" (", 1)[0] + except Exception: # noqa: BLE001 - progress display is best effort + return + bar.set_description_str(f"binja {phase.lower()}", refresh=False) + bar.total = total if total > 0 else None + bar.n = min(count, total) if total > 0 else 0 + bar.refresh() + + +def _load(bn: Any, source: Path, *, update_analysis: bool) -> Any: + try: + bv = bn.load(str(source), update_analysis=update_analysis) + except Exception as exc: # noqa: BLE001 + hint = "" + if "licen" in str(exc).lower(): + hint = " (headless use needs a Binary Ninja license that allows it)" + raise BackendError( + f"Binary Ninja could not open {source}: {exc}{hint}" + ) from exc + if bv is None: + raise BackendError(f"Binary Ninja could not open {source}") + if is_binja_database(source) and not list(getattr(bv, "segments", None) or ()): + # A save that crashed leaves an empty file, which Binary Ninja opens as + # a view with nothing mapped (and no functions) instead of failing. + try: + bv.file.close() + except Exception: # noqa: BLE001 + pass + raise BackendError( + f"Binary Ninja database {source} is empty or incomplete (no segments)" + ) + return bv + + +class BinjaHeadlessSession(BinjaSession): + """``BinjaSession`` over a view this process opened with ``binaryninja.load``.""" + + backend_label = "Binary Ninja (headless)" + # Workers reopen the cached .bndb in their own process. + parallel_safe = True + worker_backend = "binja-headless" + + def __init__( + self, + binary: Path, + *, + binja_dir: Path | None = None, + db_path: Path | None = None, + purge_cache: bool = False, + save_cache: bool = True, + progress: Progress | None = None, + ) -> None: + self.binary = Path(binary).resolve() + self.binja_dir = binja_dir + bn = import_binaryninja(binja_dir) + self._progress = progress + self._log = progress.log if progress is not None else (lambda _message: None) + self._worker_memory_mb: int | None = None + from_cache = False + if db_path is not None: + source = Path(db_path) + self._cache_db: Path | None = source + fresh = False + elif is_binja_database(self.binary): + source = self.binary + self._cache_db = self.binary + fresh = False + else: + cache = cached_database(self.binary) if save_cache else None + if cache is not None and purge_cache: + cache.unlink(missing_ok=True) + if cache is not None and cache.is_file(): + source, fresh, from_cache = cache, False, True + self._log(f"Opening cached Binary Ninja database {cache}") + else: + source, fresh = self.binary, True + self._cache_db = cache + # Analysis runs in analyze(), under the exporter's progress bar. + try: + bv = _load(bn, source, update_analysis=False) + except BackendError: + if not from_cache: + raise + # A cached database this Binary Ninja cannot read (an interrupted + # save, or one written by another version): rebuild it from the + # binary, keeping it if the binary cannot be opened either. + bv = _load(bn, self.binary, update_analysis=False) + self._log(f"Discarding unreadable cached Binary Ninja database {source}") + source.unlink(missing_ok=True) + fresh = True + super().__init__(bv=bv, bn=bn) + self._released = False + self._fresh = fresh + self._original = Path(str(bv.file.original_filename or self.binary)) + + # -- lifecycle ---------------------------------------------------------- + + def analyze(self) -> None: + self._wait_for_analysis() + if self._fresh and self._cache_db is not None: + # Keep the finished analysis for re-runs and for parallel workers. + if not can_create_databases(self._bn, self.binja_dir): + self._log( + "This Binary Ninja cannot save databases headless; rendering " + "in this process without a .bndb" + ) + self._render_without_database() + else: + self._save_database(self._cache_db) + self._fresh = False + # A worker that reopens the database holds about what this process + # holds now; the exporter sizes the worker count with it. + self._worker_memory_mb = _peak_rss_mb() + + def _save_database(self, target: Path) -> None: + self._log(f"Saving Binary Ninja database {target}") + try: + target.parent.mkdir(parents=True, exist_ok=True) + saved = bool(self._bv.create_database(str(target))) + except Exception: # noqa: BLE001 - a cache is an optimisation + saved = False + if saved: + self._log(f"Saved Binary Ninja database {target}") + return + self._log("Could not save the Binary Ninja database; rendering in this process") + try: + target.unlink(missing_ok=True) + except OSError: + pass + self._render_without_database() + + def _render_without_database(self) -> None: + """Without a .bndb, workers would each repeat the analysis: render + serially against this view instead, like the UI/MCP/RPyc sessions.""" + self._cache_db = None + self.parallel_safe = False + self.worker_backend = "binja" + + def _wait_for_analysis(self) -> None: + """``update_analysis_and_wait`` with Binary Ninja's own progress shown. + + A large binary analyzes for minutes inside one native call. It runs on + a thread so this one can show ``analysis_progress`` and, on Ctrl-C, + abort the analysis instead of waiting for the call to return. + """ + done = threading.Event() + failure: list[BaseException] = [] + + def run() -> None: + try: + BinjaSession.analyze(self) + except BaseException as exc: # noqa: BLE001 - re-raised below + failure.append(exc) + finally: + done.set() + + threading.Thread(target=run, name="tocode-binja-analysis", daemon=True).start() + bar = ( + self._progress.new_bar(desc="binja analysis", unit="fn") + if self._progress is not None + else None + ) + try: + while not done.wait(ANALYSIS_POLL_SECONDS): + if bar is not None: + _show_analysis_progress(bar, self._bv) + except KeyboardInterrupt: + abort = getattr(self._bv, "abort_analysis", None) + if callable(abort): + try: + abort() + except Exception: # noqa: BLE001 + pass + done.wait(ABORT_WAIT_SECONDS) + raise + finally: + if bar is not None: + bar.close() + if failure: + raise failure[0] + + def worker_memory_mb(self) -> int | None: + """Memory one render worker needs: this process's peak after analysis.""" + return self._worker_memory_mb + + def close(self) -> None: + bv = self._bv + super().close() + if self._released: + return + try: + bv.file.close() + except Exception: # noqa: BLE001 + pass + + def prepare_parallel_workers(self) -> None: + return None # analyze() already saved the database workers open + + def release_parallel_resources(self) -> None: + """Close the parent's view while workers render (each loads its own).""" + if self._released: + return + self._disasm_cache.clear() + self._decompile_cache.clear() + try: + self._bv.file.close() + except Exception: # noqa: BLE001 + pass + self._released = True + + def restore_parallel_resources(self) -> None: + if not self._released: + return + source = self._cache_db or self.binary + self._bv = _load(self._bn, source, update_analysis=True) + self._released = False + + def read_bytes(self, address: int, size: int) -> bytes | None: + if self._released: + # Workers rendered with their own views and the parent's view was + # closed; reopen it for the section data written after rendering. + self.restore_parallel_resources() + return super().read_bytes(address, size) + + def database_path(self) -> Path | None: + """The .bndb workers open (the input database or the analysis cache).""" + return self._cache_db + + def _input_path(self) -> str: + return str( + self.binary if not is_binja_database(self.binary) else self._original + ) + + # -- local-only enrichments ----------------------------------------------- + + def types(self) -> list[dict[str, Any]]: + bn = self._bn + try: + from binaryninja.typeprinter import TypePrinter # type: ignore[import-not-found] + + printer = TypePrinter.default + container = self._bv.type_container + except Exception: # noqa: BLE001 + return [] + # Plain C names in the header, not Binary Ninja's backtick escaping. + escaping = getattr( + getattr(bn, "TokenEscapingType", None), "NoTokenEscapingType", None + ) + options = {"escaping": escaping} if escaping is not None else {} + kinds = { + "StructureTypeClass": "struct", + "EnumerationTypeClass": "enum", + "NamedTypeReferenceClass": "typedef", + "FunctionTypeClass": "func", + } + rows: list[dict[str, Any]] = [] + for index, (name, type_obj) in enumerate(self._bv.types.items()): + if index >= MAX_TYPES: + break + try: + lines = printer.get_type_lines( + type_obj, container, bn.QualifiedName(name), **options + ) + c_decl = "\n".join(str(line) for line in lines) + except Exception: # noqa: BLE001 + c_decl = str(type_obj) + type_class = str(getattr(type_obj.type_class, "name", type_obj.type_class)) + kind = kinds.get(type_class, "typedef") + if kind == "struct" and getattr(type_obj, "type", None) is not None: + if "Union" in str(getattr(type_obj, "type", "")): + kind = "union" + width = int(getattr(type_obj, "width", 0) or 0) + rows.append( + { + "name": str(name), + "kind": kind, + "size": width or None, + "c_decl": c_decl, + } + ) + return rows + + def data_xrefs(self, addresses: Any) -> dict[int, list[tuple[int, bool]]]: + store = getattr(self._bn.LowLevelILOperation, "LLIL_STORE", None) + result: dict[int, list[tuple[int, bool]]] = {} + for address in addresses: + target = int(address) + refs: list[tuple[int, bool]] = [] + try: + for ref in self._bv.get_code_refs(target): + if len(refs) >= MAX_REFS_PER_ADDRESS: + break + write = False + function = getattr(ref, "function", None) + if function is not None and store is not None: + try: + llil = function.get_llil_at(ref.address) + write = llil is not None and llil.operation == store + except Exception: # noqa: BLE001 + write = False + refs.append((int(ref.address), write)) + except Exception: # noqa: BLE001 + continue + if refs: + result[target] = refs + return result diff --git a/src/tocode/backends/binja_mcp.py b/src/tocode/backends/binja_mcp.py new file mode 100644 index 0000000..b1f9664 --- /dev/null +++ b/src/tocode/backends/binja_mcp.py @@ -0,0 +1,764 @@ +"""Binary Ninja backend over Binary Ninja's built-in MCP server (6.0+). + +Binary Ninja 6 ships an MCP server inside the UI (default +``http://127.0.0.1:24642/mcp``). This session drives it with plain tool calls: +the analysis, decompilation, and naming are Binary Ninja's own, nothing runs +inside the Binary Ninja process except its tools, and no extra Python package +is needed (the client is standard library only). + +The server has one *active* binary view, shared with the user's UI and any +other MCP client. A session activates the view it exports, checks every few +hundred calls that it is still active (re-activating it if someone else +switched), and restores the previously active view when it closes. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +import re +from typing import Any, Iterator + +from ..errors import BackendError +from .base import BackendName +from .binja import strip_gutter as _strip_gutter +from .mcp_client import ( + McpClient, + ToolResult, + fenced_body, + front_matter, + markdown_tables, +) + +DEFAULT_MCP_URL = "http://127.0.0.1:24642/mcp" +PAGE = 1000 +ACTIVE_CHECK_EVERY = 256 +PROBE_TIMEOUT_SECONDS = 2.0 +MAX_STRING_BYTES = 65536 +MAX_TYPES = 50_000 +_INTERNAL_NAMESPACE = "BNINTERNALNAMESPACE" +_DISASM_RX = re.compile(r"^(0x[0-9a-fA-F]+)\s+(.*)$") +_CALLEE_NAME_RX = re.compile(r"^(.*?) \(0x[0-9a-fA-F]+ \(") +# Same runtime/thunk classification as the RPyc backend. +_RUNTIME_PREFIXES = ( + "__libc_csu_", + "__scrt_", + "_scrt_", + "__crt", + "_crt", + "_global__sub_i_", +) +_RUNTIME_NAMES = frozenset( + { + "_init", "_fini", "__libc_start_main", "__libc_csu_init", "__libc_csu_fini", + "frame_dummy", "register_tm_clones", "deregister_tm_clones", "__do_global_dtors_aux", + } +) # fmt: skip + + +def mcp_url() -> str: + return os.environ.get("TOCODE_BINJA_MCP_URL", "").strip() or DEFAULT_MCP_URL + + +def connect( + url: str | None = None, *, timeout: float | None = PROBE_TIMEOUT_SECONDS +) -> McpClient: + """Connect to Binary Ninja's MCP server; raises BackendError when absent.""" + client = McpClient(url or mcp_url()) + client.connect(timeout=timeout) + name = str(client.server_info.get("name", "")) + if "binaryninja" not in name.lower(): + raise BackendError( + f"the MCP server at {client.url} is {name or 'unknown'}, not Binary Ninja" + ) + return client + + +def _int(value: Any, default: int = 0) -> int: + if value is None or value == "": + return default + if isinstance(value, int): + return value + text = str(value).strip() + try: + return int(text, 16) if text.lower().startswith(("0x", "-0x")) else int(text) + except ValueError: + return default + + +def _bool(value: Any) -> bool: + return str(value).strip().lower() == "true" + + +def call(client: McpClient, tool: str, **arguments: Any) -> ToolResult: + result = client.call( + tool, {key: value for key, value in arguments.items() if value is not None} + ) + if result.is_error: + message = result.text + if result.structured: + message = str(result.structured.get("errorMessage") or message) + raise BackendError(f"Binary Ninja {tool} failed: {message[:300]}") + return result + + +def paged(client: McpClient, tool: str, **arguments: Any) -> Iterator[dict[str, str]]: + """Every row of a paginated list tool.""" + offset = 0 + while True: + result = call(client, tool, offset=offset, limit=PAGE, **arguments) + tables = markdown_tables(result.text) + yield from tables[0] if tables else [] + next_offset = front_matter(result.text).get("nextOffset") + if next_offset is None or next_offset <= offset: + return + offset = int(next_offset) + + +def paged_text(client: McpClient, tool: str, **arguments: Any) -> str: + """The full fenced body of a paginated text tool (decompile/disassembly).""" + chunks: list[str] = [] + offset = 0 + while True: + result = call(client, tool, offset=offset, limit=PAGE, **arguments) + body = fenced_body(result.text) + if body is not None: + chunks.append(body if body.endswith("\n") else body + "\n") + next_offset = front_matter(result.text).get("nextOffset") + if next_offset is None or next_offset <= offset: + return "".join(chunks) + offset = int(next_offset) + + +# -------------------------------------------------------------------------- +# Views +# -------------------------------------------------------------------------- + + +@dataclass(slots=True) +class McpView: + handle: str + open_item: str + view_type: str + architecture: str + platform: str + address_size: int + start: int + entry_point: int + path: str + active: bool + created: bool + + def describe(self) -> str: + bits = self.address_size * 8 + arch = self.architecture or "?" + flag = " [active]" if self.active else "" + return f"{self.view_type} {arch} {bits}-bit {self.path or ''}{flag}" + + +def list_views(client: McpClient) -> list[McpView]: + """Analyzable views (Raw views are skipped), in Binary Ninja's order.""" + paths = { + row.get("openItem", ""): row.get("path", "") + for row in paged(client, "bn_open_item_list") + } + views: list[McpView] = [] + for row in paged(client, "bn_binary_view_list"): + view_type = row.get("viewType", "") + if view_type == "Raw" or not _bool(row.get("available", "true")): + continue + views.append( + McpView( + handle=row.get("binaryView", ""), + open_item=row.get("openItem", ""), + view_type=view_type, + architecture=row.get("architecture", ""), + platform=row.get("platform", ""), + address_size=_int(row.get("addressSize")), + start=_int(row.get("start")), + entry_point=_int(row.get("entryPoint")), + path=paths.get(row.get("openItem", ""), ""), + active=_bool(row.get("active")), + created=_bool(row.get("created")), + ) + ) + return views + + +def active_handle(client: McpClient) -> str | None: + result = call(client, "bn_binary_view_get_active") + view = (result.structured or {}).get("binaryView") or {} + return str(view.get("handle")) if view.get("handle") else None + + +def set_active(client: McpClient, handle: str) -> None: + call(client, "bn_binary_view_set_active", binaryView=handle) + + +def find_view_for_path(views: list[McpView], path: Path) -> McpView | None: + """The view of an already open file (or of its .bndb database).""" + wanted = {str(path), str(path.resolve())} + database = {f"{item}.bndb" for item in wanted} | { + str(path.with_suffix(".bndb")), + str(path.resolve().with_suffix(".bndb")), + } + for view in views: + if view.path in wanted: + return view + for view in views: + if view.path in database: + return view + return None + + +def open_path(client: McpClient, path: Path) -> McpView: + """Open ``path`` in Binary Ninja and return its main view. + + The file stays open afterwards: Binary Ninja's UI only closes a file whose + analysis changed it through its save prompt, which must not pop up from a + batch export. + """ + result = call( + client, "bn_open_item_open", path=str(path), kind="file", setActive=False + ) + payload = result.structured or {} + item = str((payload.get("openItem") or {}).get("handle") or "") + for view in list_views(client): + if view.open_item == item: + return view + raise BackendError(f"Binary Ninja opened {path} but created no analyzable view") + + +# -------------------------------------------------------------------------- +# Session +# -------------------------------------------------------------------------- + + +class BinjaMcpSession: + """DecompilerSession driving Binary Ninja through its MCP server.""" + + backend_name: BackendName = "binja" + backend_label = "Binary Ninja (MCP)" + decompiler_label = "Binary Ninja (Pseudo C)" + analysis_command: str | None = None + parallel_safe = False # one server-wide active view: render serially + + def __init__(self, client: McpClient, view: McpView) -> None: + self.client = client + self.view = view + self._previous = active_handle(client) + self._calls = 0 + set_active(client, view.handle) + self._functions: dict[int, dict[str, Any]] | None = None + self._imports: set[int] = set() + self._segments: list[tuple[int, int, str]] | None = None + self._disasm_cache: dict[int, str] = {} + + # -- plumbing ----------------------------------------------------------- + + def _call(self, tool: str, **arguments: Any) -> ToolResult: + self._ensure_active() + return call(self.client, tool, **arguments) + + def _paged(self, tool: str, **arguments: Any) -> Iterator[dict[str, str]]: + self._ensure_active() + return paged(self.client, tool, **arguments) + + def _ensure_active(self) -> None: + self._calls += 1 + if self._calls % ACTIVE_CHECK_EVERY: + return + if active_handle(self.client) != self.view.handle: + # Another MCP client (or the user) switched views mid-export. + set_active(self.client, self.view.handle) + + # -- lifecycle ---------------------------------------------------------- + + def analyze(self) -> None: + status = self._call("bn_analysis_status").structured or {} + state = str((status.get("analysis") or {}).get("state", "")) + if state and state != "IdleState": + # Analysis of a freshly opened file runs in Binary Ninja; wait for it. + self.client.call("bn_analysis_update_and_wait", {}, timeout=None) + + def close(self) -> None: + try: + if self._previous and self._previous != self.view.handle: + set_active(self.client, self._previous) + except BackendError: + pass + + def ensure_decompiler(self) -> None: + return None # Pseudo C always ships with Binary Ninja + + # -- metadata ------------------------------------------------------------- + + def info(self) -> dict[str, Any]: + view = self.view + os_name = view.platform.split("-", 1)[0] if view.platform else "unknown" + return { + "bin": { + "arch": view.architecture or "unknown", + "bits": view.address_size * 8, + "baddr": view.start, + "os": os_name, + "format": view.view_type, + "class": view.view_type, + "type": view.view_type, + }, + "tocode": {"input_path": view.path}, + } + + def entries(self) -> list[dict[str, Any]]: + seen: set[int] = set() + rows: list[dict[str, Any]] = [] + if self.view.entry_point: + seen.add(self.view.entry_point) + rows.append({"vaddr": self.view.entry_point}) + for row in self._paged("bn_entry_point_list"): + address = _int(row.get("function") or row.get("address")) + if address and address not in seen: + seen.add(address) + rows.append({"vaddr": address}) + return rows + + def _segment_list(self) -> list[tuple[int, int, str]]: + if self._segments is None: + self._segments = [ + ( + _int(row.get("start")), + _int(row.get("end")), + ("r" if _bool(row.get("readable")) else "-") + + ("w" if _bool(row.get("writable")) else "-") + + ("x" if _bool(row.get("executable")) else "-"), + ) + for row in self._paged("bn_segment_list") + ] + return self._segments + + def _perms_at(self, address: int) -> str: + for start, end, perms in self._segment_list(): + if start <= address < end: + return perms + return "---" + + def sections(self) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for row in self._paged("bn_section_list"): + start = _int(row.get("start")) + length = _int(row.get("length")) + rows.append( + { + "name": row.get("section") or f"sec_{start:x}", + "size": length, + "vsize": length, + "type": row.get("typeName", ""), + "perm": self._perms_at(start), + "paddr": start, + "vaddr": start, + } + ) + return rows + + def imports(self) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + seen: set[tuple[int, str]] = set() + for row in self._paged("bn_import_list"): + address = _int(row.get("symbol")) + name = row.get("name", "") + if not name or (address, name) in seen: + continue + seen.add((address, name)) + namespace = row.get("namespace", "") + library = ( + None if not namespace or namespace == _INTERNAL_NAMESPACE else namespace + ) + self._imports.add(address) + rows.append( + { + "plt": address, + "name": name, + "bind": library, + "dll": library, + "type": "import", + "delay": False, + } + ) + return rows + + def exports(self) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + seen: set[int] = set() + for row in self._paged("bn_export_list"): + if row.get("binding") not in ("GlobalBinding", "WeakBinding"): + continue + address = _int(row.get("symbol")) + if address in seen: + continue + seen.add(address) + ordinal = _int(row.get("ordinal")) + rows.append( + { + "name": row.get("name", ""), + "vaddr": address, + "bind": "GLOBAL" + if row.get("binding") == "GlobalBinding" + else "WEAK", + "type": "export", + "ordinal": ordinal or None, + "is_forwarder": False, + "forwarder_target": None, + } + ) + return rows + + def symbols(self) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for row in self._paged("bn_symbol_list"): + address = _int(row.get("symbol")) + name = row.get("name", "") + kind = row.get("type", "") + rows.append( + { + "name": name, + "flagname": name, + "realname": row.get("fullName") or name, + "size": 0, + "type": kind, + "vaddr": address, + "paddr": address, + "is_imported": "Import" in kind, + } + ) + return rows + + def relocations(self) -> list[dict[str, Any]]: + return [ + { + "name": row.get("symbolName", ""), + "type": row.get("type", ""), + "vaddr": _int(row.get("address")), + "paddr": _int(row.get("address")), + "is_ifunc": False, + } + for row in self._paged("bn_relocation_list") + ] + + def strings(self) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + sections = [ + (row["vaddr"], row["vaddr"] + row["size"], row["name"]) + for row in self.sections() + ] + for row in self._paged("bn_string_list", previewBytes=256): + address = _int(row.get("string")) + length = _int(row.get("length")) + kind = row.get("type", "AsciiString") + try: + raw = bytes.fromhex(row.get("previewHex", "") or "") + except ValueError: + raw = (row.get("previewEscaped", "") or "").encode("utf-8") + if _bool(row.get("previewTruncated")) and length > len(raw): + raw = self._read(address, min(length, MAX_STRING_BYTES)) or raw + rows.append( + { + "vaddr": address, + "paddr": address, + "size": length, + "length": length, + "section": next( + ( + name + for start, end, name in sections + if start <= address < end + ), + "", + ), + "type": kind, + "string": _decode_string(raw, kind), + } + ) + return rows + + def read_bytes(self, address: int, size: int) -> bytes | None: + """Section bytes from Binary Ninja's view (the input may be a .bndb).""" + chunks: list[bytes] = [] + offset = 0 + while offset < size: + chunk = self._read(address + offset, min(MAX_STRING_BYTES, size - offset)) + if not chunk: + break + chunks.append(chunk) + offset += len(chunk) + return b"".join(chunks) + b"\0" * (size - offset) if chunks else None + + def _read(self, address: int, length: int) -> bytes | None: + try: + result = self._call( + "bn_memory_read", address=f"0x{address:x}", length=length + ) + except BackendError: + return None + hex_text = (result.structured or {}).get("hex") + return bytes.fromhex(hex_text) if isinstance(hex_text, str) else None + + def flags(self) -> list[dict[str, Any]]: + return [] + + def _function_rows(self) -> dict[int, dict[str, Any]]: + if self._functions is None: + if not self._imports: + self.imports() + rows: dict[int, dict[str, Any]] = {} + for row in self._paged("bn_function_list"): + address = _int(row.get("function")) + if address in rows: + continue # same start in another architecture (thumb/arm) + rows[address] = row + self._functions = rows + return self._functions + + def functions(self) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for address, row in self._function_rows().items(): + name = row.get("name") or f"sub_{address:x}" + definition = row.get("definition", "") + return_type, params = _split_definition(definition) + is_import = address in self._imports + is_thunk = is_import or name.startswith("j_") + size = max( + _int(row.get("highestAddress")) - _int(row.get("lowestAddress")) + 1, 0 + ) + rows.append( + { + "addr": address, + "name": name, + "size": size, + "signature": _signature(name, return_type, params), + "calltype": None, + "noreturn": False, + "stackframe": 0, + "nlocals": 0, + "nargs": len(params), + "outdegree": 0, + "indegree": 0, + "is_library": is_import, + "is_thunk": is_thunk, + "source_kind": _classify( + name, is_import=is_import, is_thunk=is_thunk + ), + "return_type": return_type, + "params": params, + } + ) + return rows + + def types(self) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for index, row in enumerate(self._paged("bn_type_list")): + if index >= MAX_TYPES: + break + name = row.get("type", "") + if not name: + continue + c_decl = row.get("definition", "") + try: + info = (self._call("bn_type_info", type=name).structured or {}).get( + "type" + ) or {} + c_decl = str(info.get("source") or c_decl) + except BackendError: + pass + rows.append( + { + "name": name, + "kind": _type_kind(row.get("class", "")), + "size": _int(row.get("width")) or None, + "c_decl": c_decl, + } + ) + return rows + + # -- per function ------------------------------------------------------- + + def disasm(self, address: int) -> str: + address = int(address) + if address not in self._disasm_cache: + body = paged_text( + self.client, "bn_function_disassembly", function=f"0x{address:x}" + ) + self._ensure_active() + lines = [] + for line in body.splitlines(): + match = _DISASM_RX.match(line.strip()) + if match: + lines.append(f"{match.group(1)}: {match.group(2)}".rstrip()) + self._disasm_cache[address] = "\n".join(lines) + return self._disasm_cache[address] + + def decompile(self, address: int) -> str: + address = int(address) + self._ensure_active() + try: + body = paged_text( + self.client, "bn_function_decompile", function=f"0x{address:x}" + ) + except BackendError as exc: + row = self._function_rows().get(address, {}) + name = row.get("name") or f"sub_{address:x}" + return f"int {name}()\n{{\n /* Binary Ninja: {exc} */\n}}\n" + text = _strip_gutter(body).strip("\n") + if not text.strip(): + row = self._function_rows().get(address, {}) + name = row.get("name") or f"sub_{address:x}" + return_type, params = _split_definition(row.get("definition", "")) + return f"{_signature(name, return_type, params)}\n{{\n}}\n" + return text + "\n" + + def function_summary(self, address: int) -> str: + address = int(address) + row = self._function_rows().get(address) + if row is None: + return f"signature: sub_{address:x}\naddress: 0x{address:x}\nargs: 0\nlocals: 0" + name = row.get("name") or f"sub_{address:x}" + return_type, params = _split_definition(row.get("definition", "")) + size = max( + _int(row.get("highestAddress")) - _int(row.get("lowestAddress")) + 1, 0 + ) + return "\n".join( + [ + f"signature: {_signature(name, return_type, params)}", + f"address: 0x{address:x}", + f"size: {size} bytes", + f"basic blocks: {row.get('basicBlockCount', '?')}", + f"args: {len(params)}", + "locals: 0", + ] + ) + + def calls_from( + self, address: int, imports: Any, functions: Any + ) -> tuple[list[int], list[str]]: + address = int(address) + edges: set[int] = set() + imported: set[str] = set() + try: + rows = list(self._paged("bn_function_callees", function=f"0x{address:x}")) + except BackendError: + return [], [] + for row in rows: + target = _int(row.get("Callee Address")) + if target == address or not target: + continue + routine = functions.get(target) + if routine is not None and not getattr(routine, "imported", False): + edges.add(target) + continue + if target in imports: + imported.add(imports[target].name) + elif routine is not None: + imported.add(routine.name) + else: + match = _CALLEE_NAME_RX.match(row.get("Callee Functions", "")) + imported.add(match.group(1) if match else f"sub_{target:x}") + return sorted(edges), sorted(imported) + + def data_xrefs(self, addresses: Any) -> dict[int, list[tuple[int, bool]]]: + return {} + + +# -------------------------------------------------------------------------- +# Helpers +# -------------------------------------------------------------------------- + + +def _split_definition(definition: str) -> tuple[str | None, list[dict[str, str]]]: + """``"int32_t(char* arg1, int32_t arg2)"`` -> return type, parameters.""" + text = definition.strip() + if not text.endswith(")"): + return (text or None), [] + depth = 0 + for index in range(len(text) - 1, -1, -1): + char = text[index] + if char == ")": + depth += 1 + elif char == "(": + depth -= 1 + if depth == 0: + return_type = text[:index].strip() or None + inner = text[index + 1 : -1] + return return_type, _parameters(inner) + return (text or None), [] + + +def _parameters(text: str) -> list[dict[str, str]]: + if not text.strip() or text.strip() == "void": + return [] + parts: list[str] = [] + depth = 0 + current: list[str] = [] + for char in text: + if char in "(<[": + depth += 1 + elif char in ")>]": + depth -= 1 + if char == "," and depth == 0: + parts.append("".join(current).strip()) + current = [] + else: + current.append(char) + parts.append("".join(current).strip()) + params: list[dict[str, str]] = [] + for index, part in enumerate(parts): + if part == "...": + params.append({"name": "...", "type": "..."}) + continue + match = re.match(r"^(.*?[\s*&])([A-Za-z_][A-Za-z0-9_]*)$", part) + if match: + params.append({"name": match.group(2), "type": match.group(1).strip()}) + else: + params.append({"name": f"arg{index + 1}", "type": part}) + return params + + +def _signature(name: str, return_type: str | None, params: list[dict[str, str]]) -> str: + args = ", ".join( + "..." if item["name"] == "..." else f"{item['type']} {item['name']}".strip() + for item in params + ) + return f"{return_type or 'int'} {name}({args or 'void'})" + + +def _classify(name: str, *, is_import: bool, is_thunk: bool) -> str: + if is_thunk: + return "thunk" + if is_import: + return "library" + lowered = name.lower() + if lowered in _RUNTIME_NAMES or lowered.startswith(_RUNTIME_PREFIXES): + return "runtime" + return "app" + + +def _type_kind(type_class: str) -> str: + return { + "StructureTypeClass": "struct", + "EnumerationTypeClass": "enum", + "NamedTypeReferenceClass": "typedef", + "FunctionTypeClass": "func", + "IntegerTypeClass": "base", + "FloatTypeClass": "base", + "BoolTypeClass": "base", + "PointerTypeClass": "typedef", + "ArrayTypeClass": "typedef", + }.get(type_class, "type") + + +def _decode_string(raw: bytes, kind: str) -> str: + lowered = kind.lower() + if "utf16" in lowered or "wide" in lowered: + return raw.decode("utf-16-le", errors="replace").rstrip("\0") + if "utf32" in lowered: + return raw.decode("utf-32-le", errors="replace").rstrip("\0") + return raw.decode("utf-8", errors="replace").rstrip("\0") diff --git a/src/tocode/backends/mcp_client.py b/src/tocode/backends/mcp_client.py new file mode 100644 index 0000000..ae9d420 --- /dev/null +++ b/src/tocode/backends/mcp_client.py @@ -0,0 +1,282 @@ +"""Minimal MCP client (JSON-RPC over streamable HTTP), standard library only. + +Enough of the Model Context Protocol to call tools on a local server such as +Binary Ninja's built-in MCP: ``initialize``, ``notifications/initialized``, +``tools/list``, and ``tools/call``. Responses may come back as plain JSON or +as a server-sent event stream; both are handled. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +import re +from typing import Any +import urllib.error +import urllib.request + +from ..errors import BackendError + +PROTOCOL_VERSION = "2025-11-25" +CLIENT_INFO = {"name": "tocode", "version": "1"} + + +@dataclass(slots=True) +class ToolResult: + text: str + structured: dict[str, Any] | None + is_error: bool + + +class McpClient: + def __init__(self, url: str, *, timeout: float | None = 600.0) -> None: + self.url = url + self.timeout = timeout + self.session_id: str | None = None + self.server_info: dict[str, Any] = {} + self.instructions: str = "" + self._next_id = 0 + + # -- protocol ------------------------------------------------------------- + + def connect(self, *, timeout: float | None = None) -> "McpClient": + result = self._request( + "initialize", + { + "protocolVersion": PROTOCOL_VERSION, + "capabilities": {}, + "clientInfo": CLIENT_INFO, + }, + timeout=timeout, + ) + self.server_info = dict(result.get("serverInfo") or {}) + self.instructions = str(result.get("instructions") or "") + self._post( + {"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, + timeout=timeout, + ) + return self + + def list_tools(self) -> list[dict[str, Any]]: + tools: list[dict[str, Any]] = [] + cursor: str | None = None + while True: + params = {"cursor": cursor} if cursor else {} + result = self._request("tools/list", params) + tools.extend(result.get("tools") or []) + cursor = result.get("nextCursor") + if not cursor: + return tools + + def call( + self, + name: str, + arguments: dict[str, Any] | None = None, + *, + timeout: float | None = None, + ) -> ToolResult: + result = self._request( + "tools/call", + {"name": name, "arguments": arguments or {}}, + timeout=timeout, + ) + texts = [ + str(item.get("text", "")) + for item in result.get("content") or [] + if isinstance(item, dict) and item.get("type") == "text" + ] + structured = result.get("structuredContent") + return ToolResult( + text="\n".join(texts), + structured=structured if isinstance(structured, dict) else None, + is_error=bool(result.get("isError", False)), + ) + + # -- transport ------------------------------------------------------------ + + def _request( + self, method: str, params: dict[str, Any], *, timeout: float | None = None + ) -> dict[str, Any]: + self._next_id += 1 + request_id = self._next_id + payload = { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params, + } + messages = self._post(payload, timeout=timeout) + for message in messages: + if message.get("id") != request_id: + continue + if "error" in message: + error = message["error"] or {} + raise BackendError( + f"MCP {method} failed: {error.get('message', error)}" + ) + result = message.get("result") + return result if isinstance(result, dict) else {} + raise BackendError(f"MCP server sent no response to {method}") + + def _post( + self, payload: dict[str, Any], *, timeout: float | None = None + ) -> list[dict[str, Any]]: + headers = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + "MCP-Protocol-Version": PROTOCOL_VERSION, + } + if self.session_id: + headers["MCP-Session-Id"] = self.session_id + request = urllib.request.Request( + self.url, + data=json.dumps(payload).encode("utf-8"), + headers=headers, + method="POST", + ) + try: + with urllib.request.urlopen( + request, timeout=self.timeout if timeout is None else timeout + ) as response: + session = response.headers.get("MCP-Session-Id") + if session: + self.session_id = session + content_type = response.headers.get("Content-Type", "") + body = response.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace")[:300] + raise BackendError( + f"MCP server at {self.url} returned HTTP {exc.code}: {detail}" + ) from exc + except (urllib.error.URLError, OSError, TimeoutError) as exc: + raise BackendError( + f"could not reach the MCP server at {self.url}: {exc}" + ) from exc + if not body.strip(): + return [] + if "text/event-stream" in content_type: + return _sse_messages(body) + try: + decoded = json.loads(body) + except ValueError as exc: + raise BackendError(f"MCP server sent invalid JSON: {body[:200]}") from exc + if isinstance(decoded, list): + return [item for item in decoded if isinstance(item, dict)] + return [decoded] if isinstance(decoded, dict) else [] + + +def _sse_messages(body: str) -> list[dict[str, Any]]: + messages: list[dict[str, Any]] = [] + data: list[str] = [] + for line in body.splitlines() + [""]: + if line.startswith("data:"): + data.append(line[5:].lstrip()) + elif not line.strip() and data: + try: + decoded = json.loads("\n".join(data)) + except ValueError: + decoded = None + if isinstance(decoded, dict): + messages.append(decoded) + data = [] + return messages + + +# -------------------------------------------------------------------------- +# Text result helpers (Binary Ninja renders lists as Markdown tables) +# -------------------------------------------------------------------------- + +_FRONT_MATTER_RX = re.compile(r"^---\n(.*?)\n---\n", re.S) + + +def front_matter(text: str) -> dict[str, Any]: + """The ``---`` YAML-ish header of a tool result (simple ``key: value`` lines).""" + match = _FRONT_MATTER_RX.match(text) + if match is None: + return {} + meta: dict[str, Any] = {} + for line in match.group(1).splitlines(): + key, sep, value = line.partition(":") + if not sep: + continue + meta[key.strip()] = _scalar(value.strip()) + return meta + + +def _scalar(value: str) -> Any: + if value in ("null", "~", ""): + return None + if value in ("true", "false"): + return value == "true" + if value.startswith('"') and value.endswith('"'): + try: + return json.loads(value) + except ValueError: + return value[1:-1] + try: + return int(value) + except ValueError: + return value + + +def split_row(line: str) -> list[str]: + """Cells of a Markdown table row; ``\\|`` is a literal pipe.""" + text = line.strip() + if text.startswith("|"): + text = text[1:] + if text.endswith("|") and not text.endswith("\\|"): + text = text[:-1] + cells: list[str] = [] + current: list[str] = [] + index = 0 + while index < len(text): + char = text[index] + if char == "\\" and index + 1 < len(text) and text[index + 1] in "|\\": + current.append(text[index + 1]) + index += 2 + continue + if char == "|": + cells.append("".join(current).strip()) + current = [] + else: + current.append(char) + index += 1 + cells.append("".join(current).strip()) + return cells + + +def markdown_tables(text: str) -> list[list[dict[str, str]]]: + """Every table in ``text`` as a list of ``{header: cell}`` rows.""" + tables: list[list[dict[str, str]]] = [] + lines = text.splitlines() + index = 0 + while index < len(lines) - 1: + line = lines[index] + divider = lines[index + 1].strip() + if ( + line.lstrip().startswith("|") + and divider.startswith("|") + and set(divider) <= set("|-: ") + ): + headers = split_row(line) + rows: list[dict[str, str]] = [] + index += 2 + while index < len(lines) and lines[index].lstrip().startswith("|"): + cells = split_row(lines[index]) + rows.append( + dict(zip(headers, cells + [""] * (len(headers) - len(cells)))) + ) + index += 1 + tables.append(rows) + continue + index += 1 + return tables + + +_FENCE_RX = re.compile(r"^(`{3,}|~{3,})[^\n]*\n(.*?)^\1\s*$", re.S | re.M) + + +def fenced_body(text: str) -> str | None: + """The content of the first fenced code block, or None.""" + match = _FENCE_RX.search(text) + return match.group(2) if match else None diff --git a/src/tocode/cli.py b/src/tocode/cli.py index 954c9ab..9754e40 100644 --- a/src/tocode/cli.py +++ b/src/tocode/cli.py @@ -10,6 +10,7 @@ from .apk import ApkExportOptions, export_apk from .apk_native import NativeOptions from .backends.asc import is_apk_input +from .backends.base import is_binja_database, probe_binja_headless from .backends.dotnet import is_dotnet_input from .errors import ToCodeError from .exporter import export_binary @@ -50,6 +51,11 @@ def build_parser() -> argparse.ArgumentParser: os.environ.get("TOCODE_BINJA_HOST", "127.0.0.1").strip() or "127.0.0.1" ) default_binja_port = _env_port("TOCODE_BINJA_PORT", 18812) + default_binja_transport = ( + os.environ.get("TOCODE_BINJA_TRANSPORT", "auto").strip().lower() or "auto" + ) + if default_binja_transport not in {"auto", "headless", "mcp", "rpyc"}: + default_binja_transport = "auto" parser = argparse.ArgumentParser( prog="tocode", description="Export a compiled binary into a source-like reverse-engineering project.", @@ -59,7 +65,7 @@ def build_parser() -> argparse.ArgumentParser: type=Path, nargs="?", default=None, - help="Input binary to export. Optional for --backend binja, which exports an open Binary Ninja view.", + help="Input binary or folder to export. Optional for --backend binja, which exports an open Binary Ninja view (a path given there is opened in Binary Ninja, or matched to an open file).", ) parser.add_argument( "-o", @@ -72,7 +78,7 @@ def build_parser() -> argparse.ArgumentParser: "--backend", choices=("auto", "ida", "r2", "angr", "binja"), default=default_backend, - help="Decompiler backend: auto prefers IDA, then r2, then angr as a last-resort fallback; binja drives Binary Ninja and is opt-in (default: TOCODE_BACKEND or auto). For APK input this selects the backend for the native .so libraries (binja is not supported there).", + help="Decompiler backend: auto prefers IDA, then Binary Ninja's Python API (headless), then r2, then angr as a last-resort fallback; binja always uses Binary Ninja, also through a running Binary Ninja (see --binja-transport) (default: TOCODE_BACKEND or auto). For APK input this selects the backend for the native .so libraries (binja needs the headless API there).", ) parser.add_argument( "--no-native", @@ -133,6 +139,31 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="APK input: export base.apk alone instead of merging sibling split_*.apk files.", ) + parser.add_argument( + "--binja-transport", + choices=("auto", "headless", "mcp", "rpyc"), + default=default_binja_transport, + help="How the binja backend reaches Binary Ninja: headless loads the input file with the Binary Ninja Python API in this process (needs a license that allows headless use), mcp uses Binary Ninja 6's built-in MCP server, rpyc uses binja-headless. auto uses headless when an input file is given and the API is installed, then MCP, then RPyc (default: TOCODE_BINJA_TRANSPORT or auto).", + ) + parser.add_argument( + "--binja-dir", + type=Path, + default=None, + help="Binary Ninja install folder (or its python/ folder) for the headless API (default: TOCODE_BINJA_DIR, BN_INSTALL_DIR, the binaryninja.pth written by Binary Ninja's install_api.py, or common install paths).", + ) + parser.add_argument( + "--binja-license", + type=Path, + default=None, + metavar="PATH", + help="Binary Ninja license file for the headless API, handed to the core instead of the license.dat in Binary Ninja's user folder; its contents are never logged (default: TOCODE_BINJA_LICENSE).", + ) + parser.add_argument( + "--binja-mcp-url", + default=None, + metavar="URL", + help="URL of Binary Ninja's MCP server (default: TOCODE_BINJA_MCP_URL or http://127.0.0.1:24642/mcp).", + ) parser.add_argument( "--binja-host", default=default_binja_host, @@ -147,14 +178,14 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--list-binja", action="store_true", - help="List the BinaryViews open in Binary Ninja with their index, then exit (binja backend).", + help="List the binary views open in Binary Ninja with their index, then exit (binja backend).", ) parser.add_argument( "--binja-view", type=int, default=None, metavar="N", - help="Export the open Binary Ninja view with this index (from --list-binja). Default: the focused view.", + help="Export the open Binary Ninja view with this index (from --list-binja). Default: the active view (MCP) or the focused view (RPyc).", ) parser.add_argument( "--all-views", @@ -233,27 +264,74 @@ def main(argv: list[str] | None = None) -> int: ) if args.binja_view is not None and args.all_views: parser.error("--binja-view and --all-views cannot be combined") + if args.binja_license is not None: + license_file = args.binja_license.expanduser().resolve() + if not license_file.is_file(): + parser.error(f"--binja-license: {license_file} is not a file") + # Through the environment so render workers and the per-binary + # processes of folder and APK exports pick it up too. + os.environ["TOCODE_BINJA_LICENSE"] = str(license_file) + if ( + args.backend == "auto" + and args.binary is not None + and is_binja_database(args.binary) + ): + # Only Binary Ninja reads a .bndb: headless when the API is installed, + # otherwise through a running Binary Ninja. + args.backend = "binja" if args.backend == "binja": - if args.binary is not None and args.binary.is_dir(): - parser.error( - "--backend binja is not supported for folder input; use ida, r2, or angr" - ) - if args.binary is not None and is_apk_input(args.binary): - parser.error( - "--backend binja is not supported for APK input; native libraries " - "use ida, r2, or angr" - ) - if ( - args.binary is not None - and not args.as_native - and args.binary.is_file() - and is_dotnet_input(args.binary) - ): + headless = ( + args.binja_transport in ("auto", "headless") + and not binja_only + and args.binary is not None + and probe_binja_headless(args.binja_dir) + ) + if args.binja_transport == "headless" and not headless: parser.error( - "--backend binja is not supported for .NET input; native code " - "uses ida, r2, or angr (or pass --as-native)" + "--binja-transport headless needs the Binary Ninja Python API and an " + "input file; pass --binja-dir (or set TOCODE_BINJA_DIR) to the " + "Binary Ninja install" ) - return _run_binja(args, progress, parser, argv) + if headless: + from .backends.binja_headless import headless_problem + + # A personal license ships the API but refuses headless use; auto + # then exports through a running Binary Ninja instead. + problem = headless_problem(args.binja_dir) + if problem is not None: + if args.binja_transport == "headless": + print(f"tocode: {problem}", file=sys.stderr) + return 1 + progress.log( + f"Binary Ninja headless unavailable ({problem}); " + "using a running Binary Ninja" + ) + headless = False + if not headless: + # A live Binary Ninja view (MCP or RPyc) exports one binary at a time. + if args.binary is not None and args.binary.is_dir(): + parser.error( + "--backend binja needs the Binary Ninja Python API (headless) " + "for folder input; use ida, r2, or angr otherwise" + ) + if args.binary is not None and is_apk_input(args.binary): + parser.error( + "--backend binja needs the Binary Ninja Python API (headless) " + "for APK input; native libraries use ida, r2, or angr otherwise" + ) + if ( + args.binary is not None + and not args.as_native + and args.binary.is_file() + and is_dotnet_input(args.binary) + ): + parser.error( + "--backend binja needs the Binary Ninja Python API (headless) " + "for .NET input; native code uses ida, r2, or angr (or pass " + "--as-native)" + ) + return _run_binja(args, progress, parser, argv) + # Headless: the regular file pipeline below, with Binary Ninja in-process. if args.binary is None: parser.error("the following arguments are required: binary") @@ -348,6 +426,7 @@ def _native_options(args: argparse.Namespace) -> NativeOptions: jobs=args.jobs, tree=args.tree, entropy=args.entropy, + binja_dir=args.binja_dir, ) @@ -494,6 +573,7 @@ def _run_one( purge_cache=args.purge_cache, binja_host=args.binja_host, binja_port=args.binja_port, + binja_dir=args.binja_dir, ) as analyzer: return export_binary( analyzer, @@ -512,6 +592,22 @@ def _run_binja( parser: argparse.ArgumentParser, argv: list[str] | None, ) -> int: + if args.binja_transport in ("auto", "mcp"): + from .backends import binja_mcp + + url = args.binja_mcp_url or binja_mcp.mcp_url() + try: + client = binja_mcp.connect(url) + except ToCodeError as exc: + if args.binja_transport == "mcp": + print(f"tocode: {exc}", file=sys.stderr) + return 1 + progress.log( + f"Binary Ninja MCP not reachable at {url} ({exc}); " + f"trying binja-headless at {args.binja_host}:{args.binja_port}" + ) + else: + return _run_binja_mcp(client, args, progress, argv) # Drive a running Binary Ninja over binja-headless. One connection is shared # across every exported view; injected views never own it (their session # close() leaves it alone), so the CLI closes it once at the end. @@ -580,6 +676,105 @@ def _run_binja( return 0 +def _run_binja_mcp( + client, args: argparse.Namespace, progress: Progress, argv: list[str] | None +) -> int: + """Export through Binary Ninja's built-in MCP server (Binary Ninja 6+).""" + from .backends import binja_mcp + + out_dir = args.out_dir.expanduser().resolve() if args.out_dir is not None else None + started = time.monotonic() + try: + views = binja_mcp.list_views(client) + if args.list_binja: + if not views: + print("No binary views are open in Binary Ninja.") + else: + print(f"Open binary views ({len(views)}, via MCP at {client.url}):") + for index, view in enumerate(views): + print(f" [{index}] {view.describe()}") + return 0 + targets: list = [] + if args.all_views: + targets = list(views) + elif args.binja_view is not None: + if not 0 <= args.binja_view < len(views): + upper = len(views) - 1 + raise ToCodeError( + f"--binja-view {args.binja_view} is out of range (open views: " + f"0..{upper if upper >= 0 else 'none'}); run --list-binja" + ) + targets = [views[args.binja_view]] + elif args.binary is not None: + path = args.binary.expanduser().resolve() + found = binja_mcp.find_view_for_path(views, path) + if found is None: + progress.log(f"Opening {path} in Binary Ninja (it stays open there)") + found = binja_mcp.open_path(client, path) + targets = [found] + else: + active = next((view for view in views if view.active), None) + if active is None: + raise ToCodeError( + "Binary Ninja has no active binary view for MCP; run --list-binja " + "and pass --binja-view N, --all-views, or a binary path" + ) + targets = [active] + if not targets: + print("tocode: no Binary Ninja views to export", file=sys.stderr) + return 1 + multi = args.all_views or len(targets) > 1 + for view in targets: + source = Path(view.path or "binaryview") + if multi: + parent = out_dir if out_dir is not None else Path.cwd() + view_out: Path | None = parent / default_output_name(source) + else: + view_out = out_dir + log_root = ( + view_out + if view_out is not None + else source.parent / default_output_name(source) + ) + progress.set_log_path(log_root / "tocode.log") + progress.log( + f"Command: {' '.join(sys.argv if argv is None else ['tocode', *argv])}" + ) + progress.log(f"Exporting Binary Ninja view {view.describe()} over MCP") + session = binja_mcp.BinjaMcpSession(client, view) + with create_analyzer( + source, backend="binja", progress=progress, session=session + ) as analyzer: + summary = export_binary( + analyzer, + out_dir=view_out, + progress=progress, + jobs=args.jobs, + tree=args.tree, + entropy=args.entropy, + restart=args.restart, + ) + if not args.quiet: + print(f"Project: {summary.root_dir}") + print( + f"Summary: functions={summary.function_count} " + f"clusters={summary.cluster_count} failures={len(summary.failed_functions)}" + ) + except KeyboardInterrupt: + progress.log("tocode: interrupted; progress saved if checkpointing had started") + print("tocode: interrupted; rerun the same command to resume", file=sys.stderr) + return 130 + except ToCodeError as exc: + progress.log(f"tocode: {exc}") + print(f"tocode: {exc}", file=sys.stderr) + return 1 + if not args.quiet: + print( + f"Exported {len(targets)} view(s) in {_format_duration(time.monotonic() - started)}" + ) + return 0 + + def _select_binja_views(args: argparse.Namespace, conn, binja_backend, bn) -> list: if args.all_views: return binja_backend.open_views(conn) diff --git a/src/tocode/exporter.py b/src/tocode/exporter.py index 0f1073c..c50f9ca 100644 --- a/src/tocode/exporter.py +++ b/src/tocode/exporter.py @@ -75,10 +75,14 @@ DEFAULT_MAX_FUNCTION_BYTES = 2 * 1024 * 1024 # Backends whose per-function rendering runs in a spawned worker so a hung # decompilation can be timed out. A worker reconstructs the session from the -# binary path, so "binja" is deliberately excluded: it renders against a live -# BinaryView / RPyc connection that cannot be pickled into a subprocess, and -# must therefore stay on the in-process serial path (_render_serial). -TIMEOUT_WORKER_BACKENDS = frozenset({"ida", "r2", "angr"}) +# binary path (or database). Binary Ninja over the UI, MCP, or RPyc ("binja") +# is deliberately excluded: it renders against a live view that cannot be +# pickled into a subprocess and stays on the in-process serial path. Headless +# Binary Ninja reopens its .bndb in each worker ("binja-headless"). +TIMEOUT_WORKER_BACKENDS = frozenset({"ida", "r2", "angr", "binja-headless"}) +# Backends whose workers each load the whole database: worker count is capped +# by available memory. +MEMORY_BOUND_BACKENDS = frozenset({"ida", "binja-headless"}) TREE_CALLING_CONVENTION_RX = re.compile( r"\b__(?:cdecl|fastcall|stdcall|thiscall|usercall|userpurge|noreturn)\b" ) @@ -576,6 +580,7 @@ class WorkerSpec: ida_domain_path: Path | None = None db_path: Path | None = None copy_db: bool = True + binja_dir: Path | None = None @dataclass(frozen=True, slots=True) @@ -752,24 +757,36 @@ def _select_render_workers(context: ExportContext) -> None: count = len(context.addresses) if context.analyzer.supports_parallel: context.requested_jobs = context.jobs - is_ida = context.analyzer.backend_name == "ida" + worker_backend = _worker_backend(context.analyzer) + memory_bound = worker_backend in MEMORY_BOUND_BACKENDS context.worker_count = choose_jobs( function_count=count, analysis_seconds=context.analyzer.analysis_seconds, requested=context.jobs, - backend=context.analyzer.backend_name, - available_memory_mb=available_memory_mb() if is_ida else None, + backend=worker_backend, + available_memory_mb=available_memory_mb() if memory_bound else None, database_size_mb=_worker_database_size_mb(context.analyzer) - if is_ida + if memory_bound else None, + measured_worker_memory_mb=_measured_worker_memory_mb(context.analyzer), ) context.render_mode = "process" if context.worker_count > 1 else "single" - if is_ida and context.jobs is not None and context.worker_count < context.jobs: + if ( + memory_bound + and context.jobs is not None + and context.worker_count < context.jobs + ): + label = "IDA" if worker_backend == "ida" else "Binary Ninja" + variable = ( + "TOCODE_IDA_WORKER_MEMORY_MB" + if worker_backend == "ida" + else "TOCODE_BINJA_WORKER_MEMORY_MB" + ) context.progress.log( f"Note: limiting to {context.worker_count} worker(s) instead of the " f"requested {context.jobs} to fit available memory " - f"(each IDA worker loads the whole database; override with " - f"TOCODE_IDA_WORKER_MEMORY_MB)." + f"(each {label} worker loads the whole database; override with " + f"{variable})." ) context.progress.log( describe_jobs( @@ -777,7 +794,7 @@ def _select_render_workers(context: ExportContext) -> None: analysis_seconds=context.analyzer.analysis_seconds, requested=context.jobs, selected=context.worker_count, - backend=context.analyzer.backend_name, + backend=worker_backend, ) ) else: @@ -1078,7 +1095,7 @@ def render_and_write_source_tree_parallel( total = sum(len(cluster.members) for cluster in clusters) if progress is not None: progress.log(f"Opening {worker_count} streaming workers for {total} functions") - progress.log("Preparing IDA worker database copies") + progress.log("Preparing worker databases") analyzer.prepare_parallel_workers() spec = _worker_spec(analyzer, copy_db=worker_count > 1) if progress is not None: @@ -1086,7 +1103,7 @@ def render_and_write_source_tree_parallel( analyzer.release_parallel_resources() if progress is not None: progress.log( - "Starting streaming worker processes; opening IDA databases may take a while" + "Starting streaming worker processes; opening their databases may take a while" ) sources: list[Path] = [] @@ -1614,9 +1631,14 @@ def _max_function_bytes() -> int: return _env_int("TOCODE_MAX_FUNCTION_BYTES", DEFAULT_MAX_FUNCTION_BYTES) +def _worker_backend(analyzer: BinaryAnalyzer) -> str: + session = getattr(analyzer, "session", None) + return str(getattr(session, "worker_backend", None) or analyzer.backend_name) + + def _uses_timeout_worker(analyzer: BinaryAnalyzer) -> bool: return ( - analyzer.backend_name in TIMEOUT_WORKER_BACKENDS + _worker_backend(analyzer) in TIMEOUT_WORKER_BACKENDS and _function_timeout_seconds() > 0 ) @@ -1977,13 +1999,14 @@ def _render_parallel( def _worker_spec(analyzer: BinaryAnalyzer, *, copy_db: bool = True) -> WorkerSpec: session = analyzer.session return WorkerSpec( - backend=analyzer.backend_name, + backend=_worker_backend(analyzer), binary=analyzer.binary, analysis_command=getattr(session, "analysis_command", None), idadir=getattr(session, "idadir", None), ida_domain_path=getattr(session, "ida_domain_path", None), db_path=_session_database_path(session), copy_db=copy_db, + binja_dir=getattr(session, "binja_dir", None), ) @@ -1994,7 +2017,7 @@ def _worker_database_size_mb(analyzer: BinaryAnalyzer) -> int | None: (such as a kernel `.i64`) does not spawn more workers than RAM can hold. """ session = analyzer.session - candidate = getattr(session, "_cache_db", None) + candidate = _session_database_path(session) if candidate is None: binary = getattr(session, "binary", None) if binary is not None and is_ida_database(Path(binary)): @@ -2008,6 +2031,11 @@ def _worker_database_size_mb(analyzer: BinaryAnalyzer) -> int | None: return max(1, size // (1024 * 1024)) +def _measured_worker_memory_mb(analyzer: BinaryAnalyzer) -> int | None: + measure = getattr(getattr(analyzer, "session", None), "worker_memory_mb", None) + return measure() if callable(measure) else None + + def _session_database_path(session: object) -> Path | None: database_path = getattr(session, "database_path", None) if callable(database_path): @@ -2062,10 +2090,23 @@ def _open_worker(spec: WorkerSpec): session = AngrSession(spec.binary) session.analyze() + elif spec.backend == "binja-headless": + # Each worker opens the parent's .bndb (the cached analysis or the input + # database) read-only; Binary Ninja databases can be opened by several + # processes, so no per-worker copy is needed. + from .backends.binja_headless import BinjaHeadlessSession + + session = BinjaHeadlessSession( + spec.binary, + binja_dir=spec.binja_dir, + db_path=spec.db_path, + save_cache=False, + ) + session.analyze() elif spec.backend == "binja": - # Unreachable in practice: binja renders against a live, non-picklable - # session (parallel_safe=False and absent from TIMEOUT_WORKER_BACKENDS), - # so it never spawns a worker. Fail loudly if that invariant ever breaks. + # Unreachable in practice: UI/MCP/RPyc binja renders against a live, + # non-picklable session (parallel_safe=False and absent from + # TIMEOUT_WORKER_BACKENDS). Fail loudly if that invariant ever breaks. raise RuntimeError("binja backend renders serially; workers unsupported") else: raise RuntimeError(f"unsupported backend for worker: {spec.backend}") @@ -2471,7 +2512,10 @@ def write_header() -> None: def write_variables() -> None: context.data_variable_count = export_variables( - analysis, root, entropy=context.entropy_enabled + analysis, + root, + entropy=context.entropy_enabled, + reader=getattr(context.analyzer.session, "read_bytes", None), ) def write_indexes() -> None: @@ -2613,7 +2657,7 @@ def publish_backend_database(context: ExportContext) -> Path | None: _copy_file_with_progress( source, target, context.progress, desc="saving database" ) - context.progress.log(f"Saved IDA database to {target}") + context.progress.log(f"Saved {context.analyzer.backend_label} database to {target}") return target.resolve() @@ -2963,6 +3007,7 @@ def build_export_agents( "- `tocode.log`: export, checkpoint, resume, and per-function render history.", "- `CLAUDE.md`: Claude entrypoint that references `AGENTS.md`.", "- `.i64` or `.idb`: exported IDA database when the IDA backend was used.", + "- `.bndb`: exported Binary Ninja database when the headless Binary Ninja backend was used.", "", "## Working Style", "", diff --git a/src/tocode/folder.py b/src/tocode/folder.py index afe0c35..a190330 100644 --- a/src/tocode/folder.py +++ b/src/tocode/folder.py @@ -672,6 +672,7 @@ def _run_jobs( tree=native.tree, entropy=native.entropy, restart=options.restart, + binja_dir=native.binja_dir, ) def settings_for(lib: NativeLib) -> JobSettings: diff --git a/src/tocode/metadata.py b/src/tocode/metadata.py index 6e529ae..a6488eb 100644 --- a/src/tocode/metadata.py +++ b/src/tocode/metadata.py @@ -1,5 +1,7 @@ from __future__ import annotations +import contextlib + from bisect import bisect_right from collections import Counter, deque import math @@ -347,18 +349,35 @@ def export_variables( root: Path, *, entropy: bool = True, + reader: Callable[[int, int], bytes | None] | None = None, ) -> int: + """Write ``data/*.bin`` and the variables documents. + + ``reader(address, size)`` reads section bytes from the backend's own view + when the input file is not a plain binary on this machine (a Binary Ninja + database reached over MCP); otherwise sections are read from the file at + their file offsets. + """ data_dir = root / "data" sections: list[dict[str, object]] = [] - with analysis.binary.path.open("rb") as handle: + with contextlib.ExitStack() as stack: + handle = ( + None + if reader is not None + else stack.enter_context(analysis.binary.path.open("rb")) + ) for segment in analysis.segments: if max(segment.vsize, segment.size) <= 0: continue file_name = None if segment.size > 0: file_name = f"{clean_path_component(segment.name)}.bin" - handle.seek(segment.paddr) - blob = handle.read(segment.size) + if handle is not None: + handle.seek(segment.paddr) + blob = handle.read(segment.size) + else: + assert reader is not None + blob = reader(segment.vaddr, segment.size) or b"" (data_dir / file_name).write_bytes(blob) if blob and entropy: segment.entropy = round(shannon_entropy(blob), 6) diff --git a/src/tocode/parallel.py b/src/tocode/parallel.py index 7b47ace..df473bc 100644 --- a/src/tocode/parallel.py +++ b/src/tocode/parallel.py @@ -18,6 +18,12 @@ # is computed from the host's real available memory and the real database size. DEFAULT_IDA_WORKER_BASE_MEMORY_MB = 768 DEFAULT_IDA_DB_RESIDENT_FACTOR = 1.5 +# A headless Binary Ninja worker loads the core, its plugins, and the .bndb +# (about 650 MB for a tiny binary on Binary Ninja 6). +DEFAULT_BINJA_WORKER_MEMORY_MB = 1024 +DEFAULT_BINJA_WORKER_BASE_MEMORY_MB = 768 +DEFAULT_BINJA_DB_RESIDENT_FACTOR = 2.0 +MAX_AUTO_BINJA_JOBS = 4 def choose_jobs( @@ -31,12 +37,18 @@ def choose_jobs( available_memory_mb: int | None = None, ida_worker_memory_mb: int | None = None, database_size_mb: int | None = None, + measured_worker_memory_mb: int | None = None, ) -> int: limit = job_limit if job_limit is not None else configured_job_limit() is_ida = backend.lower() == "ida" + is_binja = backend.lower() == "binja-headless" memory_ceiling = ( _ida_memory_ceiling(available_memory_mb, ida_worker_memory_mb, database_size_mb) if is_ida + else _binja_memory_ceiling( + available_memory_mb, database_size_mb, measured_worker_memory_mb + ) + if is_binja else None ) @@ -51,11 +63,17 @@ def choose_jobs( if function_count < MIN_FUNCTIONS_FOR_AUTO or analysis_seconds is None: return 1 - if not is_ida and analysis_seconds > FAST_ANALYSIS_SECONDS: + if not (is_ida or is_binja) and analysis_seconds > FAST_ANALYSIS_SECONDS: return 1 cpus = cpu_count if cpu_count is not None else (os.cpu_count() or 1) - backend_limit = MAX_AUTO_IDA_JOBS if is_ida else MAX_AUTO_JOBS + backend_limit = ( + MAX_AUTO_IDA_JOBS + if is_ida + else MAX_AUTO_BINJA_JOBS + if is_binja + else MAX_AUTO_JOBS + ) ceiling = min(cpus, backend_limit, limit, function_count) if memory_ceiling is not None: ceiling = min(ceiling, memory_ceiling) @@ -83,6 +101,36 @@ def _ida_memory_ceiling( return max(1, available_memory_mb // worker_memory_mb) +def _binja_memory_ceiling( + available_memory_mb: int | None, + database_size_mb: int | None, + measured_worker_memory_mb: int | None = None, +) -> int | None: + if available_memory_mb is None: + return None + worker_memory_mb = _env_int( + "TOCODE_BINJA_WORKER_MEMORY_MB", DEFAULT_BINJA_WORKER_MEMORY_MB, 256 + ) + if database_size_mb is not None and database_size_mb > 0: + estimated = DEFAULT_BINJA_WORKER_BASE_MEMORY_MB + int( + database_size_mb * DEFAULT_BINJA_DB_RESIDENT_FACTOR + ) + worker_memory_mb = max(worker_memory_mb, estimated) + # A worker reopening the database holds about what the analyzing process + # peaked at (a 38k-function binary: ~1.8 GB, far above the size estimate). + if measured_worker_memory_mb is not None and measured_worker_memory_mb > 0: + worker_memory_mb = max(worker_memory_mb, measured_worker_memory_mb) + return max(1, available_memory_mb // worker_memory_mb) + + +def _env_int(name: str, default: int, minimum: int) -> int: + raw = os.environ.get(name, "").strip() + try: + return max(minimum, int(raw)) if raw else default + except ValueError: + return default + + def describe_jobs( *, function_count: int, @@ -103,6 +151,8 @@ def describe_jobs( return f"Workers: 1 after {analysis_seconds:.2f}s analysis" if backend.lower() == "ida": return f"Workers: {selected}, IDA cache ready" + if backend.lower() == "binja-headless": + return f"Workers: {selected}, Binary Ninja database ready" return f"Workers: {selected} after {analysis_seconds:.2f}s analysis" diff --git a/tests/conftest.py b/tests/conftest.py index e014a93..a2501fc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,3 +7,12 @@ def _no_native_memory_wait(monkeypatch: pytest.MonkeyPatch) -> None: """Exports in tests must not wait for free RAM on the host running them.""" monkeypatch.setenv("TOCODE_NATIVE_MIN_FREE_MB", "0") + + +@pytest.fixture(autouse=True) +def _no_local_binary_ninja(monkeypatch: pytest.MonkeyPatch) -> None: + """`auto` must not depend on a Binary Ninja installed on the test machine; + tests that exercise the headless backend clear this explicitly.""" + monkeypatch.setenv("TOCODE_DISABLE_BINJA_HEADLESS", "1") + # The CLI's --binja-license writes this variable; restore it after each test. + monkeypatch.setenv("TOCODE_BINJA_LICENSE", "") diff --git a/tests/fake_binja/binaryninja/__init__.py b/tests/fake_binja/binaryninja/__init__.py new file mode 100644 index 0000000..d171d90 --- /dev/null +++ b/tests/fake_binja/binaryninja/__init__.py @@ -0,0 +1,468 @@ +"""A stand-in for the Binary Ninja Python API, for ToCode's tests. + +It lives on disk (not in ``sys.modules``) so that spawned render workers import +it through ``--binja-dir`` exactly as they would a real install. Every view is the +same small x86-64 ELF-like program, whatever file is loaded. Environment: + +- ``FAKE_BINJA_LOG``: file that gets one line per ``load``/``create_database``. +- ``FAKE_BINJA_LICENSE=invalid``: the core refuses headless use, like a + personal license does, unless ``core_set_license`` was given ``FAKE-LICENSE``. +- ``FAKE_BINJA_SAVE=crash``: ``create_database`` kills the process, as some + headless builds do. +- ``FAKE_BINJA_ANALYSIS_SECONDS``: how long analysis takes (ten progress steps; + ``abort_analysis`` ends it early). + +A database written by ``create_database`` is the magic line plus the original +file name; loading any other ``.bndb`` fails like a corrupt database. +""" + +from __future__ import annotations + +import enum +import os +from pathlib import Path +import signal +import threading +from types import SimpleNamespace + +_MAGIC = "FAKEBNDB\n" + + +def _note(line: str) -> None: + path = os.environ.get("FAKE_BINJA_LOG") + if path: + with open(path, "a", encoding="utf-8") as handle: + handle.write(f"{os.getpid()} {line}\n") + + +_license_data: str | None = None + + +def _check_license() -> None: + if _license_data is not None and _license_data.strip() == "FAKE-LICENSE": + return + if _license_data is not None or os.environ.get("FAKE_BINJA_LICENSE") == "invalid": + raise RuntimeError("License is not valid. Please supply a valid license.") + + +def core_set_license(licenseData: str) -> None: + global _license_data + _note("core_set_license") # never the contents + _license_data = licenseData + + +def disable_default_log() -> None: + return None + + +def core_version() -> str: + return "0.0.0" + + +def core_product() -> str: + _check_license() + return "Binary Ninja Fake" + + +class SymbolType: + ImportedFunctionSymbol = "ImportedFunctionSymbol" + ImportAddressSymbol = "ImportAddressSymbol" + ImportedDataSymbol = "ImportedDataSymbol" + FunctionSymbol = "FunctionSymbol" + DataSymbol = "DataSymbol" + + +class SymbolBinding: + GlobalBinding = "GlobalBinding" + WeakBinding = "WeakBinding" + LocalBinding = "LocalBinding" + + +class LowLevelILOperation: + LLIL_STORE = "LLIL_STORE" + LLIL_SET_REG = "LLIL_SET_REG" + + +class TokenEscapingType(enum.IntEnum): + NoTokenEscapingType = 0 + BackticksTokenEscapingType = 1 + + +class AnalysisState(enum.IntEnum): + IdleState = 2 + AnalyzeState = 5 + + +class AnalysisProgress: + def __init__(self, state: AnalysisState, count: int, total: int) -> None: + self.state = state + self.count = count + self.total = total + + def __str__(self) -> str: + if self.state == AnalysisState.AnalyzeState: + return f"Analyzing ({self.count}/{self.total})" + return "Idle" + + +class TypeClass(enum.Enum): + StructureTypeClass = 1 + IntegerTypeClass = 2 + + +class DisassemblyOption: + ShowAddress = "ShowAddress" + WaitForIL = "WaitForIL" + + +class DisassemblySettings: + def set_option(self, option, value) -> None: + return None + + +class LinearViewObject: + @classmethod + def language_representation(cls, bv, settings, language=None): + return SimpleNamespace(bv=bv, language=language) + + +class LinearViewCursor: + def __init__(self, obj) -> None: + self.obj = obj + self.address = 0 + self.done = False + + def seek_to_address(self, address: int) -> None: + self.address = int(address) + + +class QualifiedName: + def __init__(self, name) -> None: + self.name = str(name) + + def __str__(self) -> str: + return self.name + + +class _Line: + def __init__(self, text: str, address: int) -> None: + self._text = text + self.contents = SimpleNamespace(address=address) + + def __str__(self) -> str: + return self._text + + +class _Var: + def __init__(self, name: str, type_: str, identifier: int) -> None: + self.name = name + self.type = type_ + self.identifier = identifier + + +class _Function: + def __init__( + self, + view, + start: int, + name: str, + *, + size: int, + symbol_type: str, + params=(), + local_vars=(), + return_type="int32_t", + hlil=True, + llil=None, + ) -> None: + self._view = view + self.start = start + self.name = name + self.total_bytes = size + self.highest_address = start + size - 1 + self.symbol = SimpleNamespace(type=symbol_type) + self.is_thunk = False + self.parameter_vars = list(params) + self.vars = list(params) + list(local_vars) + self.return_type = return_type + self.can_return = True + self.calling_convention = SimpleNamespace(name="sysv") + self.hlil = object() if hlil else None + self.callees: list[_Function] = [] + self.callers: list[_Function] = [] + self._llil = dict(llil or {}) + + @property + def instructions(self): + self._view._check_open() + return [((["push", " ", "rbp"]), self.start), ((["ret"]), self.highest_address)] + + def get_llil_at(self, address: int): + operation = self._llil.get(int(address)) + return None if operation is None else SimpleNamespace(operation=operation) + + +class _StructType: + type_class = TypeClass.StructureTypeClass + type = "StructureVariant.StructStructureType" + width = 8 + + def __str__(self) -> str: + return "struct point" + + +class _IntType: + type_class = TypeClass.IntegerTypeClass + width = 4 + + def __str__(self) -> str: + return "int32_t" + + +class FileMetadata: + def __init__(self, filename: str, original: str) -> None: + self.filename = filename + self.original_filename = original + self.closed = False + + def close(self) -> None: + self.closed = True + + +class BinaryView: + def __init__(self, filename: str, original: str) -> None: + self.file = FileMetadata(filename, original) + self.arch = SimpleNamespace(name="x86_64") + self.address_size = 8 + self.platform = SimpleNamespace(name="linux-x86_64") + self.view_type = "ELF" + self.start = 0x400000 + self.entry_point = 0x401000 + self.analyzed = False + self.analysis_progress = AnalysisProgress(AnalysisState.IdleState, 0, 0) + self._aborted = threading.Event() + text = SimpleNamespace( + name=".text", start=0x401000, length=0x100, type="PROGBITS" + ) + rodata = SimpleNamespace( + name=".rodata", start=0x403000, length=0x20, type="PROGBITS" + ) + data = SimpleNamespace( + name=".data", start=0x404000, length=0x10, type="PROGBITS" + ) + self.sections = {".text": text, ".rodata": rodata, ".data": data} + self._segments = [ + SimpleNamespace( + start=0x401000, + length=0x100, + readable=True, + writable=False, + executable=True, + ), + SimpleNamespace( + start=0x403000, + length=0x20, + readable=True, + writable=False, + executable=False, + ), + SimpleNamespace( + start=0x404000, + length=0x10, + readable=True, + writable=True, + executable=False, + ), + ] + main = _Function( + self, + 0x401000, + "main", + size=0x40, + symbol_type=SymbolType.FunctionSymbol, + params=[_Var("argc", "int32_t", 1)], + local_vars=[_Var("total", "int64_t", 2)], + llil={ + 0x401004: LowLevelILOperation.LLIL_SET_REG, + 0x401008: LowLevelILOperation.LLIL_STORE, + }, + ) + helper = _Function( + self, + 0x401080, + "helper", + size=0x20, + symbol_type=SymbolType.FunctionSymbol, + return_type="void", + ) + puts = _Function( + self, + 0x402000, + "puts", + size=0x10, + symbol_type=SymbolType.ImportedFunctionSymbol, + hlil=False, + ) + main.callees = [helper, puts] + helper.callers = [main] + puts.callers = [main] + self.functions = [main, helper, puts] + self.entry_functions = [main] + self._by_start = {func.start: func for func in self.functions} + self._symbols = [ + SimpleNamespace( + address=0x402000, + name="puts", + short_name="puts", + type=SymbolType.ImportedFunctionSymbol, + binding=SymbolBinding.GlobalBinding, + namespace=SimpleNamespace(name="libc.so.6"), + ordinal=None, + ), + SimpleNamespace( + address=0x401000, + name="main", + short_name="main", + type=SymbolType.FunctionSymbol, + binding=SymbolBinding.GlobalBinding, + namespace=SimpleNamespace(name=None), + ordinal=None, + ), + SimpleNamespace( + address=0x401080, + name="helper", + short_name="helper", + type=SymbolType.FunctionSymbol, + binding=SymbolBinding.LocalBinding, + namespace=SimpleNamespace(name=None), + ordinal=None, + ), + SimpleNamespace( + address=0x404000, + name="counter", + short_name="counter", + type=SymbolType.DataSymbol, + binding=SymbolBinding.GlobalBinding, + namespace=SimpleNamespace(name=None), + ordinal=None, + ), + ] + self._strings = [ + SimpleNamespace(start=0x403000, value="hello", length=5, type="AsciiString") + ] + self._code_refs = { + 0x403000: [SimpleNamespace(function=main, address=0x401004)], + 0x404000: [SimpleNamespace(function=main, address=0x401008)], + } + self.types = {"point": _StructType(), "score_t": _IntType()} + self.type_container = SimpleNamespace(name="container") + self._linear = { + 0x401000: [ + "int32_t main(int32_t argc)", + "{", + " helper();", + ' return puts("hello");', + "}", + ], + 0x401080: ["void helper()", "{", " return;", "}"], + } + + @property + def segments(self): + return list(self._segments) + + def _check_open(self) -> None: + if self.file.closed: + raise RuntimeError("use of a closed BinaryView") + + def update_analysis_and_wait(self) -> None: + self._check_open() + seconds = float(os.environ.get("FAKE_BINJA_ANALYSIS_SECONDS") or 0) + for step in range(10): + if self._aborted.is_set(): + break + self.analysis_progress = AnalysisProgress( + AnalysisState.AnalyzeState, step, 10 + ) + self._aborted.wait(seconds / 10) + self.analysis_progress = AnalysisProgress(AnalysisState.IdleState, 0, 0) + self.analyzed = not self._aborted.is_set() + + def abort_analysis(self) -> None: + _note("abort_analysis") + self._aborted.set() + + def create_database(self, filename: str, progress_func=None, settings=None) -> bool: + self._check_open() + _note(f"create_database {Path(filename).name}") + if os.environ.get("FAKE_BINJA_SAVE") == "crash": + os.kill(os.getpid(), signal.SIGSEGV) + Path(filename).write_text( + _MAGIC + self.file.original_filename, encoding="utf-8" + ) + return True + + def read(self, addr: int, length: int) -> bytes: + self._check_open() + return bytes((addr + index) & 0xFF for index in range(length)) + + def get_function_at(self, address: int): + self._check_open() + return self._by_start.get(int(address)) + + def get_symbols_of_type(self, symbol_type): + return [sym for sym in self._symbols if sym.type == symbol_type] + + def get_symbols(self): + return list(self._symbols) + + def get_strings(self): + return list(self._strings) + + def get_segment_at(self, address: int): + for segment in self._segments: + if segment.start <= address < segment.start + segment.length: + return segment + return None + + def get_sections_at(self, address: int): + return [ + s for s in self.sections.values() if s.start <= address < s.start + s.length + ] + + def get_code_refs(self, addr: int, length=None, max_items=None): + return iter(self._code_refs.get(int(addr), [])) + + def get_next_linear_disassembly_lines(self, cursor): + self._check_open() + if cursor.done: + return [] + cursor.done = True + return [ + _Line(text, cursor.address) for text in self._linear.get(cursor.address, []) + ] + + +def load(source, update_analysis: bool = True, progress_func=None, options=None): + _check_license() + path = Path(source) + if path.suffix == ".bndb": + text = path.read_text(encoding="utf-8", errors="replace") + if not text: + # Like Binary Ninja, an empty .bndb opens as a view with nothing + # mapped and no functions instead of failing. + _note(f"load {path.name} update_analysis={update_analysis}") + view = BinaryView(str(path), "") + view._segments, view.functions = [], [] + return view + if not text.startswith(_MAGIC): + raise Exception(f"{path} is not a valid Binary Ninja database") + original = text[len(_MAGIC) :] + else: + path.stat() + original = str(path) + _note(f"load {path.name} update_analysis={update_analysis}") + view = BinaryView(str(path), original) + if update_analysis: + view.update_analysis_and_wait() + return view diff --git a/tests/fake_binja/binaryninja/typeprinter.py b/tests/fake_binja/binaryninja/typeprinter.py new file mode 100644 index 0000000..5196b2d --- /dev/null +++ b/tests/fake_binja/binaryninja/typeprinter.py @@ -0,0 +1,18 @@ +"""``binaryninja.typeprinter`` for the fake API: one C definition per type.""" + +from __future__ import annotations + + +class _Printer: + def get_type_lines( + self, type, container, name, padding_cols=64, collapsed=False, escaping=1 + ): + if escaping != 0: + raise AssertionError("ToCode asks for unescaped C names") + if str(type) == "struct point": + return [f"struct {name}", "{", " int32_t x;", " int32_t y;", "};"] + return [f"typedef {type} {name};"] + + +class TypePrinter: + default = _Printer() diff --git a/tests/test_binja_backend.py b/tests/test_binja_backend.py index 12da546..ca595fa 100644 --- a/tests/test_binja_backend.py +++ b/tests/test_binja_backend.py @@ -17,6 +17,7 @@ from tocode.analysis import create_analyzer from tocode.backends.base import choose_backend from tocode.backends.binja import ( + strip_gutter, BinjaSession, describe_view, focused_view, @@ -362,6 +363,13 @@ def test_disasm_and_summary() -> None: assert "callees: 1" in summary +def test_strip_gutter_drops_marker_glyphs_and_keeps_indentation() -> None: + body = "\u2753 int64_t sub_4006f0()\n{\n\u2753 /* jump -> nullptr */\n return 0;\n}" + assert strip_gutter(body) == ( + "int64_t sub_4006f0()\n{\n /* jump -> nullptr */\n return 0;\n}" + ) + + def test_decompile_renders_pseudo_c() -> None: text = _session().decompile(0x401000) # The exporter parses the prototype before the opening brace. diff --git a/tests/test_binja_headless.py b/tests/test_binja_headless.py new file mode 100644 index 0000000..801f2b6 --- /dev/null +++ b/tests/test_binja_headless.py @@ -0,0 +1,586 @@ +"""Binary Ninja headless (the ``binaryninja`` API in process), against a fake API. + +The fake lives in ``tests/fake_binja`` and is found through ``TOCODE_BINJA_DIR`` +like a real install, so spawned render workers import it too. +""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import sys +import time +from types import SimpleNamespace + +import pytest + +from tocode import analysis as analysis_module +from tocode import cli +from tocode.analysis import create_analyzer +from tocode.backends import base, binja_headless +from tocode.backends.base import ( + choose_backend, + discover_binja_dir, + probe_binja_headless, +) +from tocode.backends.binja_headless import ( + BinjaHeadlessSession, + cached_database, + headless_problem, +) +from tocode.errors import BackendError, ToCodeError +from tocode.exporter import TIMEOUT_WORKER_BACKENDS, export_binary +from tocode.parallel import choose_jobs +from tocode.progress import Progress + +FAKE_DIR = Path(__file__).parent / "fake_binja" + + +def _drop_binaryninja_modules() -> None: + for name in [ + m for m in sys.modules if m == "binaryninja" or m.startswith("binaryninja.") + ]: + sys.modules.pop(name, None) + + +@pytest.fixture +def fake_binja(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + _drop_binaryninja_modules() + if importlib.util.find_spec("binaryninja") is not None: + pytest.skip("a real Binary Ninja API is importable in this environment") + # bootstrap_binja prepends the API folder to sys.path; undo it afterwards. + monkeypatch.setattr(sys, "path", list(sys.path)) + monkeypatch.delenv("TOCODE_DISABLE_BINJA_HEADLESS", raising=False) + monkeypatch.setenv("TOCODE_BINJA_DIR", str(FAKE_DIR)) + monkeypatch.setenv("TOCODE_BINJA_CACHE_DIR", str(tmp_path / "cache")) + log = tmp_path / "binja.log" + monkeypatch.setenv("FAKE_BINJA_LOG", str(log)) + yield SimpleNamespace(dir=FAKE_DIR, log=log) + _drop_binaryninja_modules() + + +def _binary(tmp_path: Path) -> Path: + path = tmp_path / "sample.elf" + path.write_bytes(b"\x7fELF" + bytes(range(64))) + return path + + +def _events(log: Path, *, probes: bool = False) -> list[tuple[int, str]]: + """(pid, event) per fake API call, in order; without the save probe's + child process unless ``probes``.""" + if not log.exists(): + return [] + rows = [] + for line in log.read_text(encoding="utf-8").splitlines(): + pid, event = line.split(" ", 1) + rows.append((int(pid), event)) + if probes: + return rows + probe_pids = {pid for pid, event in rows if event.startswith("load probe.bin")} + return [(pid, event) for pid, event in rows if pid not in probe_pids] + + +def _quiet() -> Progress: + return Progress(enabled=False) + + +# -- backend selection ------------------------------------------------------- + + +def test_auto_prefers_ida_then_binja_then_r2(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(base, "probe_binja_headless", lambda *_: True) + monkeypatch.setattr(base, "probe_r2", lambda: True) + monkeypatch.setattr(base, "probe_ida", lambda **_: base.IdaProbe(True, "found")) + assert choose_backend("auto").selected == "ida" + monkeypatch.setattr(base, "probe_ida", lambda **_: base.IdaProbe(False, "missing")) + assert choose_backend("auto").selected == "binja" + assert choose_backend("auto", skip=frozenset({"binja"})).selected == "r2" + + +def test_bndb_input_requires_binja() -> None: + assert choose_backend("auto", input_path=Path("app.bndb")).selected == "binja" + with pytest.raises(ToCodeError, match="bndb"): + choose_backend("ida", input_path=Path("app.bndb")) + + +def test_headless_renders_in_workers() -> None: + assert "binja-headless" in TIMEOUT_WORKER_BACKENDS + assert BinjaHeadlessSession.parallel_safe is True + assert BinjaHeadlessSession.backend_name == "binja" + + +def test_probe_honors_disable_switch( + fake_binja, monkeypatch: pytest.MonkeyPatch +) -> None: + assert probe_binja_headless() is True + monkeypatch.setenv("TOCODE_DISABLE_BINJA_HEADLESS", "1") + assert probe_binja_headless() is False + + +def test_discover_binja_dir_from_install_folder(tmp_path: Path) -> None: + package = tmp_path / "binaryninja" / "python" / "binaryninja" + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + assert discover_binja_dir(tmp_path / "binaryninja") == package.parent.resolve() + + +def test_discover_binja_dir_from_install_api_pth( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for name in ("TOCODE_BINJA_DIR", "BN_INSTALL_DIR", "BINJA_DIR", "APPDATA"): + monkeypatch.delenv(name, raising=False) + python_dir = tmp_path / "install" / "python" + (python_dir / "binaryninja").mkdir(parents=True) + (python_dir / "binaryninja" / "__init__.py").write_text("") + site = tmp_path / "home" / ".local" / "lib" / "python3.12" / "site-packages" + site.mkdir(parents=True) + (site / "binaryninja.pth").write_text(f"{python_dir}\n") + monkeypatch.setenv("HOME", str(tmp_path / "home")) + assert discover_binja_dir() == python_dir.resolve() + + +def test_choose_jobs_caps_binja_workers_by_memory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("TOCODE_BINJA_WORKER_MEMORY_MB", raising=False) + + def jobs(available_memory_mb: int, measured: int | None = None) -> int: + return choose_jobs( + function_count=5000, + analysis_seconds=30.0, + requested=None, + backend="binja-headless", + cpu_count=32, + job_limit=64, + available_memory_mb=available_memory_mb, + measured_worker_memory_mb=measured, + ) + + assert jobs(64 * 1024) == 4 + assert jobs(2500) == 2 + # What the analyzing process peaked at wins over the static estimate. + assert jobs(5000, measured=1800) == 2 + assert jobs(5000, measured=200) == 4 + + +# -- the API itself ---------------------------------------------------------- + + +def test_api_not_found_names_the_flag(monkeypatch: pytest.MonkeyPatch) -> None: + _drop_binaryninja_modules() + if importlib.util.find_spec("binaryninja") is not None: + pytest.skip("a real Binary Ninja API is importable in this environment") + monkeypatch.setattr(base, "discover_binja_dir", lambda *_: None) + problem = headless_problem() + assert problem is not None and "--binja-dir" in problem + + +def test_refused_license_is_reported(fake_binja, tmp_path: Path, monkeypatch) -> None: + assert headless_problem() is None + monkeypatch.setenv("FAKE_BINJA_LICENSE", "invalid") + problem = headless_problem() + assert problem is not None and "License is not valid" in problem + with pytest.raises(BackendError, match="license"): + BinjaHeadlessSession(_binary(tmp_path)) + + +def _license(tmp_path: Path, text: str = "FAKE-LICENSE") -> Path: + path = tmp_path / "license.dat" + path.write_text(text, encoding="utf-8") + return path + + +def test_license_file_is_handed_to_the_core( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("FAKE_BINJA_LICENSE", "invalid") # no usable default license + monkeypatch.setenv("TOCODE_BINJA_LICENSE", str(_license(tmp_path))) + assert headless_problem() is None + BinjaHeadlessSession(_binary(tmp_path)).close() + # Set once per process, before the first core call; contents never logged. + events = [event for _, event in _events(fake_binja.log)] + assert events[0] == "core_set_license" + assert events.count("core_set_license") == 1 + assert "FAKE-LICENSE" not in fake_binja.log.read_text() + + +def test_license_file_the_core_rejects_is_reported( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TOCODE_BINJA_LICENSE", str(_license(tmp_path, "expired"))) + problem = headless_problem() + assert problem is not None and "License is not valid" in problem + assert "expired" not in problem + + +def test_missing_license_file_is_reported( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TOCODE_BINJA_LICENSE", str(tmp_path / "missing.dat")) + problem = headless_problem() + assert problem is not None and "cannot read the Binary Ninja license" in problem + + +# -- session: database cache --------------------------------------------------- + + +def test_analysis_is_cached_as_a_database(fake_binja, tmp_path: Path) -> None: + binary = _binary(tmp_path) + session = BinjaHeadlessSession(binary) + session.analyze() + cache = cached_database(binary) + assert cache.is_file() + assert session.database_path() == cache + session.close() + + again = BinjaHeadlessSession(binary) + again.analyze() + again.close() + assert [event for _, event in _events(fake_binja.log)] == [ + "load sample.elf update_analysis=False", + f"create_database {cache.name}", + f"load {cache.name} update_analysis=False", + ] + + +def test_save_probe_runs_once_per_core(fake_binja, tmp_path: Path) -> None: + for _ in range(2): + cached_database(_binary(tmp_path)).unlink(missing_ok=True) + session = BinjaHeadlessSession(_binary(tmp_path)) + session.analyze() + session.close() + probes = [e for _, e in _events(fake_binja.log, probes=True) if "probe.bin" in e] + assert probes == ["load probe.bin update_analysis=False"] + + +def test_core_that_crashes_saving_renders_in_process( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("FAKE_BINJA_SAVE", "crash") + binary = _binary(tmp_path) + out = tmp_path / "out" + with create_analyzer( + binary, backend="binja", binja_dir=FAKE_DIR, progress=_quiet() + ) as analyzer: + summary = export_binary(analyzer, out_dir=out, jobs=2) + assert not analyzer.supports_parallel + assert summary.function_count == 2 and not summary.failed_functions + assert not cached_database(binary).exists() + assert not list(out.glob("*.bndb")) + sources = "\n".join(path.read_text() for path in (out / "src" / "raw").rglob("*.c")) + assert "helper();" in sources + # Only the probe child crashed; this process never called create_database + # and no render worker was spawned. + events = _events(fake_binja.log) + assert {pid for pid, _ in events} == {events[0][0]} + assert not any(event.startswith("create_database") for _, event in events) + + +def test_purge_cache_rebuilds_the_database(fake_binja, tmp_path: Path) -> None: + binary = _binary(tmp_path) + first = BinjaHeadlessSession(binary) + first.analyze() + first.close() + purged = BinjaHeadlessSession(binary, purge_cache=True) + purged.analyze() + purged.close() + events = [event for _, event in _events(fake_binja.log)] + assert events.count("load sample.elf update_analysis=False") == 2 + assert sum(event.startswith("create_database") for event in events) == 2 + + +def test_unreadable_cache_is_rebuilt(fake_binja, tmp_path: Path) -> None: + binary = _binary(tmp_path) + cache = cached_database(binary) + cache.parent.mkdir(parents=True) + cache.write_text("half-written database") + log = tmp_path / "tocode.log" + session = BinjaHeadlessSession( + binary, progress=Progress(enabled=False, log_path=log) + ) + session.analyze() + session.close() + assert "Discarding unreadable cached Binary Ninja database" in log.read_text() + assert cache.read_text().startswith("FAKEBNDB") + + +def test_empty_cache_from_a_crashed_save_is_rebuilt(fake_binja, tmp_path: Path) -> None: + binary = _binary(tmp_path) + cache = cached_database(binary) + cache.parent.mkdir(parents=True) + cache.write_bytes(b"") # what a save that crashed leaves behind + out = tmp_path / "out" + with create_analyzer( + binary, backend="binja", binja_dir=FAKE_DIR, progress=_quiet() + ) as analyzer: + summary = export_binary(analyzer, out_dir=out, jobs=1) + assert summary.function_count == 2 + assert list((out / "src" / "raw").rglob("*.c")) + assert cache.read_text().startswith("FAKEBNDB") + assert (out / "sample.bndb").read_text().startswith("FAKEBNDB") + + +def test_empty_database_input_is_an_error(fake_binja, tmp_path: Path) -> None: + database = tmp_path / "app.bndb" + database.write_bytes(b"") + with pytest.raises(BackendError, match="empty or incomplete"): + BinjaHeadlessSession(database) + + +def test_analysis_progress_is_shown( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys +) -> None: + monkeypatch.setattr(binja_headless, "ANALYSIS_POLL_SECONDS", 0.02) + monkeypatch.setenv("FAKE_BINJA_ANALYSIS_SECONDS", "0.5") + session = BinjaHeadlessSession(_binary(tmp_path), progress=Progress()) + session.analyze() + session.close() + assert "binja analyzing" in capsys.readouterr().err + + +def test_ctrl_c_aborts_the_analysis( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(binja_headless, "ANALYSIS_POLL_SECONDS", 0.02) + monkeypatch.setenv("FAKE_BINJA_ANALYSIS_SECONDS", "60") + + def interrupt(bar, bv) -> None: + raise KeyboardInterrupt + + monkeypatch.setattr(binja_headless, "_show_analysis_progress", interrupt) + session = BinjaHeadlessSession(_binary(tmp_path), progress=Progress()) + started = time.monotonic() + with pytest.raises(KeyboardInterrupt): + session.analyze() + session.close() + assert time.monotonic() - started < 10 + assert "abort_analysis" in [event for _, event in _events(fake_binja.log)] + + +def test_worker_memory_is_measured_after_analysis(fake_binja, tmp_path: Path) -> None: + session = BinjaHeadlessSession(_binary(tmp_path)) + assert session.worker_memory_mb() is None + session.analyze() + if sys.platform != "win32": + measured = session.worker_memory_mb() + assert measured is not None and measured > 0 + session.close() + + +def test_database_input_is_opened_in_place(fake_binja, tmp_path: Path) -> None: + binary = _binary(tmp_path) + database = tmp_path / "sample.bndb" + database.write_text(f"FAKEBNDB\n{binary}") + session = BinjaHeadlessSession(database) + session.analyze() + assert session.database_path() == database + assert session.info()["tocode"]["input_path"] == str(binary) + session.close() + events = [event for _, event in _events(fake_binja.log)] + assert events == ["load sample.bndb update_analysis=False"] + assert not (tmp_path / "cache").exists() + + +def test_read_bytes_reopens_the_view_after_workers(fake_binja, tmp_path: Path) -> None: + session = BinjaHeadlessSession(_binary(tmp_path)) + session.analyze() + session.release_parallel_resources() + assert session.read_bytes(0x401000, 4) == b"\x00\x01\x02\x03" + session.close() + cache = cached_database(_binary(tmp_path)) + assert _events(fake_binja.log)[-1][1] == f"load {cache.name} update_analysis=True" + + +def test_types_and_data_xrefs(fake_binja, tmp_path: Path) -> None: + session = BinjaHeadlessSession(_binary(tmp_path)) + session.analyze() + types = {row["name"]: row for row in session.types()} + assert types["point"]["kind"] == "struct" + assert types["point"]["size"] == 8 + assert "int32_t x;" in types["point"]["c_decl"] + assert types["score_t"]["c_decl"] == "typedef int32_t score_t;" + assert session.data_xrefs({0x403000, 0x404000, 0x999}) == { + 0x403000: [(0x401004, False)], + 0x404000: [(0x401008, True)], + } + session.close() + + +# -- create_analyzer ----------------------------------------------------------- + + +class _FakeR2: + backend_name = "r2" + + def __init__(self, binary, analysis_command=None) -> None: + self.binary = binary + + def close(self) -> None: + return None + + +def test_auto_falls_back_when_license_refuses_headless( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("FAKE_BINJA_LICENSE", "invalid") + monkeypatch.setattr(base, "probe_ida", lambda **_: base.IdaProbe(False, "missing")) + monkeypatch.setattr(base, "probe_r2", lambda: True) + monkeypatch.setattr(analysis_module, "R2Session", _FakeR2) + with create_analyzer(_binary(tmp_path), progress=_quiet()) as analyzer: + assert isinstance(analyzer.session, _FakeR2) + + +def test_explicit_binja_reports_refused_license( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("FAKE_BINJA_LICENSE", "invalid") + with pytest.raises(BackendError, match="License is not valid"): + create_analyzer(_binary(tmp_path), backend="binja", progress=_quiet()) + + +def test_export_renders_in_workers_from_the_cached_database( + fake_binja, tmp_path: Path +) -> None: + binary = _binary(tmp_path) + out = tmp_path / "out" + with create_analyzer( + binary, backend="binja", binja_dir=FAKE_DIR, progress=_quiet() + ) as analyzer: + assert isinstance(analyzer.session, BinjaHeadlessSession) + summary = export_binary(analyzer, out_dir=out, jobs=2) + + assert summary.function_count == 2 # main and helper; the puts stub is an import + assert not summary.failed_functions + # The finished analysis ships with the export, like IDA's .i64. + assert (out / "sample.bndb").read_text().startswith("FAKEBNDB") + sources = "\n".join(path.read_text() for path in (out / "src" / "raw").rglob("*.c")) + assert "int32_t main(int32_t argc)" in sources + assert "helper();" in sources + types_header = next((out / "include").glob("*.types.h")).read_text() + assert "struct point" in types_header + # Section bytes come from the view, which the parent reopened after the + # workers finished (it closes its own view while they render). + assert (out / "data" / "text.bin").read_bytes() == bytes( + (0x401000 + index) & 0xFF for index in range(0x100) + ) + strings = json.dumps(json.loads((out / "strings.json").read_text())) + assert '"function": "main"' in strings and '"access": "read"' in strings + variables = json.dumps(json.loads((out / "data" / "variables.json").read_text())) + assert '"access": "write"' in variables + assert json.loads((out / "project.json").read_text())["backend"] == "binja" + + cache = cached_database(binary).name + events = _events(fake_binja.log) + parent = events[0][0] + worker_loads = [event for pid, event in events if pid != parent] + assert worker_loads and all( + event == f"load {cache} update_analysis=False" for event in worker_loads + ) + assert (parent, f"load {cache} update_analysis=True") in events + + +# -- CLI ------------------------------------------------------------------------- + + +def _record_run_binja(monkeypatch: pytest.MonkeyPatch) -> list: + calls: list = [] + + def fake_run_binja(args, progress, parser, argv): + calls.append(args) + return 0 + + monkeypatch.setattr(cli, "_run_binja", fake_run_binja) + return calls + + +def test_cli_exports_a_file_headless( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + calls = _record_run_binja(monkeypatch) + monkeypatch.setenv("TOCODE_FUNCTION_TIMEOUT_SECONDS", "0") # render in process + out = tmp_path / "out" + assert ( + cli.main(["--backend", "binja", str(_binary(tmp_path)), "-o", str(out), "-q"]) + == 0 + ) + assert not calls + assert (out / "sample.bndb").is_file() + assert (out / "functions.json").is_file() + + +def test_cli_auto_transport_falls_back_to_running_binja( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + calls = _record_run_binja(monkeypatch) + monkeypatch.setenv("FAKE_BINJA_LICENSE", "invalid") + assert cli.main(["--backend", "binja", str(_binary(tmp_path)), "-q"]) == 0 + assert len(calls) == 1 + + +def test_cli_headless_transport_reports_refused_license( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys +) -> None: + calls = _record_run_binja(monkeypatch) + monkeypatch.setenv("FAKE_BINJA_LICENSE", "invalid") + argv = [ + "--backend", + "binja", + "--binja-transport", + "headless", + str(_binary(tmp_path)), + "-q", + ] + assert cli.main(argv) == 1 + assert not calls + assert "License is not valid" in capsys.readouterr().err + + +def test_cli_routes_database_input_to_binja( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + calls = _record_run_binja(monkeypatch) + database = tmp_path / "app.bndb" + database.write_text("db") + assert cli.main([str(database), "-q"]) == 0 # no API here: a running Binary Ninja + assert len(calls) == 1 + assert calls[0].backend == "binja" + + +def test_cli_license_reaches_render_workers( + fake_binja, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + calls = _record_run_binja(monkeypatch) + monkeypatch.setenv("FAKE_BINJA_LICENSE", "invalid") + out = tmp_path / "out" + argv = [ + "--backend", + "binja", + "--binja-transport", + "headless", + "--binja-license", + str(_license(tmp_path)), + str(_binary(tmp_path)), + "-o", + str(out), + "-j", + "2", + "-q", + ] + assert cli.main(argv) == 0 + assert not calls + assert (out / "sample.bndb").is_file() + events = _events(fake_binja.log) + # Every process that loaded a view (the parent and each spawned worker) + # set the license first. + loaders = {pid for pid, event in events if event.startswith("load ")} + licensed = {pid for pid, event in events if event == "core_set_license"} + assert len(loaders) > 1 and loaders <= licensed + assert "FAKE-LICENSE" not in (out / "tocode.log").read_text() + + +def test_cli_rejects_a_missing_license_file(tmp_path: Path, capsys) -> None: + argv = ["--binja-license", str(tmp_path / "missing.dat"), str(_binary(tmp_path))] + with pytest.raises(SystemExit): + cli.main(argv) + assert "--binja-license" in capsys.readouterr().err diff --git a/tests/test_binja_mcp.py b/tests/test_binja_mcp.py new file mode 100644 index 0000000..06a853d --- /dev/null +++ b/tests/test_binja_mcp.py @@ -0,0 +1,736 @@ +"""Binary Ninja over its built-in MCP server, against an in-process fake.""" + +from __future__ import annotations + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import threading +from typing import Any, Iterator + +import pytest + +from tocode.backends import binja_mcp +from tocode.backends.mcp_client import ( + McpClient, + fenced_body, + front_matter, + markdown_tables, + split_row, +) +from tocode.errors import BackendError + + +def _table( + tool: str, + headers: list[str], + rows: list[list[str]], + offset: int = 0, + limit: int = 100, +) -> str: + page = rows[offset : offset + limit] + next_offset = offset + len(page) if offset + len(page) < len(rows) else None + escaped = [[cell.replace("|", "\\|") for cell in row] for row in page] + lines = [ + "---", + f'tool: "{tool}"', + f"offset: {offset}", + f"limit: {limit}", + f"count: {len(page)}", + f"total: {len(rows)}", + f"nextOffset: {'null' if next_offset is None else next_offset}", + f"truncated: {'true' if next_offset is not None else 'false'}", + "---", + "", + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + *("| " + " | ".join(row) + " |" for row in escaped), + "", + ] + return "\n".join(lines) + + +def _fenced(tool: str, lines: list[str], offset: int, limit: int, language: str) -> str: + page = lines[offset : offset + limit] + next_offset = offset + len(page) if offset + len(page) < len(lines) else None + return ( + f'---\ntool: "{tool}"\noffset: {offset}\nlimit: {limit}\ncount: {len(page)}\n' + f"total: {len(lines)}\nnextOffset: {'null' if next_offset is None else next_offset}\n---\n\n" + f"````{language}\n" + "\n".join(page) + "\n````\n" + ) + + +MAIN = 0x401000 +HELPER = 0x401100 +STUB = 0x402000 +DECOMPILE = { + MAIN: [ + "🌐 int32_t main(int32_t argc, char** argv)", + "", + "{", + " helper(argc);", + ' return puts("a|b");', + "}", + ], + HELPER: [" int32_t helper(int32_t arg1)", "", "{", " return arg1 + 1;", "}"], +} + + +class FakeBinja: + """State and tool handlers of a fake Binary Ninja MCP server.""" + + def __init__(self) -> None: + self.active = "view_2" + self.calls: list[tuple[str, dict[str, Any]]] = [] + self.opened: list[str] = [] + + def view(self, handle: str) -> dict[str, Any]: + return { + "handle": handle, + "openItem": "item_1" if handle == "view_1" else "item_2", + "viewType": "ELF", + "architecture": "x86_64", + "platform": "linux-x86_64", + "addressSize": 8, + "start": "0x400000", + "end": "0x403000", + "entryPoint": hex(MAIN), + } + + def handle( + self, name: str, args: dict[str, Any] + ) -> tuple[str, dict[str, Any] | None, bool]: + self.calls.append((name, args)) + offset, limit = args.get("offset", 0), args.get("limit", 100) + views = [ + [ + "view_1", + "item_1", + "ELF", + "true", + "true" if self.active == "view_1" else "false", + "true", + "x86_64", + "linux-x86_64", + "8", + "0x400000", + hex(MAIN), + ], + [ + "view_1r", + "item_1", + "Raw", + "true", + "false", + "true", + "", + "", + "4", + "0x0", + "0x0", + ], + [ + "view_2", + "item_2", + "ELF", + "true", + "true" if self.active == "view_2" else "false", + "true", + "x86_64", + "linux-x86_64", + "8", + "0x400000", + hex(MAIN), + ], + ] + if name == "bn_binary_view_list": + return ( + _table( + name, + [ + "binaryView", + "openItem", + "viewType", + "created", + "active", + "available", + "architecture", + "platform", + "addressSize", + "start", + "entryPoint", + ], + views, + offset, + limit, + ), + None, + False, + ) + if name == "bn_open_item_list": + items = [ + ["item_1", "file", "/bins/app", "/bins/app"], + ["item_2", "database", "/bins/other.bndb", "/bins/other.bndb"], + ] + items += [ + [f"item_{9 + i}", "file", path, path] + for i, path in enumerate(self.opened) + ] + return ( + _table( + name, + ["openItem", "kind", "displayName", "path"], + items, + offset, + limit, + ), + None, + False, + ) + if name == "bn_binary_view_get_active": + return "", {"binaryView": self.view(self.active)}, False + if name == "bn_binary_view_set_active": + self.active = args["binaryView"] + return "", {"binaryView": self.view(self.active), "changed": True}, False + if name == "bn_analysis_status": + return "", {"analysis": {"state": "IdleState"}}, False + if name == "bn_function_list": + rows = [ + [ + hex(MAIN), + "x86_64", + "main", + hex(MAIN), + hex(MAIN + 0x3F), + "int32_t(int32_t argc, char** argv)", + "3", + ], + [ + hex(HELPER), + "x86_64", + "helper", + hex(HELPER), + hex(HELPER + 0xF), + "int32_t(int32_t arg1)", + "1", + ], + [ + hex(STUB), + "x86_64", + "puts", + hex(STUB), + hex(STUB + 5), + "int32_t(char const* s)", + "1", + ], + ] + return ( + _table( + name, + [ + "function", + "arch", + "name", + "lowestAddress", + "highestAddress", + "definition", + "basicBlockCount", + ], + rows, + offset, + limit, + ), + None, + False, + ) + if name == "bn_import_list": + rows = [ + [ + hex(STUB), + "puts", + "ImportedFunctionSymbol", + "libc.so.6", + "0", + "NoBinding", + ] + ] + return ( + _table( + name, + ["symbol", "name", "type", "namespace", "ordinal", "binding"], + rows, + offset, + limit, + ), + None, + False, + ) + if name == "bn_export_list": + rows = [ + [ + hex(MAIN), + "main", + "FunctionSymbol", + "BNINTERNALNAMESPACE", + "0", + "GlobalBinding", + ] + ] + return ( + _table( + name, + ["symbol", "name", "type", "namespace", "ordinal", "binding"], + rows, + offset, + limit, + ), + None, + False, + ) + if name == "bn_symbol_list": + rows = [ + [ + hex(MAIN), + "main", + "FunctionSymbol", + "BNINTERNALNAMESPACE", + "0", + "GlobalBinding", + "main", + ] + ] + return ( + _table( + name, + [ + "symbol", + "name", + "type", + "namespace", + "ordinal", + "binding", + "fullName", + ], + rows, + offset, + limit, + ), + None, + False, + ) + if name == "bn_section_list": + rows = [ + [".text", "0x401000", "0x402000", "0x1000", "PROGBITS"], + [".plt", "0x402000", "0x402100", "0x100", "PROGBITS"], + ] + return ( + _table( + name, + ["section", "start", "end", "length", "typeName"], + rows, + offset, + limit, + ), + None, + False, + ) + if name == "bn_segment_list": + rows = [["0x400000", "0x400000", "0x403000", "true", "false", "true"]] + return ( + _table( + name, + ["segment", "start", "end", "readable", "writable", "executable"], + rows, + offset, + limit, + ), + None, + False, + ) + if name == "bn_string_list": + rows = [["0x401f00", "AsciiString", "0x3", "a|b", "617c62", "false"]] + return ( + _table( + name, + [ + "string", + "type", + "length", + "previewEscaped", + "previewHex", + "previewTruncated", + ], + rows, + offset, + limit, + ), + None, + False, + ) + if name == "bn_relocation_list": + rows = [ + [ + "0x403000", + hex(STUB), + "x86_64", + hex(STUB), + "puts", + "ELFJumpSlotRelocationType", + ] + ] + return ( + _table( + name, + [ + "address", + "target", + "architecture", + "symbol", + "symbolName", + "type", + ], + rows, + offset, + limit, + ), + None, + False, + ) + if name == "bn_entry_point_list": + return ( + _table( + name, + ["address", "function", "arch", "name"], + [[hex(MAIN), hex(MAIN), "x86_64", "main"]], + offset, + limit, + ), + None, + False, + ) + if name == "bn_type_list": + return ( + _table( + name, + ["type", "id", "class", "definition", "width"], + [["point", "x", "StructureTypeClass", "struct point", "8"]], + offset, + limit, + ), + None, + False, + ) + if name == "bn_type_info": + return ( + "", + { + "type": { + "source": "struct point\n{\n int32_t x;\n int32_t y;\n};" + } + }, + False, + ) + if name == "bn_function_decompile": + address = int(args["function"], 16) + return ( + _fenced( + name, + DECOMPILE[address], + offset, + args.get("limit", 1000), + "bn-pseudoc", + ), + None, + False, + ) + if name == "bn_function_disassembly": + address = int(args["function"], 16) + lines = [ + " int32_t f()", + "", + f"0x{address:x} push rbp", + f"0x{address + 1:x} call helper", + ] + return ( + _fenced(name, lines, offset, args.get("limit", 1000), "asm"), + None, + False, + ) + if name == "bn_function_callees": + address = int(args["function"], 16) + rows = ( + [ + [ + "0x401010", + "x86_64", + hex(HELPER), + f"helper ({hex(HELPER)} (x86_64))", + ], + ["0x401020", "x86_64", hex(STUB), f"puts ({hex(STUB)} (x86_64))"], + ] + if address == MAIN + else [] + ) + return ( + _table( + name, + ["Callsite", "Architecture", "Callee Address", "Callee Functions"], + rows, + offset, + limit, + ), + None, + False, + ) + if name == "bn_memory_read": + data = bytes([0x90]) * int(args["length"]) + return "", {"hex": data.hex(), "bytesRead": len(data)}, False + if name == "bn_open_item_open": + self.opened.append(args["path"]) + return "", {"openItem": {"handle": f"item_{8 + len(self.opened)}"}}, False + return ( + json.dumps({"errorCode": "unknown_tool"}), + {"errorCode": "unknown_tool", "errorMessage": name}, + True, + ) + + +@pytest.fixture +def server() -> Iterator[tuple[str, FakeBinja]]: + fake = FakeBinja() + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_args: Any) -> None: + return + + def do_POST(self) -> None: # noqa: N802 - http.server API + payload = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + if "id" not in payload: + self.send_response(202) + self.end_headers() + return + method = payload["method"] + if method == "initialize": + result: dict[str, Any] = { + "serverInfo": {"name": "binaryninja_ui_mcp"}, + "capabilities": {}, + } + elif method == "tools/call": + text, structured, error = fake.handle( + payload["params"]["name"], payload["params"]["arguments"] + ) + result = {"content": [{"type": "text", "text": text}], "isError": error} + if structured is not None: + result["structuredContent"] = structured + else: + result = {} + body = json.dumps( + {"jsonrpc": "2.0", "id": payload["id"], "result": result} + ).encode() + self.send_response(200) + if payload["params"].get("name") == "bn_section_list": + # Exercise the event-stream response form too. + body = b"event: message\ndata: " + body + b"\n\n" + self.send_header("Content-Type", "text/event-stream") + else: + self.send_header("Content-Type", "application/json") + self.send_header("MCP-Session-Id", "s1") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{httpd.server_address[1]}/mcp", fake + finally: + httpd.shutdown() + + +# -- client helpers ----------------------------------------------------------- + + +def test_markdown_helpers_handle_escaped_pipes_and_front_matter() -> None: + text = _table("t", ["a", "b"], [["x|y", "1"]]) + assert ( + front_matter(text)["tool"] == "t" and front_matter(text)["nextOffset"] is None + ) + assert markdown_tables(text) == [[{"a": "x|y", "b": "1"}]] + assert split_row("| a \\| b | c |") == ["a | b", "c"] + assert fenced_body("````asm\nline\n````\n") == "line\n" + + +def test_client_rejects_a_server_that_is_not_binary_ninja(server) -> None: + url, _fake = server + client = McpClient(url).connect() + assert client.session_id == "s1" + with pytest.raises(BackendError, match="could not reach"): + binja_mcp.connect("http://127.0.0.1:1/mcp", timeout=1) + + +# -- session ------------------------------------------------------------------ + + +def test_session_maps_tools_and_restores_the_active_view(server) -> None: + url, fake = server + client = binja_mcp.connect(url) + views = binja_mcp.list_views(client) + assert [view.handle for view in views] == ["view_1", "view_2"] # Raw skipped + assert views[0].path == "/bins/app" and views[1].active + + session = binja_mcp.BinjaMcpSession(client, views[0]) + assert fake.active == "view_1" + assert session.info()["bin"]["baddr"] == 0x400000 + + functions = {row["addr"]: row for row in session.functions()} + assert functions[MAIN]["signature"] == "int32_t main(int32_t argc, char** argv)" + assert functions[MAIN]["params"] == [ + {"name": "argc", "type": "int32_t"}, + {"name": "argv", "type": "char**"}, + ] + assert functions[STUB]["is_library"] and functions[STUB]["is_thunk"] + assert functions[MAIN]["size"] == 0x40 + imports = session.imports() + assert imports[0]["dll"] == "libc.so.6" + assert session.strings()[0]["string"] == "a|b" + assert session.sections()[0]["perm"] == "r-x" # via the event-stream response + assert "int32_t x;" in session.types()[0]["c_decl"] + + text = session.decompile(MAIN) + assert text.startswith( + "int32_t main(int32_t argc, char** argv)\n" + ) # gutter glyph gone + assert 'puts("a|b")' in text + assert session.disasm(MAIN).splitlines()[0] == f"0x{MAIN:x}: push rbp" + + class Routine: + def __init__(self, name: str, imported: bool) -> None: + self.name, self.imported = name, imported + + class Import: + name = "puts" + + known = { + MAIN: Routine("main", False), + HELPER: Routine("helper", False), + STUB: Routine("puts", True), + } + assert session.calls_from(MAIN, {STUB: Import()}, known) == ([HELPER], ["puts"]) + + session.close() + assert fake.active == "view_2" # the user's view is active again + + +def test_decompile_follows_pagination(server, monkeypatch: pytest.MonkeyPatch) -> None: + url, fake = server + monkeypatch.setattr(binja_mcp, "PAGE", 2) + client = binja_mcp.connect(url) + session = binja_mcp.BinjaMcpSession(client, binja_mcp.list_views(client)[0]) + + text = session.decompile(MAIN) + + assert text.count("\n") == len(DECOMPILE[MAIN]) + pages = [args for name, args in fake.calls if name == "bn_function_decompile"] + assert [args["offset"] for args in pages] == [0, 2, 4] + + +def test_find_view_for_path_matches_files_and_databases(server) -> None: + url, _fake = server + views = binja_mcp.list_views(binja_mcp.connect(url)) + by_file = binja_mcp.find_view_for_path(views, Path("/bins/app")) + by_database = binja_mcp.find_view_for_path(views, Path("/bins/other")) + assert by_file is not None and by_file.handle == "view_1" + assert by_database is not None and by_database.handle == "view_2" + assert binja_mcp.find_view_for_path(views, Path("/bins/none")) is None + + +# -- CLI ------------------------------------------------------------------------ + + +def test_cli_lists_views_over_mcp(server, capsys) -> None: + from tocode.cli import main + + url, _fake = server + assert main(["--backend", "binja", "--binja-mcp-url", url, "--list-binja"]) == 0 + out = capsys.readouterr().out + assert ( + "[0] ELF x86_64 64-bit /bins/app" in out and "[1]" in out and "[active]" in out + ) + + +def test_cli_exports_a_view_over_mcp(server, tmp_path: Path) -> None: + from tocode.cli import main + + url, fake = server + out = tmp_path / "out" + code = main( + [ + "--backend", + "binja", + "--binja-mcp-url", + url, + "--binja-view", + "0", + "-o", + str(out), + "-q", + ] + ) + + assert code == 0 + sources = "".join(path.read_text() for path in (out / "src" / "raw").rglob("*.c")) + assert ( + "int32_t main(int32_t argc, char** argv)" in sources + and "helper(argc);" in sources + ) + project = json.loads((out / "project.json").read_text()) + assert project["backend"] == "binja" and project["image_base"] == "0x400000" + # Section bytes come from Binary Ninja's view, not from a local file. + assert (out / "data" / "text.bin").read_bytes() == b"\x90" * 0x1000 + assert fake.active == "view_2" + + +def test_cli_falls_back_to_rpyc_when_mcp_is_down(monkeypatch, capsys) -> None: + from tocode import cli + from tocode.backends import binja as binja_backend + + seen = {} + + def fake_rpyc(host, port): + seen["rpyc"] = (host, port) + raise BackendError("no binja-headless") + + monkeypatch.setattr(binja_backend, "connect_rpyc", fake_rpyc) + + assert ( + cli.main( + [ + "--backend", + "binja", + "--binja-mcp-url", + "http://127.0.0.1:1/mcp", + "--list-binja", + ] + ) + == 1 + ) + assert seen["rpyc"] == ("127.0.0.1", 18812) + assert "trying binja-headless" in capsys.readouterr().err + + seen.clear() + assert ( + cli.main( + [ + "--backend", + "binja", + "--binja-transport", + "mcp", + "--binja-mcp-url", + "http://127.0.0.1:1/mcp", + "--list-binja", + ] + ) + == 1 + ) + assert "rpyc" not in seen # forced MCP never falls back From eab2ee117176a68e3ed90393291e138ee860fb89 Mon Sep 17 00:00:00 2001 From: buzzer-re <22428720+buzzer-re@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:48:53 -0300 Subject: [PATCH 4/4] Simplify the README Trim implementation detail from the Binary Ninja and APK native sections and move the resume note out of the example's code block. Co-Authored-By: Claude Opus 5.5 --- README.md | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 2be8e88..c1963e7 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ tocode Vendor.Lib.1.0.0.nupkg - `src/raw//.java`: one decompiled Java file per class, folders follow Java packages (no clustering, no summaries). - `AndroidManifest.xml` + `manifest.json`: decoded and parsed manifest (permissions, components with intent filters and exported state, application attributes, split manifests). - `classes.json`, `functions.json`, `function-index.json`, `strings.json`, `imports.json`, `exports.json` (exported components + JNI methods), `reachable.json` (from manifest components), `package-graph.json`, `sections.json`, `triage.json`. -- `lib//*.so`: every native library extracted; `native///`: a complete nested ToCode export per library (all ABIs), produced on a background thread while the DEX side decompiles. `native-libs.json` records the status of each (each library runs in its own process; one failing or being OOM-killed never fails the APK export, and the native thread waits for `TOCODE_NATIVE_MIN_FREE_MB`, default 1024 MB, of free memory before each library). Pass `--no-native` to skip the native decompilation (libraries are still extracted). `--backend` picks the native backend (`auto`/`ida`/`binja`/`r2`/`angr`; `binja` needs the headless API here). +- `lib//*.so`: every native library; `native///`: a full ToCode export of each one, made while the DEX side decompiles. `native-libs.json` records the status of each; a library that fails never fails the APK export. `--no-native` skips them, and `--backend` picks their backend. - `data/apk/**`: every other APK entry verbatim; `data/res/**/*.xml` and `data/resources.json`: decoded binary XML and `resources.arsc`. `base.apk` automatically merges sibling `split_*.apk` files (config and ABI splits) into the same project; `--no-splits` exports it alone. The default output directory is `_decompiler`. @@ -219,13 +219,13 @@ codex ``` #### From an ongoing RE work -``` -Interrupted exports save progress automatically. Rerun the same command to resume from cached function renders, or add `--restart` to ignore the saved checkpoint and start over. +``` tocode firmwareX.bin.i64 -o firmwareX_decompiled/ -... ``` +Interrupted exports save progress automatically. Rerun the same command to resume, or add `--restart` to start over. + ### Binary Ninja The Binary Ninja backend uses Binary Ninja's own analysis and Pseudo C. ToCode reaches Binary Ninja in one of three ways, tried in this order (`--binja-transport headless`, `mcp` or `rpyc` forces one): @@ -242,9 +242,9 @@ tocode ./program.bndb -o out/ # a Binary Ninja database always goes t tocode ./firmware/ --backend binja # folders, APK native libraries, and .NET native code too ``` -ToCode finds the API through `--binja-dir` or `TOCODE_BINJA_DIR` (the install folder or its `python/` folder), `BN_INSTALL_DIR`, the `binaryninja.pth` written by Binary Ninja's `scripts/install_api.py`, or the default install locations. The finished analysis is cached as a `.bndb` (in `~/.cache/tocode/binja`, or `TOCODE_BINJA_CACHE_DIR`), so a re-run and every render worker open it instead of analyzing again; `--purge-cache` rebuilds it. The export includes it as `.bndb`, like IDA's `.i64`. Each worker loads Binary Ninja and the database, so free memory caps the worker count (`TOCODE_BINJA_WORKER_MEMORY_MB`, default 1024 per worker). +If ToCode can't find Binary Ninja, pass `--binja-dir` with its install folder. -If the license refuses headless use, ToCode moves on: `--backend binja` to a running Binary Ninja, `auto` to radare2 or angr. `TOCODE_DISABLE_BINJA_HEADLESS=1` skips the API altogether. +If your license doesn't allow headless use, ToCode uses a running Binary Ninja instead (below). **MCP (Binary Ninja 6 and newer).** Turn on the MCP server in Binary Ninja, then: @@ -256,9 +256,7 @@ tocode --backend binja --all-views -o out/ # every open view, one folder e tocode --backend binja ./program -o out/ # export that file (opened in Binary Ninja if it isn't) ``` -With a headless license, a file argument is exported headless; add `--binja-transport mcp` to export it through the UI instead. - -ToCode connects to `http://127.0.0.1:24642/mcp` by default; set `--binja-mcp-url` or `TOCODE_BINJA_MCP_URL` to change it. MCP has one active view shared with the UI. ToCode switches it to the view it exports and switches it back when done. A file ToCode opens for you stays open in Binary Ninja, because Binary Ninja only closes analyzed files through its save prompt. Section data (`data/*.bin`) is read from Binary Ninja's view, so exporting a `.bndb` works. +With a headless license, a file argument is exported headless; add `--binja-transport mcp` to use the UI instead. ToCode connects to `http://127.0.0.1:24642/mcp`; `--binja-mcp-url` changes it. **RPC bridge (older versions).** If the MCP server doesn't answer, ToCode falls back to the bridge automatically. Run this once in Binary Ninja's Python console to start the bridge: @@ -285,7 +283,7 @@ print("ToCode RPC server listening on 127.0.0.1:18812") It exposes the whole Binary Ninja Python VM with no authentication, so keep it bound to `127.0.0.1`. The [binja-headless](https://github.com/hugsy/binja-headless) plugin exposes the same interface if you prefer a packaged option. -The same commands work over the bridge (the default view is the focused one). Use `--binja-host` / `--binja-port` (or `TOCODE_BINJA_HOST` / `TOCODE_BINJA_PORT`; default `127.0.0.1:18812`) to reach Binary Ninja on another machine. +The same commands work over the bridge. Use `--binja-host` / `--binja-port` to reach Binary Ninja on another machine. Already scripting inside Binary Ninja? Skip the server and export the live view directly: