From 5ba5dd951f2c9799ffe1c1c655600d7b7768f43a Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:51:09 -0700 Subject: [PATCH 1/2] Start adding components of the `Execute` module These components basically comprise the innards of what used to exist solely within the private codebase's `LLScriptExecuteLuau`. In the future `LLScriptExecuteLuau` will be a small wrapper around these building blocks instead. It's a lot easier to have them here when trying to build out things like VM sharing, GC pacing changes, pre-emption tweaks, etc. --- CMakeLists.txt | 11 +- Executor/include/Luau/ByteStream.h | 137 ++ Executor/include/Luau/Executor.h | 438 ++++++ Executor/include/Luau/Script.h | 238 ++++ Executor/src/Executor.cpp | 299 +++++ Executor/src/Logging.cpp | 64 + Executor/src/Script.cpp | 930 +++++++++++++ Makefile | 15 +- Sources.cmake | 12 + VM/include/lua.h | 68 +- VM/src/ares.cpp | 33 +- VM/src/ares.h | 7 +- VM/src/ldo.cpp | 49 +- VM/src/lllevents.cpp | 4 +- VM/src/llltimers.cpp | 8 +- build-cmd.sh | 3 + tests/SLExecutor.test.cpp | 2017 ++++++++++++++++++++++++++++ 17 files changed, 4261 insertions(+), 72 deletions(-) create mode 100644 Executor/include/Luau/ByteStream.h create mode 100644 Executor/include/Luau/Executor.h create mode 100644 Executor/include/Luau/Script.h create mode 100644 Executor/src/Executor.cpp create mode 100644 Executor/src/Logging.cpp create mode 100644 Executor/src/Script.cpp create mode 100644 tests/SLExecutor.test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b9881b2..d1149009 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,7 @@ if (LUAU_BUILD_SHARED) add_library(Luau.Analysis SHARED) add_library(Luau.CodeGen SHARED) add_library(Luau.VM SHARED) + add_library(Luau.Executor SHARED) add_library(Luau.Require SHARED) add_library(isocline SHARED) else() @@ -75,6 +76,7 @@ else() add_library(Luau.Analysis STATIC) add_library(Luau.CodeGen STATIC) add_library(Luau.VM STATIC) + add_library(Luau.Executor STATIC) add_library(Luau.Require STATIC) add_library(isocline STATIC) endif() @@ -161,6 +163,12 @@ target_compile_features(Luau.VM PRIVATE cxx_std_17) target_include_directories(Luau.VM PUBLIC VM/include "${PACKAGE_INCLUDE_DIR}") target_link_libraries(Luau.VM PUBLIC Luau.Common) +# ServerLua: per-script execution engine shared with the script host +target_compile_features(Luau.Executor PUBLIC cxx_std_17) +target_include_directories(Luau.Executor PUBLIC Executor/include) +target_link_libraries(Luau.Executor PUBLIC Luau.VM) +target_link_libraries(Luau.Executor PRIVATE Luau.VM.Internals) + target_compile_features(Luau.Require PUBLIC cxx_std_17) target_include_directories(Luau.Require PUBLIC Require/include) target_link_libraries(Luau.Require PUBLIC Luau.Config Luau.VM) @@ -219,6 +227,7 @@ target_compile_options(Luau.CLI.lib PRIVATE ${LUAU_OPTIONS}) target_compile_options(Luau.LSLBuiltins PRIVATE ${LUAU_OPTIONS}) target_compile_options(Luau.CodeGen PRIVATE ${LUAU_OPTIONS}) target_compile_options(Luau.VM PRIVATE ${LUAU_OPTIONS}) +target_compile_options(Luau.Executor PRIVATE ${LUAU_OPTIONS}) target_compile_options(isocline PRIVATE ${LUAU_OPTIONS} ${ISOCLINE_OPTIONS}) if(LUAU_EXTERN_C) @@ -345,7 +354,7 @@ if(LUAU_BUILD_TESTS) target_compile_definitions(Luau.Conformance PRIVATE DOCTEST_CONFIG_DOUBLE_STRINGIFY DOCTEST_CONFIG_USE_STD_HEADERS) target_include_directories(Luau.Conformance PRIVATE extern VM/src "${PACKAGE_INCLUDE_DIR}") target_link_directories(Luau.Conformance PRIVATE "${PACKAGE_LIB_DIR}") - target_link_libraries(Luau.Conformance PRIVATE Luau.Analysis Luau.Bytecode Luau.Inliner Luau.Compiler Luau.CodeGen Luau.VM Luau.LSLBuiltins "${TAILSLIDE_LIBRARY}") + target_link_libraries(Luau.Conformance PRIVATE Luau.Analysis Luau.Bytecode Luau.Inliner Luau.Compiler Luau.CodeGen Luau.VM Luau.Executor Luau.LSLBuiltins "${TAILSLIDE_LIBRARY}") if(CMAKE_SYSTEM_NAME MATCHES "Android|iOS") set(LUAU_CONFORMANCE_SOURCE_DIR "Client/Luau/tests/conformance") diff --git a/Executor/include/Luau/ByteStream.h b/Executor/include/Luau/ByteStream.h new file mode 100644 index 00000000..81d68e2d --- /dev/null +++ b/Executor/include/Luau/ByteStream.h @@ -0,0 +1,137 @@ +// ServerLua: little-endian bytestream primitives for the wrappers we place +// around ares-serialized state. I regret some of my design decisions here +// and this probably should not need to be part of the public API. +#pragma once + +#include +#include +#include + +namespace Luau +{ +namespace Executor +{ + +struct ByteWriter +{ + std::string& out; + + void writeU8(uint8_t value) { out.push_back((char)value); } + + void writeU32(uint32_t value) + { + for (int i = 0; i < 4; ++i) + writeU8((uint8_t)(value >> (i * 8))); + } + + void writeS32(int32_t value) { writeU32((uint32_t)value); } + + void writeU64(uint64_t value) + { + writeU32((uint32_t)value); + writeU32((uint32_t)(value >> 32)); + } + + void writeF32(float value) + { + uint32_t rep; + memcpy(&rep, &value, sizeof(rep)); + writeU32(rep); + } + + void writeF64(double value) + { + uint64_t rep; + memcpy(&rep, &value, sizeof(rep)); + writeU64(rep); + } + + void writeBytes(const char* data, size_t len) { out.append(data, len); } + + // Length-prefixed, so it round-trips embedded nulls + void writeString(const char* data, size_t len) + { + writeU32((uint32_t)len); + writeBytes(data, len); + } + + void writeString(const std::string& value) { writeString(value.data(), value.size()); } +}; + +struct ByteReader +{ + const char* data; + size_t remaining; + + bool readBytes(void* dest, size_t len) + { + if (len > remaining) + return false; + memcpy(dest, data, len); + data += len; + remaining -= len; + return true; + } + + bool readU8(uint8_t& value) { return readBytes(&value, sizeof(value)); } + + bool readU32(uint32_t& value) + { + uint8_t buf[4]; + if (!readBytes(buf, sizeof(buf))) + return false; + value = (uint32_t)buf[0] | ((uint32_t)buf[1] << 8) | ((uint32_t)buf[2] << 16) | ((uint32_t)buf[3] << 24); + return true; + } + + bool readS32(int32_t& value) + { + uint32_t rep; + if (!readU32(rep)) + return false; + value = (int32_t)rep; + return true; + } + + bool readU64(uint64_t& value) + { + uint32_t low; + uint32_t high; + if (!readU32(low) || !readU32(high)) + return false; + value = (uint64_t)low | ((uint64_t)high << 32); + return true; + } + + bool readF32(float& value) + { + uint32_t rep; + if (!readU32(rep)) + return false; + memcpy(&value, &rep, sizeof(value)); + return true; + } + + bool readF64(double& value) + { + uint64_t rep; + if (!readU64(rep)) + return false; + memcpy(&value, &rep, sizeof(value)); + return true; + } + + bool readString(std::string& value) + { + uint32_t len; + if (!readU32(len) || len > remaining) + return false; + value.assign(data, len); + data += len; + remaining -= len; + return true; + } +}; + +} // namespace Executor +} // namespace Luau diff --git a/Executor/include/Luau/Executor.h b/Executor/include/Luau/Executor.h new file mode 100644 index 00000000..6c6d9d40 --- /dev/null +++ b/Executor/include/Luau/Executor.h @@ -0,0 +1,438 @@ +// ServerLua: per-script execution engine shared with the script host. +#pragma once + +#include +#include + +#include "lua.h" + +namespace Luau +{ +namespace Executor +{ + +// Memory category for allocations that should not be billed to the script +constexpr int kSystemMemcat = 0; +// Memory category for allocations attributable to the script instance +constexpr int kUserMemcat = LUA_FIRST_USER_MEMCAT; +// Default (and normal maximum) per-script memory limit +constexpr int kDefaultMemoryLimit = 1024 * 128; + +// Identifies which class a persisted payload belongs to, and which layout it +// used. Bump `version` whenever that class's fields change. +struct StateFingerprint +{ + char tag[4]; + uint32_t version; +}; + +// Log levels for LogCallback +enum class LogLevel : uint8_t +{ + Debug = 0, + Info, + Warn, +}; + +// Parameters that define the sealed image consumed by buildImage() +struct ImageConfig +{ + // Luau bytecode with any host asset header already stripped off + const char* bytecode = nullptr; + size_t bytecodeSize = 0; + bool isLSL = false; + uint32_t apiVersion = 0; + // We want to leave open the possibility that we can change bytecode behind + // people's backs for upgrading reasons. Keep around the amount we want to + // actually "charge" them for the bytecode size for memory accounting purposes + // so this doesn't break scripts. + size_t chargedBytecodeSize = 0; + const char* chunkname = "=lua_script"; + // Identifier used in build log messages. Probably an asset UUID. + const char* name = ""; +}; + +// Per-placement parameters that can differ between two scripts running the +// same asset, consumed by instantiateScript(). +struct ScriptConfig +{ + // Opaque identifier used as the log source for the script's messages + const char* scriptId = ""; + int memoryLimit = kDefaultMemoryLimit; + // Opaque per-script host context copied onto the Script. In lscript terms, + // this is the `LLScriptExecuteLuau` instance. + void* hostContext = nullptr; +}; + +class IEnvironment; +class IImage; +class IProvisioner; +class Script; + +// Receives an engine log message, already formatted. `source` identifies the +// origin for attribution and throttling. +using LogCallback = void (*)(LogLevel level, const char* source, const char* message); + +// Process-wide log sink, same shape as `Luau::assertHandler()`. Messages are +// dropped while it's null. +inline LogCallback& logCallback() +{ + static LogCallback callback = nullptr; + return callback; +} + +// Format a message and hand it to the installed LogCallback. `source` +// identifies the origin for attribution and throttling. +void logDebug(const char* source, const char* fmt, ...) LUA_PRINTF_ATTR(2, 3); +void logInfo(const char* source, const char* fmt, ...) LUA_PRINTF_ATTR(2, 3); +void logWarn(const char* source, const char* fmt, ...) LUA_PRINTF_ATTR(2, 3); + +// Give the embedder a chance to plop their own things into the environment before it's +// fully set up. This is called before GC fixing / ares perms registration. +using PopulateEnvironmentCallback = void (*)(IEnvironment& environment, lua_State* L); + +// Host callback wiring, identical for every script of a provisioner. +struct HostCallbacks +{ + lua_clockProvider clockProvider = nullptr; + lua_clockProvider performanceClockProvider = nullptr; + lua_randomProvider randomProvider = nullptr; + lua_setTimerEventCallback setTimerEventCb = nullptr; + lua_eventHandlerRegistrationCallback eventHandlerRegistrationCb = nullptr; + lua_clockProvider quantaClockProvider = nullptr; + PopulateEnvironmentCallback populateEnvironment = nullptr; +}; + +// An environment is... basically just a Lua VM with some particular settings. It's +// intended for multiple of these to be able to be living at any given moment, one +// for LSL, one for a particular version of the Lua API, etc. +class IEnvironment +{ +public: + virtual ~IEnvironment() = default; + + virtual IProvisioner& getProvisioner() const = 0; + virtual bool isLSL() const = 0; + virtual uint32_t getAPIVersion() const = 0; + + // The root lua_State this environment owns. Images' clone threads and + // forkservers all hang off it + virtual lua_State* getBaseState() const = 0; + // Threaddata seed for every non-script thread in the VM (base state, + // forkservers, build-time clones) + virtual lua_SLRuntimeState& getRuntimeState() = 0; +}; + +// The environment a `Provisioner` mints by default. Public so a host can +// subclass it and hand its own back from `Provisioner::makeEnvironment()`. +class Environment : public IEnvironment +{ +public: + Environment(IProvisioner& provisioner, bool is_lsl, uint32_t api_version); + ~Environment() override; + + Environment(const Environment&) = delete; + Environment& operator=(const Environment&) = delete; + + IProvisioner& getProvisioner() const override { return mProvisioner; } + bool isLSL() const override { return mIsLSL; } + uint32_t getAPIVersion() const override { return mAPIVersion; } + + lua_State* getBaseState() const override { return mBaseState; } + + // `kind` stays `LUA_SLSTATE_BARE` so the engine callbacks know there is + // no `Script` behind our threads. + lua_SLRuntimeState& getRuntimeState() override { return mRuntimeState; } + + // Stands the VM up. Run by the provisioner immediately after construction, + // so a subclass' constructor goes first and has nothing built to trip over. + // Templated on the script type because `lua_Callbacks` is per-VM, so the + // handlers have to be chosen here rather than per script. + template + void build() + { + lua_State* L = openVM(); + S::installVMCallbacks(L); + + // Assigned last so nothing can reach a VM that isn't wired up yet + mBaseState = L; + } + +private: + // Everything up to the callbacks, kept out of the template so the library + // setup stays in the .cpp + lua_State* openVM(); + + IProvisioner& mProvisioner; + bool mIsLSL = false; + uint32_t mAPIVersion = 0; + + lua_SLRuntimeState mRuntimeState; + lua_State* mBaseState = nullptr; +}; + +// A particular Instance of a given script's run state. Effectively, this is just +// an RAII helper that holds onto a `lua_State*` in the registry and unrefs it +// when it dies. Necessarily separate from `Script` because `Script`s need to keep +// a stable object identity across reset() / deserialize(). Instance is probably +// not a very descriptive name... +class Instance +{ +public: + Instance() = default; + + // Adopts `thread`, releasing registry ref `ref` against `anchor_state` on + // destruction. + Instance(lua_State* thread, lua_State* anchor_state, int ref) + : mThread(thread) + , mAnchorState(anchor_state) + , mRef(ref) + { + } + + Instance(const Instance&) = delete; + Instance& operator=(const Instance&) = delete; + + Instance(Instance&& other) noexcept + : mThread(other.mThread) + , mAnchorState(other.mAnchorState) + , mRef(other.mRef) + { + other.mThread = nullptr; + other.mAnchorState = nullptr; + other.mRef = LUA_NOREF; + } + + Instance& operator=(Instance&& other) noexcept + { + if (this != &other) + { + // Clear out our thread, we're taking another. + releaseThread(); + + mThread = other.mThread; + mAnchorState = other.mAnchorState; + mRef = other.mRef; + other.mThread = nullptr; + other.mAnchorState = nullptr; + other.mRef = LUA_NOREF; + } + return *this; + } + + ~Instance() { releaseThread(); } + + explicit operator bool() const { return mThread != nullptr; } + lua_State* thread() const { return mThread; } + +private: + void releaseThread() + { + if (mThread != nullptr) + { + // Unrefing it should cause it to be swept up by GC. + lua_unref(mAnchorState, mRef); + } + mThread = nullptr; + mAnchorState = nullptr; + mRef = LUA_NOREF; + } + + lua_State* mThread = nullptr; + lua_State* mAnchorState = nullptr; + int mRef = LUA_NOREF; +}; + + +class IImage +{ +public: + virtual ~IImage() = default; + + virtual bool isValid() const = 0; + // Load-failure message when isValid() is false + virtual const std::string& getError() const = 0; + virtual const std::string& getName() const = 0; + virtual IEnvironment& getEnvironment() const = 0; + virtual IProvisioner& getProvisioner() const = 0; + virtual bool isLSL() const = 0; + virtual uint32_t getAPIVersion() const = 0; + + // Forks off an instance using the forkserver, either with the default + // state blob, or a provided one if we're resuming. + virtual Instance forkInstance(lua_SLRuntimeState* owner, const std::string* blob = nullptr) = 0; + + // Serializes `instance` against this image's forkserver into `out` + virtual bool serializeInstance(const Instance& instance, std::string& out) = 0; + + // The environment's bare runtime state, seed for each Script's live copy + virtual const lua_SLRuntimeState& getRuntimeState() const = 0; + + // Full asset size charged against every script of this image + virtual size_t getChargedBytecodeSize() const = 0; + + // Pristine pre-fork objects excluded from per-script memory accounting + virtual const lua_OpaqueGCObjectSet& getFreeObjects() const = 0; +}; + +// The image a `Provisioner` mints by default. Public so a host can subclass it +// and hand its own back from `Provisioner::makeImage()`. +class Image : public IImage +{ +public: + Image(std::shared_ptr environment, const ImageConfig& config); + ~Image() override; + + Image(const Image&) = delete; + Image& operator=(const Image&) = delete; + + bool isValid() const override { return mForkerState != nullptr; } + const std::string& getError() const override { return mError; } + const std::string& getName() const override { return mName; } + IEnvironment& getEnvironment() const override { return *mEnvironment; } + IProvisioner& getProvisioner() const override { return mEnvironment->getProvisioner(); } + bool isLSL() const override { return mIsLSL; } + uint32_t getAPIVersion() const override { return mAPIVersion; } + + Instance forkInstance(lua_SLRuntimeState* owner, const std::string* blob) override; + bool serializeInstance(const Instance& instance, std::string& out) override; + + const lua_SLRuntimeState& getRuntimeState() const override { return mEnvironment->getRuntimeState(); } + size_t getChargedBytecodeSize() const override { return mChargedBytecodeSize; } + const lua_OpaqueGCObjectSet& getFreeObjects() const override { return mFreeObjects; } + + // Loads the bytecode and stands up the forkserver. Run by the provisioner + // immediately after construction, same as `Environment::build()`. A failure + // leaves `isValid()` false with `getError()` explaining, and the + // environment still usable. + void build(const ImageConfig& config); + +private: + // Source string for log messages + const char* logSource() const { return mName.c_str(); } + + // Keeps the environment alive for as long as the image exists + std::shared_ptr mEnvironment; + + bool mIsLSL = false; + uint32_t mAPIVersion = 0; + + // Ares forkserver thread holding the pristine post-load snapshot, + // anchored in the environment's registry + lua_State* mForkerState = nullptr; + int mForkerRef = LUA_NOREF; + // Pristine pre-fork objects excluded from per-script memory accounting + lua_OpaqueGCObjectSet mFreeObjects; + + size_t mChargedBytecodeSize = 0; + std::string mName; + std::string mError; +}; + + +class IProvisioner +{ +public: + virtual ~IProvisioner() = default; + + virtual const HostCallbacks& getCallbacks() const = 0; + + virtual std::shared_ptr createEnvironment(bool is_lsl, uint32_t api_version) = 0; + virtual std::shared_ptr buildImage(std::shared_ptr environment, const ImageConfig& config) = 0; + virtual std::shared_ptr