feat(router): add lmdeploy router

.github/scripts/check_lmdeploy.py

@@ -20,6 +20,8 @@ def check_module_init(root: str):
             continue
         elif d.startswith('lmdeploy/monitoring'):
             continue
+        elif d.startswith('lmdeploy/router'):
+            continue
         elif d.startswith('lmdeploy/serve/turbomind/triton_models'):
             continue
         elif d.startswith('lmdeploy/serve/turbomind/triton_python_backend'):

.github/workflows/router_release.yml

@@ -0,0 +1,64 @@
+name: router-release
+
+on:
+  workflow_dispatch:
+    inputs:
+      upload:
+        description: 'Upload lmdeploy-router distributions to PyPI'
+        required: true
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    defaults:
+      run:
+        working-directory: lmdeploy/router
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Build router wheel
+        run: |
+          python -m pip install --upgrade build setuptools-rust wheel
+          python -m build --wheel --outdir dist .
+      - name: Inspect distributions
+        run: ls -lh dist/*
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: lmdeploy-router-${{ github.sha }}
+          path: lmdeploy/router/dist/*
+          if-no-files-found: error
+          retention-days: 14
+
+  publish:
+    if: ${{ inputs.upload }}
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment: prod
+    steps:
+      - name: Download router artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: lmdeploy-router-${{ github.sha }}
+          path: dist
+      - name: Publish lmdeploy-router to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.ROUTER_PYPI_TOKEN }}
+        run: |
+          python -m pip install --upgrade twine
+          twine check dist/*
+          twine upload dist/*

.github/workflows/router_test.yml

@@ -0,0 +1,64 @@
+name: router-test
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/router_test.yml"
+      - "lmdeploy/router/**"
+  push:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/router_test.yml"
+      - "lmdeploy/router/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  rust:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    defaults:
+      run:
+        working-directory: lmdeploy/router
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v6
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy, rustfmt
+      - name: Cache Rust artifacts
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: lmdeploy/router
+      - name: Check formatting
+        run: cargo fmt --check
+      - name: Static analysis
+        run: cargo clippy --all-targets --locked -- -D warnings
+      - name: Rust tests
+        run: cargo test --all-targets --locked
+
+  python:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: lmdeploy/router
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install test dependencies
+        run: python -m pip install -e .[dev]
+      - name: Python unit tests
+        run: python -m pytest py_test/unit
+      - name: Python mock integration tests
+        run: >-
+          python -m pytest py_test/integration -m 'integration and not pd_engine and not lmdeploy'

.gitignore

@@ -58,6 +58,9 @@ lmdeploy/turbomind/_turbomind*.so
 lmdeploy/turbomind/_turbomind*.pyd
 lmdeploy/turbomind/lib/
 dist/
+lmdeploy/router/target/
+lmdeploy/router/.pytest_cache/
+lmdeploy/router/.tokenizer_cache/
 examples/cpp/llama/*.csv
 *.npy
 *.weight
@@ -74,6 +77,8 @@ wscripts/
 *.bin
 *config.json
 *generate_config.json
+!lmdeploy/router/examples/configs/*.json
+!lmdeploy/router/test_prompt_input.json
 !lmdeploy/turbomind/hf_repo/config.json
 
 # Pytorch

docker/Dockerfile

@@ -1,6 +1,7 @@
 # Base images
 ARG IMAGE_TYPE=final
 ARG CUDA_VERSION=cu128
+ARG BUILD_ROUTER=1
 
 FROM nvidia/cuda:13.0.3-devel-ubuntu24.04 AS cu130
 ENV CUDA_VERSION_SHORT=cu130
@@ -19,8 +20,9 @@ FROM cu128 AS cu12
 # Builder image
 FROM ${CUDA_VERSION} AS dev
 ARG PYTHON_VERSION=3.12
+ARG BUILD_ROUTER=1
 
-ENV PATH=/opt/py3/bin:/root/.local/bin:${PATH}
+ENV PATH=/opt/py3/bin:/root/.local/bin:/root/.cargo/bin:${PATH}
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=Etc/UTC
 
@@ -41,7 +43,10 @@ RUN --mount=type=cache,target=/root/.cache \
     rm -rf /var/lib/apt/lists/* && \
     wget -qO- https://astral.sh/uv/install.sh | sh && \
     uv venv -p python${PYTHON_VERSION} --seed /opt/py3 && \
-    pip install --upgrade pip build
+    pip install --upgrade pip build && \
+    if [ "${BUILD_ROUTER}" != "0" ]; then \
+        curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal; \
+    fi
 
 # Pip deps only (cache until requirements/common.txt or runtime_cuda.txt changes). No full repo COPY yet.
 FROM dev AS runtime-deps-pip
@@ -79,10 +84,30 @@ WORKDIR /opt/lmdeploy
 RUN --mount=type=cache,target=/root/.cache \
     docker/build.sh
 
+# Build the standalone Rust router in the same image by default. Pass
+# --build-arg BUILD_ROUTER=0 to skip Rust installation and compilation. The
+# disabled branch posts a small marker because later bind mounts require the
+# same artifact path in both variants.
+FROM dev AS router-builder
+ARG BUILD_ROUTER=1
+ENV PATH=/opt/py3/bin:/root/.local/bin:/root/.cargo/bin:${PATH}
+WORKDIR /src
+COPY lmdeploy/router/Cargo.toml lmdeploy/router/Cargo.lock ./
+COPY lmdeploy/router/src ./src
+RUN if [ "${BUILD_ROUTER}" != "0" ]; then \
+        cargo build --release --locked --no-default-features --bin lmdeploy-router && \
+        install -D -m 0755 target/release/lmdeploy-router /src/router/lmdeploy-router; \
+    else \
+        mkdir -p /src/router && printf 'router disabled\n' > /src/router/lmdeploy-router; \
+    fi
+
 FROM runtime-deps-pip AS wheels
 
 COPY --from=thirdparty-wheels /wheels /wheels
 COPY --from=lmdeploy-wheel /wheels /wheels
+# Keep a stable artifact path across enabled and disabled builds.
+RUN --mount=type=bind,target=/router-build-stage,from=router-builder,source=/src/router \
+    cp /router-build-stage/lmdeploy-router /wheels/lmdeploy-router
 
 # Runtime image
 FROM nvidia/cuda:13.0.3-base-ubuntu24.04 AS cu130-base
@@ -100,6 +125,7 @@ FROM cu128-base AS cu12-base
 
 FROM ${CUDA_VERSION}-base AS final
 ARG PYTHON_VERSION=3.12
+ARG BUILD_ROUTER=1
 ENV PYTHON_VERSION=${PYTHON_VERSION}
 
 # Install the CUDA runtime requirements explicitly so torch is selected from
@@ -114,4 +140,9 @@ RUN --mount=type=cache,target=/root/.cache \
 # explicitly set ptxas path for triton
 ENV PATH=/opt/py3/bin:$PATH
 ENV TRITON_PTXAS_PATH=/usr/local/cuda/bin/ptxas
+RUN --mount=type=bind,target=/wheels-build-stage,from=wheels,source=/wheels \
+    if [ "${BUILD_ROUTER}" != "0" ]; then \
+        install -m 0755 /wheels-build-stage/lmdeploy-router /opt/lmdeploy-router; \
+    fi
+ENV PATH=/opt:${PATH}
 FROM ${IMAGE_TYPE}

docs/en/advance/metrics.md

@@ -72,7 +72,7 @@ As an example, we use the model `Qwen/Qwen2.5-7B-Instruct` with `DP=2, TP=2`. St
 
 ```bash
 # Proxy server
-lmdeploy serve proxy --server-port 8000 --routing-strategy 'min_expected_latency' --serving-strategy Hybrid --log-level INFO
+lmdeploy serve proxy --server-port 8000 --routing-strategy 'cache_aware' --serving-strategy Hybrid --log-level INFO
 
 # API server
 LMDEPLOY_DP_MASTER_ADDR=127.0.0.1 \
@@ -87,7 +87,7 @@ lmdeploy serve api_server \
     --node-rank 0
 ```
 
-You should be able to see multiple API servers added to the proxy server list. Details can be found in `lmdeploy/serve/proxy/proxy_config.json`.
+You should be able to see multiple API servers in the router node list at `http://0.0.0.0:8000/nodes/status`.
 
 For example, you may have the following API servers:
 

docs/en/api/openapi.rst

@@ -14,11 +14,3 @@ OpenAI Compatible API Endpoints
 .. openapi:: ../_static/openai.yaml
     :request:
     :paths: /v1/responses
-
-
-Proxy Server API
-----------------
-
-.. openapi:: ../_static/proxy.yaml
-    :request:
-    :examples:

docs/en/conf.py

@@ -24,7 +24,6 @@
 from lmdeploy.serve.openai.api_server import ServerContext  # noqa: E402
 from lmdeploy.serve.openai.endpoints import create_openai_router  # noqa: E402
 from lmdeploy.serve.openai.responses import create_responses_router  # noqa: E402
-from lmdeploy.serve.proxy.proxy import app as proxy_server  # noqa: E402
 
 version_file = '../../lmdeploy/version.py'
 with open(version_file) as f:
@@ -74,9 +73,6 @@ def metrics():
 with open(spec_dir / 'openai.yaml', 'w', encoding='utf-8') as f:
     f.write(safe_dump(openai_server.openapi()))
 
-with open(spec_dir / 'proxy.yaml', 'w', encoding='utf-8') as f:
-    f.write(safe_dump(proxy_server.openapi()))
-
 # -- General configuration ---------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.

docs/en/llm/api_server.md

@@ -2,6 +2,7 @@
 
 This article primarily discusses the deployment of a single LLM model across multiple GPUs on a single node, providing a service that is compatible with the OpenAI interface, as well as the usage of the service API.
 For the sake of convenience, we refer to this service as `api_server`. Regarding parallel services with multiple models, please refer to the guide about [Request Distribution Server](proxy_server.md).
+When using the router entry point `lmdeploy serve proxy`, install the separate `lmdeploy-router` package first.
 
 In the following sections, we will first introduce methods for starting the service, choosing the appropriate one based on your application scenario.