Reject top-k queries with k greater than the number of indexed series

.github/workflows/ci.yml

@@ -8,88 +8,90 @@
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    # Pinned: on ubuntu-latest (26.04) HerculesRange RandomWalk and
+    # CoconutTest.QueryBoundLookup fail; they pass on 24.04. Tracked separately.
+    runs-on: ubuntu-24.04
 
     strategy:
       matrix:
         build-type: [Release, Debug]
         # mpi: [ON, OFF] # Enable or disable MPI (COMMENTED OUT)
         # cuda: [ON, OFF] # Enable or disable CUDA (COMMENTED OUT)
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           submodules: recursive
 
       - name: Initialize submodules (fallback)
         run: |
           git submodule update --init --recursive
 
       - name: Verify pybind11 submodule
         run: |
           ls -la extern/pybind11/CMakeLists.txt || echo "pybind11 CMakeLists.txt not found"
 
       - name: Install system dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y \
             build-essential \
             cmake \
             libomp-dev \
             libopenblas-dev \
             libopenmpi-dev \
             openmpi-bin \
             openmpi-common
 
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.12"
 
       - name: Install Python dependencies
         run: |
           python -m pip install --upgrade pip
           pip install numpy==2.2.6 mpi4py==4.0.3
 
       - name: Create build directory
         run: mkdir build
 
       - name: Configure CMake
         run: |
           cd build
           cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DBUILD_ODYSSEY=ON -DBUILD_SING=OFF -DBUILD_PYTHON=ON -DBUILD_BENCHMARK=ON -DBUILD_DEMO=ON -DBUILD_TESTS=ON -DDEBUG_MSG=ON
 
       - name: Build project
         run: |
           cd build
           make -j$(nproc)
 
       - name: Run tests
         run: |
           cd build
           # Run parameterized tests (AstronomyData, RandomWalkData) and Odyssey test
           #ctest -R "AstronomyData|RandomWalkData|test_Odyssey_L2Square" --output-on-failure --verbose
           ctest -R "AstronomyData " --output-on-failure --verbose
 
       - name: Run individual test executables (fallback)
         if: failure()
         run: |
           cd build
           # Run non-Odyssey test executables with AstronomyData filter
           for test_exe in $(find . -name "test_*" -type f -executable ! -name "*Odyssey*"); do
             echo "Running $test_exe (Astronomy only)"
             if ! $test_exe --gtest_filter="*AstronomyData*" --gtest_output=xml:${test_exe##*/}_results.xml; then
               echo "Test $test_exe failed"
               exit 1
             fi
           done
           # Run Odyssey test under mpirun if present
           if [ -f ./tests/test_Odyssey_L2Square ]; then
             echo "Running test_Odyssey_L2Square under mpirun (--oversubscribe for limited cores)"
             if ! mpirun --oversubscribe -np 4 ./tests/test_Odyssey_L2Square --gtest_filter="*AstronomyData*"; then
               echo "test_Odyssey_L2Square failed"
               exit 1
             fi
           fi
 

.github/workflows/pypi-release.yml

@@ -18,7 +18,9 @@ concurrency:
 
 jobs:
   publish:
-    runs-on: ubuntu-latest
+    # Pinned: ubuntu-latest moved to 26.04, whose toolchain auditwheel cannot
+    # turn into a manylinux wheel ("too-recent versioned symbols").
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
 

.github/workflows/release-check.yml

@@ -17,7 +17,9 @@ on:
 
 jobs:
   dry-run:
-    runs-on: ubuntu-latest
+    # Pinned: ubuntu-latest moved to 26.04, whose toolchain auditwheel cannot
+    # turn into a manylinux wheel ("too-recent versioned symbols").
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
 

.github/workflows/simple-ci.yml

@@ -6,76 +6,78 @@
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    # Pinned: on ubuntu-latest (26.04) HerculesRange RandomWalk and
+    # CoconutTest.QueryBoundLookup fail; they pass on 24.04. Tracked separately.
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           submodules: recursive
 
       - name: Initialize submodules (fallback)
         run: |
           git submodule update --init --recursive
 
       - name: Verify pybind11 submodule
         run: |
           ls -la extern/pybind11/CMakeLists.txt || echo "pybind11 CMakeLists.txt not found"
 
       - name: Install system dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y \
             build-essential \
             cmake \
             libomp-dev
 
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.12"
 
       - name: Install basic Python dependencies
         run: |
           python -m pip install --upgrade pip
           pip install numpy
 
       - name: Create build directory
         run: mkdir -p build
 
       - name: Configure CMake (minimal configuration)
         run: |
           cd build
           cmake .. \
             -DCMAKE_BUILD_TYPE=Release \
             -DBUILD_ODYSSEY=OFF \
             -DBUILD_SING=OFF \
             -DBUILD_PYTHON=OFF \
             -DBUILD_BENCHMARK=OFF \
             -DBUILD_DEMO=OFF \
             -DBUILD_TESTS=ON
 
       - name: Build project
         run: |
           cd build
           make -j$(nproc)
 
       - name: Run tests
         run: |
           cd build
           ctest --output-on-failure --verbose
 
       - name: List and run test executables directly
         if: failure()
         run: |
           cd build/tests
           echo "Available test executables:"
           ls -la test_* 2>/dev/null || echo "No test executables found"
 
           # Run each test executable if they exist
           for test_exe in test_*; do
             if [ -x "$test_exe" ]; then
               echo "Running $test_exe"
               ./$test_exe || echo "Test $test_exe failed but continuing..."
             fi
           done

CONTINUITY.md

@@ -64,6 +64,11 @@ These algorithms do not all share the same dependency profile or feature set.
 ## Important Behavioral Conventions
 
 - Top-k search is the baseline capability.
+- Every top-k entry point rejects `k` greater than the number of indexed series with
+  `std::invalid_argument` (`ValueError` in Python), via
+  `SimilaritySearchAlgorithm::checkTopKNotGreaterThanDatabase`. Results are never padded by
+  repeating the last neighbor or with placeholder ids. New algorithms must call it too;
+  `tests/test_TopK_KGreaterThanN.cpp` and `tests/test_topk_k_greater_than_n.py` cover it.
 - Range search is modeled through `SearchConfig` and is not universally implemented.
 - Streaming insert is implemented by `BruteForceSearch`, `LbBruteforce`, `Messi`, and `Coconut`;
   the base implementation still throws for algorithms that do not support it.

README.md

@@ -122,6 +122,11 @@ use insertion counts, not arbitrary user timestamps. Resident SAX uses O(N) RAM;
 See [the implementation notes](docs/coconut-disk-streaming.md). Earlier results
 from the unmerged RAM insertion buffer do not evaluate disk-based streaming.
 
+### Top-k with k larger than the dataset
+
+Every algorithm rejects a top-k query whose `k` is greater than the number of indexed series:
+C++ throws `std::invalid_argument` and Python raises `ValueError`. Results are never padded.
+
 ### Range (distance-r) queries
 
 Algorithms that implement range search answer range queries through

lib/algos/Coconut.cpp

@@ -503,6 +503,7 @@ namespace
         requireBuilt();
         if (distance_type != DistanceType::L2_SQUARED) throw std::runtime_error("Coconut top-k search only supports L2_SQUARED");
         if (!k) throw std::runtime_error("Coconut top-k search requires k > 0");
+        checkTopKNotGreaterThanDatabase(k);
         checkFinite(query, n_query, dim); set_active_breakpoints(nullptr, nullptr);
         const auto &s = *state_; auto runs = s.snapshot();
         for (const auto &run : runs) run->validate();

lib/algos/Fresh.cpp

@@ -1518,6 +1518,7 @@ void Fresh::searchIndex(const float *query, idx_t n_query, const SearchConfig &c
 
 void Fresh::searchIndex(const float *query, idx_t n_query, idx_t k, idx_t *I, float *D)
 {
+    checkTopKNotGreaterThanDatabase(k);
     if (this->distance_type == DistanceType::L2_SQUARED)
         searchIndexL2Squared(query, n_query, k, I, D);
     else if (this->distance_type == DistanceType::DTW)

lib/algos/Hercules.cpp

@@ -725,6 +725,7 @@ void Hercules::searchIndex(const float *query, idx_t n_query, idx_t k, idx_t *I,
         throw std::runtime_error("Hercules::searchIndex: index not built");
     if (config_.index_dir.empty())
         throw std::runtime_error("Hercules::searchIndex: index_dir not set");
+    checkTopKNotGreaterThanDatabase(k);
 
     for (idx_t q = 0; q < n_query; q++)
         hercules_knn_search(root_, query + q * this->dim, (int)this->dim, k,

lib/algos/Messi.cpp

@@ -1350,6 +1350,7 @@ namespace daisy
 
     void Messi::searchIndex(const float *query, const idx_t n_query, const idx_t k, idx_t *I, float *D)
     {
+        checkTopKNotGreaterThanDatabase(k);
         if (this->distance_type == DistanceType::L2_SQUARED)
         {
             searchIndexL2Squared(query, n_query, k, I, D);

lib/algos/ParIS.cpp

@@ -186,6 +186,7 @@ namespace daisy
 
     void ParIS::searchIndex(const float *query, const idx_t n_query, const idx_t k, idx_t *I, float *D)
     {
+        checkTopKNotGreaterThanDatabase(k);
         if (this->distance_type == DistanceType::L2_SQUARED)
         {
             searchIndexL2Squared(query, n_query, k, I, D);

lib/algos/SimilaritySearchAlgorithm.hpp

@@ -6,6 +6,7 @@
 #include <cfloat>
 #include <vector>
 #include <stdexcept>
+#include <string>
 
 #include "../distance_computers/DistanceComputer.hpp"
 #include "../isax/iSAXSearch.hpp"
@@ -119,18 +120,25 @@ namespace daisy
         }
 
     protected:
+        // Top-k cannot return more neighbors than indexed series. Every top-k
+        // entry point calls this so the caller gets an error (std::invalid_argument,
+        // ValueError in Python) instead of padded or placeholder results.
+        void checkTopKNotGreaterThanDatabase(const idx_t k) const
+        {
+            if (k > n_database)
+                throw std::invalid_argument("k (" + std::to_string(k) +
+                                            ") cannot be greater than the number of indexed series (" +
+                                            std::to_string(n_database) + ")");
+        }
+
         bool validateSearchParams(const idx_t k, const idx_t n_query) const
         {
             if (k == 0)
             {
                 std::cerr << "[Error] k must be greater than 0\n";
                 return false;
             }
-            if (k > n_database)
-            {
-                std::cerr << "[Error] k (" << k << ") cannot be greater than database size (" << n_database << ")\n";
-                return false;
-            }
+            checkTopKNotGreaterThanDatabase(k);
             if (n_query == 0)
             {
                 std::cerr << "[Error] n_query must be greater than 0\n";

lib/algos/Sing.cpp

@@ -1699,6 +1699,7 @@ namespace daisy
 
     void Sing::searchIndex(const float *query, const idx_t n_query, const idx_t k, idx_t *I, float *D)
     {
+        checkTopKNotGreaterThanDatabase(k);
         if (this->distance_type == DistanceType::L2_SQUARED && this->index != nullptr)
         {
             searchIndexL2Square(query, n_query, k, I, D);

lib/algos/Sofa.cpp

@@ -867,6 +867,7 @@ void Sofa::searchIndexDTW(const float *query, idx_t n_query, idx_t k, idx_t *I,
 
 void Sofa::searchIndex(const float *query, const idx_t n_query, const idx_t k, idx_t *I, float *D)
 {
+    checkTopKNotGreaterThanDatabase(k);
     if (this->distance_type == DistanceType::L2_SQUARED)
         searchIndexL2Squared(query, n_query, k, I, D);
     else if (this->distance_type == DistanceType::DTW)

lib/algos/odyssey/Odyssey.cpp

@@ -2231,11 +2231,7 @@ namespace daisy
             std::cerr << "[Error] k must be greater than 0\n";
             return false;
         }
-        if (k > n_database)
-        {
-            std::cerr << "[Error] k (" << k << ") cannot be greater than database size (" << n_database << ")\n";
-            return false;
-        }
+        checkTopKNotGreaterThanDatabase(k);
         if (n_query == 0)
         {
             std::cerr << "[Error] n_query must be greater than 0\n";

tests/CMakeLists.txt

@@ -383,6 +383,33 @@ gtest_discover_tests(
     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
 )
 
+# ////// TOP-K: k > n is rejected by every algorithm //////
+add_executable(
+    test_TopK_KGreaterThanN
+    test_TopK_KGreaterThanN.cpp
+    test_utils.cpp
+)
+target_link_libraries(
+    test_TopK_KGreaterThanN
+    PRIVATE
+    GTest::gtest_main
+    dino_lib
+    commons_lib
+    stdc++fs
+)
+target_include_directories(test_TopK_KGreaterThanN
+    PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../lib
+    ${CMAKE_CURRENT_SOURCE_DIR}/../commons
+)
+if(BUILD_COCONUT)
+    target_compile_definitions(test_TopK_KGreaterThanN PRIVATE DAISY_TEST_COCONUT=1)
+endif()
+gtest_discover_tests(
+    test_TopK_KGreaterThanN
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+
 # ////// MESSI streaming //////
 add_executable(
     test_Messi_Streaming

tests/test_TopK_KGreaterThanN.cpp

@@ -0,0 +1,144 @@
+// Every top-k entry point must reject k > number of indexed series with
+// std::invalid_argument (ValueError in Python) instead of returning padded or
+// placeholder results. k == n must keep working.
+#include "test_utils.hpp"
+#if DAISY_TEST_COCONUT
+#include "../lib/algos/Coconut.hpp"
+#endif
+
+#include <cstdio>
+#include <functional>
+#include <memory>
+#include <random>
+#include <string>
+#include <vector>
+
+using daisy::idx_t;
+
+namespace
+{
+    constexpr idx_t N = 64;
+    constexpr idx_t DIM = 64;
+    constexpr idx_t NQ = 2;
+
+    std::vector<float> randomSeries(idx_t n, unsigned seed)
+    {
+        std::mt19937 rng(seed);
+        std::normal_distribution<float> dist(0.0f, 1.0f);
+        std::vector<float> v(n * DIM);
+        for (float &x : v)
+            x = dist(rng);
+        return v;
+    }
+
+    struct Fixture
+    {
+        std::vector<float> data = randomSeries(N, 7);
+        std::vector<float> queries = randomSeries(NQ, 11);
+        std::string data_file;
+
+        Fixture()
+        {
+            data_file = ::testing::TempDir() + "daisy_topk_k_gt_n.bin";
+            FILE *f = std::fopen(data_file.c_str(), "wb");
+            std::fwrite(data.data(), sizeof(float), data.size(), f);
+            std::fclose(f);
+        }
+        ~Fixture() { std::remove(data_file.c_str()); }
+    };
+
+    void expectKGreaterThanNRejected(daisy::SimilaritySearchAlgorithm &search, const float *queries)
+    {
+        std::vector<idx_t> I(NQ * (N + 1));
+        std::vector<float> D(NQ * (N + 1));
+        EXPECT_THROW(search.searchIndex(queries, NQ, N + 1, I.data(), D.data()), std::invalid_argument);
+
+        daisy::SearchConfig config;
+        config.k = N + 1;
+        std::vector<std::vector<idx_t>> VI;
+        std::vector<std::vector<float>> VD;
+        EXPECT_THROW(search.searchIndex(queries, NQ, config, VI, VD), std::invalid_argument);
+
+        EXPECT_NO_THROW(search.searchIndex(queries, NQ, N, I.data(), D.data()));
+    }
+}
+
+class TopKGreaterThanN : public ::testing::TestWithParam<daisy::DistanceType>
+{
+protected:
+    Fixture fx;
+};
+
+TEST_P(TopKGreaterThanN, BruteForceSearch)
+{
+    daisy::BruteForceSearch search(GetParam());
+    search.buildIndex(fx.data.data(), N, DIM);
+    expectKGreaterThanNRejected(search, fx.queries.data());
+}
+
+TEST_P(TopKGreaterThanN, LbBruteforce)
+{
+    daisy::LbBruteforce search(GetParam());
+    search.buildIndex(fx.data.data(), N, DIM);
+    expectKGreaterThanNRejected(search, fx.queries.data());
+}
+
+TEST_P(TopKGreaterThanN, Messi)
+{
+    daisy::Messi search(GetParam());
+    search.buildIndex(fx.data.data(), N, DIM);
+    expectKGreaterThanNRejected(search, fx.queries.data());
+}
+
+TEST_P(TopKGreaterThanN, ParIS)
+{
+    daisy::ParIS search(GetParam());
+    search.buildIndex(fx.data_file, DIM, N);
+    expectKGreaterThanNRejected(search, fx.queries.data());
+}
+
+TEST_P(TopKGreaterThanN, Fresh)
+{
+    daisy::Fresh search(GetParam());
+    search.buildIndex(fx.data.data(), N, DIM);
+    expectKGreaterThanNRejected(search, fx.queries.data());
+}
+
+TEST_P(TopKGreaterThanN, DumpyOS)
+{
+    daisy::DumpyOS search(GetParam());
+    search.buildIndex(fx.data.data(), N, DIM);
+    expectKGreaterThanNRejected(search, fx.queries.data());
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    Metrics, TopKGreaterThanN,
+    ::testing::Values(daisy::DistanceType::L2_SQUARED, daisy::DistanceType::DTW),
+    [](const ::testing::TestParamInfo<daisy::DistanceType> &info)
+    { return info.param == daisy::DistanceType::DTW ? std::string("DTW") : std::string("L2Squared"); });
+
+// L2-only algorithms.
+TEST(TopKGreaterThanNL2, Hercules)
+{
+    Fixture fx;
+    daisy::HerculesConfig cfg;
+    cfg.index_dir = ::testing::TempDir() + "daisy_topk_k_gt_n_hercules";
+    daisy::Hercules search(daisy::DistanceType::L2_SQUARED, cfg);
+    search.buildIndex(fx.data.data(), N, DIM);
+    expectKGreaterThanNRejected(search, fx.queries.data());
+}
+
+#if DAISY_TEST_COCONUT
+TEST(TopKGreaterThanNL2, Coconut)
+{
+    Fixture fx;
+    daisy::Coconut search(daisy::DistanceType::L2_SQUARED);
+    search.buildIndex(fx.data.data(), N, DIM);
+    expectKGreaterThanNRejected(search, fx.queries.data());
+
+    std::vector<idx_t> I(NQ * (N + 1));
+    std::vector<float> D(NQ * (N + 1));
+    EXPECT_THROW(search.searchIndexRecent(fx.queries.data(), NQ, N + 1, N, I.data(), D.data()),
+                 std::invalid_argument);
+}
+#endif