Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 33 additions & 68 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1489,64 +1489,7 @@ if(BUILD_TESTING)
"${GECODE_TEST_BLACKBOX_LOG}.dll_parallel")
endif()

set(GECODE_CHECK_TESTS
Branch::Int::Dense::3
Int::Arithmetic::Abs
Int::Arithmetic::ArgMax
Int::Arithmetic::Max::Nary
Int::Cumulative::Man::Fix::0::4
Int::Distinct::Random
Int::Extensional::TupleSet::Sparse::IncrementalDelta
Int::Extensional::TupleSet::Auto::DefaultDispatch
Int::Linear::Bool::Int::Lq
Int::MiniModel::LinExpr::Bool::352
NoGoods::Queens
Search::DFS::Sol::Binary::Nary::Binary::1::1::1)
if(GECODE_ENABLE_FLATZINC)
list(INSERT GECODE_CHECK_TESTS 1
FlatZinc::Options
FlatZinc::magic_square
FlatZinc::blackbox)
endif()
if(GECODE_ENABLE_SET_VARS)
list(APPEND GECODE_CHECK_TESTS
Set::Dom::Dom::Gr
Set::RelOp::ConstSSI::Union
Set::Sequence::SeqU1
Set::Wait)
endif()
if(GECODE_ENABLE_FLOAT_VARS)
set(GECODE_FLOAT_CHECK_TESTS
Float::Arithmetic::PositiveNRootBounds
Float::Arithmetic::PowConsistency
Float::Arithmetic::MultZeroEndpoint
Float::Arithmetic::Pow::N::2::XY::Sol::C
Float::Arithmetic::NRoot::N::2::XY::Sol::C
Float::Arithmetic::Mult::XYZ::Sol::C)
list(APPEND GECODE_CHECK_TESTS ${GECODE_FLOAT_CHECK_TESTS})

# Keep the fast CI selection honest when Float regressions are added or
# the list above is edited. These names must remain registered in the
# ordinary check target rather than living only in ad-hoc test commands.
set(GECODE_REQUIRED_FLOAT_CHECK_TESTS
Float::Arithmetic::PositiveNRootBounds
Float::Arithmetic::PowConsistency
Float::Arithmetic::MultZeroEndpoint
Float::Arithmetic::Pow::N::2::XY::Sol::C
Float::Arithmetic::NRoot::N::2::XY::Sol::C
Float::Arithmetic::Mult::XYZ::Sol::C)
foreach(gecode_required_float_test ${GECODE_REQUIRED_FLOAT_CHECK_TESTS})
if(NOT gecode_required_float_test IN_LIST GECODE_CHECK_TESTS)
message(FATAL_ERROR
"Required Float check test is missing: ${gecode_required_float_test}")
endif()
endforeach()
endif()

set(GECODE_CHECK_ARGS -iter 2 -threads 0 -fixprob 1)
foreach(gecode_check_test ${GECODE_CHECK_TESTS})
list(APPEND GECODE_CHECK_ARGS -test ${gecode_check_test})
endforeach()
set(GECODE_CHECK_ARGS -iter 2 -threads 0 -fixprob 1 -tag check)

# Keep ctest robust while leaving the large test binaries out of the default build.
set(GECODE_TEST_BUILD_TARGETS gecode-test)
Expand All @@ -1573,16 +1516,6 @@ if(BUILD_TESTING)
ENVIRONMENT "${GECODE_TEST_BLACKBOX_ENV}")
endif()
set(GECODE_CHECK_DEPENDS gecode-test)
if(GECODE_ENABLE_FLOAT_VARS)
add_custom_target(verify-gecode-check-tests
COMMAND ${CMAKE_COMMAND}
"-DTEST_EXECUTABLE=$<TARGET_FILE:gecode-test>"
"-DREQUIRED_TESTS=${GECODE_REQUIRED_FLOAT_CHECK_TESTS}"
-P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/VerifyTestSelection.cmake
DEPENDS gecode-test
VERBATIM)
list(APPEND GECODE_CHECK_DEPENDS verify-gecode-check-tests)
endif()
if(GECODE_ENABLE_FLATZINC)
list(APPEND GECODE_CHECK_DEPENDS
gecode-test-blackbox-exec
Expand All @@ -1595,11 +1528,35 @@ if(BUILD_TESTING)
$<TARGET_FILE:gecode-test> ${GECODE_CHECK_ARGS}
DEPENDS ${GECODE_CHECK_DEPENDS}
USES_TERMINAL)
add_custom_target(check-normal
COMMAND ${CMAKE_COMMAND} -E env
GECODE_TEST_BLACKBOX_EXEC=$<TARGET_FILE:gecode-test-blackbox-exec>
GECODE_TEST_BLACKBOX_DLL=$<TARGET_FILE:gecode-test-blackbox-dll>
GECODE_TEST_BLACKBOX_LOG=${GECODE_TEST_BLACKBOX_LOG}
$<TARGET_FILE:gecode-test> -tag normal
DEPENDS ${GECODE_CHECK_DEPENDS}
USES_TERMINAL)
add_custom_target(check-sweep
COMMAND ${CMAKE_COMMAND} -E env
GECODE_TEST_BLACKBOX_EXEC=$<TARGET_FILE:gecode-test-blackbox-exec>
GECODE_TEST_BLACKBOX_DLL=$<TARGET_FILE:gecode-test-blackbox-dll>
GECODE_TEST_BLACKBOX_LOG=${GECODE_TEST_BLACKBOX_LOG}
$<TARGET_FILE:gecode-test> -tag sweep
DEPENDS ${GECODE_CHECK_DEPENDS}
USES_TERMINAL)
else()
add_custom_target(check
COMMAND $<TARGET_FILE:gecode-test> ${GECODE_CHECK_ARGS}
DEPENDS ${GECODE_CHECK_DEPENDS}
USES_TERMINAL)
add_custom_target(check-normal
COMMAND $<TARGET_FILE:gecode-test> -tag normal
DEPENDS gecode-test
USES_TERMINAL)
add_custom_target(check-sweep
COMMAND $<TARGET_FILE:gecode-test> -tag sweep
DEPENDS gecode-test
USES_TERMINAL)
endif()
if(GECODE_ENABLE_FAULT_INJECTION)
set(GECODE_FAULT_CHECK_ARGS -iter 1 -threads 1 -test "^Fault::")
Expand All @@ -1617,10 +1574,18 @@ if(BUILD_TESTING)
message(WARNING "Skipping gecode-test/check targets because required modules are disabled")
add_custom_target(check
COMMAND ${CMAKE_COMMAND} -E echo "Skipping check target because required modules are disabled")
add_custom_target(check-normal
COMMAND ${CMAKE_COMMAND} -E echo "Skipping check-normal target because required modules are disabled")
add_custom_target(check-sweep
COMMAND ${CMAKE_COMMAND} -E echo "Skipping check-sweep target because required modules are disabled")
endif()
else()
add_custom_target(check
COMMAND ${CMAKE_COMMAND} -E echo "Skipping check target because BUILD_TESTING is OFF")
add_custom_target(check-normal
COMMAND ${CMAKE_COMMAND} -E echo "Skipping check-normal target because BUILD_TESTING is OFF")
add_custom_target(check-sweep
COMMAND ${CMAKE_COMMAND} -E echo "Skipping check-sweep target because BUILD_TESTING is OFF")
endif()

# ---------------------------------------------------------------------------
Expand Down
58 changes: 9 additions & 49 deletions Makefile.in
Original file line number Diff line number Diff line change
Expand Up @@ -1441,67 +1441,27 @@ test: mkcompiledirs $(BLACKBOXFIXTURES)
@$(MAKE) $(VARIMP) $(TESTEXE) \
$(TESTPUBLICRUNNERSMOKEEXE) $(TESTPUBLICINTSMOKEEXE)

CHECKTESTS = Branch::Int::Dense::3 \
Int::Arithmetic::Abs \
Int::Arithmetic::ArgMax \
Int::Arithmetic::Max::Nary \
Int::Cumulative::Man::Fix::0::4 \
Int::Distinct::Random \
Int::Extensional::TupleSet::Sparse::IncrementalDelta \
Int::Extensional::TupleSet::Auto::DefaultDispatch \
Int::Linear::Bool::Int::Lq \
Int::MiniModel::LinExpr::Bool::352 \
NoGoods::Queens \
Search::DFS::Sol::Binary::Nary::Binary::1::1::1

ifeq "@enable_set_vars@" "yes"
CHECKTESTS += \
Set::Dom::Dom::Gr \
Set::RelOp::ConstSSI::Union \
Set::Sequence::SeqU1 \
Set::Wait
endif

ifeq "@enable_flatzinc@" "yes"
CHECKTESTS += FlatZinc::magic_square FlatZinc::Options FlatZinc::blackbox
BLACKBOXCHECKENV = \
GECODE_TEST_BLACKBOX_EXEC=$(abspath $(BLACKBOXEXEC)) \
GECODE_TEST_BLACKBOX_DLL=$(abspath $(BLACKBOXDLL)) \
GECODE_TEST_BLACKBOX_LOG=$(BLACKBOXLOG)
endif

ifeq "@enable_float_vars@" "yes"
FLOATCHECKTESTS = Float::Arithmetic::PositiveNRootBounds \
Float::Arithmetic::PowConsistency \
Float::Arithmetic::MultZeroEndpoint \
Float::Arithmetic::Pow::N::2::XY::Sol::C \
Float::Arithmetic::NRoot::N::2::XY::Sol::C \
Float::Arithmetic::Mult::XYZ::Sol::C
CHECKTESTS += $(FLOATCHECKTESTS)
REQUIREDFLOATCHECKTESTS = Float::Arithmetic::PositiveNRootBounds \
Float::Arithmetic::PowConsistency \
Float::Arithmetic::MultZeroEndpoint \
Float::Arithmetic::Pow::N::2::XY::Sol::C \
Float::Arithmetic::NRoot::N::2::XY::Sol::C \
Float::Arithmetic::Mult::XYZ::Sol::C
endif

# A basic integrity test
check: test
$(RUNENVIRONMENT) $(TESTPUBLICRUNNERSMOKEEXE)
$(RUNENVIRONMENT) $(TESTPUBLICINTSMOKEEXE)
@for t in $(REQUIREDFLOATCHECKTESTS); do \
case " $(CHECKTESTS) " in *" $$t "*) ;; \
*) echo "Required Float check test is missing: $$t" >&2; exit 1 ;; \
esac; \
if ! $(RUNENVIRONMENT) $(TESTEXE) -list | grep -Fqx "$$t"; then \
echo "Required Float check test is not registered: $$t" >&2; \
exit 1; \
fi; \
done
$(BLACKBOXCHECKENV) $(RUNENVIRONMENT) \
$(TESTEXE) -iter 2 -threads 0 -fixprob 1 \
$(CHECKTESTS:%=-test %)
$(TESTEXE) -iter 2 -threads 0 -fixprob 1 -tag check

# The normal test suite without sweep tests
check-normal: test
$(BLACKBOXCHECKENV) $(RUNENVIRONMENT) $(TESTEXE) -tag normal

# Really heavy sweep tests
check-sweep: test
$(BLACKBOXCHECKENV) $(RUNENVIRONMENT) $(TESTEXE) -tag sweep

.PHONY: regenerate
regenerate:
Expand Down
9 changes: 9 additions & 0 deletions changelog.in
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,15 @@ Date: unreleased
[DESCRIPTION]
This is the development changelog for the next Gecode release.

[ENTRY]
Module: test
What: new
Rank: major
[DESCRIPTION]
Add self-declared check, normal, and sweep tags to the public test harness.
The standard check targets use a focused integrity selection, while normal
testing retains broad coverage without the exhaustive multi-hour sweeps.

[ENTRY]
Module: test
What: new
Expand Down
37 changes: 37 additions & 0 deletions docs/cmake-build.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,43 @@ cmake --install build/vs2022-vcpkg --config Release --prefix C:/path/to/install
Visual Studio is a multi-config generator, so use `--config Release` (or
`Debug`) for build/install/check commands rather than `CMAKE_BUILD_TYPE`.

## Test Targets

When `BUILD_TESTING=ON`, CMake builds the `gecode-test` test runner on demand.
The standard `check` target runs the basic integrity suite:

```bash
cmake --build build --target check
```

Two additional targets expose broader tag-based suites:

```bash
cmake --build build --target check-normal
cmake --build build --target check-sweep
```

`check-normal` runs the normal test suite. `check-sweep` runs tests tagged as
heavy sweep tests and is intended for deliberate, longer-running validation.

The test runner can also be invoked directly with tags:

```bash
gecode-test -tag check
gecode-test -tag normal
gecode-test -tag sweep
gecode-test -tag normal -tag sweep
gecode-test -tag all
```

With no `-tag` option, the runner does not restrict tests by tag. Repeated tags
form a union, while tag and name filters intersect. `-tag all` explicitly
selects every known tag.

Use `gecode-test -list-tags` to list known tags and
`gecode-test -list-with-tags` to inspect test assignments. Listing always shows
all registered tests, regardless of selection filters.

## Build Conventions and Key Options

### Common CMake options
Expand Down
32 changes: 32 additions & 0 deletions docs/public-test-harness.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,38 @@ or `RM_PMI` when the propagator supports only part of the reification API.
`testsearch` and `testfix` can disable the corresponding checks for constraints
where those checks do not apply.

## Select tests by tag

Tests created with the one-argument `Test::Base` constructor have the `normal`
tag. Any other membership is assigned explicitly when the test is constructed:

```c++
ConsumerSmoke()
: Test::Base("Package::ConsumerSmoke", Test::TestTag::sweep) {}
```

The runner recognizes the `check`, `normal`, and `sweep` tags. Repeating
`-tag` selects their union. Without `-tag`, the runner does not restrict tests
by tag; `-tag all` is the explicit equivalent:

```bash
./consumer-smoke -tag normal
./consumer-smoke -tag normal -tag sweep
./consumer-smoke -tag all
```

Name and tag filters intersect: a test must match both when both `-test` and
`-tag` are present.

Use `-list-tags` to list the recognized tags and `-list-with-tags` to show the
tags assigned to every registered test. Gecode's tests declare their tags at
registration, just like downstream tests; test names do not trigger implicit
classification. The listing commands always show all registered tests; they do
not apply `-test`, `-tag`, or `-start` filters.

The runner uses the same option model as Gecode's own `gecode-test` binary.
The supported public seam is the runner function, not a separate alternate CLI.

## Run and reproduce tests

List the registered tests:
Expand Down
7 changes: 6 additions & 1 deletion test/branch.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -732,8 +732,13 @@ namespace Test { namespace Branch {
}

#ifdef GECODE_HAS_SET_VARS
SetTest::SetTest(TestTags tags, const std::string& s, int a,
const Gecode::IntSet& d)
: Base("Set::Branch::"+s,tags), arity(a), dom(d) {
}

SetTest::SetTest(const std::string& s, int a, const Gecode::IntSet& d)
: Base("Set::Branch::"+s), arity(a), dom(d) {
: SetTest(TestTag::normal,s,a,d) {
}

bool
Expand Down
3 changes: 3 additions & 0 deletions test/branch.hh
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ namespace Test {
/// Domain of variables
Gecode::IntSet dom;
public:
/// Construct and register test with explicitly assigned tags
SetTest(TestTags tags, const std::string& s, int a,
const Gecode::IntSet& d);
/// Construct and register test
SetTest(const std::string& s, int a, const Gecode::IntSet& d);
/// Perform test
Expand Down
8 changes: 4 additions & 4 deletions test/branch/set.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ namespace Test { namespace Branch {
class Set : public SetTest {
public:
/// Create and register test
Set(const std::string& s, const Gecode::IntSet& d, int n)
: SetTest(s,n,d) {}
Set(TestTags tags, const std::string& s, const Gecode::IntSet& d, int n)
: SetTest(tags,s,n,d) {}
/// Post propagators on variables \a x
virtual void post(Gecode::Space& home, Gecode::SetVarArray& x) {
Gecode::SetVarArgs xx(x.size()-1);
Expand All @@ -57,8 +57,8 @@ namespace Test { namespace Branch {
const int v_sparse[6] = {-100,-10,0,10,100,1000};
Gecode::IntSet d_sparse(v_sparse,6);

Set d_3("Dense::3",d_dense,3);
Set s_3("Sparse::3",d_sparse,3);
Set d_3(TestTag::normal,"Dense::3",d_dense,3);
Set s_3(TestTag::sweep,"Sparse::3",d_sparse,3);
}

}}
Expand Down
Loading
Loading