diff --git a/.github/workflows/blender-smoke.yml b/.github/workflows/blender-smoke.yml index b2bd7ea..e67acee 100644 --- a/.github/workflows/blender-smoke.yml +++ b/.github/workflows/blender-smoke.yml @@ -3,7 +3,10 @@ name: Blender Smoke Test # Executes the snippets' and skills' headline examples inside REAL Blender, headless, # on the current stable (5.2.x LTS) and the fallback LTS (4.5.x) on every PR, # plus 5.1.x on the weekly cron, and fails on any error or -# empty-output assertion. py_compile (in validate.yml) cannot catch API-level regressions +# empty-output assertion. Shipped examples go through tests/smoke/run_example.py +# (catalog.json): SKIP is exit 77 + SMOKE_SKIP below --min-version, never exit 0. +# Post-exit sidecars are opt-in. A leg with zero PASSes is red. +# py_compile (in validate.yml) cannot catch API-level regressions # like the EEVEE-id inversion, the slotted-actions boundary, the driver TypeError, or the # dead SDF link -- this gate runs the code so those surface in CI, not in users' files. @@ -33,6 +36,18 @@ jobs: steps: - uses: actions/checkout@v7 + - name: Harness protocol unit tests + run: | + set -euo pipefail + python3 tests/smoke/test_harness.py -v + + - name: Init smoke status log + run: | + set -euo pipefail + # runner.temp is illegal in job-level env; set it from the step shell. + echo "BDT_SMOKE_STATUS=$RUNNER_TEMP/smoke-status.jsonl" >> "$GITHUB_ENV" + : > "$RUNNER_TEMP/smoke-status.jsonl" + - name: Install Blender runtime libraries run: | set -euo pipefail @@ -117,518 +132,74 @@ jobs: --output "$RUNNER_TEMP/out/render.png" --engine CYCLES test -s "$RUNNER_TEMP/out/render.png" || { echo "::error::render PNG missing/empty"; exit 1; } - - name: Shipped example - swatch grid renders and self-verifies - run: | - set -euo pipefail - # Run the SHIPPED example file (not a copy). It asserts non-black AND - # distinct-region-count == material-count internally and exits non-zero on - # failure. Cycles (CPU) for pixels on the GPU-less runner; the example still - # asserts the version-correct EEVEE engine id before rendering. Small/low-sample - # for speed -- region detection holds at this size. - xvfb-run -a "$BLENDER" --background \ - --python examples/swatch-grid/swatch_grid.py -- \ - --output "$RUNNER_TEMP/out/swatch.png" --engine cycles --samples 8 --width 640 - test -s "$RUNNER_TEMP/out/swatch.png" || { echo "::error::swatch grid output missing/empty"; exit 1; } - - - name: Shipped example - turntable correctness (slotted actions) - run: | - set -euo pipefail - # Frame-independent check only (no render): asserts the rotation keys drive - # playback via the cross-version channelbag path; exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/turntable/turntable.py -- - - - name: Shipped example - GN SDF remesh correctness (GridToMesh) - run: | - set -euo pipefail - # Frame-independent check only (no render): asserts the SDF->GridToMesh remesh - # yields geometry (eval vcount > 0 and != base); exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/gn-sdf-remesh/gn_sdf_remesh.py -- - - - name: Shipped example - depsgraph-evaluated export - run: | - set -euo pipefail - # Frame-independent check only (no render): asserts wm.obj_export ships the - # depsgraph-evaluated (modifier-applied) geometry -- exported vcount == evaluated - # > base. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/depsgraph-export/depsgraph_export.py -- - - - name: Shipped example - wave displace (foreach bulk IO) - run: | - set -euo pipefail - # Frame-independent check only (no render): 9409 verts displaced via one - # foreach_get + one foreach_set; asserts the Z span matches the amplitude - # and every vertex matches the closed-form wave. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/wave-displace/wave_displace.py -- - - - name: Shipped example - driver wave (driver_namespace + depsgraph) - run: | - set -euo pipefail - # Frame-independent check only (no render): sixteen SCRIPTED drivers call a - # driver_namespace function; asserts the driven values on the evaluated copy - # AND the flushed-back original both match the closed form. Exits non-zero - # on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/driver-wave/driver_wave.py -- - - - name: Shipped example - bmesh gear (ownership + watertight topology) - run: | - set -euo pipefail - # Frame-independent check only (no render): parametric gear via bmesh with - # bm.free() in try/finally; asserts closed-form vert/edge/face counts and - # that every edge borders exactly two faces. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/bmesh-gear/bmesh_gear.py -- - - - name: Shipped example - shader node group (interface sockets + sharing) - run: | - set -euo pipefail - # Frame-independent check only (no render): TintedGloss group declared via - # tree.interface.new_socket, instanced by two materials; asserts the sockets - # exist, the group datablock is shared (users == 2), and the per-instance - # Tint values differ. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/shader-node-group/shader_node_group.py -- - - - name: Shipped example - temp-override join (context override) - run: | - set -euo pipefail - # Frame-independent check only (no render): three-step staircase joined under - # bpy.context.temp_override; asserts one mesh remains, sources are gone, - # topology verts=24 faces=18, and local Z spans all steps. Exits non-zero - # on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/temp-override-join/temp_override_join.py -- - - - name: Shipped example - GN instance grid (Instance on Points) - run: | - set -euo pipefail - # Frame-independent check only (no render): generative GN tree instances a - # cube on a 3x3 grid and realizes; asserts eval verts=72 faces=54, Set - # Material carries Lime, and corner center matches the grid. Exits non-zero - # on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/gn-instance-grid/gn_instance_grid.py -- - - - name: Shipped example - GN modifier inputs (5.1 dict vs 5.2 RNA) - run: | - set -euo pipefail - # Frame-independent check only (no render): one GN tree, three modifier - # copies; writes Scale 1/2/3 via version-appropriate path; asserts - # readback and evaluated Z-extent match. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/gn-modifier-inputs/gn_modifier_inputs.py -- - - - name: Shipped example - shape-key blend (data API + evaluated mesh) - run: | - set -euo pipefail - # Frame-independent check only (no render): relative Tall key lifts+flares - # at value=0.5; asserts undeformed mesh stays at Basis and every evaluated - # vert matches basis + value*(key-basis). Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/shape-key-blend/shape_key_blend.py -- - - - name: Shipped example - curve bevel arc (Bezier + bevel_depth) - run: | - set -euo pipefail - # Frame-independent check only (no render): beveled Bezier semicircle via - # curve data API; asserts 8 points, bevel_depth=0.15, fill caps, eval - # verts=1044 faces=1028, tube rests on floor. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/curve-bevel-arc/curve_bevel_arc.py -- - - - name: Shipped example - compositor glare (Fog Glow + 5.x node group) - run: | - set -euo pipefail - # Two tiny 1-sample Cycles renders (compositor on vs off, ~seconds each): - # asserts the cross-version compositor plumbing (compositing_node_group - # on 5.x, scene.node_tree on 4.x), the Glare Fog Glow configuration, a - # strictly-falling halo with the compositor on, and exactly-zero halo - # pixels with it off. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/compositor-glare/compositor_glare.py -- - - - name: Shipped example - damped-track aim (DAMPED_TRACK data API) - run: | - set -euo pipefail - # Frame-independent check only (no render): twelve needles aim at an - # emissive core via Object.constraints.new('DAMPED_TRACK') on TRACK_Z; - # asserts one unmuted constraint per needle and evaluated local +Z - # aligns toward the core (dot ≥ 0.998). Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/damped-track-aim/damped_track_aim.py -- - - - name: Shipped example - color attribute wheel (color_attributes CORNER domain) - run: | - set -euo pipefail - # Frame-independent check only (no render): an HSV wheel disc colored via - # mesh.color_attributes.new() on the CORNER domain; asserts the attribute - # is sized to loop count (not vertex count), is active_color, and probe - # loops match the closed-form HSV for their vertex. Exits non-zero on - # failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/color-attribute-wheel/color_attribute_wheel.py -- - - - name: Shipped example - parent inverse orrery (matrix_parent_inverse) - run: | - set -euo pipefail - # Frame-independent check only (no render): a data-API orrery whose arms, - # planets, and moon are parented with the keep-world idiom; asserts bare - # `.parent =` teleports a probe, matrix_parent_inverse restores it exactly, - # matrix_world is stale until view_layer.update(), and every orbit lands - # on its closed form. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/parent-inverse-orrery/parent_inverse_orrery.py -- - - - name: Shipped example - grease pencil rosette (GPv3 attribute API) - run: | - set -euo pipefail - # Frame-independent check only (no render): five nested rose curves drawn - # as GPv3 strokes; asserts the version-gated datablock address (4.5: - # grease_pencils_v3 with grease_pencils still legacy; 5.x: grease_pencils - # is GPv3, legacy names gone), lazy attribute-layer materialization from - # point writes, and a closed-form round-trip of every position through - # the raw POINT attribute buffer. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/grease-pencil-rosette/grease_pencil_rosette.py -- - - - name: Shipped example - gp line art contour (LINEART modifier + thickness trap) - run: | - set -euo pipefail - # Check only (no render): GPv3 object + LINEART modifier; asserts - # source_object is load-bearing (clear → 0 strokes), contour+crease - # off → 0, restore recovers silhouette strokes, and thickness is - # 4.5-only (AttributeError on 5.1; portable path is radius). - # Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/gp-lineart-contour/gp_lineart_contour.py -- - - - name: Shipped example - armature bend (edit_bones + LBS closed form) - run: | - set -euo pipefail - # Check only (no render): a four-bone chain skins a tapered tube via - # name-bound vertex groups; asserts edit_bones is empty outside edit - # mode and holds the closed-form chain inside it, then compares every - # depsgraph-evaluated vertex against re-implemented linear blend - # skinning (root ring pinned, tip deflected — a straight tube fails). - # Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/armature-bend/armature_bend.py -- - - - name: Shipped example - text version stamp (TextCurve solids + version string) + - name: Canary - expected skip is SKIP not PASS run: | set -euo pipefail - # Check only (no render): a TextCurve whose body is the live - # bpy.app.version_string; asserts the Curve subclass and built-in - # font contracts, that flat text is filled but planar, the closed - # form z-extent = 2*(extrude+bevel_depth), body edits regenerating - # geometry, that version_string starts with the dotted version tuple - # ("4.5.11 LTS" is not bare semver), and that a Mesh reference dies - # at to_mesh_clear(). Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/text-version-stamp/text_version_stamp.py -- + python3 tests/smoke/run_example.py \ + --name canary-skip \ + --blender "$BLENDER" \ + --script tests/smoke/canary_skip.py \ + --series "${{ matrix.series }}" \ + --min-version 99.0 \ + --xvfb - - name: Shipped example - image pixels testcard (flat RGBA buffer + save lifecycle) + - name: Canary - unexpected skip is red run: | - set -euo pipefail - # Check only (no render): a procedural broadcast test card written with - # one pixels.foreach_set; asserts the buffer is always flat RGBA - # (channels == 4 even with alpha=False, RGB-stride writes raise), the - # byte round-trip error is <= 0.5/255 and strictly > 0 (storage really - # is 8-bit) while float_buffer=True round-trips at float32 precision, - # scale() reallocates so stale-size bulk reads raise, and save() flips - # source to FILE and drops the buffer (pixels silently re-load from - # disk — proven with an imposter file) while save_render() preserves - # it. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/image-pixels-testcard/image_pixels_testcard.py -- - - - name: Shipped example - uv layer grid (calc_uvs silent no-op) - run: | - set -euo pipefail - # Check only (no render): create_grid(..., calc_uvs=True) without a - # pre-existing UV layer leaves 0 layers (silent no-op); pre-create + - # calc_uvs fills loops to the closed-form grid UVs - # ((x/size+1)/2, (y/size+1)/2) within 1e-6 including mesh persistence; - # explicit loop assignment is the calc_uvs-free fallback. Exits - # non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/uv-layer-grid/uv_layer_grid.py -- - - - name: Shipped example - png vs exr alpha (float PNG false unpremul) - run: | - set -euo pipefail - # Check only (no render): float_buffer=True Image.save() to PNG writes - # RGBA16 and unpremultiplies as if associated-alpha, so straight-authored - # dark values at low alpha clamp to white (closed-form err 0.98 at - # RGB 0.02 / a=1/255); OpenEXR round-trips within 1e-5; byte→PNG is - # straight RGBA8; EXR color_mode=RGB drops alpha to opaque. Exits - # non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/png-exr-alpha/png_exr_alpha.py -- - - - name: Shipped example - vse cut list (sequencer 4.5 vs 5.x rename) - run: | - set -euo pipefail - # Check only (no render): asserts the accessor rename (.sequences - # gone on 5.x, bridged on 4.5), the new_effect end-kwarg rename - # (frame_end on 4.5, length on 5.x — the wrong kwarg TypeErrors), - # closed-form spans on each version's canonical accessors - # (frame_final_* vs left_handle/right_handle/duration with the - # deprecation bridge), GC wiring and clamping, scene-strip span, - # and a save/reload round-trip. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/vse-cut-list/vse_cut_list.py -- - # Compositing witness on a tiny Cycles render: mosaic cells carry - # their strip colors, the cross cell is a true mid blend, and a - # consumed cross source never composites independently. - xvfb-run -a "$BLENDER" --background \ - --python examples/vse-cut-list/vse_cut_list.py -- \ - --check-pixels --engine cycles - - - name: Shipped example - glTF export round-trip (+Y-up, applied modifiers, UVs) - run: | - set -euo pipefail - # Check only (no render): a 35-part supply crate exported GLTF_SEPARATE - # and re-imported; asserts the exporter/importer kwarg set still exists - # (RNA guard), the on-disk +Y-up conversion ((x,y,z)->(x,z,-y) baked - # into POSITION bounds, no node rotation), disk vertex count == - # evaluated loop count (export_apply), V-flipped TEXCOORD_0, and the - # full round-trip of positions/normals/UVs/per-triangle material - # bindings within float tolerances. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/gltf-export-roundtrip/gltf_export_roundtrip.py -- - - - name: Shipped example - LOD decimate chain (depsgraph + ratio bounds) - run: | - set -euo pipefail - # Check only (no render): a lathed retro rocket evaluated at LOD0/1/2 - # through the Decimate modifier; asserts closed-form base topology and - # bbox, evaluated-vs-original non-destructiveness (obj.data unchanged), - # evaluated triangle counts within 5% of ratio*base, and silhouette - # (bbox) preservation within 1e-3. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/lod-decimate-chain/lod_decimate_chain.py -- - - - name: Shipped example - vertex weight limit (4-influence cap + LBS) - run: | - set -euo pipefail - # Check only (no render): a five-bone mech arm with rich 5-influence - # boots, pruned to the 4-influence engine cap via the data API; - # asserts pre/post-limit influence counts, unit weight sums, pose - # preservation (pre vs post pruning), exact LBS from the mesh's own - # deform layer, and a pinned Root mount. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/vertex-weight-limit/vertex_weight_limit.py -- - - - name: Shipped example - triangulate + tangents (mikktspace contract) - run: | - set -euo pipefail - # Check only (no render): a lathed buckler with polar/strip/planar - # UV charts; asserts closed-form loop-triangle count, unit and - # orthogonal tangent frames, bitangent == sign*(n x t), tangents - # matching the independent edge/UV-delta formula on smooth-field - # triangles, zero flips off seams, and the re-fetched UV layer - # surviving calc_tangents. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/triangulate-tangents/triangulate_tangents.py -- - - - name: Shipped example - glTF skin round-trip (skins, weights, deform) - run: | - set -euo pipefail - # Check only (no render): a 7-bone mech scorpion exported with - # export_skins and re-imported; asserts skins[0] joints, JOINTS_0/ - # WEIGHTS_0 unit sums, bone/parent/rest-matrix round-trip, bit-exact - # weights via position-key lookup, and identical deformation of the - # re-imported rig. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/gltf-skin-roundtrip/gltf_skin_roundtrip.py -- - - - name: Shipped example - VSE gamma cross (blend-curve closed form) - run: | - set -euo pipefail - # Check only: renders tiny frames across a GAMMA_CROSS and asserts - # every sample against the gamma-0.5 closed form ((1-t)*sqrt(A) + - # t*sqrt(B))^2 with t = (frame-start)/duration, plus the material - # lerp deviation at mid. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/vse-gamma-cross/vse_gamma_cross.py -- - - - name: Shipped example - light link studio (receiver collections in pixels) - run: | - set -euo pipefail - # Check only (two tiny Cycles renders): a key linked to the hero - # collection must light only the hero (luminance ratio >= 3x at - # projected centers), and unlinking must raise the decoy >= 50% - # with hero drift <= 5%. Also guards the Object-level API path - # (ld.light_linking is an AttributeError). Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/light-link-studio/light_link_studio.py -- - - - name: Shipped example - collision hull proxy (compound convex pieces) - run: | - set -euo pipefail - # Check only (no render): a lathed fire hydrant prop plus its - # four-piece compound collision, each piece a bmesh convex_hull of - # a coarse sec(pi/n)-inflated cage; asserts per-piece watertightness, - # positive signed volume, render-vertex containment and hull - # convexity by closed-form plane tests, Euler characteristic 2, and - # the 255-face per-piece engine budget. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/collision-hull-proxy/collision_hull_proxy.py -- - - - name: Shipped example - custom normals + shade by angle (post-4.1 shading contract) - run: | - set -euo pipefail - # Check only (no render): a jerry can prop; asserts the legacy - # shading API stays removed (use_auto_smooth/use_custom_normals/ - # calc_normals), set_sharp_from_angle sharp sets matching an - # independent dihedral recompute exactly, evaluated normal welds/ - # splits, custom split normals surviving depsgraph evaluation - # within int16 quantization, and the version-gated shade_auto_smooth - # operator divergence (CANCELLED headless on 4.5, FINISHED on 5.1). - # Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/custom-normals-shade/custom_normals_shade.py -- - - - name: Shipped example - sky texture sun elevation (Nishita / MS + zenith rise) - run: | - set -euo pipefail - # Check only (no gallery still): World ShaderNodeTexSky drives - # Background Color; sky_type is NISHITA on 4.5 and MULTIPLE_SCATTERING - # on 5.1 (NISHITA gone); dust_density is 4.5-only (aerosol_density on - # 5.1). Two tiny Cycles OPEN_EXR zenith probes assert sun_elevation - # 8→55 deg brightens zenith (rise gate ≥ 1.25). Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/sky-texture-sun-elevation/sky_texture_sun_elevation.py -- - - - name: Shipped example - mesh hygiene audit (topology ingest checklist) - run: | - set -euo pipefail - # Check only (no render): stepped utility pedestal; asserts no ngons, - # no loose verts, manifold edges (exactly 2 faces), no zero-area faces, - # positive signed volume, Euler V-E+F == 2. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/mesh-hygiene-audit/mesh_hygiene_audit.py -- - - - name: Shipped example - prop origin transform (base origin + scale + MPI) - run: | - set -euo pipefail - # Check only (no render): street pedestal; asserts scale (1,1,1) after - # data-API apply, local bbox min.z == 0, world AABB unchanged, bare - # parent jumps accessory, matrix_parent_inverse restores world location, - # and matrix_world stays stale until view_layer.update(). Exits non-zero - # on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/prop-origin-transform/prop_origin_transform.py -- - - - name: Shipped example - soccer ball Goldberg (polyhedron invariants + panel binding) - run: | - set -euo pipefail - # Check only (no render): a bmesh icosphere truncated at 1/3 per edge - # into the Goldberg polyhedron; asserts closed-form counts (60/90/32, - # Euler 2), 12-pentagon/20-hexagon census, uniform degree 3, - # watertightness, edge-length uniformity, face planarity vs an - # independent Newell normal, centroid-at-origin circumsphere, and - # panel materials bound by face vertex count. Exits non-zero on - # failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/soccer-ball-goldberg/soccer_ball_goldberg.py -- - - - name: Shipped example - car mirror symmetry (Mirror modifier + depsgraph) - run: | - set -euo pipefail - # Check only (no render): a hatchback lofted as one half and - # completed by the Mirror modifier; asserts the datablock keeps the - # half (126/231/106, 28 centerline verts), the evaluated mesh is - # exactly 2n-c (224/434/212), watertight with Euler 2, every - # evaluated vertex has an exact negated-X partner, and wheels/lamps - # mirror about object origins parked on the symmetry plane. Exits - # non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/car-mirror-symmetry/car_mirror_symmetry.py -- - - - name: Shipped example - attribute domain shear (POINT vs CORNER semantics) - run: | - set -euo pipefail - # Check only (no render): an 8-wedge pinwheel around one shared hub - # vertex; asserts CORNER attr sizing (3K loops) and exact per-wedge - # hub colors, POINT sizing (K+1 verts), and that a naive per-wedge - # POINT loop shears to the last write at every shared vertex with - # the measured deviation equal to the palette closed form. Exits - # non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/attribute-domain-shear/attribute_domain_shear.py -- - - - name: Shipped example - degenerate bevel weld (half-dimension collapse) - run: | - set -euo pipefail - # Check only (no render): a 1.6x0.4x1.0 slab beveled at safe (0.10) - # and threshold (0.20 == min/2) offsets; asserts zero vs exactly 12 - # zero-area faces (4 min-axis edges x 3 segments), 16 coincident - # verts (4 x (segments+1)), min_area collapse > 1e5x, and a stdlib - # GLB re-parse counting 32 degenerate triangles shipped versus 0. - # Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/degenerate-bevel-weld/degenerate_bevel_weld.py -- + set +e + python3 tests/smoke/run_example.py --no-record \ + --name canary-skip-unexpected \ + --blender "$BLENDER" \ + --script tests/smoke/canary_skip.py \ + --series "${{ matrix.series }}" \ + --min-version 4.0 \ + --xvfb + code=$? + set -e + [ "$code" -eq 1 ] || { echo "::error::unexpected skip did not fail (got $code)"; exit 1; } + echo "unexpected skip exited 1 (correct)" - - name: Shipped example - modular kit snap (tile boundary contract) + - name: Canary - post-exit sidecar run: | set -euo pipefail - # Check only (no render): a 4x3x3 corridor segment; asserts 16 - # boundary verts on x in {0, TILE} within 1e-6, end-ring coincidence - # within 1e-6, a linked duplicate's joint positions matching within - # 1e-6, shell bbox == declared tile (dev 0), manifold shell except - # the 16 open-end edges, watertight detail parts contained inside - # the tile, and reuse hygiene (identity scales, kit names, pivot - # origins) across 21 parts. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/modular-kit-snap/modular_kit_snap.py -- + python3 tests/smoke/run_example.py \ + --name canary-sidecar \ + --blender "$BLENDER" \ + --script tests/smoke/canary_sidecar.py \ + --series "${{ matrix.series }}" \ + --expect-sidecar "$RUNNER_TEMP/out/canary.sidecar" \ + --sidecar-contains sidecar-ok \ + --xvfb - - name: Shipped example - lightmap UV channel (two-layer bake contract) + - name: Canary - missing sidecar is red run: | - set -euo pipefail - # Check only (no render): a 21-part market cart; asserts per-part - # UVMap/UVLight layers with active == UVLight and - # active_render == UVMap re-asserted after the ops (they clear - # both flags), UV0 loops untouched by the UV1 unwrap (drift 0), - # every UV1 loop inside [0,1], zero overlapping islands via an - # independent binned SAT scan, measured min island distance >= - # half the nominal pack margin, and watertight parts. Exits - # non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/lightmap-uv-channel/lightmap_uv_channel.py -- + set +e + python3 tests/smoke/run_example.py --no-record \ + --name canary-sidecar-missing \ + --blender "$BLENDER" \ + --script tests/smoke/canary_sidecar.py \ + --series "${{ matrix.series }}" \ + --expect-sidecar "$RUNNER_TEMP/out/missing.sidecar" \ + --sidecar-contains sidecar-ok \ + --xvfb \ + -- --omit-sidecar + code=$? + set -e + [ "$code" -eq 1 ] || { echo "::error::missing sidecar did not fail (got $code)"; exit 1; } + echo "missing sidecar exited 1 (correct)" - - name: Shipped example - socket attach points (spawn mount contract) + - name: Shipped examples run: | set -euo pipefail - # Check only (no render): a 21-part survey drone with 7 SKT_ socket - # empties and 7 seated modules; asserts every socket's evaluated - # world matrix equals the authored transform within 1e-6 with an - # orthonormal right-handed basis, socket +Z == the mount pad's - # Newell normal and origin == that face's centroid (two independent - # derivations), the documented up-axis fallback rule, modules seated - # with a strictly identity local transform, rigid invariance under a - # re-pose, and that freezing the root transform preserves world - # matrices while clearing all 7 parent-inverses and pushing the - # root scale down into the children. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/socket-attach-points/socket_attach_points.py -- + python3 tests/smoke/run_catalog.py \ + --blender "$BLENDER" \ + --series "${{ matrix.series }}" \ + --status "$BDT_SMOKE_STATUS" \ + --out "$RUNNER_TEMP/out" \ + --xvfb \ + --catalog tests/smoke/catalog.json - - name: Shipped example - vertex colour AO (baked occlusion attribute) + - name: Smoke summary + if: ${{ always() && env.BLENDER != '' }} run: | set -euo pipefail - # Check only (no render): a deterministic cosine-weighted hemisphere - # AO integrator validated against the closed form - # 1 - 0.5(1 - 1/sqrt(1+k^2)) at six probe distances, an unoccluded - # plate baking to exactly 1.0, strict monotonicity with distance, - # then the same integrator run over an 11-part stone well: values in - # [0,1] with real spread, FLOAT_COLOR exact round-trip vs BYTE_COLOR - # matching an independent sRGB quantisation model, both attributes - # surviving depsgraph evaluation at deviation 0 with element counts - # intact, and reuse hygiene. Exits non-zero on failure. - xvfb-run -a "$BLENDER" --background \ - --python examples/vertex-color-ao/vertex_color_ao.py -- + python3 tests/smoke/summarize.py --status "$BDT_SMOKE_STATUS" diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index a69a8ec..1f9a6f4 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -270,3 +270,12 @@ jobs: print(f'Counts verified: {skill_count} skills, {rule_count} rules, {template_count} {template_word}, {snippet_count} snippets, {example_count} examples') PYEOF + + validate-harness: + name: Validate smoke harness protocol + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Harness unit tests + run: python3 tests/smoke/test_harness.py -v diff --git a/AGENTS.md b/AGENTS.md index 1ac7f8f..aa6fab6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -157,8 +157,12 @@ way, and a one-paragraph rationale. 30 to 80 lines is the right size. the manifest `version` line (see `release.yml` below) — never hand-edit it. - `blender-smoke.yml` executes every shipped example (check-only, no render) plus snippet/template smoke tests inside REAL headless Blender, on - 5.2 LTS and 4.5 LTS for every PR (5.1 on the weekly cron). A new example is not - shipped until it has a step here. + 5.2 LTS and 4.5 LTS for every PR (5.1 on the weekly cron). Examples run + through `tests/smoke/run_example.py` (catalog: `tests/smoke/catalog.json`). + SKIP is exit 77 plus a `SMOKE_SKIP:` reason, and only when `--min-version` + is above this Blender; exit 0 with that marker is a vacuous pass and fails. + Post-exit sidecars are opt-in (`--expect-sidecar`). A leg with zero PASSes + is red. A new example is not shipped until it has a catalog row. - `drift-check.yml` consumes `Developer-Tools-Directory/.github/actions/ drift-check@v1.15` to enforce ecosystem standards-version markers. - `release.yml` auto-bumps the version, tags, force-updates floating tags diff --git a/CLAUDE.md b/CLAUDE.md index 6d2b2e7..e074316 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -130,6 +130,20 @@ Stage with **explicit paths only** — never `git add -A` or `git add .`. Cursor - **After gallery regeneration** (`python scripts/build_gallery.py`), read the **generated HTML** character by character — the `` text and witnesses callouts in `docs/gallery/index.html` and `docs/gallery//index.html` — not just `examples/gallery.json`. Precedent: the `teaches.split(".")[0]` bug truncated 14/21 card alts at dotted API paths like `bmesh.ops` while the source JSON looked fine (fixed in PR #68). - **Playwright gallery captures:** gallery `` tags lazy-load, so force them first (`document.querySelectorAll('img').forEach(i => i.loading = 'eager')`, then wait). **Scroll the target card into view and take a viewport capture** — `scrollIntoView({block:'center', behavior:'instant'})`, short wait, `browser_take_screenshot` with `fullPage` omitted. A `fullPage` capture is NOT a workaround: on a tall gallery page it renders every card image blank even when the images are verified loaded (`complete === true`, `naturalWidth === 1280`, `opacity === 1`) — measured on the 45-card grid at 1425x4516. Verify load state via `browser_evaluate` rather than trusting the pixels. +## Smoke skip and post-exit + +The host runner is `tests/smoke/run_example.py`. Shipped examples are listed in +`tests/smoke/catalog.json` (not a new YAML step per example). + +- **SKIP:** print `SMOKE_SKIP: ` and `sys.exit(77)`. Legal only when the + catalog/runner `--min-version` is **above** this Blender. Exit 0 with that + marker is FAIL (vacuous). Skip on a version that should run is FAIL. +- **Post-exit sidecar:** opt-in `--expect-sidecar PATH`. The example writes + `$BDT_SMOKE_SIDECAR` (set by the runner). The harness asserts after Blender + exits. Not a gallery still. +- **Summary:** `tests/smoke/summarize.py` prints passed/skipped/failed. Zero + PASSes makes the job red even if every example skipped cleanly. + ## Example-Run Process - The canonical example-creation prompt lives at `docs/new-example-prompt.md`; keep it in agreement with this file and `AGENTS.md`. diff --git a/docs/new-example-prompt.md b/docs/new-example-prompt.md index 194533b..9121d45 100644 --- a/docs/new-example-prompt.md +++ b/docs/new-example-prompt.md @@ -46,7 +46,11 @@ you write code: Where an API diverges between 4.5, 5.1, and 5.2, asserting each side's actual contract is part of the witness—version-gate explicitly and document the divergence rather than papering over it. Version-gate on the `bpy.app.version` tuple, not -`version_string`, which is not bare semver on LTS builds. If you discover an +`version_string`, which is not bare semver on LTS builds. If an example cannot +run on a matrix leg, it must print `SMOKE_SKIP: ` and `sys.exit(77)` — +exit 0 is a pass, not a skip. Set catalog `min_version` so a skip on a version +that should run is FAIL. Post-exit sidecars (`$BDT_SMOKE_SIDECAR`, harness +`--expect-sidecar`) are opt-in and are not gallery stills. If you discover an undocumented hazard while authoring (a crash, a dangling reference, an ordering constraint), that discovery belongs in the code comments and README; it is often more valuable than the original subject. @@ -67,8 +71,8 @@ The example must: Complete every integration required for a shipped example. Infer the exact current shape from neighboring examples and repository configuration, including the example -directory, README, gallery metadata and assets, plugin manifest, smoke workflow, -top-level README, and generated gallery pages. After regenerating the gallery with +directory, README, gallery metadata and assets, plugin manifest, smoke catalog +(`tests/smoke/catalog.json`), top-level README, and generated gallery pages. After regenerating the gallery with `python scripts/build_gallery.py`, read the **generated** output character by character—not only `examples/gallery.json` source fields. Open `docs/gallery/index.html` and `docs/gallery//index.html` and inspect the diff --git a/tests/smoke/canary_sidecar.py b/tests/smoke/canary_sidecar.py new file mode 100644 index 0000000..81afcb6 --- /dev/null +++ b/tests/smoke/canary_sidecar.py @@ -0,0 +1,23 @@ +"""Harness canary: write $BDT_SMOKE_SIDECAR then exit 0. + +The host runner asserts the file after this process dies. Pass --omit-sidecar +to prove a missing sidecar is FAIL. Not a shipped example. No gallery still. +""" +import os +import sys + +omit = "--omit-sidecar" in sys.argv +path = os.environ.get("BDT_SMOKE_SIDECAR") +if not omit: + if not path: + print("ERROR: BDT_SMOKE_SIDECAR unset", file=sys.stderr) + sys.exit(1) + parent = os.path.dirname(path) + if parent: + os.makedirs(parent, exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write("sidecar-ok\n") + print(f"wrote sidecar {path}", flush=True) +else: + print("omitting sidecar (canary red path)", flush=True) +sys.exit(0) diff --git a/tests/smoke/canary_skip.py b/tests/smoke/canary_skip.py new file mode 100644 index 0000000..c300944 --- /dev/null +++ b/tests/smoke/canary_skip.py @@ -0,0 +1,8 @@ +"""Harness canary: always skip. Legal only when --min-version is above this Blender. + +Prints SMOKE_SKIP and exits 77. Not a shipped example. +""" +import sys + +print("SMOKE_SKIP: harness canary (always skip)", flush=True) +sys.exit(77) diff --git a/tests/smoke/catalog.json b/tests/smoke/catalog.json new file mode 100644 index 0000000..644de31 --- /dev/null +++ b/tests/smoke/catalog.json @@ -0,0 +1,57 @@ +[ + { + "name": "swatch-grid", + "script": "examples/swatch-grid/swatch_grid.py", + "args": ["--output", "$OUT/swatch.png", "--engine", "cycles", "--samples", "8", "--width", "640"], + "expect_file": "$OUT/swatch.png" + }, + {"name": "turntable", "script": "examples/turntable/turntable.py"}, + {"name": "gn-sdf-remesh", "script": "examples/gn-sdf-remesh/gn_sdf_remesh.py"}, + {"name": "depsgraph-export", "script": "examples/depsgraph-export/depsgraph_export.py"}, + {"name": "wave-displace", "script": "examples/wave-displace/wave_displace.py"}, + {"name": "driver-wave", "script": "examples/driver-wave/driver_wave.py"}, + {"name": "bmesh-gear", "script": "examples/bmesh-gear/bmesh_gear.py"}, + {"name": "shader-node-group", "script": "examples/shader-node-group/shader_node_group.py"}, + {"name": "temp-override-join", "script": "examples/temp-override-join/temp_override_join.py"}, + {"name": "gn-instance-grid", "script": "examples/gn-instance-grid/gn_instance_grid.py"}, + {"name": "gn-modifier-inputs", "script": "examples/gn-modifier-inputs/gn_modifier_inputs.py"}, + {"name": "shape-key-blend", "script": "examples/shape-key-blend/shape_key_blend.py"}, + {"name": "curve-bevel-arc", "script": "examples/curve-bevel-arc/curve_bevel_arc.py"}, + {"name": "compositor-glare", "script": "examples/compositor-glare/compositor_glare.py"}, + {"name": "damped-track-aim", "script": "examples/damped-track-aim/damped_track_aim.py"}, + {"name": "color-attribute-wheel", "script": "examples/color-attribute-wheel/color_attribute_wheel.py"}, + {"name": "parent-inverse-orrery", "script": "examples/parent-inverse-orrery/parent_inverse_orrery.py"}, + {"name": "grease-pencil-rosette", "script": "examples/grease-pencil-rosette/grease_pencil_rosette.py"}, + {"name": "gp-lineart-contour", "script": "examples/gp-lineart-contour/gp_lineart_contour.py"}, + {"name": "armature-bend", "script": "examples/armature-bend/armature_bend.py"}, + {"name": "text-version-stamp", "script": "examples/text-version-stamp/text_version_stamp.py"}, + {"name": "image-pixels-testcard", "script": "examples/image-pixels-testcard/image_pixels_testcard.py"}, + {"name": "uv-layer-grid", "script": "examples/uv-layer-grid/uv_layer_grid.py"}, + {"name": "png-exr-alpha", "script": "examples/png-exr-alpha/png_exr_alpha.py"}, + {"name": "vse-cut-list", "script": "examples/vse-cut-list/vse_cut_list.py"}, + { + "name": "vse-cut-list-pixels", + "script": "examples/vse-cut-list/vse_cut_list.py", + "args": ["--check-pixels", "--engine", "cycles"] + }, + {"name": "gltf-export-roundtrip", "script": "examples/gltf-export-roundtrip/gltf_export_roundtrip.py"}, + {"name": "lod-decimate-chain", "script": "examples/lod-decimate-chain/lod_decimate_chain.py"}, + {"name": "vertex-weight-limit", "script": "examples/vertex-weight-limit/vertex_weight_limit.py"}, + {"name": "triangulate-tangents", "script": "examples/triangulate-tangents/triangulate_tangents.py"}, + {"name": "gltf-skin-roundtrip", "script": "examples/gltf-skin-roundtrip/gltf_skin_roundtrip.py"}, + {"name": "vse-gamma-cross", "script": "examples/vse-gamma-cross/vse_gamma_cross.py"}, + {"name": "light-link-studio", "script": "examples/light-link-studio/light_link_studio.py"}, + {"name": "collision-hull-proxy", "script": "examples/collision-hull-proxy/collision_hull_proxy.py"}, + {"name": "custom-normals-shade", "script": "examples/custom-normals-shade/custom_normals_shade.py"}, + {"name": "sky-texture-sun-elevation", "script": "examples/sky-texture-sun-elevation/sky_texture_sun_elevation.py"}, + {"name": "mesh-hygiene-audit", "script": "examples/mesh-hygiene-audit/mesh_hygiene_audit.py"}, + {"name": "prop-origin-transform", "script": "examples/prop-origin-transform/prop_origin_transform.py"}, + {"name": "soccer-ball-goldberg", "script": "examples/soccer-ball-goldberg/soccer_ball_goldberg.py"}, + {"name": "car-mirror-symmetry", "script": "examples/car-mirror-symmetry/car_mirror_symmetry.py"}, + {"name": "attribute-domain-shear", "script": "examples/attribute-domain-shear/attribute_domain_shear.py"}, + {"name": "degenerate-bevel-weld", "script": "examples/degenerate-bevel-weld/degenerate_bevel_weld.py"}, + {"name": "modular-kit-snap", "script": "examples/modular-kit-snap/modular_kit_snap.py"}, + {"name": "lightmap-uv-channel", "script": "examples/lightmap-uv-channel/lightmap_uv_channel.py"}, + {"name": "socket-attach-points", "script": "examples/socket-attach-points/socket_attach_points.py"}, + {"name": "vertex-color-ao", "script": "examples/vertex-color-ao/vertex_color_ao.py"} +] diff --git a/tests/smoke/protocol.py b/tests/smoke/protocol.py new file mode 100644 index 0000000..c0c3957 --- /dev/null +++ b/tests/smoke/protocol.py @@ -0,0 +1,132 @@ +"""Skip / pass / fail classification for the host smoke harness. + +Blender example scripts have two legal ways to finish: + +* exit 0 — ran and passed (then optional post-exit sidecar) +* exit 77 with a ``SMOKE_SKIP: `` line — unsupported on this Blender + +Anything else is FAIL, including the vacuous-pass vector: printing a skip +marker and exiting 0, or exiting 77 on a version that should run. +""" +from __future__ import annotations + +import os +from typing import Optional, Tuple + +SKIP_EXIT = 77 +SKIP_PREFIX = "SMOKE_SKIP:" + +PASS = "PASS" +SKIP = "SKIP" +FAIL = "FAIL" + + +def parse_version(spec: str) -> Tuple[int, ...]: + """'5.2' or '5.2.1' or '5.0' -> comparable tuple (pad to 3).""" + parts = [] + for p in spec.strip().split("."): + if not p.isdigit(): + raise ValueError(f"not a version spec: {spec!r}") + parts.append(int(p)) + if not parts: + raise ValueError(f"empty version spec: {spec!r}") + while len(parts) < 3: + parts.append(0) + return tuple(parts[:3]) + + +def parse_skip_reason(output: str) -> Optional[str]: + for line in output.splitlines(): + s = line.strip() + if s.startswith(SKIP_PREFIX): + reason = s[len(SKIP_PREFIX):].strip() + return reason or None + return None + + +def sidecar_ok(path: Optional[str], contains: Optional[str]) -> Tuple[bool, str]: + if not path: + return False, "sidecar path not set" + if not os.path.isfile(path): + return False, f"missing post-exit sidecar {path}" + if os.path.getsize(path) == 0: + return False, f"empty post-exit sidecar {path}" + if contains: + with open(path, encoding="utf-8") as fh: + text = fh.read() + if contains not in text: + return False, f"sidecar missing expected text {contains!r}" + return True, "" + + +def classify( + *, + proc_exit: int, + output: str, + min_version: Optional[str] = None, + blender_version: Optional[str] = None, + forbid_skip: bool = False, + expect_sidecar: bool = False, + sidecar_path: Optional[str] = None, + sidecar_contains: Optional[str] = None, +) -> Tuple[str, str]: + """Return (PASS|SKIP|FAIL, detail). + + Skip is legal only when ``min_version`` is set and ``blender_version`` + is strictly below it (and ``forbid_skip`` is false). An expected skip + does not require a sidecar. + """ + reason = parse_skip_reason(output) + skipped = proc_exit == SKIP_EXIT + + if proc_exit == 0 and reason is not None: + return FAIL, "SMOKE_SKIP marker with exit 0 (vacuous skip)" + + if skipped: + if not reason: + return FAIL, f"exit {SKIP_EXIT} without SMOKE_SKIP reason" + if forbid_skip: + return FAIL, f"skipped where skip is forbidden: {reason}" + if min_version is None: + return FAIL, f"skipped with no --min-version (unexpected): {reason}" + if blender_version is None: + return FAIL, f"skipped but blender version unknown: {reason}" + if parse_version(blender_version) >= parse_version(min_version): + return ( + FAIL, + f"skipped on {blender_version} but min-version {min_version} " + f"(should run): {reason}", + ) + return SKIP, reason + + if proc_exit != 0: + return FAIL, f"blender exit {proc_exit}" + + if expect_sidecar: + ok, detail = sidecar_ok(sidecar_path, sidecar_contains) + if not ok: + return FAIL, detail + + return PASS, "" + + +def summarize_records(records): + """records: iterable of dicts with 'status' key. + + Returns (passed, skipped, failed, harness_exit). + harness_exit is 0 on mixed pass+skip, 1 if any FAIL, 2 if zero PASS. + """ + passed = skipped = failed = 0 + for rec in records: + st = rec["status"] + if st == PASS: + passed += 1 + elif st == SKIP: + skipped += 1 + else: + failed += 1 + if failed: + return passed, skipped, failed, 1 + if passed == 0: + return passed, skipped, failed, 2 + return passed, skipped, failed, 0 diff --git a/tests/smoke/run_catalog.py b/tests/smoke/run_catalog.py new file mode 100644 index 0000000..9b0d1a8 --- /dev/null +++ b/tests/smoke/run_catalog.py @@ -0,0 +1,76 @@ +"""Run every shipped example through run_example.classify via run_example.py. + +Stops on the first FAIL (same fail-fast as the previous per-step YAML). +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def main(argv=None): + p = argparse.ArgumentParser() + p.add_argument("--blender", required=True) + p.add_argument("--catalog", default=os.path.join(HERE, "catalog.json")) + p.add_argument("--series", required=True) + p.add_argument("--status", default=os.environ.get("BDT_SMOKE_STATUS")) + p.add_argument("--out", required=True, help="scratch dir for --output renders") + p.add_argument("--xvfb", action="store_true") + args = p.parse_args(argv) + + with open(args.catalog, encoding="utf-8") as fh: + catalog = json.load(fh) + + os.makedirs(args.out, exist_ok=True) + runner = os.path.join(HERE, "run_example.py") + n = 0 + for item in catalog: + n += 1 + name = item["name"] + script = item["script"] + extra = list(item.get("args") or []) + extra = [a.replace("$OUT", args.out) for a in extra] + cmd = [ + sys.executable, + runner, + "--name", + name, + "--blender", + args.blender, + "--script", + script, + "--series", + args.series, + "--status", + args.status or "", + ] + if args.xvfb: + cmd.append("--xvfb") + if item.get("min_version"): + cmd.extend(["--min-version", item["min_version"]]) + if extra: + cmd.append("--") + cmd.extend(extra) + print(f"::group::{name}", flush=True) + code = subprocess.call(cmd) + print("::endgroup::", flush=True) + if code != 0: + print(f"catalog abort at {name} (exit {code})", file=sys.stderr) + return code + expect = item.get("expect_file") + if expect: + expect = expect.replace("$OUT", args.out) + if not os.path.isfile(expect) or os.path.getsize(expect) == 0: + print(f"ERROR: expected output missing {expect}", file=sys.stderr) + return 1 + print(f"catalog finished {n} entries", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/smoke/run_example.py b/tests/smoke/run_example.py new file mode 100644 index 0000000..32d28a0 --- /dev/null +++ b/tests/smoke/run_example.py @@ -0,0 +1,138 @@ +"""Host-side runner: invoke one Blender script, classify PASS / SKIP / FAIL. + +Usage: + python tests/smoke/run_example.py --name NAME --blender BIN --script PATH + [--series 5.2] [--min-version 5.0] [--xvfb] + [--expect-sidecar FILE] [--sidecar-contains TEXT] + [--forbid-skip] [--status FILE] + -- extra args passed after Blender's `--` +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from datetime import datetime, timezone + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from protocol import classify + + +def _now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def append_status(path, record): + if not path: + return + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(record, ensure_ascii=True) + "\n") + + +def build_cmd(args): + cmd = [args.blender, "--background", "--python", args.script, "--"] + cmd.extend(args.script_args) + if args.xvfb: + cmd = ["xvfb-run", "-a"] + cmd + return cmd + + +def run(args): + sidecar = args.expect_sidecar + env = os.environ.copy() + if sidecar: + env["BDT_SMOKE_SIDECAR"] = sidecar + parent = os.path.dirname(sidecar) + if parent: + os.makedirs(parent, exist_ok=True) + if os.path.exists(sidecar): + os.remove(sidecar) + + cmd = build_cmd(args) + print(f"=== run_example {args.name} ===", flush=True) + print(" ".join(cmd), flush=True) + + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=env, + ) + chunks = [] + assert proc.stdout is not None + for line in proc.stdout: + sys.stdout.write(line) + sys.stdout.flush() + chunks.append(line) + proc.wait() + output = "".join(chunks) + proc_exit = proc.returncode if proc.returncode is not None else 1 + + status, detail = classify( + proc_exit=proc_exit, + output=output, + min_version=args.min_version, + blender_version=args.series, + forbid_skip=args.forbid_skip, + expect_sidecar=bool(sidecar), + sidecar_path=sidecar, + sidecar_contains=args.sidecar_contains, + ) + record = { + "name": args.name, + "status": status, + "detail": detail, + "proc_exit": proc_exit, + "ts": _now(), + } + append_status(args.status, record) + + if status == "PASS": + print(f"[PASS] {args.name}", flush=True) + return 0 + if status == "SKIP": + print(f"[SKIP] {args.name}: {detail}", flush=True) + # Expected skip must not fail the YAML step; summarize counts it. + return 0 + print(f"[FAIL] {args.name}: {detail}", file=sys.stderr, flush=True) + return 1 + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--name", required=True) + p.add_argument("--blender", required=True) + p.add_argument("--script", required=True) + p.add_argument("--series", default=None, help="matrix series, e.g. 5.2 or 4.5") + p.add_argument("--min-version", default=None) + p.add_argument("--xvfb", action="store_true") + p.add_argument("--expect-sidecar", default=None) + p.add_argument("--sidecar-contains", default=None) + p.add_argument("--forbid-skip", action="store_true") + p.add_argument( + "--no-record", + action="store_true", + help="do not append to the status file (inverted canaries)", + ) + p.add_argument( + "--status", + default=os.environ.get("BDT_SMOKE_STATUS"), + help="JSONL status file (default $BDT_SMOKE_STATUS)", + ) + p.add_argument("script_args", nargs=argparse.REMAINDER) + args = p.parse_args(argv) + if args.script_args and args.script_args[0] == "--": + args.script_args = args.script_args[1:] + if args.no_record: + args.status = None + elif not args.status: + args.status = None + return run(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/smoke/run_smoke.py b/tests/smoke/run_smoke.py index 591f9e1..25152ab 100644 --- a/tests/smoke/run_smoke.py +++ b/tests/smoke/run_smoke.py @@ -9,19 +9,32 @@ """ import bpy, sys, os, tempfile +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from protocol import PASS, SKIP, FAIL, summarize_records + ARGS = sys.argv[sys.argv.index("--") + 1:] if "--" in sys.argv else [] OUT = ARGS[0] if ARGS else tempfile.mkdtemp() os.makedirs(OUT, exist_ok=True) V = bpy.app.version print(f"=== smoke on Blender {V[0]}.{V[1]}.{V[2]} -> {OUT} ===") +_RESULTS = [] + def require(example, cond, evidence): """Pass-or-die assertion. On failure print which example failed and exit 1.""" status = "ok" if cond else "FAIL" print(f"[{status}] {example}: {evidence}") if not cond: + _RESULTS.append({"name": example, "status": FAIL, "detail": evidence}) print(f"SMOKE FAILED at example '{example}' on Blender {V[0]}.{V[1]}.{V[2]}") sys.exit(1) + _RESULTS.append({"name": example, "status": PASS, "detail": evidence}) + + +def skip(example, reason): + """Record a version skip. Does not fail. A run with zero PASSes still fails.""" + print(f"[SKIP] {example}: {reason}") + _RESULTS.append({"name": example, "status": SKIP, "detail": reason}) def reset(): bpy.ops.wm.read_factory_settings(use_empty=True) @@ -182,4 +195,12 @@ def on_save_pre(arg, *a): captured['t'] = type(arg).__name__ f"Blender {V[0]}.{V[1]}.{V[2]}: {e}") sys.exit(1) +passed, skipped, failed, code = summarize_records(_RESULTS) +print( + f"=== SMOKE SUMMARY: {passed} passed, {skipped} skipped, {failed} failed " + f"on Blender {V[0]}.{V[1]}.{V[2]} ===" +) +if code == 2: + print("SMOKE FAILED: every check skipped — job is not green") + sys.exit(2) print(f"=== ALL SMOKE CHECKS PASSED on Blender {V[0]}.{V[1]}.{V[2]} ===") diff --git a/tests/smoke/summarize.py b/tests/smoke/summarize.py new file mode 100644 index 0000000..9f44458 --- /dev/null +++ b/tests/smoke/summarize.py @@ -0,0 +1,74 @@ +"""Print PASS/SKIP/FAIL counts from a JSONL status file. + +Exit 0 if at least one PASS and no FAIL. +Exit 1 if any FAIL. +Exit 2 if every recorded example skipped (or nothing ran) — a green all-skip is illegal. + +Writes GitHub step summary when $GITHUB_STEP_SUMMARY is set. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from protocol import summarize_records + + +def load(path): + if not path or not os.path.isfile(path): + return [] + records = [] + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line: + records.append(json.loads(line)) + return records + + +def render(records, passed, skipped, failed): + lines = [ + "## Smoke summary", + "", + f"**{passed} passed**, **{skipped} skipped**, **{failed} failed**", + "", + "| Example | Result | Detail |", + "| --- | --- | --- |", + ] + for rec in records: + detail = (rec.get("detail") or "").replace("|", "\\|") + lines.append(f"| {rec['name']} | {rec['status']} | {detail} |") + lines.append("") + return "\n".join(lines) + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--status", + default=os.environ.get("BDT_SMOKE_STATUS"), + required=False, + ) + args = p.parse_args(argv) + records = load(args.status) + passed, skipped, failed, code = summarize_records(records) + text = render(records, passed, skipped, failed) + print(text) + if code == 2: + print( + "FAIL: every example skipped (or none ran); job is not green", + file=sys.stderr, + ) + gh = os.environ.get("GITHUB_STEP_SUMMARY") + if gh: + with open(gh, "a", encoding="utf-8") as fh: + fh.write(text) + fh.write("\n") + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/smoke/test_harness.py b/tests/smoke/test_harness.py new file mode 100644 index 0000000..58b8e0e --- /dev/null +++ b/tests/smoke/test_harness.py @@ -0,0 +1,163 @@ +"""Stdlib tests for skip / sidecar classification. No Blender required. + +These are the red-path proofs for the harness itself: unexpected skip is +FAIL, vacuous skip (marker + exit 0) is FAIL, missing sidecar is FAIL, +all-skip summary is not green. +""" +from __future__ import annotations + +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from protocol import FAIL, PASS, SKIP, SKIP_EXIT, classify, summarize_records + + +class ClassifySkip(unittest.TestCase): + def test_pass_exit_0(self): + st, _ = classify(proc_exit=0, output="ok\n") + self.assertEqual(st, PASS) + + def test_expected_skip_below_floor(self): + st, detail = classify( + proc_exit=SKIP_EXIT, + output="SMOKE_SKIP: Bundles require Blender 5.0+\n", + min_version="5.0", + blender_version="4.5", + ) + self.assertEqual(st, SKIP) + self.assertIn("Bundles", detail) + + def test_unexpected_skip_on_supported_version_is_fail(self): + st, detail = classify( + proc_exit=SKIP_EXIT, + output="SMOKE_SKIP: Bundles require Blender 5.0+\n", + min_version="5.0", + blender_version="5.2", + ) + self.assertEqual(st, FAIL) + self.assertIn("should run", detail) + + def test_skip_without_min_version_is_fail(self): + st, _ = classify( + proc_exit=SKIP_EXIT, + output="SMOKE_SKIP: because I felt like it\n", + ) + self.assertEqual(st, FAIL) + + def test_forbid_skip_is_fail(self): + st, _ = classify( + proc_exit=SKIP_EXIT, + output="SMOKE_SKIP: no\n", + min_version="99.0", + blender_version="4.5", + forbid_skip=True, + ) + self.assertEqual(st, FAIL) + + def test_vacuous_skip_exit_0_with_marker_is_fail(self): + st, detail = classify( + proc_exit=0, + output="SMOKE_SKIP: pretending\n", + min_version="5.0", + blender_version="4.5", + ) + self.assertEqual(st, FAIL) + self.assertIn("vacuous", detail) + + def test_skip_exit_without_reason_is_fail(self): + st, _ = classify(proc_exit=SKIP_EXIT, output="no marker\n") + self.assertEqual(st, FAIL) + + def test_blender_nonzero_is_fail(self): + st, detail = classify(proc_exit=12, output="ERROR\n") + self.assertEqual(st, FAIL) + self.assertIn("12", detail) + + +class ClassifySidecar(unittest.TestCase): + def test_missing_sidecar_is_fail(self): + st, detail = classify( + proc_exit=0, + output="ok\n", + expect_sidecar=True, + sidecar_path=os.path.join(tempfile.gettempdir(), "no-such-sidecar-bdt"), + ) + self.assertEqual(st, FAIL) + self.assertIn("missing", detail) + + def test_sidecar_present_is_pass(self): + fd, path = tempfile.mkstemp(prefix="bdt-sidecar-") + os.close(fd) + try: + with open(path, "w", encoding="utf-8") as fh: + fh.write("sidecar-ok\n") + st, _ = classify( + proc_exit=0, + output="ok\n", + expect_sidecar=True, + sidecar_path=path, + sidecar_contains="sidecar-ok", + ) + self.assertEqual(st, PASS) + finally: + os.remove(path) + + def test_sidecar_wrong_contents_is_fail(self): + fd, path = tempfile.mkstemp(prefix="bdt-sidecar-") + os.close(fd) + try: + with open(path, "w", encoding="utf-8") as fh: + fh.write("nope\n") + st, _ = classify( + proc_exit=0, + output="ok\n", + expect_sidecar=True, + sidecar_path=path, + sidecar_contains="sidecar-ok", + ) + self.assertEqual(st, FAIL) + finally: + os.remove(path) + + def test_expected_skip_does_not_require_sidecar(self): + st, _ = classify( + proc_exit=SKIP_EXIT, + output="SMOKE_SKIP: not on 4.5\n", + min_version="5.1", + blender_version="4.5", + expect_sidecar=True, + sidecar_path="/nonexistent", + ) + self.assertEqual(st, SKIP) + + +class Summarize(unittest.TestCase): + def test_pass_and_skip_is_green(self): + p, s, f, code = summarize_records( + [{"status": PASS}, {"status": SKIP, "detail": "x"}] + ) + self.assertEqual((p, s, f, code), (1, 1, 0, 0)) + + def test_any_fail_is_red(self): + _, _, _, code = summarize_records( + [{"status": PASS}, {"status": FAIL, "detail": "x"}] + ) + self.assertEqual(code, 1) + + def test_all_skip_is_not_green(self): + p, s, f, code = summarize_records( + [{"status": SKIP, "detail": "a"}, {"status": SKIP, "detail": "b"}] + ) + self.assertEqual((p, s, f), (0, 2, 0)) + self.assertEqual(code, 2) + + def test_empty_is_not_green(self): + _, _, _, code = summarize_records([]) + self.assertEqual(code, 2) + + +if __name__ == "__main__": + unittest.main()