From 4469ea7aca937d57dbf627d3724bafabe414d1ea Mon Sep 17 00:00:00 2001 From: Mike Langmayr <1809691+mikelangmayr@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:48:40 -0700 Subject: [PATCH 1/4] Give RAW captures their own output stream and gate reads on RAWENABLE --- .github/workflows/emulator-integration.yml | 29 ++++++++++ README.md | 6 ++- camerad/archon_controller.cpp | 51 +++++++++++++++++- camerad/archon_controller.h | 8 +++ camerad/archon_interface.cpp | 1 + utils/fits_writer.cpp | 50 +++++++++-------- utils/fits_writer.h | 16 ++++-- utils/frame_output.h | 5 ++ utils/shared_memory_writer.cpp | 62 ++++++++++++---------- utils/shared_memory_writer.h | 26 ++++++--- 10 files changed, 190 insertions(+), 64 deletions(-) diff --git a/.github/workflows/emulator-integration.yml b/.github/workflows/emulator-integration.yml index 86be029..aba24a6 100644 --- a/.github/workflows/emulator-integration.yml +++ b/.github/workflows/emulator-integration.yml @@ -77,6 +77,35 @@ jobs: bin/camerad-shm-reader ci_frame_outputs_shm /tmp/ci_milk_shm ls /tmp/ci_fits_test/ci_frame_outputs_*.fits + image_size() { + bin/camerad-shm-reader ci_frame_outputs_shm /tmp/ci_milk_shm \ + | grep -o 'size=[0-9]*x[0-9]*' | head -1 + } + before=$(image_size) + + # read must refuse to fetch until capture is actually enabled + send "raw read" | grep -q "ERROR" || { echo "raw read ran with RAWENABLE=0"; exit 1; } + + # The Archon captures raw alongside the frame, so enabling it has to + # precede the exposure it should appear in + send "raw set RAWENABLE 1" + resp=$(send "expose 1") + echo "expose -> $resp" + echo "$resp" | grep -q "DONE" || exit 1 + + resp=$(send "raw read") + echo "raw read -> $resp" + echo "$resp" | grep -q "DONE" || exit 1 + sleep 1 + + # RAW has its own geometry, so it has to land in its own outputs rather + # than collide with the image file or resize the image's SHM stream + bin/camerad-shm-reader ci_frame_outputs_shm_raw /tmp/ci_milk_shm + ls /tmp/ci_fits_test/ci_frame_outputs_*_raw.fits + + after=$(image_size) + test "$before" = "$after" || { echo "image stream changed: $before -> $after"; exit 1; } + pkill -f 'bin/camerad' || true pkill -f 'bin/camerad-emulator' || true diff --git a/README.md b/README.md index ce112e5..8c9aec6 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,10 @@ Logging follows `LOG_STDERR` from the `.cfg`; pass `log_to_stderr=` to override Every instrument publishes each acquired frame to one or more outputs, configured entirely via `.cfg` file keys (`Camera::Interface::configure_frame_outputs()` builds them from `Camera::apply_config_overrides()`, called once at startup for every instrument, not just HISPEC). Both outputs are independent; either, both, or neither can be enabled per instrument. +### Streams + +Every frame carries an optional stream name so that one exposure can deliver outputs of different geometry without them colliding. The primary image leaves it empty; an Archon `raw read` sets it to `raw`. FITS appends the name to the filename (`image_00000123_raw.fits`) and shared memory appends it to the segment name (`camera_raw`), so a RAW capture neither overwrites the image file nor resizes the image stream. + ### FITS Writes one FITS file per frame asynchronously (a queue plus a dedicated writer thread, so the readout thread never blocks on disk I/O). @@ -225,7 +229,7 @@ Publishes each frame as an [ImageStreamIO](https://github.com/milk-org/ImageStre | `SHM_RING_BUFFER_SIZE` | `4` | Depth of ImageStreamIO's internal history ring buffer (`CBsize`); the live frame a real-time reader sees is separate from this | | `SHM_DIR` | (unset) | Base directory ImageStreamIO writes into. If unset, ImageStreamIO falls back to its own default resolution (`MILK_SHM_DIR` env var, then `/milk/shm`). If set, it must already exist and be writable. | -Frame geometry (width/height/pixel depth) isn't a config key: it's fixed for an ImageStreamIO stream's whole life, so the writer (re)creates the stream automatically whenever it sees the geometry change from what's currently allocated. +Frame geometry (width/height/pixel depth) isn't a config key: it's fixed for an ImageStreamIO stream's whole life, so the writer (re)creates a stream automatically whenever it sees the geometry change from what's currently allocated for that stream. ## Heater & Sensor Control diff --git a/camerad/archon_controller.cpp b/camerad/archon_controller.cpp index b50d761..a53d7c2 100644 --- a/camerad/archon_controller.cpp +++ b/camerad/archon_controller.cpp @@ -1776,6 +1776,7 @@ namespace Camera { this->get_configmap_value("BIGBUF", mode->bigbuf); this->get_configmap_value("FRAMEMODE", mode->geometry.framemode); this->get_configmap_value("RAWENABLE", mode->rawenable); + this->get_configmap_value("RAWENABLE", this->rawinfo.enable); this->get_configmap_value("RAWSEL", this->rawinfo.adchan); this->get_configmap_value("RAWSAMPLES", this->rawinfo.samples); this->get_configmap_value("RAWSTARTLINE", this->rawinfo.startline); @@ -2415,6 +2416,7 @@ namespace Camera { } // refresh cached geometry from the now-updated configmap + this->get_configmap_value("RAWENABLE", this->rawinfo.enable); this->get_configmap_value("RAWSEL", this->rawinfo.adchan); this->get_configmap_value("RAWSAMPLES", this->rawinfo.samples); this->get_configmap_value("RAWSTARTLINE", this->rawinfo.startline); @@ -2426,6 +2428,34 @@ namespace Camera { /***** Camera::ArchonController::set_raw_config **************************/ + /***** Camera::ArchonController::raw_frame_keys **************************/ + /** + * @brief build the provenance keys describing a RAW capture + * @details An AD channel and an ADM channel arrive as an identical uint16 + * block sampled at different rates, so a reader cannot interpret + * the samples without RAWSEL and the selected module's type. + * @return shared_ptr suitable for FrameMetadata::frame_keys + */ + std::shared_ptr ArchonController::raw_frame_keys() const { + // RAWSEL indexes four channels per slot, starting at the first AD slot + const int channel = this->rawinfo.adchan; + const int slot = AD_SLOT_FIRST + channel / MAXNADCHAN; + + auto keys = std::make_shared(); + keys->addkey("RAWSEL", channel, "Archon RAWSEL, channel captured"); + keys->addkey("RAWSLOT", slot, "backplane slot RAWSEL selects"); + if (slot >= 1 && slot <= static_cast(this->modtype.size())) { + keys->addkey("RAWMODTY", this->modtype[slot-1], "MODn_TYPE of the selected slot"); + } + keys->addkey("RAWSAMP", static_cast(this->rawinfo.samples), "RAWSAMPLES"); + keys->addkey("RAWSLINE", static_cast(this->rawinfo.startline), "RAWSTARTLINE"); + keys->addkey("RAWELINE", static_cast(this->rawinfo.endline), "RAWENDLINE"); + keys->addkey("RAWSPIX", static_cast(this->rawinfo.startpixel), "RAWSTARTPIXEL"); + return keys; + } + /***** Camera::ArchonController::raw_frame_keys **************************/ + + /***** Camera::ArchonController::read_raw *******************************/ /** * @brief retrieve RAW (pre-CDS) data from the newest frame buffer @@ -2437,6 +2467,14 @@ namespace Camera { long ArchonController::read_raw(std::string &retstring) { const std::string function("Camera::ArchonController::read_raw"); + // Without this the fetch returns whatever sits at the raw offset, since a + // controller with capture off is indistinguishable from one reporting nothing + if (this->rawinfo.enable == 0) { + logwrite(function, "ERROR RAW capture is disabled"); + retstring = "RAW capture is disabled; set it with \"raw set RAWENABLE 1\""; + return ERROR; + } + long error = this->get_frame_status(); if (error != NO_ERROR) { logwrite(function, "ERROR getting frame status"); @@ -2444,6 +2482,16 @@ namespace Camera { return error; } + const auto index = this->frameinfo.index.load(); + + // Raw is captured with the frame, so a buffer filled before RAWENABLE was set + // reports zero and raw_geometry() falls back to inferring from the config keys + if (this->frameinfo.bufrawblocks[index] == 0 || + this->frameinfo.bufrawlines[index] == 0) { + logwrite(function, "WARNING controller reports no raw data in this buffer; " + "using configured geometry"); + } + const raw_geometry_t geom = this->raw_geometry(); if (geom.samples == 0 || geom.lines == 0) { logwrite(function, "ERROR RAW geometry is empty; check RAW config"); @@ -2472,7 +2520,6 @@ namespace Camera { static_cast(geom.samples) * sizeof(uint16_t)); } - const auto index = this->frameinfo.index.load(); const size_t payload_bytes = payload_samples * sizeof(uint16_t); Camera::FrameMetadata meta; @@ -2481,6 +2528,8 @@ namespace Camera { meta.width = geom.samples; meta.height = geom.lines; meta.bytes_per_pixel = sizeof(uint16_t); + meta.stream = RAW_STREAM; + meta.frame_keys = this->raw_frame_keys(); this->interface->dispatch_frame(reinterpret_cast(samples.data()), payload_bytes, meta); diff --git a/camerad/archon_controller.h b/camerad/archon_controller.h index aa96558..68695c7 100644 --- a/camerad/archon_controller.h +++ b/camerad/archon_controller.h @@ -47,6 +47,12 @@ constexpr int MODTYPE_ADLN = 15; constexpr int MODTYPE_UNKNOWN = 16; constexpr int MODTYPE_ADM = 17; +// First backplane slot an AD or ADM module may occupy, which RAWSEL indexes from +constexpr int AD_SLOT_FIRST = 5; + +// Stream label keeping a pre-CDS RAW capture in outputs of its own +const std::string RAW_STREAM = "raw"; + /** * Archon commands */ @@ -253,6 +259,7 @@ namespace Camera { /** @brief Archon RAW (pre-CDS) capture configuration, mirrors ACF keywords */ struct rawinfo_t { + int enable{0}; // RAWENABLE: whether the controller captures raw at all int adchan{0}; // RAWSEL: AD channel captured uint16_t samples{0}; // RAWSAMPLES: 16-bit samples per line uint16_t startline{0}; // RAWSTARTLINE @@ -383,6 +390,7 @@ namespace Camera { uint32_t raw_frame_bytes() const; // padded, size-aware byte count for a RAW fetch long set_raw_config(const std::string &args, std::string &retstring); long get_raw_config(std::string &retstring); + std::shared_ptr raw_frame_keys() const; long read_raw(std::string &retstring); diff --git a/camerad/archon_interface.cpp b/camerad/archon_interface.cpp index 9654c55..7a8f69b 100644 --- a/camerad/archon_interface.cpp +++ b/camerad/archon_interface.cpp @@ -968,6 +968,7 @@ namespace Camera { retstring.append( " set .. set RAW keyword(s) then apply\n" ); retstring.append( " read retrieve RAW data in-band as 16-bit samples\n" ); retstring.append( " Keys: RAWENABLE RAWSEL RAWSTARTLINE RAWENDLINE RAWSTARTPIXEL RAWSAMPLES\n" ); + retstring.append( " RAWENABLE must be set before the exposure the raw data comes from\n" ); return HELP; } diff --git a/utils/fits_writer.cpp b/utils/fits_writer.cpp index 8325e87..86f9a5f 100644 --- a/utils/fits_writer.cpp +++ b/utils/fits_writer.cpp @@ -133,7 +133,7 @@ namespace Camera { cv_.notify_all(); if (worker_.joinable()) worker_.join(); - close_cube(); // worker has exited, safe to touch its state here + close_cubes(); // worker has exited, safe to touch its state here started_.store(false); const std::string function("Camera::FitsWriter::close"); @@ -217,7 +217,7 @@ namespace Camera { } if (frame.end_of_exposure) { - close_cube(); + close_cubes(); continue; } @@ -231,13 +231,13 @@ namespace Camera { long FitsWriter::write_fits_file(const QueuedFrame &frame) { if (cube_enabled_.load()) return write_cube_frame(frame); - if (cube_fits_) close_cube(); // datacube was just turned off mid-cube; finalize it + if (!cubes_.empty()) close_cubes(); // datacube was just turned off mid-cube; finalize it const std::string function("Camera::FitsWriter::write_fits_file"); const auto &meta = frame.meta; const size_t npixels = static_cast(meta.width) * meta.height; - const std::string filename = make_filename(meta.frame_number); + const std::string filename = make_filename(meta); // CCfits requires a non-existing path; "!" prefix would overwrite, // but make_filename() already resolved any conflict const int bitpix = (meta.bytes_per_pixel == 2) ? USHORT_IMG : ULONG_IMG; @@ -291,21 +291,22 @@ namespace Camera { const auto &meta = frame.meta; const size_t npixels = static_cast(meta.width) * meta.height; const int bitpix = (meta.bytes_per_pixel == 2) ? USHORT_IMG : ULONG_IMG; + OpenCube &cube = cubes_[meta.stream]; try { - if (!cube_fits_) { - const std::string filename = make_filename(meta.frame_number); + if (!cube.fits) { + const std::string filename = make_filename(meta); long axes[2] = {0, 0}; // NAXIS=0: header-only primary, matches v1's cube primary - cube_fits_ = std::make_unique(filename, bitpix, 0, axes); - add_keys_from(cube_fits_->pHDU(), meta.header_set.get()); - cube_extension_count_ = 0; + cube.fits = std::make_unique(filename, bitpix, 0, axes); + add_keys_from(cube.fits->pHDU(), meta.header_set.get()); + cube.extension_count = 0; logwrite(function, "opened cube " + filename); } std::vector ext_axes = { static_cast(meta.width), static_cast(meta.height) }; - const std::string extname = std::to_string(cube_extension_count_ + 1); - auto *ext = cube_fits_->addImage(extname, bitpix, ext_axes); + const std::string extname = std::to_string(cube.extension_count + 1); + auto *ext = cube.fits->addImage(extname, bitpix, ext_axes); if (bitpix == USHORT_IMG) { ext->addKey("BZERO", 32768, "offset for signed short int"); @@ -327,8 +328,8 @@ namespace Camera { std::valarray data(src, npixels); ext->write(first_pixel, npixels, data); } - cube_fits_->flush(); - ++cube_extension_count_; + cube.fits->flush(); + ++cube.extension_count; } catch (const CCfits::FitsException &e) { logwrite(function, "ERROR FITS exception writing cube extension: " + e.message()); @@ -342,12 +343,14 @@ namespace Camera { return NO_ERROR; } - void FitsWriter::close_cube() { - if (!cube_fits_) return; - logwrite("Camera::FitsWriter::close_cube", - "closed cube with " + std::to_string(cube_extension_count_) + " extensions"); - cube_fits_.reset(); - cube_extension_count_ = 0; + void FitsWriter::close_cubes() { + for (const auto &[stream, cube] : cubes_) { + if (!cube.fits) continue; + logwrite("Camera::FitsWriter::close_cubes", + "closed cube " + (stream.empty() ? "image" : stream) + " with " + + std::to_string(cube.extension_count) + " extensions"); + } + cubes_.clear(); } std::string FitsWriter::resolve_output_dir() { @@ -369,12 +372,15 @@ namespace Camera { return dir; } - std::string FitsWriter::make_filename(uint64_t frame_number) { + std::string FitsWriter::make_filename(const FrameMetadata &meta) { char num[32]; std::snprintf(num, sizeof(num), "%08llu", - static_cast(frame_number)); + static_cast(meta.frame_number)); - const std::string prefix = resolve_output_dir() + "/" + cfg_.basename + "_" + num; + // Stream last, so a frame and its off-geometry companions sort adjacently + const std::string stream_part = meta.stream.empty() ? "" : "_" + meta.stream; + const std::string prefix = + resolve_output_dir() + "/" + cfg_.basename + "_" + num + stream_part; std::string filename = prefix + ".fits"; int suffix = 1; diff --git a/utils/fits_writer.h b/utils/fits_writer.h index 0b11b19..c69affc 100644 --- a/utils/fits_writer.h +++ b/utils/fits_writer.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -68,8 +69,8 @@ namespace Camera { void worker_loop(); long write_fits_file(const QueuedFrame &frame); long write_cube_frame(const QueuedFrame &frame); - void close_cube(); - std::string make_filename(uint64_t frame_number); + void close_cubes(); + std::string make_filename(const FrameMetadata &meta); // Resolve the target directory, creating today's autodir subdir on demand. // Worker-thread only (uses cur_date_dir_ without locking). std::string resolve_output_dir(); @@ -79,9 +80,14 @@ namespace Camera { // Cache of the last autodir path created; touched only on the worker thread std::string cur_date_dir_; - // Datacube state — worker-thread only, no locking needed - std::unique_ptr cube_fits_; - uint32_t cube_extension_count_{0}; + /// One datacube being appended to, per FrameMetadata::stream + struct OpenCube { + std::unique_ptr fits; + uint32_t extension_count{0}; + }; + + // Datacube state, worker-thread only, no locking needed + std::map cubes_; std::deque queue_; mutable std::mutex mtx_; diff --git a/utils/frame_output.h b/utils/frame_output.h index 334bcc9..b37328b 100644 --- a/utils/frame_output.h +++ b/utils/frame_output.h @@ -22,6 +22,11 @@ namespace Camera { uint32_t bytes_per_pixel{0}; uint64_t sequence_number{0}; // monotonic per-stream counter + // Names the output stream a frame belongs to; empty is the primary image. + // Outputs keep streams apart by it, so an off-geometry frame cannot + // collide with or resize the image stream + std::string stream; + // FITS keys that vary per frame (e.g. per read within one exposure); rebuilt each frame std::shared_ptr frame_keys; diff --git a/utils/shared_memory_writer.cpp b/utils/shared_memory_writer.cpp index 6e3527a..6bb30eb 100644 --- a/utils/shared_memory_writer.cpp +++ b/utils/shared_memory_writer.cpp @@ -108,24 +108,31 @@ namespace Camera { return ERROR; } - if (meta.width != allocated_width_ || - meta.height != allocated_height_ || - meta.bytes_per_pixel != allocated_bytes_per_pixel_) { - if (this->recreate(meta.width, meta.height, meta.bytes_per_pixel) != NO_ERROR) { + const auto [entry, inserted] = segments_.try_emplace(meta.stream); + Segment &segment = entry->second; + if (inserted) { + segment.name = meta.stream.empty() ? segment_name_ + : segment_name_ + "_" + meta.stream; + } + + if (meta.width != segment.allocated_width || + meta.height != segment.allocated_height || + meta.bytes_per_pixel != segment.allocated_bytes_per_pixel) { + if (this->recreate(segment, meta.width, meta.height, meta.bytes_per_pixel) != NO_ERROR) { return ERROR; } } void* buffer = nullptr; - if (ImageStreamIO_writeBuffer(&image_, &buffer) != IMAGESTREAMIO_SUCCESS) { - logwrite(function, "ERROR ImageStreamIO_writeBuffer failed"); + if (ImageStreamIO_writeBuffer(&segment.image, &buffer) != IMAGESTREAMIO_SUCCESS) { + logwrite(function, "ERROR ImageStreamIO_writeBuffer failed for \"" + segment.name + "\""); return ERROR; } std::memcpy(buffer, data, frame_bytes); - this->write_keywords(meta); + write_keywords(segment.image, meta); - ImageStreamIO_UpdateIm(&image_); + ImageStreamIO_UpdateIm(&segment.image); frames_written_.fetch_add(1, std::memory_order_relaxed); return NO_ERROR; @@ -143,23 +150,24 @@ namespace Camera { void SharedMemoryWriter::close() { const std::string function("Camera::SharedMemoryWriter::close"); - ImageStreamIO_destroyIm(&image_); - allocated_width_ = allocated_height_ = allocated_bytes_per_pixel_ = 0; - - if (opened_) { - logwrite(function, "closed \"" + segment_name_ + "\""); + for (auto &entry : segments_) { + ImageStreamIO_destroyIm(&entry.second.image); + logwrite(function, "closed \"" + entry.second.name + "\""); } + segments_.clear(); opened_ = false; } - long SharedMemoryWriter::recreate(uint32_t width, uint32_t height, uint32_t bytes_per_pixel) { + long SharedMemoryWriter::recreate(Segment &segment, uint32_t width, uint32_t height, + uint32_t bytes_per_pixel) { const std::string function("Camera::SharedMemoryWriter::recreate"); - ImageStreamIO_destroyIm(&image_); - allocated_width_ = allocated_height_ = allocated_bytes_per_pixel_ = 0; + ImageStreamIO_destroyIm(&segment.image); + segment.allocated_width = segment.allocated_height = 0; + segment.allocated_bytes_per_pixel = 0; char path[STRINGMAXLEN_FILE_NAME]; - ImageStreamIO_filename(path, sizeof(path), segment_name_.c_str()); + ImageStreamIO_filename(path, sizeof(path), segment.name.c_str()); // Diagnostic even when the library's internal unlink-and-retry self-heals a // same-owner crash leftover, so an operator can see it happened @@ -172,12 +180,12 @@ namespace Camera { uint32_t size[2] = {width, height}; const errno_t status = ImageStreamIO_createIm( - &image_, segment_name_.c_str(), 2, size, datatype, + &segment.image, segment.name.c_str(), 2, size, datatype, 1 /* shared */, NUM_KEYWORDS, static_cast(ring_buffer_size_)); if (status != IMAGESTREAMIO_SUCCESS) { const std::string blocker = describe_path(path); - logwrite(function, "ERROR ImageStreamIO_createIm failed for \"" + segment_name_ + + logwrite(function, "ERROR ImageStreamIO_createIm failed for \"" + segment.name + "\" (" + std::to_string(width) + "x" + std::to_string(height) + ")" + (blocker.empty() ? "" : "; blocked by " + blocker)); return ERROR; @@ -187,23 +195,23 @@ namespace Camera { logwrite(function, "WARNING chmod failed for \"" + std::string(path) + "\""); } - allocated_width_ = width; - allocated_height_ = height; - allocated_bytes_per_pixel_ = bytes_per_pixel; + segment.allocated_width = width; + segment.allocated_height = height; + segment.allocated_bytes_per_pixel = bytes_per_pixel; - logwrite(function, "created \"" + segment_name_ + "\" (" + + logwrite(function, "created \"" + segment.name + "\" (" + std::to_string(width) + "x" + std::to_string(height) + ", " + std::to_string(bytes_per_pixel) + " bytes/px, " + std::to_string(ring_buffer_size_) + " frames)"); return NO_ERROR; } - void SharedMemoryWriter::write_keywords(const FrameMetadata &meta) { - set_long_keyword(image_.kw[0], "FRAMENO", + void SharedMemoryWriter::write_keywords(IMAGE &image, const FrameMetadata &meta) { + set_long_keyword(image.kw[0], "FRAMENO", static_cast(meta.frame_number), "Frame number"); - set_long_keyword(image_.kw[1], "TIMESTMP", + set_long_keyword(image.kw[1], "TIMESTMP", static_cast(meta.timestamp), "Archon timestamp (0.01 us units)"); - set_long_keyword(image_.kw[2], "SEQNUM", + set_long_keyword(image.kw[2], "SEQNUM", static_cast(meta.sequence_number), "Sequence number"); } diff --git a/utils/shared_memory_writer.h b/utils/shared_memory_writer.h index c1def17..6537869 100644 --- a/utils/shared_memory_writer.h +++ b/utils/shared_memory_writer.h @@ -12,11 +12,12 @@ #include #include #include +#include #include namespace Camera { - /// FrameOutput that publishes frames as an ImageStreamIO shared-memory image stream + /// FrameOutput that publishes frames as ImageStreamIO shared-memory image streams class SharedMemoryWriter : public FrameOutput { public: SharedMemoryWriter(const std::string &segment_name, @@ -30,6 +31,15 @@ namespace Camera { OutputStatus status() const override; private: + /// One ImageStreamIO stream, created on the first frame that names it + struct Segment { + IMAGE image{}; + std::string name; + uint32_t allocated_width{0}; + uint32_t allocated_height{0}; + uint32_t allocated_bytes_per_pixel{0}; + }; + std::atomic frames_written_{0}; std::string segment_name_; @@ -37,16 +47,16 @@ namespace Camera { std::string shm_dir_; bool opened_{false}; - IMAGE image_{}; - uint32_t allocated_width_{0}; - uint32_t allocated_height_{0}; - uint32_t allocated_bytes_per_pixel_{0}; + // Keyed by FrameMetadata::stream, so a RAW capture gets its own segment + // instead of tearing down and resizing the image stream + std::map segments_; // Destroys any existing stream and creates one for the given frame shape - long recreate(uint32_t width, uint32_t height, uint32_t bytes_per_pixel); + long recreate(Segment &segment, uint32_t width, uint32_t height, + uint32_t bytes_per_pixel); - // Writes FRAMENO/TIMESTMP/SEQNUM into image_.kw[] - void write_keywords(const FrameMetadata &meta); + // Writes FRAMENO/TIMESTMP/SEQNUM into image.kw[] + static void write_keywords(IMAGE &image, const FrameMetadata &meta); }; } From 4dab8d7ee627f0fab6e837b591e5b0012d95af74 Mon Sep 17 00:00:00 2001 From: Mike Langmayr <1809691+mikelangmayr@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:50:30 -0700 Subject: [PATCH 2/4] Decide the RAW geometry fallback in one place --- camerad/archon_controller.cpp | 21 ++++++++++----------- camerad/archon_controller.h | 3 ++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/camerad/archon_controller.cpp b/camerad/archon_controller.cpp index 9e1ba52..65fba77 100644 --- a/camerad/archon_controller.cpp +++ b/camerad/archon_controller.cpp @@ -2345,8 +2345,9 @@ namespace Camera { geom.samples = static_cast(this->rawinfo.samples); geom.blocks_per_line = static_cast(this->frameinfo.bufrawblocks[index]); geom.lines = static_cast(this->frameinfo.bufrawlines[index]); + geom.from_config = (geom.blocks_per_line == 0 || geom.lines == 0); - if (geom.blocks_per_line == 0 || geom.lines == 0) { + if (geom.from_config) { geom.blocks_per_line = (static_cast(geom.samples) * sizeof(uint16_t) + BLOCK_LEN - 1) / BLOCK_LEN; const int span = this->rawinfo.endline - this->rawinfo.startline + 1; @@ -2497,16 +2498,6 @@ namespace Camera { return error; } - const auto index = this->frameinfo.index.load(); - - // Raw is captured with the frame, so a buffer filled before RAWENABLE was set - // reports zero and raw_geometry() falls back to inferring from the config keys - if (this->frameinfo.bufrawblocks[index] == 0 || - this->frameinfo.bufrawlines[index] == 0) { - logwrite(function, "WARNING controller reports no raw data in this buffer; " - "using configured geometry"); - } - const raw_geometry_t geom = this->raw_geometry(); if (geom.samples == 0 || geom.lines == 0) { logwrite(function, "ERROR RAW geometry is empty; check RAW config"); @@ -2514,6 +2505,13 @@ namespace Camera { return ERROR; } + // Raw is captured with the frame, so a buffer filled before RAWENABLE was set + // holds none and reports zero, leaving only the config keys to size the fetch + if (geom.from_config) { + logwrite(function, "WARNING controller reports no raw data in this buffer; " + "using configured geometry"); + } + const size_t fetch_bytes = static_cast(geom.blocks_per_line) * geom.lines * BLOCK_LEN; std::shared_ptr raw_buffer(new char[fetch_bytes]); char* bufptr = raw_buffer.get(); @@ -2535,6 +2533,7 @@ namespace Camera { static_cast(geom.samples) * sizeof(uint16_t)); } + const auto index = this->frameinfo.index.load(); const size_t payload_bytes = payload_samples * sizeof(uint16_t); Camera::FrameMetadata meta; diff --git a/camerad/archon_controller.h b/camerad/archon_controller.h index 9d3e3e4..f8962a9 100644 --- a/camerad/archon_controller.h +++ b/camerad/archon_controller.h @@ -47,7 +47,7 @@ constexpr int MODTYPE_ADLN = 15; constexpr int MODTYPE_UNKNOWN = 16; constexpr int MODTYPE_ADM = 17; -// First backplane slot an AD or ADM module may occupy, which RAWSEL indexes from +// AD and ADM modules are restricted to slots 5-8, the range RAWSEL addresses constexpr int AD_SLOT_FIRST = 5; // Stream label keeping a pre-CDS RAW capture in outputs of its own @@ -384,6 +384,7 @@ namespace Camera { uint32_t samples; // valid 16-bit samples per line (RAWSAMPLES) uint32_t blocks_per_line; // 1024-byte blocks per line, padded per Archon uint32_t lines; // number of raw lines (RAWENDLINE-RAWSTARTLINE+1) + bool from_config{false}; // controller reported nothing, so this is inferred }; static bool is_raw_config_key(const std::string &key); From b6198778202336119ff8b20030d85ff38247e219 Mon Sep 17 00:00:00 2001 From: Mike Langmayr <1809691+mikelangmayr@users.noreply.github.com> Date: Thu, 24 Sep 2026 11:37:58 -0700 Subject: [PATCH 3/4] Report every candidate RAW slot instead of deriving one from RAWSEL --- camerad/archon_controller.cpp | 21 +++++++++++---------- camerad/archon_controller.h | 1 + 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/camerad/archon_controller.cpp b/camerad/archon_controller.cpp index 65fba77..27e836f 100644 --- a/camerad/archon_controller.cpp +++ b/camerad/archon_controller.cpp @@ -2448,21 +2448,22 @@ namespace Camera { /** * @brief build the provenance keys describing a RAW capture * @details An AD channel and an ADM channel arrive as an identical uint16 - * block sampled at different rates, so a reader cannot interpret - * the samples without RAWSEL and the selected module's type. + * block sampled at different rates, so the samples cannot be + * interpreted without RAWSEL and the installed module types. * @return shared_ptr suitable for FrameMetadata::frame_keys */ std::shared_ptr ArchonController::raw_frame_keys() const { - // RAWSEL indexes four channels per slot, starting at the first AD slot - const int channel = this->rawinfo.adchan; - const int slot = AD_SLOT_FIRST + channel / MAXNADCHAN; - auto keys = std::make_shared(); - keys->addkey("RAWSEL", channel, "Archon RAWSEL, channel captured"); - keys->addkey("RAWSLOT", slot, "backplane slot RAWSEL selects"); - if (slot >= 1 && slot <= static_cast(this->modtype.size())) { - keys->addkey("RAWMODTY", this->modtype[slot-1], "MODn_TYPE of the selected slot"); + keys->addkey("RAWSEL", this->rawinfo.adchan, "Archon RAWSEL, channel captured"); + + // RAWSEL's channels per slot is 4 in the manual but 18 in the Archon GUI, + // so report every candidate slot rather than derive one from an unknown stride + for (int slot = AD_SLOT_FIRST; + slot <= AD_SLOT_LAST && slot <= static_cast(this->modtype.size()); ++slot) { + keys->addkey("RAWMOD"+std::to_string(slot), this->modtype[slot-1], + "MODn_TYPE of slot "+std::to_string(slot)); } + keys->addkey("RAWSAMP", static_cast(this->rawinfo.samples), "RAWSAMPLES"); keys->addkey("RAWSLINE", static_cast(this->rawinfo.startline), "RAWSTARTLINE"); keys->addkey("RAWELINE", static_cast(this->rawinfo.endline), "RAWENDLINE"); diff --git a/camerad/archon_controller.h b/camerad/archon_controller.h index f8962a9..b777ef7 100644 --- a/camerad/archon_controller.h +++ b/camerad/archon_controller.h @@ -49,6 +49,7 @@ constexpr int MODTYPE_ADM = 17; // AD and ADM modules are restricted to slots 5-8, the range RAWSEL addresses constexpr int AD_SLOT_FIRST = 5; +constexpr int AD_SLOT_LAST = 8; // Stream label keeping a pre-CDS RAW capture in outputs of its own const std::string RAW_STREAM = "raw"; From 77990eb8c8cb475569eb69e6e52f446ca7394674 Mon Sep 17 00:00:00 2001 From: Mike Langmayr <1809691+mikelangmayr@users.noreply.github.com> Date: Thu, 24 Sep 2026 11:46:42 -0700 Subject: [PATCH 4/4] Document raw sample retrieval and the RAWSEL mapping ambiguity --- docs/commands/controller.md | 94 +++++++++++++++++++++++++++++++++++-- docs/fits/index.md | 6 ++- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/docs/commands/controller.md b/docs/commands/controller.md index d37c36d..6425d9a 100644 --- a/docs/commands/controller.md +++ b/docs/commands/controller.md @@ -27,10 +27,10 @@ confirming that a reply came back. The commands most worth knowing: The server itself parses `FRAME`, `STATUS` and `SYSTEM` replies for its own bookkeeping, so use `native` for inspection rather than as a control path. -Beyond `native`, the Archon-specific commands are `raw` for configuration memory, `getp` and `setp` -for parameters, `inreg` for a VCPU input register, `loadtiming` and `readacf` for loading, `mode` -for camera modes, `autofetch_mode`, and `heater` and `sensor` for the thermal modules. The -[heater and sensor](#heater-and-sensor) syntax is below. +Beyond `native`, the Archon-specific commands are `raw` for pre-CDS sample capture, `getp` and +`setp` for parameters, `inreg` for a VCPU input register, `loadtiming` and `readacf` for loading, +`mode` for camera modes, `autofetch_mode`, and `heater` and `sensor` for the thermal modules. The +[raw samples](#raw-samples) and [heater and sensor](#heater-and-sensor) syntax is below. ## ARC (AstroCam) @@ -45,6 +45,92 @@ and returns success, so the command replies `DONE` while doing nothing. Passing three-letter DSP commands through to an ARC controller is therefore not currently possible from the command interface. +## Raw samples + +The Archon can capture a window of unprocessed ADC samples from a single channel alongside the +normal CDS frame, which is how a detector engineer inspects the output waveform for settling times, +sampling windows, reset level stability and clock feed through. `raw` configures and retrieves it. + +``` +raw [ config | set [...] | read ] +``` + +| Form | Effect | +|---|---| +| `raw config` | Report the six RAW keywords | +| `raw set ...` | Write the keyword(s) to configuration memory, then apply | +| `raw read` | Fetch the raw region of the newest buffer and dispatch it as a frame | + +The keywords are `RAWENABLE`, `RAWSEL`, `RAWSTARTLINE`, `RAWENDLINE`, `RAWSTARTPIXEL` and +`RAWSAMPLES`. `RAWSAMPLES` is rounded up to a whole 1024-byte block per line. + +:::{warning} +`RAWENABLE` takes effect at capture time, not at read time. The controller acquires raw samples +while it fills the frame buffer, so enabling it after an exposure cannot populate that buffer. Set +it before the exposure the samples should come from. + +`raw set` can only write keywords that already exist in the loaded ACF, so an ACF that omits them +cannot be driven from the command interface at all. +::: + +`raw read` refuses while `RAWENABLE` is 0. A controller with capture disabled reports zero raw +blocks and lines, which is indistinguishable from one that reports nothing, so the fetch would +otherwise return whatever happens to sit at the raw offset and label it as raw data. + +The result is dispatched on its own stream, named `raw`, so it never collides with the image. The +FITS writer gives it a separate file and the shared-memory writer a separate segment. See +[frame output keys](../configuration/frame-outputs.md). + +### Interpreting the samples + +An AD channel and an ADM channel both arrive as an identical block of `uint16`, but they are not +sampled the same way: + +AD +: 16 bits at 100 MHz, so consecutive samples are 10 ns apart. + +ADM +: 18 bits at 12.5 MHz, truncated to 16 bits, with each value repeated eight times and dithered so + that averaging the eight recovers the original 18-bit sample. The effective period is 80 ns. + +Nothing in the data itself distinguishes the two, so `raw read` records the provenance in the +header instead. + +| Keyword | Meaning | +|---|---| +| `RAWSEL` | The channel selector in force for the capture | +| `RAWMOD5` to `RAWMOD8` | `MODn_TYPE` of each slot `RAWSEL` can address | +| `RAWSAMP` | `RAWSAMPLES` | +| `RAWSLINE`, `RAWELINE` | `RAWSTARTLINE`, `RAWENDLINE` | +| `RAWSPIX` | `RAWSTARTPIXEL` | + +Every candidate slot is reported rather than the one slot `RAWSEL` selects, because the mapping +from `RAWSEL` to a slot is not reliably known. See below. + +### Module types and the RAWSEL range + +The Archon manual is incomplete here and partly stale, and the GUI source distributed with the +controller is the better reference. `archongui/src/archon.h` carries the full module type list, +where the manual stops at `16+: Unknown`: + +| Type | Module | +|---|---| +| 6 | Atlas | +| 16 | DriverX | +| 17 | ADM | +| 18 | Unknown, the sentinel | + +`archongui/src/archongui.cpp` then builds its Raw Channel Select control with +`for (i = 1; i <= 72; i++)` and stores the zero-based index as `RAWSEL`, so the selector spans 0 to +71. That is 18 channels across each of slots 5 to 8, matching the ADM channel layout. The manual +instead documents `RAWSEL` as 0 to 15, four per slot, described purely in terms of AD modules. + +:::{note} +The two strides disagree, and on a chassis with a mix of AD and ADM modules they resolve the same +`RAWSEL` to different slots. `camerad` therefore derives nothing from `RAWSEL` and records it +verbatim beside the type of every slot it could refer to, leaving the reader to resolve it. +::: + ## Heater and sensor For Archon **Heater** and **HeaterX** modules. Both require firmware to be loaded and a diff --git a/docs/fits/index.md b/docs/fits/index.md index 29c65a6..4702b71 100644 --- a/docs/fits/index.md +++ b/docs/fits/index.md @@ -9,12 +9,16 @@ in [frame output keys](../configuration/frame-outputs.md). The writer builds each name itself: ``` -/_.fits +/_[_].fits ``` The frame number is zero-padded to eight digits. If that path already exists the writer appends `_1`, `_2` and so on until it finds a free name, so a file is never silently overwritten. +The stream is omitted for the primary image and present for anything published alongside it, such +as the `raw` stream from [raw samples](../commands/controller.md#raw-samples). It comes last so a +frame and its companions sort together. + `FITS_AUTODIR` puts all of this inside a `YYYYMMDD` subdirectory of `FITS_OUTPUT_DIR`. :::{warning}