From a928c11d006e6fbd99d75fff9c461dcb7b79df0a Mon Sep 17 00:00:00 2001 From: Rye Date: Thu, 22 Oct 2026 11:27:16 -0400 Subject: [PATCH 1/3] A J2C image takes its byte targets from its own codestream's layout calcDataSizeJ2C estimates a discard level's bytes from the texture's dimensions alone: 128 KB for a 2048 at discard 1, about 9% of a median 1.39 MB file. Second Life's LRCP codestreams hold only their first quality layers in that much, and its single-layer ones are cut inside a low resolution, so a decode at the level comes out soft or very blurry: 2048 multi-layer at discard 1 measured 43.9 dB PSNR at the median and 29.9 dB at p10 against the decode of the whole file, single-layer 23.9 dB. ALJ2CLayout reads SIZ, COD and the first SOT without OpenJPEG. A codestream of one tile in one tile-part, the shape all 52,000 complete entries of a texture cache have, ends at that SOT's offset plus Psot plus the EOC, and all of it sits in the first 600 bytes a fetch reads. LLImageJ2C parses it whenever setData or appendData changes its data (LLImageFormatted::onDataChanged) and in updateData, and calcDataSize and calcDiscardLevelBytes follow it: the target for discard d is the codestream's length times the fraction that decodes within 45 dB of the whole file's decode at that level, by the texture's longer side. The quality layers' byte tiers are fixed, so a small file needs a larger share: 90% at discard 1 up to 128 pixels, 65% from 1024. The sizes-only estimate stays for a codestream whose headers are not in hand. The fractions were calibrated on half of that cache and checked on the other half: at its targets every class of longer side and layer count decodes within 45 dB for 93.5% of its files or more. Against the starting table of 75/50/25/10/10% for every size, 2048s take 65/30/12/5/3%. TextureReverseByteRange decoded at half the estimate, which put back the softness the targets remove, and TextureNewByteRange chose between the estimate's two formulas. Both settings go, with LLImage's copies: the fractions carry the quality bar, and the estimate is always the smaller formula. Tests: the parser over the viewer's encoder and OpenJPEG's (layered, single-layer, four tiles, every cut of the header, a missing SOT), and decodes at every target of layered and single-layer codestreams, which pass 45 dB where the sizes-only estimate gives 30-36 dB at discard 1. With calcDataSize back on the estimate they fail at every level. Co-Authored-By: Claude Opus 5.5 --- .../llappappearanceutility.cpp | 5 +- indra/llimage/CMakeLists.txt | 12 +- indra/llimage/alj2clayout.cpp | 204 ++++++++ indra/llimage/alj2clayout.h | 70 +++ indra/llimage/llimage.cpp | 8 +- indra/llimage/llimage.h | 10 +- indra/llimage/llimagej2c.cpp | 52 +- indra/llimage/llimagej2c.h | 10 + indra/llimage/tests/alj2clayout_test.cpp | 462 ++++++++++++++++++ indra/llkdu/tests/llimagej2ckdu_test.cpp | 2 + indra/newview/app_settings/settings.xml | 22 - indra/newview/llappviewer.cpp | 2 +- .../default/xui/pt/settings_comments.xml | 4 - 13 files changed, 803 insertions(+), 60 deletions(-) create mode 100644 indra/llimage/alj2clayout.cpp create mode 100644 indra/llimage/alj2clayout.h create mode 100644 indra/llimage/tests/alj2clayout_test.cpp diff --git a/indra/llappearanceutility/llappappearanceutility.cpp b/indra/llappearanceutility/llappappearanceutility.cpp index 54740d4f3dc..fb6bb0fa061 100644 --- a/indra/llappearanceutility/llappappearanceutility.cpp +++ b/indra/llappearanceutility/llappappearanceutility.cpp @@ -467,10 +467,7 @@ bool LLAppAppearanceUtility::init() LL_DEBUGS() << "BakeSize: " << mBakeTextureSize << LL_ENDL; // Initialize classes. - // Values taken from settings.xml. - const bool USE_TEXTURE_NEW_BYTE_RANGE=true; - const S32 TEXTURE_REVERSE_BYTE_RANGE=50; - LLImage::initClass(USE_TEXTURE_NEW_BYTE_RANGE, TEXTURE_REVERSE_BYTE_RANGE); + LLImage::initClass(); const bool SKIP_ANALYZE_ALPHA=true; LLTranslationBridge::ptr_t trans = std::make_shared(); diff --git a/indra/llimage/CMakeLists.txt b/indra/llimage/CMakeLists.txt index 37e1167b9e4..bfaaa40ec4a 100644 --- a/indra/llimage/CMakeLists.txt +++ b/indra/llimage/CMakeLists.txt @@ -5,6 +5,7 @@ add_library(llimage STATIC) target_sources( llimage PRIVATE + alj2clayout.cpp llimagebmp.cpp llimage.cpp llimagedimensionsinfo.cpp @@ -25,6 +26,7 @@ target_sources( PUBLIC FILE_SET HEADERS FILES + alj2clayout.h llimage.h llimagebmp.h llimagedimensionsinfo.h @@ -67,6 +69,12 @@ if(AL_BUILD_TESTS) LIBRARIES llmath llcommon ) - # The locks images are read and written under, over the library itself. - al_add_test_binary(llimage_tests PROJECT llimage SOURCES tests/llimage_test.cpp LIBRARIES llimage) + # The library itself: the locks images are read and written under, and the + # J2C codestream layout and the byte targets it gives. + al_add_test_binary( + llimage_tests + PROJECT llimage + SOURCES tests/llimage_test.cpp tests/alj2clayout_test.cpp + LIBRARIES llimage ll::openjpeg + ) endif() diff --git a/indra/llimage/alj2clayout.cpp b/indra/llimage/alj2clayout.cpp new file mode 100644 index 00000000000..8100e9367d1 --- /dev/null +++ b/indra/llimage/alj2clayout.cpp @@ -0,0 +1,204 @@ +/** + * @file alj2clayout.cpp + * @brief ALJ2CLayout: a JPEG 2000 codestream's layout, read from its headers + * + * $LicenseInfo:firstyear=2026&license=viewerlgpl$ + * Alchemy Viewer Source Code + * Copyright (C) 2026, Rye + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "alj2clayout.h" + +#include "llimage.h" + +namespace +{ + constexpr U8 MARKER = 0xFF; + constexpr U8 SOC = 0x4F; + constexpr U8 SIZ = 0x51; + constexpr U8 COD = 0x52; + constexpr U8 SOT = 0x90; + + // The marker, Lsot, Isot, Psot, TPsot and TNsot. + constexpr S32 SOT_FIELDS_SIZE = 12; + // SOT's fields and the SOD that ends the shortest tile-part header. + constexpr S32 MIN_TILE_PART_SIZE = SOT_FIELDS_SIZE + 2; + constexpr S32 EOC_SIZE = 2; + + constexpr S32 SIZ_MIN_BODY = 38; + constexpr S32 COD_MIN_BODY = 10; + + // The fraction of a codestream that decodes at each discard level within 45 dB PSNR of + // the whole codestream's decode at that level, by the texture's longer side, for nine in + // ten Second Life textures of each size whether single or multi-layer. The quality + // layers' byte tiers are fixed, so they take a larger share of a smaller file. + struct StreamingFractions + { + S32 mMaxSide; + F64 mFraction[MAX_DISCARD_LEVEL + 1]; + }; + constexpr StreamingFractions STREAMING_FRACTIONS[] = { + { 128, { 1.0, 0.90, 0.75, 0.60, 0.45, 0.45 } }, + { 256, { 1.0, 0.75, 0.50, 0.30, 0.14, 0.10 } }, + { 512, { 1.0, 0.70, 0.45, 0.20, 0.10, 0.06 } }, + { 1024, { 1.0, 0.65, 0.40, 0.12, 0.08, 0.04 } }, + { S32_MAX, { 1.0, 0.65, 0.30, 0.12, 0.05, 0.03 } }, + }; + + F64 streamingFraction(S32 side, S32 discard) + { + for (const StreamingFractions& row : STREAMING_FRACTIONS) + { + if (side <= row.mMaxSide) + { + return row.mFraction[discard]; + } + } + return 1.0; + } + + U32 read16(const U8* p) + { + return (U32(p[0]) << 8) | U32(p[1]); + } + + U32 read32(const U8* p) + { + return (U32(p[0]) << 24) | (U32(p[1]) << 16) | (U32(p[2]) << 8) | U32(p[3]); + } + + U64 ceilDiv(U64 numerator, U64 denominator) + { + return (numerator + denominator - 1) / denominator; + } +} + +// static +ALJ2CLayout ALJ2CLayout::parse(const U8* data, S32 size) +{ + if (!data || size < 4 || data[0] != MARKER || data[1] != SOC) + { + return {}; + } + + ALJ2CLayout layout; + bool have_siz = false; + bool have_cod = false; + S32 pos = 2; + while (pos + 4 <= size && data[pos + 1] != SOT) + { + if (data[pos] != MARKER) + { + return {}; + } + + // The segment length counts itself but not the marker. + const S32 length = S32(read16(data + pos + 2)); + if (length < 2 || pos + 2 + length > size) + { + return {}; + } + + const U8* body = data + pos + 4; + const S32 body_size = length - 2; + const U8 marker = data[pos + 1]; + if (marker == SIZ) + { + if (body_size < SIZ_MIN_BODY) + { + return {}; + } + const U64 x = read32(body + 2); + const U64 y = read32(body + 6); + const U64 x_origin = read32(body + 10); + const U64 y_origin = read32(body + 14); + const U64 tile_width = read32(body + 18); + const U64 tile_height = read32(body + 22); + const U64 tile_x_origin = read32(body + 26); + const U64 tile_y_origin = read32(body + 30); + if (x <= x_origin || y <= y_origin || x - x_origin > U16_MAX || y - y_origin > U16_MAX + || !tile_width || !tile_height || x <= tile_x_origin || y <= tile_y_origin) + { + return {}; + } + const U64 tiles = ceilDiv(x - tile_x_origin, tile_width) * ceilDiv(y - tile_y_origin, tile_height); + layout.mWidth = S32(x - x_origin); + layout.mHeight = S32(y - y_origin); + layout.mComponents = S32(read16(body + 34)); + layout.mTiles = S32(llmin(tiles, U64(S32_MAX))); + have_siz = true; + } + else if (marker == COD) + { + if (body_size < COD_MIN_BODY) + { + return {}; + } + layout.mProgression = body[1]; + layout.mLayers = S32(read16(body + 2)); + layout.mLevels = body[5]; + have_cod = true; + } + pos += 2 + length; + } + + if (!have_siz || !have_cod || pos + SOT_FIELDS_SIZE > size || data[pos] != MARKER || data[pos + 1] != SOT) + { + return {}; + } + + layout.mHeaderLength = pos; + + const U32 tile_part_length = read32(data + pos + 6); + const U8 tile_parts = data[pos + 11]; + if (layout.mTiles == 1 && tile_parts == 1 && tile_part_length >= MIN_TILE_PART_SIZE + && tile_part_length <= U32(MAX_IMAGE_DATA_SIZE)) + { + layout.mLength = pos + S32(tile_part_length) + EOC_SIZE; + } + return layout; +} + +S32 ALJ2CLayout::calcDataSize(S32 discard) const +{ + if (!isValid()) + { + return 0; + } + discard = llclamp(discard, 0, MAX_DISCARD_LEVEL); + const S32 bytes = S32(F64(mLength) * streamingFraction(llmax(mWidth, mHeight), discard)); + return llclamp(bytes, llmin(FIRST_PACKET_SIZE, mLength), mLength); +} + +S32 ALJ2CLayout::calcDiscardLevelBytes(S32 bytes) const +{ + if (!isValid()) + { + return MAX_DISCARD_LEVEL; + } + for (S32 discard = 0; discard < MAX_DISCARD_LEVEL; ++discard) + { + if (bytes >= calcDataSize(discard)) + { + return discard; + } + } + return MAX_DISCARD_LEVEL; +} diff --git a/indra/llimage/alj2clayout.h b/indra/llimage/alj2clayout.h new file mode 100644 index 00000000000..2c7c33bed4e --- /dev/null +++ b/indra/llimage/alj2clayout.h @@ -0,0 +1,70 @@ +/** + * @file alj2clayout.h + * @brief ALJ2CLayout: a JPEG 2000 codestream's layout, read from its headers + * + * $LicenseInfo:firstyear=2026&license=viewerlgpl$ + * Alchemy Viewer Source Code + * Copyright (C) 2026, Rye + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * $/LicenseInfo$ + */ + +#pragma once + +#include "stdtypes.h" + +// What a JPEG 2000 codestream's main header (SIZ, COD) and first tile-part header (SOT) say +// about it, and the bytes from its start that a decode at each discard level needs. +// +// The total length is known only for a codestream of one tile in one tile-part, the shape +// every Second Life texture has: the first SOT's Psot then runs to the EOC. +struct ALJ2CLayout +{ + enum EProgression : S32 + { + LRCP = 0, + RLCP = 1, + RPCL = 2, + PCRL = 3, + CPRL = 4, + }; + + S32 mWidth = 0; + S32 mHeight = 0; + S32 mComponents = 0; + S32 mTiles = 0; + S32 mProgression = -1; + S32 mLayers = 0; + // Decomposition levels; the codestream holds mLevels + 1 resolutions. + S32 mLevels = 0; + // Offset of the first SOT marker, which ends the main header. + S32 mHeaderLength = 0; + // The whole codestream, EOC included. 0 when the headers do not fix it. + S32 mLength = 0; + + // Reads a codestream's headers from its first bytes. The layout is invalid when they + // stop short of the first SOT's fields or are not a codestream. + static ALJ2CLayout parse(const U8* data, S32 size); + + bool isValid() const { return mLength > 0; } + + // Bytes from the start of the codestream that decode at `discard` within 45 dB PSNR of + // a decode at that level from the whole of it. + S32 calcDataSize(S32 discard) const; + + // The finest discard level the first `bytes` of the codestream decode. + S32 calcDiscardLevelBytes(S32 bytes) const; +}; diff --git a/indra/llimage/llimage.cpp b/indra/llimage/llimage.cpp index f07fc178a18..b0397b95675 100644 --- a/indra/llimage/llimage.cpp +++ b/indra/llimage/llimage.cpp @@ -588,14 +588,10 @@ static void bilinear_scale(const U8 *src, U32 srcW, U32 srcH, U32 srcCh, U32 src //static thread_local std::string LLImage::sLastThreadErrorMessage; -bool LLImage::sUseNewByteRange = false; -S32 LLImage::sMinimalReverseByteRangePercent = 75; //static -void LLImage::initClass(bool use_new_byte_range, S32 minimal_reverse_byte_range_percent) +void LLImage::initClass() { - sUseNewByteRange = use_new_byte_range; - sMinimalReverseByteRangePercent = minimal_reverse_byte_range_percent; } //static @@ -2503,6 +2499,7 @@ void LLImageFormatted::setData(U8 *data, S32 size) setDataAndSize(data, size); // Access private LLImageBase members sGlobalFormattedMemory += getDataSize(); + onDataChanged(); } } @@ -2523,6 +2520,7 @@ void LLImageFormatted::appendData(U8 *data, S32 size) reallocateData(newsize); memcpy(getData() + cursize, data, size); ll_aligned_free_16(data); + onDataChanged(); } } } diff --git a/indra/llimage/llimage.h b/indra/llimage/llimage.h index 5d1b9c83bc9..ec2daf38fb7 100644 --- a/indra/llimage/llimage.h +++ b/indra/llimage/llimage.h @@ -100,19 +100,14 @@ typedef enum e_image_codec class LLImage { public: - static void initClass(bool use_new_byte_range = false, S32 minimal_reverse_byte_range_percent = 75); + static void initClass(); static void cleanupClass(); static const std::string& getLastThreadError(); static void setLastError(const std::string& message); - static bool useNewByteRange() { return sUseNewByteRange; } - static S32 getReverseByteRangePercent() { return sMinimalReverseByteRangePercent; } - protected: static thread_local std::string sLastThreadErrorMessage; - static bool sUseNewByteRange; - static S32 sMinimalReverseByteRangePercent; }; //============================================================================ @@ -475,6 +470,9 @@ class LLImageFormatted : public LLImageBase protected: bool copyData(U8 *data, S32 size); // calls updateData() + // Called with the data lock held once setData or appendData has changed the data. + virtual void onDataChanged() {} + protected: S8 mCodec; S8 mDecoding; diff --git a/indra/llimage/llimagej2c.cpp b/indra/llimage/llimagej2c.cpp index e5b127ee8d2..567e80a9f55 100644 --- a/indra/llimage/llimagej2c.cpp +++ b/indra/llimage/llimagej2c.cpp @@ -85,6 +85,24 @@ S8 LLImageJ2C::getRawDiscardLevel() return mRawDiscardLevel; } +// virtual +void LLImageJ2C::deleteData() +{ + LLImageDataLock lock(this); + + mLayout = {}; + LLImageFormatted::deleteData(); +} + +// virtual +void LLImageJ2C::onDataChanged() +{ + if (!mLayout.isValid()) + { + mLayout = ALJ2CLayout::parse(getData(), getDataSize()); + } +} + bool LLImageJ2C::updateData() { bool res = true; @@ -100,6 +118,7 @@ bool LLImageJ2C::updateData() } else { + mLayout = ALJ2CLayout::parse(getData(), getDataSize()); res = mImpl->getMetadata(*this); } @@ -223,7 +242,8 @@ S32 LLImageJ2C::calcHeaderSizeJ2C() S32 LLImageJ2C::calcDataSizeJ2C(S32 w, S32 h, S32 comp, S32 discard_level, F32 rate) { // Estimates the byte budget needed to decode at `discard_level`. Smaller - // discards (higher quality) take more bytes. + // discards (higher quality) take more bytes. It sizes a codestream whose + // headers are not in hand; ALJ2CLayout sizes one whose headers are. // // This is the historical sqrt-based "Byte Range Study" formula from the // THX1138 KDU work (see @@ -234,8 +254,7 @@ S32 LLImageJ2C::calcDataSizeJ2C(S32 w, S32 h, S32 comp, S32 discard_level, F32 r // indra/vcpkg/ports/openjpeg/, OJ tolerates truncation natively and the // inflated curve only delays progressive previews unnecessarily. // - // Two estimators are computed and the smaller picked when - // `LLImage::useNewByteRange()` is on (default true): + // Two estimators are computed and the smaller picked: // * `new_bytes` = sqrt(w*h) * comp * rate * 1000 / layer_factor // — KDU-derived approximation tracking the rate-distortion curve. // * `old_bytes` = w * h * comp * rate @@ -271,16 +290,9 @@ S32 LLImageJ2C::calcDataSizeJ2C(S32 w, S32 h, S32 comp, S32 discard_level, F32 r S32 new_bytes = (S32)(sqrt((F32)(w * h)) * (F32)components * rate * 1000.f / layer_factor); S32 old_bytes = (S32)((F32)(w * h * components) * rate); - S32 bytes = (LLImage::useNewByteRange() && (new_bytes < old_bytes)) ? new_bytes : old_bytes; - // Floor at MIN_LAYER_SIZE rather than calcHeaderSizeJ2C() (== FIRST_PACKET_SIZE, - // 600). At a 50% reverse-byte-range threshold, the old floor triggered - // decode at ~300 bytes — past the main header but before any packet - // body data was present. With OpenJPEG patched to tolerate truncated - // tile-parts, the empty decode no longer errors out but still produces - // mid-grey via the DC level shift on zero coefficients (until the - // openjpeg tcd.c overlay patch lands too). MIN_LAYER_SIZE (2000) gives - // a 1000-byte threshold, which is enough for the main header plus at - // least one packet body of low-resolution coefficient data. + S32 bytes = llmin(new_bytes, old_bytes); + // MIN_LAYER_SIZE holds the main header and at least one packet body of + // low-resolution coefficient data. bytes = llmax(bytes, MIN_LAYER_SIZE); return bytes; } @@ -293,6 +305,11 @@ S32 LLImageJ2C::calcHeaderSize() // calcDataSize() returns how many bytes to read to load discard_level (including header) S32 LLImageJ2C::calcDataSize(S32 discard_level) { + if (mLayout.isValid()) + { + return mLayout.calcDataSize(discard_level); + } + discard_level = llclamp(discard_level, 0, MAX_DISCARD_LEVEL); if ( mAreaUsedForDataSizeCalcs != (getHeight() * getWidth()) || (mDataSizes[0] == 0)) @@ -312,6 +329,11 @@ S32 LLImageJ2C::calcDataSize(S32 discard_level) S32 LLImageJ2C::calcDiscardLevelBytes(S32 bytes) { llassert(bytes >= 0); + if (mLayout.isValid()) + { + return mLayout.calcDiscardLevelBytes(bytes); + } + S32 discard_level = 0; if (bytes == 0) { @@ -319,9 +341,7 @@ S32 LLImageJ2C::calcDiscardLevelBytes(S32 bytes) } while (1) { - S32 bytes_needed = calcDataSize(discard_level); - // Use TextureReverseByteRange percent (see settings.xml) of the optimal size to qualify as correct rendering for the given discard level - if (bytes >= (bytes_needed*LLImage::getReverseByteRangePercent()/100)) + if (bytes >= calcDataSize(discard_level)) { break; } diff --git a/indra/llimage/llimagej2c.h b/indra/llimage/llimagej2c.h index 8158347fb27..ef831faef11 100644 --- a/indra/llimage/llimagej2c.h +++ b/indra/llimage/llimagej2c.h @@ -27,6 +27,7 @@ #ifndef LL_LLIMAGEJ2C_H #define LL_LLIMAGEJ2C_H +#include "alj2clayout.h" #include "llimage.h" #include "llassettype.h" @@ -45,6 +46,7 @@ class LLImageJ2C : public LLImageFormatted // Base class overrides /*virtual*/ std::string getExtension() { return std::string("j2c"); } + /*virtual*/ void deleteData(); /*virtual*/ bool updateData(); /*virtual*/ bool decode(LLImageRaw *raw_imagep, F32 decode_time); /*virtual*/ bool decodeChannels(LLImageRaw *raw_imagep, F32 decode_time, S32 first_channel, S32 max_channel_count); @@ -74,6 +76,10 @@ class LLImageJ2C : public LLImageFormatted static S32 calcHeaderSizeJ2C(); static S32 calcDataSizeJ2C(S32 w, S32 h, S32 comp, S32 discard_level, F32 rate = DEFAULT_COMPRESSION_RATE); + // The codestream's layout, read from the data once it holds the main header and the + // first SOT. calcDataSize() and calcDiscardLevelBytes() follow it whenever it is valid. + const ALJ2CLayout& getLayout() const { return mLayout; } + static std::string getEngineInfo(); protected: @@ -83,6 +89,10 @@ class LLImageJ2C : public LLImageFormatted void decodeFailed(); void updateRawDiscardLevel(); + /*virtual*/ void onDataChanged(); + + ALJ2CLayout mLayout; + S32 mMaxBytes; // Maximum number of bytes of data to use... S32 mDataSizes[MAX_DISCARD_LEVEL+1]; // Size of data required to reach a given level diff --git a/indra/llimage/tests/alj2clayout_test.cpp b/indra/llimage/tests/alj2clayout_test.cpp new file mode 100644 index 00000000000..66c87e924b2 --- /dev/null +++ b/indra/llimage/tests/alj2clayout_test.cpp @@ -0,0 +1,462 @@ +/** + * @file alj2clayout_test.cpp + * @brief Tests for the J2C codestream layout and the byte targets it gives + * + * $LicenseInfo:firstyear=2026&license=viewerlgpl$ + * Alchemy Viewer Source Code + * Copyright (C) 2026, Rye + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "alj2clayout.h" +#include "llimagej2c.h" + +#include "openjpeg.h" + +#include + +#include +#include +#include +#include + +namespace +{ + constexpr F64 QUALITY_BAR_DB = 45.0; + constexpr F64 IDENTICAL_DB = 99.0; + + U32 hash(U32 x, U32 y, U32 seed) + { + U32 h = x * 0x8da6b343u ^ y * 0xd8163841u ^ seed * 0xcb1ab31fu; + h ^= h >> 13; + h *= 0x5bd1e995u; + h ^= h >> 15; + return h; + } + + // Smoothly interpolated lattice noise in [0, 1], one lattice cell every `period` pixels. + F64 valueNoise(S32 x, S32 y, S32 period, U32 seed) + { + const S32 cx = x / period; + const S32 cy = y / period; + const F64 fx = F64(x % period) / period; + const F64 fy = F64(y % period) / period; + const F64 sx = fx * fx * (3.0 - 2.0 * fx); + const F64 sy = fy * fy * (3.0 - 2.0 * fy); + auto corner = [&](S32 dx, S32 dy) { return F64(hash(cx + dx, cy + dy, seed) & 0xffff) / 65535.0; }; + const F64 top = corner(0, 0) + (corner(1, 0) - corner(0, 0)) * sx; + const F64 bottom = corner(0, 1) + (corner(1, 1) - corner(0, 1)) * sx; + return top + (bottom - top) * sy; + } + + // A texture with detail at every scale, as photographs and painted textures have it: noise + // octaves whose amplitude falls with frequency, a different mix in each channel, and a few + // hard-edged shapes. + LLPointer makeTexture(S32 size, S32 components, U32 seed) + { + LLPointer raw = new LLImageRaw(U16(size), U16(size), S8(components)); + U8* data = raw->getData(); + for (S32 y = 0; y < size; ++y) + { + for (S32 x = 0; x < size; ++x) + { + for (S32 c = 0; c < components; ++c) + { + F64 value = 0.0; + F64 amplitude = 0.5; + for (S32 period = size / 2; period >= 1; period /= 2) + { + value += amplitude * (valueNoise(x, y, period, seed + c * 977) - 0.5); + amplitude *= 0.75; + } + const bool stripe = ((x + 2 * y) / (size / 8)) % 3 == 0; + const bool disc = (x - size / 3) * (x - size / 3) + (y - size / 2) * (y - size / 2) < size * size / 25; + value += (stripe ? 0.12 : 0.0) - (disc ? 0.18 : 0.0); + data[(y * size + x) * components + c] = U8(std::clamp(128.0 + 255.0 * value, 0.0, 255.0)); + } + } + } + return raw; + } + + // The viewer's own encoder: LRCP with quality layers of 600 and 2000 bytes, then four + // times the last, up to an eighth of the raw size. + LLPointer encodeLayered(const LLImageRaw* raw) + { + LLPointer j2c = new LLImageJ2C; + REQUIRE(j2c->encode(raw, 0.f)); + return j2c; + } + + struct MemoryOut + { + std::vector mBytes; + size_t mPos = 0; + size_t mEnd = 0; + }; + + OPJ_SIZE_T writeOut(void* buffer, OPJ_SIZE_T bytes, void* user) + { + MemoryOut* out = static_cast(user); + if (out->mPos + bytes > out->mBytes.size()) + { + out->mBytes.resize(out->mPos + bytes); + } + memcpy(out->mBytes.data() + out->mPos, buffer, bytes); + out->mPos += bytes; + out->mEnd = std::max(out->mEnd, out->mPos); + return bytes; + } + + OPJ_OFF_T skipOut(OPJ_OFF_T bytes, void* user) + { + MemoryOut* out = static_cast(user); + out->mPos += size_t(bytes); + if (out->mPos > out->mBytes.size()) + { + out->mBytes.resize(out->mPos); + } + out->mEnd = std::max(out->mEnd, out->mPos); + return bytes; + } + + OPJ_BOOL seekOut(OPJ_OFF_T pos, void* user) + { + static_cast(user)->mPos = size_t(pos); + return OPJ_TRUE; + } + + // An LRCP codestream at half the raw size in one quality layer, which orders its packets + // by resolution alone, as Second Life's single-layer textures are; or the same cut into + // four tiles. + std::vector encodeSingleLayer(const LLImageRaw* raw, bool four_tiles = false) + { + const S32 size = raw->getWidth(); + const S32 components = raw->getComponents(); + + opj_cparameters_t parameters; + opj_set_default_encoder_parameters(¶meters); + parameters.cod_format = OPJ_CODEC_J2K; + parameters.prog_order = OPJ_LRCP; + parameters.irreversible = 1; + parameters.cp_disto_alloc = 1; + parameters.tcp_numlayers = 1; + parameters.tcp_rates[0] = 2.f; + parameters.tcp_mct = components >= 3 ? 1 : 0; + if (four_tiles) + { + parameters.tile_size_on = OPJ_TRUE; + parameters.cp_tdx = size / 2; + parameters.cp_tdy = size / 2; + } + + std::vector component_parameters(components); + for (opj_image_cmptparm_t& component : component_parameters) + { + component = {}; + component.dx = 1; + component.dy = 1; + component.w = size; + component.h = size; + component.prec = 8; + } + opj_image_t* image = opj_image_create(components, component_parameters.data(), components >= 3 ? OPJ_CLRSPC_SRGB : OPJ_CLRSPC_GRAY); + REQUIRE(image); + image->x1 = size; + image->y1 = size; + const U8* data = raw->getData(); + for (S32 i = 0; i < size * size; ++i) + { + for (S32 c = 0; c < components; ++c) + { + image->comps[c].data[i] = data[i * components + c]; + } + } + + MemoryOut out; + opj_codec_t* codec = opj_create_compress(OPJ_CODEC_J2K); + opj_stream_t* stream = opj_stream_create(OPJ_J2K_STREAM_CHUNK_SIZE, OPJ_FALSE); + opj_stream_set_user_data(stream, &out, nullptr); + opj_stream_set_write_function(stream, writeOut); + opj_stream_set_skip_function(stream, skipOut); + opj_stream_set_seek_function(stream, seekOut); + const bool encoded = opj_setup_encoder(codec, ¶meters, image) + && opj_start_compress(codec, image, stream) + && opj_encode(codec, stream) + && opj_end_compress(codec, stream); + opj_stream_destroy(stream); + opj_destroy_codec(codec); + opj_image_destroy(image); + REQUIRE(encoded); + + out.mBytes.resize(out.mEnd); + return out.mBytes; + } + + LLPointer imageFrom(const U8* data, S32 size) + { + LLPointer j2c = new LLImageJ2C; + U8* copy = (U8*)ll_aligned_malloc_16(size); + memcpy(copy, data, size); + j2c->setData(copy, size); + return j2c; + } + + // The decode at `discard` of the first `bytes` of a codestream, as the decode thread runs it. + LLPointer decodePrefix(const U8* data, S32 bytes, S32 discard) + { + LLPointer j2c = imageFrom(data, bytes); + REQUIRE(j2c->updateData()); + j2c->setDiscardLevel(S8(discard)); + LLPointer raw = new LLImageRaw(j2c->getWidth(), j2c->getHeight(), j2c->getComponents()); + REQUIRE(j2c->decode(raw, 0.f)); + return raw; + } + + F64 psnr(const LLImageRaw* decoded, const LLImageRaw* reference) + { + REQUIRE(decoded->getWidth() == reference->getWidth()); + REQUIRE(decoded->getHeight() == reference->getHeight()); + REQUIRE(decoded->getComponents() == reference->getComponents()); + const S32 count = decoded->getDataSize(); + F64 squared_error = 0.0; + for (S32 i = 0; i < count; ++i) + { + const F64 difference = F64(decoded->getData()[i]) - F64(reference->getData()[i]); + squared_error += difference * difference; + } + if (squared_error == 0.0) + { + return IDENTICAL_DB; + } + return 10.0 * std::log10(255.0 * 255.0 * count / squared_error); + } + + // PSNR of a decode at `discard` from the first `bytes` against the decode at `discard` + // from all of them. + F64 prefixQuality(const std::vector& codestream, S32 bytes, S32 discard) + { + const S32 size = S32(codestream.size()); + LLPointer reference = decodePrefix(codestream.data(), size, discard); + LLPointer prefix = decodePrefix(codestream.data(), std::min(bytes, size), discard); + return psnr(prefix, reference); + } + + std::vector bytesOf(const LLImageJ2C* j2c) + { + return std::vector(j2c->getData(), j2c->getData() + j2c->getDataSize()); + } + + // Offset of the first SOT marker, found by walking the main header's segments. + S32 firstTilePartOffset(const std::vector& codestream) + { + S32 pos = 2; + while (codestream[pos + 1] != 0x90) + { + pos += 2 + ((codestream[pos + 2] << 8) | codestream[pos + 3]); + } + return pos; + } + + TEST_CASE("alj2clayout: the viewer's encoder writes a layout the parser reads whole", "[alj2clayout]") + { + LLPointer raw = makeTexture(256, 3, 1); + LLPointer j2c = encodeLayered(raw); + const std::vector codestream = bytesOf(j2c); + + const ALJ2CLayout layout = ALJ2CLayout::parse(codestream.data(), S32(codestream.size())); + CHECK(layout.isValid()); + CHECK(layout.mWidth == 256); + CHECK(layout.mHeight == 256); + CHECK(layout.mComponents == 3); + CHECK(layout.mTiles == 1); + CHECK(layout.mProgression == ALJ2CLayout::LRCP); + CHECK(layout.mLayers > 1); + CHECK(layout.mLevels == 5); + CHECK(layout.mHeaderLength == firstTilePartOffset(codestream)); + CHECK(layout.mLength == S32(codestream.size())); + } + + TEST_CASE("alj2clayout: a single layer codestream's length is its first tile-part's", "[alj2clayout]") + { + LLPointer raw = makeTexture(256, 4, 2); + const std::vector codestream = encodeSingleLayer(raw); + + const ALJ2CLayout layout = ALJ2CLayout::parse(codestream.data(), S32(codestream.size())); + CHECK(layout.isValid()); + CHECK(layout.mComponents == 4); + CHECK(layout.mLayers == 1); + CHECK(layout.mLength == S32(codestream.size())); + } + + TEST_CASE("alj2clayout: the first packet's worth of bytes holds the whole layout", "[alj2clayout]") + { + LLPointer raw = makeTexture(512, 3, 3); + const std::vector codestream = bytesOf(encodeLayered(raw)); + REQUIRE(S32(codestream.size()) > FIRST_PACKET_SIZE); + + const ALJ2CLayout whole = ALJ2CLayout::parse(codestream.data(), S32(codestream.size())); + const ALJ2CLayout first = ALJ2CLayout::parse(codestream.data(), FIRST_PACKET_SIZE); + CHECK(first.isValid()); + CHECK(first.mLength == whole.mLength); + CHECK(first.mHeaderLength == whole.mHeaderLength); + CHECK(first.mLayers == whole.mLayers); + } + + TEST_CASE("alj2clayout: a header cut anywhere before the first SOT's fields end is no layout", "[alj2clayout]") + { + LLPointer raw = makeTexture(128, 3, 4); + const std::vector codestream = bytesOf(encodeLayered(raw)); + const S32 sot_fields_end = firstTilePartOffset(codestream) + 12; + + for (S32 size = 0; size < sot_fields_end; ++size) + { + CAPTURE(size); + CHECK_FALSE(ALJ2CLayout::parse(codestream.data(), size).isValid()); + } + CHECK(ALJ2CLayout::parse(codestream.data(), sot_fields_end).isValid()); + } + + TEST_CASE("alj2clayout: a main header with no SOT after it is no layout", "[alj2clayout]") + { + LLPointer raw = makeTexture(128, 3, 5); + std::vector codestream = bytesOf(encodeLayered(raw)); + codestream.resize(firstTilePartOffset(codestream)); + codestream.push_back(0xFF); + codestream.push_back(0xD9); + + CHECK_FALSE(ALJ2CLayout::parse(codestream.data(), S32(codestream.size())).isValid()); + } + + TEST_CASE("alj2clayout: bytes that are not a codestream are no layout", "[alj2clayout]") + { + LLPointer raw = makeTexture(128, 3, 6); + std::vector codestream = bytesOf(encodeLayered(raw)); + + CHECK_FALSE(ALJ2CLayout::parse(nullptr, 600).isValid()); + std::vector shifted(codestream.begin() + 1, codestream.end()); + CHECK_FALSE(ALJ2CLayout::parse(shifted.data(), S32(shifted.size())).isValid()); + + // A segment length that runs past the data. + codestream[4] = 0xFF; + codestream[5] = 0xFF; + CHECK_FALSE(ALJ2CLayout::parse(codestream.data(), S32(codestream.size())).isValid()); + } + + TEST_CASE("alj2clayout: a codestream of several tiles leaves its length unknown", "[alj2clayout]") + { + LLPointer raw = makeTexture(256, 3, 7); + const std::vector codestream = encodeSingleLayer(raw, true); + + const ALJ2CLayout layout = ALJ2CLayout::parse(codestream.data(), S32(codestream.size())); + CHECK(layout.mTiles == 4); + CHECK(layout.mWidth == 256); + CHECK_FALSE(layout.isValid()); + CHECK(layout.calcDataSize(1) == 0); + } + + TEST_CASE("alj2clayout: an image learns its layout as the data comes in", "[alj2clayout]") + { + LLPointer raw = makeTexture(256, 3, 8); + const std::vector codestream = bytesOf(encodeLayered(raw)); + const S32 sot_fields_end = firstTilePartOffset(codestream) + 12; + + LLPointer j2c = new LLImageJ2C; + S32 appended = 0; + while (appended < S32(codestream.size())) + { + const S32 chunk = std::min(100, S32(codestream.size()) - appended); + U8* data = (U8*)ll_aligned_malloc_16(chunk); + memcpy(data, codestream.data() + appended, chunk); + j2c->appendData(data, chunk); + appended += chunk; + CAPTURE(appended); + CHECK(j2c->getLayout().isValid() == (appended >= sot_fields_end)); + } + CHECK(j2c->getLayout().mLength == S32(codestream.size())); + + const S32 whole = j2c->getLayout().mLength; + CHECK(j2c->calcDataSize(0) == whole); + CHECK(j2c->calcDiscardLevelBytes(whole) == 0); + } + + TEST_CASE("alj2clayout: targets grow with resolution and each reaches its own discard level", "[alj2clayout]") + { + LLPointer raw = makeTexture(1024, 3, 9); + const std::vector codestream = bytesOf(encodeLayered(raw)); + const ALJ2CLayout layout = ALJ2CLayout::parse(codestream.data(), S32(codestream.size())); + REQUIRE(layout.isValid()); + + CHECK(layout.calcDataSize(0) == layout.mLength); + for (S32 discard = 1; discard <= MAX_DISCARD_LEVEL; ++discard) + { + CAPTURE(discard); + const S32 target = layout.calcDataSize(discard); + CHECK(target <= layout.calcDataSize(discard - 1)); + CHECK(target >= FIRST_PACKET_SIZE); + CHECK(layout.calcDiscardLevelBytes(target) <= discard); + if (discard < MAX_DISCARD_LEVEL) + { + CHECK(layout.calcDiscardLevelBytes(target - 1) > discard); + } + } + CHECK(layout.calcDiscardLevelBytes(0) == MAX_DISCARD_LEVEL); + } + + // Each codestream is decoded at every discard level from the bytes its image asks for, + // and at discard 1 from the bytes the size-only estimate gives a codestream whose + // headers are not in hand, which cuts both kinds short. + void checkTargets(const std::vector& codestream, S32 size, S32 components) + { + LLPointer j2c = imageFrom(codestream.data(), S32(codestream.size())); + REQUIRE(j2c->getLayout().isValid()); + + for (S32 discard = 1; discard <= MAX_DISCARD_LEVEL; ++discard) + { + CAPTURE(discard, codestream.size()); + const S32 target = j2c->calcDataSize(discard); + CAPTURE(target); + CHECK(prefixQuality(codestream, target, discard) >= QUALITY_BAR_DB); + } + + const S32 estimate = LLImageJ2C::calcDataSizeJ2C(size, size, components, 1); + CAPTURE(estimate, codestream.size()); + CHECK(prefixQuality(codestream, estimate, 1) < QUALITY_BAR_DB); + } + + TEST_CASE("alj2clayout: a layered codestream decodes within the quality bar at every target", "[alj2clayout]") + { + for (S32 size : { 512, 1024 }) + { + CAPTURE(size); + LLPointer raw = makeTexture(size, 3, 10 + size); + checkTargets(bytesOf(encodeLayered(raw)), size, 3); + } + } + + TEST_CASE("alj2clayout: a single layer codestream decodes within the quality bar at every target", "[alj2clayout]") + { + for (S32 size : { 512, 1024 }) + { + CAPTURE(size); + LLPointer raw = makeTexture(size, 3, 20 + size); + checkTargets(encodeSingleLayer(raw), size, 3); + } + } +} diff --git a/indra/llkdu/tests/llimagej2ckdu_test.cpp b/indra/llkdu/tests/llimagej2ckdu_test.cpp index c8235c3fd71..3202e8e24b4 100644 --- a/indra/llkdu/tests/llimagej2ckdu_test.cpp +++ b/indra/llkdu/tests/llimagej2ckdu_test.cpp @@ -113,6 +113,8 @@ S8 LLImageJ2C::getRawDiscardLevel() { return 0; } void LLImageJ2C::resetLastError() { } void LLImageJ2C::setLastError(const std::string&, const std::string&) { } bool LLImageJ2C::updateData() { return false; } +void LLImageJ2C::deleteData() { } +void LLImageJ2C::onDataChanged() { } void LLImageJ2C::updateRawDiscardLevel() { } LLKDUMemIn::LLKDUMemIn(const U8*, const U32, const U16, const U16, const U8, kdu_core::siz_params*) { } diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index d55515b8ae8..115c87ce36c 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -12247,28 +12247,6 @@ Value 0 - TextureNewByteRange - - Comment - Use the new more accurate byte range computation for j2c discard levels - Persist - 1 - Type - Boolean - Value - 1 - - TextureReverseByteRange - - Comment - Minimal percent of the optimal byte range allowed to render a given discard level - Persist - 1 - Type - S32 - Value - 50 - TextureSaveLocation Comment diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 88d466a5cdf..83a79411111 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -2333,7 +2333,7 @@ bool LLAppViewer::initThreads() static const bool enable_threads = true; - LLImage::initClass(gSavedSettings.getBOOL("TextureNewByteRange"),gSavedSettings.getS32("TextureReverseByteRange")); + LLImage::initClass(); LLLFSThread::initClass(enable_threads && true); // TODO: fix crashes associated with this shutdo diff --git a/indra/newview/skins/default/xui/pt/settings_comments.xml b/indra/newview/skins/default/xui/pt/settings_comments.xml index 852efe5d6ef..713c62f5d4e 100644 --- a/indra/newview/skins/default/xui/pt/settings_comments.xml +++ b/indra/newview/skins/default/xui/pt/settings_comments.xml @@ -2555,10 +2555,6 @@ Se TRUE, sempre carregar texturas em resolução total (discard = 0). TextureLoggingThreshold Especifica o limiar em bytes a partir do qual os dados de download de textura devem ser enviados ao sim. - TextureNewByteRange - Usar o novo cálculo de faixa de bytes mais preciso para os níveis de descarte j2c. - TextureReverseByteRange - Percentual mínimo da faixa de bytes ótima permitido para renderizar um dado nível de descarte. TextureSaveLocation Localização atual para salvar texturas em massa no disco. TextureScaleMaxAreaFactor From c01cee2ddbbbe9b80a279460086279579b1f7846 Mon Sep 17 00:00:00 2001 From: Rye Date: Thu, 22 Oct 2026 11:27:51 -0400 Subject: [PATCH 2/3] A texture fetch asks for its codestream's byte target and decodes at the level its bytes reach With RenderMaxTextureResolution at 1024, 2048 textures stayed blurry after a relog until the setting was toggled. The fresh worker that asked for discard 1 read calcDataSizeJ2C's estimate from the cache (128 KB of a complete 1.39 MB entry), and CACHE_POST, like the HTTP completion, recorded the requested discard as loaded. A decode of the first quality layers at reduce 1 therefore counted as discard 1, the DONE -> INIT loop had nothing left to fetch, and the cap, which pins 2048s at discard 1, made it permanent. Toggling to 2048 asked for discard 0, which fetches everything. The worker copies the codestream's layout from its formatted image once the headers are in (updateLayout) and keeps it for the texture's later requests, since endWork() lets go of the image after each one. It sizes every request for discard 1 and up from it, in createRequest and setDesiredDiscard as well as in the running request; discard 0 still asks for everything. When the headers arrive with a cache read the estimate sized and the entry filled that read, it reads on from the entry instead of decoding what it has. CACHE_POST and the HTTP completion set mLoadedDiscard to the finest level the bytes reach and never finer than the request (calcLoadedDiscard), so a response short of the target decodes as a preview and the DONE -> INIT loop fetches the rest. DONE weighs that level as well as the decode's, which a codestream with fewer resolutions than the level loaded makes finer than the bytes support. A cache read that needs half the codestream or more takes all of it: local disk is cheap next to a second read. A request whose network fetch fails with data in hand decodes what it holds and finishes, as before (mFetchFailed). Without that, a preview short of the target made DONE ask again after every failure, with no backoff for ordinary textures: a decode, a cache read and a request per cycle for as long as the server failed. Co-Authored-By: Claude Opus 5.5 --- indra/newview/lltexturefetch.cpp | 94 ++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 6 deletions(-) diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 40fc6dc3d5f..e1364718f60 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -483,6 +483,21 @@ class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler // Locks: Mw (ctor invokes without lock) void setDesiredDiscard(S32 discard, S32 size); + // Locks: Mw + // The bytes a request for `discard` needs: from the codestream's layout once it is known, + // `estimate` until then. A request for discard 0 keeps `estimate`, which asks for all of it. + S32 calcDesiredSize(S32 discard, S32 estimate) const; + + // Locks: Mw + // Takes the layout from the formatted image once it holds the codestream's headers, and + // sizes the request by it. True when this call found it. + bool updateLayout(); + + // Locks: Mw + // The discard level `bytes` of formatted data decode a request for `discard` at: the + // finest they reach, never finer than the request. + S32 calcLoadedDiscard(S32 discard, S32 bytes) const; + // Locks: Mw void removeFromCache(); @@ -546,6 +561,8 @@ class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler LLPointer mFormattedImage; LLPointer mRawImage, mAuxImage; + // The codestream's layout, once mFormattedImage has held its headers. + ALJ2CLayout mLayout; FTType mFTType; LLUUID mID; LLHost mHost; @@ -578,7 +595,8 @@ class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler mRequestedOffset, mDesiredSize, mFileSize, - mCachedSize; + mCachedSize, + mCacheReadEnd; // where the last cache read was asked to stop e_request_state mSentRequest; handle_t mDecodeHandle; // DecodeResponders alive that point at this worker. A responder lives on past completed() @@ -590,6 +608,9 @@ class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler bool mWritten; bool mNeedsAux; bool mHaveAllData; + // The last network request failed: the worker decodes what it holds and finishes rather + // than asking again. + bool mFetchFailed; bool mInLocalCache; bool mInCache; bool mCanUseHTTP; @@ -909,6 +930,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mFileSize(0), mSkippedStatesTime(0), mCachedSize(0), + mCacheReadEnd(0), mLoaded(false), mSentRequest(UNSENT), mDecodeHandle(0), @@ -917,6 +939,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mWritten(false), mNeedsAux(false), mHaveAllData(false), + mFetchFailed(false), mInLocalCache(false), mInCache(false), mCanUseHTTP(true), @@ -996,6 +1019,7 @@ LLTextureFetchWorker::~LLTextureFetchWorker() // Locks: Mw (ctor invokes without lock) void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) { + size = calcDesiredSize(discard, size); bool prioritize = false; if (mDesiredDiscard != discard) { @@ -1025,6 +1049,42 @@ void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) } } +// Locks: Mw +S32 LLTextureFetchWorker::calcDesiredSize(S32 discard, S32 estimate) const +{ + return (mLayout.isValid() && discard > 0) ? mLayout.calcDataSize(discard) : estimate; +} + +// Locks: Mw +bool LLTextureFetchWorker::updateLayout() +{ + if (mLayout.isValid() || mFormattedImage.isNull() || mFormattedImage->getCodec() != IMG_CODEC_J2C) + { + return false; + } + + { + LLImageDataSharedLock lock(mFormattedImage); + mLayout = static_cast(mFormattedImage.get())->getLayout(); + } + if (!mLayout.isValid()) + { + return false; + } + mDesiredSize = llmax(calcDesiredSize(mDesiredDiscard, mDesiredSize), TEXTURE_CACHE_ENTRY_SIZE); + return true; +} + +// Locks: Mw +S32 LLTextureFetchWorker::calcLoadedDiscard(S32 discard, S32 bytes) const +{ + if (mHaveAllData || !mLayout.isValid()) + { + return discard; + } + return llmax(discard, mLayout.calcDiscardLevelBytes(bytes)); +} + // Locks: Mw void LLTextureFetchWorker::setImagePriority(F32 priority) { @@ -1179,6 +1239,8 @@ bool LLTextureFetchWorker::doWork(S32 param) mHttpReplySize = 0; mHttpReplyOffset = 0; mHaveAllData = false; + mFetchFailed = false; + mCacheReadEnd = 0; mCacheReadHandle = LLTextureCache::nullHandle(); mCacheWriteHandle = LLTextureCache::nullHandle(); setState(LOAD_FROM_TEXTURE_CACHE); @@ -1203,6 +1265,13 @@ bool LLTextureFetchWorker::doWork(S32 param) return doWork(param); // return false; } + if (mLayout.isValid() && 2 * S64(mDesiredSize) >= mLayout.mLength) + { + // Reading the rest of a local entry now costs less than reading it later + // for a finer level. + size = llmax(size, mLayout.mLength - offset); + } + mCacheReadEnd = offset + size; mFileSize = 0; mLoaded = false; @@ -1266,12 +1335,19 @@ bool LLTextureFetchWorker::doWork(S32 param) { LL_PROFILE_ZONE_NAMED_CATEGORY_THREAD("tfwdw - CACHE_POST"); mCachedSize = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; + if (updateLayout() && mCachedSize < mDesiredSize && !mHaveAllData && mCachedSize >= mCacheReadEnd) + { + // The headers just read size the request past a read the entry filled, so it + // may hold more. + setState(LOAD_FROM_TEXTURE_CACHE); + return doWork(param); + } // Successfully loaded if ((mCachedSize >= mDesiredSize) || mHaveAllData) { // we have enough data, decode it llassert_always(mFormattedImage->getDataSize() > 0); - mLoadedDiscard = mDesiredDiscard; + mLoadedDiscard = calcLoadedDiscard(mDesiredDiscard, mCachedSize); if (mLoadedDiscard < 0) { LL_WARNS(LOG_TXT) << mID << " mLoadedDiscard is " << mLoadedDiscard @@ -1646,6 +1722,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (cur_size > 0) { // Use available data + mFetchFailed = true; mLoadedDiscard = mFormattedImage->getDiscardLevel(); if (mLoadedDiscard < 0) { @@ -1766,7 +1843,8 @@ bool LLTextureFetchWorker::doWork(S32 param) mHttpReplySize = 0; mHttpReplyOffset = 0; - mLoadedDiscard = mRequestedDiscard; + updateLayout(); + mLoadedDiscard = calcLoadedDiscard(mRequestedDiscard, total_size); if (mLoadedDiscard < 0 || (mLoadedDiscard > MAX_DISCARD_LEVEL && mFormattedImage->getCodec() == IMG_CODEC_J2C)) { LL_WARNS(LOG_TXT) << mID << " mLoadedDiscard is " << mLoadedDiscard @@ -1891,6 +1969,7 @@ bool LLTextureFetchWorker::doWork(S32 param) LL_DEBUGS(LOG_TXT) << mID << ": Decode of cached file failed (removed), retrying" << LL_ENDL; llassert_always(mDecodeHandle == 0); mFormattedImage = NULL; + mLayout = {}; ++mRetryAttempt; setState(INIT); //return false; @@ -1992,13 +2071,15 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DONE) { LL_PROFILE_ZONE_NAMED_CATEGORY_THREAD("tfwdw - DONE"); - if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard) + // The decode can come out finer than the bytes support when the codestream has fewer + // resolutions than the level loaded, so the bytes' level counts as well. + if (mDecodedDiscard >= 0 && !mFetchFailed && mDesiredDiscard < llmax(mDecodedDiscard, mLoadedDiscard)) { // More data was requested, return to INIT setState(INIT); LL_DEBUGS(LOG_TXT) << mID << " more data requested, returning to INIT: " - << " mDecodedDiscard " << mDecodedDiscard << ">= 0 && mDesiredDiscard " << mDesiredDiscard - << "<" << " mDecodedDiscard " << mDecodedDiscard << LL_ENDL; + << " mDecodedDiscard " << mDecodedDiscard << " mLoadedDiscard " << mLoadedDiscard + << " > mDesiredDiscard " << mDesiredDiscard << LL_ENDL; // return false; return doWork(param); } @@ -2669,6 +2750,7 @@ S32 LLTextureFetch::createRequest(FTType f_type, const std::string& url, const L return CREATE_REQUEST_ERROR_ABORTED; // need to wait for previous aborted request to complete } worker->lockWorkMutex(); // +Mw + desired_size = worker->calcDesiredSize(desired_discard, desired_size); if (worker->mState == LLTextureFetchWorker::DONE && worker->mDesiredSize == llmax(desired_size, TEXTURE_CACHE_ENTRY_SIZE) && worker->mDesiredDiscard == desired_discard) { worker->unlockWorkMutex(); // -Mw From a6ce80e3099aeff7824ebd288e95931e3bc812e1 Mon Sep 17 00:00:00 2001 From: Rye Date: Thu, 22 Oct 2026 11:28:08 -0400 Subject: [PATCH 3/3] LLImageJ2COJ::getMetadata no longer seeds byte budgets from tile-part ends The SOT walker mapped tile-part ends to discard levels on the assumption of one tile-part per resolution. Every Second Life texture has one tile-part, so every level got the whole file, and a partial walk mapped discard 0 to the last tile-part walked rather than the end of the codestream. LLImageJ2C takes its targets from ALJ2CLayout now. getMetadata's guess at mDiscardLevel, which updateData overwrote, goes with it. The registry's openjpeg 2.5.4#31 drops opj_j2k_walk_tile_part_ends; nothing here calls it, so the viewer builds against either port version. Co-Authored-By: Claude Opus 5.5 --- indra/llimagej2coj/llimagej2coj.cpp | 74 +---------------------------- 1 file changed, 2 insertions(+), 72 deletions(-) diff --git a/indra/llimagej2coj/llimagej2coj.cpp b/indra/llimagej2coj/llimagej2coj.cpp index 52081f246f4..1130e113643 100644 --- a/indra/llimagej2coj/llimagej2coj.cpp +++ b/indra/llimagej2coj/llimagej2coj.cpp @@ -305,8 +305,7 @@ class JPEG2KDecode : public JPEG2KBase U32 dataSize, S32& widthOut, S32& heightOut, - S32& components, - S32& discard_level) + S32& components) { parameters.flags |= OPJ_DPARAMETERS_DUMP_FLAG; @@ -372,21 +371,6 @@ class JPEG2KDecode : public JPEG2KBase heightOut = S32(tilesH * tileDimY); components = codestream_info->nbcomps; - // The maximum discard level is bounded by the codestream's resolution - // count (you can't ask OpenJPEG to reduce below the lowest encoded - // resolution). Prior versions of this code derived it from log2(tile - // count), which returns 0 for single-tile codestreams — the common - // case for SL textures — and effectively disabled progressive decode. - discard_level = 0; - if (codestream_info->m_default_tile_info.tccp_info) - { - U32 numres = codestream_info->m_default_tile_info.tccp_info[0].numresolutions; - if (numres > 0) - { - discard_level = llclamp(S32(numres) - 1, 0, MAX_DISCARD_LEVEL); - } - } - return true; } @@ -1104,11 +1088,10 @@ bool LLImageJ2COJ::getMetadata(LLImageJ2C &base) S32 width = 0; S32 height = 0; S32 components = 0; - S32 discard_level = 0; U32 dataSize = base.getDataSize(); U8* data = base.getData(); - bool header_read = decode.readHeader(data, dataSize, width, height, components, discard_level); + bool header_read = decode.readHeader(data, dataSize, width, height, components); if (!header_read) { std::string opj_err = decode.getLastError(); @@ -1116,60 +1099,7 @@ bool LLImageJ2COJ::getMetadata(LLImageJ2C &base) return false; } - base.mDiscardLevel = discard_level; base.setSize(width, height, components); - // Override the synthetic byte-budget estimate with actual codestream - // tile-part boundaries when we can parse them. Our patched OpenJPEG - // tolerates truncated tile-parts (see indra/vcpkg/ports/openjpeg/), so - // seeding from a *partial* walk is now safe: the fetcher triggers - // decode earlier, OJ produces whatever pixels it can from the available - // bytes, and the texture refines as more data arrives. - // - // Mapping: discard 0 (highest quality) gets the end of the last walked - // tile-part; discard MAX_DISCARD_LEVEL gets the end of the first. Single - // tile-part files (typical OJ-encoded uploads) collapse to one value - // across every discard. - // Up-front cap on tile-part-ends. SL textures rarely exceed ~6 tile-parts - // (one per resolution); a 64-entry buffer is comfortably oversized and - // matches MAX_NB_LAYERS. The OJ walker fills out_num_ends with the - // actual count. - constexpr OPJ_UINT32 MAX_TILE_PART_ENDS = 64; - OPJ_SIZE_T tile_part_ends[MAX_TILE_PART_ENDS]; - OPJ_UINT32 num_tile_part_ends = 0; - OPJ_BOOL complete = OPJ_FALSE; - bool walked = opj_j2k_walk_tile_part_ends(data, (OPJ_SIZE_T)dataSize, - tile_part_ends, - &num_tile_part_ends, - MAX_TILE_PART_ENDS, - &complete) == OPJ_TRUE; - if (walked && num_tile_part_ends > 0) - { - S32 n = (S32)num_tile_part_ends; - for (S32 d = 0; d <= MAX_DISCARD_LEVEL; d++) - { - S32 idx = llclamp(n - 1 - d, 0, n - 1); - base.mDataSizes[d] = (S32)tile_part_ends[idx]; - } - // Mark the cache valid so calcDataSize() returns our values rather - // than recomputing via calcDataSizeJ2C. - base.mAreaUsedForDataSizeCalcs = base.getHeight() * base.getWidth(); - - LL_DEBUGS("Texture") << "LLImageJ2COJ: SOT walker seeded mDataSizes from " - << n << " tile-part(s) (" - << (complete ? "complete" : "partial") - << "); buf=" << dataSize - << " last_tp_end=" << tile_part_ends[n - 1] - << " d0=" << base.mDataSizes[0] - << " d5=" << base.mDataSizes[MAX_DISCARD_LEVEL] - << LL_ENDL; - } - else - { - LL_DEBUGS("Texture") << "LLImageJ2COJ: SOT walker failed (buf=" << dataSize - << "), falling back to calcDataSizeJ2C estimate" - << LL_ENDL; - } - return true; }