fix(atomicmarket): read the stored template mint of an unlinked listing and cap the collection names so the effective mint filters hold on a large collection

CHANGELOG.md

@@ -25,6 +25,8 @@ Adds an effective mint filter and sort to the market listing endpoints, so a cli
 - The effective mint of an asset is its `original_mint` from `atomicassets_original_mints` when a link row with a mint number exists, else its template mint. An asset with neither is ignored. A listing matches when at least one of its assets has an effective mint and every such asset lies in the range, so a listing of an asset with no link is still found by its template mint. (#234)
 - The listing endpoints above, except `/v1/sales/templates`, accept `sort=effective_mint`, which orders by the lowest effective mint of the listing. It returns only listings where an asset has an effective mint, with or without a bound, and the matching `_count` endpoint counts the same listings. (#234)
 - A request that uses `min_effective_mint`, `max_effective_mint` or `sort=effective_mint` needs `collection_name` and returns 400 without it, because without that scope each listing needs its own lookup of the link table. The `template_mint` filters and sorts keep their queries. (#234)
+- A request that uses one of these three parameters also returns 400 with more than 50 collection names, counted before deduplication, because a longer list leaves the indexed filter path of `/v2/sales`. A request without these parameters accepts any number of names. (#235)
+- For a listing with no linked asset, the effective mint filters and sort read the stored template mint range of the listing, the value the `template_mint` filter reads, so the cost does not grow with the asset rows of an unlinked collection. A listing with a linked asset still reads the mint of each of its assets. A listing whose stored range is not written yet reads the mint of each of its assets. On `/v2/sales` the stored range follows a change of an asset mint after the filter table refresh, as for the `template_mint` filter. (#235)
 
 ## [2.5.0]
 

src/api/namespaces/atomicmarket/effective-mint-suite.ts

@@ -9,12 +9,25 @@ export type MintAssetSpec = {
     original?: number | null,
 };
 
+/**
+ * The template mint range the filler stores on a listing: the lowest and highest template
+ * mint of its assets, or 'empty' when no asset has one.
+ */
+export function storedTemplateRange(assets: MintAssetSpec[]): string {
+    const mints = assets.map(a => a.template).filter((m): m is number => typeof m === 'number');
+
+    return mints.length ? `[${Math.min(...mints)},${Math.max(...mints)}]` : 'empty';
+}
+
 export type EffectiveMintSuiteOptions = {
+    // True when makeListing stores the given range. The sales filter table derives its own.
+    canOverrideStoredRange?: boolean,
     client: AtomicMarketTestClient,
     txit: any,
-    // listingTemplateMint is the template mint range the listing stores, as [n,n], for the
+    // listingTemplateMint is the template mint range the listing stores, as [n,n], or null for no
+    // stored range yet, for the
     // handlers that read a stored column. The sales filter table derives it from the assets.
-    makeListing: (collectionName: string, assets: MintAssetSpec[], listingTemplateMint?: number) => Promise<number>,
+    makeListing: (collectionName: string, assets: MintAssetSpec[], listingTemplateMint?: number | null) => Promise<number>,
     query: (params: Record<string, string>) => Promise<number[]>,
     count: (params: Record<string, string>) => Promise<number>,
 };
@@ -53,7 +66,7 @@ async function expectApiError(promise: Promise<any>, code: number, message: stri
 }
 
 export function defineEffectiveMintSuite(options: EffectiveMintSuiteOptions): void {
-    const {client, txit, makeListing, query, count} = options;
+    const {client, txit, makeListing, query, count, canOverrideStoredRange} = options;
 
     async function setup(): Promise<string> {
         return (await client.createCollection()).collection_name;
@@ -177,6 +190,144 @@ export function defineEffectiveMintSuite(options: EffectiveMintSuiteOptions): vo
                 .to.deep.equal([both]);
         });
 
+        txit('matches an unlinked listing by the template mint range it stores', async () => {
+            const c = await setup();
+            const single = await makeListing(c, [{template: 5}]);
+            const multi = await makeListing(c, [{template: 4}, {template: 6}]);
+            await makeListing(c, [{template: 4}, {template: 60}]);
+            await makeListing(c, [{template: 3}]);
+
+            expect((await query({collection_name: c, min_effective_mint: '4', max_effective_mint: '6'})).sort())
+                .to.deep.equal([single, multi].sort());
+        });
+
+        txit('reads the aggregate, not the stored range, of a linked listing', async () => {
+            const c = await setup();
+            const originalIn = await makeListing(c, [{original: 5, template: 500}]);
+            await makeListing(c, [{original: 50, template: 5}]);
+
+            expect(await query({collection_name: c, min_effective_mint: '4', max_effective_mint: '6'}))
+                .to.deep.equal([originalIn]);
+        });
+
+        txit('accepts a bound above the int4 range on an unlinked listing', async () => {
+            const c = await setup();
+            const low = await makeListing(c, [{template: 5}]);
+            const high = await makeListing(c, [{template: 2000000000}]);
+            const linked = await makeListing(c, [{original: 3000000000, template: 5}]);
+            const huge = '9007199254740991';
+
+            expect((await query({collection_name: c, min_effective_mint: '4', max_effective_mint: huge})).sort())
+                .to.deep.equal([low, high, linked].sort());
+            expect((await query({collection_name: c, max_effective_mint: huge})).sort())
+                .to.deep.equal([low, high, linked].sort());
+            expect(await query({collection_name: c, min_effective_mint: huge})).to.deep.equal([]);
+        });
+
+        txit('accepts a minimum alone and a maximum alone on unlinked listings', async () => {
+            const c = await setup();
+            const low = await makeListing(c, [{template: 2}]);
+            const high = await makeListing(c, [{template: 20}]);
+            await makeListing(c, [{}]);
+
+            expect(await query({collection_name: c, min_effective_mint: '10'})).to.deep.equal([high]);
+            expect(await query({collection_name: c, max_effective_mint: '10'})).to.deep.equal([low]);
+            expect((await query({collection_name: c, min_effective_mint: '2', max_effective_mint: '20'})).sort())
+                .to.deep.equal([low, high].sort());
+        });
+
+        txit('orders linked and unlinked listings together and counts the same rows', async () => {
+            const c = await setup();
+            const linkedLow = await makeListing(c, [{original: 1, template: 90}]);
+            const unlinkedLow = await makeListing(c, [{template: 3}]);
+            const linkedMid = await makeListing(c, [{original: 7}]);
+            const unlinkedMulti = await makeListing(c, [{template: 12}, {template: 9}]);
+            await makeListing(c, [{}]);
+
+            const asc = {collection_name: c, sort: 'effective_mint', order: 'asc'};
+
+            expect(await query(asc)).to.deep.equal([linkedLow, unlinkedLow, linkedMid, unlinkedMulti]);
+            expect(await query({...asc, order: 'desc'}))
+                .to.deep.equal([unlinkedMulti, linkedMid, unlinkedLow, linkedLow]);
+            expect(await count(asc)).to.equal(4);
+        });
+
+        // The sales filter table derives its range from the assets, so only the stored-column
+        // sources can hold a range that differs from them. /v2/sales proves it in its own file.
+        if (canOverrideStoredRange) {
+            txit('reads the stored range of an unlinked listing', async () => {
+                const c = await setup();
+                // the stored range holds the mints of the listing, so a range that differs from the
+                // asset decides, which proves the filter reads the stored column
+                const storedIn = await makeListing(c, [{template: 500}], 5);
+                await makeListing(c, [{template: 5}], 500);
+
+                expect(await query({collection_name: c, min_effective_mint: '4', max_effective_mint: '6'}))
+                    .to.deep.equal([storedIn]);
+                expect(await query({collection_name: c, sort: 'effective_mint', order: 'asc', max_effective_mint: '6'}))
+                    .to.deep.equal([storedIn]);
+            });
+
+            txit('reads the assets of a listing whose stored range is not written yet', async () => {
+                const c = await setup();
+                const unlinkedNull = await makeListing(c, [{template: 5}], null);
+                const linkedNull = await makeListing(c, [{original: 6, template: 500}], null);
+                const third = await makeListing(c, [{template: 50}], null);
+                await makeListing(c, [{}], null);
+
+                const range = {collection_name: c, min_effective_mint: '4', max_effective_mint: '7'};
+                const sort = {collection_name: c, sort: 'effective_mint', order: 'asc'};
+
+                expect((await query(range)).sort()).to.deep.equal([unlinkedNull, linkedNull].sort());
+                expect(await query({...range, sort: 'effective_mint', order: 'asc'}))
+                    .to.deep.equal([unlinkedNull, linkedNull]);
+                expect(await count(range)).to.equal(2);
+                expect(await query(sort)).to.deep.equal([unlinkedNull, linkedNull, third]);
+                expect(await count(sort)).to.equal(3);
+            });
+        }
+
+        describe('collection name cap', () => {
+            const others = (n: number): string[] => Array.from({length: n}, (_, i) =>
+                `zz${String.fromCharCode(97 + Math.floor(i / 26))}${String.fromCharCode(97 + (i % 26))}`);
+            const CAP_MESSAGE = 'The effective mint filters and sort accept at most 50 collection names';
+
+            txit('accepts 50 collection names with a mint parameter', async () => {
+                const c = await setup();
+                const id = await makeListing(c, [{original: 5}]);
+                const names = [c, ...others(49)].join(',');
+
+                expect(await query({collection_name: names, min_effective_mint: '1'})).to.deep.equal([id]);
+                expect(await query({collection_name: names, sort: 'effective_mint'})).to.deep.equal([id]);
+            });
+
+            txit('rejects 51 collection names with a mint parameter', async () => {
+                const c = await setup();
+                await makeListing(c, [{original: 5}]);
+                const names = [c, ...others(50)].join(',');
+
+                await expectApiError(query({collection_name: names, min_effective_mint: '1'}), 400, CAP_MESSAGE);
+                await expectApiError(query({collection_name: names, max_effective_mint: '9'}), 400, CAP_MESSAGE);
+                await expectApiError(query({collection_name: names, sort: 'effective_mint'}), 400, CAP_MESSAGE);
+                await expectApiError(count({collection_name: names, min_effective_mint: '1'}), 400, CAP_MESSAGE);
+            });
+
+            txit('counts 51 copies of one name as 51', async () => {
+                const c = await setup();
+
+                await expectApiError(
+                    query({collection_name: Array(51).fill(c).join(','), min_effective_mint: '1'}), 400, CAP_MESSAGE
+                );
+            });
+
+            txit('keeps 51 collection names working without a mint parameter', async () => {
+                const c = await setup();
+                const id = await makeListing(c, [{original: 5}]);
+
+                expect(await query({collection_name: [c, ...others(50)].join(',')})).to.deep.equal([id]);
+            });
+        });
+
         txit('rejects the mint parameters without collection_name', async () => {
             await expectApiError(query({min_effective_mint: '1'}), 400, SCOPE_MESSAGE);
             await expectApiError(query({max_effective_mint: '1'}), 400, SCOPE_MESSAGE);

src/api/namespaces/atomicmarket/handlers/auctions.integration.test.ts

@@ -4,7 +4,7 @@ import {initAtomicMarketTest} from '../test';
 import {RequestValues} from '../../utils';
 import {getTestContext} from '../../../../utils/test';
 import {getAuctionAction, getAuctionsAction, getAuctionsCountAction} from './auctions';
-import {createMintAsset, defineEffectiveMintSuite} from '../effective-mint-suite';
+import {createMintAsset, defineEffectiveMintSuite, storedTemplateRange} from '../effective-mint-suite';
 import {AuctionApiState} from '../index';
 import sinon from 'sinon';
 import {clearMarketVersionCache, MARKET_VERSION_CACHE_TTL_MS} from '../market-version';
@@ -337,12 +337,15 @@ describe('auction handler', () => {
     });
 
     defineEffectiveMintSuite({
+        canOverrideStoredRange: true,
         client,
         txit,
         makeListing: async (collectionName, assets, templateMint) => {
             const listing = await client.createAuction({
                 collection_name: collectionName,
-                ...(templateMint === undefined ? {} : {template_mint: `[${templateMint},${templateMint}]`}),
+                template_mint: templateMint === undefined
+                    ? storedTemplateRange(assets)
+                    : (templateMint === null ? null : `[${templateMint},${templateMint}]`),
             });
             for (const [i, spec] of assets.entries()) {
                 const asset_id = await createMintAsset(client, collectionName, spec);

src/api/namespaces/atomicmarket/handlers/buyoffers.integration.test.ts

@@ -3,7 +3,7 @@ import {initAtomicMarketTest} from '../test';
 import {RequestValues} from '../../utils';
 import {getTestContext} from '../../../../utils/test';
 import {getBuyOfferAction, getBuyOffersAction, getBuyOffersCountAction} from './buyoffers';
-import {createMintAsset, defineEffectiveMintSuite} from '../effective-mint-suite';
+import {createMintAsset, defineEffectiveMintSuite, storedTemplateRange} from '../effective-mint-suite';
 import {ApiError} from '../../../error';
 
 // TODO add more tests
@@ -163,12 +163,15 @@ describe('buy offer handler', () => {
     });
 
     defineEffectiveMintSuite({
+        canOverrideStoredRange: true,
         client,
         txit,
         makeListing: async (collectionName, assets, templateMint) => {
             const listing = await client.createBuyOffer({
                 collection_name: collectionName,
-                ...(templateMint === undefined ? {} : {template_mint: `[${templateMint},${templateMint}]`}),
+                template_mint: templateMint === undefined
+                    ? storedTemplateRange(assets)
+                    : (templateMint === null ? null : `[${templateMint},${templateMint}]`),
             });
             for (const [i, spec] of assets.entries()) {
                 const asset_id = await createMintAsset(client, collectionName, spec);

src/api/namespaces/atomicmarket/handlers/sales.get-sales.integration.test.ts

@@ -1,7 +1,7 @@
 import 'mocha';
 import {expect} from 'chai';
 import {getSaleAction, getSalesAction, getSalesCountAction} from './sales';
-import {createMintAsset, defineEffectiveMintSuite} from '../effective-mint-suite';
+import {createMintAsset, defineEffectiveMintSuite, storedTemplateRange} from '../effective-mint-suite';
 import {SaleApiState} from '../index';
 import {OfferState} from '../../../../filler/handlers/atomicassets';
 import {SaleState} from '../../../../filler/handlers/atomicmarket';
@@ -773,6 +773,7 @@ describe('AtomicMarket Sales API', () => {
     });
 
     defineEffectiveMintSuite({
+        canOverrideStoredRange: true,
         client,
         txit,
         makeListing: async (collectionName, assets, templateMint) => {
@@ -784,7 +785,9 @@ describe('AtomicMarket Sales API', () => {
 
             return (await client.createSale({
                 offer_id, collection_name: collectionName,
-                ...(templateMint === undefined ? {} : {template_mint: `[${templateMint},${templateMint}]`}),
+                template_mint: templateMint === undefined
+                    ? storedTemplateRange(assets)
+                    : (templateMint === null ? null : `[${templateMint},${templateMint}]`),
             })).sale_id;
         },
         query: async (values) => await getSalesIds(values),

src/api/namespaces/atomicmarket/handlers/sales2.get-sales-templates.integration.test.ts

@@ -459,6 +459,40 @@ describe('AtomicMarket Sales API', () => {
             })).to.deep.equal([both]);
         });
 
+        txit('reads the stored range of an unlinked listing, not its assets', async () => {
+            const c = (await client.createCollection()).collection_name;
+            const storedOut = await templateSale(c, {template: 5});
+            const storedIn = await templateSale(c, {template: 500});
+            await client.refreshSalesFilters();
+            await client.query('UPDATE atomicmarket_sales_filters SET template_mint = $2 WHERE sale_id = $1', [storedOut, '[500,500]']);
+            await client.query('UPDATE atomicmarket_sales_filters SET template_mint = $2 WHERE sale_id = $1', [storedIn, '[5,5]']);
+
+            // no refresh here, so the stored range the updates wrote is the one read
+            const result = await getSalesTemplatesV2Action({
+                symbol: 'TEST', collection_name: c, min_effective_mint: '4', max_effective_mint: '6',
+            }, getTestContext(client));
+
+            expect(result.map((s: any) => s.sale_id)).to.deep.equal([storedIn]);
+        });
+
+        txit('rejects 51 collection names with a bound', async () => {
+            const c = (await client.createCollection()).collection_name;
+            await templateSale(c, {original: 5});
+            const names = [c, ...Array.from({length: 50}, (_, i) =>
+                `zz${String.fromCharCode(97 + Math.floor(i / 26))}${String.fromCharCode(97 + (i % 26))}`)].join(',');
+
+            let err: any;
+            try {
+                await getSalesIds({collection_name: names, min_effective_mint: '1'});
+            } catch (e) {
+                err = e;
+            }
+
+            expect(err).to.be.instanceof(ApiError);
+            expect(err.code).to.equal(400);
+            expect(err.message).to.equal('The effective mint filters and sort accept at most 50 collection names');
+        });
+
         txit('requires collection_name and a valid range', async () => {
             const c = (await client.createCollection()).collection_name;
             await templateSale(c, {original: 5});