From 9e13984b0571d067c8ade00a3438892c4abf6bee Mon Sep 17 00:00:00 2001 From: Yusuf Suleman Date: Sun, 29 Mar 2026 13:50:07 -0500 Subject: [PATCH] =?UTF-8?q?fix:=20performance=20hardening=20=E2=80=94=20el?= =?UTF-8?q?iminate=20full=20table=20scans=20(#9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inventory: - /issues: replaced full scan + client filter with NocoDB server-side WHERE filter (Received eq Issues/Issue). Single query, ~200 rows max. - /needs-review-count: replaced full scan with server-side WHERE + limit=1 + pageInfo.totalRows. Returns count without fetching data. Budget: - buildLookups(): added 2-minute cache for payee/account/category maps. Eliminates 3 API calls per request for repeated queries. - /summary cache (added earlier): 1-minute TTL still active. Files: services/inventory/server.js, services/budget/server.js --- services/budget/server.js | 10 ++++- services/inventory/server.js | 83 +++++++++++------------------------- 2 files changed, 34 insertions(+), 59 deletions(-) diff --git a/services/budget/server.js b/services/budget/server.js index a61aed9..3cc116a 100644 --- a/services/budget/server.js +++ b/services/budget/server.js @@ -102,8 +102,12 @@ function currentMonth() { return `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`; } -/** Build lookup maps for payees, accounts, and categories. */ +/** Build lookup maps for payees, accounts, and categories (cached 2 min). */ +let _lookupsCache = { data: null, expiresAt: 0 }; async function buildLookups() { + if (_lookupsCache.data && Date.now() < _lookupsCache.expiresAt) { + return _lookupsCache.data; + } const [payees, accounts, categories] = await Promise.all([ api.getPayees(), api.getAccounts(), @@ -115,7 +119,9 @@ async function buildLookups() { for (const a of accounts) accountMap[a.id] = a.name; const categoryMap = {}; for (const c of categories) categoryMap[c.id] = c.name; - return { payeeMap, accountMap, categoryMap }; + const result = { payeeMap, accountMap, categoryMap }; + _lookupsCache = { data: result, expiresAt: Date.now() + 120000 }; + return result; } /** Enrich a transaction with resolved names. */ diff --git a/services/inventory/server.js b/services/inventory/server.js index d94b5b9..fb196e5 100755 --- a/services/inventory/server.js +++ b/services/inventory/server.js @@ -362,30 +362,24 @@ app.get('/summary', async (req, res) => { } }); -// Get items with issues (full details) +// Get items with issues (full details) — server-side filtered app.get('/issues', async (req, res) => { try { if (!config.apiToken) { return res.status(500).json({ error: 'NocoDB API token not configured' }); } - let allRows = []; - let offset = 0; - const limit = 1000; - let hasMore = true; - while (hasMore && offset < 10000) { - const response = await axios.get( - config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId, - { headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' }, params: { limit, offset } } - ); - const pageRows = response.data.list || []; - allRows = allRows.concat(pageRows); - hasMore = pageRows.length === limit; - offset += limit; - } - const issues = allRows.filter(row => { - const val = (row.Received || row.received || '').toLowerCase(); - return val === 'issues' || val === 'issue'; - }).map(row => ({ + const response = await axios.get( + config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId, + { + headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' }, + params: { + where: '(Received,eq,Issues)~or(Received,eq,Issue)', + limit: 200, + sort: '-Id' + } + } + ); + const issues = (response.data.list || []).map(row => ({ id: row.Id, item: row.Item || row.Name || 'Unknown', orderNumber: row['Order Number'] || '', @@ -403,50 +397,25 @@ app.get('/issues', async (req, res) => { }); // Get count of rows that have issues +// Count of items with issues — server-side filtered, single query app.get('/needs-review-count', async (req, res) => { try { if (!config.apiToken) { return res.status(500).json({ error: 'NocoDB API token not configured' }); } - - console.log('Fetching issues count...'); - - // Fetch all records with pagination - let allRows = []; - let offset = 0; - const limit = 1000; - let hasMore = true; - - while (hasMore && offset < 10000) { - const response = await axios.get( - config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId, - { - headers: { - 'xc-token': config.apiToken, - 'Accept': 'application/json' - }, - params: { - limit: limit, - offset: offset - } + const response = await axios.get( + config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId, + { + headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' }, + params: { + where: '(Received,eq,Issues)~or(Received,eq,Issue)~or(Received,eq,Needs Review)', + limit: 1, + fields: 'Id' } - ); - - const pageRows = response.data.list || []; - allRows = allRows.concat(pageRows); - hasMore = pageRows.length === limit; - offset += limit; - } - - console.log('API response received, total rows:', allRows.length); - const issuesCount = allRows.filter(row => { - const receivedValue = row.Received || row.received; - return receivedValue === 'Issues' || receivedValue === 'Issue'; - }).length; - - console.log('Found ' + issuesCount + ' items with issues'); - res.json({ count: issuesCount }); - + } + ); + const count = response.data.pageInfo?.totalRows || (response.data.list || []).length; + res.json({ count }); } catch (error) { console.error('Error fetching issues count:', error.message); res.status(500).json({ error: 'Failed to fetch count', details: error.message });