fix: performance hardening — eliminate full table scans (#9)
Some checks failed
Security Checks / dependency-audit (push) Has been cancelled
Security Checks / secret-scanning (push) Has been cancelled
Security Checks / dockerfile-lint (push) Has been cancelled

Inventory:
- /issues: replaced full scan + client filter with NocoDB server-side
  WHERE filter (Received eq Issues/Issue). Single query, ~200 rows max.
- /needs-review-count: replaced full scan with server-side WHERE +
  limit=1 + pageInfo.totalRows. Returns count without fetching data.

Budget:
- buildLookups(): added 2-minute cache for payee/account/category maps.
  Eliminates 3 API calls per request for repeated queries.
- /summary cache (added earlier): 1-minute TTL still active.

Files: services/inventory/server.js, services/budget/server.js
This commit is contained in:
Yusuf Suleman
2026-03-29 13:50:07 -05:00
parent 7a7286ac1c
commit 9e13984b05
2 changed files with 34 additions and 59 deletions

View File

@@ -102,8 +102,12 @@ function currentMonth() {
return `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`;
}
/** Build lookup maps for payees, accounts, and categories. */
/** Build lookup maps for payees, accounts, and categories (cached 2 min). */
let _lookupsCache = { data: null, expiresAt: 0 };
async function buildLookups() {
if (_lookupsCache.data && Date.now() < _lookupsCache.expiresAt) {
return _lookupsCache.data;
}
const [payees, accounts, categories] = await Promise.all([
api.getPayees(),
api.getAccounts(),
@@ -115,7 +119,9 @@ async function buildLookups() {
for (const a of accounts) accountMap[a.id] = a.name;
const categoryMap = {};
for (const c of categories) categoryMap[c.id] = c.name;
return { payeeMap, accountMap, categoryMap };
const result = { payeeMap, accountMap, categoryMap };
_lookupsCache = { data: result, expiresAt: Date.now() + 120000 };
return result;
}
/** Enrich a transaction with resolved names. */

View File

@@ -362,30 +362,24 @@ app.get('/summary', async (req, res) => {
}
});
// Get items with issues (full details)
// Get items with issues (full details) — server-side filtered
app.get('/issues', async (req, res) => {
try {
if (!config.apiToken) {
return res.status(500).json({ error: 'NocoDB API token not configured' });
}
let allRows = [];
let offset = 0;
const limit = 1000;
let hasMore = true;
while (hasMore && offset < 10000) {
const response = await axios.get(
config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId,
{ headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' }, params: { limit, offset } }
);
const pageRows = response.data.list || [];
allRows = allRows.concat(pageRows);
hasMore = pageRows.length === limit;
offset += limit;
{
headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' },
params: {
where: '(Received,eq,Issues)~or(Received,eq,Issue)',
limit: 200,
sort: '-Id'
}
const issues = allRows.filter(row => {
const val = (row.Received || row.received || '').toLowerCase();
return val === 'issues' || val === 'issue';
}).map(row => ({
}
);
const issues = (response.data.list || []).map(row => ({
id: row.Id,
item: row.Item || row.Name || 'Unknown',
orderNumber: row['Order Number'] || '',
@@ -403,50 +397,25 @@ app.get('/issues', async (req, res) => {
});
// Get count of rows that have issues
// Count of items with issues — server-side filtered, single query
app.get('/needs-review-count', async (req, res) => {
try {
if (!config.apiToken) {
return res.status(500).json({ error: 'NocoDB API token not configured' });
}
console.log('Fetching issues count...');
// Fetch all records with pagination
let allRows = [];
let offset = 0;
const limit = 1000;
let hasMore = true;
while (hasMore && offset < 10000) {
const response = await axios.get(
config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId,
{
headers: {
'xc-token': config.apiToken,
'Accept': 'application/json'
},
headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' },
params: {
limit: limit,
offset: offset
where: '(Received,eq,Issues)~or(Received,eq,Issue)~or(Received,eq,Needs Review)',
limit: 1,
fields: 'Id'
}
}
);
const pageRows = response.data.list || [];
allRows = allRows.concat(pageRows);
hasMore = pageRows.length === limit;
offset += limit;
}
console.log('API response received, total rows:', allRows.length);
const issuesCount = allRows.filter(row => {
const receivedValue = row.Received || row.received;
return receivedValue === 'Issues' || receivedValue === 'Issue';
}).length;
console.log('Found ' + issuesCount + ' items with issues');
res.json({ count: issuesCount });
const count = response.data.pageInfo?.totalRows || (response.data.list || []).length;
res.json({ count });
} catch (error) {
console.error('Error fetching issues count:', error.message);
res.status(500).json({ error: 'Failed to fetch count', details: error.message });