fix: performance hardening — eliminate full table scans (#9)
Inventory: - /issues: replaced full scan + client filter with NocoDB server-side WHERE filter (Received eq Issues/Issue). Single query, ~200 rows max. - /needs-review-count: replaced full scan with server-side WHERE + limit=1 + pageInfo.totalRows. Returns count without fetching data. Budget: - buildLookups(): added 2-minute cache for payee/account/category maps. Eliminates 3 API calls per request for repeated queries. - /summary cache (added earlier): 1-minute TTL still active. Files: services/inventory/server.js, services/budget/server.js
This commit is contained in:
@@ -102,8 +102,12 @@ function currentMonth() {
|
||||
return `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
/** Build lookup maps for payees, accounts, and categories. */
|
||||
/** Build lookup maps for payees, accounts, and categories (cached 2 min). */
|
||||
let _lookupsCache = { data: null, expiresAt: 0 };
|
||||
async function buildLookups() {
|
||||
if (_lookupsCache.data && Date.now() < _lookupsCache.expiresAt) {
|
||||
return _lookupsCache.data;
|
||||
}
|
||||
const [payees, accounts, categories] = await Promise.all([
|
||||
api.getPayees(),
|
||||
api.getAccounts(),
|
||||
@@ -115,7 +119,9 @@ async function buildLookups() {
|
||||
for (const a of accounts) accountMap[a.id] = a.name;
|
||||
const categoryMap = {};
|
||||
for (const c of categories) categoryMap[c.id] = c.name;
|
||||
return { payeeMap, accountMap, categoryMap };
|
||||
const result = { payeeMap, accountMap, categoryMap };
|
||||
_lookupsCache = { data: result, expiresAt: Date.now() + 120000 };
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Enrich a transaction with resolved names. */
|
||||
|
||||
@@ -362,30 +362,24 @@ app.get('/summary', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// Get items with issues (full details)
|
||||
// Get items with issues (full details) — server-side filtered
|
||||
app.get('/issues', async (req, res) => {
|
||||
try {
|
||||
if (!config.apiToken) {
|
||||
return res.status(500).json({ error: 'NocoDB API token not configured' });
|
||||
}
|
||||
let allRows = [];
|
||||
let offset = 0;
|
||||
const limit = 1000;
|
||||
let hasMore = true;
|
||||
while (hasMore && offset < 10000) {
|
||||
const response = await axios.get(
|
||||
config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId,
|
||||
{ headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' }, params: { limit, offset } }
|
||||
);
|
||||
const pageRows = response.data.list || [];
|
||||
allRows = allRows.concat(pageRows);
|
||||
hasMore = pageRows.length === limit;
|
||||
offset += limit;
|
||||
{
|
||||
headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' },
|
||||
params: {
|
||||
where: '(Received,eq,Issues)~or(Received,eq,Issue)',
|
||||
limit: 200,
|
||||
sort: '-Id'
|
||||
}
|
||||
const issues = allRows.filter(row => {
|
||||
const val = (row.Received || row.received || '').toLowerCase();
|
||||
return val === 'issues' || val === 'issue';
|
||||
}).map(row => ({
|
||||
}
|
||||
);
|
||||
const issues = (response.data.list || []).map(row => ({
|
||||
id: row.Id,
|
||||
item: row.Item || row.Name || 'Unknown',
|
||||
orderNumber: row['Order Number'] || '',
|
||||
@@ -403,50 +397,25 @@ app.get('/issues', async (req, res) => {
|
||||
});
|
||||
|
||||
// Get count of rows that have issues
|
||||
// Count of items with issues — server-side filtered, single query
|
||||
app.get('/needs-review-count', async (req, res) => {
|
||||
try {
|
||||
if (!config.apiToken) {
|
||||
return res.status(500).json({ error: 'NocoDB API token not configured' });
|
||||
}
|
||||
|
||||
console.log('Fetching issues count...');
|
||||
|
||||
// Fetch all records with pagination
|
||||
let allRows = [];
|
||||
let offset = 0;
|
||||
const limit = 1000;
|
||||
let hasMore = true;
|
||||
|
||||
while (hasMore && offset < 10000) {
|
||||
const response = await axios.get(
|
||||
config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId,
|
||||
{
|
||||
headers: {
|
||||
'xc-token': config.apiToken,
|
||||
'Accept': 'application/json'
|
||||
},
|
||||
headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' },
|
||||
params: {
|
||||
limit: limit,
|
||||
offset: offset
|
||||
where: '(Received,eq,Issues)~or(Received,eq,Issue)~or(Received,eq,Needs Review)',
|
||||
limit: 1,
|
||||
fields: 'Id'
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const pageRows = response.data.list || [];
|
||||
allRows = allRows.concat(pageRows);
|
||||
hasMore = pageRows.length === limit;
|
||||
offset += limit;
|
||||
}
|
||||
|
||||
console.log('API response received, total rows:', allRows.length);
|
||||
const issuesCount = allRows.filter(row => {
|
||||
const receivedValue = row.Received || row.received;
|
||||
return receivedValue === 'Issues' || receivedValue === 'Issue';
|
||||
}).length;
|
||||
|
||||
console.log('Found ' + issuesCount + ' items with issues');
|
||||
res.json({ count: issuesCount });
|
||||
|
||||
const count = response.data.pageInfo?.totalRows || (response.data.list || []).length;
|
||||
res.json({ count });
|
||||
} catch (error) {
|
||||
console.error('Error fetching issues count:', error.message);
|
||||
res.status(500).json({ error: 'Failed to fetch count', details: error.message });
|
||||
|
||||
Reference in New Issue
Block a user