fix: performance hardening — eliminate full table scans (#9)
Inventory: - /issues: replaced full scan + client filter with NocoDB server-side WHERE filter (Received eq Issues/Issue). Single query, ~200 rows max. - /needs-review-count: replaced full scan with server-side WHERE + limit=1 + pageInfo.totalRows. Returns count without fetching data. Budget: - buildLookups(): added 2-minute cache for payee/account/category maps. Eliminates 3 API calls per request for repeated queries. - /summary cache (added earlier): 1-minute TTL still active. Files: services/inventory/server.js, services/budget/server.js
This commit is contained in:
@@ -102,8 +102,12 @@ function currentMonth() {
|
|||||||
return `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`;
|
return `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Build lookup maps for payees, accounts, and categories. */
|
/** Build lookup maps for payees, accounts, and categories (cached 2 min). */
|
||||||
|
let _lookupsCache = { data: null, expiresAt: 0 };
|
||||||
async function buildLookups() {
|
async function buildLookups() {
|
||||||
|
if (_lookupsCache.data && Date.now() < _lookupsCache.expiresAt) {
|
||||||
|
return _lookupsCache.data;
|
||||||
|
}
|
||||||
const [payees, accounts, categories] = await Promise.all([
|
const [payees, accounts, categories] = await Promise.all([
|
||||||
api.getPayees(),
|
api.getPayees(),
|
||||||
api.getAccounts(),
|
api.getAccounts(),
|
||||||
@@ -115,7 +119,9 @@ async function buildLookups() {
|
|||||||
for (const a of accounts) accountMap[a.id] = a.name;
|
for (const a of accounts) accountMap[a.id] = a.name;
|
||||||
const categoryMap = {};
|
const categoryMap = {};
|
||||||
for (const c of categories) categoryMap[c.id] = c.name;
|
for (const c of categories) categoryMap[c.id] = c.name;
|
||||||
return { payeeMap, accountMap, categoryMap };
|
const result = { payeeMap, accountMap, categoryMap };
|
||||||
|
_lookupsCache = { data: result, expiresAt: Date.now() + 120000 };
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Enrich a transaction with resolved names. */
|
/** Enrich a transaction with resolved names. */
|
||||||
|
|||||||
@@ -362,30 +362,24 @@ app.get('/summary', async (req, res) => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Get items with issues (full details)
|
// Get items with issues (full details) — server-side filtered
|
||||||
app.get('/issues', async (req, res) => {
|
app.get('/issues', async (req, res) => {
|
||||||
try {
|
try {
|
||||||
if (!config.apiToken) {
|
if (!config.apiToken) {
|
||||||
return res.status(500).json({ error: 'NocoDB API token not configured' });
|
return res.status(500).json({ error: 'NocoDB API token not configured' });
|
||||||
}
|
}
|
||||||
let allRows = [];
|
|
||||||
let offset = 0;
|
|
||||||
const limit = 1000;
|
|
||||||
let hasMore = true;
|
|
||||||
while (hasMore && offset < 10000) {
|
|
||||||
const response = await axios.get(
|
const response = await axios.get(
|
||||||
config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId,
|
config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId,
|
||||||
{ headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' }, params: { limit, offset } }
|
{
|
||||||
);
|
headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' },
|
||||||
const pageRows = response.data.list || [];
|
params: {
|
||||||
allRows = allRows.concat(pageRows);
|
where: '(Received,eq,Issues)~or(Received,eq,Issue)',
|
||||||
hasMore = pageRows.length === limit;
|
limit: 200,
|
||||||
offset += limit;
|
sort: '-Id'
|
||||||
}
|
}
|
||||||
const issues = allRows.filter(row => {
|
}
|
||||||
const val = (row.Received || row.received || '').toLowerCase();
|
);
|
||||||
return val === 'issues' || val === 'issue';
|
const issues = (response.data.list || []).map(row => ({
|
||||||
}).map(row => ({
|
|
||||||
id: row.Id,
|
id: row.Id,
|
||||||
item: row.Item || row.Name || 'Unknown',
|
item: row.Item || row.Name || 'Unknown',
|
||||||
orderNumber: row['Order Number'] || '',
|
orderNumber: row['Order Number'] || '',
|
||||||
@@ -403,50 +397,25 @@ app.get('/issues', async (req, res) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Get count of rows that have issues
|
// Get count of rows that have issues
|
||||||
|
// Count of items with issues — server-side filtered, single query
|
||||||
app.get('/needs-review-count', async (req, res) => {
|
app.get('/needs-review-count', async (req, res) => {
|
||||||
try {
|
try {
|
||||||
if (!config.apiToken) {
|
if (!config.apiToken) {
|
||||||
return res.status(500).json({ error: 'NocoDB API token not configured' });
|
return res.status(500).json({ error: 'NocoDB API token not configured' });
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('Fetching issues count...');
|
|
||||||
|
|
||||||
// Fetch all records with pagination
|
|
||||||
let allRows = [];
|
|
||||||
let offset = 0;
|
|
||||||
const limit = 1000;
|
|
||||||
let hasMore = true;
|
|
||||||
|
|
||||||
while (hasMore && offset < 10000) {
|
|
||||||
const response = await axios.get(
|
const response = await axios.get(
|
||||||
config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId,
|
config.ncodbUrl + '/api/v1/db/data/noco/' + config.baseId + '/' + config.tableId,
|
||||||
{
|
{
|
||||||
headers: {
|
headers: { 'xc-token': config.apiToken, 'Accept': 'application/json' },
|
||||||
'xc-token': config.apiToken,
|
|
||||||
'Accept': 'application/json'
|
|
||||||
},
|
|
||||||
params: {
|
params: {
|
||||||
limit: limit,
|
where: '(Received,eq,Issues)~or(Received,eq,Issue)~or(Received,eq,Needs Review)',
|
||||||
offset: offset
|
limit: 1,
|
||||||
|
fields: 'Id'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
const count = response.data.pageInfo?.totalRows || (response.data.list || []).length;
|
||||||
const pageRows = response.data.list || [];
|
res.json({ count });
|
||||||
allRows = allRows.concat(pageRows);
|
|
||||||
hasMore = pageRows.length === limit;
|
|
||||||
offset += limit;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('API response received, total rows:', allRows.length);
|
|
||||||
const issuesCount = allRows.filter(row => {
|
|
||||||
const receivedValue = row.Received || row.received;
|
|
||||||
return receivedValue === 'Issues' || receivedValue === 'Issue';
|
|
||||||
}).length;
|
|
||||||
|
|
||||||
console.log('Found ' + issuesCount + ' items with issues');
|
|
||||||
res.json({ count: issuesCount });
|
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error fetching issues count:', error.message);
|
console.error('Error fetching issues count:', error.message);
|
||||||
res.status(500).json({ error: 'Failed to fetch count', details: error.message });
|
res.status(500).json({ error: 'Failed to fetch count', details: error.message });
|
||||||
|
|||||||
Reference in New Issue
Block a user