mirror of https://github.com/msgbyte/tailchat
perf(admin-next): optimize mongodb's performance on statistical quantities
moonrailgun
2 years ago
parent
98e78e28f8
commit
11154f735b
@ -0,0 +1,4 @@
|
||||
fork from https://github.com/NathanAdhitya/express-mongoose-ra-json-server
|
||||
|
||||
modify:
|
||||
- count logic in get `/`
|
||||
@ -0,0 +1,259 @@
|
||||
import { RequestHandler, Router } from 'express';
|
||||
import { LeanDocument } from 'mongoose';
|
||||
import statusMessages from './statusMessages';
|
||||
import { ADPBaseModel, ADPBaseSchema } from './utils/baseModel.interface';
|
||||
import castFilter from './utils/castFilter';
|
||||
import convertId from './utils/convertId';
|
||||
import filterGetList from './utils/filterGetList';
|
||||
import { filterReadOnly } from './utils/filterReadOnly';
|
||||
import parseQuery from './utils/parseQuery';
|
||||
import virtualId from './utils/virtualId';
|
||||
|
||||
// Export certain helper functions for custom reuse.
|
||||
export { default as virtualId } from './utils/virtualId';
|
||||
export { default as convertId } from './utils/convertId';
|
||||
export { default as castFilter } from './utils/castFilter';
|
||||
export { default as parseQuery } from './utils/parseQuery';
|
||||
export { default as filterGetList } from './utils/filterGetList';
|
||||
export { filterReadOnly } from './utils/filterReadOnly';
|
||||
export { default as statusMessages } from './statusMessages';
|
||||
|
||||
export interface raExpressMongooseCapabilities {
|
||||
list?: boolean;
|
||||
get?: boolean;
|
||||
create?: boolean;
|
||||
update?: boolean;
|
||||
delete?: boolean;
|
||||
}
|
||||
|
||||
export interface raExpressMongooseOptions<T> {
|
||||
/** Fields to search from ?q (used for autofill and search) */
|
||||
q?: string[];
|
||||
|
||||
/** Base name for ACLs (e.g. list operation does baseName.list) */
|
||||
aclName?: string;
|
||||
|
||||
/** Fields to allow regex based search (non-exact search) */
|
||||
allowedRegexFields?: string[];
|
||||
|
||||
/** Read-only fields to filter out during create and update */
|
||||
readOnlyFields?: string[];
|
||||
|
||||
/** Function to transform inputs received in create and update */
|
||||
inputTransformer?: (input: Partial<T>) => Promise<Partial<T>>;
|
||||
|
||||
/** Additional queries for list, e.g. deleted/hidden flag. */
|
||||
listQuery?: Record<string, any>;
|
||||
|
||||
/** Max rows from a get operation to prevent accidental server suicide (default 100) */
|
||||
maxRows?: number;
|
||||
|
||||
/** Extra selects for mongoose queries (in the case that certain fields are hidden by default) */
|
||||
extraSelects?: string;
|
||||
|
||||
/** Disable or enable certain parts. */
|
||||
capabilities?: raExpressMongooseCapabilities;
|
||||
|
||||
/** Specify a custom express.js router */
|
||||
router?: Router;
|
||||
|
||||
/** Specify an ACL middleware to check against permissions */
|
||||
ACLMiddleware?: (name: string) => RequestHandler;
|
||||
}
|
||||
|
||||
export function raExpressMongoose<T extends ADPBaseModel, I>(
|
||||
model: T,
|
||||
options?: raExpressMongooseOptions<I>
|
||||
) {
|
||||
const {
|
||||
q,
|
||||
allowedRegexFields = [],
|
||||
readOnlyFields,
|
||||
inputTransformer = (input: any) => input,
|
||||
listQuery,
|
||||
extraSelects,
|
||||
maxRows = 100,
|
||||
capabilities,
|
||||
aclName,
|
||||
router = Router(),
|
||||
ACLMiddleware,
|
||||
} = options ?? {};
|
||||
|
||||
const {
|
||||
list: canList = true,
|
||||
get: canGet = true,
|
||||
create: canCreate = true,
|
||||
update: canUpdate = true,
|
||||
delete: canDelete = true,
|
||||
} = capabilities ?? {};
|
||||
|
||||
/** getList, getMany, getManyReference */
|
||||
if (canList)
|
||||
router.get(
|
||||
'/',
|
||||
aclName && ACLMiddleware
|
||||
? ACLMiddleware(`${aclName}.list`)
|
||||
: (req, res, next) => next(),
|
||||
async (req, res) => {
|
||||
const filterQuery = {
|
||||
...listQuery,
|
||||
...parseQuery(
|
||||
castFilter(
|
||||
convertId(filterGetList(req.query)),
|
||||
model,
|
||||
allowedRegexFields
|
||||
),
|
||||
model,
|
||||
allowedRegexFields,
|
||||
q
|
||||
),
|
||||
};
|
||||
let query = model.find(filterQuery);
|
||||
|
||||
if (req.query._sort && req.query._order)
|
||||
query = query.sort({
|
||||
[typeof req.query._sort === 'string'
|
||||
? req.query._sort === 'id'
|
||||
? '_id'
|
||||
: req.query._sort
|
||||
: '_id']: req.query._order === 'ASC' ? 1 : -1,
|
||||
});
|
||||
|
||||
if (req.query._start)
|
||||
query = query.skip(
|
||||
parseInt(
|
||||
typeof req.query._start === 'string' ? req.query._start : '0'
|
||||
)
|
||||
);
|
||||
|
||||
if (req.query._end)
|
||||
query = query.limit(
|
||||
Math.min(
|
||||
parseInt(
|
||||
typeof req.query._end === 'string' ? req.query._end : '0'
|
||||
) -
|
||||
(req.query._start
|
||||
? parseInt(
|
||||
typeof req.query._start === 'string'
|
||||
? req.query._start
|
||||
: '0'
|
||||
)
|
||||
: 0),
|
||||
maxRows
|
||||
)
|
||||
);
|
||||
else query = query.limit(maxRows);
|
||||
|
||||
if (extraSelects) query = query.select(extraSelects);
|
||||
|
||||
if (Object.keys(filterQuery).length === 0) {
|
||||
res.set(
|
||||
'X-Total-Count',
|
||||
(await model.estimatedDocumentCount()).toString()
|
||||
);
|
||||
} else {
|
||||
res.set(
|
||||
'X-Total-Count',
|
||||
(await model.countDocuments(filterQuery)).toString()
|
||||
);
|
||||
}
|
||||
|
||||
return res.json(
|
||||
virtualId((await query.lean()) as LeanDocument<ADPBaseSchema>)
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
/** getOne, getMany */
|
||||
if (canGet)
|
||||
router.get(
|
||||
'/:id',
|
||||
aclName && ACLMiddleware
|
||||
? ACLMiddleware(`${aclName}.list`)
|
||||
: (req, res, next) => next(),
|
||||
async (req, res) => {
|
||||
await model
|
||||
.findById(req.params.id)
|
||||
.select(extraSelects)
|
||||
.lean()
|
||||
.then((result) => res.json(virtualId(result)))
|
||||
.catch((e) => {
|
||||
return statusMessages.error(res, 400, e);
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
/** create */
|
||||
if (canCreate)
|
||||
router.post(
|
||||
'/',
|
||||
aclName && ACLMiddleware
|
||||
? ACLMiddleware(`${aclName}.create`)
|
||||
: (req, res, next) => next(),
|
||||
async (req, res) => {
|
||||
// eslint-disable-next-line new-cap
|
||||
const result = convertId(
|
||||
await inputTransformer(filterReadOnly<I>(req.body, readOnlyFields))
|
||||
);
|
||||
const newData = {
|
||||
...result,
|
||||
};
|
||||
|
||||
const newEntry = new model(newData);
|
||||
await newEntry
|
||||
.save()
|
||||
.then((result) => res.json(virtualId(result)))
|
||||
.catch((e: any) => {
|
||||
return statusMessages.error(res, 400, e, 'Bad request');
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
/** update */
|
||||
if (canUpdate)
|
||||
router.put(
|
||||
'/:id',
|
||||
aclName && ACLMiddleware
|
||||
? ACLMiddleware(`${aclName}.edit`)
|
||||
: (req, res, next) => next(),
|
||||
async (req, res) => {
|
||||
const updateData = {
|
||||
...(await convertId(
|
||||
await inputTransformer(filterReadOnly<I>(req.body, readOnlyFields))
|
||||
)),
|
||||
};
|
||||
|
||||
await model
|
||||
.findOneAndUpdate({ _id: req.params.id }, updateData, {
|
||||
new: true,
|
||||
runValidators: true,
|
||||
})
|
||||
.lean()
|
||||
.then((result) => res.json(virtualId(result)))
|
||||
.catch((e) => {
|
||||
return statusMessages.error(res, 400, e, 'Bad request');
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
/**
|
||||
* delete
|
||||
*/
|
||||
if (canDelete)
|
||||
router.delete(
|
||||
'/:id',
|
||||
aclName && ACLMiddleware
|
||||
? ACLMiddleware(`${aclName}.delete`)
|
||||
: (req, res, next) => next(),
|
||||
async (req, res) => {
|
||||
await model
|
||||
.findOneAndDelete({ _id: req.params.id })
|
||||
.then((result) => res.json(virtualId(result)))
|
||||
.catch((e) => {
|
||||
return statusMessages.error(res, 404, e, 'Element does not exist');
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
return router;
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
/**
|
||||
* @file statusMessages
|
||||
* @description handles status messages / error responses
|
||||
*/
|
||||
|
||||
import { Response } from 'express';
|
||||
|
||||
/**
|
||||
* Handles rejections other than errors. 400, 401, etc.
|
||||
*/
|
||||
function reject(res: Response, status: number, reason?: any) {
|
||||
return res.status(status).json({ message: reason ?? 'Invalid request' });
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles errors
|
||||
*/
|
||||
function error(res: Response, status: number, e: Error, message?: string) {
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
return res.status(status).json({ message, error: e.message });
|
||||
}
|
||||
}
|
||||
|
||||
export default { reject, error };
|
||||
@ -0,0 +1,7 @@
|
||||
import { Model, Document } from 'mongoose';
|
||||
|
||||
export interface ADPBaseSchema {
|
||||
_id: string;
|
||||
}
|
||||
|
||||
export type ADPBaseModel = Model<ADPBaseSchema & Document & any>;
|
||||
@ -0,0 +1,14 @@
|
||||
/** Turns id into _id for search queries */
|
||||
export default function convertId<T extends Record<string, unknown>>(obj: T) {
|
||||
if (obj.id) {
|
||||
const newObject = {
|
||||
_id: obj.id,
|
||||
...obj,
|
||||
};
|
||||
|
||||
delete newObject.id;
|
||||
return newObject;
|
||||
} else {
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
export const filterGetListParams = [
|
||||
'_sort',
|
||||
'_order',
|
||||
'_start',
|
||||
'_end',
|
||||
] as const;
|
||||
|
||||
/** Removes _sort, _order, _start, _end from a query. */
|
||||
export default function filterGetList<T extends Record<string, unknown>>(
|
||||
obj: T
|
||||
) {
|
||||
const filtered: any = {};
|
||||
Object.entries(obj).forEach(([index, value]) => {
|
||||
if (!filterGetListParams.includes(index as any)) filtered[index] = value;
|
||||
});
|
||||
return filtered as Omit<T, (typeof filterGetListParams)[number]>;
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
/** Makes sure that it does not modify crucial and sacred parts mutates the original object. */
|
||||
export function filterReadOnly<T extends {}>(
|
||||
obj: T,
|
||||
readOnlyFields?: string[]
|
||||
) {
|
||||
if (!readOnlyFields) return obj as T;
|
||||
|
||||
readOnlyFields.forEach((v) => {
|
||||
delete obj[v];
|
||||
});
|
||||
|
||||
return obj as Partial<T>;
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
import { ADPBaseModel } from './baseModel.interface';
|
||||
import castFilter from './castFilter';
|
||||
|
||||
interface parseQueryParam {
|
||||
q?: string;
|
||||
$or?: any;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns ?q into $or queries, deletes q
|
||||
* @param {Object} results Original object with the q field
|
||||
* @param {string[]} fields Fields to apply q to
|
||||
*/
|
||||
export default function parseQuery<
|
||||
T extends parseQueryParam,
|
||||
M extends ADPBaseModel
|
||||
>(
|
||||
result: T,
|
||||
model: M,
|
||||
allowedRegexes: string[],
|
||||
fields?: string[]
|
||||
): T & { $or?: any } {
|
||||
if (!fields) return result;
|
||||
if (result.q) {
|
||||
if (!Array.isArray(result.$or)) result.$or = [];
|
||||
fields.forEach((field) => {
|
||||
const newFilter = { [field]: result.q };
|
||||
result.$or.push(castFilter(newFilter, model, allowedRegexes));
|
||||
});
|
||||
delete result.q;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
export default function virtualId<T extends { _id: string }>(
|
||||
arr: T[]
|
||||
): Array<T & { id: string }>;
|
||||
export default function virtualId<T extends { _id: string }>(
|
||||
doc: T
|
||||
): T & { id: string };
|
||||
|
||||
/** Virtual ID (_id to id) for react-admin */
|
||||
export default function virtualId<T extends { _id: string }>(el: Array<T> | T) {
|
||||
if (Array.isArray(el)) {
|
||||
return el.map((e) => {
|
||||
return {
|
||||
id: e._id,
|
||||
...e,
|
||||
_id: undefined,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
return { id: el._id, ...el, _id: undefined };
|
||||
}
|
||||
Loading…
Reference in New Issue