Commit daf626ee authored by ThinhNC's avatar ThinhNC

Merge branch 'feat/system-config-and-feature-flags' into 'develop'

feat(system-config): implement dynamic configuration module and feature flags

See merge request !16
parents f5821c6b 0b96ce87
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
"db:migrate:reset": "node scripts/prisma-run.js migrate reset", "db:migrate:reset": "node scripts/prisma-run.js migrate reset",
"db:migrate:status": "node scripts/prisma-run.js migrate status", "db:migrate:status": "node scripts/prisma-run.js migrate status",
"db:seed": "node scripts/prisma-run.js db seed -- --tsx prisma/seed.ts", "db:seed": "node scripts/prisma-run.js db seed -- --tsx prisma/seed.ts",
"config:sync": "tsx scripts/sync-system-configs.ts",
"lint": "eslint .", "lint": "eslint .",
"format": "prettier --write .", "format": "prettier --write .",
"test": "jest" "test": "jest"
......
-- CreateTable
CREATE TABLE "system_configs" (
"id" UUID NOT NULL,
"key" TEXT NOT NULL,
"value" JSONB NOT NULL,
"description" TEXT,
"category" TEXT NOT NULL DEFAULT 'GENERAL',
"is_public" BOOLEAN NOT NULL DEFAULT false,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "system_configs_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "system_configs_key_key" ON "system_configs"("key");
-- CreateIndex
CREATE INDEX "system_configs_category_idx" ON "system_configs"("category");
-- CreateIndex
CREATE INDEX "system_configs_is_public_idx" ON "system_configs"("is_public");
...@@ -468,3 +468,18 @@ model RolePermission { ...@@ -468,3 +468,18 @@ model RolePermission {
@@map("role_permissions") @@map("role_permissions")
} }
model SystemConfig {
id String @id @default(uuid()) @db.Uuid
key String @unique
value Json
description String?
category String @default("GENERAL")
isPublic Boolean @default(false) @map("is_public")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
@@index([category])
@@index([isPublic])
@@map("system_configs")
}
...@@ -17,6 +17,7 @@ import { ...@@ -17,6 +17,7 @@ import {
SYSTEM_PERMISSIONS_CATALOG, SYSTEM_PERMISSIONS_CATALOG,
SYSTEM_ROLE_DEFAULT_PERMISSIONS, SYSTEM_ROLE_DEFAULT_PERMISSIONS,
} from "../src/common/constants/permission.constant"; } from "../src/common/constants/permission.constant";
import { DEFAULT_SYSTEM_CONFIGS } from "../src/common/constants/system-config.constant";
const prisma = new PrismaClient(); const prisma = new PrismaClient();
...@@ -597,10 +598,32 @@ async function seedCrawlJobsAndPages(userId: string) { ...@@ -597,10 +598,32 @@ async function seedCrawlJobsAndPages(userId: string) {
}); });
} }
async function seedSystemConfigs(): Promise<void> {
console.log("Seeding default system configs...");
for (const item of DEFAULT_SYSTEM_CONFIGS) {
await prisma.systemConfig.upsert({
where: { key: item.key },
update: {
description: item.description ?? null,
category: item.category,
isPublic: item.isPublic,
},
create: {
key: item.key,
value: item.value as any,
description: item.description ?? null,
category: item.category,
isPublic: item.isPublic,
},
});
}
}
async function main() { async function main() {
const permissionMap = await seedPermissions(); const permissionMap = await seedPermissions();
const roleMap = await seedRoles(permissionMap); const roleMap = await seedRoles(permissionMap);
await seedUsers(roleMap); await seedUsers(roleMap);
await seedSystemConfigs();
const crawlerUser = await prisma.user.findUnique({ const crawlerUser = await prisma.user.findUnique({
where: { email: "crawl@crawl.local" }, where: { email: "crawl@crawl.local" },
......
import "dotenv/config";
import { PrismaClient } from "@prisma/client";
import { DEFAULT_SYSTEM_CONFIGS } from "../src/common/constants/system-config.constant";
const prisma = new PrismaClient();
async function main() {
console.log(`Synchronizing ${DEFAULT_SYSTEM_CONFIGS.length} system configs to database...`);
let upserted = 0;
for (const item of DEFAULT_SYSTEM_CONFIGS) {
await prisma.systemConfig.upsert({
where: { key: item.key },
update: {
description: item.description ?? null,
category: item.category,
isPublic: item.isPublic,
},
create: {
key: item.key,
value: item.value as any,
description: item.description ?? null,
category: item.category,
isPublic: item.isPublic,
},
});
upserted++;
console.log(`[✔] Upserted: ${item.key} (${item.category}, public: ${item.isPublic})`);
}
const allowedKeys = new Set(DEFAULT_SYSTEM_CONFIGS.map((c) => c.key));
const deleteResult = await prisma.systemConfig.deleteMany({
where: {
key: {
notIn: Array.from(allowedKeys),
},
},
});
if (deleteResult.count > 0) {
console.log(`[x] Cleaned up ${deleteResult.count} obsolete/sensitive keys from database.`);
}
const allConfigs = await prisma.systemConfig.findMany({
orderBy: [{ category: "asc" }, { key: "asc" }],
select: { key: true, category: true, isPublic: true, value: true, description: true },
});
console.log(`\n============================ SUMMARY ============================`);
console.log(`Total configs in DB: ${allConfigs.length}`);
console.log(`Upserted configs: ${upserted}`);
const categories = Array.from(new Set(allConfigs.map((c) => c.category)));
for (const cat of categories) {
const inCat = allConfigs.filter((c) => c.category === cat);
console.log(`Category [${cat}]: ${inCat.length} configs`);
}
}
main()
.catch((err) => {
console.error("Sync error:", err);
process.exit(1);
})
.finally(async () => {
await prisma.$disconnect();
});
...@@ -12,6 +12,7 @@ import routes from "./routes"; ...@@ -12,6 +12,7 @@ import routes from "./routes";
import swaggerDocument from "./docs/swagger.json"; import swaggerDocument from "./docs/swagger.json";
import healthRoute from "./modules/health/health.route"; import healthRoute from "./modules/health/health.route";
import { rateLimitMiddleware } from "./middlewares/rate-limit.middleware"; import { rateLimitMiddleware } from "./middlewares/rate-limit.middleware";
import { maintenanceMiddleware } from "./middlewares/maintenance.middleware";
import { envConfig } from "./config/env.config"; import { envConfig } from "./config/env.config";
import { parseTrustProxy } from "./common/helpers/proxy.helper"; import { parseTrustProxy } from "./common/helpers/proxy.helper";
...@@ -46,7 +47,7 @@ app.use(express.urlencoded({ extended: true })); ...@@ -46,7 +47,7 @@ app.use(express.urlencoded({ extended: true }));
app.use("/health", healthRoute); app.use("/health", healthRoute);
app.use("/api-docs", swaggerUi.serve, swaggerUi.setup(swaggerDocument)); app.use("/api-docs", swaggerUi.serve, swaggerUi.setup(swaggerDocument));
app.use("/api/v1", rateLimitMiddleware, routes); app.use("/api/v1", rateLimitMiddleware, maintenanceMiddleware, routes);
app.use(notFoundMiddleware); app.use(notFoundMiddleware);
app.use(errorMiddleware); app.use(errorMiddleware);
......
...@@ -32,6 +32,10 @@ export const AUDIT_ACTIONS = { ...@@ -32,6 +32,10 @@ export const AUDIT_ACTIONS = {
PERMISSION_REVOKED: "PERMISSION_REVOKED", PERMISSION_REVOKED: "PERMISSION_REVOKED",
SUPER_ADMIN_ASSIGN_ATTEMPT: "SUPER_ADMIN_ASSIGN_ATTEMPT", SUPER_ADMIN_ASSIGN_ATTEMPT: "SUPER_ADMIN_ASSIGN_ATTEMPT",
PRIVILEGE_ESCALATION_BLOCKED: "PRIVILEGE_ESCALATION_BLOCKED", PRIVILEGE_ESCALATION_BLOCKED: "PRIVILEGE_ESCALATION_BLOCKED",
SYSTEM_CONFIG_CREATED: "SYSTEM_CONFIG_CREATED",
SYSTEM_CONFIG_UPDATED: "SYSTEM_CONFIG_UPDATED",
SYSTEM_CONFIG_TOGGLED: "SYSTEM_CONFIG_TOGGLED",
SYSTEM_CONFIG_DELETED: "SYSTEM_CONFIG_DELETED",
} as const; } as const;
export type AuditAction = (typeof AUDIT_ACTIONS)[keyof typeof AUDIT_ACTIONS]; export type AuditAction = (typeof AUDIT_ACTIONS)[keyof typeof AUDIT_ACTIONS];
...@@ -12,3 +12,4 @@ export * from "./crawl-page-status.constant"; ...@@ -12,3 +12,4 @@ export * from "./crawl-page-status.constant";
export * from "./webhook.constant"; export * from "./webhook.constant";
export * from "./system-role.constant"; export * from "./system-role.constant";
export * from "./permission.constant"; export * from "./permission.constant";
export * from "./system-config.constant";
...@@ -68,6 +68,10 @@ export const PERMISSIONS = { ...@@ -68,6 +68,10 @@ export const PERMISSIONS = {
// Dashboard // Dashboard
DASHBOARD_READ: "dashboard.read", DASHBOARD_READ: "dashboard.read",
DASHBOARD_READ_ALL: "dashboard.read_all", DASHBOARD_READ_ALL: "dashboard.read_all",
// System Configs
SYSTEM_CONFIG_READ: "system_configs.read",
SYSTEM_CONFIG_MANAGE: "system_configs.manage",
} as const; } as const;
export type PermissionSlug = (typeof PERMISSIONS)[keyof typeof PERMISSIONS]; export type PermissionSlug = (typeof PERMISSIONS)[keyof typeof PERMISSIONS];
...@@ -471,6 +475,24 @@ export const SYSTEM_PERMISSIONS_CATALOG: PermissionDefinition[] = [ ...@@ -471,6 +475,24 @@ export const SYSTEM_PERMISSIONS_CATALOG: PermissionDefinition[] = [
action: "read_all", action: "read_all",
isSystem: true, isSystem: true,
}, },
// System Configs
{
name: "View System Configs",
slug: PERMISSIONS.SYSTEM_CONFIG_READ,
description: "Xem danh sách và chi tiết cấu hình hệ thống & feature flags",
resource: "system_configs",
action: "read",
isSystem: true,
},
{
name: "Manage System Configs",
slug: PERMISSIONS.SYSTEM_CONFIG_MANAGE,
description: "Thêm, cập nhật, bật/tắt hoặc xóa cấu hình hệ thống & feature flags",
resource: "system_configs",
action: "manage",
isSystem: true,
},
]; ];
export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record< export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
...@@ -525,6 +547,8 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record< ...@@ -525,6 +547,8 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
PERMISSIONS.AUDIT_LOGS_READ, PERMISSIONS.AUDIT_LOGS_READ,
PERMISSIONS.DASHBOARD_READ, PERMISSIONS.DASHBOARD_READ,
PERMISSIONS.DASHBOARD_READ_ALL, PERMISSIONS.DASHBOARD_READ_ALL,
PERMISSIONS.SYSTEM_CONFIG_READ,
PERMISSIONS.SYSTEM_CONFIG_MANAGE,
], ],
[SYSTEM_ROLE_SLUGS.CRAWLER_USER]: [ [SYSTEM_ROLE_SLUGS.CRAWLER_USER]: [
PERMISSIONS.CRAWL_JOBS_CREATE, PERMISSIONS.CRAWL_JOBS_CREATE,
......
This diff is collapsed.
import Redis from "ioredis";
import { envConfig } from "../../config/env.config";
let publisherClient: Redis | null = null;
let subscriberClient: Redis | null = null;
export function getRedisPublisher(): Redis | null {
if (!envConfig.redis.enabled) return null;
if (!publisherClient) {
try {
publisherClient = new Redis({
host: envConfig.redis.host,
port: envConfig.redis.port,
maxRetriesPerRequest: 1,
lazyConnect: true,
connectTimeout: 2000,
retryStrategy: () => null,
enableOfflineQueue: false,
});
publisherClient.on("error", () => {
// Suppress unhandled redis error crashes
});
} catch {
publisherClient = null;
}
}
return publisherClient;
}
export function getRedisSubscriber(): Redis | null {
if (!envConfig.redis.enabled) return null;
if (!subscriberClient) {
try {
subscriberClient = new Redis({
host: envConfig.redis.host,
port: envConfig.redis.port,
maxRetriesPerRequest: 1,
lazyConnect: true,
connectTimeout: 2000,
retryStrategy: () => null,
enableOfflineQueue: false,
});
subscriberClient.on("error", () => {
// Suppress unhandled redis error crashes
});
} catch {
subscriberClient = null;
}
}
return subscriberClient;
}
...@@ -3443,4 +3443,251 @@ export const swaggerPaths: Record<string, any> = { ...@@ -3443,4 +3443,251 @@ export const swaggerPaths: Record<string, any> = {
}, },
}, },
}, },
"/system/public": {
get: {
tags: ["System Config"],
summary: "Lấy cấu hình công khai và Feature Flags",
description: "Cho phép client/frontend đọc toàn bộ cấu hình có isPublic: true mà không cần đăng nhập.",
responses: {
200: {
description: "Lấy cấu hình công khai thành công",
content: {
"application/json": {
schema: {
type: "object",
properties: {
success: { type: "boolean", example: true },
data: {
type: "object",
properties: {
configs: {
type: "array",
items: { $ref: "#/components/schemas/SystemConfig" },
},
map: { type: "object" },
},
},
},
},
},
},
},
},
},
},
"/system/configs": {
get: {
tags: ["System Config"],
summary: "Danh sách cấu hình hệ thống",
description: "Lấy danh sách cấu hình và cờ tính năng, hỗ trợ tìm kiếm và lọc theo danh mục (Yêu cầu quyền SYSTEM_CONFIG_READ).",
parameters: [
{
name: "category",
in: "query",
schema: {
type: "string",
enum: ["GENERAL", "FEATURE_FLAG", "INTEGRATION", "SECURITY"],
},
description: "Lọc theo danh mục cấu hình",
},
{
name: "search",
in: "query",
schema: { type: "string" },
description: "Tìm kiếm theo khóa hoặc mô tả",
},
{
name: "page",
in: "query",
schema: { type: "integer", default: 1 },
},
{
name: "limit",
in: "query",
schema: { type: "integer", default: 20 },
},
],
responses: {
200: {
description: "Lấy danh sách thành công",
content: {
"application/json": {
schema: {
type: "object",
properties: {
success: { type: "boolean", example: true },
data: {
type: "object",
properties: {
items: {
type: "array",
items: { $ref: "#/components/schemas/SystemConfig" },
},
total: { type: "integer" },
page: { type: "integer" },
limit: { type: "integer" },
},
},
},
},
},
},
},
401: { description: "Chưa xác thực" },
403: { description: "Không có quyền SYSTEM_CONFIG_READ" },
},
},
post: {
tags: ["System Config"],
summary: "Tạo cấu hình mới",
description: "Tạo mới một khóa cấu hình hoặc Feature Flag (Yêu cầu quyền SYSTEM_CONFIG_MANAGE).",
requestBody: {
required: true,
content: {
"application/json": {
schema: { $ref: "#/components/schemas/CreateSystemConfigRequest" },
},
},
},
responses: {
201: {
description: "Tạo cấu hình thành công",
content: {
"application/json": {
schema: {
type: "object",
properties: {
success: { type: "boolean", example: true },
data: { $ref: "#/components/schemas/SystemConfig" },
message: { type: "string" },
},
},
},
},
},
400: { description: "Dữ liệu không hợp lệ" },
409: { description: "Khóa cấu hình đã tồn tại" },
},
},
},
"/system/configs/{key}": {
get: {
tags: ["System Config"],
summary: "Chi tiết một cấu hình",
parameters: [
{
name: "key",
in: "path",
required: true,
schema: { type: "string" },
description: "Khóa định danh cấu hình",
},
],
responses: {
200: {
description: "Lấy chi tiết thành công",
content: {
"application/json": {
schema: {
type: "object",
properties: {
success: { type: "boolean", example: true },
data: { $ref: "#/components/schemas/SystemConfig" },
},
},
},
},
},
404: { description: "Không tìm thấy cấu hình" },
},
},
put: {
tags: ["System Config"],
summary: "Cập nhật cấu hình",
parameters: [
{
name: "key",
in: "path",
required: true,
schema: { type: "string" },
},
],
requestBody: {
required: true,
content: {
"application/json": {
schema: { $ref: "#/components/schemas/UpdateSystemConfigRequest" },
},
},
},
responses: {
200: {
description: "Cập nhật thành công",
content: {
"application/json": {
schema: {
type: "object",
properties: {
success: { type: "boolean", example: true },
data: { $ref: "#/components/schemas/SystemConfig" },
},
},
},
},
},
404: { description: "Không tìm thấy cấu hình" },
},
},
delete: {
tags: ["System Config"],
summary: "Xóa cấu hình",
parameters: [
{
name: "key",
in: "path",
required: true,
schema: { type: "string" },
},
],
responses: {
200: { description: "Xóa thành công" },
404: { description: "Không tìm thấy cấu hình" },
},
},
},
"/system/features/{key}/toggle": {
patch: {
tags: ["System Config"],
summary: "Bật/tắt nhanh Feature Flag",
description: "Chuyển đổi trạng thái boolean (true <-> false) cho một cờ tính năng (Yêu cầu quyền SYSTEM_CONFIG_MANAGE).",
parameters: [
{
name: "key",
in: "path",
required: true,
schema: { type: "string" },
},
],
responses: {
200: {
description: "Chuyển đổi trạng thái thành công",
content: {
"application/json": {
schema: {
type: "object",
properties: {
success: { type: "boolean", example: true },
data: { $ref: "#/components/schemas/SystemConfig" },
message: { type: "string" },
},
},
},
},
},
400: { description: "Cấu hình không phải boolean Feature Flag" },
404: { description: "Không tìm thấy cấu hình" },
},
},
},
}; };
This diff is collapsed.
...@@ -811,6 +811,50 @@ const rawSchemas = { ...@@ -811,6 +811,50 @@ const rawSchemas = {
}, },
}, },
}, },
SystemConfig: {
type: "object",
properties: {
id: { type: "string", format: "uuid" },
key: { type: "string", example: "feature.ai.enabled" },
value: { example: true },
description: { type: "string", nullable: true, example: "Kích hoạt AI" },
category: {
type: "string",
enum: ["GENERAL", "FEATURE_FLAG", "INTEGRATION", "SECURITY"],
example: "FEATURE_FLAG",
},
isPublic: { type: "boolean", example: true },
createdAt: { type: "string", format: "date-time" },
updatedAt: { type: "string", format: "date-time" },
},
},
CreateSystemConfigRequest: {
type: "object",
required: ["key", "value"],
properties: {
key: { type: "string", example: "feature.new_module.enabled" },
value: { example: true },
description: { type: "string", example: "Bật tắt tính năng mới" },
category: {
type: "string",
enum: ["GENERAL", "FEATURE_FLAG", "INTEGRATION", "SECURITY"],
example: "FEATURE_FLAG",
},
isPublic: { type: "boolean", example: false },
},
},
UpdateSystemConfigRequest: {
type: "object",
properties: {
value: { example: false },
description: { type: "string", example: "Mô tả mới" },
category: {
type: "string",
enum: ["GENERAL", "FEATURE_FLAG", "INTEGRATION", "SECURITY"],
},
isPublic: { type: "boolean" },
},
},
}; };
const outputFile = "./src/docs/swagger.json"; const outputFile = "./src/docs/swagger.json";
......
import { Request, Response, NextFunction } from "express";
import { systemConfigService } from "../modules/system-config/system-config.service";
import { ERROR_CODE } from "../common/errors/error-code";
/**
* Middleware kiểm tra chế độ bảo trì toàn hệ thống (feature.maintenance_mode.enabled)
* Khi bảo trì được bật, chặn các request từ người dùng thông thường,
* ngoại trừ các endpoint quản trị cấu hình, đăng nhập và health check.
*/
export async function maintenanceMiddleware(
req: Request,
res: Response,
next: NextFunction,
) {
// Bỏ qua các endpoint thiết yếu để Admin vẫn có thể đăng nhập và tắt chế độ bảo trì
const publicPaths = [
"/system",
"/auth/login",
"/auth/refresh",
"/api-docs",
"/health",
];
const isPublicOrAdminExempt = publicPaths.some(
(prefix) => req.path === prefix || req.path.startsWith(prefix + "/"),
);
if (isPublicOrAdminExempt) {
return next();
}
const isMaintenanceMode = await systemConfigService.isFeatureEnabled(
"feature.maintenance_mode.enabled",
false,
);
if (isMaintenanceMode) {
// Nếu là Admin thì cho phép qua
const user = (req as any).user;
if (user?.role === "ADMIN") {
return next();
}
return res.status(503).json({
success: false,
message:
"Hệ thống đang trong chế độ bảo trì định kỳ để nâng cấp. Vui lòng quay lại sau ít phút.",
code: ERROR_CODE.INTERNAL_SERVER_ERROR,
});
}
next();
}
import rateLimit, { RateLimitRequestHandler } from "express-rate-limit"; import rateLimit, { RateLimitRequestHandler } from "express-rate-limit";
import { envConfig } from "../config/env.config"; import { envConfig } from "../config/env.config";
import { ERROR_CODE } from "../common/errors/error-code"; import { ERROR_CODE } from "../common/errors/error-code";
import { systemConfigService } from "../modules/system-config/system-config.service";
/** /**
* Global API rate limit per IP, configurable for each environment. * Global API rate limit per IP, configurable for each environment.
...@@ -9,7 +10,11 @@ import { ERROR_CODE } from "../common/errors/error-code"; ...@@ -9,7 +10,11 @@ import { ERROR_CODE } from "../common/errors/error-code";
*/ */
export const rateLimitMiddleware: RateLimitRequestHandler = rateLimit({ export const rateLimitMiddleware: RateLimitRequestHandler = rateLimit({
windowMs: envConfig.rateLimit.windowMs, windowMs: envConfig.rateLimit.windowMs,
max: envConfig.rateLimit.max, max: async () =>
systemConfigService.get<number>(
"rate_limit.max_requests",
envConfig.rateLimit.max,
),
standardHeaders: true, standardHeaders: true,
legacyHeaders: false, legacyHeaders: false,
message: { message: {
......
...@@ -32,6 +32,7 @@ import { ...@@ -32,6 +32,7 @@ import {
createUtcDateFromZonedParts, createUtcDateFromZonedParts,
} from "../../common/helpers/schedule-calculator.helper"; } from "../../common/helpers/schedule-calculator.helper";
import { PermissionService } from "../permissions/permission.service"; import { PermissionService } from "../permissions/permission.service";
import { systemConfigService } from "../system-config/system-config.service";
interface AuthJwtPayload { interface AuthJwtPayload {
id: string; id: string;
...@@ -279,6 +280,18 @@ export class AuthService { ...@@ -279,6 +280,18 @@ export class AuthService {
} }
async register(data: RegisterDto): Promise<MeDto> { async register(data: RegisterDto): Promise<MeDto> {
const isRegistrationEnabled = await systemConfigService.isFeatureEnabled(
"feature.registration.enabled",
true,
);
if (!isRegistrationEnabled) {
throw new AppError(
"Tính năng đăng ký tài khoản hiện đang tạm khóa bởi Quản trị viên.",
403,
ERROR_CODE.FORBIDDEN,
);
}
const existing = await this.repository.findByEmail(data.email); const existing = await this.repository.findByEmail(data.email);
if (existing) { if (existing) {
......
import FirecrawlApp from "@mendable/firecrawl-js"; import FirecrawlApp from "@mendable/firecrawl-js";
import { firecrawlConfig } from "../../config/firecrawl.config"; import { firecrawlConfig } from "../../config/firecrawl.config";
import { systemConfigService } from "../system-config/system-config.service";
let cachedApiKey: string | null = null;
let cachedBaseUrl: string | null = null;
let firecrawlClient: FirecrawlApp | null = null; let firecrawlClient: FirecrawlApp | null = null;
export async function getDynamicFirecrawlClient(): Promise<FirecrawlApp> {
const apiKey = await systemConfigService.get<string>(
"integration.firecrawl.api_key",
firecrawlConfig.apiKey,
);
const baseUrl = await systemConfigService.get<string>(
"integration.firecrawl.base_url",
firecrawlConfig.baseUrl,
);
const effectiveKey = apiKey || firecrawlConfig.apiKey;
const effectiveUrl = baseUrl || firecrawlConfig.baseUrl;
if (!effectiveKey) {
throw new Error(
"FIRECRAWL_API_KEY is not set. Add it in System Config or .env file before using the crawler.",
);
}
if (
!firecrawlClient ||
cachedApiKey !== effectiveKey ||
cachedBaseUrl !== effectiveUrl
) {
cachedApiKey = effectiveKey;
cachedBaseUrl = effectiveUrl;
firecrawlClient = new FirecrawlApp({
apiKey: effectiveKey,
apiUrl: effectiveUrl,
});
}
return firecrawlClient;
}
export function getFirecrawlClient(): FirecrawlApp { export function getFirecrawlClient(): FirecrawlApp {
if (!firecrawlClient) { if (!firecrawlClient) {
if (!firecrawlConfig.apiKey) { if (!firecrawlConfig.apiKey) {
......
import { Request, Response, NextFunction } from "express";
import { systemConfigService } from "./system-config.service";
import {
CreateSystemConfigDto,
UpdateSystemConfigDto,
SystemConfigQueryDto,
} from "./system-config.dto";
export class SystemConfigController {
private readonly service = systemConfigService;
getPublic = async (
req: Request,
res: Response,
next: NextFunction,
): Promise<void> => {
try {
const result = await this.service.getPublicConfigs();
res.json({
success: true,
data: result,
});
} catch (error) {
next(error);
}
};
findAll = async (
req: Request,
res: Response,
next: NextFunction,
): Promise<void> => {
try {
const query = req.query as SystemConfigQueryDto;
const result = await this.service.findAll(query);
res.json({
success: true,
data: result,
});
} catch (error) {
next(error);
}
};
findByKey = async (
req: Request,
res: Response,
next: NextFunction,
): Promise<void> => {
try {
const { key } = req.params;
const result = await this.service.findByKey(key);
res.json({
success: true,
data: result,
});
} catch (error) {
next(error);
}
};
create = async (
req: Request,
res: Response,
next: NextFunction,
): Promise<void> => {
try {
const dto: CreateSystemConfigDto = req.body;
const result = await this.service.create(dto, {
actorId: req.user.id,
ipAddress: req.ip,
userAgent: req.headers["user-agent"] as string,
});
res.status(201).json({
success: true,
data: result,
message: "Tạo cấu hình mới thành công",
});
} catch (error) {
next(error);
}
};
update = async (
req: Request,
res: Response,
next: NextFunction,
): Promise<void> => {
try {
const { key } = req.params;
const dto: UpdateSystemConfigDto = req.body;
const result = await this.service.update(key, dto, {
actorId: req.user.id,
ipAddress: req.ip,
userAgent: req.headers["user-agent"] as string,
});
res.json({
success: true,
data: result,
message: "Cập nhật cấu hình thành công",
});
} catch (error) {
next(error);
}
};
toggleFeature = async (
req: Request,
res: Response,
next: NextFunction,
): Promise<void> => {
try {
const { key } = req.params;
const result = await this.service.toggleFeature(key, {
actorId: req.user.id,
ipAddress: req.ip,
userAgent: req.headers["user-agent"] as string,
});
res.json({
success: true,
data: result,
message: `Đã ${result.value ? "bật" : "tắt"} tính năng thành công`,
});
} catch (error) {
next(error);
}
};
delete = async (
req: Request,
res: Response,
next: NextFunction,
): Promise<void> => {
try {
const { key } = req.params;
await this.service.delete(key, {
actorId: req.user.id,
ipAddress: req.ip,
userAgent: req.headers["user-agent"] as string,
});
res.json({
success: true,
message: "Xóa cấu hình thành công",
});
} catch (error) {
next(error);
}
};
}
import { SystemConfigCategory } from "../../common/constants/system-config.constant";
export interface CreateSystemConfigDto {
key: string;
value: unknown;
description?: string;
category?: SystemConfigCategory;
isPublic?: boolean;
}
export interface UpdateSystemConfigDto {
value?: unknown;
description?: string;
category?: SystemConfigCategory;
isPublic?: boolean;
}
export interface SystemConfigQueryDto {
category?: SystemConfigCategory;
search?: string;
isPublic?: boolean | string;
page?: number | string;
limit?: number | string;
}
export interface SystemConfigResponseDto {
id: string;
key: string;
value: unknown;
description: string | null;
category: string;
isPublic: boolean;
createdAt: Date;
updatedAt: Date;
}
export interface PublicConfigsResponseDto {
configs: Array<{
key: string;
value: unknown;
category: string;
description: string | null;
}>;
map: Record<string, unknown>;
}
export interface SystemConfigEventPayload {
key?: string;
action: "create" | "update" | "toggle" | "delete" | "invalidate";
timestamp: number;
}
import { prisma } from "../../database/prisma.client";
import { Prisma } from "@prisma/client";
import {
CreateSystemConfigDto,
UpdateSystemConfigDto,
SystemConfigQueryDto,
} from "./system-config.dto";
import { DefaultSystemConfigItem } from "../../common/constants/system-config.constant";
export class SystemConfigRepository {
async findAll(query: SystemConfigQueryDto = {}) {
const where: Prisma.SystemConfigWhereInput = {};
if (query.category) {
where.category = query.category;
}
if (query.isPublic !== undefined) {
where.isPublic =
typeof query.isPublic === "boolean"
? query.isPublic
: query.isPublic === "true";
}
if (query.search) {
where.OR = [
{ key: { contains: query.search, mode: "insensitive" } },
{ description: { contains: query.search, mode: "insensitive" } },
];
}
const page = Math.max(1, Number(query.page) || 1);
const limit = Math.min(Math.max(1, Number(query.limit) || 20), 100);
const skip = (page - 1) * limit;
const [items, total] = await Promise.all([
prisma.systemConfig.findMany({
where,
orderBy: [{ category: "asc" }, { key: "asc" }],
skip,
take: limit,
}),
prisma.systemConfig.count({ where }),
]);
return {
items,
total,
page,
limit,
};
}
async findByKey(key: string) {
return prisma.systemConfig.findUnique({
where: { key },
});
}
async findPublicConfigs() {
return prisma.systemConfig.findMany({
where: { isPublic: true },
orderBy: { key: "asc" },
});
}
async create(data: CreateSystemConfigDto) {
return prisma.systemConfig.create({
data: {
key: data.key,
value: data.value as Prisma.InputJsonValue,
description: data.description ?? null,
category: data.category,
isPublic: data.isPublic ?? false,
},
});
}
async update(key: string, data: UpdateSystemConfigDto) {
const updatePayload: Prisma.SystemConfigUpdateInput = {};
if (data.value !== undefined) {
updatePayload.value = data.value as Prisma.InputJsonValue;
}
if (data.description !== undefined) {
updatePayload.description = data.description;
}
if (data.category !== undefined) {
updatePayload.category = data.category;
}
if (data.isPublic !== undefined) {
updatePayload.isPublic = data.isPublic;
}
return prisma.systemConfig.update({
where: { key },
data: updatePayload,
});
}
async delete(key: string) {
return prisma.systemConfig.delete({
where: { key },
});
}
async ensureDefault(item: DefaultSystemConfigItem) {
return prisma.systemConfig.upsert({
where: { key: item.key },
update: {
description: item.description ?? null,
category: item.category,
isPublic: item.isPublic,
},
create: {
key: item.key,
value: item.value as Prisma.InputJsonValue,
description: item.description ?? null,
category: item.category,
isPublic: item.isPublic,
},
});
}
}
import { Router } from "express";
import { SystemConfigController } from "./system-config.controller";
import { authMiddleware } from "../../middlewares/auth.middleware";
import { requirePermission } from "../../middlewares/permission.middleware";
import {
validate,
validateQuery,
validateParams,
} from "../../middlewares/validate.middleware";
import {
createSystemConfigSchema,
updateSystemConfigSchema,
systemConfigKeyParamSchema,
systemConfigQuerySchema,
} from "./system-config.validation";
import { PERMISSIONS } from "../../common/constants/permission.constant";
const router = Router();
const controller = new SystemConfigController();
// 1. GET /api/v1/system/public (Public client access)
router.get("/public", controller.getPublic);
// 2. GET /api/v1/system/configs (Admin list configs with search and category filter)
router.get(
"/configs",
authMiddleware,
requirePermission(PERMISSIONS.SYSTEM_CONFIG_READ),
validateQuery(systemConfigQuerySchema),
controller.findAll,
);
// 3. GET /api/v1/system/configs/:key (Get single config detail)
router.get(
"/configs/:key",
authMiddleware,
requirePermission(PERMISSIONS.SYSTEM_CONFIG_READ),
validateParams(systemConfigKeyParamSchema),
controller.findByKey,
);
// 4. POST /api/v1/system/configs (Create new config)
router.post(
"/configs",
authMiddleware,
requirePermission(PERMISSIONS.SYSTEM_CONFIG_MANAGE),
validate(createSystemConfigSchema),
controller.create,
);
// 5. PUT /api/v1/system/configs/:key (Update config)
router.put(
"/configs/:key",
authMiddleware,
requirePermission(PERMISSIONS.SYSTEM_CONFIG_MANAGE),
validateParams(systemConfigKeyParamSchema),
validate(updateSystemConfigSchema),
controller.update,
);
// 6. PATCH /api/v1/system/features/:key/toggle (Quick toggle for boolean Feature Flag)
router.patch(
"/features/:key/toggle",
authMiddleware,
requirePermission(PERMISSIONS.SYSTEM_CONFIG_MANAGE),
validateParams(systemConfigKeyParamSchema),
controller.toggleFeature,
);
// 7. DELETE /api/v1/system/configs/:key (Delete config)
router.delete(
"/configs/:key",
authMiddleware,
requirePermission(PERMISSIONS.SYSTEM_CONFIG_MANAGE),
validateParams(systemConfigKeyParamSchema),
controller.delete,
);
export default router;
This diff is collapsed.
import { z } from "zod";
import { SYSTEM_CONFIG_CATEGORY } from "../../common/constants/system-config.constant";
export const createSystemConfigSchema = z.object({
key: z
.string({ required_error: "Khóa cấu hình là bắt buộc" })
.min(2, "Khóa cấu hình phải có ít nhất 2 ký tự")
.max(100, "Khóa cấu hình tối đa 100 ký tự")
.regex(
/^[a-zA-Z0-9_.-]+$/,
"Khóa cấu hình chỉ được chứa chữ cái, chữ số, dấu chấm (.), gạch dưới (_) và gạch ngang (-)",
),
value: z
.any()
.refine((val) => val !== undefined, {
message: "Giá trị cấu hình không được để trống",
}),
description: z.string().max(500, "Mô tả tối đa 500 ký tự").optional().nullable(),
category: z
.nativeEnum(SYSTEM_CONFIG_CATEGORY)
.optional()
.default(SYSTEM_CONFIG_CATEGORY.GENERAL),
isPublic: z.boolean().optional().default(false),
});
export const updateSystemConfigSchema = z
.object({
value: z.any().optional(),
description: z.string().max(500, "Mô tả tối đa 500 ký tự").optional().nullable(),
category: z.nativeEnum(SYSTEM_CONFIG_CATEGORY).optional(),
isPublic: z.boolean().optional(),
})
.refine(
(data) =>
data.value !== undefined ||
data.description !== undefined ||
data.category !== undefined ||
data.isPublic !== undefined,
{
message: "Phải cung cấp ít nhất một trường để cập nhật",
},
);
export const systemConfigKeyParamSchema = z.object({
key: z.string().min(1, "Khóa cấu hình không được để trống"),
});
export const systemConfigQuerySchema = z.object({
category: z.nativeEnum(SYSTEM_CONFIG_CATEGORY).optional(),
search: z.string().optional(),
isPublic: z
.union([z.boolean(), z.enum(["true", "false"])])
.optional(),
page: z
.union([z.number(), z.string()])
.optional()
.transform((val) => {
if (val === undefined) return 1;
const parsed = typeof val === "string" ? parseInt(val, 10) : val;
return isNaN(parsed) || parsed < 1 ? 1 : parsed;
}),
limit: z
.union([z.number(), z.string()])
.optional()
.transform((val) => {
if (val === undefined) return 20;
const parsed = typeof val === "string" ? parseInt(val, 10) : val;
return isNaN(parsed) || parsed < 1 ? 20 : Math.min(parsed, 100);
}),
});
...@@ -4,6 +4,7 @@ import { UserQueryDto } from "./user.dto"; ...@@ -4,6 +4,7 @@ import { UserQueryDto } from "./user.dto";
import { envConfig } from "../../config/env.config"; import { envConfig } from "../../config/env.config";
import { ROLES } from "../../common/constants/role.constant"; import { ROLES } from "../../common/constants/role.constant";
import { SYSTEM_ROLE_SLUGS } from "../../common/constants/system-role.constant"; import { SYSTEM_ROLE_SLUGS } from "../../common/constants/system-role.constant";
import { systemConfigService } from "../system-config/system-config.service";
export class UserRepository { export class UserRepository {
async findAll(query: UserQueryDto = {}) { async findAll(query: UserQueryDto = {}) {
...@@ -94,7 +95,7 @@ export class UserRepository { ...@@ -94,7 +95,7 @@ export class UserRepository {
}); });
} }
create(data: { async create(data: {
email: string; email: string;
passwordHash: string; passwordHash: string;
fullName?: string; fullName?: string;
...@@ -104,6 +105,19 @@ export class UserRepository { ...@@ -104,6 +105,19 @@ export class UserRepository {
maxJobsPerDayLimit?: number; maxJobsPerDayLimit?: number;
maxConcurrentJobsLimit?: number; maxConcurrentJobsLimit?: number;
}): Promise<User> { }): Promise<User> {
const defaultMaxPages = await systemConfigService.get<number>(
"quota.user_max_pages",
envConfig.quota.defaultMaxPages,
);
const defaultMaxJobsPerDay = await systemConfigService.get<number>(
"quota.user_max_jobs_per_day",
envConfig.quota.defaultMaxJobsPerDay,
);
const defaultMaxConcurrentJobs = await systemConfigService.get<number>(
"quota.user_max_concurrent_jobs",
envConfig.quota.defaultMaxConcurrentJobs,
);
return prisma.user.create({ return prisma.user.create({
data: { data: {
email: data.email, email: data.email,
...@@ -111,12 +125,10 @@ export class UserRepository { ...@@ -111,12 +125,10 @@ export class UserRepository {
fullName: data.fullName, fullName: data.fullName,
avatarUrl: data.avatarUrl, avatarUrl: data.avatarUrl,
role: data.role ?? ROLES.CRAWLER_USER, role: data.role ?? ROLES.CRAWLER_USER,
maxPagesLimit: data.maxPagesLimit ?? envConfig.quota.defaultMaxPages, maxPagesLimit: data.maxPagesLimit ?? defaultMaxPages,
maxJobsPerDayLimit: maxJobsPerDayLimit: data.maxJobsPerDayLimit ?? defaultMaxJobsPerDay,
data.maxJobsPerDayLimit ?? envConfig.quota.defaultMaxJobsPerDay,
maxConcurrentJobsLimit: maxConcurrentJobsLimit:
data.maxConcurrentJobsLimit ?? data.maxConcurrentJobsLimit ?? defaultMaxConcurrentJobs,
envConfig.quota.defaultMaxConcurrentJobs,
}, },
}); });
} }
......
...@@ -22,6 +22,7 @@ import { CRAWL_MODE } from "../common/constants/crawl-mode.constant"; ...@@ -22,6 +22,7 @@ import { CRAWL_MODE } from "../common/constants/crawl-mode.constant";
import { ASSET_TYPE } from "../common/constants/asset-type.constant"; import { ASSET_TYPE } from "../common/constants/asset-type.constant";
import { CRAWL_PAGE_STATUS } from "../common/constants/crawl-page-status.constant"; import { CRAWL_PAGE_STATUS } from "../common/constants/crawl-page-status.constant";
import { WEBHOOK_EVENT } from "../common/constants/webhook.constant"; import { WEBHOOK_EVENT } from "../common/constants/webhook.constant";
import { systemConfigService } from "../modules/system-config/system-config.service";
// Lazy getters — instantiated on first use so Jest mocks replace constructors before creation // Lazy getters — instantiated on first use so Jest mocks replace constructors before creation
const getJobRepository = () => new CrawlJobRepository(); const getJobRepository = () => new CrawlJobRepository();
const getPageRepository = () => new CrawlPageRepository(); const getPageRepository = () => new CrawlPageRepository();
...@@ -704,20 +705,30 @@ export async function processCrawlJob(job: Job): Promise<void> { ...@@ -704,20 +705,30 @@ export async function processCrawlJob(job: Job): Promise<void> {
}); });
} }
// Generate diff_report.json upon successful job completion // Generate diff_report.json upon successful job completion (if feature flag is enabled)
try { const isDiffEnabled = await systemConfigService.isFeatureEnabled(
const { ChangeDetectionService } = "feature.change_detection.enabled",
await import("../modules/change-detection/change-detection.service"); true,
const changeDetectionService = new ChangeDetectionService(); );
const diffReport = if (isDiffEnabled) {
await changeDetectionService.generateAndSaveDiffReport(jobId); try {
const { ChangeDetectionService } =
await import("../modules/change-detection/change-detection.service");
const changeDetectionService = new ChangeDetectionService();
const diffReport =
await changeDetectionService.generateAndSaveDiffReport(jobId);
console.log(
`[Worker] Diff report generated for job ${jobId}: ${diffReport.summary.newPagesCount} new, ${diffReport.summary.modifiedPagesCount} modified, ${diffReport.summary.deletedPagesCount} deleted, ${diffReport.summary.unchangedPagesCount} unchanged`,
);
} catch (diffErr: unknown) {
console.error(
`[Worker] Failed to generate diff report for job ${jobId}:`,
getErrorMessage(diffErr),
);
}
} else {
console.log( console.log(
`[Worker] Diff report generated for job ${jobId}: ${diffReport.summary.newPagesCount} new, ${diffReport.summary.modifiedPagesCount} modified, ${diffReport.summary.deletedPagesCount} deleted, ${diffReport.summary.unchangedPagesCount} unchanged`, `[Worker] Skipped diff report for job ${jobId}: feature.change_detection.enabled is false`,
);
} catch (diffErr: unknown) {
console.error(
`[Worker] Failed to generate diff report for job ${jobId}:`,
getErrorMessage(diffErr),
); );
} }
} catch (error: unknown) { } catch (error: unknown) {
...@@ -735,46 +746,52 @@ export async function processCrawlJob(job: Job): Promise<void> { ...@@ -735,46 +746,52 @@ export async function processCrawlJob(job: Job): Promise<void> {
(updatedJob.status === JOB_STATUS.COMPLETED || (updatedJob.status === JOB_STATUS.COMPLETED ||
updatedJob.status === JOB_STATUS.FAILED) updatedJob.status === JOB_STATUS.FAILED)
) { ) {
const { WebhookDeliveryService } = const isWebhookEnabled = await systemConfigService.isFeatureEnabled(
await import("../modules/webhooks/webhook-delivery.service"); "feature.webhook.deliveries.enabled",
const webhookDeliveryService = new WebhookDeliveryService(); true,
const event =
updatedJob.status === JOB_STATUS.COMPLETED
? WEBHOOK_EVENT.JOB_COMPLETED
: WEBHOOK_EVENT.JOB_FAILED;
void logStep(
jobId,
updatedJob.status === JOB_STATUS.COMPLETED ? "INFO" : "ERROR",
updatedJob.status,
`Job finished with status ${updatedJob.status}${updatedJob.errorMessage ? `: ${updatedJob.errorMessage}` : ""}`,
);
const diffSummary =
updatedJob &&
typeof updatedJob === "object" &&
"diffSummary" in updatedJob
? ((updatedJob as { diffSummary: unknown }).diffSummary ?? null)
: null;
const payload = {
jobId: updatedJob.id,
status: updatedJob.status,
startUrl: updatedJob.startUrl,
mode: updatedJob.mode,
totalPages: updatedJob.totalPages,
successPages: updatedJob.successPages,
failedPages: updatedJob.failedPages,
errorMessage: updatedJob.errorMessage,
diffSummary,
startedAt: updatedJob.startedAt,
finishedAt: updatedJob.finishedAt,
};
void webhookDeliveryService.dispatch(
updatedJob.id,
updatedJob.userId,
event,
payload,
); );
if (isWebhookEnabled) {
const { WebhookDeliveryService } =
await import("../modules/webhooks/webhook-delivery.service");
const webhookDeliveryService = new WebhookDeliveryService();
const event =
updatedJob.status === JOB_STATUS.COMPLETED
? WEBHOOK_EVENT.JOB_COMPLETED
: WEBHOOK_EVENT.JOB_FAILED;
void logStep(
jobId,
updatedJob.status === JOB_STATUS.COMPLETED ? "INFO" : "ERROR",
updatedJob.status,
`Job finished with status ${updatedJob.status}${updatedJob.errorMessage ? `: ${updatedJob.errorMessage}` : ""}`,
);
const diffSummary =
updatedJob &&
typeof updatedJob === "object" &&
"diffSummary" in updatedJob
? ((updatedJob as { diffSummary: unknown }).diffSummary ?? null)
: null;
const payload = {
jobId: updatedJob.id,
status: updatedJob.status,
startUrl: updatedJob.startUrl,
mode: updatedJob.mode,
totalPages: updatedJob.totalPages,
successPages: updatedJob.successPages,
failedPages: updatedJob.failedPages,
errorMessage: updatedJob.errorMessage,
diffSummary,
startedAt: updatedJob.startedAt,
finishedAt: updatedJob.finishedAt,
};
void webhookDeliveryService.dispatch(
updatedJob.id,
updatedJob.userId,
event,
payload,
);
}
} }
} catch (webhookErr) { } catch (webhookErr) {
console.error( console.error(
......
...@@ -12,6 +12,7 @@ import healthRoute from "../modules/health/health.route"; ...@@ -12,6 +12,7 @@ import healthRoute from "../modules/health/health.route";
import dashboardRoute from "../modules/dashboard/dashboard.route"; import dashboardRoute from "../modules/dashboard/dashboard.route";
import roleRoute from "../modules/roles/role.route"; import roleRoute from "../modules/roles/role.route";
import permissionRoute from "../modules/permissions/permission.route"; import permissionRoute from "../modules/permissions/permission.route";
import systemConfigRoute from "../modules/system-config/system-config.route";
const router = Router(); const router = Router();
...@@ -20,6 +21,7 @@ router.use("/auth", authRoute); ...@@ -20,6 +21,7 @@ router.use("/auth", authRoute);
router.use("/users", userRoute); router.use("/users", userRoute);
router.use("/roles", roleRoute); router.use("/roles", roleRoute);
router.use("/permissions", permissionRoute); router.use("/permissions", permissionRoute);
router.use("/system", systemConfigRoute);
router.use("/dashboard", dashboardRoute); router.use("/dashboard", dashboardRoute);
router.use("/crawl-jobs", crawlJobRoute); router.use("/crawl-jobs", crawlJobRoute);
router.use("/crawl-schedules", crawlScheduleRoute); router.use("/crawl-schedules", crawlScheduleRoute);
......
...@@ -45,7 +45,19 @@ async function bootstrap() { ...@@ -45,7 +45,19 @@ async function bootstrap() {
initLocalStorage(); initLocalStorage();
const { systemConfigService } = await import(
"./modules/system-config/system-config.service"
);
try {
await systemConfigService.ensureDefaultConfigs();
console.log("[Server] Default system configs initialized successfully.");
} catch (err) {
console.warn("[Server] Failed to initialize default system configs:", err);
}
if (isRedisAvailable) { if (isRedisAvailable) {
systemConfigService.initRedisSubscriber();
await import("./queues/webhook.worker"); await import("./queues/webhook.worker");
console.log("[Server] Webhook worker initialized in background."); console.log("[Server] Webhook worker initialized in background.");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment