Commit 5b123692 authored by ThinhNC's avatar ThinhNC

feat(security,rbac): resolve 17 audit findings and migrate routes to dynamic permissions

parent e0406ff9
/*
Warnings:
- You are about to drop the `CrawlJobLog` table. If the table is not empty, all the data it contains will be lost.
*/
-- DropForeignKey
ALTER TABLE "CrawlJobLog" DROP CONSTRAINT "CrawlJobLog_job_id_fkey";
-- DropForeignKey
ALTER TABLE "crawl_assets" DROP CONSTRAINT "crawl_assets_crawl_job_id_fkey";
-- DropTable
DROP TABLE "CrawlJobLog";
-- CreateTable
CREATE TABLE "crawl_job_logs" (
"id" UUID NOT NULL,
"job_id" UUID NOT NULL,
"level" "LogLevel" NOT NULL,
"step" TEXT NOT NULL,
"message" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "crawl_job_logs_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE INDEX "crawl_job_logs_job_id_created_at_idx" ON "crawl_job_logs"("job_id", "created_at");
-- CreateIndex
CREATE INDEX "audit_logs_user_id_created_at_idx" ON "audit_logs"("user_id", "created_at");
-- CreateIndex
CREATE INDEX "audit_logs_action_idx" ON "audit_logs"("action");
-- CreateIndex
CREATE INDEX "audit_logs_created_at_idx" ON "audit_logs"("created_at");
-- CreateIndex
CREATE INDEX "audit_logs_ip_address_idx" ON "audit_logs"("ip_address");
-- CreateIndex
CREATE INDEX "crawl_assets_crawl_job_id_idx" ON "crawl_assets"("crawl_job_id");
-- CreateIndex
CREATE INDEX "crawl_jobs_user_id_created_at_idx" ON "crawl_jobs"("user_id", "created_at");
-- AddForeignKey
ALTER TABLE "crawl_assets" ADD CONSTRAINT "crawl_assets_crawl_job_id_fkey" FOREIGN KEY ("crawl_job_id") REFERENCES "crawl_jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "crawl_job_logs" ADD CONSTRAINT "crawl_job_logs_job_id_fkey" FOREIGN KEY ("job_id") REFERENCES "crawl_jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;
...@@ -152,6 +152,7 @@ model CrawlJob { ...@@ -152,6 +152,7 @@ model CrawlJob {
@@index([status]) @@index([status])
@@index([createdAt]) @@index([createdAt])
@@index([userId, status]) @@index([userId, status])
@@index([userId, createdAt])
@@index([scheduleId]) @@index([scheduleId])
@@map("crawl_jobs") @@map("crawl_jobs")
} }
...@@ -206,7 +207,7 @@ model CrawlAsset { ...@@ -206,7 +207,7 @@ model CrawlAsset {
createdAt DateTime @default(now()) @map("created_at") createdAt DateTime @default(now()) @map("created_at")
page CrawlPage? @relation(fields: [pageId], references: [id], onDelete: SetNull) page CrawlPage? @relation(fields: [pageId], references: [id], onDelete: SetNull)
crawlJob CrawlJob? @relation(fields: [crawlJobId], references: [id]) crawlJob CrawlJob? @relation(fields: [crawlJobId], references: [id], onDelete: Cascade)
crawlJobId String? @map("crawl_job_id") @db.Uuid crawlJobId String? @map("crawl_job_id") @db.Uuid
@@index([pageId]) @@index([pageId])
......
...@@ -19,19 +19,12 @@ const app = express(); ...@@ -19,19 +19,12 @@ const app = express();
app.set("trust proxy", parseTrustProxy(envConfig.trustProxy)); app.set("trust proxy", parseTrustProxy(envConfig.trustProxy));
app.use( app.use(helmet());
helmet({
contentSecurityPolicy: false, // Vô hiệu hóa CSP để Swagger UI load stylesheet bình thường
}),
);
app.use( app.use(
cors({ cors({
origin: (origin, callback) => { origin: (origin, callback) => {
if (!origin) return callback(null, true); if (!origin) return callback(null, true);
if ( if (envConfig.cors.allowedOrigins.includes(origin)) {
envConfig.cors.allowedOrigins.includes(origin) ||
envConfig.cors.allowedOrigins.includes("*")
) {
return callback(null, true); return callback(null, true);
} }
return callback(new Error(`Origin ${origin} not allowed by CORS`)); return callback(new Error(`Origin ${origin} not allowed by CORS`));
...@@ -46,7 +39,12 @@ app.use(express.json()); ...@@ -46,7 +39,12 @@ app.use(express.json());
app.use(express.urlencoded({ extended: true })); app.use(express.urlencoded({ extended: true }));
app.use("/health", healthRoute); app.use("/health", healthRoute);
app.use("/api-docs", swaggerUi.serve, swaggerUi.setup(swaggerDocument)); app.use(
"/api-docs",
helmet({ contentSecurityPolicy: false }),
swaggerUi.serve,
swaggerUi.setup(swaggerDocument),
);
app.use("/api/v1", rateLimitMiddleware, routes); app.use("/api/v1", rateLimitMiddleware, routes);
app.use(notFoundMiddleware); app.use(notFoundMiddleware);
......
...@@ -41,6 +41,26 @@ export const PERMISSIONS = { ...@@ -41,6 +41,26 @@ export const PERMISSIONS = {
EXPORTS_READ_ALL: "exports.read_all", EXPORTS_READ_ALL: "exports.read_all",
EXPORTS_CREATE: "exports.create", EXPORTS_CREATE: "exports.create",
EXPORTS_DOWNLOAD: "exports.download", EXPORTS_DOWNLOAD: "exports.download",
EXPORTS_DELETE: "exports.delete",
// Webhooks
WEBHOOKS_READ: "webhooks.read",
WEBHOOKS_CREATE: "webhooks.create",
WEBHOOKS_UPDATE: "webhooks.update",
WEBHOOKS_DELETE: "webhooks.delete",
WEBHOOKS_TEST: "webhooks.test",
// Extraction Templates
EXTRACTION_TEMPLATES_READ: "extraction_templates.read",
EXTRACTION_TEMPLATES_CREATE: "extraction_templates.create",
EXTRACTION_TEMPLATES_UPDATE: "extraction_templates.update",
EXTRACTION_TEMPLATES_DELETE: "extraction_templates.delete",
// API Keys
API_KEYS_READ: "api_keys.read",
API_KEYS_CREATE: "api_keys.create",
API_KEYS_UPDATE: "api_keys.update",
API_KEYS_DELETE: "api_keys.delete",
// Audit Logs // Audit Logs
AUDIT_LOGS_READ: "audit_logs.read", AUDIT_LOGS_READ: "audit_logs.read",
...@@ -305,6 +325,124 @@ export const SYSTEM_PERMISSIONS_CATALOG: PermissionDefinition[] = [ ...@@ -305,6 +325,124 @@ export const SYSTEM_PERMISSIONS_CATALOG: PermissionDefinition[] = [
action: "download", action: "download",
isSystem: true, isSystem: true,
}, },
{
name: "Delete Export",
slug: PERMISSIONS.EXPORTS_DELETE,
description: "Xóa tệp trích xuất dữ liệu",
resource: "exports",
action: "delete",
isSystem: true,
},
// Webhooks
{
name: "View Webhooks",
slug: PERMISSIONS.WEBHOOKS_READ,
description: "Xem cấu hình và nhật ký gửi webhook",
resource: "webhooks",
action: "read",
isSystem: true,
},
{
name: "Create Webhook",
slug: PERMISSIONS.WEBHOOKS_CREATE,
description: "Tạo cấu hình webhook mới",
resource: "webhooks",
action: "create",
isSystem: true,
},
{
name: "Update Webhook",
slug: PERMISSIONS.WEBHOOKS_UPDATE,
description: "Cập nhật cấu hình webhook hoặc gửi lại",
resource: "webhooks",
action: "update",
isSystem: true,
},
{
name: "Delete Webhook",
slug: PERMISSIONS.WEBHOOKS_DELETE,
description: "Xóa cấu hình webhook",
resource: "webhooks",
action: "delete",
isSystem: true,
},
{
name: "Test Webhook",
slug: PERMISSIONS.WEBHOOKS_TEST,
description: "Gửi kiểm thử ping webhook",
resource: "webhooks",
action: "test",
isSystem: true,
},
// Extraction Templates
{
name: "View Extraction Templates",
slug: PERMISSIONS.EXTRACTION_TEMPLATES_READ,
description: "Xem mẫu bóc tách dữ liệu",
resource: "extraction_templates",
action: "read",
isSystem: true,
},
{
name: "Create Extraction Template",
slug: PERMISSIONS.EXTRACTION_TEMPLATES_CREATE,
description: "Tạo mẫu bóc tách dữ liệu mới",
resource: "extraction_templates",
action: "create",
isSystem: true,
},
{
name: "Update Extraction Template",
slug: PERMISSIONS.EXTRACTION_TEMPLATES_UPDATE,
description: "Cập nhật mẫu bóc tách dữ liệu",
resource: "extraction_templates",
action: "update",
isSystem: true,
},
{
name: "Delete Extraction Template",
slug: PERMISSIONS.EXTRACTION_TEMPLATES_DELETE,
description: "Xóa mẫu bóc tách dữ liệu",
resource: "extraction_templates",
action: "delete",
isSystem: true,
},
// API Keys
{
name: "View API Keys",
slug: PERMISSIONS.API_KEYS_READ,
description: "Xem danh sách khóa API",
resource: "api_keys",
action: "read",
isSystem: true,
},
{
name: "Create API Key",
slug: PERMISSIONS.API_KEYS_CREATE,
description: "Tạo khóa API mới",
resource: "api_keys",
action: "create",
isSystem: true,
},
{
name: "Update API Key",
slug: PERMISSIONS.API_KEYS_UPDATE,
description: "Kích hoạt hoặc vô hiệu hóa khóa API",
resource: "api_keys",
action: "update",
isSystem: true,
},
{
name: "Delete API Key",
slug: PERMISSIONS.API_KEYS_DELETE,
description: "Thu hồi và xóa khóa API",
resource: "api_keys",
action: "delete",
isSystem: true,
},
// Audit Logs // Audit Logs
{ {
...@@ -370,6 +508,20 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record< ...@@ -370,6 +508,20 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
PERMISSIONS.EXPORTS_READ_ALL, PERMISSIONS.EXPORTS_READ_ALL,
PERMISSIONS.EXPORTS_CREATE, PERMISSIONS.EXPORTS_CREATE,
PERMISSIONS.EXPORTS_DOWNLOAD, PERMISSIONS.EXPORTS_DOWNLOAD,
PERMISSIONS.EXPORTS_DELETE,
PERMISSIONS.WEBHOOKS_READ,
PERMISSIONS.WEBHOOKS_CREATE,
PERMISSIONS.WEBHOOKS_UPDATE,
PERMISSIONS.WEBHOOKS_DELETE,
PERMISSIONS.WEBHOOKS_TEST,
PERMISSIONS.EXTRACTION_TEMPLATES_READ,
PERMISSIONS.EXTRACTION_TEMPLATES_CREATE,
PERMISSIONS.EXTRACTION_TEMPLATES_UPDATE,
PERMISSIONS.EXTRACTION_TEMPLATES_DELETE,
PERMISSIONS.API_KEYS_READ,
PERMISSIONS.API_KEYS_CREATE,
PERMISSIONS.API_KEYS_UPDATE,
PERMISSIONS.API_KEYS_DELETE,
PERMISSIONS.AUDIT_LOGS_READ, PERMISSIONS.AUDIT_LOGS_READ,
PERMISSIONS.DASHBOARD_READ, PERMISSIONS.DASHBOARD_READ,
PERMISSIONS.DASHBOARD_READ_ALL, PERMISSIONS.DASHBOARD_READ_ALL,
...@@ -379,6 +531,7 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record< ...@@ -379,6 +531,7 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
PERMISSIONS.CRAWL_JOBS_READ, PERMISSIONS.CRAWL_JOBS_READ,
PERMISSIONS.CRAWL_JOBS_CANCEL, PERMISSIONS.CRAWL_JOBS_CANCEL,
PERMISSIONS.CRAWL_JOBS_RETRY, PERMISSIONS.CRAWL_JOBS_RETRY,
PERMISSIONS.CRAWL_JOBS_DELETE,
PERMISSIONS.CRAWL_SCHEDULES_CREATE, PERMISSIONS.CRAWL_SCHEDULES_CREATE,
PERMISSIONS.CRAWL_SCHEDULES_READ, PERMISSIONS.CRAWL_SCHEDULES_READ,
PERMISSIONS.CRAWL_SCHEDULES_UPDATE, PERMISSIONS.CRAWL_SCHEDULES_UPDATE,
...@@ -387,12 +540,30 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record< ...@@ -387,12 +540,30 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
PERMISSIONS.EXPORTS_READ, PERMISSIONS.EXPORTS_READ,
PERMISSIONS.EXPORTS_CREATE, PERMISSIONS.EXPORTS_CREATE,
PERMISSIONS.EXPORTS_DOWNLOAD, PERMISSIONS.EXPORTS_DOWNLOAD,
PERMISSIONS.EXPORTS_DELETE,
PERMISSIONS.DASHBOARD_READ, PERMISSIONS.DASHBOARD_READ,
PERMISSIONS.WEBHOOKS_READ,
PERMISSIONS.WEBHOOKS_CREATE,
PERMISSIONS.WEBHOOKS_UPDATE,
PERMISSIONS.WEBHOOKS_DELETE,
PERMISSIONS.WEBHOOKS_TEST,
PERMISSIONS.EXTRACTION_TEMPLATES_READ,
PERMISSIONS.EXTRACTION_TEMPLATES_CREATE,
PERMISSIONS.EXTRACTION_TEMPLATES_UPDATE,
PERMISSIONS.EXTRACTION_TEMPLATES_DELETE,
PERMISSIONS.API_KEYS_READ,
PERMISSIONS.API_KEYS_CREATE,
PERMISSIONS.API_KEYS_UPDATE,
PERMISSIONS.API_KEYS_DELETE,
], ],
[SYSTEM_ROLE_SLUGS.VIEWER]: [ [SYSTEM_ROLE_SLUGS.VIEWER]: [
PERMISSIONS.CRAWL_JOBS_READ, PERMISSIONS.CRAWL_JOBS_READ,
PERMISSIONS.CRAWL_SCHEDULES_READ, PERMISSIONS.CRAWL_SCHEDULES_READ,
PERMISSIONS.EXPORTS_READ, PERMISSIONS.EXPORTS_READ,
PERMISSIONS.EXPORTS_DOWNLOAD,
PERMISSIONS.DASHBOARD_READ, PERMISSIONS.DASHBOARD_READ,
PERMISSIONS.WEBHOOKS_READ,
PERMISSIONS.EXTRACTION_TEMPLATES_READ,
PERMISSIONS.API_KEYS_READ,
], ],
}; };
...@@ -31,6 +31,7 @@ export const ERROR_CODE = { ...@@ -31,6 +31,7 @@ export const ERROR_CODE = {
PRIVILEGE_ESCALATION_DENIED: "PRIVILEGE_ESCALATION_DENIED", PRIVILEGE_ESCALATION_DENIED: "PRIVILEGE_ESCALATION_DENIED",
SYSTEM_ROLE_PROTECTED: "SYSTEM_ROLE_PROTECTED", SYSTEM_ROLE_PROTECTED: "SYSTEM_ROLE_PROTECTED",
CANNOT_REMOVE_LAST_SUPER_ADMIN: "CANNOT_REMOVE_LAST_SUPER_ADMIN", CANNOT_REMOVE_LAST_SUPER_ADMIN: "CANNOT_REMOVE_LAST_SUPER_ADMIN",
RATE_LIMIT_EXCEEDED: "RATE_LIMIT_EXCEEDED",
} as const; } as const;
export type ErrorCode = keyof typeof ERROR_CODE; export type ErrorCode = keyof typeof ERROR_CODE;
...@@ -14,6 +14,8 @@ import { ...@@ -14,6 +14,8 @@ import {
DATA_QUALITY_MIN_SCORE, DATA_QUALITY_MIN_SCORE,
DATA_CONTRACT_HASH_ALGORITHM, DATA_CONTRACT_HASH_ALGORITHM,
} from "../constants/data-contract.constant"; } from "../constants/data-contract.constant";
import { ASSET_TYPE } from "../constants/asset-type.constant";
import { CRAWL_PAGE_STATUS } from "../constants/crawl-page-status.constant";
/** /**
* Normalize một URL để phục vụ deduplicate và so sánh. * Normalize một URL để phục vụ deduplicate và so sánh.
...@@ -321,7 +323,7 @@ export function transformImages(assets: CrawlAsset[]): ImageRecord[] { ...@@ -321,7 +323,7 @@ export function transformImages(assets: CrawlAsset[]): ImageRecord[] {
const seenUrls = new Set<string>(); const seenUrls = new Set<string>();
return assets return assets
.filter((a) => { .filter((a) => {
if (a.assetType !== "IMAGE") return false; if (a.assetType !== ASSET_TYPE.IMAGE) return false;
if (seenUrls.has(a.url)) return false; if (seenUrls.has(a.url)) return false;
seenUrls.add(a.url); seenUrls.add(a.url);
return true; return true;
...@@ -366,7 +368,7 @@ export function transformPageToRecord( ...@@ -366,7 +368,7 @@ export function transformPageToRecord(
const { page, assets, tables = [], jobDomain, seenContentHashes } = options; const { page, assets, tables = [], jobDomain, seenContentHashes } = options;
const normalizedUrl = page.normalizedUrl || normalizeUrl(page.url); const normalizedUrl = page.normalizedUrl || normalizeUrl(page.url);
const isSuccess = page.status === "SUCCESS"; const isSuccess = page.status === CRAWL_PAGE_STATUS.SUCCESS;
// Clean text từ markdownContent // Clean text từ markdownContent
const rawMarkdown = page.markdownContent ?? null; const rawMarkdown = page.markdownContent ?? null;
......
...@@ -32,11 +32,60 @@ export function errorMiddleware( ...@@ -32,11 +32,60 @@ export function errorMiddleware(
return; return;
} }
// Handle Prisma Known Request Errors
const prismaError = error as { code?: string; meta?: { target?: string[] } };
if (
prismaError.code &&
typeof prismaError.code === "string" &&
prismaError.code.startsWith("P")
) {
switch (prismaError.code) {
case "P2002": {
const target = Array.isArray(prismaError.meta?.target)
? prismaError.meta.target.join(", ")
: "field";
res.status(409).json({
success: false,
message: `A record with this ${target} already exists.`,
code: ERROR_CODE.DUPLICATE_ENTRY,
});
return;
}
case "P2023": {
res.status(400).json({
success: false,
message: "Invalid input format or malformed identifier.",
code: ERROR_CODE.VALIDATION_ERROR,
});
return;
}
case "P2025": {
res.status(404).json({
success: false,
message: "Requested record not found.",
code: ERROR_CODE.NOT_FOUND,
});
return;
}
case "P2003": {
res.status(400).json({
success: false,
message:
"Referenced record does not exist or relation constraint failed.",
code: ERROR_CODE.VALIDATION_ERROR,
});
return;
}
default:
break;
}
}
console.error("[Unhandled Error]", error); console.error("[Unhandled Error]", error);
res.status(500).json({ res.status(500).json({
success: false, success: false,
message: "Internal server error", message: "Internal server error",
code: "INTERNAL_SERVER_ERROR", code: ERROR_CODE.INTERNAL_SERVER_ERROR,
}); });
} }
import { Request, Response, NextFunction } from "express"; import { Request, Response, NextFunction } from "express";
import { PermissionSlug } from "../common/constants/permission.constant"; import {
PermissionSlug,
SYSTEM_ROLE_DEFAULT_PERMISSIONS,
} from "../common/constants/permission.constant";
import { SystemRoleSlug } from "../common/constants/system-role.constant";
import { AppError } from "../common/errors/app-error"; import { AppError } from "../common/errors/app-error";
import { ERROR_CODE } from "../common/errors/error-code"; import { ERROR_CODE } from "../common/errors/error-code";
import { PermissionService } from "../modules/permissions/permission.service"; import { PermissionService } from "../modules/permissions/permission.service";
...@@ -12,12 +16,19 @@ async function resolveUserPermissions(req: Request): Promise<string[]> { ...@@ -12,12 +16,19 @@ async function resolveUserPermissions(req: Request): Promise<string[]> {
} }
const permissions = await permissionService.getUserPermissions(req.user.id); const permissions = await permissionService.getUserPermissions(req.user.id);
req.user.permissions = permissions;
if (!req.user.roles) { if (!req.user.roles) {
req.user.roles = await permissionService.getUserRoles(req.user.id); req.user.roles = await permissionService.getUserRoles(req.user.id);
} }
if (permissions.length === 0 && req.user.role) {
const defaultPerms =
SYSTEM_ROLE_DEFAULT_PERMISSIONS[req.user.role as SystemRoleSlug] || [];
req.user.permissions = defaultPerms;
return defaultPerms;
}
req.user.permissions = permissions;
return permissions; return permissions;
} }
......
import rateLimit, { RateLimitRequestHandler } from "express-rate-limit"; import rateLimit, { RateLimitRequestHandler } from "express-rate-limit";
import { envConfig } from "../config/env.config"; import { envConfig } from "../config/env.config";
import { ERROR_CODE } from "../common/errors/error-code";
/** /**
* Global API rate limit per IP, configurable for each environment. * Global API rate limit per IP, configurable for each environment.
...@@ -14,7 +15,7 @@ export const rateLimitMiddleware: RateLimitRequestHandler = rateLimit({ ...@@ -14,7 +15,7 @@ export const rateLimitMiddleware: RateLimitRequestHandler = rateLimit({
message: { message: {
success: false, success: false,
message: "Bạn đã gửi quá nhiều yêu cầu. Vui lòng thử lại sau.", message: "Bạn đã gửi quá nhiều yêu cầu. Vui lòng thử lại sau.",
code: "RATE_LIMIT_EXCEEDED", code: ERROR_CODE.RATE_LIMIT_EXCEEDED,
}, },
}); });
...@@ -26,6 +27,6 @@ export const authRateLimiter: RateLimitRequestHandler = rateLimit({ ...@@ -26,6 +27,6 @@ export const authRateLimiter: RateLimitRequestHandler = rateLimit({
message: { message: {
success: false, success: false,
message: "Quá nhiều yêu cầu xác thực. Vui lòng thử lại sau 1 phút.", message: "Quá nhiều yêu cầu xác thực. Vui lòng thử lại sau 1 phút.",
code: "RATE_LIMIT_EXCEEDED", code: ERROR_CODE.RATE_LIMIT_EXCEEDED,
}, },
}); });
...@@ -7,6 +7,7 @@ export const ALLOWED_AVATAR_MIME_TYPES = [ ...@@ -7,6 +7,7 @@ export const ALLOWED_AVATAR_MIME_TYPES = [
"image/jpeg", "image/jpeg",
"image/png", "image/png",
"image/webp", "image/webp",
"image/gif",
] as const; ] as const;
export const MAX_AVATAR_SIZE_BYTES = 5 * 1024 * 1024; // 5MB export const MAX_AVATAR_SIZE_BYTES = 5 * 1024 * 1024; // 5MB
......
import { Router } from "express"; import { Router } from "express";
import { ApiKeyController } from "./api-key.controller"; import { ApiKeyController } from "./api-key.controller";
import { authMiddleware } from "../../middlewares/auth.middleware"; import { authMiddleware } from "../../middlewares/auth.middleware";
import { requirePermission } from "../../middlewares/permission.middleware";
import { PERMISSIONS } from "../../common/constants/permission.constant";
import { validate } from "../../middlewares/validate.middleware"; import { validate } from "../../middlewares/validate.middleware";
import { import {
createApiKeySchema, createApiKeySchema,
...@@ -13,16 +15,28 @@ const controller = new ApiKeyController(); ...@@ -13,16 +15,28 @@ const controller = new ApiKeyController();
router.post( router.post(
"/", "/",
authMiddleware, authMiddleware,
requirePermission(PERMISSIONS.API_KEYS_CREATE),
validate(createApiKeySchema), validate(createApiKeySchema),
controller.create, controller.create,
); );
router.get("/", authMiddleware, controller.list); router.get(
"/",
authMiddleware,
requirePermission(PERMISSIONS.API_KEYS_READ),
controller.list,
);
router.patch( router.patch(
"/:id", "/:id",
authMiddleware, authMiddleware,
requirePermission(PERMISSIONS.API_KEYS_UPDATE),
validate(updateApiKeyStatusSchema), validate(updateApiKeyStatusSchema),
controller.setActive, controller.setActive,
); );
router.delete("/:id", authMiddleware, controller.revoke); router.delete(
"/:id",
authMiddleware,
requirePermission(PERMISSIONS.API_KEYS_DELETE),
controller.revoke,
);
export default router; export default router;
...@@ -3,8 +3,8 @@ import { AuditLogController } from "./audit-log.controller"; ...@@ -3,8 +3,8 @@ import { AuditLogController } from "./audit-log.controller";
import { authMiddleware } from "../../middlewares/auth.middleware"; import { authMiddleware } from "../../middlewares/auth.middleware";
import { validateQuery } from "../../middlewares/validate.middleware"; import { validateQuery } from "../../middlewares/validate.middleware";
import { listAuditLogsQuerySchema } from "./audit-log.validation"; import { listAuditLogsQuerySchema } from "./audit-log.validation";
import { requireRole } from "../../middlewares/role.middleware"; import { requirePermission } from "../../middlewares/permission.middleware";
import { ROLES } from "../../common/constants/role.constant"; import { PERMISSIONS } from "../../common/constants/permission.constant";
const router = Router(); const router = Router();
const controller = new AuditLogController(); const controller = new AuditLogController();
...@@ -12,7 +12,7 @@ const controller = new AuditLogController(); ...@@ -12,7 +12,7 @@ const controller = new AuditLogController();
router.get( router.get(
"/", "/",
authMiddleware, authMiddleware,
requireRole(ROLES.ADMIN), requirePermission(PERMISSIONS.AUDIT_LOGS_READ),
validateQuery(listAuditLogsQuerySchema), validateQuery(listAuditLogsQuerySchema),
controller.findAll, controller.findAll,
); );
......
import { prisma } from "../../database/prisma.client"; import { prisma } from "../../database/prisma.client";
import { ROLES } from "../../common/constants/role.constant"; import { ROLES } from "../../common/constants/role.constant";
import { SYSTEM_ROLE_SLUGS } from "../../common/constants/system-role.constant";
export class AuthRepository { export class AuthRepository {
findByEmail(email: string) { findByEmail(email: string) {
...@@ -14,20 +15,37 @@ export class AuthRepository { ...@@ -14,20 +15,37 @@ export class AuthRepository {
}); });
} }
createUser(data: { async createUser(data: {
email: string; email: string;
passwordHash: string; passwordHash: string;
fullName?: string; fullName?: string;
isActive?: boolean; isActive?: boolean;
}) { }) {
return prisma.user.create({ return prisma.$transaction(async (tx) => {
data: { const user = await tx.user.create({
email: data.email, data: {
passwordHash: data.passwordHash, email: data.email,
fullName: data.fullName, passwordHash: data.passwordHash,
role: ROLES.CRAWLER_USER, fullName: data.fullName,
isActive: data.isActive ?? true, role: ROLES.CRAWLER_USER,
}, isActive: data.isActive ?? true,
},
});
const defaultRole = await tx.role.findUnique({
where: { slug: SYSTEM_ROLE_SLUGS.CRAWLER_USER },
});
if (defaultRole) {
await tx.userRoleAssignment.create({
data: {
userId: user.id,
roleId: defaultRole.id,
},
});
}
return user;
}); });
} }
......
...@@ -5,7 +5,10 @@ import { ...@@ -5,7 +5,10 @@ import {
copyRefreshTokenToBody, copyRefreshTokenToBody,
} from "../../middlewares/auth.middleware"; } from "../../middlewares/auth.middleware";
import { authRateLimiter } from "../../middlewares/rate-limit.middleware"; import { authRateLimiter } from "../../middlewares/rate-limit.middleware";
import { validate } from "../../middlewares/validate.middleware"; import {
validate,
validateParams,
} from "../../middlewares/validate.middleware";
import { uploadAvatarMiddleware } from "../../middlewares/upload.middleware"; import { uploadAvatarMiddleware } from "../../middlewares/upload.middleware";
import { import {
loginSchema, loginSchema,
...@@ -20,6 +23,7 @@ import { ...@@ -20,6 +23,7 @@ import {
changePasswordSchema, changePasswordSchema,
requestDeactivationSchema, requestDeactivationSchema,
confirmDeactivationSchema, confirmDeactivationSchema,
avatarFileNameParamsSchema,
} from "./auth.validation"; } from "./auth.validation";
const router = Router(); const router = Router();
...@@ -75,9 +79,13 @@ router.post( ...@@ -75,9 +79,13 @@ router.post(
controller.uploadAvatar(req, res, next); controller.uploadAvatar(req, res, next);
}, },
); );
router.get("/avatar/:fileName", (req, res, next) => { router.get(
controller.getAvatar(req, res, next); "/avatar/:fileName",
}); validateParams(avatarFileNameParamsSchema),
(req, res, next) => {
controller.getAvatar(req, res, next);
},
);
router.post( router.post(
"/change-password", "/change-password",
authMiddleware, authMiddleware,
......
...@@ -104,3 +104,14 @@ export const requestDeactivationSchema = z.object({ ...@@ -104,3 +104,14 @@ export const requestDeactivationSchema = z.object({
export const confirmDeactivationSchema = z.object({ export const confirmDeactivationSchema = z.object({
token: z.string().min(1, "Thiếu mã xác nhận vô hiệu hóa."), token: z.string().min(1, "Thiếu mã xác nhận vô hiệu hóa."),
}); });
export const avatarFileNameParamsSchema = z.object({
fileName: z
.string()
.trim()
.regex(
/^[a-zA-Z0-9_.-]+\.(jpg|jpeg|png|webp|gif)$/i,
"Invalid avatar filename format",
)
.refine((name) => !name.includes(".."), "Path traversal is not allowed"),
});
...@@ -14,6 +14,7 @@ import { buildJobRootFilePath } from "../../common/helpers/file.helper"; ...@@ -14,6 +14,7 @@ import { buildJobRootFilePath } from "../../common/helpers/file.helper";
import { normalizeUrl } from "../../common/helpers/data-contract.helper"; import { normalizeUrl } from "../../common/helpers/data-contract.helper";
import { AppError } from "../../common/errors/app-error"; import { AppError } from "../../common/errors/app-error";
import { ERROR_CODE } from "../../common/errors/error-code"; import { ERROR_CODE } from "../../common/errors/error-code";
import { CrawlPageStatus } from "../../common/constants/crawl-page-status.constant";
/** /**
* Minimal page shape required for diff comparison. * Minimal page shape required for diff comparison.
...@@ -26,7 +27,7 @@ type DiffPage = { ...@@ -26,7 +27,7 @@ type DiffPage = {
normalizedUrl: string; normalizedUrl: string;
contentHash: string | null; contentHash: string | null;
wordCount: number; wordCount: number;
status: import("@prisma/client").CrawlPageStatus; status: CrawlPageStatus;
statusCode: number | null; statusCode: number | null;
title: string | null; title: string | null;
crawledAt: Date | null; crawledAt: Date | null;
......
...@@ -41,11 +41,14 @@ export class CrawlExportController { ...@@ -41,11 +41,14 @@ export class CrawlExportController {
const result = await this.service.findAllByUser(req.user.id, page, limit); const result = await this.service.findAllByUser(req.user.id, page, limit);
res.json({ res.json({
success: true, success: true,
data: result.items, data: {
pagination: { items: result.items,
total: result.total, meta: {
page: result.page, total: result.total,
limit: result.limit, page: result.page,
limit: result.limit,
totalPages: Math.ceil(result.total / (result.limit || 1)),
},
}, },
}); });
} catch (error) { } catch (error) {
......
import { Router } from "express"; import { Router } from "express";
import { CrawlExportController } from "./crawl-export.controller"; import { CrawlExportController } from "./crawl-export.controller";
import { authMiddleware } from "../../middlewares/auth.middleware"; import { authMiddleware } from "../../middlewares/auth.middleware";
import { requireRole } from "../../middlewares/role.middleware"; import { requirePermission } from "../../middlewares/permission.middleware";
import { ROLES } from "../../common/constants/role.constant"; import { PERMISSIONS } from "../../common/constants/permission.constant";
import {
validateQuery,
validateParams,
} from "../../middlewares/validate.middleware";
import {
crawlExportQuerySchema,
crawlExportParamsSchema,
} from "./crawl-export.validation";
const router = Router(); const router = Router();
const controller = new CrawlExportController(); const controller = new CrawlExportController();
...@@ -10,19 +18,22 @@ const controller = new CrawlExportController(); ...@@ -10,19 +18,22 @@ const controller = new CrawlExportController();
router.get( router.get(
"/", "/",
authMiddleware, authMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.EXPORTS_READ),
validateQuery(crawlExportQuerySchema),
controller.findAll, controller.findAll,
); );
router.get( router.get(
"/:exportId/download", "/:exportId/download",
authMiddleware, authMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.EXPORTS_DOWNLOAD),
validateParams(crawlExportParamsSchema),
controller.download, controller.download,
); );
router.delete( router.delete(
"/:exportId", "/:exportId",
authMiddleware, authMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER), requirePermission(PERMISSIONS.EXPORTS_DELETE),
validateParams(crawlExportParamsSchema),
controller.delete, controller.delete,
); );
......
import { z } from "zod";
export const crawlExportQuerySchema = z.object({
page: z.coerce.number().int().min(1).default(1),
limit: z.coerce.number().int().min(1).max(100).default(20),
});
export const crawlExportParamsSchema = z.object({
exportId: z.string().uuid("Invalid export ID format"),
});
...@@ -7,10 +7,12 @@ import { ...@@ -7,10 +7,12 @@ import {
createExportSchema, createExportSchema,
listCrawlJobsQuerySchema, listCrawlJobsQuerySchema,
getAssetsQuerySchema, getAssetsQuerySchema,
jobLogsQuerySchema,
diffQuerySchema,
} from "./crawl-job.validation"; } from "./crawl-job.validation";
import { crawlPageQuerySchema } from "../crawl-pages/crawl-page.validation"; import { crawlPageQuerySchema } from "../crawl-pages/crawl-page.validation";
import { requireRole } from "../../middlewares/role.middleware"; import { requirePermission } from "../../middlewares/permission.middleware";
import { ROLES } from "../../common/constants/role.constant"; import { PERMISSIONS } from "../../common/constants/permission.constant";
const router = Router(); const router = Router();
const controller = new CrawlJobController(); const controller = new CrawlJobController();
...@@ -18,7 +20,7 @@ const controller = new CrawlJobController(); ...@@ -18,7 +20,7 @@ const controller = new CrawlJobController();
router.post( router.post(
"/", "/",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER), requirePermission(PERMISSIONS.CRAWL_JOBS_CREATE),
validate(createCrawlJobSchema), validate(createCrawlJobSchema),
(req, res, next) => { (req, res, next) => {
controller.create(req, res, next); controller.create(req, res, next);
...@@ -27,70 +29,71 @@ router.post( ...@@ -27,70 +29,71 @@ router.post(
router.get( router.get(
"/", "/",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_JOBS_READ),
validateQuery(listCrawlJobsQuerySchema), validateQuery(listCrawlJobsQuerySchema),
controller.findAll, controller.findAll,
); );
router.get( router.get(
"/:id", "/:id",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_JOBS_READ),
controller.findById, controller.findById,
); );
router.delete( router.delete(
"/:id", "/:id",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER), requirePermission(PERMISSIONS.CRAWL_JOBS_DELETE),
controller.delete, controller.delete,
); );
router.post( router.post(
"/:id/rerun", "/:id/rerun",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER), requirePermission(PERMISSIONS.CRAWL_JOBS_RETRY),
controller.rerun, controller.rerun,
); );
router.get( router.get(
"/:id/logs", "/:id/logs",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_JOBS_READ),
validateQuery(jobLogsQuerySchema),
controller.getLogs, controller.getLogs,
); );
router.get( router.get(
"/:id/events", "/:id/events",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_JOBS_READ),
controller.streamEvents, controller.streamEvents,
); );
router.post( router.post(
"/:id/cancel", "/:id/cancel",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER), requirePermission(PERMISSIONS.CRAWL_JOBS_CANCEL),
controller.cancel, controller.cancel,
); );
router.get( router.get(
"/:id/pages", "/:id/pages",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_JOBS_READ),
validateQuery(crawlPageQuerySchema), validateQuery(crawlPageQuerySchema),
controller.getPages, controller.getPages,
); );
router.get( router.get(
"/:id/pages/preview", "/:id/pages/preview",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_JOBS_READ),
validateQuery(crawlPageQuerySchema), validateQuery(crawlPageQuerySchema),
controller.getPagesPreview, controller.getPagesPreview,
); );
router.get( router.get(
"/:id/exports", "/:id/exports",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.EXPORTS_READ),
controller.getExports, controller.getExports,
); );
router.post( router.post(
"/:id/exports", "/:id/exports",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER), requirePermission(PERMISSIONS.EXPORTS_CREATE),
validate(createExportSchema), validate(createExportSchema),
(req, res, next) => { (req, res, next) => {
controller.createExport(req, res, next); controller.createExport(req, res, next);
...@@ -99,26 +102,28 @@ router.post( ...@@ -99,26 +102,28 @@ router.post(
router.get( router.get(
"/:id/download", "/:id/download",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.EXPORTS_DOWNLOAD),
controller.download, controller.download,
); );
router.get( router.get(
"/:id/assets", "/:id/assets",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_JOBS_READ),
validateQuery(getAssetsQuerySchema), validateQuery(getAssetsQuerySchema),
controller.getAssets, controller.getAssets,
); );
router.get( router.get(
"/:id/diff", "/:id/diff",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_JOBS_READ),
validateQuery(diffQuerySchema),
controller.getDiff, controller.getDiff,
); );
router.get( router.get(
"/:id/diff/download", "/:id/diff/download",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_JOBS_READ),
validateQuery(diffQuerySchema),
controller.downloadDiff, controller.downloadDiff,
); );
export default router; export default router;
...@@ -12,7 +12,10 @@ import { ...@@ -12,7 +12,10 @@ import {
import { crawlQueue } from "../../queues/crawl.queue"; import { crawlQueue } from "../../queues/crawl.queue";
import { ROLES } from "../../common/constants/role.constant"; import { ROLES } from "../../common/constants/role.constant";
import { JOB_STATUS } from "../../common/constants/job-status.constant"; import { JOB_STATUS } from "../../common/constants/job-status.constant";
import { EXPORT_TYPE } from "../../common/constants/export-type.constant"; import {
EXPORT_TYPE,
EXPORT_STATUS,
} from "../../common/constants/export-type.constant";
import { DEFAULT_TIMEZONE } from "../../common/constants/timezone.constant"; import { DEFAULT_TIMEZONE } from "../../common/constants/timezone.constant";
import { CRAWL_MODE } from "../../common/constants/crawl-mode.constant"; import { CRAWL_MODE } from "../../common/constants/crawl-mode.constant";
import { CreateCrawlJobDto, CrawlJobQueryDto } from "./crawl-job.dto"; import { CreateCrawlJobDto, CrawlJobQueryDto } from "./crawl-job.dto";
...@@ -246,7 +249,7 @@ export class CrawlJobService { ...@@ -246,7 +249,7 @@ export class CrawlJobService {
for (const exportRecord of exports) { for (const exportRecord of exports) {
if ( if (
exportRecord.exportType === EXPORT_TYPE.ZIP && exportRecord.exportType === EXPORT_TYPE.ZIP &&
exportRecord.status === JOB_STATUS.COMPLETED && exportRecord.status === EXPORT_STATUS.COMPLETED &&
(await storage.exists(exportRecord.filePath)) (await storage.exists(exportRecord.filePath))
) { ) {
return exportRecord; return exportRecord;
......
...@@ -74,3 +74,15 @@ export const getAssetsQuerySchema = z.object({ ...@@ -74,3 +74,15 @@ export const getAssetsQuerySchema = z.object({
page: z.coerce.number().int().min(1).default(1), page: z.coerce.number().int().min(1).default(1),
limit: z.coerce.number().int().min(1).max(500).default(50), limit: z.coerce.number().int().min(1).max(500).default(50),
}); });
export const jobLogsQuerySchema = z.object({
page: z.coerce.number().int().min(1).default(1),
limit: z.coerce.number().int().min(1).max(100).default(50),
});
export const diffQuerySchema = z.object({
compareWithJobId: z
.string()
.uuid("Invalid compareWithJobId format")
.optional(),
});
...@@ -6,9 +6,10 @@ import { ...@@ -6,9 +6,10 @@ import {
createCrawlScheduleSchema, createCrawlScheduleSchema,
updateCrawlScheduleSchema, updateCrawlScheduleSchema,
crawlScheduleQuerySchema, crawlScheduleQuerySchema,
crawlScheduleHistoryQuerySchema,
} from "./crawl-schedule.validation"; } from "./crawl-schedule.validation";
import { requireRole } from "../../middlewares/role.middleware"; import { requirePermission } from "../../middlewares/permission.middleware";
import { ROLES } from "../../common/constants/role.constant"; import { PERMISSIONS } from "../../common/constants/permission.constant";
const router = Router(); const router = Router();
const controller = new CrawlScheduleController(); const controller = new CrawlScheduleController();
...@@ -16,7 +17,7 @@ const controller = new CrawlScheduleController(); ...@@ -16,7 +17,7 @@ const controller = new CrawlScheduleController();
router.post( router.post(
"/", "/",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER), requirePermission(PERMISSIONS.CRAWL_SCHEDULES_CREATE),
validate(createCrawlScheduleSchema), validate(createCrawlScheduleSchema),
controller.create, controller.create,
); );
...@@ -24,7 +25,7 @@ router.post( ...@@ -24,7 +25,7 @@ router.post(
router.get( router.get(
"/", "/",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_SCHEDULES_READ),
validateQuery(crawlScheduleQuerySchema), validateQuery(crawlScheduleQuerySchema),
controller.findAll, controller.findAll,
); );
...@@ -32,14 +33,14 @@ router.get( ...@@ -32,14 +33,14 @@ router.get(
router.get( router.get(
"/:id", "/:id",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_SCHEDULES_READ),
controller.findById, controller.findById,
); );
router.patch( router.patch(
"/:id", "/:id",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER), requirePermission(PERMISSIONS.CRAWL_SCHEDULES_UPDATE),
validate(updateCrawlScheduleSchema), validate(updateCrawlScheduleSchema),
controller.update, controller.update,
); );
...@@ -47,21 +48,22 @@ router.patch( ...@@ -47,21 +48,22 @@ router.patch(
router.delete( router.delete(
"/:id", "/:id",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER), requirePermission(PERMISSIONS.CRAWL_SCHEDULES_DELETE),
controller.delete, controller.delete,
); );
router.post( router.post(
"/:id/run", "/:id/run",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER), requirePermission(PERMISSIONS.CRAWL_SCHEDULES_RUN),
controller.triggerRun, controller.triggerRun,
); );
router.get( router.get(
"/:id/history", "/:id/history",
apiKeyOrAuthMiddleware, apiKeyOrAuthMiddleware,
requireRole(ROLES.ADMIN, ROLES.CRAWLER_USER, ROLES.VIEWER), requirePermission(PERMISSIONS.CRAWL_SCHEDULES_READ),
validateQuery(crawlScheduleHistoryQuerySchema),
controller.getHistory, controller.getHistory,
); );
......
...@@ -106,3 +106,8 @@ export const crawlScheduleQuerySchema = z.object({ ...@@ -106,3 +106,8 @@ export const crawlScheduleQuerySchema = z.object({
.default("createdAt"), .default("createdAt"),
order: z.enum(["asc", "desc"]).optional().default("desc"), order: z.enum(["asc", "desc"]).optional().default("desc"),
}); });
export const crawlScheduleHistoryQuerySchema = z.object({
page: z.coerce.number().int().min(1).default(1),
limit: z.coerce.number().int().min(1).max(100).default(20),
});
import { prisma } from "../../database/prisma.client"; import { prisma } from "../../database/prisma.client";
import { ROLES } from "../../common/constants/role.constant"; import { ROLES } from "../../common/constants/role.constant";
import { JOB_STATUS } from "../../common/constants/job-status.constant"; import { JOB_STATUS } from "../../common/constants/job-status.constant";
import { CrawlPageStatus } from "@prisma/client"; import { CRAWL_PAGE_STATUS } from "../../common/constants/crawl-page-status.constant";
export class DashboardRepository { export class DashboardRepository {
async getStats(userId: string, role: string) { async getStats(userId: string, role: string) {
...@@ -12,41 +12,24 @@ export class DashboardRepository { ...@@ -12,41 +12,24 @@ export class DashboardRepository {
const exportWhere = isGlobal ? {} : { job: { userId } }; const exportWhere = isGlobal ? {} : { job: { userId } };
const [ const [
totalJobs, jobStatusGroups,
completedJobs, pageStatusGroups,
failedJobs,
runningJobs,
pendingJobs,
totalPagesCrawled, totalPagesCrawled,
successfulPages,
failedPages,
activeSchedules, activeSchedules,
totalSchedules, totalSchedules,
totalExports, totalExports,
] = await Promise.all([ ] = await Promise.all([
prisma.crawlJob.count({ where: jobWhere }), prisma.crawlJob.groupBy({
prisma.crawlJob.count({ by: ["status"],
where: { ...jobWhere, status: JOB_STATUS.COMPLETED }, _count: { status: true },
where: jobWhere,
}), }),
prisma.crawlJob.count({ prisma.crawlPage.groupBy({
where: { ...jobWhere, status: JOB_STATUS.FAILED }, by: ["status"],
}), _count: { status: true },
prisma.crawlJob.count({ where: pageWhere,
where: { ...jobWhere, status: JOB_STATUS.RUNNING },
}),
prisma.crawlJob.count({
where: {
...jobWhere,
status: { in: [JOB_STATUS.PENDING, JOB_STATUS.QUEUED] },
},
}), }),
prisma.crawlPage.count({ where: pageWhere }), prisma.crawlPage.count({ where: pageWhere }),
prisma.crawlPage.count({
where: { ...pageWhere, status: CrawlPageStatus.SUCCESS },
}),
prisma.crawlPage.count({
where: { ...pageWhere, status: CrawlPageStatus.FAILED },
}),
prisma.crawlSchedule.count({ prisma.crawlSchedule.count({
where: { ...scheduleWhere, isActive: true }, where: { ...scheduleWhere, isActive: true },
}), }),
...@@ -54,18 +37,32 @@ export class DashboardRepository { ...@@ -54,18 +37,32 @@ export class DashboardRepository {
prisma.crawlExport.count({ where: exportWhere }), prisma.crawlExport.count({ where: exportWhere }),
]); ]);
const jobCounts: Record<string, number> = {};
let totalJobs = 0;
for (const group of jobStatusGroups) {
jobCounts[group.status] = group._count.status;
totalJobs += group._count.status;
}
const pageCounts: Record<string, number> = {};
for (const group of pageStatusGroups) {
pageCounts[group.status] = group._count.status;
}
return { return {
jobs: { jobs: {
total: totalJobs, total: totalJobs,
completed: completedJobs, completed: jobCounts[JOB_STATUS.COMPLETED] ?? 0,
failed: failedJobs, failed: jobCounts[JOB_STATUS.FAILED] ?? 0,
running: runningJobs, running: jobCounts[JOB_STATUS.RUNNING] ?? 0,
pending: pendingJobs, pending:
(jobCounts[JOB_STATUS.PENDING] ?? 0) +
(jobCounts[JOB_STATUS.QUEUED] ?? 0),
}, },
pages: { pages: {
total: totalPagesCrawled, total: totalPagesCrawled,
successful: successfulPages, successful: pageCounts[CRAWL_PAGE_STATUS.SUCCESS] ?? 0,
failed: failedPages, failed: pageCounts[CRAWL_PAGE_STATUS.FAILED] ?? 0,
}, },
schedules: { schedules: {
total: totalSchedules, total: totalSchedules,
......
import { Router } from "express"; import { Router } from "express";
import { DashboardController } from "./dashboard.controller"; import { DashboardController } from "./dashboard.controller";
import { authMiddleware } from "../../middlewares/auth.middleware"; import { authMiddleware } from "../../middlewares/auth.middleware";
import { requirePermission } from "../../middlewares/permission.middleware";
import { PERMISSIONS } from "../../common/constants/permission.constant";
const router = Router(); const router = Router();
const controller = new DashboardController(); const controller = new DashboardController();
router.get("/stats", authMiddleware, controller.getStats); router.get(
"/stats",
authMiddleware,
requirePermission(PERMISSIONS.DASHBOARD_READ),
controller.getStats,
);
export default router; export default router;
...@@ -60,17 +60,22 @@ export class JsonExportService extends BaseExportService { ...@@ -60,17 +60,22 @@ export class JsonExportService extends BaseExportService {
fs.writeFileSync(filePath, JSON.stringify(wrapper, null, 2), "utf-8"); fs.writeFileSync(filePath, JSON.stringify(wrapper, null, 2), "utf-8");
// 1. Xuất pages.raw.json (Lọc bỏ các trường dữ liệu sạch & chất lượng) // 1. Xuất pages.raw.json (Lọc bỏ các trường dữ liệu sạch & chất lượng)
const rawPages = pages.map( const rawPages = pages.map((page) => ({
({ id: page.id,
cleanText: _cleanText, jobId: page.jobId,
mainContent: _mainContent, url: page.url,
wordCount: _wordCount, normalizedUrl: page.normalizedUrl,
contentHash: _contentHash, status: page.status,
dataQualityScore: _dataQualityScore, statusCode: page.statusCode,
warnings: _warnings, errorMessage: page.errorMessage,
...rawFields title: page.title,
}) => rawFields, description: page.description,
); rawMarkdown: page.rawMarkdown,
links: page.links,
images: page.images,
tables: page.tables,
crawledAt: page.crawledAt,
}));
const { filePath: rawFilePath } = buildJobDataRawFilePath( const { filePath: rawFilePath } = buildJobDataRawFilePath(
job.id, job.id,
JOB_EXPORT_FILES.PAGES_RAW_JSON, JOB_EXPORT_FILES.PAGES_RAW_JSON,
...@@ -79,9 +84,27 @@ export class JsonExportService extends BaseExportService { ...@@ -79,9 +84,27 @@ export class JsonExportService extends BaseExportService {
fs.writeFileSync(rawFilePath, JSON.stringify(rawWrapper, null, 2), "utf-8"); fs.writeFileSync(rawFilePath, JSON.stringify(rawWrapper, null, 2), "utf-8");
// 2. Xuất pages.clean.json (Lọc bỏ trường rawMarkdown) // 2. Xuất pages.clean.json (Lọc bỏ trường rawMarkdown)
const cleanPages = pages.map( const cleanPages = pages.map((page) => ({
({ rawMarkdown: _rawMarkdown, ...cleanFields }) => cleanFields, id: page.id,
); jobId: page.jobId,
url: page.url,
normalizedUrl: page.normalizedUrl,
status: page.status,
statusCode: page.statusCode,
errorMessage: page.errorMessage,
title: page.title,
description: page.description,
cleanText: page.cleanText,
mainContent: page.mainContent,
wordCount: page.wordCount,
contentHash: page.contentHash,
dataQualityScore: page.dataQualityScore,
warnings: page.warnings,
links: page.links,
images: page.images,
tables: page.tables,
crawledAt: page.crawledAt,
}));
const { filePath: cleanFilePath } = buildJobDataCleanFilePath( const { filePath: cleanFilePath } = buildJobDataCleanFilePath(
job.id, job.id,
JOB_EXPORT_FILES.PAGES_CLEAN_JSON, JOB_EXPORT_FILES.PAGES_CLEAN_JSON,
......
...@@ -11,6 +11,7 @@ import { ...@@ -11,6 +11,7 @@ import {
buildCrawlResultZipName, buildCrawlResultZipName,
} from "../../common/constants/storage-path.constant"; } from "../../common/constants/storage-path.constant";
import { EXPORT_MIME_TYPES } from "../../common/constants/export-type.constant"; import { EXPORT_MIME_TYPES } from "../../common/constants/export-type.constant";
import { CRAWL_PAGE_STATUS } from "../../common/constants/crawl-page-status.constant";
import { import {
buildJobLogsFilePath, buildJobLogsFilePath,
buildJobRootFilePath, buildJobRootFilePath,
...@@ -101,9 +102,13 @@ export class ZipExportService extends BaseExportService { ...@@ -101,9 +102,13 @@ export class ZipExportService extends BaseExportService {
private writeSummary(job: CrawlJob & { pages: CrawlPage[] }): void { private writeSummary(job: CrawlJob & { pages: CrawlPage[] }): void {
const { filePath } = buildJobRootFilePath(job.id, JOB_EXPORT_FILES.SUMMARY); const { filePath } = buildJobRootFilePath(job.id, JOB_EXPORT_FILES.SUMMARY);
const successPages = job.pages.filter((p) => p.status === "SUCCESS").length; const successPages = job.pages.filter(
(p) => p.status === CRAWL_PAGE_STATUS.SUCCESS,
).length;
const failedPages = job.pages.filter( const failedPages = job.pages.filter(
(p) => p.status !== "SUCCESS" && p.status !== "SKIPPED", (p) =>
p.status !== CRAWL_PAGE_STATUS.SUCCESS &&
p.status !== CRAWL_PAGE_STATUS.SKIPPED,
).length; ).length;
const summary = { const summary = {
...@@ -125,9 +130,9 @@ export class ZipExportService extends BaseExportService { ...@@ -125,9 +130,9 @@ export class ZipExportService extends BaseExportService {
const errors = job.pages const errors = job.pages
.filter( .filter(
(p) => (p) =>
p.status !== "SUCCESS" && p.status !== CRAWL_PAGE_STATUS.SUCCESS &&
p.status !== "PENDING" && p.status !== CRAWL_PAGE_STATUS.PENDING &&
p.status !== "SKIPPED", p.status !== CRAWL_PAGE_STATUS.SKIPPED,
) )
.map((p) => ({ .map((p) => ({
url: p.url, url: p.url,
...@@ -145,12 +150,14 @@ export class ZipExportService extends BaseExportService { ...@@ -145,12 +150,14 @@ export class ZipExportService extends BaseExportService {
JOB_EXPORT_FILES.DATA_QUALITY_JSON, JOB_EXPORT_FILES.DATA_QUALITY_JSON,
); );
const successPages = job.pages.filter((p) => p.status === "SUCCESS"); const successPages = job.pages.filter(
(p) => p.status === CRAWL_PAGE_STATUS.SUCCESS,
);
const errorPages = job.pages.filter( const errorPages = job.pages.filter(
(p) => (p) =>
p.status !== "SUCCESS" && p.status !== CRAWL_PAGE_STATUS.SUCCESS &&
p.status !== "SKIPPED" && p.status !== CRAWL_PAGE_STATUS.SKIPPED &&
p.status !== "PENDING", p.status !== CRAWL_PAGE_STATUS.PENDING,
); );
const seenHashes = new Set<string>(); const seenHashes = new Set<string>();
......
import { Router } from "express"; import { Router } from "express";
import { ExtractionTemplateController } from "./extraction-template.controller"; import { ExtractionTemplateController } from "./extraction-template.controller";
import { authMiddleware } from "../../middlewares/auth.middleware"; import { authMiddleware } from "../../middlewares/auth.middleware";
import { requirePermission } from "../../middlewares/permission.middleware";
import { PERMISSIONS } from "../../common/constants/permission.constant";
import { validate } from "../../middlewares/validate.middleware"; import { validate } from "../../middlewares/validate.middleware";
import { import {
createExtractionTemplateSchema, createExtractionTemplateSchema,
...@@ -12,14 +14,32 @@ const controller = new ExtractionTemplateController(); ...@@ -12,14 +14,32 @@ const controller = new ExtractionTemplateController();
router.use(authMiddleware); router.use(authMiddleware);
router.post("/", validate(createExtractionTemplateSchema), controller.create); router.post(
router.get("/", controller.findAll); "/",
router.get("/:id", controller.findById); requirePermission(PERMISSIONS.EXTRACTION_TEMPLATES_CREATE),
validate(createExtractionTemplateSchema),
controller.create,
);
router.get(
"/",
requirePermission(PERMISSIONS.EXTRACTION_TEMPLATES_READ),
controller.findAll,
);
router.get(
"/:id",
requirePermission(PERMISSIONS.EXTRACTION_TEMPLATES_READ),
controller.findById,
);
router.patch( router.patch(
"/:id", "/:id",
requirePermission(PERMISSIONS.EXTRACTION_TEMPLATES_UPDATE),
validate(updateExtractionTemplateSchema), validate(updateExtractionTemplateSchema),
controller.update, controller.update,
); );
router.delete("/:id", controller.delete); router.delete(
"/:id",
requirePermission(PERMISSIONS.EXTRACTION_TEMPLATES_DELETE),
controller.delete,
);
export default router; export default router;
import { HealthService } from "../health.service"; import { HealthService } from "../health.service";
import { prisma } from "../../../database/prisma.client"; import { HealthRepository } from "../health.repository";
jest.mock("../../../database/prisma.client", () => ({ jest.mock("../health.repository");
prisma: {
$queryRaw: jest.fn(),
},
}));
jest.mock("../../../queues/crawl.queue", () => ({ jest.mock("../../../queues/crawl.queue", () => ({
crawlQueue: { crawlQueue: {
...@@ -28,10 +24,12 @@ jest.mock("../../../queues/webhook.queue", () => ({ ...@@ -28,10 +24,12 @@ jest.mock("../../../queues/webhook.queue", () => ({
describe("HealthService", () => { describe("HealthService", () => {
let service: HealthService; let service: HealthService;
let mockHealthRepo: jest.Mocked<HealthRepository>;
beforeEach(() => { beforeEach(() => {
jest.clearAllMocks(); jest.clearAllMocks();
service = new HealthService(); mockHealthRepo = new HealthRepository() as jest.Mocked<HealthRepository>;
service = new HealthService(mockHealthRepo);
}); });
describe("getLiveness", () => { describe("getLiveness", () => {
...@@ -45,7 +43,7 @@ describe("HealthService", () => { ...@@ -45,7 +43,7 @@ describe("HealthService", () => {
describe("getReadiness", () => { describe("getReadiness", () => {
it("returns ready status when database is up", async () => { it("returns ready status when database is up", async () => {
(prisma.$queryRaw as jest.Mock).mockResolvedValue([{ 1: 1 }]); mockHealthRepo.pingDatabase.mockResolvedValue();
const result = await service.getReadiness(); const result = await service.getReadiness();
expect(result.status).toBe("ready"); expect(result.status).toBe("ready");
...@@ -54,7 +52,7 @@ describe("HealthService", () => { ...@@ -54,7 +52,7 @@ describe("HealthService", () => {
}); });
it("returns unhealthy status when database query fails", async () => { it("returns unhealthy status when database query fails", async () => {
(prisma.$queryRaw as jest.Mock).mockRejectedValue( mockHealthRepo.pingDatabase.mockRejectedValue(
new Error("Connection timeout"), new Error("Connection timeout"),
); );
......
import { prisma } from "../../database/prisma.client";
export class HealthRepository {
async pingDatabase(): Promise<void> {
await prisma.$queryRaw`SELECT 1`;
}
}
import { prisma } from "../../database/prisma.client"; import { HealthRepository } from "./health.repository";
import { crawlQueue } from "../../queues/crawl.queue"; import { crawlQueue } from "../../queues/crawl.queue";
import { webhookQueue } from "../../queues/webhook.queue"; import { webhookQueue } from "../../queues/webhook.queue";
import { getErrorMessage } from "../../common/helpers/error-mapping.helper"; import { getErrorMessage } from "../../common/helpers/error-mapping.helper";
...@@ -13,6 +13,8 @@ export interface QueueCountMetrics { ...@@ -13,6 +13,8 @@ export interface QueueCountMetrics {
export type QueueMetricsResult = QueueCountMetrics | "unavailable" | null; export type QueueMetricsResult = QueueCountMetrics | "unavailable" | null;
export class HealthService { export class HealthService {
constructor(private readonly repository = new HealthRepository()) {}
getLiveness() { getLiveness() {
return { return {
status: "ok", status: "ok",
...@@ -32,7 +34,7 @@ export class HealthService { ...@@ -32,7 +34,7 @@ export class HealthService {
// 1. Check Database // 1. Check Database
const dbStart = Date.now(); const dbStart = Date.now();
try { try {
await prisma.$queryRaw`SELECT 1`; await this.repository.pingDatabase();
checks.database = { checks.database = {
status: "up", status: "up",
latencyMs: Date.now() - dbStart, latencyMs: Date.now() - dbStart,
......
...@@ -76,6 +76,22 @@ export class RoleRepository { ...@@ -76,6 +76,22 @@ export class RoleRepository {
}); });
} }
async findByIds(ids: string[]) {
return prisma.role.findMany({
where: { id: { in: ids } },
include: {
rolePermissions: {
include: {
permission: true,
},
},
_count: {
select: { userRoles: true },
},
},
});
}
async findBySlug(slug: string) { async findBySlug(slug: string) {
return prisma.role.findUnique({ return prisma.role.findUnique({
where: { slug }, where: { slug },
......
...@@ -154,6 +154,21 @@ export class UserRepository { ...@@ -154,6 +154,21 @@ export class UserRepository {
where: { userId: id }, where: { userId: id },
}); });
await tx.crawlSchedule.updateMany({
where: { userId: id },
data: { isActive: false },
});
await tx.apiKey.updateMany({
where: { userId: id },
data: { isActive: false },
});
await tx.webhookConfig.updateMany({
where: { userId: id },
data: { isActive: false },
});
return user; return user;
}); });
} }
......
...@@ -261,12 +261,9 @@ export class UserService { ...@@ -261,12 +261,9 @@ export class UserService {
await this.findById(targetUserId); await this.findById(targetUserId);
// 2. Fetch target roles to validate // 2. Fetch target roles to validate
const targetRoles = await Promise.all( const targetRoles = await this.roleRepository.findByIds(roleIds);
roleIds.map((id) => this.roleRepository.findById(id)),
);
const missingRole = targetRoles.find((r) => !r); if (targetRoles.length !== roleIds.length) {
if (missingRole || targetRoles.length !== roleIds.length) {
throw new AppError( throw new AppError(
"One or more roles not found", "One or more roles not found",
404, 404,
......
...@@ -7,6 +7,7 @@ import { getErrorMessage } from "../../common/helpers/error-mapping.helper"; ...@@ -7,6 +7,7 @@ import { getErrorMessage } from "../../common/helpers/error-mapping.helper";
import { AppError } from "../../common/errors/app-error"; import { AppError } from "../../common/errors/app-error";
import { ERROR_CODE } from "../../common/errors/error-code"; import { ERROR_CODE } from "../../common/errors/error-code";
import { WEBHOOK_DELIVERY_STATUS } from "../../common/constants/webhook.constant";
export class WebhookDeliveryService { export class WebhookDeliveryService {
private readonly repository = new WebhookRepository(); private readonly repository = new WebhookRepository();
...@@ -40,7 +41,7 @@ export class WebhookDeliveryService { ...@@ -40,7 +41,7 @@ export class WebhookDeliveryService {
crawlJobId, crawlJobId,
event, event,
payload, payload,
status: "PENDING", status: WEBHOOK_DELIVERY_STATUS.PENDING,
attempt: 1, attempt: 1,
}); });
...@@ -100,7 +101,7 @@ export class WebhookDeliveryService { ...@@ -100,7 +101,7 @@ export class WebhookDeliveryService {
: JSON.stringify(response.data); : JSON.stringify(response.data);
await this.repository.updateDelivery(deliveryId, { await this.repository.updateDelivery(deliveryId, {
status: "SUCCESS", status: WEBHOOK_DELIVERY_STATUS.SUCCESS,
statusCode: response.status, statusCode: response.status,
responseBody: responseBody.substring(0, 2000), // Limit size stored in DB responseBody: responseBody.substring(0, 2000), // Limit size stored in DB
deliveredAt: new Date(), deliveredAt: new Date(),
...@@ -141,7 +142,7 @@ export class WebhookDeliveryService { ...@@ -141,7 +142,7 @@ export class WebhookDeliveryService {
*/ */
async markFailed(deliveryId: string, errorReason: string): Promise<void> { async markFailed(deliveryId: string, errorReason: string): Promise<void> {
await this.repository.updateDelivery(deliveryId, { await this.repository.updateDelivery(deliveryId, {
status: "FAILED", status: WEBHOOK_DELIVERY_STATUS.FAILED,
errorMessage: errorMessage:
`Max attempts exhausted. Last error: ${errorReason}`.substring(0, 1000), `Max attempts exhausted. Last error: ${errorReason}`.substring(0, 1000),
}); });
...@@ -162,7 +163,7 @@ export class WebhookDeliveryService { ...@@ -162,7 +163,7 @@ export class WebhookDeliveryService {
} }
const updated = await this.repository.updateDelivery(deliveryId, { const updated = await this.repository.updateDelivery(deliveryId, {
status: "PENDING", status: WEBHOOK_DELIVERY_STATUS.PENDING,
attempt: 1, attempt: 1,
errorMessage: null, errorMessage: null,
}); });
......
import { Router } from "express"; import { Router } from "express";
import { WebhookController } from "./webhook.controller"; import { WebhookController } from "./webhook.controller";
import { authMiddleware } from "../../middlewares/auth.middleware"; import { authMiddleware } from "../../middlewares/auth.middleware";
import { requirePermission } from "../../middlewares/permission.middleware";
import { PERMISSIONS } from "../../common/constants/permission.constant";
import { validate, validateQuery } from "../../middlewares/validate.middleware"; import { validate, validateQuery } from "../../middlewares/validate.middleware";
import { import {
createWebhookConfigSchema, createWebhookConfigSchema,
...@@ -14,24 +16,47 @@ const controller = new WebhookController(); ...@@ -14,24 +16,47 @@ const controller = new WebhookController();
router.post( router.post(
"/configs", "/configs",
authMiddleware, authMiddleware,
requirePermission(PERMISSIONS.WEBHOOKS_CREATE),
validate(createWebhookConfigSchema), validate(createWebhookConfigSchema),
controller.createConfig, controller.createConfig,
); );
router.get("/configs", authMiddleware, controller.listConfigs); router.get(
"/configs",
authMiddleware,
requirePermission(PERMISSIONS.WEBHOOKS_READ),
controller.listConfigs,
);
router.patch( router.patch(
"/configs/:id", "/configs/:id",
authMiddleware, authMiddleware,
requirePermission(PERMISSIONS.WEBHOOKS_UPDATE),
validate(updateWebhookConfigSchema), validate(updateWebhookConfigSchema),
controller.updateConfig, controller.updateConfig,
); );
router.delete("/configs/:id", authMiddleware, controller.deleteConfig); router.delete(
router.post("/configs/:id/test", authMiddleware, controller.testConfig); "/configs/:id",
authMiddleware,
requirePermission(PERMISSIONS.WEBHOOKS_DELETE),
controller.deleteConfig,
);
router.post(
"/configs/:id/test",
authMiddleware,
requirePermission(PERMISSIONS.WEBHOOKS_TEST),
controller.testConfig,
);
router.get( router.get(
"/deliveries", "/deliveries",
authMiddleware, authMiddleware,
requirePermission(PERMISSIONS.WEBHOOKS_READ),
validateQuery(listWebhookDeliveriesQuerySchema), validateQuery(listWebhookDeliveriesQuerySchema),
controller.listDeliveries, controller.listDeliveries,
); );
router.post("/deliveries/:id/redeliver", authMiddleware, controller.redeliver); router.post(
"/deliveries/:id/redeliver",
authMiddleware,
requirePermission(PERMISSIONS.WEBHOOKS_UPDATE),
controller.redeliver,
);
export default router; export default router;
...@@ -19,6 +19,9 @@ import { ...@@ -19,6 +19,9 @@ import {
import { runExtractionIfTemplate } from "../modules/extraction-templates/extraction-runner"; import { runExtractionIfTemplate } from "../modules/extraction-templates/extraction-runner";
import { JOB_STATUS } from "../common/constants/job-status.constant"; import { JOB_STATUS } from "../common/constants/job-status.constant";
import { CRAWL_MODE } from "../common/constants/crawl-mode.constant"; import { CRAWL_MODE } from "../common/constants/crawl-mode.constant";
import { ASSET_TYPE } from "../common/constants/asset-type.constant";
import { CRAWL_PAGE_STATUS } from "../common/constants/crawl-page-status.constant";
import { WEBHOOK_EVENT } from "../common/constants/webhook.constant";
// Lazy getters — instantiated on first use so Jest mocks replace constructors before creation // Lazy getters — instantiated on first use so Jest mocks replace constructors before creation
const getJobRepository = () => new CrawlJobRepository(); const getJobRepository = () => new CrawlJobRepository();
const getPageRepository = () => new CrawlPageRepository(); const getPageRepository = () => new CrawlPageRepository();
...@@ -58,7 +61,7 @@ export async function savePageAssets( ...@@ -58,7 +61,7 @@ export async function savePageAssets(
assetsBatch.push({ assetsBatch.push({
jobId, jobId,
pageId, pageId,
assetType: "IMAGE" as const, assetType: ASSET_TYPE.IMAGE,
url: img.url, url: img.url,
sourceUrl: item.url, sourceUrl: item.url,
altText: img.alt || undefined, altText: img.alt || undefined,
...@@ -73,7 +76,7 @@ export async function savePageAssets( ...@@ -73,7 +76,7 @@ export async function savePageAssets(
assetsBatch.push({ assetsBatch.push({
jobId, jobId,
pageId, pageId,
assetType: "LINK" as const, assetType: ASSET_TYPE.LINK,
url: link.url, url: link.url,
sourceUrl: item.url, sourceUrl: item.url,
altText: link.text || undefined, altText: link.text || undefined,
...@@ -87,7 +90,7 @@ export async function savePageAssets( ...@@ -87,7 +90,7 @@ export async function savePageAssets(
assetsBatch.push({ assetsBatch.push({
jobId, jobId,
pageId, pageId,
assetType: "PDF" as const, assetType: ASSET_TYPE.PDF,
url: pdfUrl, url: pdfUrl,
sourceUrl: item.url, sourceUrl: item.url,
orderIndex: index + 1, orderIndex: index + 1,
...@@ -196,7 +199,7 @@ export async function persistBatchResults( ...@@ -196,7 +199,7 @@ export async function persistBatchResults(
const normalized = getPageProcessor().normalizeFailedPage( const normalized = getPageProcessor().normalizeFailedPage(
{ url: blockedUrl, error: "Blocked by robots.txt" }, { url: blockedUrl, error: "Blocked by robots.txt" },
jobId, jobId,
"BLOCKED", CRAWL_PAGE_STATUS.BLOCKED,
); );
await getPageRepository().upsert(normalized); await getPageRepository().upsert(normalized);
failedCount++; failedCount++;
...@@ -581,8 +584,8 @@ export async function processCrawlJob(job: Job): Promise<void> { ...@@ -581,8 +584,8 @@ export async function processCrawlJob(job: Job): Promise<void> {
const event = const event =
updatedJob.status === JOB_STATUS.COMPLETED updatedJob.status === JOB_STATUS.COMPLETED
? "job.completed" ? WEBHOOK_EVENT.JOB_COMPLETED
: "job.failed"; : WEBHOOK_EVENT.JOB_FAILED;
void logStep( void logStep(
jobId, jobId,
updatedJob.status === JOB_STATUS.COMPLETED ? "INFO" : "ERROR", updatedJob.status === JOB_STATUS.COMPLETED ? "INFO" : "ERROR",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment