feat: initialize backend source with Express, TypeScript, Prisma, and pnpm

.env.example

+NODE_ENV=development
+PORT=3000
+
+# Database Configuration
+DB_HOST=27.74.255.96
+DB_PORT=5430
+DB_USER=postgres
+DB_PASSWORD="your_password_here"
+DB_NAME=datacrawler
+
+JWT_ACCESS_SECRET=change_me_access_secret
+JWT_REFRESH_SECRET=change_me_refresh_secret
+JWT_ACCESS_EXPIRES_IN=1d
+JWT_REFRESH_EXPIRES_IN=7d
+
+FIRECRAWL_API_KEY=your_firecrawl_api_key
+FIRECRAWL_BASE_URL=https://api.firecrawl.dev
+
+REDIS_HOST=127.0.0.1
+REDIS_PORT=6379
+
+STORAGE_DRIVER=local
+STORAGE_EXPORT_DIR=storage/exports
+
+MAX_CRAWL_PAGES=100
+MAX_CRAWL_DEPTH=3

.gitignore

+node_modules/
+dist/
+.env
+*.log
+storage/exports/*
+!storage/exports/.gitkeep
+.DS_Store

.npmrc

+shamefully-hoist=true

docker-compose.yml

+version: '3.8'
+
+services:
+  postgres:
+    image: postgres:16-alpine
+    container_name: crawl_data_postgres
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: crawl_data_db
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+  redis:
+    image: redis:7-alpine
+    container_name: crawl_data_redis
+    ports:
+      - "6379:6379"
+
+volumes:
+  postgres_data:

package.json

+{
+  "name": "data-crawler-be",
+  "version": "1.0.0",
+  "description": "Backend API for data crawling service",
+  "main": "dist/server.js",
+  "packageManager": "pnpm@9.15.0",
+  "scripts": {
+    "dev": "ts-node-dev --respawn --transpile-only src/server.ts",
+    "worker": "ts-node-dev --respawn --transpile-only src/queues/crawl.worker.ts",
+    "build": "tsc",
+    "start": "node dist/server.js",
+
+    "prisma:generate": "node scripts/prisma-run.js generate",
+    "prisma:studio": "node scripts/prisma-run.js studio",
+
+    "db:migrate": "node scripts/prisma-run.js migrate dev",
+    "db:migrate:init": "node scripts/prisma-run.js migrate dev --name init",
+    "db:migrate:deploy": "node scripts/prisma-run.js migrate deploy",
+    "db:migrate:reset": "node scripts/prisma-run.js migrate reset",
+    "db:migrate:status": "node scripts/prisma-run.js migrate status",
+
+    "db:seed": "node scripts/prisma-run.js db seed -- --tsx prisma/seed.ts",
+
+    "lint": "eslint .",
+    "format": "prettier --write ."
+  },
+  "dependencies": {
+    "@prisma/client": "^5.22.0",
+    "archiver": "^7.0.1",
+    "bcryptjs": "^2.4.3",
+    "bullmq": "^5.34.0",
+    "cors": "^2.8.5",
+    "dotenv": "^16.4.7",
+    "exceljs": "^4.4.0",
+    "express": "^4.21.2",
+    "@mendable/firecrawl-js": "^1.19.0",
+    "helmet": "^8.0.0",
+    "ioredis": "^5.4.2",
+    "json2csv": "^6.0.0-alpha.2",
+    "jsonwebtoken": "^9.0.2",
+    "morgan": "^1.10.0",
+    "turndown": "^7.2.0",
+    "zod": "^3.24.1"
+  },
+  "devDependencies": {
+    "@types/archiver": "^6.0.3",
+    "@types/bcryptjs": "^2.4.6",
+    "@types/cors": "^2.8.17",
+    "@types/express": "^4.17.21",
+    "@types/jsonwebtoken": "^9.0.7",
+    "@types/morgan": "^1.9.9",
+    "@types/node": "^22.10.2",
+    "@types/turndown": "^5.0.5",
+    "eslint": "^9.17.0",
+    "prettier": "^3.4.2",
+    "prisma": "^5.22.0",
+    "ts-node": "^10.9.2",
+    "ts-node-dev": "^2.0.0",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.2"
+  }
+}

prisma/schema.prisma

+generator client {
+  provider = "prisma-client-js"
+}
+
+datasource db {
+  provider = "postgresql"
+  url      = env("DATABASE_URL")
+}
+
+enum UserRole {
+  ADMIN
+  CRAWLER_USER
+  VIEWER
+}
+
+enum CrawlJobStatus {
+  PENDING
+  RUNNING
+  COMPLETED
+  PARTIAL_COMPLETED
+  FAILED
+  CANCELED
+  BLOCKED
+}
+
+enum CrawlMode {
+  SCRAPE
+  CRAWL
+  SITEMAP
+  URL_LIST
+}
+
+enum CrawlPageStatus {
+  PENDING
+  SUCCESS
+  FAILED
+  BLOCKED
+  REQUIRES_LOGIN
+  CAPTCHA_DETECTED
+  PAYWALL_DETECTED
+  TIMEOUT
+  SKIPPED
+}
+
+enum ExportType {
+  JSON
+  CSV
+  XLSX
+  MARKDOWN
+  MARKDOWN_ZIP
+  FULL_ZIP
+}
+
+enum ExportStatus {
+  PENDING
+  PROCESSING
+  COMPLETED
+  FAILED
+}
+
+enum AssetType {
+  IMAGE
+  LINK
+  PDF
+  FILE
+  VIDEO
+  OTHER
+}
+
+model User {
+  id           String     @id @default(uuid()) @db.Uuid
+  email        String     @unique
+  passwordHash String     @map("password_hash")
+  fullName     String?    @map("full_name")
+  role         UserRole   @default(CRAWLER_USER)
+  isActive     Boolean    @default(true) @map("is_active")
+
+  crawlJobs    CrawlJob[]
+
+  createdAt    DateTime   @default(now()) @map("created_at")
+  updatedAt    DateTime   @updatedAt @map("updated_at")
+
+  @@map("users")
+}
+
+model CrawlJob {
+  id            String          @id @default(uuid()) @db.Uuid
+  userId        String          @map("user_id") @db.Uuid
+
+  startUrl      String          @map("start_url")
+  domain        String?
+  mode          CrawlMode       @default(SCRAPE)
+  status        CrawlJobStatus  @default(PENDING)
+
+  maxPages      Int             @default(20) @map("max_pages")
+  maxDepth      Int             @default(1) @map("max_depth")
+
+  totalPages    Int             @default(0) @map("total_pages")
+  successPages  Int             @default(0) @map("success_pages")
+  failedPages   Int             @default(0) @map("failed_pages")
+
+  errorMessage  String?         @map("error_message")
+
+  startedAt     DateTime?       @map("started_at")
+  finishedAt    DateTime?       @map("finished_at")
+  createdAt     DateTime        @default(now()) @map("created_at")
+  updatedAt     DateTime        @updatedAt @map("updated_at")
+
+  user          User            @relation(fields: [userId], references: [id])
+  pages         CrawlPage[]
+  exports       CrawlExport[]
+  assets        CrawlAsset[]
+
+  @@index([userId])
+  @@index([status])
+  @@index([createdAt])
+  @@map("crawl_jobs")
+}
+
+model CrawlPage {
+  id              String          @id @default(uuid()) @db.Uuid
+  jobId           String          @map("job_id") @db.Uuid
+
+  url             String
+  title           String?
+  description     String?
+  markdownContent String?         @map("markdown_content")
+  htmlContentPath String?         @map("html_content_path")
+
+  status          CrawlPageStatus @default(PENDING)
+  statusCode      Int?            @map("status_code")
+  errorMessage    String?         @map("error_message")
+
+  crawledAt       DateTime?       @map("crawled_at")
+  createdAt       DateTime        @default(now()) @map("created_at")
+  updatedAt       DateTime        @updatedAt @map("updated_at")
+
+  job             CrawlJob        @relation(fields: [jobId], references: [id], onDelete: Cascade)
+  assets          CrawlAsset[]
+
+  @@index([jobId])
+  @@index([status])
+  @@map("crawl_pages")
+}
+
+model CrawlAsset {
+  id          String      @id @default(uuid()) @db.Uuid
+  jobId       String      @map("job_id") @db.Uuid
+  pageId      String?     @map("page_id") @db.Uuid
+
+  assetType   AssetType   @map("asset_type")
+  url         String
+  sourceUrl   String?     @map("source_url")
+  altText     String?     @map("alt_text")
+  mimeType    String?     @map("mime_type")
+  orderIndex  Int?        @map("order_index")
+  cssSelector String?     @map("css_selector")
+  domPath     String?     @map("dom_path")
+
+  createdAt   DateTime    @default(now()) @map("created_at")
+
+  job         CrawlJob    @relation(fields: [jobId], references: [id], onDelete: Cascade)
+  page        CrawlPage?  @relation(fields: [pageId], references: [id], onDelete: SetNull)
+
+  @@index([jobId])
+  @@index([pageId])
+  @@index([assetType])
+  @@map("crawl_assets")
+}
+
+model CrawlExport {
+  id          String       @id @default(uuid()) @db.Uuid
+  jobId       String       @map("job_id") @db.Uuid
+
+  exportType  ExportType   @map("export_type")
+  status      ExportStatus @default(PENDING)
+
+  fileName    String       @map("file_name")
+  filePath    String       @map("file_path")
+  fileSize    Int?         @map("file_size")
+  mimeType    String?      @map("mime_type")
+
+  errorMessage String?     @map("error_message")
+
+  createdAt   DateTime     @default(now()) @map("created_at")
+  updatedAt   DateTime     @updatedAt @map("updated_at")
+
+  job         CrawlJob     @relation(fields: [jobId], references: [id], onDelete: Cascade)
+
+  @@index([jobId])
+  @@index([exportType])
+  @@map("crawl_exports")
+}

prisma/seed.ts

+import { PrismaClient, UserRole } from '@prisma/client';
+import bcrypt from 'bcryptjs';
+
+const prisma = new PrismaClient();
+
+async function main() {
+  const passwordHash = await bcrypt.hash('Admin@123456', 10);
+
+  await prisma.user.upsert({
+    where: { email: 'admin@crawl.local' },
+    update: {},
+    create: {
+      email: 'admin@crawl.local',
+      passwordHash,
+      fullName: 'System Admin',
+      role: UserRole.ADMIN,
+      isActive: true,
+    },
+  });
+
+  console.log('Seed completed');
+}
+
+main()
+  .catch((error) => {
+    console.error(error);
+    process.exit(1);
+  })
+  .finally(async () => {
+    await prisma.$disconnect();
+  });

scripts/prisma-run.js

+require('dotenv').config();
+const { spawn } = require('child_process');
+
+const password = encodeURIComponent(process.env.DB_PASSWORD || '');
+const user = encodeURIComponent(process.env.DB_USER || 'postgres');
+process.env.DATABASE_URL = `postgresql://${user}:${password}@${process.env.DB_HOST || 'localhost'}:${process.env.DB_PORT || '5432'}/${process.env.DB_NAME || 'datacrawler'}?schema=public`;
+
+const args = process.argv.slice(2);
+const cmd = process.platform === 'win32' ? 'npx.cmd' : 'npx';
+const child = spawn(cmd, ['prisma', ...args], {
+  stdio: 'inherit',
+  env: process.env,
+  shell: true,
+});
+
+child.on('exit', (code) => process.exit(code ?? 1));

src/app.ts

+import express from 'express';
+import cors from 'cors';
+import helmet from 'helmet';
+import morgan from 'morgan';
+import { errorMiddleware } from './middlewares/error.middleware';
+import routes from './routes';
+
+const app = express();
+
+app.use(helmet());
+app.use(cors());
+app.use(morgan('dev'));
+app.use(express.json());
+app.use(express.urlencoded({ extended: true }));
+
+app.use('/api/v1', routes);
+
+app.use(errorMiddleware);
+
+export default app;

src/common/constants/export-type.constant.ts

+export const EXPORT_TYPE = {
+  JSON: 'JSON',
+  CSV: 'CSV',
+  XLSX: 'XLSX',
+  MARKDOWN: 'MARKDOWN',
+  MARKDOWN_ZIP: 'MARKDOWN_ZIP',
+  FULL_ZIP: 'FULL_ZIP',
+} as const;
+
+export type ExportType = keyof typeof EXPORT_TYPE;

src/common/constants/job-status.constant.ts

+export const JOB_STATUS = {
+  PENDING: 'PENDING',
+  RUNNING: 'RUNNING',
+  COMPLETED: 'COMPLETED',
+  PARTIAL_COMPLETED: 'PARTIAL_COMPLETED',
+  FAILED: 'FAILED',
+  CANCELED: 'CANCELED',
+  BLOCKED: 'BLOCKED',
+} as const;
+
+export type JobStatus = keyof typeof JOB_STATUS;

src/common/constants/role.constant.ts

+export const ROLES = {
+  ADMIN: 'ADMIN',
+  CRAWLER_USER: 'CRAWLER_USER',
+  VIEWER: 'VIEWER',
+} as const;
+
+export type Role = keyof typeof ROLES;

src/common/errors/app-error.ts

+export class AppError extends Error {
+  public readonly statusCode: number;
+  public readonly code?: string;
+  public readonly isOperational: boolean;
+
+  constructor(message: string, statusCode: number = 500, code?: string) {
+    super(message);
+    this.statusCode = statusCode;
+    this.code = code;
+    this.isOperational = true;
+    Object.setPrototypeOf(this, AppError.prototype);
+  }
+}

src/common/errors/error-code.ts

+export const ERROR_CODE = {
+  UNAUTHORIZED: 'UNAUTHORIZED',
+  FORBIDDEN: 'FORBIDDEN',
+  NOT_FOUND: 'NOT_FOUND',
+  VALIDATION_ERROR: 'VALIDATION_ERROR',
+  INTERNAL_SERVER_ERROR: 'INTERNAL_SERVER_ERROR',
+  INVALID_CREDENTIALS: 'INVALID_CREDENTIALS',
+  USER_INACTIVE: 'USER_INACTIVE',
+  TOKEN_EXPIRED: 'TOKEN_EXPIRED',
+  TOKEN_INVALID: 'TOKEN_INVALID',
+  DUPLICATE_ENTRY: 'DUPLICATE_ENTRY',
+  CRAWL_JOB_NOT_FOUND: 'CRAWL_JOB_NOT_FOUND',
+  CRAWL_JOB_ALREADY_COMPLETED: 'CRAWL_JOB_ALREADY_COMPLETED',
+  PRIVATE_IP_BLOCKED: 'PRIVATE_IP_BLOCKED',
+  INVALID_URL: 'INVALID_URL',
+} as const;
+
+export type ErrorCode = keyof typeof ERROR_CODE;

src/common/helpers/file.helper.ts

+import path from 'path';
+import fs from 'fs';
+
+export function ensureDirExists(dirPath: string): void {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+  }
+}
+
+export function buildExportFilePath(exportDir: string, fileName: string): string {
+  return path.join(exportDir, fileName);
+}
+
+export function getFileSizeBytes(filePath: string): number {
+  try {
+    const stat = fs.statSync(filePath);
+    return stat.size;
+  } catch {
+    return 0;
+  }
+}
+
+export function deleteFile(filePath: string): void {
+  try {
+    if (fs.existsSync(filePath)) {
+      fs.unlinkSync(filePath);
+    }
+  } catch {
+    // ignore
+  }
+}

src/common/helpers/slug.helper.ts

+export function toSlug(text: string): string {
+  return text
+    .toLowerCase()
+    .trim()
+    .replace(/[^\w\s-]/g, '')
+    .replace(/[\s_-]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+export function generateExportFileName(jobId: string, exportType: string): string {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const ext = getExtensionByType(exportType);
+  return `export-${jobId.slice(0, 8)}-${timestamp}.${ext}`;
+}
+
+function getExtensionByType(exportType: string): string {
+  const map: Record<string, string> = {
+    JSON: 'json',
+    CSV: 'csv',
+    XLSX: 'xlsx',
+    MARKDOWN: 'md',
+    MARKDOWN_ZIP: 'zip',
+    FULL_ZIP: 'zip',
+  };
+  return map[exportType] || 'bin';
+}

src/common/helpers/url.helper.ts

+import { AppError } from '../errors/app-error';
+import { ERROR_CODE } from '../errors/error-code';
+
+const PRIVATE_IP_PATTERNS = [
+  /^localhost$/i,
+  /^127\./,
+  /^10\./,
+  /^172\.(1[6-9]|2\d|3[01])\./,
+  /^192\.168\./,
+  /^::1$/,
+  /^fd[0-9a-f]{2}:/i,
+];
+
+export function validateUrl(url: string): URL {
+  let parsed: URL;
+
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new AppError('Invalid URL format', 400, ERROR_CODE.INVALID_URL);
+  }
+
+  if (!['http:', 'https:'].includes(parsed.protocol)) {
+    throw new AppError('Only HTTP and HTTPS URLs are allowed', 400, ERROR_CODE.INVALID_URL);
+  }
+
+  const hostname = parsed.hostname;
+
+  for (const pattern of PRIVATE_IP_PATTERNS) {
+    if (pattern.test(hostname)) {
+      throw new AppError(
+        'Private or local IP addresses are not allowed',
+        403,
+        ERROR_CODE.PRIVATE_IP_BLOCKED,
+      );
+    }
+  }
+
+  return parsed;
+}
+
+export function extractDomain(url: string): string {
+  try {
+    const parsed = new URL(url);
+    return parsed.hostname;
+  } catch {
+    return '';
+  }
+}

src/common/types/express.d.ts

+import { UserRole } from '@prisma/client';
+
+declare global {
+  namespace Express {
+    interface Request {
+      user: {
+        id: string;
+        email: string;
+        role: UserRole;
+      };
+    }
+  }
+}

src/config/database.config.ts

+import { envConfig } from './env.config';
+
+export const databaseConfig = {
+  url: envConfig.databaseUrl,
+  host: envConfig.database.host,
+  port: envConfig.database.port,
+  user: envConfig.database.user,
+  password: envConfig.database.password,
+  name: envConfig.database.name,
+};

src/config/env.config.ts

+export const envConfig = {
+  nodeEnv: process.env.NODE_ENV || 'development',
+  port: parseInt(process.env.PORT || '3000', 10),
+  database: {
+    host: process.env.DB_HOST || 'localhost',
+    port: parseInt(process.env.DB_PORT || '5432', 10),
+    user: process.env.DB_USER || 'postgres',
+    password: process.env.DB_PASSWORD || '',
+    name: process.env.DB_NAME || 'datacrawler',
+  },
+  get databaseUrl() {
+    return `postgresql://${this.database.user}:${encodeURIComponent(this.database.password)}@${this.database.host}:${this.database.port}/${this.database.name}?schema=public`;
+  },
+  jwt: {
+    accessSecret: process.env.JWT_ACCESS_SECRET || 'default_access_secret',
+    refreshSecret: process.env.JWT_REFRESH_SECRET || 'default_refresh_secret',
+    accessExpiresIn: process.env.JWT_ACCESS_EXPIRES_IN || '1d',
+    refreshExpiresIn: process.env.JWT_REFRESH_EXPIRES_IN || '7d',
+  },
+  firecrawl: {
+    apiKey: process.env.FIRECRAWL_API_KEY || '',
+    baseUrl: process.env.FIRECRAWL_BASE_URL || 'https://api.firecrawl.dev',
+  },
+  redis: {
+    host: process.env.REDIS_HOST || '127.0.0.1',
+    port: parseInt(process.env.REDIS_PORT || '6379', 10),
+  },
+  storage: {
+    driver: process.env.STORAGE_DRIVER || 'local',
+    exportDir: process.env.STORAGE_EXPORT_DIR || 'storage/exports',
+  },
+  crawl: {
+    maxPages: parseInt(process.env.MAX_CRAWL_PAGES || '100', 10),
+    maxDepth: parseInt(process.env.MAX_CRAWL_DEPTH || '3', 10),
+  },
+};

src/config/firecrawl.config.ts

+import { envConfig } from './env.config';
+
+export const firecrawlConfig = envConfig.firecrawl;

src/config/jwt.config.ts

+import { envConfig } from './env.config';
+
+export const jwtConfig = envConfig.jwt;

src/config/storage.config.ts

+import { envConfig } from './env.config';
+
+export const storageConfig = envConfig.storage;

src/database/prisma.client.ts

+import { PrismaClient } from '@prisma/client';
+
+export const prisma = new PrismaClient({
+  log: process.env.NODE_ENV === 'development'
+    ? ['query', 'error', 'warn']
+    : ['error', 'warn'],
+});

src/middlewares/auth.middleware.ts

+import { Request, Response, NextFunction } from 'express';
+import jwt from 'jsonwebtoken';
+import { jwtConfig } from '../config/jwt.config';
+import { AppError } from '../common/errors/app-error';
+import { ERROR_CODE } from '../common/errors/error-code';
+
+export function authMiddleware(req: Request, res: Response, next: NextFunction): void {
+  const authHeader = req.headers.authorization;
+
+  if (!authHeader || !authHeader.startsWith('Bearer ')) {
+    next(new AppError('Unauthorized', 401, ERROR_CODE.UNAUTHORIZED));
+    return;
+  }
+
+  const token = authHeader.split(' ')[1];
+
+  try {
+    const payload = jwt.verify(token, jwtConfig.accessSecret) as {
+      id: string;
+      email: string;
+      role: string;
+    };
+
+    req.user = {
+      id: payload.id,
+      email: payload.email,
+      role: payload.role as any,
+    };
+
+    next();
+  } catch (error) {
+    if (error instanceof jwt.TokenExpiredError) {
+      next(new AppError('Token expired', 401, ERROR_CODE.TOKEN_EXPIRED));
+    } else {
+      next(new AppError('Invalid token', 401, ERROR_CODE.TOKEN_INVALID));
+    }
+  }
+}

src/middlewares/error.middleware.ts

+import { Request, Response, NextFunction } from 'express';
+import { AppError } from '../common/errors/app-error';
+
+export function errorMiddleware(
+  error: Error,
+  req: Request,
+  res: Response,
+  next: NextFunction,
+): void {
+  if (error instanceof AppError) {
+    res.status(error.statusCode).json({
+      success: false,
+      message: error.message,
+      code: error.code,
+    });
+    return;
+  }
+
+  console.error('[Unhandled Error]', error);
+
+  res.status(500).json({
+    success: false,
+    message: 'Internal server error',
+    code: 'INTERNAL_SERVER_ERROR',
+  });
+}

src/middlewares/rate-limit.middleware.ts

+import { Request, Response, NextFunction } from 'express';
+
+const requestCounts = new Map<string, { count: number; resetAt: number }>();
+
+const WINDOW_MS = 15 * 60 * 1000;
+const MAX_REQUESTS = 100;
+
+export function rateLimitMiddleware(req: Request, res: Response, next: NextFunction): void {
+  const ip = req.ip || req.socket.remoteAddress || 'unknown';
+  const now = Date.now();
+
+  const record = requestCounts.get(ip);
+
+  if (!record || now > record.resetAt) {
+    requestCounts.set(ip, { count: 1, resetAt: now + WINDOW_MS });
+    next();
+    return;
+  }
+
+  record.count += 1;
+
+  if (record.count > MAX_REQUESTS) {
+    res.status(429).json({
+      success: false,
+      message: 'Too many requests, please try again later',
+      code: 'RATE_LIMIT_EXCEEDED',
+    });
+    return;
+  }
+
+  next();
+}

src/middlewares/role.middleware.ts

+import { Request, Response, NextFunction } from 'express';
+import { UserRole } from '@prisma/client';
+import { AppError } from '../common/errors/app-error';
+import { ERROR_CODE } from '../common/errors/error-code';
+
+export function requireRole(...roles: UserRole[]) {
+  return (req: Request, res: Response, next: NextFunction): void => {
+    if (!req.user) {
+      next(new AppError('Unauthorized', 401, ERROR_CODE.UNAUTHORIZED));
+      return;
+    }
+
+    if (!roles.includes(req.user.role)) {
+      next(new AppError('Forbidden', 403, ERROR_CODE.FORBIDDEN));
+      return;
+    }
+
+    next();
+  };
+}

src/middlewares/validate.middleware.ts

+import { Request, Response, NextFunction } from 'express';
+import { ZodSchema } from 'zod';
+import { AppError } from '../common/errors/app-error';
+import { ERROR_CODE } from '../common/errors/error-code';
+
+export function validate(schema: ZodSchema) {
+  return (req: Request, res: Response, next: NextFunction): void => {
+    const result = schema.safeParse(req.body);
+
+    if (!result.success) {
+      const messages = result.error.errors
+        .map((e) => `${e.path.join('.')}: ${e.message}`)
+        .join(', ');
+      next(new AppError(messages, 422, ERROR_CODE.VALIDATION_ERROR));
+      return;
+    }
+
+    req.body = result.data;
+    next();
+  };
+}

src/modules/auth/auth.controller.ts

+import { Request, Response, NextFunction } from 'express';
+import { AuthService } from './auth.service';
+
+export class AuthController {
+  private readonly service = new AuthService();
+
+  login = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const { email, password } = req.body;
+      const result = await this.service.login(email, password);
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+
+  me = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const result = await this.service.getMe(req.user.id);
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+}

src/modules/auth/auth.dto.ts

+export interface LoginDto {
+  email: string;
+  password: string;
+}
+
+export interface AuthTokensDto {
+  accessToken: string;
+  refreshToken: string;
+}
+
+export interface MeDto {
+  id: string;
+  email: string;
+  fullName: string | null;
+  role: string;
+  isActive: boolean;
+  createdAt: Date;
+}

src/modules/auth/auth.repository.ts

+import { prisma } from '../../database/prisma.client';
+
+export class AuthRepository {
+  findByEmail(email: string) {
+    return prisma.user.findUnique({
+      where: { email },
+    });
+  }
+
+  findById(id: string) {
+    return prisma.user.findUnique({
+      where: { id },
+    });
+  }
+}

src/modules/auth/auth.route.ts

+import { Router } from 'express';
+import { AuthController } from './auth.controller';
+import { authMiddleware } from '../../middlewares/auth.middleware';
+import { validate } from '../../middlewares/validate.middleware';
+import { loginSchema } from './auth.validation';
+
+const router = Router();
+const controller = new AuthController();
+
+router.post('/login', validate(loginSchema), controller.login);
+router.get('/me', authMiddleware, controller.me);
+
+export default router;

src/modules/auth/auth.service.ts

+import bcrypt from 'bcryptjs';
+import jwt from 'jsonwebtoken';
+import { AuthRepository } from './auth.repository';
+import { AppError } from '../../common/errors/app-error';
+import { ERROR_CODE } from '../../common/errors/error-code';
+import { jwtConfig } from '../../config/jwt.config';
+
+export class AuthService {
+  private readonly repository = new AuthRepository();
+
+  async login(email: string, password: string) {
+    const user = await this.repository.findByEmail(email);
+
+    if (!user) {
+      throw new AppError('Invalid credentials', 401, ERROR_CODE.INVALID_CREDENTIALS);
+    }
+
+    if (!user.isActive) {
+      throw new AppError('Account is inactive', 403, ERROR_CODE.USER_INACTIVE);
+    }
+
+    const isPasswordValid = await bcrypt.compare(password, user.passwordHash);
+
+    if (!isPasswordValid) {
+      throw new AppError('Invalid credentials', 401, ERROR_CODE.INVALID_CREDENTIALS);
+    }
+
+    const payload = { id: user.id, email: user.email, role: user.role };
+
+    const accessToken = jwt.sign(payload, jwtConfig.accessSecret, {
+      expiresIn: jwtConfig.accessExpiresIn as any,
+    });
+
+    const refreshToken = jwt.sign(payload, jwtConfig.refreshSecret, {
+      expiresIn: jwtConfig.refreshExpiresIn as any,
+    });
+
+    return {
+      accessToken,
+      refreshToken,
+      user: {
+        id: user.id,
+        email: user.email,
+        fullName: user.fullName,
+        role: user.role,
+      },
+    };
+  }
+
+  async getMe(userId: string) {
+    const user = await this.repository.findById(userId);
+
+    if (!user || !user.isActive) {
+      throw new AppError('User not found', 404, ERROR_CODE.NOT_FOUND);
+    }
+
+    return {
+      id: user.id,
+      email: user.email,
+      fullName: user.fullName,
+      role: user.role,
+      isActive: user.isActive,
+      createdAt: user.createdAt,
+    };
+  }
+}

src/modules/auth/auth.validation.ts

+import { z } from 'zod';
+
+export const loginSchema = z.object({
+  email: z.string().email('Invalid email format'),
+  password: z.string().min(1, 'Password is required'),
+});

src/modules/crawl-assets/crawl-asset.dto.ts

+import { AssetType } from '@prisma/client';
+
+export interface CreateCrawlAssetDto {
+  jobId: string;
+  pageId?: string;
+  assetType: AssetType;
+  url: string;
+  sourceUrl?: string;
+  altText?: string;
+  mimeType?: string;
+  orderIndex?: number;
+  cssSelector?: string;
+  domPath?: string;
+}

src/modules/crawl-assets/crawl-asset.repository.ts

+import { prisma } from '../../database/prisma.client';
+import { AssetType } from '@prisma/client';
+
+export class CrawlAssetRepository {
+  create(data: {
+    jobId: string;
+    pageId?: string;
+    assetType: AssetType;
+    url: string;
+    sourceUrl?: string;
+    altText?: string;
+    mimeType?: string;
+    orderIndex?: number;
+    cssSelector?: string;
+    domPath?: string;
+  }) {
+    return prisma.crawlAsset.create({ data });
+  }
+
+  createMany(assets: Array<{
+    jobId: string;
+    pageId?: string;
+    assetType: AssetType;
+    url: string;
+    sourceUrl?: string;
+    altText?: string;
+    mimeType?: string;
+  }>) {
+    return prisma.crawlAsset.createMany({ data: assets });
+  }
+
+  findByJobId(jobId: string) {
+    return prisma.crawlAsset.findMany({
+      where: { jobId },
+      orderBy: { createdAt: 'asc' },
+    });
+  }
+
+  findByPageId(pageId: string) {
+    return prisma.crawlAsset.findMany({
+      where: { pageId },
+    });
+  }
+}

src/modules/crawl-assets/crawl-asset.service.ts

+import { CrawlAssetRepository } from './crawl-asset.repository';
+import { AssetType } from '@prisma/client';
+
+export class CrawlAssetService {
+  private readonly repository = new CrawlAssetRepository();
+
+  async findByJobId(jobId: string) {
+    return this.repository.findByJobId(jobId);
+  }
+
+  async create(data: {
+    jobId: string;
+    pageId?: string;
+    assetType: AssetType;
+    url: string;
+    sourceUrl?: string;
+    altText?: string;
+    mimeType?: string;
+  }) {
+    return this.repository.create(data);
+  }
+
+  async createMany(assets: Array<{
+    jobId: string;
+    pageId?: string;
+    assetType: AssetType;
+    url: string;
+    sourceUrl?: string;
+    altText?: string;
+    mimeType?: string;
+  }>) {
+    return this.repository.createMany(assets);
+  }
+}

src/modules/crawl-exports/crawl-export.controller.ts

+import { Request, Response, NextFunction } from 'express';
+import fs from 'fs';
+import { CrawlExportService } from './crawl-export.service';
+import { AppError } from '../../common/errors/app-error';
+
+export class CrawlExportController {
+  private readonly service = new CrawlExportService();
+
+  download = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const exportRecord = await this.service.findById(req.params.exportId);
+
+      if (!fs.existsSync(exportRecord.filePath)) {
+        next(new AppError('Export file not found on disk', 404));
+        return;
+      }
+
+      res.download(exportRecord.filePath, exportRecord.fileName);
+    } catch (error) {
+      next(error);
+    }
+  };
+}

src/modules/crawl-exports/crawl-export.dto.ts

+import { ExportType, ExportStatus } from '@prisma/client';
+
+export interface CreateCrawlExportDto {
+  jobId: string;
+  exportType: ExportType;
+  fileName: string;
+  filePath: string;
+  fileSize?: number;
+  mimeType?: string;
+}
+
+export interface UpdateCrawlExportDto {
+  status?: ExportStatus;
+  fileSize?: number;
+  errorMessage?: string;
+}

src/modules/crawl-exports/crawl-export.repository.ts

+import { prisma } from '../../database/prisma.client';
+import { ExportStatus, ExportType } from '@prisma/client';
+
+export class CrawlExportRepository {
+  create(data: {
+    jobId: string;
+    exportType: ExportType;
+    fileName: string;
+    filePath: string;
+    fileSize?: number;
+    mimeType?: string;
+  }) {
+    return prisma.crawlExport.create({ data });
+  }
+
+  findByJobId(jobId: string) {
+    return prisma.crawlExport.findMany({
+      where: { jobId },
+      orderBy: { createdAt: 'desc' },
+    });
+  }
+
+  findById(id: string) {
+    return prisma.crawlExport.findUnique({
+      where: { id },
+    });
+  }
+
+  update(id: string, data: { status?: ExportStatus; fileSize?: number; errorMessage?: string }) {
+    return prisma.crawlExport.update({
+      where: { id },
+      data,
+    });
+  }
+}

src/modules/crawl-exports/crawl-export.route.ts

+import { Router } from 'express';
+import { CrawlExportController } from './crawl-export.controller';
+import { authMiddleware } from '../../middlewares/auth.middleware';
+
+const router = Router();
+const controller = new CrawlExportController();
+
+router.get('/:exportId/download', authMiddleware, controller.download);
+
+export default router;

src/modules/crawl-exports/crawl-export.service.ts

+import { CrawlExportRepository } from './crawl-export.repository';
+import { CrawlJobRepository } from '../crawl-jobs/crawl-job.repository';
+import { AppError } from '../../common/errors/app-error';
+import { ERROR_CODE } from '../../common/errors/error-code';
+import { ExportType } from '@prisma/client';
+
+export class CrawlExportService {
+  private readonly repository = new CrawlExportRepository();
+  private readonly jobRepository = new CrawlJobRepository();
+
+  async findByJobId(jobId: string) {
+    return this.repository.findByJobId(jobId);
+  }
+
+  async findById(id: string) {
+    const exportRecord = await this.repository.findById(id);
+
+    if (!exportRecord) {
+      throw new AppError('Export not found', 404, ERROR_CODE.NOT_FOUND);
+    }
+
+    return exportRecord;
+  }
+
+  async createExport(userId: string, jobId: string, exportType: ExportType) {
+    const job = await this.jobRepository.findById(jobId);
+
+    if (!job || job.userId !== userId) {
+      throw new AppError('Crawl job not found', 404, ERROR_CODE.CRAWL_JOB_NOT_FOUND);
+    }
+
+    const { ExportService } = await import('../exports/export.service');
+    const exportService = new ExportService();
+
+    return exportService.generate(job, exportType);
+  }
+}

src/modules/crawl-jobs/crawl-job.controller.ts

+import { Request, Response, NextFunction } from 'express';
+import { CrawlJobService } from './crawl-job.service';
+import { CrawlPageService } from '../crawl-pages/crawl-page.service';
+import { CrawlExportService } from '../crawl-exports/crawl-export.service';
+
+export class CrawlJobController {
+  private readonly service = new CrawlJobService();
+  private readonly pageService = new CrawlPageService();
+  private readonly exportService = new CrawlExportService();
+
+  create = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const userId = req.user.id;
+      const result = await this.service.create(userId, req.body);
+
+      res.status(201).json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+
+  findAll = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const userId = req.user.id;
+      const result = await this.service.findAllByUser(userId, req.query);
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+
+  findById = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const userId = req.user.id;
+      const result = await this.service.findById(userId, req.params.id);
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+
+  cancel = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const userId = req.user.id;
+      const result = await this.service.cancel(userId, req.params.id);
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+
+  getPages = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      await this.service.findById(req.user.id, req.params.id);
+      const result = await this.pageService.findByJobId(req.params.id);
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+
+  getExports = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      await this.service.findById(req.user.id, req.params.id);
+      const result = await this.exportService.findByJobId(req.params.id);
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+
+  createExport = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const result = await this.exportService.createExport(
+        req.user.id,
+        req.params.id,
+        req.body.exportType,
+      );
+
+      res.status(201).json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+}

src/modules/crawl-jobs/crawl-job.dto.ts

+import { CrawlJobStatus, CrawlMode } from '@prisma/client';
+
+export interface CreateCrawlJobDto {
+  startUrl: string;
+  mode?: CrawlMode;
+  maxPages?: number;
+  maxDepth?: number;
+}
+
+export interface CrawlJobQueryDto {
+  status?: CrawlJobStatus;
+  page?: number;
+  limit?: number;
+}
+
+export interface UpdateCrawlJobStatusDto {
+  status: CrawlJobStatus;
+  errorMessage?: string;
+  startedAt?: Date;
+  finishedAt?: Date;
+}

src/modules/crawl-jobs/crawl-job.repository.ts

+import { prisma } from '../../database/prisma.client';
+import { CrawlJobStatus } from '@prisma/client';
+
+export class CrawlJobRepository {
+  create(data: {
+    userId: string;
+    startUrl: string;
+    domain?: string;
+    mode: any;
+    maxPages?: number;
+    maxDepth?: number;
+  }) {
+    return prisma.crawlJob.create({
+      data: {
+        userId: data.userId,
+        startUrl: data.startUrl,
+        domain: data.domain,
+        mode: data.mode,
+        maxPages: data.maxPages ?? 20,
+        maxDepth: data.maxDepth ?? 1,
+      },
+    });
+  }
+
+  findAllByUser(userId: string, query: any) {
+    return prisma.crawlJob.findMany({
+      where: { userId },
+      orderBy: { createdAt: 'desc' },
+      include: {
+        exports: true,
+      },
+    });
+  }
+
+  findAll() {
+    return prisma.crawlJob.findMany({
+      orderBy: { createdAt: 'desc' },
+    });
+  }
+
+  findById(id: string) {
+    return prisma.crawlJob.findUnique({
+      where: { id },
+      include: {
+        pages: true,
+        exports: true,
+        assets: true,
+      },
+    });
+  }
+
+  updateStatus(id: string, status: CrawlJobStatus, extra?: {
+    errorMessage?: string;
+    startedAt?: Date;
+    finishedAt?: Date;
+    totalPages?: number;
+    successPages?: number;
+    failedPages?: number;
+  }) {
+    return prisma.crawlJob.update({
+      where: { id },
+      data: { status, ...extra },
+    });
+  }
+}

src/modules/crawl-jobs/crawl-job.route.ts

+import { Router } from 'express';
+import { CrawlJobController } from './crawl-job.controller';
+import { authMiddleware } from '../../middlewares/auth.middleware';
+import { validate } from '../../middlewares/validate.middleware';
+import { createCrawlJobSchema } from './crawl-job.validation';
+
+const router = Router();
+const controller = new CrawlJobController();
+
+router.post('/', authMiddleware, validate(createCrawlJobSchema), controller.create);
+router.get('/', authMiddleware, controller.findAll);
+router.get('/:id', authMiddleware, controller.findById);
+router.post('/:id/cancel', authMiddleware, controller.cancel);
+router.get('/:id/pages', authMiddleware, controller.getPages);
+router.get('/:id/exports', authMiddleware, controller.getExports);
+router.post('/:id/exports', authMiddleware, controller.createExport);
+
+export default router;

src/modules/crawl-jobs/crawl-job.service.ts

+import { CrawlJobRepository } from './crawl-job.repository';
+import { AppError } from '../../common/errors/app-error';
+import { ERROR_CODE } from '../../common/errors/error-code';
+import { validateUrl, extractDomain } from '../../common/helpers/url.helper';
+import { crawlQueue } from '../../queues/crawl.queue';
+
+export class CrawlJobService {
+  private readonly repository = new CrawlJobRepository();
+
+  async create(userId: string, payload: any) {
+    const parsed = validateUrl(payload.startUrl);
+    const domain = extractDomain(payload.startUrl);
+
+    const job = await this.repository.create({
+      userId,
+      startUrl: parsed.href,
+      domain,
+      mode: payload.mode ?? 'SCRAPE',
+      maxPages: payload.maxPages,
+      maxDepth: payload.maxDepth,
+    });
+
+    await crawlQueue.add('crawl-job', {
+      jobId: job.id,
+    });
+
+    return job;
+  }
+
+  async findAllByUser(userId: string, query: any) {
+    return this.repository.findAllByUser(userId, query);
+  }
+
+  async findById(userId: string, jobId: string) {
+    const job = await this.repository.findById(jobId);
+
+    if (!job || job.userId !== userId) {
+      throw new AppError('Crawl job not found', 404, ERROR_CODE.CRAWL_JOB_NOT_FOUND);
+    }
+
+    return job;
+  }
+
+  async cancel(userId: string, jobId: string) {
+    const job = await this.findById(userId, jobId);
+
+    if (job.status === 'COMPLETED') {
+      throw new AppError('Completed job cannot be canceled', 400, ERROR_CODE.CRAWL_JOB_ALREADY_COMPLETED);
+    }
+
+    return this.repository.updateStatus(jobId, 'CANCELED');
+  }
+}

src/modules/crawl-jobs/crawl-job.validation.ts

+import { z } from 'zod';
+
+export const createCrawlJobSchema = z.object({
+  startUrl: z.string().url('Invalid URL format'),
+  mode: z.enum(['SCRAPE', 'CRAWL', 'SITEMAP', 'URL_LIST']).optional(),
+  maxPages: z.number().int().min(1).max(1000).optional(),
+  maxDepth: z.number().int().min(1).max(10).optional(),
+});

src/modules/crawl-pages/crawl-page.dto.ts

+import { CrawlPageStatus } from '@prisma/client';
+
+export interface CreateCrawlPageDto {
+  jobId: string;
+  url: string;
+  title?: string;
+  description?: string;
+  markdownContent?: string;
+  htmlContentPath?: string;
+  status?: CrawlPageStatus;
+  statusCode?: number;
+  errorMessage?: string;
+  crawledAt?: Date;
+}
+
+export interface UpdateCrawlPageDto {
+  title?: string;
+  description?: string;
+  markdownContent?: string;
+  htmlContentPath?: string;
+  status?: CrawlPageStatus;
+  statusCode?: number;
+  errorMessage?: string;
+  crawledAt?: Date;
+}

src/modules/crawl-pages/crawl-page.repository.ts

+import { prisma } from '../../database/prisma.client';
+import { CrawlPageStatus } from '@prisma/client';
+
+export class CrawlPageRepository {
+  create(data: {
+    jobId: string;
+    url: string;
+    title?: string;
+    description?: string;
+    markdownContent?: string;
+    htmlContentPath?: string;
+    status?: CrawlPageStatus;
+    statusCode?: number;
+    errorMessage?: string;
+    crawledAt?: Date;
+  }) {
+    return prisma.crawlPage.create({ data });
+  }
+
+  findByJobId(jobId: string) {
+    return prisma.crawlPage.findMany({
+      where: { jobId },
+      orderBy: { createdAt: 'asc' },
+    });
+  }
+
+  findById(id: string) {
+    return prisma.crawlPage.findUnique({
+      where: { id },
+      include: { assets: true },
+    });
+  }
+
+  update(id: string, data: {
+    title?: string;
+    description?: string;
+    markdownContent?: string;
+    htmlContentPath?: string;
+    status?: CrawlPageStatus;
+    statusCode?: number;
+    errorMessage?: string;
+    crawledAt?: Date;
+  }) {
+    return prisma.crawlPage.update({
+      where: { id },
+      data,
+    });
+  }
+
+  countByJobId(jobId: string) {
+    return prisma.crawlPage.count({ where: { jobId } });
+  }
+
+  countByJobIdAndStatus(jobId: string, status: CrawlPageStatus) {
+    return prisma.crawlPage.count({ where: { jobId, status } });
+  }
+}

src/modules/crawl-pages/crawl-page.service.ts

+import { CrawlPageRepository } from './crawl-page.repository';
+import { CrawlPageStatus } from '@prisma/client';
+
+export class CrawlPageService {
+  private readonly repository = new CrawlPageRepository();
+
+  async findByJobId(jobId: string) {
+    return this.repository.findByJobId(jobId);
+  }
+
+  async create(data: {
+    jobId: string;
+    url: string;
+    title?: string;
+    description?: string;
+    markdownContent?: string;
+    status?: CrawlPageStatus;
+    statusCode?: number;
+    errorMessage?: string;
+    crawledAt?: Date;
+  }) {
+    return this.repository.create(data);
+  }
+
+  async update(id: string, data: {
+    title?: string;
+    description?: string;
+    markdownContent?: string;
+    status?: CrawlPageStatus;
+    statusCode?: number;
+    errorMessage?: string;
+    crawledAt?: Date;
+  }) {
+    return this.repository.update(id, data);
+  }
+}

src/modules/exports/csv-export.service.ts

+import fs from 'fs';
+import path from 'path';
+import { CrawlJob, CrawlPage } from '@prisma/client';
+import { ensureDirExists } from '../../common/helpers/file.helper';
+import { storageConfig } from '../../config/storage.config';
+
+export class CsvExportService {
+  async export(job: CrawlJob & { pages: CrawlPage[] }): Promise<{ fileName: string; filePath: string }> {
+    const exportDir = storageConfig.exportDir;
+    ensureDirExists(exportDir);
+
+    const fileName = `crawl-${job.id}-${Date.now()}.csv`;
+    const filePath = path.join(exportDir, fileName);
+
+    const headers = ['url', 'title', 'description', 'status', 'statusCode', 'crawledAt'];
+    const rows = job.pages.map((page) => [
+      page.url,
+      this.escapeCsv(page.title ?? ''),
+      this.escapeCsv(page.description ?? ''),
+      page.status,
+      page.statusCode ?? '',
+      page.crawledAt?.toISOString() ?? '',
+    ]);
+
+    const csv = [headers, ...rows].map((r) => r.join(',')).join('\n');
+    fs.writeFileSync(filePath, csv, 'utf-8');
+
+    return { fileName, filePath };
+  }
+
+  private escapeCsv(value: string): string {
+    if (value.includes(',') || value.includes('"') || value.includes('\n')) {
+      return `"${value.replace(/"/g, '""')}"`;
+    }
+    return value;
+  }
+}

src/modules/exports/export.service.ts

+import { CrawlJob, CrawlPage, ExportType } from '@prisma/client';
+import { prisma } from '../../database/prisma.client';
+import { CrawlExportRepository } from '../crawl-exports/crawl-export.repository';
+import { JsonExportService } from './json-export.service';
+import { CsvExportService } from './csv-export.service';
+import { XlsxExportService } from './xlsx-export.service';
+import { MarkdownExportService } from './markdown-export.service';
+import { getFileSizeBytes } from '../../common/helpers/file.helper';
+
+const MIME_TYPE: Record<string, string> = {
+  JSON: 'application/json',
+  CSV: 'text/csv',
+  XLSX: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  MARKDOWN: 'text/markdown',
+  MARKDOWN_ZIP: 'application/zip',
+  FULL_ZIP: 'application/zip',
+};
+
+export class ExportService {
+  private readonly exportRepository = new CrawlExportRepository();
+
+  async generate(job: CrawlJob, exportType: ExportType) {
+    const jobWithPages = await prisma.crawlJob.findUnique({
+      where: { id: job.id },
+      include: { pages: true },
+    });
+
+    const fullJob = jobWithPages as CrawlJob & { pages: CrawlPage[] };
+
+    let fileName: string;
+    let filePath: string;
+
+    switch (exportType) {
+      case 'JSON': {
+        const service = new JsonExportService();
+        ({ fileName, filePath } = await service.export(fullJob));
+        break;
+      }
+      case 'CSV': {
+        const service = new CsvExportService();
+        ({ fileName, filePath } = await service.export(fullJob));
+        break;
+      }
+      case 'XLSX': {
+        const service = new XlsxExportService();
+        ({ fileName, filePath } = await service.export(fullJob));
+        break;
+      }
+      case 'MARKDOWN': {
+        const service = new MarkdownExportService();
+        ({ fileName, filePath } = await service.exportSingleMd(fullJob));
+        break;
+      }
+      case 'MARKDOWN_ZIP':
+      case 'FULL_ZIP': {
+        const service = new MarkdownExportService();
+        ({ fileName, filePath } = await service.exportMarkdownZip(fullJob));
+        break;
+      }
+      default:
+        throw new Error(`Unsupported export type: ${exportType}`);
+    }
+
+    const fileSize = getFileSizeBytes(filePath);
+
+    const exportRecord = await this.exportRepository.create({
+      jobId: job.id,
+      exportType,
+      fileName,
+      filePath,
+      fileSize,
+      mimeType: MIME_TYPE[exportType],
+    });
+
+    await this.exportRepository.update(exportRecord.id, { status: 'COMPLETED' });
+
+    return exportRecord;
+  }
+}

src/modules/exports/json-export.service.ts

+import fs from 'fs';
+import path from 'path';
+import { CrawlJob, CrawlPage } from '@prisma/client';
+import { ensureDirExists } from '../../common/helpers/file.helper';
+import { storageConfig } from '../../config/storage.config';
+
+export class JsonExportService {
+  async export(job: CrawlJob & { pages: CrawlPage[] }): Promise<{ fileName: string; filePath: string }> {
+    const exportDir = storageConfig.exportDir;
+    ensureDirExists(exportDir);
+
+    const fileName = `crawl-${job.id}-${Date.now()}.json`;
+    const filePath = path.join(exportDir, fileName);
+
+    const data = {
+      jobId: job.id,
+      startUrl: job.startUrl,
+      domain: job.domain,
+      mode: job.mode,
+      status: job.status,
+      totalPages: job.totalPages,
+      exportedAt: new Date().toISOString(),
+      pages: job.pages.map((page) => ({
+        url: page.url,
+        title: page.title,
+        description: page.description,
+        markdownContent: page.markdownContent,
+        status: page.status,
+        statusCode: page.statusCode,
+        crawledAt: page.crawledAt,
+      })),
+    };
+
+    fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');
+
+    return { fileName, filePath };
+  }
+}

src/modules/exports/markdown-export.service.ts

+import fs from 'fs';
+import path from 'path';
+import archiver from 'archiver';
+import { CrawlJob, CrawlPage } from '@prisma/client';
+import { ensureDirExists } from '../../common/helpers/file.helper';
+import { storageConfig } from '../../config/storage.config';
+
+export class MarkdownExportService {
+  async exportSingleMd(job: CrawlJob & { pages: CrawlPage[] }): Promise<{ fileName: string; filePath: string }> {
+    const exportDir = storageConfig.exportDir;
+    ensureDirExists(exportDir);
+
+    const fileName = `crawl-${job.id}-${Date.now()}.md`;
+    const filePath = path.join(exportDir, fileName);
+
+    const lines: string[] = [`# Crawl Report: ${job.startUrl}`, ''];
+
+    for (const page of job.pages) {
+      lines.push(`## ${page.title ?? page.url}`);
+      lines.push(`**URL:** ${page.url}`);
+      if (page.description) lines.push(`**Description:** ${page.description}`);
+      lines.push('');
+      if (page.markdownContent) {
+        lines.push(page.markdownContent);
+        lines.push('');
+      }
+      lines.push('---', '');
+    }
+
+    fs.writeFileSync(filePath, lines.join('\n'), 'utf-8');
+
+    return { fileName, filePath };
+  }
+
+  async exportMarkdownZip(job: CrawlJob & { pages: CrawlPage[] }): Promise<{ fileName: string; filePath: string }> {
+    const exportDir = storageConfig.exportDir;
+    ensureDirExists(exportDir);
+
+    const fileName = `crawl-${job.id}-${Date.now()}.zip`;
+    const filePath = path.join(exportDir, fileName);
+
+    await new Promise<void>((resolve, reject) => {
+      const output = fs.createWriteStream(filePath);
+      const archive = archiver('zip', { zlib: { level: 9 } });
+
+      output.on('close', resolve);
+      archive.on('error', reject);
+      archive.pipe(output);
+
+      job.pages.forEach((page, idx) => {
+        const mdContent = page.markdownContent ?? `# ${page.url}\n\nNo content available.`;
+        const mdFileName = `page-${String(idx + 1).padStart(3, '0')}.md`;
+        archive.append(mdContent, { name: mdFileName });
+      });
+
+      archive.finalize();
+    });
+
+    return { fileName, filePath };
+  }
+}

src/modules/exports/xlsx-export.service.ts

+import path from 'path';
+import ExcelJS from 'exceljs';
+import { CrawlJob, CrawlPage } from '@prisma/client';
+import { ensureDirExists } from '../../common/helpers/file.helper';
+import { storageConfig } from '../../config/storage.config';
+
+export class XlsxExportService {
+  async export(job: CrawlJob & { pages: CrawlPage[] }): Promise<{ fileName: string; filePath: string }> {
+    const exportDir = storageConfig.exportDir;
+    ensureDirExists(exportDir);
+
+    const fileName = `crawl-${job.id}-${Date.now()}.xlsx`;
+    const filePath = path.join(exportDir, fileName);
+
+    const workbook = new ExcelJS.Workbook();
+    const sheet = workbook.addWorksheet('Pages');
+
+    sheet.columns = [
+      { header: 'URL', key: 'url', width: 50 },
+      { header: 'Title', key: 'title', width: 40 },
+      { header: 'Description', key: 'description', width: 50 },
+      { header: 'Status', key: 'status', width: 20 },
+      { header: 'Status Code', key: 'statusCode', width: 15 },
+      { header: 'Crawled At', key: 'crawledAt', width: 25 },
+    ];
+
+    for (const page of job.pages) {
+      sheet.addRow({
+        url: page.url,
+        title: page.title ?? '',
+        description: page.description ?? '',
+        status: page.status,
+        statusCode: page.statusCode ?? '',
+        crawledAt: page.crawledAt?.toISOString() ?? '',
+      });
+    }
+
+    await workbook.xlsx.writeFile(filePath);
+
+    return { fileName, filePath };
+  }
+}

src/modules/firecrawl/firecrawl.client.ts

+import FirecrawlApp from '@mendable/firecrawl-js';
+import { firecrawlConfig } from '../../config/firecrawl.config';
+
+let firecrawlClient: FirecrawlApp | null = null;
+
+export function getFirecrawlClient(): FirecrawlApp {
+  if (!firecrawlClient) {
+    firecrawlClient = new FirecrawlApp({
+      apiKey: firecrawlConfig.apiKey,
+    });
+  }
+  return firecrawlClient;
+}

src/modules/firecrawl/firecrawl.dto.ts

+export interface FirecrawlScrapeDto {
+  url: string;
+  formats?: string[];
+}
+
+export interface FirecrawlCrawlDto {
+  url: string;
+  maxPages?: number;
+  maxDepth?: number;
+}
+
+export interface FirecrawlPageResult {
+  url: string;
+  title?: string;
+  description?: string;
+  markdown?: string;
+  html?: string;
+  statusCode?: number;
+  links?: string[];
+  images?: Array<{ url: string; alt?: string }>;
+}

src/modules/firecrawl/firecrawl.service.ts

+import { getFirecrawlClient } from './firecrawl.client';
+import { FirecrawlPageResult } from './firecrawl.dto';
+
+export class FirecrawlService {
+  async scrapePage(url: string): Promise<FirecrawlPageResult> {
+    const client = getFirecrawlClient();
+
+    const result = await client.scrapeUrl(url, {
+      formats: ['markdown', 'html'],
+    }) as any;
+
+    return {
+      url,
+      title: result.metadata?.title,
+      description: result.metadata?.description,
+      markdown: result.markdown,
+      html: result.html,
+      statusCode: result.metadata?.statusCode,
+    };
+  }
+
+  async crawlSite(url: string, maxPages: number = 20, maxDepth: number = 1) {
+    const client = getFirecrawlClient();
+
+    const result = await client.crawlUrl(url, {
+      limit: maxPages,
+      maxDepth,
+      scrapeOptions: {
+        formats: ['markdown', 'html'],
+      },
+    } as any);
+
+    return result;
+  }
+}

src/modules/users/user.controller.ts

+import { Request, Response, NextFunction } from 'express';
+import { UserService } from './user.service';
+
+export class UserController {
+  private readonly service = new UserService();
+
+  findAll = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const result = await this.service.findAll();
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+
+  findById = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const result = await this.service.findById(req.params.id);
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+
+  create = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const result = await this.service.create(req.body);
+
+      res.status(201).json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+
+  update = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+      const result = await this.service.update(req.params.id, req.body);
+
+      res.json({
+        success: true,
+        data: result,
+      });
+    } catch (error) {
+      next(error);
+    }
+  };
+}

src/modules/users/user.dto.ts

+export interface CreateUserDto {
+  email: string;
+  password: string;
+  fullName?: string;
+  role?: string;
+}
+
+export interface UpdateUserDto {
+  fullName?: string;
+  isActive?: boolean;
+  role?: string;
+}
+
+export interface UserResponseDto {
+  id: string;
+  email: string;
+  fullName: string | null;
+  role: string;
+  isActive: boolean;
+  createdAt: Date;
+  updatedAt: Date;
+}

src/modules/users/user.repository.ts

+import { prisma } from '../../database/prisma.client';
+import { UserRole } from '@prisma/client';
+
+export class UserRepository {
+  findAll() {
+    return prisma.user.findMany({
+      orderBy: { createdAt: 'desc' },
+    });
+  }
+
+  findById(id: string) {
+    return prisma.user.findUnique({
+      where: { id },
+    });
+  }
+
+  findByEmail(email: string) {
+    return prisma.user.findUnique({
+      where: { email },
+    });
+  }
+
+  create(data: {
+    email: string;
+    passwordHash: string;
+    fullName?: string;
+    role?: UserRole;
+  }) {
+    return prisma.user.create({
+      data: {
+        email: data.email,
+        passwordHash: data.passwordHash,
+        fullName: data.fullName,
+        role: data.role ?? 'CRAWLER_USER',
+      },
+    });
+  }
+
+  update(id: string, data: { fullName?: string; isActive?: boolean; role?: UserRole }) {
+    return prisma.user.update({
+      where: { id },
+      data,
+    });
+  }
+}

src/modules/users/user.route.ts

+import { Router } from 'express';
+import { UserController } from './user.controller';
+import { authMiddleware } from '../../middlewares/auth.middleware';
+import { requireRole } from '../../middlewares/role.middleware';
+import { validate } from '../../middlewares/validate.middleware';
+import { createUserSchema, updateUserSchema } from './user.validation';
+
+const router = Router();
+const controller = new UserController();
+
+router.get('/', authMiddleware, requireRole('ADMIN'), controller.findAll);
+router.get('/:id', authMiddleware, requireRole('ADMIN'), controller.findById);
+router.post('/', authMiddleware, requireRole('ADMIN'), validate(createUserSchema), controller.create);
+router.put('/:id', authMiddleware, requireRole('ADMIN'), validate(updateUserSchema), controller.update);
+
+export default router;

src/modules/users/user.service.ts

+import bcrypt from 'bcryptjs';
+import { UserRepository } from './user.repository';
+import { AppError } from '../../common/errors/app-error';
+import { ERROR_CODE } from '../../common/errors/error-code';
+import { UserRole } from '@prisma/client';
+
+export class UserService {
+  private readonly repository = new UserRepository();
+
+  async findAll() {
+    return this.repository.findAll();
+  }
+
+  async findById(id: string) {
+    const user = await this.repository.findById(id);
+
+    if (!user) {
+      throw new AppError('User not found', 404, ERROR_CODE.NOT_FOUND);
+    }
+
+    return user;
+  }
+
+  async create(data: { email: string; password: string; fullName?: string; role?: string }) {
+    const existing = await this.repository.findByEmail(data.email);
+
+    if (existing) {
+      throw new AppError('Email already exists', 409, ERROR_CODE.DUPLICATE_ENTRY);
+    }
+
+    const passwordHash = await bcrypt.hash(data.password, 10);
+
+    return this.repository.create({
+      email: data.email,
+      passwordHash,
+      fullName: data.fullName,
+      role: data.role as UserRole | undefined,
+    });
+  }
+
+  async update(id: string, data: { fullName?: string; isActive?: boolean; role?: string }) {
+    await this.findById(id);
+
+    return this.repository.update(id, {
+      fullName: data.fullName,
+      isActive: data.isActive,
+      role: data.role as UserRole | undefined,
+    });
+  }
+}

src/modules/users/user.validation.ts

+import { z } from 'zod';
+
+export const createUserSchema = z.object({
+  email: z.string().email('Invalid email format'),
+  password: z.string().min(8, 'Password must be at least 8 characters'),
+  fullName: z.string().optional(),
+  role: z.enum(['ADMIN', 'CRAWLER_USER', 'VIEWER']).optional(),
+});
+
+export const updateUserSchema = z.object({
+  fullName: z.string().optional(),
+  isActive: z.boolean().optional(),
+  role: z.enum(['ADMIN', 'CRAWLER_USER', 'VIEWER']).optional(),
+});

src/queues/crawl.queue.ts

+import { Queue } from 'bullmq';
+import { envConfig } from '../config/env.config';
+
+export const crawlQueue = new Queue('crawl-jobs', {
+  connection: {
+    host: envConfig.redis.host,
+    port: envConfig.redis.port,
+  },
+  defaultJobOptions: {
+    attempts: 3,
+    backoff: {
+      type: 'exponential',
+      delay: 5000,
+    },
+  },
+});

src/queues/crawl.worker.ts

+import 'dotenv/config';
+import { Worker, Job } from 'bullmq';
+import { envConfig } from '../config/env.config';
+import { CrawlJobRepository } from '../modules/crawl-jobs/crawl-job.repository';
+import { CrawlPageRepository } from '../modules/crawl-pages/crawl-page.repository';
+import { CrawlAssetRepository } from '../modules/crawl-assets/crawl-asset.repository';
+import { FirecrawlService } from '../modules/firecrawl/firecrawl.service';
+
+const jobRepository = new CrawlJobRepository();
+const pageRepository = new CrawlPageRepository();
+const assetRepository = new CrawlAssetRepository();
+const firecrawlService = new FirecrawlService();
+
+async function processCrawlJob(job: Job<{ jobId: string }>) {
+  const { jobId } = job.data;
+
+  await jobRepository.updateStatus(jobId, 'RUNNING', {
+    startedAt: new Date(),
+  });
+
+  try {
+    const crawlJob = await jobRepository.findById(jobId);
+
+    if (!crawlJob) {
+      throw new Error(`Job ${jobId} not found`);
+    }
+
+    if (crawlJob.mode === 'SCRAPE') {
+      const result = await firecrawlService.scrapePage(crawlJob.startUrl);
+
+      const page = await pageRepository.create({
+        jobId,
+        url: result.url,
+        title: result.title,
+        description: result.description,
+        markdownContent: result.markdown,
+        status: 'SUCCESS',
+        statusCode: result.statusCode,
+        crawledAt: new Date(),
+      });
+
+      if (result.images && result.images.length > 0) {
+        await assetRepository.createMany(
+          result.images.map((img) => ({
+            jobId,
+            pageId: page.id,
+            assetType: 'IMAGE' as const,
+            url: img.url,
+            altText: img.alt,
+          })),
+        );
+      }
+
+      await jobRepository.updateStatus(jobId, 'COMPLETED', {
+        finishedAt: new Date(),
+        totalPages: 1,
+        successPages: 1,
+        failedPages: 0,
+      });
+    } else {
+      const result = await firecrawlService.crawlSite(
+        crawlJob.startUrl,
+        crawlJob.maxPages,
+        crawlJob.maxDepth,
+      ) as any;
+
+      const pages = result?.data ?? [];
+      let successCount = 0;
+      let failedCount = 0;
+
+      for (const item of pages) {
+        try {
+          await pageRepository.create({
+            jobId,
+            url: item.metadata?.url ?? item.url ?? crawlJob.startUrl,
+            title: item.metadata?.title,
+            description: item.metadata?.description,
+            markdownContent: item.markdown,
+            status: 'SUCCESS',
+            statusCode: item.metadata?.statusCode,
+            crawledAt: new Date(),
+          });
+          successCount++;
+        } catch {
+          failedCount++;
+        }
+      }
+
+      await jobRepository.updateStatus(jobId, 'COMPLETED', {
+        finishedAt: new Date(),
+        totalPages: pages.length,
+        successPages: successCount,
+        failedPages: failedCount,
+      });
+    }
+  } catch (error: any) {
+    await jobRepository.updateStatus(jobId, 'FAILED', {
+      finishedAt: new Date(),
+      errorMessage: error?.message ?? 'Unknown error',
+    });
+    throw error;
+  }
+}
+
+const worker = new Worker('crawl-jobs', processCrawlJob, {
+  connection: {
+    host: envConfig.redis.host,
+    port: envConfig.redis.port,
+  },
+  concurrency: 3,
+});
+
+worker.on('completed', (job) => {
+  console.log(`[Worker] Job ${job.id} completed`);
+});
+
+worker.on('failed', (job, err) => {
+  console.error(`[Worker] Job ${job?.id} failed: ${err.message}`);
+});
+
+console.log('[Worker] Crawl worker started');

src/routes/index.ts

+import { Router } from 'express';
+import authRoute from '../modules/auth/auth.route';
+import userRoute from '../modules/users/user.route';
+import crawlJobRoute from '../modules/crawl-jobs/crawl-job.route';
+import crawlExportRoute from '../modules/crawl-exports/crawl-export.route';
+
+const router = Router();
+
+router.get('/health', (req, res) => {
+  res.json({ status: 'ok', timestamp: new Date().toISOString() });
+});
+
+router.use('/auth', authRoute);
+router.use('/users', userRoute);
+router.use('/crawl-jobs', crawlJobRoute);
+router.use('/exports', crawlExportRoute);
+
+export default router;

src/server.ts

+import 'dotenv/config';
+import app from './app';
+import { envConfig } from './config/env.config';
+
+const PORT = envConfig.port;
+
+app.listen(PORT, () => {
+  console.log(`Server running on port ${PORT} in ${envConfig.nodeEnv} mode`);
+});

tsconfig.json

+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": ["ES2020"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}