Skip to content

U
upgrade-data-crawler-fe
Commit 5a27bdf6 authored by ThinhNC's avatar ThinhNC
Browse files
Options

feat(client): implement silent refresh token, swagger types, and zod schemas

parent cda8a149
Hide whitespace changes
Inline Side-by-side
Showing with 1003 additions and 32 deletions
.env.example
View file @ 5a27bdf6
# Backend API Base URL
NEXT_PUBLIC_API_URL=http://localhost:9999/api/v1
NEXT_PUBLIC_API_URL=http://localhost:9898/api/v1
# App Settings
NEXT_PUBLIC_APP_NAME="Data Crawler System"
......
eslint.config.mjs
View file @ 5a27bdf6
......@@ -12,6 +12,7 @@ const eslintConfig = defineConfig([
"out/**",
"build/**",
"next-env.d.ts",
".agents/**",
]),
]);
......
src/app/error.tsx
View file @ 5a27bdf6
"use client";
import React, { useEffect } from "react";
import { useRouter } from "next/navigation";
import { Button } from "@/components/ui/button";
import { AlertTriangle, RefreshCw } from "lucide-react";
......@@ -11,6 +12,8 @@ export default function ErrorBoundary({
error: Error & { digest?: string };
reset: () => void;
}) {
const router = useRouter();
useEffect(() => {
console.error("App Error Boundary caught:", error);
}, [error]);
......@@ -29,7 +32,7 @@ export default function ErrorBoundary({
<RefreshCw className="mr-2 h-4 w-4" />
Thử Lại
</Button>
<Button variant="outline" onClick={() => (window.location.href = "/")}>
<Button variant="outline" onClick={() => router.push("/")}>
Về Trang Chủ
</Button>
</div>
......
src/app/layout.tsx
View file @ 5a27bdf6
......@@ -28,6 +28,7 @@ export default function RootLayout({
return (
<html lang="vi" suppressHydrationWarning>
<body
suppressHydrationWarning
className={`${geistSans.variable} ${geistMono.variable} antialiased min-h-screen flex flex-col bg-background text-foreground selection:bg-emerald-500/30 selection:text-emerald-400 transition-colors duration-200`}
>
<Providers>
......
src/components/common/theme-toggle.tsx
View file @ 5a27bdf6
"use client";
import React, { useEffect, useState } from "react";
import React, { useSyncExternalStore } from "react";
import { useTheme } from "next-themes";
import { Monitor, Moon, Sun } from "lucide-react";
import { useLanguage } from "@/providers/language-provider";
const emptySubscribe = () => () => {};
export function ThemeToggle() {
const { theme, setTheme } = useTheme();
const { t } = useLanguage();
const [mounted, setMounted] = useState(false);
useEffect(() => {
setMounted(true);
}, []);
const mounted = useSyncExternalStore(emptySubscribe, () => true, () => false);
if (!mounted) {
return (
......
src/components/crawler/crawler-task-form.tsx
View file @ 5a27bdf6
......@@ -11,7 +11,7 @@ import { useCreateCrawlerTask } from "@/hooks/use-crawler";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Loader2, PlusCircle, Sparkles } from "lucide-react";
import { Loader2, PlusCircle } from "lucide-react";
interface CrawlerTaskFormProps {
onSuccess?: () => void;
......
src/components/crawler/crawler-task-table.tsx
View file @ 5a27bdf6
......@@ -98,7 +98,7 @@ export function CrawlerTaskTable() {
<Layers className="h-10 w-10 text-slate-600" />
<h3 className="font-semibold text-slate-300">Chưa có tác vụ crawl nào</h3>
<p className="text-xs text-slate-500 max-w-sm">
Bấm nút "Tạo Tác Vụ Mới" phía trên để bắt đầu cấu hình và thu thập dữ liệu web.
Bấm nút &quot;Tạo Tác Vụ Mới&quot; phía trên để bắt đầu cấu hình và thu thập dữ liệu web.
</p>
</div>
);
......
src/lib/api-client.ts
View file @ 5a27bdf6
import axios, { AxiosError, AxiosInstance, InternalAxiosRequestConfig } from "axios";
import axios, {
AxiosError,
AxiosInstance,
InternalAxiosRequestConfig,
} from "axios";
export interface ApiErrorResponse {
message: string;
......@@ -10,7 +14,11 @@ export class ApiError extends Error {
statusCode?: number;
errors?: Record<string, string[]>;
constructor(message: string, statusCode?: number, errors?: Record<string, string[]>) {
constructor(
message: string,
statusCode?: number,
errors?: Record<string, string[]>
) {
super(message);
this.name = "ApiError";
this.statusCode = statusCode;
......@@ -18,18 +26,42 @@ export class ApiError extends Error {
}
}
const baseURL = process.env.NEXT_PUBLIC_API_URL || "http://localhost:8000/api";
interface CustomAxiosRequestConfig extends InternalAxiosRequestConfig {
_retry?: boolean;
}
const baseURL =
process.env.NEXT_PUBLIC_API_URL || "http://localhost:9898/api/v1";
export const apiClient: AxiosInstance = axios.create({
baseURL,
timeout: 30000,
withCredentials: true,
headers: {
"Content-Type": "application/json",
Accept: "application/json",
},
});
// Request Interceptor
// Queue management for silent token refresh
let isRefreshing = false;
let failedQueue: Array<{
resolve: (token: string) => void;
reject: (error: unknown) => void;
}> = [];
const processQueue = (error: unknown, token: string | null = null) => {
failedQueue.forEach((prom) => {
if (error) {
prom.reject(error);
} else if (token) {
prom.resolve(token);
}
});
failedQueue = [];
};
// Request Interceptor: Attach bearer token from localStorage if available
apiClient.interceptors.request.use(
(config: InternalAxiosRequestConfig) => {
if (typeof window !== "undefined") {
......@@ -43,15 +75,110 @@ apiClient.interceptors.request.use(
(error) => Promise.reject(error)
);
// Response Interceptor
// Response Interceptor: Handle 401 and perform Silent Refresh Token
apiClient.interceptors.response.use(
(response) => response,
(error: AxiosError<ApiErrorResponse>) => {
async (error: AxiosError<ApiErrorResponse>) => {
const originalRequest = error.config as CustomAxiosRequestConfig | undefined;
if (!originalRequest) {
return Promise.reject(
new ApiError(error.message || "Đã xảy ra lỗi mạng không xác định")
);
}
const statusCode = error.response?.status;
const isAuthEndpoint =
originalRequest.url?.includes("/auth/login") ||
originalRequest.url?.includes("/auth/refresh") ||
originalRequest.url?.includes("/auth/register");
// If 401 Unauthorized occurs on an authenticated endpoint and has not yet been retried
if (statusCode === 401 && !originalRequest._retry && !isAuthEndpoint) {
if (isRefreshing) {
return new Promise<unknown>((resolve, reject) => {
failedQueue.push({
resolve: (token: string) => {
if (originalRequest.headers) {
originalRequest.headers.Authorization = `Bearer ${token}`;
}
resolve(apiClient(originalRequest));
},
reject: (err: unknown) => {
reject(err);
},
});
});
}
originalRequest._retry = true;
isRefreshing = true;
const storedRefreshToken =
typeof window !== "undefined"
? localStorage.getItem("refresh_token")
: null;
try {
const refreshResponse = await axios.post<{
success: boolean;
message?: string;
data: {
accessToken: string;
refreshToken?: string;
};
}>(
`${baseURL}/auth/refresh`,
{ refreshToken: storedRefreshToken },
{ withCredentials: true }
);
const newAccessToken = refreshResponse.data?.data?.accessToken;
const newRefreshToken = refreshResponse.data?.data?.refreshToken;
if (!newAccessToken) {
throw new Error("Không nhận được Access Token mới.");
}
if (typeof window !== "undefined") {
localStorage.setItem("auth_token", newAccessToken);
if (newRefreshToken) {
localStorage.setItem("refresh_token", newRefreshToken);
}
}
apiClient.defaults.headers.common.Authorization = `Bearer ${newAccessToken}`;
if (originalRequest.headers) {
originalRequest.headers.Authorization = `Bearer ${newAccessToken}`;
}
processQueue(null, newAccessToken);
return apiClient(originalRequest);
} catch (refreshErr) {
processQueue(refreshErr, null);
if (typeof window !== "undefined") {
localStorage.removeItem("auth_token");
localStorage.removeItem("refresh_token");
window.dispatchEvent(new CustomEvent("auth:unauthorized"));
}
const refreshMessage =
(refreshErr as AxiosError<ApiErrorResponse>)?.response?.data
?.message || "Phiên đăng nhập đã hết hạn. Vui lòng đăng nhập lại.";
return Promise.reject(
new ApiError(refreshMessage, 401, error.response?.data?.errors)
);
} finally {
isRefreshing = false;
}
}
const message =
error.response?.data?.message ||
error.message ||
"Đã có lỗi xảy ra khi kết nối tới máy chủ.";
const statusCode = error.response?.status;
const errors = error.response?.data?.errors;
return Promise.reject(new ApiError(message, statusCode, errors));
......
src/providers/language-provider.tsx
View file @ 5a27bdf6
"use client";
import React, { createContext, useContext, useEffect, useState, ReactNode } from "react";
import React, { createContext, useContext, useState, ReactNode } from "react";
import { Locale, translations, TranslationKeys } from "@/lib/i18n/translations";
interface LanguageContextType {
......@@ -14,24 +14,20 @@ const LanguageContext = createContext<LanguageContextType | undefined>(undefined
const LANGUAGE_STORAGE_KEY = "data_crawler_lang";
export function LanguageProvider({ children }: { children: ReactNode }) {
const [locale, setLocaleState] = useState<Locale>("vi");
const [isMounted, setIsMounted] = useState(false);
useEffect(() => {
setIsMounted(true);
const saved = localStorage.getItem(LANGUAGE_STORAGE_KEY) as Locale | null;
if (saved && (saved === "vi" || saved === "en")) {
setLocaleState(saved);
} else {
// Auto-detect browser language
const [locale, setLocaleState] = useState<Locale>(() => {
if (typeof window !== "undefined") {
const saved = localStorage.getItem(LANGUAGE_STORAGE_KEY) as Locale | null;
if (saved && (saved === "vi" || saved === "en")) {
return saved;
}
const browserLang = navigator.language?.toLowerCase() || "";
if (browserLang.startsWith("vi")) {
setLocaleState("vi");
} else {
setLocaleState("en");
return "vi";
}
return "en";
}
}, []);
return "vi";
});
const setLocale = (newLocale: Locale) => {
setLocaleState(newLocale);
......
src/schemas/auth.schema.ts 0 → 100644
View file @ 5a27bdf6
import { z } from "zod";
export const loginSchema = z.object({
email: z
.string()
.trim()
.min(1, "Email không được để trống")
.email("Email không đúng định dạng"),
password: z
.string()
.min(6, "Mật khẩu phải có ít nhất 6 ký tự"),
});
export const registerSchema = z
.object({
email: z
.string()
.trim()
.min(1, "Email không được để trống")
.email("Email không đúng định dạng"),
password: z
.string()
.min(8, "Mật khẩu phải có ít nhất 8 ký tự")
.regex(/[A-Z]/, "Mật khẩu phải chứa ít nhất 1 chữ hoa")
.regex(/[a-z]/, "Mật khẩu phải chứa ít nhất 1 chữ thường")
.regex(/[0-9]/, "Mật khẩu phải chứa ít nhất 1 chữ số")
.regex(/[^A-Za-z0-9]/, "Mật khẩu phải chứa ít nhất 1 ký tự đặc biệt"),
confirmPassword: z.string().min(1, "Vui lòng xác nhận mật khẩu"),
fullName: z
.string()
.trim()
.min(2, "Họ và tên tối thiểu 2 ký tự")
.max(100, "Họ và tên tối đa 100 ký tự")
.optional()
.or(z.literal("")),
})
.refine((data) => data.password === data.confirmPassword, {
message: "Mật khẩu xác nhận không khớp",
path: ["confirmPassword"],
});
export const updateProfileSchema = z.object({
fullName: z
.string()
.trim()
.min(2, "Họ và tên tối thiểu 2 ký tự")
.max(100, "Họ và tên không được quá 100 ký tự")
.optional(),
});
export const changePasswordSchema = z
.object({
currentPassword: z.string().min(1, "Vui lòng nhập mật khẩu hiện tại"),
newPassword: z
.string()
.min(8, "Mật khẩu mới phải có ít nhất 8 ký tự")
.regex(/[A-Z]/, "Mật khẩu mới phải chứa ít nhất 1 chữ hoa")
.regex(/[0-9]/, "Mật khẩu mới phải chứa ít nhất 1 chữ số"),
confirmPassword: z.string().min(1, "Vui lòng xác nhận mật khẩu mới"),
})
.refine((data) => data.newPassword === data.confirmPassword, {
message: "Mật khẩu xác nhận không khớp",
path: ["confirmPassword"],
})
.refine((data) => data.currentPassword !== data.newPassword, {
message: "Mật khẩu mới không được trùng với mật khẩu cũ",
path: ["newPassword"],
});
export const forgotPasswordSchema = z.object({
email: z
.string()
.trim()
.min(1, "Email không được để trống")
.email("Email không đúng định dạng"),
});
export const resetPasswordSchema = z
.object({
token: z.string().min(1, "Mã token xác thực không được để trống"),
password: z
.string()
.min(8, "Mật khẩu mới phải có ít nhất 8 ký tự")
.regex(/[A-Z]/, "Mật khẩu mới phải chứa ít nhất 1 chữ hoa")
.regex(/[0-9]/, "Mật khẩu mới phải chứa ít nhất 1 chữ số"),
confirmPassword: z.string().min(1, "Vui lòng xác nhận mật khẩu"),
})
.refine((data) => data.password === data.confirmPassword, {
message: "Mật khẩu xác nhận không khớp",
path: ["confirmPassword"],
});
export type LoginInput = z.infer<typeof loginSchema>;
export type RegisterInput = z.infer<typeof registerSchema>;
export type UpdateProfileInput = z.infer<typeof updateProfileSchema>;
export type ChangePasswordInput = z.infer<typeof changePasswordSchema>;
export type ForgotPasswordInput = z.infer<typeof forgotPasswordSchema>;
export type ResetPasswordInput = z.infer<typeof resetPasswordSchema>;
src/schemas/crawl-job.schema.ts 0 → 100644
View file @ 5a27bdf6
import { z } from "zod";
export const crawlModeEnum = z.enum([
"SCRAPE",
"CRAWL",
"SITEMAP",
"URL_LIST",
]);
export const crawlJobStatusEnum = z.enum([
"PENDING",
"QUEUED",
"RUNNING",
"PROCESSING_EXPORT",
"COMPLETED",
"FAILED",
"CANCELED",
"EXPIRED",
]);
export const createCrawlJobSchema = z
.object({
startUrl: z
.string()
.trim()
.url("Vui lòng nhập định dạng URL hợp lệ (vd: https://example.com)")
.optional()
.or(z.literal("")),
mode: crawlModeEnum.default("SCRAPE"),
maxPages: z
.number()
.int("Số trang phải là số nguyên")
.min(1, "Tối thiểu phải cào 1 trang")
.max(1000, "Tối đa cào 1000 trang trong một phiên")
.default(20),
maxDepth: z
.number()
.int("Độ sâu phải là số nguyên")
.min(1, "Độ sâu tối thiểu là 1")
.max(10, "Độ sâu tối đa là 10")
.default(1),
urls: z
.array(
z
.string()
.trim()
.url("Danh sách URL chứa đường dẫn không hợp lệ")
)
.max(1000, "Danh sách tối đa 1000 URLs")
.optional()
.default([]),
delayMs: z
.number()
.int()
.min(100, "Độ trễ tối thiểu 100ms")
.max(30000, "Độ trễ tối đa 30000ms")
.optional()
.default(1000),
respectRobotsTxt: z.boolean().default(true),
scheduleId: z.string().uuid().optional(),
})
.superRefine((data, ctx) => {
if (data.mode === "URL_LIST") {
if (!data.urls || data.urls.length === 0) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Vui lòng nhập ít nhất 1 URL khi chọn chế độ Danh sách URL",
path: ["urls"],
});
}
if (data.startUrl && data.startUrl.trim() !== "") {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message:
"Không cần nhập URL bắt đầu khi chọn chế độ Danh sách URL",
path: ["startUrl"],
});
}
} else {
if (!data.startUrl || data.startUrl.trim() === "") {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "URL bắt đầu là bắt buộc khi không dùng chế độ Danh sách URL",
path: ["startUrl"],
});
}
}
});
export const listCrawlJobsQuerySchema = z.object({
status: crawlJobStatusEnum.optional(),
mode: crawlModeEnum.optional(),
search: z.string().trim().optional(),
sortBy: z
.enum([
"createdAt",
"updatedAt",
"startUrl",
"status",
"mode",
"totalPages",
])
.optional(),
order: z.enum(["asc", "desc"]).optional().default("desc"),
page: z.coerce.number().int().min(1).optional().default(1),
limit: z.coerce.number().int().min(1).max(100).optional().default(20),
});
export type CreateCrawlJobInput = z.infer<typeof createCrawlJobSchema>;
export type ListCrawlJobsQueryInput = z.infer<typeof listCrawlJobsQuerySchema>;
src/schemas/crawl-schedule.schema.ts 0 → 100644
View file @ 5a27bdf6
import { z } from "zod";
import { crawlModeEnum } from "./crawl-job.schema";
export const scheduleFrequencyEnum = z.enum([
"DAILY",
"WEEKLY",
"MONTHLY",
"CUSTOM",
]);
export function isValidCronExpression(cron: string | null | undefined): boolean {
if (!cron || typeof cron !== "string") return false;
const parts = cron.trim().split(/\s+/);
if (parts.length !== 5) return false;
return parts.every((p) => p.length > 0);
}
export const createCrawlScheduleSchema = z
.object({
name: z
.string()
.trim()
.min(1, "Tên lịch cào không được để trống")
.max(150, "Tên lịch cào không được quá 150 ký tự"),
startUrl: z
.string()
.trim()
.url("Vui lòng nhập URL bắt đầu hợp lệ"),
mode: crawlModeEnum.default("SCRAPE"),
frequency: scheduleFrequencyEnum.default("DAILY"),
cronExpression: z.string().trim().optional(),
hour: z
.number()
.int()
.min(0, "Giờ từ 0 đến 23")
.max(23, "Giờ từ 0 đến 23")
.default(0),
minute: z
.number()
.int()
.min(0, "Phút từ 0 đến 59")
.max(59, "Phút từ 0 đến 59")
.default(0),
dayOfWeek: z.number().int().min(0).max(6).optional(),
dayOfMonth: z.number().int().min(1).max(31).optional(),
timezone: z.string().trim().default("Asia/Ho_Chi_Minh"),
maxPages: z.number().int().min(1).max(1000).default(20),
maxDepth: z.number().int().min(1).max(10).default(1),
urls: z
.array(z.string().trim().url("URL không hợp lệ"))
.optional()
.default([]),
isActive: z.boolean().default(true),
autoDiff: z.boolean().default(true),
})
.superRefine((data, ctx) => {
if (data.mode === "URL_LIST") {
if (!data.urls || data.urls.length === 0) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Danh sách URL phải có ít nhất 1 đường dẫn khi chọn chế độ URL_LIST",
path: ["urls"],
});
}
}
if (data.frequency === "CUSTOM") {
if (!data.cronExpression || !isValidCronExpression(data.cronExpression)) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message:
"Vui lòng nhập biểu thức Cron chuẩn 5 trường (vd: '0 2 * * *') khi chọn CUSTOM",
path: ["cronExpression"],
});
}
}
});
export const updateCrawlScheduleSchema = z
.object({
name: z.string().trim().min(1).max(150).optional(),
startUrl: z.string().trim().url("Vui lòng nhập URL hợp lệ").optional(),
mode: crawlModeEnum.optional(),
frequency: scheduleFrequencyEnum.optional(),
cronExpression: z.string().trim().optional(),
hour: z.number().int().min(0).max(23).optional(),
minute: z.number().int().min(0).max(59).optional(),
dayOfWeek: z.number().int().min(0).max(6).optional(),
dayOfMonth: z.number().int().min(1).max(31).optional(),
timezone: z.string().trim().optional(),
maxPages: z.number().int().min(1).max(1000).optional(),
maxDepth: z.number().int().min(1).max(10).optional(),
urls: z.array(z.string().trim().url()).optional(),
isActive: z.boolean().optional(),
autoDiff: z.boolean().optional(),
})
.superRefine((data, ctx) => {
if (
data.frequency === "CUSTOM" &&
data.cronExpression !== undefined &&
!isValidCronExpression(data.cronExpression)
) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message:
"Vui lòng nhập biểu thức Cron chuẩn 5 trường khi tần suất là CUSTOM",
path: ["cronExpression"],
});
}
});
export const crawlScheduleQuerySchema = z.object({
search: z.string().trim().optional(),
frequency: scheduleFrequencyEnum.optional(),
isActive: z.boolean().optional(),
page: z.coerce.number().int().min(1).default(1),
limit: z.coerce.number().int().min(1).max(100).default(20),
sortBy: z
.enum([
"createdAt",
"updatedAt",
"name",
"frequency",
"nextRunAt",
"lastRunAt",
"isActive",
])
.optional()
.default("createdAt"),
order: z.enum(["asc", "desc"]).optional().default("desc"),
});
export type CreateCrawlScheduleInput = z.infer<typeof createCrawlScheduleSchema>;
export type UpdateCrawlScheduleInput = z.infer<typeof updateCrawlScheduleSchema>;
export type CrawlScheduleQueryInput = z.infer<typeof crawlScheduleQuerySchema>;
src/schemas/export.schema.ts 0 → 100644
View file @ 5a27bdf6
import { z } from "zod";
export const exportTypeEnum = z.enum([
"JSON",
"CSV",
"XLSX",
"MARKDOWN",
"ZIP",
]);
export const exportStatusEnum = z.enum([
"PENDING",
"PROCESSING",
"COMPLETED",
"FAILED",
]);
export const createExportSchema = z.object({
jobId: z.string().uuid("Mã Job không hợp lệ"),
exportType: exportTypeEnum.default("JSON"),
fileName: z.string().trim().optional(),
});
export const exportQuerySchema = z.object({
jobId: z.string().uuid().optional(),
exportType: exportTypeEnum.optional(),
status: exportStatusEnum.optional(),
page: z.coerce.number().int().min(1).default(1),
limit: z.coerce.number().int().min(1).max(100).default(20),
});
export type CreateExportInput = z.infer<typeof createExportSchema>;
export type ExportQueryInput = z.infer<typeof exportQuerySchema>;
src/schemas/extraction-template.schema.ts 0 → 100644
View file @ 5a27bdf6
import { z } from "zod";
export const extractionFieldSchema = z.object({
name: z.string().trim().min(1, "Tên trường không được để trống"),
selector: z.string().trim().min(1, "CSS Selector không được để trống"),
attr: z
.string()
.trim()
.min(1, 'Thuộc tính không được để trống (dùng "innerText" hoặc "text" cho nội dung văn bản)'),
required: z.boolean().default(false),
});
export const createExtractionTemplateSchema = z.object({
name: z.string().trim().min(1, "Tên mẫu không được để trống"),
domain: z
.string()
.trim()
.min(1, "Tên miền (Domain) không được để trống")
.regex(
/^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/,
"Tên miền phải đúng định dạng hostname (ví dụ: vnexpress.net, dantri.com.vn)"
),
fields: z
.array(extractionFieldSchema)
.min(1, "Cần cấu hình ít nhất 1 trường trích xuất"),
});
export const updateExtractionTemplateSchema = z.object({
name: z.string().trim().min(1, "Tên mẫu không được để trống").optional(),
fields: z
.array(extractionFieldSchema)
.min(1, "Cần cấu hình ít nhất 1 trường trích xuất")
.optional(),
});
export type ExtractionFieldInput = z.infer<typeof extractionFieldSchema>;
export type CreateExtractionTemplateInput = z.infer<
typeof createExtractionTemplateSchema
>;
export type UpdateExtractionTemplateInput = z.infer<
typeof updateExtractionTemplateSchema
>;
src/schemas/index.ts 0 → 100644
View file @ 5a27bdf6
export * from "./auth.schema";
export * from "./crawl-job.schema";
export * from "./crawl-schedule.schema";
export * from "./extraction-template.schema";
export * from "./export.schema";
export * from "./crawler.schema";
src/types/auth.ts 0 → 100644
View file @ 5a27bdf6
export type UserRole = "ADMIN" | "CRAWLER_USER" | "VIEWER";
export interface User {
id: string;
email: string;
fullName: string | null;
avatarUrl: string | null;
role: UserRole;
isActive: boolean;
maxPagesLimit: number;
maxJobsPerDayLimit: number;
maxConcurrentJobsLimit: number;
createdAt: string;
updatedAt: string;
}
export interface AuthTokens {
accessToken: string;
refreshToken: string;
}
export interface AuthUserSummary {
id: string;
email: string;
fullName: string | null;
avatarUrl?: string | null;
role: string;
}
export interface LoginResponse {
accessToken: string;
refreshToken: string;
user: AuthUserSummary;
}
export interface MeDto {
id: string;
email: string;
fullName: string | null;
avatarUrl: string | null;
role: string;
isActive: boolean;
createdAt: string;
}
export interface UserQuota {
maxPagesLimit: number;
maxJobsPerDayLimit: number;
maxConcurrentJobsLimit: number;
}
export interface UserUsage {
jobsUsedToday: number;
jobsRemainingToday: number;
concurrentJobsRunning: number;
concurrentJobsAvailable: number;
totalPagesCrawled: number;
}
export interface UserUsageDto {
quota: UserQuota;
usage: UserUsage;
resetAt: string;
}
export interface RolePermissionItemDto {
id: string;
name: string;
slug: string;
resource: string;
action: string;
}
export interface Role {
id: string;
name: string;
slug: string;
description: string | null;
isSystem: boolean;
isActive: boolean;
createdAt: string;
updatedAt: string;
permissions?: RolePermissionItemDto[];
userCount?: number;
}
export interface Permission {
id: string;
name: string;
slug: string;
description: string | null;
resource: string;
action: string;
isSystem: boolean;
createdAt: string;
updatedAt: string;
}
src/types/crawl-job.ts 0 → 100644
View file @ 5a27bdf6
export type CrawlJobStatus =
| "PENDING"
| "QUEUED"
| "RUNNING"
| "PROCESSING_EXPORT"
| "COMPLETED"
| "FAILED"
| "CANCELED"
| "EXPIRED";
export type CrawlMode = "SCRAPE" | "CRAWL" | "SITEMAP" | "URL_LIST";
export type CrawlPageStatus =
| "PENDING"
| "SUCCESS"
| "FAILED"
| "BLOCKED"
| "REQUIRES_LOGIN"
| "CAPTCHA_DETECTED"
| "PAYWALL_DETECTED"
| "TIMEOUT"
| "SKIPPED";
export type AssetType =
| "IMAGE"
| "LINK"
| "PDF"
| "FILE"
| "VIDEO"
| "OTHER";
export type LogLevel = "INFO" | "WARNING" | "ERROR";
export interface CrawlJob {
id: string;
userId: string;
startUrl: string;
domain: string | null;
mode: CrawlMode;
status: CrawlJobStatus;
maxPages: number;
maxDepth: number;
urls: string[];
totalPages: number;
successPages: number;
failedPages: number;
timeoutMs: number;
retryCount: number;
respectRobotsTxt: boolean;
userAgent: string | null;
delayMs: number;
errorMessage: string | null;
firecrawlJobId: string | null;
scheduleId: string | null;
diffReportPath: string | null;
diffSummary: Record<string, unknown> | null;
startedAt: string | null;
finishedAt: string | null;
createdAt: string;
updatedAt: string;
}
export interface CrawlPage {
id: string;
jobId: string;
url: string;
title: string | null;
description: string | null;
markdownContent: string | null;
htmlContentPath: string | null;
content: string | null;
status: CrawlPageStatus;
statusCode: number | null;
errorMessage: string | null;
hasSensitiveData: boolean;
crawledAt: string | null;
normalizedUrl: string;
contentHash: string | null;
wordCount: number;
dataQualityScore: number | null;
warnings: string[];
structuredData: Record<string, unknown> | null;
createdAt: string;
updatedAt: string;
}
export interface CrawlAsset {
id: string;
pageId: string | null;
crawlJobId: string | null;
assetType: AssetType;
url: string;
sourceUrl: string | null;
altText: string | null;
mimeType: string | null;
orderIndex: number | null;
cssSelector: string | null;
domPath: string | null;
createdAt: string;
}
export interface CrawlJobLog {
id: string;
jobId: string;
level: LogLevel;
step: string;
message: string;
createdAt: string;
}
export interface CreateCrawlJobDto {
startUrl?: string;
mode?: CrawlMode;
maxPages?: number;
maxDepth?: number;
urls?: string[];
scheduleId?: string;
}
export interface CrawlJobQueryDto {
status?: CrawlJobStatus;
mode?: CrawlMode;
search?: string;
sortBy?:
| "createdAt"
| "updatedAt"
| "startUrl"
| "status"
| "mode"
| "totalPages";
order?: "asc" | "desc";
page?: number;
limit?: number;
}
export interface UpdateCrawlJobStatusDto {
status: CrawlJobStatus;
errorMessage?: string;
startedAt?: string;
finishedAt?: string;
}
export interface DiffSummaryDto {
previousJobId?: string;
changesDetected?: boolean;
addedUrls?: string[];
removedUrls?: string[];
modifiedUrls?: string[];
}
src/types/crawl-schedule.ts 0 → 100644
View file @ 5a27bdf6
import { CrawlMode } from "./crawl-job";
export type ScheduleFrequency = "DAILY" | "WEEKLY" | "MONTHLY" | "CUSTOM";
export interface CrawlSchedule {
id: string;
userId: string;
name: string;
startUrl: string;
domain: string | null;
mode: CrawlMode;
frequency: ScheduleFrequency;
cronExpression: string | null;
hour: number;
minute: number;
dayOfWeek: number | null;
dayOfMonth: number | null;
timezone: string;
maxPages: number;
maxDepth: number;
urls: string[];
isActive: boolean;
autoDiff: boolean;
lastRunAt: string | null;
nextRunAt: string | null;
createdAt: string;
updatedAt: string;
}
export interface CreateCrawlScheduleDto {
name: string;
startUrl: string;
mode?: CrawlMode;
frequency?: ScheduleFrequency;
cronExpression?: string;
hour?: number;
minute?: number;
dayOfWeek?: number;
dayOfMonth?: number;
timezone?: string;
maxPages?: number;
maxDepth?: number;
urls?: string[];
isActive?: boolean;
autoDiff?: boolean;
}
export interface UpdateCrawlScheduleDto {
name?: string;
startUrl?: string;
mode?: CrawlMode;
frequency?: ScheduleFrequency;
cronExpression?: string;
hour?: number;
minute?: number;
dayOfWeek?: number;
dayOfMonth?: number;
timezone?: string;
maxPages?: number;
maxDepth?: number;
urls?: string[];
isActive?: boolean;
autoDiff?: boolean;
}
export interface CrawlScheduleQueryDto {
search?: string;
frequency?: ScheduleFrequency;
isActive?: boolean;
page?: number;
limit?: number;
sortBy?: string;
order?: "asc" | "desc";
}
src/types/dashboard.ts 0 → 100644
View file @ 5a27bdf6
import { UserUsageDto } from "./auth";
export interface DashboardJobsMetric {
total: number;
completed: number;
failed: number;
running: number;
pending: number;
}
export interface DashboardPagesMetric {
total: number;
successful: number;
failed: number;
}
export interface DashboardSchedulesMetric {
total: number;
active: number;
}
export interface DashboardExportsMetric {
total: number;
}
export interface DashboardStats {
jobs: DashboardJobsMetric;
pages: DashboardPagesMetric;
schedules: DashboardSchedulesMetric;
exports: DashboardExportsMetric;
quotaAndUsage?: UserUsageDto;
}
src/types/export.ts 0 → 100644
View file @ 5a27bdf6
export type ExportType = "JSON" | "CSV" | "XLSX" | "MARKDOWN" | "ZIP";
export type ExportStatus = "PENDING" | "PROCESSING" | "COMPLETED" | "FAILED";
export interface CrawlExport {
id: string;
jobId: string;
exportType: ExportType;
status: ExportStatus;
fileName: string;
filePath: string;
fileSize: number | null;
mimeType: string | null;
checksum: string | null;
errorMessage: string | null;
createdAt: string;
updatedAt: string;
expiredAt: string | null;
downloadUrl?: string;
}
export interface CreateCrawlExportDto {
jobId: string;
exportType: ExportType;
fileName?: string;
}
export interface CrawlExportQueryDto {
jobId?: string;
exportType?: ExportType;
status?: ExportStatus;
page?: number;
limit?: number;
}
src/types/extraction-template.ts 0 → 100644
View file @ 5a27bdf6
export interface ExtractionField {
name: string;
selector: string;
attr: string;
required: boolean;
}
export interface ExtractionTemplate {
id: string;
userId: string;
name: string;
domain: string;
fields: ExtractionField[];
createdAt: string;
updatedAt: string;
}
export interface CreateExtractionTemplateDto {
name: string;
domain: string;
fields: ExtractionField[];
}
export interface UpdateExtractionTemplateDto {
name?: string;
fields?: ExtractionField[];
}
src/types/index.ts 0 → 100644
View file @ 5a27bdf6
export * from "./api";
export * from "./auth";
export * from "./crawl-job";
export * from "./crawl-schedule";
export * from "./extraction-template";
export * from "./export";
export * from "./dashboard";
export * from "./crawler";
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment