Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
U
upgrade-data-crawler-be
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ThinhNC
upgrade-data-crawler-be
Commits
5b123692
Commit
5b123692
authored
Sep 05, 2026
by
ThinhNC
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
feat(security,rbac): resolve 17 audit findings and migrate routes to dynamic permissions
parent
e0406ff9
Changes
39
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
39 changed files
with
716 additions
and
196 deletions
+716
-196
SKILL.md
.claude/skills/code-review-and-quality/SKILL.md
+39
-28
migration.sql
...wl_asset_cascade_and_job_user_created_index/migration.sql
+53
-0
schema.prisma
prisma/schema.prisma
+2
-1
app.ts
src/app.ts
+8
-10
permission.constant.ts
src/common/constants/permission.constant.ts
+171
-0
error-code.ts
src/common/errors/error-code.ts
+1
-0
data-contract.helper.ts
src/common/helpers/data-contract.helper.ts
+4
-2
error.middleware.ts
src/middlewares/error.middleware.ts
+50
-1
permission.middleware.ts
src/middlewares/permission.middleware.ts
+13
-2
rate-limit.middleware.ts
src/middlewares/rate-limit.middleware.ts
+3
-2
upload.middleware.ts
src/middlewares/upload.middleware.ts
+1
-0
api-key.route.ts
src/modules/api-keys/api-key.route.ts
+16
-2
audit-log.route.ts
src/modules/audit-logs/audit-log.route.ts
+3
-3
auth.repository.ts
src/modules/auth/auth.repository.ts
+27
-9
auth.route.ts
src/modules/auth/auth.route.ts
+12
-4
auth.validation.ts
src/modules/auth/auth.validation.ts
+11
-0
change-detection.service.ts
src/modules/change-detection/change-detection.service.ts
+2
-1
crawl-export.controller.ts
src/modules/crawl-exports/crawl-export.controller.ts
+8
-5
crawl-export.route.ts
src/modules/crawl-exports/crawl-export.route.ts
+16
-5
crawl-export.validation.ts
src/modules/crawl-exports/crawl-export.validation.ts
+10
-0
crawl-job.route.ts
src/modules/crawl-jobs/crawl-job.route.ts
+23
-18
crawl-job.service.ts
src/modules/crawl-jobs/crawl-job.service.ts
+5
-2
crawl-job.validation.ts
src/modules/crawl-jobs/crawl-job.validation.ts
+12
-0
crawl-schedule.route.ts
src/modules/crawl-schedules/crawl-schedule.route.ts
+11
-9
crawl-schedule.validation.ts
src/modules/crawl-schedules/crawl-schedule.validation.ts
+5
-0
dashboard.repository.ts
src/modules/dashboard/dashboard.repository.ts
+31
-34
dashboard.route.ts
src/modules/dashboard/dashboard.route.ts
+8
-1
json-export.service.ts
src/modules/exports/json-export.service.ts
+37
-14
zip-export.service.ts
src/modules/exports/zip-export.service.ts
+16
-9
extraction-template.route.ts
...modules/extraction-templates/extraction-template.route.ts
+24
-4
health.service.test.ts
src/modules/health/__tests__/health.service.test.ts
+7
-9
health.repository.ts
src/modules/health/health.repository.ts
+7
-0
health.service.ts
src/modules/health/health.service.ts
+4
-2
role.repository.ts
src/modules/roles/role.repository.ts
+16
-0
user.repository.ts
src/modules/users/user.repository.ts
+15
-0
user.service.ts
src/modules/users/user.service.ts
+2
-5
webhook-delivery.service.ts
src/modules/webhooks/webhook-delivery.service.ts
+5
-4
webhook.route.ts
src/modules/webhooks/webhook.route.ts
+29
-4
crawl.worker.processor.ts
src/queues/crawl.worker.processor.ts
+9
-6
No files found.
.claude/skills/code-review-and-quality/SKILL.md
View file @
5b123692
This diff is collapsed.
Click to expand it.
prisma/migrations/20260905103359_add_crawl_asset_cascade_and_job_user_created_index/migration.sql
0 → 100644
View file @
5b123692
/*
Warnings:
- You are about to drop the `CrawlJobLog` table. If the table is not empty, all the data it contains will be lost.
*/
-- DropForeignKey
ALTER
TABLE
"CrawlJobLog"
DROP
CONSTRAINT
"CrawlJobLog_job_id_fkey"
;
-- DropForeignKey
ALTER
TABLE
"crawl_assets"
DROP
CONSTRAINT
"crawl_assets_crawl_job_id_fkey"
;
-- DropTable
DROP
TABLE
"CrawlJobLog"
;
-- CreateTable
CREATE
TABLE
"crawl_job_logs"
(
"id"
UUID
NOT
NULL
,
"job_id"
UUID
NOT
NULL
,
"level"
"LogLevel"
NOT
NULL
,
"step"
TEXT
NOT
NULL
,
"message"
TEXT
NOT
NULL
,
"created_at"
TIMESTAMP
(
3
)
NOT
NULL
DEFAULT
CURRENT_TIMESTAMP
,
CONSTRAINT
"crawl_job_logs_pkey"
PRIMARY
KEY
(
"id"
)
);
-- CreateIndex
CREATE
INDEX
"crawl_job_logs_job_id_created_at_idx"
ON
"crawl_job_logs"
(
"job_id"
,
"created_at"
);
-- CreateIndex
CREATE
INDEX
"audit_logs_user_id_created_at_idx"
ON
"audit_logs"
(
"user_id"
,
"created_at"
);
-- CreateIndex
CREATE
INDEX
"audit_logs_action_idx"
ON
"audit_logs"
(
"action"
);
-- CreateIndex
CREATE
INDEX
"audit_logs_created_at_idx"
ON
"audit_logs"
(
"created_at"
);
-- CreateIndex
CREATE
INDEX
"audit_logs_ip_address_idx"
ON
"audit_logs"
(
"ip_address"
);
-- CreateIndex
CREATE
INDEX
"crawl_assets_crawl_job_id_idx"
ON
"crawl_assets"
(
"crawl_job_id"
);
-- CreateIndex
CREATE
INDEX
"crawl_jobs_user_id_created_at_idx"
ON
"crawl_jobs"
(
"user_id"
,
"created_at"
);
-- AddForeignKey
ALTER
TABLE
"crawl_assets"
ADD
CONSTRAINT
"crawl_assets_crawl_job_id_fkey"
FOREIGN
KEY
(
"crawl_job_id"
)
REFERENCES
"crawl_jobs"
(
"id"
)
ON
DELETE
CASCADE
ON
UPDATE
CASCADE
;
-- AddForeignKey
ALTER
TABLE
"crawl_job_logs"
ADD
CONSTRAINT
"crawl_job_logs_job_id_fkey"
FOREIGN
KEY
(
"job_id"
)
REFERENCES
"crawl_jobs"
(
"id"
)
ON
DELETE
CASCADE
ON
UPDATE
CASCADE
;
prisma/schema.prisma
View file @
5b123692
...
@@ -152,6 +152,7 @@ model CrawlJob {
...
@@ -152,6 +152,7 @@ model CrawlJob {
@@
index
([
status
])
@@
index
([
status
])
@@
index
([
createdAt
])
@@
index
([
createdAt
])
@@
index
([
userId
,
status
])
@@
index
([
userId
,
status
])
@@
index
([
userId
,
createdAt
])
@@
index
([
scheduleId
])
@@
index
([
scheduleId
])
@@
map
(
"crawl_jobs"
)
@@
map
(
"crawl_jobs"
)
}
}
...
@@ -206,7 +207,7 @@ model CrawlAsset {
...
@@ -206,7 +207,7 @@ model CrawlAsset {
createdAt
DateTime
@
default
(
now
())
@
map
(
"created_at"
)
createdAt
DateTime
@
default
(
now
())
@
map
(
"created_at"
)
page
CrawlPage
?
@
relation
(
fields
:
[
pageId
],
references
:
[
id
],
onDelete
:
SetNull
)
page
CrawlPage
?
@
relation
(
fields
:
[
pageId
],
references
:
[
id
],
onDelete
:
SetNull
)
crawlJob
CrawlJob
?
@
relation
(
fields
:
[
crawlJobId
],
references
:
[
id
])
crawlJob
CrawlJob
?
@
relation
(
fields
:
[
crawlJobId
],
references
:
[
id
]
,
onDelete
:
Cascade
)
crawlJobId
String
?
@
map
(
"crawl_job_id"
)
@
db
.
Uuid
crawlJobId
String
?
@
map
(
"crawl_job_id"
)
@
db
.
Uuid
@@
index
([
pageId
])
@@
index
([
pageId
])
...
...
src/app.ts
View file @
5b123692
...
@@ -19,19 +19,12 @@ const app = express();
...
@@ -19,19 +19,12 @@ const app = express();
app
.
set
(
"trust proxy"
,
parseTrustProxy
(
envConfig
.
trustProxy
));
app
.
set
(
"trust proxy"
,
parseTrustProxy
(
envConfig
.
trustProxy
));
app
.
use
(
app
.
use
(
helmet
());
helmet
({
contentSecurityPolicy
:
false
,
// Vô hiệu hóa CSP để Swagger UI load stylesheet bình thường
}),
);
app
.
use
(
app
.
use
(
cors
({
cors
({
origin
:
(
origin
,
callback
)
=>
{
origin
:
(
origin
,
callback
)
=>
{
if
(
!
origin
)
return
callback
(
null
,
true
);
if
(
!
origin
)
return
callback
(
null
,
true
);
if
(
if
(
envConfig
.
cors
.
allowedOrigins
.
includes
(
origin
))
{
envConfig
.
cors
.
allowedOrigins
.
includes
(
origin
)
||
envConfig
.
cors
.
allowedOrigins
.
includes
(
"*"
)
)
{
return
callback
(
null
,
true
);
return
callback
(
null
,
true
);
}
}
return
callback
(
new
Error
(
`Origin
${
origin
}
not allowed by CORS`
));
return
callback
(
new
Error
(
`Origin
${
origin
}
not allowed by CORS`
));
...
@@ -46,7 +39,12 @@ app.use(express.json());
...
@@ -46,7 +39,12 @@ app.use(express.json());
app
.
use
(
express
.
urlencoded
({
extended
:
true
}));
app
.
use
(
express
.
urlencoded
({
extended
:
true
}));
app
.
use
(
"/health"
,
healthRoute
);
app
.
use
(
"/health"
,
healthRoute
);
app
.
use
(
"/api-docs"
,
swaggerUi
.
serve
,
swaggerUi
.
setup
(
swaggerDocument
));
app
.
use
(
"/api-docs"
,
helmet
({
contentSecurityPolicy
:
false
}),
swaggerUi
.
serve
,
swaggerUi
.
setup
(
swaggerDocument
),
);
app
.
use
(
"/api/v1"
,
rateLimitMiddleware
,
routes
);
app
.
use
(
"/api/v1"
,
rateLimitMiddleware
,
routes
);
app
.
use
(
notFoundMiddleware
);
app
.
use
(
notFoundMiddleware
);
...
...
src/common/constants/permission.constant.ts
View file @
5b123692
...
@@ -41,6 +41,26 @@ export const PERMISSIONS = {
...
@@ -41,6 +41,26 @@ export const PERMISSIONS = {
EXPORTS_READ_ALL
:
"exports.read_all"
,
EXPORTS_READ_ALL
:
"exports.read_all"
,
EXPORTS_CREATE
:
"exports.create"
,
EXPORTS_CREATE
:
"exports.create"
,
EXPORTS_DOWNLOAD
:
"exports.download"
,
EXPORTS_DOWNLOAD
:
"exports.download"
,
EXPORTS_DELETE
:
"exports.delete"
,
// Webhooks
WEBHOOKS_READ
:
"webhooks.read"
,
WEBHOOKS_CREATE
:
"webhooks.create"
,
WEBHOOKS_UPDATE
:
"webhooks.update"
,
WEBHOOKS_DELETE
:
"webhooks.delete"
,
WEBHOOKS_TEST
:
"webhooks.test"
,
// Extraction Templates
EXTRACTION_TEMPLATES_READ
:
"extraction_templates.read"
,
EXTRACTION_TEMPLATES_CREATE
:
"extraction_templates.create"
,
EXTRACTION_TEMPLATES_UPDATE
:
"extraction_templates.update"
,
EXTRACTION_TEMPLATES_DELETE
:
"extraction_templates.delete"
,
// API Keys
API_KEYS_READ
:
"api_keys.read"
,
API_KEYS_CREATE
:
"api_keys.create"
,
API_KEYS_UPDATE
:
"api_keys.update"
,
API_KEYS_DELETE
:
"api_keys.delete"
,
// Audit Logs
// Audit Logs
AUDIT_LOGS_READ
:
"audit_logs.read"
,
AUDIT_LOGS_READ
:
"audit_logs.read"
,
...
@@ -305,6 +325,124 @@ export const SYSTEM_PERMISSIONS_CATALOG: PermissionDefinition[] = [
...
@@ -305,6 +325,124 @@ export const SYSTEM_PERMISSIONS_CATALOG: PermissionDefinition[] = [
action
:
"download"
,
action
:
"download"
,
isSystem
:
true
,
isSystem
:
true
,
},
},
{
name
:
"Delete Export"
,
slug
:
PERMISSIONS
.
EXPORTS_DELETE
,
description
:
"Xóa tệp trích xuất dữ liệu"
,
resource
:
"exports"
,
action
:
"delete"
,
isSystem
:
true
,
},
// Webhooks
{
name
:
"View Webhooks"
,
slug
:
PERMISSIONS
.
WEBHOOKS_READ
,
description
:
"Xem cấu hình và nhật ký gửi webhook"
,
resource
:
"webhooks"
,
action
:
"read"
,
isSystem
:
true
,
},
{
name
:
"Create Webhook"
,
slug
:
PERMISSIONS
.
WEBHOOKS_CREATE
,
description
:
"Tạo cấu hình webhook mới"
,
resource
:
"webhooks"
,
action
:
"create"
,
isSystem
:
true
,
},
{
name
:
"Update Webhook"
,
slug
:
PERMISSIONS
.
WEBHOOKS_UPDATE
,
description
:
"Cập nhật cấu hình webhook hoặc gửi lại"
,
resource
:
"webhooks"
,
action
:
"update"
,
isSystem
:
true
,
},
{
name
:
"Delete Webhook"
,
slug
:
PERMISSIONS
.
WEBHOOKS_DELETE
,
description
:
"Xóa cấu hình webhook"
,
resource
:
"webhooks"
,
action
:
"delete"
,
isSystem
:
true
,
},
{
name
:
"Test Webhook"
,
slug
:
PERMISSIONS
.
WEBHOOKS_TEST
,
description
:
"Gửi kiểm thử ping webhook"
,
resource
:
"webhooks"
,
action
:
"test"
,
isSystem
:
true
,
},
// Extraction Templates
{
name
:
"View Extraction Templates"
,
slug
:
PERMISSIONS
.
EXTRACTION_TEMPLATES_READ
,
description
:
"Xem mẫu bóc tách dữ liệu"
,
resource
:
"extraction_templates"
,
action
:
"read"
,
isSystem
:
true
,
},
{
name
:
"Create Extraction Template"
,
slug
:
PERMISSIONS
.
EXTRACTION_TEMPLATES_CREATE
,
description
:
"Tạo mẫu bóc tách dữ liệu mới"
,
resource
:
"extraction_templates"
,
action
:
"create"
,
isSystem
:
true
,
},
{
name
:
"Update Extraction Template"
,
slug
:
PERMISSIONS
.
EXTRACTION_TEMPLATES_UPDATE
,
description
:
"Cập nhật mẫu bóc tách dữ liệu"
,
resource
:
"extraction_templates"
,
action
:
"update"
,
isSystem
:
true
,
},
{
name
:
"Delete Extraction Template"
,
slug
:
PERMISSIONS
.
EXTRACTION_TEMPLATES_DELETE
,
description
:
"Xóa mẫu bóc tách dữ liệu"
,
resource
:
"extraction_templates"
,
action
:
"delete"
,
isSystem
:
true
,
},
// API Keys
{
name
:
"View API Keys"
,
slug
:
PERMISSIONS
.
API_KEYS_READ
,
description
:
"Xem danh sách khóa API"
,
resource
:
"api_keys"
,
action
:
"read"
,
isSystem
:
true
,
},
{
name
:
"Create API Key"
,
slug
:
PERMISSIONS
.
API_KEYS_CREATE
,
description
:
"Tạo khóa API mới"
,
resource
:
"api_keys"
,
action
:
"create"
,
isSystem
:
true
,
},
{
name
:
"Update API Key"
,
slug
:
PERMISSIONS
.
API_KEYS_UPDATE
,
description
:
"Kích hoạt hoặc vô hiệu hóa khóa API"
,
resource
:
"api_keys"
,
action
:
"update"
,
isSystem
:
true
,
},
{
name
:
"Delete API Key"
,
slug
:
PERMISSIONS
.
API_KEYS_DELETE
,
description
:
"Thu hồi và xóa khóa API"
,
resource
:
"api_keys"
,
action
:
"delete"
,
isSystem
:
true
,
},
// Audit Logs
// Audit Logs
{
{
...
@@ -370,6 +508,20 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
...
@@ -370,6 +508,20 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
PERMISSIONS
.
EXPORTS_READ_ALL
,
PERMISSIONS
.
EXPORTS_READ_ALL
,
PERMISSIONS
.
EXPORTS_CREATE
,
PERMISSIONS
.
EXPORTS_CREATE
,
PERMISSIONS
.
EXPORTS_DOWNLOAD
,
PERMISSIONS
.
EXPORTS_DOWNLOAD
,
PERMISSIONS
.
EXPORTS_DELETE
,
PERMISSIONS
.
WEBHOOKS_READ
,
PERMISSIONS
.
WEBHOOKS_CREATE
,
PERMISSIONS
.
WEBHOOKS_UPDATE
,
PERMISSIONS
.
WEBHOOKS_DELETE
,
PERMISSIONS
.
WEBHOOKS_TEST
,
PERMISSIONS
.
EXTRACTION_TEMPLATES_READ
,
PERMISSIONS
.
EXTRACTION_TEMPLATES_CREATE
,
PERMISSIONS
.
EXTRACTION_TEMPLATES_UPDATE
,
PERMISSIONS
.
EXTRACTION_TEMPLATES_DELETE
,
PERMISSIONS
.
API_KEYS_READ
,
PERMISSIONS
.
API_KEYS_CREATE
,
PERMISSIONS
.
API_KEYS_UPDATE
,
PERMISSIONS
.
API_KEYS_DELETE
,
PERMISSIONS
.
AUDIT_LOGS_READ
,
PERMISSIONS
.
AUDIT_LOGS_READ
,
PERMISSIONS
.
DASHBOARD_READ
,
PERMISSIONS
.
DASHBOARD_READ
,
PERMISSIONS
.
DASHBOARD_READ_ALL
,
PERMISSIONS
.
DASHBOARD_READ_ALL
,
...
@@ -379,6 +531,7 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
...
@@ -379,6 +531,7 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
PERMISSIONS
.
CRAWL_JOBS_READ
,
PERMISSIONS
.
CRAWL_JOBS_READ
,
PERMISSIONS
.
CRAWL_JOBS_CANCEL
,
PERMISSIONS
.
CRAWL_JOBS_CANCEL
,
PERMISSIONS
.
CRAWL_JOBS_RETRY
,
PERMISSIONS
.
CRAWL_JOBS_RETRY
,
PERMISSIONS
.
CRAWL_JOBS_DELETE
,
PERMISSIONS
.
CRAWL_SCHEDULES_CREATE
,
PERMISSIONS
.
CRAWL_SCHEDULES_CREATE
,
PERMISSIONS
.
CRAWL_SCHEDULES_READ
,
PERMISSIONS
.
CRAWL_SCHEDULES_READ
,
PERMISSIONS
.
CRAWL_SCHEDULES_UPDATE
,
PERMISSIONS
.
CRAWL_SCHEDULES_UPDATE
,
...
@@ -387,12 +540,30 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
...
@@ -387,12 +540,30 @@ export const SYSTEM_ROLE_DEFAULT_PERMISSIONS: Record<
PERMISSIONS
.
EXPORTS_READ
,
PERMISSIONS
.
EXPORTS_READ
,
PERMISSIONS
.
EXPORTS_CREATE
,
PERMISSIONS
.
EXPORTS_CREATE
,
PERMISSIONS
.
EXPORTS_DOWNLOAD
,
PERMISSIONS
.
EXPORTS_DOWNLOAD
,
PERMISSIONS
.
EXPORTS_DELETE
,
PERMISSIONS
.
DASHBOARD_READ
,
PERMISSIONS
.
DASHBOARD_READ
,
PERMISSIONS
.
WEBHOOKS_READ
,
PERMISSIONS
.
WEBHOOKS_CREATE
,
PERMISSIONS
.
WEBHOOKS_UPDATE
,
PERMISSIONS
.
WEBHOOKS_DELETE
,
PERMISSIONS
.
WEBHOOKS_TEST
,
PERMISSIONS
.
EXTRACTION_TEMPLATES_READ
,
PERMISSIONS
.
EXTRACTION_TEMPLATES_CREATE
,
PERMISSIONS
.
EXTRACTION_TEMPLATES_UPDATE
,
PERMISSIONS
.
EXTRACTION_TEMPLATES_DELETE
,
PERMISSIONS
.
API_KEYS_READ
,
PERMISSIONS
.
API_KEYS_CREATE
,
PERMISSIONS
.
API_KEYS_UPDATE
,
PERMISSIONS
.
API_KEYS_DELETE
,
],
],
[
SYSTEM_ROLE_SLUGS
.
VIEWER
]:
[
[
SYSTEM_ROLE_SLUGS
.
VIEWER
]:
[
PERMISSIONS
.
CRAWL_JOBS_READ
,
PERMISSIONS
.
CRAWL_JOBS_READ
,
PERMISSIONS
.
CRAWL_SCHEDULES_READ
,
PERMISSIONS
.
CRAWL_SCHEDULES_READ
,
PERMISSIONS
.
EXPORTS_READ
,
PERMISSIONS
.
EXPORTS_READ
,
PERMISSIONS
.
EXPORTS_DOWNLOAD
,
PERMISSIONS
.
DASHBOARD_READ
,
PERMISSIONS
.
DASHBOARD_READ
,
PERMISSIONS
.
WEBHOOKS_READ
,
PERMISSIONS
.
EXTRACTION_TEMPLATES_READ
,
PERMISSIONS
.
API_KEYS_READ
,
],
],
};
};
src/common/errors/error-code.ts
View file @
5b123692
...
@@ -31,6 +31,7 @@ export const ERROR_CODE = {
...
@@ -31,6 +31,7 @@ export const ERROR_CODE = {
PRIVILEGE_ESCALATION_DENIED
:
"PRIVILEGE_ESCALATION_DENIED"
,
PRIVILEGE_ESCALATION_DENIED
:
"PRIVILEGE_ESCALATION_DENIED"
,
SYSTEM_ROLE_PROTECTED
:
"SYSTEM_ROLE_PROTECTED"
,
SYSTEM_ROLE_PROTECTED
:
"SYSTEM_ROLE_PROTECTED"
,
CANNOT_REMOVE_LAST_SUPER_ADMIN
:
"CANNOT_REMOVE_LAST_SUPER_ADMIN"
,
CANNOT_REMOVE_LAST_SUPER_ADMIN
:
"CANNOT_REMOVE_LAST_SUPER_ADMIN"
,
RATE_LIMIT_EXCEEDED
:
"RATE_LIMIT_EXCEEDED"
,
}
as
const
;
}
as
const
;
export
type
ErrorCode
=
keyof
typeof
ERROR_CODE
;
export
type
ErrorCode
=
keyof
typeof
ERROR_CODE
;
src/common/helpers/data-contract.helper.ts
View file @
5b123692
...
@@ -14,6 +14,8 @@ import {
...
@@ -14,6 +14,8 @@ import {
DATA_QUALITY_MIN_SCORE
,
DATA_QUALITY_MIN_SCORE
,
DATA_CONTRACT_HASH_ALGORITHM
,
DATA_CONTRACT_HASH_ALGORITHM
,
}
from
"../constants/data-contract.constant"
;
}
from
"../constants/data-contract.constant"
;
import
{
ASSET_TYPE
}
from
"../constants/asset-type.constant"
;
import
{
CRAWL_PAGE_STATUS
}
from
"../constants/crawl-page-status.constant"
;
/**
/**
* Normalize một URL để phục vụ deduplicate và so sánh.
* Normalize một URL để phục vụ deduplicate và so sánh.
...
@@ -321,7 +323,7 @@ export function transformImages(assets: CrawlAsset[]): ImageRecord[] {
...
@@ -321,7 +323,7 @@ export function transformImages(assets: CrawlAsset[]): ImageRecord[] {
const
seenUrls
=
new
Set
<
string
>
();
const
seenUrls
=
new
Set
<
string
>
();
return
assets
return
assets
.
filter
((
a
)
=>
{
.
filter
((
a
)
=>
{
if
(
a
.
assetType
!==
"IMAGE"
)
return
false
;
if
(
a
.
assetType
!==
ASSET_TYPE
.
IMAGE
)
return
false
;
if
(
seenUrls
.
has
(
a
.
url
))
return
false
;
if
(
seenUrls
.
has
(
a
.
url
))
return
false
;
seenUrls
.
add
(
a
.
url
);
seenUrls
.
add
(
a
.
url
);
return
true
;
return
true
;
...
@@ -366,7 +368,7 @@ export function transformPageToRecord(
...
@@ -366,7 +368,7 @@ export function transformPageToRecord(
const
{
page
,
assets
,
tables
=
[],
jobDomain
,
seenContentHashes
}
=
options
;
const
{
page
,
assets
,
tables
=
[],
jobDomain
,
seenContentHashes
}
=
options
;
const
normalizedUrl
=
page
.
normalizedUrl
||
normalizeUrl
(
page
.
url
);
const
normalizedUrl
=
page
.
normalizedUrl
||
normalizeUrl
(
page
.
url
);
const
isSuccess
=
page
.
status
===
"SUCCESS"
;
const
isSuccess
=
page
.
status
===
CRAWL_PAGE_STATUS
.
SUCCESS
;
// Clean text từ markdownContent
// Clean text từ markdownContent
const
rawMarkdown
=
page
.
markdownContent
??
null
;
const
rawMarkdown
=
page
.
markdownContent
??
null
;
...
...
src/middlewares/error.middleware.ts
View file @
5b123692
...
@@ -32,11 +32,60 @@ export function errorMiddleware(
...
@@ -32,11 +32,60 @@ export function errorMiddleware(
return
;
return
;
}
}
// Handle Prisma Known Request Errors
const
prismaError
=
error
as
{
code
?:
string
;
meta
?:
{
target
?:
string
[]
}
};
if
(
prismaError
.
code
&&
typeof
prismaError
.
code
===
"string"
&&
prismaError
.
code
.
startsWith
(
"P"
)
)
{
switch
(
prismaError
.
code
)
{
case
"P2002"
:
{
const
target
=
Array
.
isArray
(
prismaError
.
meta
?.
target
)
?
prismaError
.
meta
.
target
.
join
(
", "
)
:
"field"
;
res
.
status
(
409
).
json
({
success
:
false
,
message
:
`A record with this
${
target
}
already exists.`
,
code
:
ERROR_CODE
.
DUPLICATE_ENTRY
,
});
return
;
}
case
"P2023"
:
{
res
.
status
(
400
).
json
({
success
:
false
,
message
:
"Invalid input format or malformed identifier."
,
code
:
ERROR_CODE
.
VALIDATION_ERROR
,
});
return
;
}
case
"P2025"
:
{
res
.
status
(
404
).
json
({
success
:
false
,
message
:
"Requested record not found."
,
code
:
ERROR_CODE
.
NOT_FOUND
,
});
return
;
}
case
"P2003"
:
{
res
.
status
(
400
).
json
({
success
:
false
,
message
:
"Referenced record does not exist or relation constraint failed."
,
code
:
ERROR_CODE
.
VALIDATION_ERROR
,
});
return
;
}
default
:
break
;
}
}
console
.
error
(
"[Unhandled Error]"
,
error
);
console
.
error
(
"[Unhandled Error]"
,
error
);
res
.
status
(
500
).
json
({
res
.
status
(
500
).
json
({
success
:
false
,
success
:
false
,
message
:
"Internal server error"
,
message
:
"Internal server error"
,
code
:
"INTERNAL_SERVER_ERROR"
,
code
:
ERROR_CODE
.
INTERNAL_SERVER_ERROR
,
});
});
}
}
src/middlewares/permission.middleware.ts
View file @
5b123692
import
{
Request
,
Response
,
NextFunction
}
from
"express"
;
import
{
Request
,
Response
,
NextFunction
}
from
"express"
;
import
{
PermissionSlug
}
from
"../common/constants/permission.constant"
;
import
{
PermissionSlug
,
SYSTEM_ROLE_DEFAULT_PERMISSIONS
,
}
from
"../common/constants/permission.constant"
;
import
{
SystemRoleSlug
}
from
"../common/constants/system-role.constant"
;
import
{
AppError
}
from
"../common/errors/app-error"
;
import
{
AppError
}
from
"../common/errors/app-error"
;
import
{
ERROR_CODE
}
from
"../common/errors/error-code"
;
import
{
ERROR_CODE
}
from
"../common/errors/error-code"
;
import
{
PermissionService
}
from
"../modules/permissions/permission.service"
;
import
{
PermissionService
}
from
"../modules/permissions/permission.service"
;
...
@@ -12,12 +16,19 @@ async function resolveUserPermissions(req: Request): Promise<string[]> {
...
@@ -12,12 +16,19 @@ async function resolveUserPermissions(req: Request): Promise<string[]> {
}
}
const
permissions
=
await
permissionService
.
getUserPermissions
(
req
.
user
.
id
);
const
permissions
=
await
permissionService
.
getUserPermissions
(
req
.
user
.
id
);
req
.
user
.
permissions
=
permissions
;
if
(
!
req
.
user
.
roles
)
{
if
(
!
req
.
user
.
roles
)
{
req
.
user
.
roles
=
await
permissionService
.
getUserRoles
(
req
.
user
.
id
);
req
.
user
.
roles
=
await
permissionService
.
getUserRoles
(
req
.
user
.
id
);
}
}
if
(
permissions
.
length
===
0
&&
req
.
user
.
role
)
{
const
defaultPerms
=
SYSTEM_ROLE_DEFAULT_PERMISSIONS
[
req
.
user
.
role
as
SystemRoleSlug
]
||
[];
req
.
user
.
permissions
=
defaultPerms
;
return
defaultPerms
;
}
req
.
user
.
permissions
=
permissions
;
return
permissions
;
return
permissions
;
}
}
...
...
src/middlewares/rate-limit.middleware.ts
View file @
5b123692
import
rateLimit
,
{
RateLimitRequestHandler
}
from
"express-rate-limit"
;
import
rateLimit
,
{
RateLimitRequestHandler
}
from
"express-rate-limit"
;
import
{
envConfig
}
from
"../config/env.config"
;
import
{
envConfig
}
from
"../config/env.config"
;
import
{
ERROR_CODE
}
from
"../common/errors/error-code"
;
/**
/**
* Global API rate limit per IP, configurable for each environment.
* Global API rate limit per IP, configurable for each environment.
...
@@ -14,7 +15,7 @@ export const rateLimitMiddleware: RateLimitRequestHandler = rateLimit({
...
@@ -14,7 +15,7 @@ export const rateLimitMiddleware: RateLimitRequestHandler = rateLimit({
message
:
{
message
:
{
success
:
false
,
success
:
false
,
message
:
"Bạn đã gửi quá nhiều yêu cầu. Vui lòng thử lại sau."
,
message
:
"Bạn đã gửi quá nhiều yêu cầu. Vui lòng thử lại sau."
,
code
:
"RATE_LIMIT_EXCEEDED"
,
code
:
ERROR_CODE
.
RATE_LIMIT_EXCEEDED
,
},
},
});
});
...
@@ -26,6 +27,6 @@ export const authRateLimiter: RateLimitRequestHandler = rateLimit({
...
@@ -26,6 +27,6 @@ export const authRateLimiter: RateLimitRequestHandler = rateLimit({
message
:
{
message
:
{
success
:
false
,
success
:
false
,
message
:
"Quá nhiều yêu cầu xác thực. Vui lòng thử lại sau 1 phút."
,
message
:
"Quá nhiều yêu cầu xác thực. Vui lòng thử lại sau 1 phút."
,
code
:
"RATE_LIMIT_EXCEEDED"
,
code
:
ERROR_CODE
.
RATE_LIMIT_EXCEEDED
,
},
},
});
});
src/middlewares/upload.middleware.ts
View file @
5b123692
...
@@ -7,6 +7,7 @@ export const ALLOWED_AVATAR_MIME_TYPES = [
...
@@ -7,6 +7,7 @@ export const ALLOWED_AVATAR_MIME_TYPES = [
"image/jpeg"
,
"image/jpeg"
,
"image/png"
,
"image/png"
,
"image/webp"
,
"image/webp"
,
"image/gif"
,
]
as
const
;
]
as
const
;
export
const
MAX_AVATAR_SIZE_BYTES
=
5
*
1024
*
1024
;
// 5MB
export
const
MAX_AVATAR_SIZE_BYTES
=
5
*
1024
*
1024
;
// 5MB
...
...
src/modules/api-keys/api-key.route.ts
View file @
5b123692
import
{
Router
}
from
"express"
;
import
{
Router
}
from
"express"
;
import
{
ApiKeyController
}
from
"./api-key.controller"
;
import
{
ApiKeyController
}
from
"./api-key.controller"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
requirePermission
}
from
"../../middlewares/permission.middleware"
;
import
{
PERMISSIONS
}
from
"../../common/constants/permission.constant"
;
import
{
validate
}
from
"../../middlewares/validate.middleware"
;
import
{
validate
}
from
"../../middlewares/validate.middleware"
;
import
{
import
{
createApiKeySchema
,
createApiKeySchema
,
...
@@ -13,16 +15,28 @@ const controller = new ApiKeyController();
...
@@ -13,16 +15,28 @@ const controller = new ApiKeyController();
router
.
post
(
router
.
post
(
"/"
,
"/"
,
authMiddleware
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
API_KEYS_CREATE
),
validate
(
createApiKeySchema
),
validate
(
createApiKeySchema
),
controller
.
create
,
controller
.
create
,
);
);
router
.
get
(
"/"
,
authMiddleware
,
controller
.
list
);
router
.
get
(
"/"
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
API_KEYS_READ
),
controller
.
list
,
);
router
.
patch
(
router
.
patch
(
"/:id"
,
"/:id"
,
authMiddleware
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
API_KEYS_UPDATE
),
validate
(
updateApiKeyStatusSchema
),
validate
(
updateApiKeyStatusSchema
),
controller
.
setActive
,
controller
.
setActive
,
);
);
router
.
delete
(
"/:id"
,
authMiddleware
,
controller
.
revoke
);
router
.
delete
(
"/:id"
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
API_KEYS_DELETE
),
controller
.
revoke
,
);
export
default
router
;
export
default
router
;
src/modules/audit-logs/audit-log.route.ts
View file @
5b123692
...
@@ -3,8 +3,8 @@ import { AuditLogController } from "./audit-log.controller";
...
@@ -3,8 +3,8 @@ import { AuditLogController } from "./audit-log.controller";
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
validateQuery
}
from
"../../middlewares/validate.middleware"
;
import
{
validateQuery
}
from
"../../middlewares/validate.middleware"
;
import
{
listAuditLogsQuerySchema
}
from
"./audit-log.validation"
;
import
{
listAuditLogsQuerySchema
}
from
"./audit-log.validation"
;
import
{
require
Role
}
from
"../../middlewares/role
.middleware"
;
import
{
require
Permission
}
from
"../../middlewares/permission
.middleware"
;
import
{
ROLES
}
from
"../../common/constants/role
.constant"
;
import
{
PERMISSIONS
}
from
"../../common/constants/permission
.constant"
;
const
router
=
Router
();
const
router
=
Router
();
const
controller
=
new
AuditLogController
();
const
controller
=
new
AuditLogController
();
...
@@ -12,7 +12,7 @@ const controller = new AuditLogController();
...
@@ -12,7 +12,7 @@ const controller = new AuditLogController();
router
.
get
(
router
.
get
(
"/"
,
"/"
,
authMiddleware
,
authMiddleware
,
require
Role
(
ROLES
.
ADMIN
),
require
Permission
(
PERMISSIONS
.
AUDIT_LOGS_READ
),
validateQuery
(
listAuditLogsQuerySchema
),
validateQuery
(
listAuditLogsQuerySchema
),
controller
.
findAll
,
controller
.
findAll
,
);
);
...
...
src/modules/auth/auth.repository.ts
View file @
5b123692
import
{
prisma
}
from
"../../database/prisma.client"
;
import
{
prisma
}
from
"../../database/prisma.client"
;
import
{
ROLES
}
from
"../../common/constants/role.constant"
;
import
{
ROLES
}
from
"../../common/constants/role.constant"
;
import
{
SYSTEM_ROLE_SLUGS
}
from
"../../common/constants/system-role.constant"
;
export
class
AuthRepository
{
export
class
AuthRepository
{
findByEmail
(
email
:
string
)
{
findByEmail
(
email
:
string
)
{
...
@@ -14,20 +15,37 @@ export class AuthRepository {
...
@@ -14,20 +15,37 @@ export class AuthRepository {
});
});
}
}
createUser
(
data
:
{
async
createUser
(
data
:
{
email
:
string
;
email
:
string
;
passwordHash
:
string
;
passwordHash
:
string
;
fullName
?:
string
;
fullName
?:
string
;
isActive
?:
boolean
;
isActive
?:
boolean
;
})
{
})
{
return
prisma
.
user
.
create
({
return
prisma
.
$transaction
(
async
(
tx
)
=>
{
data
:
{
const
user
=
await
tx
.
user
.
create
({
email
:
data
.
email
,
data
:
{
passwordHash
:
data
.
passwordHash
,
email
:
data
.
email
,
fullName
:
data
.
fullName
,
passwordHash
:
data
.
passwordHash
,
role
:
ROLES
.
CRAWLER_USER
,
fullName
:
data
.
fullName
,
isActive
:
data
.
isActive
??
true
,
role
:
ROLES
.
CRAWLER_USER
,
},
isActive
:
data
.
isActive
??
true
,
},
});
const
defaultRole
=
await
tx
.
role
.
findUnique
({
where
:
{
slug
:
SYSTEM_ROLE_SLUGS
.
CRAWLER_USER
},
});
if
(
defaultRole
)
{
await
tx
.
userRoleAssignment
.
create
({
data
:
{
userId
:
user
.
id
,
roleId
:
defaultRole
.
id
,
},
});
}
return
user
;
});
});
}
}
...
...
src/modules/auth/auth.route.ts
View file @
5b123692
...
@@ -5,7 +5,10 @@ import {
...
@@ -5,7 +5,10 @@ import {
copyRefreshTokenToBody
,
copyRefreshTokenToBody
,
}
from
"../../middlewares/auth.middleware"
;
}
from
"../../middlewares/auth.middleware"
;
import
{
authRateLimiter
}
from
"../../middlewares/rate-limit.middleware"
;
import
{
authRateLimiter
}
from
"../../middlewares/rate-limit.middleware"
;
import
{
validate
}
from
"../../middlewares/validate.middleware"
;
import
{
validate
,
validateParams
,
}
from
"../../middlewares/validate.middleware"
;
import
{
uploadAvatarMiddleware
}
from
"../../middlewares/upload.middleware"
;
import
{
uploadAvatarMiddleware
}
from
"../../middlewares/upload.middleware"
;
import
{
import
{
loginSchema
,
loginSchema
,
...
@@ -20,6 +23,7 @@ import {
...
@@ -20,6 +23,7 @@ import {
changePasswordSchema
,
changePasswordSchema
,
requestDeactivationSchema
,
requestDeactivationSchema
,
confirmDeactivationSchema
,
confirmDeactivationSchema
,
avatarFileNameParamsSchema
,
}
from
"./auth.validation"
;
}
from
"./auth.validation"
;
const
router
=
Router
();
const
router
=
Router
();
...
@@ -75,9 +79,13 @@ router.post(
...
@@ -75,9 +79,13 @@ router.post(
controller
.
uploadAvatar
(
req
,
res
,
next
);
controller
.
uploadAvatar
(
req
,
res
,
next
);
},
},
);
);
router
.
get
(
"/avatar/:fileName"
,
(
req
,
res
,
next
)
=>
{
router
.
get
(
controller
.
getAvatar
(
req
,
res
,
next
);
"/avatar/:fileName"
,
});
validateParams
(
avatarFileNameParamsSchema
),
(
req
,
res
,
next
)
=>
{
controller
.
getAvatar
(
req
,
res
,
next
);
},
);
router
.
post
(
router
.
post
(
"/change-password"
,
"/change-password"
,
authMiddleware
,
authMiddleware
,
...
...
src/modules/auth/auth.validation.ts
View file @
5b123692
...
@@ -104,3 +104,14 @@ export const requestDeactivationSchema = z.object({
...
@@ -104,3 +104,14 @@ export const requestDeactivationSchema = z.object({
export
const
confirmDeactivationSchema
=
z
.
object
({
export
const
confirmDeactivationSchema
=
z
.
object
({
token
:
z
.
string
().
min
(
1
,
"Thiếu mã xác nhận vô hiệu hóa."
),
token
:
z
.
string
().
min
(
1
,
"Thiếu mã xác nhận vô hiệu hóa."
),
});
});
export
const
avatarFileNameParamsSchema
=
z
.
object
({
fileName
:
z
.
string
()
.
trim
()
.
regex
(
/^
[
a-zA-Z0-9_.-
]
+
\.(
jpg|jpeg|png|webp|gif
)
$/i
,
"Invalid avatar filename format"
,
)
.
refine
((
name
)
=>
!
name
.
includes
(
".."
),
"Path traversal is not allowed"
),
});
src/modules/change-detection/change-detection.service.ts
View file @
5b123692
...
@@ -14,6 +14,7 @@ import { buildJobRootFilePath } from "../../common/helpers/file.helper";
...
@@ -14,6 +14,7 @@ import { buildJobRootFilePath } from "../../common/helpers/file.helper";
import
{
normalizeUrl
}
from
"../../common/helpers/data-contract.helper"
;
import
{
normalizeUrl
}
from
"../../common/helpers/data-contract.helper"
;
import
{
AppError
}
from
"../../common/errors/app-error"
;
import
{
AppError
}
from
"../../common/errors/app-error"
;
import
{
ERROR_CODE
}
from
"../../common/errors/error-code"
;
import
{
ERROR_CODE
}
from
"../../common/errors/error-code"
;
import
{
CrawlPageStatus
}
from
"../../common/constants/crawl-page-status.constant"
;
/**
/**
* Minimal page shape required for diff comparison.
* Minimal page shape required for diff comparison.
...
@@ -26,7 +27,7 @@ type DiffPage = {
...
@@ -26,7 +27,7 @@ type DiffPage = {
normalizedUrl
:
string
;
normalizedUrl
:
string
;
contentHash
:
string
|
null
;
contentHash
:
string
|
null
;
wordCount
:
number
;
wordCount
:
number
;
status
:
import
(
"@prisma/client"
).
CrawlPageStatus
;
status
:
CrawlPageStatus
;
statusCode
:
number
|
null
;
statusCode
:
number
|
null
;
title
:
string
|
null
;
title
:
string
|
null
;
crawledAt
:
Date
|
null
;
crawledAt
:
Date
|
null
;
...
...
src/modules/crawl-exports/crawl-export.controller.ts
View file @
5b123692
...
@@ -41,11 +41,14 @@ export class CrawlExportController {
...
@@ -41,11 +41,14 @@ export class CrawlExportController {
const
result
=
await
this
.
service
.
findAllByUser
(
req
.
user
.
id
,
page
,
limit
);
const
result
=
await
this
.
service
.
findAllByUser
(
req
.
user
.
id
,
page
,
limit
);
res
.
json
({
res
.
json
({
success
:
true
,
success
:
true
,
data
:
result
.
items
,
data
:
{
pagination
:
{
items
:
result
.
items
,
total
:
result
.
total
,
meta
:
{
page
:
result
.
page
,
total
:
result
.
total
,
limit
:
result
.
limit
,
page
:
result
.
page
,
limit
:
result
.
limit
,
totalPages
:
Math
.
ceil
(
result
.
total
/
(
result
.
limit
||
1
)),
},
},
},
});
});
}
catch
(
error
)
{
}
catch
(
error
)
{
...
...
src/modules/crawl-exports/crawl-export.route.ts
View file @
5b123692
import
{
Router
}
from
"express"
;
import
{
Router
}
from
"express"
;
import
{
CrawlExportController
}
from
"./crawl-export.controller"
;
import
{
CrawlExportController
}
from
"./crawl-export.controller"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
requireRole
}
from
"../../middlewares/role.middleware"
;
import
{
requirePermission
}
from
"../../middlewares/permission.middleware"
;
import
{
ROLES
}
from
"../../common/constants/role.constant"
;
import
{
PERMISSIONS
}
from
"../../common/constants/permission.constant"
;
import
{
validateQuery
,
validateParams
,
}
from
"../../middlewares/validate.middleware"
;
import
{
crawlExportQuerySchema
,
crawlExportParamsSchema
,
}
from
"./crawl-export.validation"
;
const
router
=
Router
();
const
router
=
Router
();
const
controller
=
new
CrawlExportController
();
const
controller
=
new
CrawlExportController
();
...
@@ -10,19 +18,22 @@ const controller = new CrawlExportController();
...
@@ -10,19 +18,22 @@ const controller = new CrawlExportController();
router
.
get
(
router
.
get
(
"/"
,
"/"
,
authMiddleware
,
authMiddleware
,
requireRole
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
requirePermission
(
PERMISSIONS
.
EXPORTS_READ
),
validateQuery
(
crawlExportQuerySchema
),
controller
.
findAll
,
controller
.
findAll
,
);
);
router
.
get
(
router
.
get
(
"/:exportId/download"
,
"/:exportId/download"
,
authMiddleware
,
authMiddleware
,
requireRole
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
requirePermission
(
PERMISSIONS
.
EXPORTS_DOWNLOAD
),
validateParams
(
crawlExportParamsSchema
),
controller
.
download
,
controller
.
download
,
);
);
router
.
delete
(
router
.
delete
(
"/:exportId"
,
"/:exportId"
,
authMiddleware
,
authMiddleware
,
requireRole
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
),
requirePermission
(
PERMISSIONS
.
EXPORTS_DELETE
),
validateParams
(
crawlExportParamsSchema
),
controller
.
delete
,
controller
.
delete
,
);
);
...
...
src/modules/crawl-exports/crawl-export.validation.ts
0 → 100644
View file @
5b123692
import
{
z
}
from
"zod"
;
export
const
crawlExportQuerySchema
=
z
.
object
({
page
:
z
.
coerce
.
number
().
int
().
min
(
1
).
default
(
1
),
limit
:
z
.
coerce
.
number
().
int
().
min
(
1
).
max
(
100
).
default
(
20
),
});
export
const
crawlExportParamsSchema
=
z
.
object
({
exportId
:
z
.
string
().
uuid
(
"Invalid export ID format"
),
});
src/modules/crawl-jobs/crawl-job.route.ts
View file @
5b123692
...
@@ -7,10 +7,12 @@ import {
...
@@ -7,10 +7,12 @@ import {
createExportSchema
,
createExportSchema
,
listCrawlJobsQuerySchema
,
listCrawlJobsQuerySchema
,
getAssetsQuerySchema
,
getAssetsQuerySchema
,
jobLogsQuerySchema
,
diffQuerySchema
,
}
from
"./crawl-job.validation"
;
}
from
"./crawl-job.validation"
;
import
{
crawlPageQuerySchema
}
from
"../crawl-pages/crawl-page.validation"
;
import
{
crawlPageQuerySchema
}
from
"../crawl-pages/crawl-page.validation"
;
import
{
require
Role
}
from
"../../middlewares/role
.middleware"
;
import
{
require
Permission
}
from
"../../middlewares/permission
.middleware"
;
import
{
ROLES
}
from
"../../common/constants/role
.constant"
;
import
{
PERMISSIONS
}
from
"../../common/constants/permission
.constant"
;
const
router
=
Router
();
const
router
=
Router
();
const
controller
=
new
CrawlJobController
();
const
controller
=
new
CrawlJobController
();
...
@@ -18,7 +20,7 @@ const controller = new CrawlJobController();
...
@@ -18,7 +20,7 @@ const controller = new CrawlJobController();
router
.
post
(
router
.
post
(
"/"
,
"/"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
),
require
Permission
(
PERMISSIONS
.
CRAWL_JOBS_CREATE
),
validate
(
createCrawlJobSchema
),
validate
(
createCrawlJobSchema
),
(
req
,
res
,
next
)
=>
{
(
req
,
res
,
next
)
=>
{
controller
.
create
(
req
,
res
,
next
);
controller
.
create
(
req
,
res
,
next
);
...
@@ -27,70 +29,71 @@ router.post(
...
@@ -27,70 +29,71 @@ router.post(
router
.
get
(
router
.
get
(
"/"
,
"/"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
require
Permission
(
PERMISSIONS
.
CRAWL_JOBS_READ
),
validateQuery
(
listCrawlJobsQuerySchema
),
validateQuery
(
listCrawlJobsQuerySchema
),
controller
.
findAll
,
controller
.
findAll
,
);
);
router
.
get
(
router
.
get
(
"/:id"
,
"/:id"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
require
Permission
(
PERMISSIONS
.
CRAWL_JOBS_READ
),
controller
.
findById
,
controller
.
findById
,
);
);
router
.
delete
(
router
.
delete
(
"/:id"
,
"/:id"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
),
require
Permission
(
PERMISSIONS
.
CRAWL_JOBS_DELETE
),
controller
.
delete
,
controller
.
delete
,
);
);
router
.
post
(
router
.
post
(
"/:id/rerun"
,
"/:id/rerun"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
),
require
Permission
(
PERMISSIONS
.
CRAWL_JOBS_RETRY
),
controller
.
rerun
,
controller
.
rerun
,
);
);
router
.
get
(
router
.
get
(
"/:id/logs"
,
"/:id/logs"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
requireRole
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
requirePermission
(
PERMISSIONS
.
CRAWL_JOBS_READ
),
validateQuery
(
jobLogsQuerySchema
),
controller
.
getLogs
,
controller
.
getLogs
,
);
);
router
.
get
(
router
.
get
(
"/:id/events"
,
"/:id/events"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
require
Permission
(
PERMISSIONS
.
CRAWL_JOBS_READ
),
controller
.
streamEvents
,
controller
.
streamEvents
,
);
);
router
.
post
(
router
.
post
(
"/:id/cancel"
,
"/:id/cancel"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
),
require
Permission
(
PERMISSIONS
.
CRAWL_JOBS_CANCEL
),
controller
.
cancel
,
controller
.
cancel
,
);
);
router
.
get
(
router
.
get
(
"/:id/pages"
,
"/:id/pages"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
require
Permission
(
PERMISSIONS
.
CRAWL_JOBS_READ
),
validateQuery
(
crawlPageQuerySchema
),
validateQuery
(
crawlPageQuerySchema
),
controller
.
getPages
,
controller
.
getPages
,
);
);
router
.
get
(
router
.
get
(
"/:id/pages/preview"
,
"/:id/pages/preview"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
require
Permission
(
PERMISSIONS
.
CRAWL_JOBS_READ
),
validateQuery
(
crawlPageQuerySchema
),
validateQuery
(
crawlPageQuerySchema
),
controller
.
getPagesPreview
,
controller
.
getPagesPreview
,
);
);
router
.
get
(
router
.
get
(
"/:id/exports"
,
"/:id/exports"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
require
Permission
(
PERMISSIONS
.
EXPORTS_READ
),
controller
.
getExports
,
controller
.
getExports
,
);
);
router
.
post
(
router
.
post
(
"/:id/exports"
,
"/:id/exports"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
),
require
Permission
(
PERMISSIONS
.
EXPORTS_CREATE
),
validate
(
createExportSchema
),
validate
(
createExportSchema
),
(
req
,
res
,
next
)
=>
{
(
req
,
res
,
next
)
=>
{
controller
.
createExport
(
req
,
res
,
next
);
controller
.
createExport
(
req
,
res
,
next
);
...
@@ -99,26 +102,28 @@ router.post(
...
@@ -99,26 +102,28 @@ router.post(
router
.
get
(
router
.
get
(
"/:id/download"
,
"/:id/download"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
require
Permission
(
PERMISSIONS
.
EXPORTS_DOWNLOAD
),
controller
.
download
,
controller
.
download
,
);
);
router
.
get
(
router
.
get
(
"/:id/assets"
,
"/:id/assets"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
require
Permission
(
PERMISSIONS
.
CRAWL_JOBS_READ
),
validateQuery
(
getAssetsQuerySchema
),
validateQuery
(
getAssetsQuerySchema
),
controller
.
getAssets
,
controller
.
getAssets
,
);
);
router
.
get
(
router
.
get
(
"/:id/diff"
,
"/:id/diff"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
requireRole
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
requirePermission
(
PERMISSIONS
.
CRAWL_JOBS_READ
),
validateQuery
(
diffQuerySchema
),
controller
.
getDiff
,
controller
.
getDiff
,
);
);
router
.
get
(
router
.
get
(
"/:id/diff/download"
,
"/:id/diff/download"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
requireRole
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
requirePermission
(
PERMISSIONS
.
CRAWL_JOBS_READ
),
validateQuery
(
diffQuerySchema
),
controller
.
downloadDiff
,
controller
.
downloadDiff
,
);
);
export
default
router
;
export
default
router
;
src/modules/crawl-jobs/crawl-job.service.ts
View file @
5b123692
...
@@ -12,7 +12,10 @@ import {
...
@@ -12,7 +12,10 @@ import {
import
{
crawlQueue
}
from
"../../queues/crawl.queue"
;
import
{
crawlQueue
}
from
"../../queues/crawl.queue"
;
import
{
ROLES
}
from
"../../common/constants/role.constant"
;
import
{
ROLES
}
from
"../../common/constants/role.constant"
;
import
{
JOB_STATUS
}
from
"../../common/constants/job-status.constant"
;
import
{
JOB_STATUS
}
from
"../../common/constants/job-status.constant"
;
import
{
EXPORT_TYPE
}
from
"../../common/constants/export-type.constant"
;
import
{
EXPORT_TYPE
,
EXPORT_STATUS
,
}
from
"../../common/constants/export-type.constant"
;
import
{
DEFAULT_TIMEZONE
}
from
"../../common/constants/timezone.constant"
;
import
{
DEFAULT_TIMEZONE
}
from
"../../common/constants/timezone.constant"
;
import
{
CRAWL_MODE
}
from
"../../common/constants/crawl-mode.constant"
;
import
{
CRAWL_MODE
}
from
"../../common/constants/crawl-mode.constant"
;
import
{
CreateCrawlJobDto
,
CrawlJobQueryDto
}
from
"./crawl-job.dto"
;
import
{
CreateCrawlJobDto
,
CrawlJobQueryDto
}
from
"./crawl-job.dto"
;
...
@@ -246,7 +249,7 @@ export class CrawlJobService {
...
@@ -246,7 +249,7 @@ export class CrawlJobService {
for
(
const
exportRecord
of
exports
)
{
for
(
const
exportRecord
of
exports
)
{
if
(
if
(
exportRecord
.
exportType
===
EXPORT_TYPE
.
ZIP
&&
exportRecord
.
exportType
===
EXPORT_TYPE
.
ZIP
&&
exportRecord
.
status
===
JOB
_STATUS
.
COMPLETED
&&
exportRecord
.
status
===
EXPORT
_STATUS
.
COMPLETED
&&
(
await
storage
.
exists
(
exportRecord
.
filePath
))
(
await
storage
.
exists
(
exportRecord
.
filePath
))
)
{
)
{
return
exportRecord
;
return
exportRecord
;
...
...
src/modules/crawl-jobs/crawl-job.validation.ts
View file @
5b123692
...
@@ -74,3 +74,15 @@ export const getAssetsQuerySchema = z.object({
...
@@ -74,3 +74,15 @@ export const getAssetsQuerySchema = z.object({
page
:
z
.
coerce
.
number
().
int
().
min
(
1
).
default
(
1
),
page
:
z
.
coerce
.
number
().
int
().
min
(
1
).
default
(
1
),
limit
:
z
.
coerce
.
number
().
int
().
min
(
1
).
max
(
500
).
default
(
50
),
limit
:
z
.
coerce
.
number
().
int
().
min
(
1
).
max
(
500
).
default
(
50
),
});
});
export
const
jobLogsQuerySchema
=
z
.
object
({
page
:
z
.
coerce
.
number
().
int
().
min
(
1
).
default
(
1
),
limit
:
z
.
coerce
.
number
().
int
().
min
(
1
).
max
(
100
).
default
(
50
),
});
export
const
diffQuerySchema
=
z
.
object
({
compareWithJobId
:
z
.
string
()
.
uuid
(
"Invalid compareWithJobId format"
)
.
optional
(),
});
src/modules/crawl-schedules/crawl-schedule.route.ts
View file @
5b123692
...
@@ -6,9 +6,10 @@ import {
...
@@ -6,9 +6,10 @@ import {
createCrawlScheduleSchema
,
createCrawlScheduleSchema
,
updateCrawlScheduleSchema
,
updateCrawlScheduleSchema
,
crawlScheduleQuerySchema
,
crawlScheduleQuerySchema
,
crawlScheduleHistoryQuerySchema
,
}
from
"./crawl-schedule.validation"
;
}
from
"./crawl-schedule.validation"
;
import
{
require
Role
}
from
"../../middlewares/role
.middleware"
;
import
{
require
Permission
}
from
"../../middlewares/permission
.middleware"
;
import
{
ROLES
}
from
"../../common/constants/role
.constant"
;
import
{
PERMISSIONS
}
from
"../../common/constants/permission
.constant"
;
const
router
=
Router
();
const
router
=
Router
();
const
controller
=
new
CrawlScheduleController
();
const
controller
=
new
CrawlScheduleController
();
...
@@ -16,7 +17,7 @@ const controller = new CrawlScheduleController();
...
@@ -16,7 +17,7 @@ const controller = new CrawlScheduleController();
router
.
post
(
router
.
post
(
"/"
,
"/"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
),
require
Permission
(
PERMISSIONS
.
CRAWL_SCHEDULES_CREATE
),
validate
(
createCrawlScheduleSchema
),
validate
(
createCrawlScheduleSchema
),
controller
.
create
,
controller
.
create
,
);
);
...
@@ -24,7 +25,7 @@ router.post(
...
@@ -24,7 +25,7 @@ router.post(
router
.
get
(
router
.
get
(
"/"
,
"/"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
require
Permission
(
PERMISSIONS
.
CRAWL_SCHEDULES_READ
),
validateQuery
(
crawlScheduleQuerySchema
),
validateQuery
(
crawlScheduleQuerySchema
),
controller
.
findAll
,
controller
.
findAll
,
);
);
...
@@ -32,14 +33,14 @@ router.get(
...
@@ -32,14 +33,14 @@ router.get(
router
.
get
(
router
.
get
(
"/:id"
,
"/:id"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
require
Permission
(
PERMISSIONS
.
CRAWL_SCHEDULES_READ
),
controller
.
findById
,
controller
.
findById
,
);
);
router
.
patch
(
router
.
patch
(
"/:id"
,
"/:id"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
),
require
Permission
(
PERMISSIONS
.
CRAWL_SCHEDULES_UPDATE
),
validate
(
updateCrawlScheduleSchema
),
validate
(
updateCrawlScheduleSchema
),
controller
.
update
,
controller
.
update
,
);
);
...
@@ -47,21 +48,22 @@ router.patch(
...
@@ -47,21 +48,22 @@ router.patch(
router
.
delete
(
router
.
delete
(
"/:id"
,
"/:id"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
),
require
Permission
(
PERMISSIONS
.
CRAWL_SCHEDULES_DELETE
),
controller
.
delete
,
controller
.
delete
,
);
);
router
.
post
(
router
.
post
(
"/:id/run"
,
"/:id/run"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
require
Role
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
),
require
Permission
(
PERMISSIONS
.
CRAWL_SCHEDULES_RUN
),
controller
.
triggerRun
,
controller
.
triggerRun
,
);
);
router
.
get
(
router
.
get
(
"/:id/history"
,
"/:id/history"
,
apiKeyOrAuthMiddleware
,
apiKeyOrAuthMiddleware
,
requireRole
(
ROLES
.
ADMIN
,
ROLES
.
CRAWLER_USER
,
ROLES
.
VIEWER
),
requirePermission
(
PERMISSIONS
.
CRAWL_SCHEDULES_READ
),
validateQuery
(
crawlScheduleHistoryQuerySchema
),
controller
.
getHistory
,
controller
.
getHistory
,
);
);
...
...
src/modules/crawl-schedules/crawl-schedule.validation.ts
View file @
5b123692
...
@@ -106,3 +106,8 @@ export const crawlScheduleQuerySchema = z.object({
...
@@ -106,3 +106,8 @@ export const crawlScheduleQuerySchema = z.object({
.
default
(
"createdAt"
),
.
default
(
"createdAt"
),
order
:
z
.
enum
([
"asc"
,
"desc"
]).
optional
().
default
(
"desc"
),
order
:
z
.
enum
([
"asc"
,
"desc"
]).
optional
().
default
(
"desc"
),
});
});
export
const
crawlScheduleHistoryQuerySchema
=
z
.
object
({
page
:
z
.
coerce
.
number
().
int
().
min
(
1
).
default
(
1
),
limit
:
z
.
coerce
.
number
().
int
().
min
(
1
).
max
(
100
).
default
(
20
),
});
src/modules/dashboard/dashboard.repository.ts
View file @
5b123692
import
{
prisma
}
from
"../../database/prisma.client"
;
import
{
prisma
}
from
"../../database/prisma.client"
;
import
{
ROLES
}
from
"../../common/constants/role.constant"
;
import
{
ROLES
}
from
"../../common/constants/role.constant"
;
import
{
JOB_STATUS
}
from
"../../common/constants/job-status.constant"
;
import
{
JOB_STATUS
}
from
"../../common/constants/job-status.constant"
;
import
{
C
rawlPageStatus
}
from
"@prisma/clie
nt"
;
import
{
C
RAWL_PAGE_STATUS
}
from
"../../common/constants/crawl-page-status.consta
nt"
;
export
class
DashboardRepository
{
export
class
DashboardRepository
{
async
getStats
(
userId
:
string
,
role
:
string
)
{
async
getStats
(
userId
:
string
,
role
:
string
)
{
...
@@ -12,41 +12,24 @@ export class DashboardRepository {
...
@@ -12,41 +12,24 @@ export class DashboardRepository {
const
exportWhere
=
isGlobal
?
{}
:
{
job
:
{
userId
}
};
const
exportWhere
=
isGlobal
?
{}
:
{
job
:
{
userId
}
};
const
[
const
[
totalJobs
,
jobStatusGroups
,
completedJobs
,
pageStatusGroups
,
failedJobs
,
runningJobs
,
pendingJobs
,
totalPagesCrawled
,
totalPagesCrawled
,
successfulPages
,
failedPages
,
activeSchedules
,
activeSchedules
,
totalSchedules
,
totalSchedules
,
totalExports
,
totalExports
,
]
=
await
Promise
.
all
([
]
=
await
Promise
.
all
([
prisma
.
crawlJob
.
count
({
where
:
jobWhere
}),
prisma
.
crawlJob
.
groupBy
({
prisma
.
crawlJob
.
count
({
by
:
[
"status"
],
where
:
{
...
jobWhere
,
status
:
JOB_STATUS
.
COMPLETED
},
_count
:
{
status
:
true
},
where
:
jobWhere
,
}),
}),
prisma
.
crawlJob
.
count
({
prisma
.
crawlPage
.
groupBy
({
where
:
{
...
jobWhere
,
status
:
JOB_STATUS
.
FAILED
},
by
:
[
"status"
],
}),
_count
:
{
status
:
true
},
prisma
.
crawlJob
.
count
({
where
:
pageWhere
,
where
:
{
...
jobWhere
,
status
:
JOB_STATUS
.
RUNNING
},
}),
prisma
.
crawlJob
.
count
({
where
:
{
...
jobWhere
,
status
:
{
in
:
[
JOB_STATUS
.
PENDING
,
JOB_STATUS
.
QUEUED
]
},
},
}),
}),
prisma
.
crawlPage
.
count
({
where
:
pageWhere
}),
prisma
.
crawlPage
.
count
({
where
:
pageWhere
}),
prisma
.
crawlPage
.
count
({
where
:
{
...
pageWhere
,
status
:
CrawlPageStatus
.
SUCCESS
},
}),
prisma
.
crawlPage
.
count
({
where
:
{
...
pageWhere
,
status
:
CrawlPageStatus
.
FAILED
},
}),
prisma
.
crawlSchedule
.
count
({
prisma
.
crawlSchedule
.
count
({
where
:
{
...
scheduleWhere
,
isActive
:
true
},
where
:
{
...
scheduleWhere
,
isActive
:
true
},
}),
}),
...
@@ -54,18 +37,32 @@ export class DashboardRepository {
...
@@ -54,18 +37,32 @@ export class DashboardRepository {
prisma
.
crawlExport
.
count
({
where
:
exportWhere
}),
prisma
.
crawlExport
.
count
({
where
:
exportWhere
}),
]);
]);
const
jobCounts
:
Record
<
string
,
number
>
=
{};
let
totalJobs
=
0
;
for
(
const
group
of
jobStatusGroups
)
{
jobCounts
[
group
.
status
]
=
group
.
_count
.
status
;
totalJobs
+=
group
.
_count
.
status
;
}
const
pageCounts
:
Record
<
string
,
number
>
=
{};
for
(
const
group
of
pageStatusGroups
)
{
pageCounts
[
group
.
status
]
=
group
.
_count
.
status
;
}
return
{
return
{
jobs
:
{
jobs
:
{
total
:
totalJobs
,
total
:
totalJobs
,
completed
:
completedJobs
,
completed
:
jobCounts
[
JOB_STATUS
.
COMPLETED
]
??
0
,
failed
:
failedJobs
,
failed
:
jobCounts
[
JOB_STATUS
.
FAILED
]
??
0
,
running
:
runningJobs
,
running
:
jobCounts
[
JOB_STATUS
.
RUNNING
]
??
0
,
pending
:
pendingJobs
,
pending
:
(
jobCounts
[
JOB_STATUS
.
PENDING
]
??
0
)
+
(
jobCounts
[
JOB_STATUS
.
QUEUED
]
??
0
),
},
},
pages
:
{
pages
:
{
total
:
totalPagesCrawled
,
total
:
totalPagesCrawled
,
successful
:
successfulPages
,
successful
:
pageCounts
[
CRAWL_PAGE_STATUS
.
SUCCESS
]
??
0
,
failed
:
failedPages
,
failed
:
pageCounts
[
CRAWL_PAGE_STATUS
.
FAILED
]
??
0
,
},
},
schedules
:
{
schedules
:
{
total
:
totalSchedules
,
total
:
totalSchedules
,
...
...
src/modules/dashboard/dashboard.route.ts
View file @
5b123692
import
{
Router
}
from
"express"
;
import
{
Router
}
from
"express"
;
import
{
DashboardController
}
from
"./dashboard.controller"
;
import
{
DashboardController
}
from
"./dashboard.controller"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
requirePermission
}
from
"../../middlewares/permission.middleware"
;
import
{
PERMISSIONS
}
from
"../../common/constants/permission.constant"
;
const
router
=
Router
();
const
router
=
Router
();
const
controller
=
new
DashboardController
();
const
controller
=
new
DashboardController
();
router
.
get
(
"/stats"
,
authMiddleware
,
controller
.
getStats
);
router
.
get
(
"/stats"
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
DASHBOARD_READ
),
controller
.
getStats
,
);
export
default
router
;
export
default
router
;
src/modules/exports/json-export.service.ts
View file @
5b123692
...
@@ -60,17 +60,22 @@ export class JsonExportService extends BaseExportService {
...
@@ -60,17 +60,22 @@ export class JsonExportService extends BaseExportService {
fs
.
writeFileSync
(
filePath
,
JSON
.
stringify
(
wrapper
,
null
,
2
),
"utf-8"
);
fs
.
writeFileSync
(
filePath
,
JSON
.
stringify
(
wrapper
,
null
,
2
),
"utf-8"
);
// 1. Xuất pages.raw.json (Lọc bỏ các trường dữ liệu sạch & chất lượng)
// 1. Xuất pages.raw.json (Lọc bỏ các trường dữ liệu sạch & chất lượng)
const
rawPages
=
pages
.
map
(
const
rawPages
=
pages
.
map
((
page
)
=>
({
({
id
:
page
.
id
,
cleanText
:
_cleanText
,
jobId
:
page
.
jobId
,
mainContent
:
_mainContent
,
url
:
page
.
url
,
wordCount
:
_wordCount
,
normalizedUrl
:
page
.
normalizedUrl
,
contentHash
:
_contentHash
,
status
:
page
.
status
,
dataQualityScore
:
_dataQualityScore
,
statusCode
:
page
.
statusCode
,
warnings
:
_warnings
,
errorMessage
:
page
.
errorMessage
,
...
rawFields
title
:
page
.
title
,
})
=>
rawFields
,
description
:
page
.
description
,
);
rawMarkdown
:
page
.
rawMarkdown
,
links
:
page
.
links
,
images
:
page
.
images
,
tables
:
page
.
tables
,
crawledAt
:
page
.
crawledAt
,
}));
const
{
filePath
:
rawFilePath
}
=
buildJobDataRawFilePath
(
const
{
filePath
:
rawFilePath
}
=
buildJobDataRawFilePath
(
job
.
id
,
job
.
id
,
JOB_EXPORT_FILES
.
PAGES_RAW_JSON
,
JOB_EXPORT_FILES
.
PAGES_RAW_JSON
,
...
@@ -79,9 +84,27 @@ export class JsonExportService extends BaseExportService {
...
@@ -79,9 +84,27 @@ export class JsonExportService extends BaseExportService {
fs
.
writeFileSync
(
rawFilePath
,
JSON
.
stringify
(
rawWrapper
,
null
,
2
),
"utf-8"
);
fs
.
writeFileSync
(
rawFilePath
,
JSON
.
stringify
(
rawWrapper
,
null
,
2
),
"utf-8"
);
// 2. Xuất pages.clean.json (Lọc bỏ trường rawMarkdown)
// 2. Xuất pages.clean.json (Lọc bỏ trường rawMarkdown)
const
cleanPages
=
pages
.
map
(
const
cleanPages
=
pages
.
map
((
page
)
=>
({
({
rawMarkdown
:
_rawMarkdown
,
...
cleanFields
})
=>
cleanFields
,
id
:
page
.
id
,
);
jobId
:
page
.
jobId
,
url
:
page
.
url
,
normalizedUrl
:
page
.
normalizedUrl
,
status
:
page
.
status
,
statusCode
:
page
.
statusCode
,
errorMessage
:
page
.
errorMessage
,
title
:
page
.
title
,
description
:
page
.
description
,
cleanText
:
page
.
cleanText
,
mainContent
:
page
.
mainContent
,
wordCount
:
page
.
wordCount
,
contentHash
:
page
.
contentHash
,
dataQualityScore
:
page
.
dataQualityScore
,
warnings
:
page
.
warnings
,
links
:
page
.
links
,
images
:
page
.
images
,
tables
:
page
.
tables
,
crawledAt
:
page
.
crawledAt
,
}));
const
{
filePath
:
cleanFilePath
}
=
buildJobDataCleanFilePath
(
const
{
filePath
:
cleanFilePath
}
=
buildJobDataCleanFilePath
(
job
.
id
,
job
.
id
,
JOB_EXPORT_FILES
.
PAGES_CLEAN_JSON
,
JOB_EXPORT_FILES
.
PAGES_CLEAN_JSON
,
...
...
src/modules/exports/zip-export.service.ts
View file @
5b123692
...
@@ -11,6 +11,7 @@ import {
...
@@ -11,6 +11,7 @@ import {
buildCrawlResultZipName
,
buildCrawlResultZipName
,
}
from
"../../common/constants/storage-path.constant"
;
}
from
"../../common/constants/storage-path.constant"
;
import
{
EXPORT_MIME_TYPES
}
from
"../../common/constants/export-type.constant"
;
import
{
EXPORT_MIME_TYPES
}
from
"../../common/constants/export-type.constant"
;
import
{
CRAWL_PAGE_STATUS
}
from
"../../common/constants/crawl-page-status.constant"
;
import
{
import
{
buildJobLogsFilePath
,
buildJobLogsFilePath
,
buildJobRootFilePath
,
buildJobRootFilePath
,
...
@@ -101,9 +102,13 @@ export class ZipExportService extends BaseExportService {
...
@@ -101,9 +102,13 @@ export class ZipExportService extends BaseExportService {
private
writeSummary
(
job
:
CrawlJob
&
{
pages
:
CrawlPage
[]
}):
void
{
private
writeSummary
(
job
:
CrawlJob
&
{
pages
:
CrawlPage
[]
}):
void
{
const
{
filePath
}
=
buildJobRootFilePath
(
job
.
id
,
JOB_EXPORT_FILES
.
SUMMARY
);
const
{
filePath
}
=
buildJobRootFilePath
(
job
.
id
,
JOB_EXPORT_FILES
.
SUMMARY
);
const
successPages
=
job
.
pages
.
filter
((
p
)
=>
p
.
status
===
"SUCCESS"
).
length
;
const
successPages
=
job
.
pages
.
filter
(
(
p
)
=>
p
.
status
===
CRAWL_PAGE_STATUS
.
SUCCESS
,
).
length
;
const
failedPages
=
job
.
pages
.
filter
(
const
failedPages
=
job
.
pages
.
filter
(
(
p
)
=>
p
.
status
!==
"SUCCESS"
&&
p
.
status
!==
"SKIPPED"
,
(
p
)
=>
p
.
status
!==
CRAWL_PAGE_STATUS
.
SUCCESS
&&
p
.
status
!==
CRAWL_PAGE_STATUS
.
SKIPPED
,
).
length
;
).
length
;
const
summary
=
{
const
summary
=
{
...
@@ -125,9 +130,9 @@ export class ZipExportService extends BaseExportService {
...
@@ -125,9 +130,9 @@ export class ZipExportService extends BaseExportService {
const
errors
=
job
.
pages
const
errors
=
job
.
pages
.
filter
(
.
filter
(
(
p
)
=>
(
p
)
=>
p
.
status
!==
"SUCCESS"
&&
p
.
status
!==
CRAWL_PAGE_STATUS
.
SUCCESS
&&
p
.
status
!==
"PENDING"
&&
p
.
status
!==
CRAWL_PAGE_STATUS
.
PENDING
&&
p
.
status
!==
"SKIPPED"
,
p
.
status
!==
CRAWL_PAGE_STATUS
.
SKIPPED
,
)
)
.
map
((
p
)
=>
({
.
map
((
p
)
=>
({
url
:
p
.
url
,
url
:
p
.
url
,
...
@@ -145,12 +150,14 @@ export class ZipExportService extends BaseExportService {
...
@@ -145,12 +150,14 @@ export class ZipExportService extends BaseExportService {
JOB_EXPORT_FILES
.
DATA_QUALITY_JSON
,
JOB_EXPORT_FILES
.
DATA_QUALITY_JSON
,
);
);
const
successPages
=
job
.
pages
.
filter
((
p
)
=>
p
.
status
===
"SUCCESS"
);
const
successPages
=
job
.
pages
.
filter
(
(
p
)
=>
p
.
status
===
CRAWL_PAGE_STATUS
.
SUCCESS
,
);
const
errorPages
=
job
.
pages
.
filter
(
const
errorPages
=
job
.
pages
.
filter
(
(
p
)
=>
(
p
)
=>
p
.
status
!==
"SUCCESS"
&&
p
.
status
!==
CRAWL_PAGE_STATUS
.
SUCCESS
&&
p
.
status
!==
"SKIPPED"
&&
p
.
status
!==
CRAWL_PAGE_STATUS
.
SKIPPED
&&
p
.
status
!==
"PENDING"
,
p
.
status
!==
CRAWL_PAGE_STATUS
.
PENDING
,
);
);
const
seenHashes
=
new
Set
<
string
>
();
const
seenHashes
=
new
Set
<
string
>
();
...
...
src/modules/extraction-templates/extraction-template.route.ts
View file @
5b123692
import
{
Router
}
from
"express"
;
import
{
Router
}
from
"express"
;
import
{
ExtractionTemplateController
}
from
"./extraction-template.controller"
;
import
{
ExtractionTemplateController
}
from
"./extraction-template.controller"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
requirePermission
}
from
"../../middlewares/permission.middleware"
;
import
{
PERMISSIONS
}
from
"../../common/constants/permission.constant"
;
import
{
validate
}
from
"../../middlewares/validate.middleware"
;
import
{
validate
}
from
"../../middlewares/validate.middleware"
;
import
{
import
{
createExtractionTemplateSchema
,
createExtractionTemplateSchema
,
...
@@ -12,14 +14,32 @@ const controller = new ExtractionTemplateController();
...
@@ -12,14 +14,32 @@ const controller = new ExtractionTemplateController();
router
.
use
(
authMiddleware
);
router
.
use
(
authMiddleware
);
router
.
post
(
"/"
,
validate
(
createExtractionTemplateSchema
),
controller
.
create
);
router
.
post
(
router
.
get
(
"/"
,
controller
.
findAll
);
"/"
,
router
.
get
(
"/:id"
,
controller
.
findById
);
requirePermission
(
PERMISSIONS
.
EXTRACTION_TEMPLATES_CREATE
),
validate
(
createExtractionTemplateSchema
),
controller
.
create
,
);
router
.
get
(
"/"
,
requirePermission
(
PERMISSIONS
.
EXTRACTION_TEMPLATES_READ
),
controller
.
findAll
,
);
router
.
get
(
"/:id"
,
requirePermission
(
PERMISSIONS
.
EXTRACTION_TEMPLATES_READ
),
controller
.
findById
,
);
router
.
patch
(
router
.
patch
(
"/:id"
,
"/:id"
,
requirePermission
(
PERMISSIONS
.
EXTRACTION_TEMPLATES_UPDATE
),
validate
(
updateExtractionTemplateSchema
),
validate
(
updateExtractionTemplateSchema
),
controller
.
update
,
controller
.
update
,
);
);
router
.
delete
(
"/:id"
,
controller
.
delete
);
router
.
delete
(
"/:id"
,
requirePermission
(
PERMISSIONS
.
EXTRACTION_TEMPLATES_DELETE
),
controller
.
delete
,
);
export
default
router
;
export
default
router
;
src/modules/health/__tests__/health.service.test.ts
View file @
5b123692
import
{
HealthService
}
from
"../health.service"
;
import
{
HealthService
}
from
"../health.service"
;
import
{
prisma
}
from
"../../../database/prisma.client
"
;
import
{
HealthRepository
}
from
"../health.repository
"
;
jest
.
mock
(
"../../../database/prisma.client"
,
()
=>
({
jest
.
mock
(
"../health.repository"
);
prisma
:
{
$queryRaw
:
jest
.
fn
(),
},
}));
jest
.
mock
(
"../../../queues/crawl.queue"
,
()
=>
({
jest
.
mock
(
"../../../queues/crawl.queue"
,
()
=>
({
crawlQueue
:
{
crawlQueue
:
{
...
@@ -28,10 +24,12 @@ jest.mock("../../../queues/webhook.queue", () => ({
...
@@ -28,10 +24,12 @@ jest.mock("../../../queues/webhook.queue", () => ({
describe
(
"HealthService"
,
()
=>
{
describe
(
"HealthService"
,
()
=>
{
let
service
:
HealthService
;
let
service
:
HealthService
;
let
mockHealthRepo
:
jest
.
Mocked
<
HealthRepository
>
;
beforeEach
(()
=>
{
beforeEach
(()
=>
{
jest
.
clearAllMocks
();
jest
.
clearAllMocks
();
service
=
new
HealthService
();
mockHealthRepo
=
new
HealthRepository
()
as
jest
.
Mocked
<
HealthRepository
>
;
service
=
new
HealthService
(
mockHealthRepo
);
});
});
describe
(
"getLiveness"
,
()
=>
{
describe
(
"getLiveness"
,
()
=>
{
...
@@ -45,7 +43,7 @@ describe("HealthService", () => {
...
@@ -45,7 +43,7 @@ describe("HealthService", () => {
describe
(
"getReadiness"
,
()
=>
{
describe
(
"getReadiness"
,
()
=>
{
it
(
"returns ready status when database is up"
,
async
()
=>
{
it
(
"returns ready status when database is up"
,
async
()
=>
{
(
prisma
.
$queryRaw
as
jest
.
Mock
).
mockResolvedValue
([{
1
:
1
}]
);
mockHealthRepo
.
pingDatabase
.
mockResolvedValue
(
);
const
result
=
await
service
.
getReadiness
();
const
result
=
await
service
.
getReadiness
();
expect
(
result
.
status
).
toBe
(
"ready"
);
expect
(
result
.
status
).
toBe
(
"ready"
);
...
@@ -54,7 +52,7 @@ describe("HealthService", () => {
...
@@ -54,7 +52,7 @@ describe("HealthService", () => {
});
});
it
(
"returns unhealthy status when database query fails"
,
async
()
=>
{
it
(
"returns unhealthy status when database query fails"
,
async
()
=>
{
(
prisma
.
$queryRaw
as
jest
.
Mock
)
.
mockRejectedValue
(
mockHealthRepo
.
pingDatabase
.
mockRejectedValue
(
new
Error
(
"Connection timeout"
),
new
Error
(
"Connection timeout"
),
);
);
...
...
src/modules/health/health.repository.ts
0 → 100644
View file @
5b123692
import
{
prisma
}
from
"../../database/prisma.client"
;
export
class
HealthRepository
{
async
pingDatabase
():
Promise
<
void
>
{
await
prisma
.
$queryRaw
`SELECT 1`
;
}
}
src/modules/health/health.service.ts
View file @
5b123692
import
{
prisma
}
from
"../../database/prisma.client
"
;
import
{
HealthRepository
}
from
"./health.repository
"
;
import
{
crawlQueue
}
from
"../../queues/crawl.queue"
;
import
{
crawlQueue
}
from
"../../queues/crawl.queue"
;
import
{
webhookQueue
}
from
"../../queues/webhook.queue"
;
import
{
webhookQueue
}
from
"../../queues/webhook.queue"
;
import
{
getErrorMessage
}
from
"../../common/helpers/error-mapping.helper"
;
import
{
getErrorMessage
}
from
"../../common/helpers/error-mapping.helper"
;
...
@@ -13,6 +13,8 @@ export interface QueueCountMetrics {
...
@@ -13,6 +13,8 @@ export interface QueueCountMetrics {
export
type
QueueMetricsResult
=
QueueCountMetrics
|
"unavailable"
|
null
;
export
type
QueueMetricsResult
=
QueueCountMetrics
|
"unavailable"
|
null
;
export
class
HealthService
{
export
class
HealthService
{
constructor
(
private
readonly
repository
=
new
HealthRepository
())
{}
getLiveness
()
{
getLiveness
()
{
return
{
return
{
status
:
"ok"
,
status
:
"ok"
,
...
@@ -32,7 +34,7 @@ export class HealthService {
...
@@ -32,7 +34,7 @@ export class HealthService {
// 1. Check Database
// 1. Check Database
const
dbStart
=
Date
.
now
();
const
dbStart
=
Date
.
now
();
try
{
try
{
await
prisma
.
$queryRaw
`SELECT 1`
;
await
this
.
repository
.
pingDatabase
()
;
checks
.
database
=
{
checks
.
database
=
{
status
:
"up"
,
status
:
"up"
,
latencyMs
:
Date
.
now
()
-
dbStart
,
latencyMs
:
Date
.
now
()
-
dbStart
,
...
...
src/modules/roles/role.repository.ts
View file @
5b123692
...
@@ -76,6 +76,22 @@ export class RoleRepository {
...
@@ -76,6 +76,22 @@ export class RoleRepository {
});
});
}
}
async
findByIds
(
ids
:
string
[])
{
return
prisma
.
role
.
findMany
({
where
:
{
id
:
{
in
:
ids
}
},
include
:
{
rolePermissions
:
{
include
:
{
permission
:
true
,
},
},
_count
:
{
select
:
{
userRoles
:
true
},
},
},
});
}
async
findBySlug
(
slug
:
string
)
{
async
findBySlug
(
slug
:
string
)
{
return
prisma
.
role
.
findUnique
({
return
prisma
.
role
.
findUnique
({
where
:
{
slug
},
where
:
{
slug
},
...
...
src/modules/users/user.repository.ts
View file @
5b123692
...
@@ -154,6 +154,21 @@ export class UserRepository {
...
@@ -154,6 +154,21 @@ export class UserRepository {
where
:
{
userId
:
id
},
where
:
{
userId
:
id
},
});
});
await
tx
.
crawlSchedule
.
updateMany
({
where
:
{
userId
:
id
},
data
:
{
isActive
:
false
},
});
await
tx
.
apiKey
.
updateMany
({
where
:
{
userId
:
id
},
data
:
{
isActive
:
false
},
});
await
tx
.
webhookConfig
.
updateMany
({
where
:
{
userId
:
id
},
data
:
{
isActive
:
false
},
});
return
user
;
return
user
;
});
});
}
}
...
...
src/modules/users/user.service.ts
View file @
5b123692
...
@@ -261,12 +261,9 @@ export class UserService {
...
@@ -261,12 +261,9 @@ export class UserService {
await
this
.
findById
(
targetUserId
);
await
this
.
findById
(
targetUserId
);
// 2. Fetch target roles to validate
// 2. Fetch target roles to validate
const
targetRoles
=
await
Promise
.
all
(
const
targetRoles
=
await
this
.
roleRepository
.
findByIds
(
roleIds
);
roleIds
.
map
((
id
)
=>
this
.
roleRepository
.
findById
(
id
)),
);
const
missingRole
=
targetRoles
.
find
((
r
)
=>
!
r
);
if
(
targetRoles
.
length
!==
roleIds
.
length
)
{
if
(
missingRole
||
targetRoles
.
length
!==
roleIds
.
length
)
{
throw
new
AppError
(
throw
new
AppError
(
"One or more roles not found"
,
"One or more roles not found"
,
404
,
404
,
...
...
src/modules/webhooks/webhook-delivery.service.ts
View file @
5b123692
...
@@ -7,6 +7,7 @@ import { getErrorMessage } from "../../common/helpers/error-mapping.helper";
...
@@ -7,6 +7,7 @@ import { getErrorMessage } from "../../common/helpers/error-mapping.helper";
import
{
AppError
}
from
"../../common/errors/app-error"
;
import
{
AppError
}
from
"../../common/errors/app-error"
;
import
{
ERROR_CODE
}
from
"../../common/errors/error-code"
;
import
{
ERROR_CODE
}
from
"../../common/errors/error-code"
;
import
{
WEBHOOK_DELIVERY_STATUS
}
from
"../../common/constants/webhook.constant"
;
export
class
WebhookDeliveryService
{
export
class
WebhookDeliveryService
{
private
readonly
repository
=
new
WebhookRepository
();
private
readonly
repository
=
new
WebhookRepository
();
...
@@ -40,7 +41,7 @@ export class WebhookDeliveryService {
...
@@ -40,7 +41,7 @@ export class WebhookDeliveryService {
crawlJobId
,
crawlJobId
,
event
,
event
,
payload
,
payload
,
status
:
"PENDING"
,
status
:
WEBHOOK_DELIVERY_STATUS
.
PENDING
,
attempt
:
1
,
attempt
:
1
,
});
});
...
@@ -100,7 +101,7 @@ export class WebhookDeliveryService {
...
@@ -100,7 +101,7 @@ export class WebhookDeliveryService {
:
JSON
.
stringify
(
response
.
data
);
:
JSON
.
stringify
(
response
.
data
);
await
this
.
repository
.
updateDelivery
(
deliveryId
,
{
await
this
.
repository
.
updateDelivery
(
deliveryId
,
{
status
:
"SUCCESS"
,
status
:
WEBHOOK_DELIVERY_STATUS
.
SUCCESS
,
statusCode
:
response
.
status
,
statusCode
:
response
.
status
,
responseBody
:
responseBody
.
substring
(
0
,
2000
),
// Limit size stored in DB
responseBody
:
responseBody
.
substring
(
0
,
2000
),
// Limit size stored in DB
deliveredAt
:
new
Date
(),
deliveredAt
:
new
Date
(),
...
@@ -141,7 +142,7 @@ export class WebhookDeliveryService {
...
@@ -141,7 +142,7 @@ export class WebhookDeliveryService {
*/
*/
async
markFailed
(
deliveryId
:
string
,
errorReason
:
string
):
Promise
<
void
>
{
async
markFailed
(
deliveryId
:
string
,
errorReason
:
string
):
Promise
<
void
>
{
await
this
.
repository
.
updateDelivery
(
deliveryId
,
{
await
this
.
repository
.
updateDelivery
(
deliveryId
,
{
status
:
"FAILED"
,
status
:
WEBHOOK_DELIVERY_STATUS
.
FAILED
,
errorMessage
:
errorMessage
:
`Max attempts exhausted. Last error:
${
errorReason
}
`
.
substring
(
0
,
1000
),
`Max attempts exhausted. Last error:
${
errorReason
}
`
.
substring
(
0
,
1000
),
});
});
...
@@ -162,7 +163,7 @@ export class WebhookDeliveryService {
...
@@ -162,7 +163,7 @@ export class WebhookDeliveryService {
}
}
const
updated
=
await
this
.
repository
.
updateDelivery
(
deliveryId
,
{
const
updated
=
await
this
.
repository
.
updateDelivery
(
deliveryId
,
{
status
:
"PENDING"
,
status
:
WEBHOOK_DELIVERY_STATUS
.
PENDING
,
attempt
:
1
,
attempt
:
1
,
errorMessage
:
null
,
errorMessage
:
null
,
});
});
...
...
src/modules/webhooks/webhook.route.ts
View file @
5b123692
import
{
Router
}
from
"express"
;
import
{
Router
}
from
"express"
;
import
{
WebhookController
}
from
"./webhook.controller"
;
import
{
WebhookController
}
from
"./webhook.controller"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
authMiddleware
}
from
"../../middlewares/auth.middleware"
;
import
{
requirePermission
}
from
"../../middlewares/permission.middleware"
;
import
{
PERMISSIONS
}
from
"../../common/constants/permission.constant"
;
import
{
validate
,
validateQuery
}
from
"../../middlewares/validate.middleware"
;
import
{
validate
,
validateQuery
}
from
"../../middlewares/validate.middleware"
;
import
{
import
{
createWebhookConfigSchema
,
createWebhookConfigSchema
,
...
@@ -14,24 +16,47 @@ const controller = new WebhookController();
...
@@ -14,24 +16,47 @@ const controller = new WebhookController();
router
.
post
(
router
.
post
(
"/configs"
,
"/configs"
,
authMiddleware
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
WEBHOOKS_CREATE
),
validate
(
createWebhookConfigSchema
),
validate
(
createWebhookConfigSchema
),
controller
.
createConfig
,
controller
.
createConfig
,
);
);
router
.
get
(
"/configs"
,
authMiddleware
,
controller
.
listConfigs
);
router
.
get
(
"/configs"
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
WEBHOOKS_READ
),
controller
.
listConfigs
,
);
router
.
patch
(
router
.
patch
(
"/configs/:id"
,
"/configs/:id"
,
authMiddleware
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
WEBHOOKS_UPDATE
),
validate
(
updateWebhookConfigSchema
),
validate
(
updateWebhookConfigSchema
),
controller
.
updateConfig
,
controller
.
updateConfig
,
);
);
router
.
delete
(
"/configs/:id"
,
authMiddleware
,
controller
.
deleteConfig
);
router
.
delete
(
router
.
post
(
"/configs/:id/test"
,
authMiddleware
,
controller
.
testConfig
);
"/configs/:id"
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
WEBHOOKS_DELETE
),
controller
.
deleteConfig
,
);
router
.
post
(
"/configs/:id/test"
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
WEBHOOKS_TEST
),
controller
.
testConfig
,
);
router
.
get
(
router
.
get
(
"/deliveries"
,
"/deliveries"
,
authMiddleware
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
WEBHOOKS_READ
),
validateQuery
(
listWebhookDeliveriesQuerySchema
),
validateQuery
(
listWebhookDeliveriesQuerySchema
),
controller
.
listDeliveries
,
controller
.
listDeliveries
,
);
);
router
.
post
(
"/deliveries/:id/redeliver"
,
authMiddleware
,
controller
.
redeliver
);
router
.
post
(
"/deliveries/:id/redeliver"
,
authMiddleware
,
requirePermission
(
PERMISSIONS
.
WEBHOOKS_UPDATE
),
controller
.
redeliver
,
);
export
default
router
;
export
default
router
;
src/queues/crawl.worker.processor.ts
View file @
5b123692
...
@@ -19,6 +19,9 @@ import {
...
@@ -19,6 +19,9 @@ import {
import
{
runExtractionIfTemplate
}
from
"../modules/extraction-templates/extraction-runner"
;
import
{
runExtractionIfTemplate
}
from
"../modules/extraction-templates/extraction-runner"
;
import
{
JOB_STATUS
}
from
"../common/constants/job-status.constant"
;
import
{
JOB_STATUS
}
from
"../common/constants/job-status.constant"
;
import
{
CRAWL_MODE
}
from
"../common/constants/crawl-mode.constant"
;
import
{
CRAWL_MODE
}
from
"../common/constants/crawl-mode.constant"
;
import
{
ASSET_TYPE
}
from
"../common/constants/asset-type.constant"
;
import
{
CRAWL_PAGE_STATUS
}
from
"../common/constants/crawl-page-status.constant"
;
import
{
WEBHOOK_EVENT
}
from
"../common/constants/webhook.constant"
;
// Lazy getters — instantiated on first use so Jest mocks replace constructors before creation
// Lazy getters — instantiated on first use so Jest mocks replace constructors before creation
const
getJobRepository
=
()
=>
new
CrawlJobRepository
();
const
getJobRepository
=
()
=>
new
CrawlJobRepository
();
const
getPageRepository
=
()
=>
new
CrawlPageRepository
();
const
getPageRepository
=
()
=>
new
CrawlPageRepository
();
...
@@ -58,7 +61,7 @@ export async function savePageAssets(
...
@@ -58,7 +61,7 @@ export async function savePageAssets(
assetsBatch
.
push
({
assetsBatch
.
push
({
jobId
,
jobId
,
pageId
,
pageId
,
assetType
:
"IMAGE"
as
const
,
assetType
:
ASSET_TYPE
.
IMAGE
,
url
:
img
.
url
,
url
:
img
.
url
,
sourceUrl
:
item
.
url
,
sourceUrl
:
item
.
url
,
altText
:
img
.
alt
||
undefined
,
altText
:
img
.
alt
||
undefined
,
...
@@ -73,7 +76,7 @@ export async function savePageAssets(
...
@@ -73,7 +76,7 @@ export async function savePageAssets(
assetsBatch
.
push
({
assetsBatch
.
push
({
jobId
,
jobId
,
pageId
,
pageId
,
assetType
:
"LINK"
as
const
,
assetType
:
ASSET_TYPE
.
LINK
,
url
:
link
.
url
,
url
:
link
.
url
,
sourceUrl
:
item
.
url
,
sourceUrl
:
item
.
url
,
altText
:
link
.
text
||
undefined
,
altText
:
link
.
text
||
undefined
,
...
@@ -87,7 +90,7 @@ export async function savePageAssets(
...
@@ -87,7 +90,7 @@ export async function savePageAssets(
assetsBatch
.
push
({
assetsBatch
.
push
({
jobId
,
jobId
,
pageId
,
pageId
,
assetType
:
"PDF"
as
const
,
assetType
:
ASSET_TYPE
.
PDF
,
url
:
pdfUrl
,
url
:
pdfUrl
,
sourceUrl
:
item
.
url
,
sourceUrl
:
item
.
url
,
orderIndex
:
index
+
1
,
orderIndex
:
index
+
1
,
...
@@ -196,7 +199,7 @@ export async function persistBatchResults(
...
@@ -196,7 +199,7 @@ export async function persistBatchResults(
const
normalized
=
getPageProcessor
().
normalizeFailedPage
(
const
normalized
=
getPageProcessor
().
normalizeFailedPage
(
{
url
:
blockedUrl
,
error
:
"Blocked by robots.txt"
},
{
url
:
blockedUrl
,
error
:
"Blocked by robots.txt"
},
jobId
,
jobId
,
"BLOCKED"
,
CRAWL_PAGE_STATUS
.
BLOCKED
,
);
);
await
getPageRepository
().
upsert
(
normalized
);
await
getPageRepository
().
upsert
(
normalized
);
failedCount
++
;
failedCount
++
;
...
@@ -581,8 +584,8 @@ export async function processCrawlJob(job: Job): Promise<void> {
...
@@ -581,8 +584,8 @@ export async function processCrawlJob(job: Job): Promise<void> {
const event =
const event =
updatedJob.status === JOB_STATUS.COMPLETED
updatedJob.status === JOB_STATUS.COMPLETED
?
"job.completed"
?
WEBHOOK_EVENT.JOB_COMPLETED
:
"job.failed"
;
:
WEBHOOK_EVENT.JOB_FAILED
;
void logStep(
void logStep(
jobId,
jobId,
updatedJob.status === JOB_STATUS.COMPLETED ? "INFO" : "ERROR",
updatedJob.status === JOB_STATUS.COMPLETED ? "INFO" : "ERROR",
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment