Add project files.

.dockerignore

+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
\ No newline at end of file

.gitattributes

+###############################################################################
+# Set default behavior to automatically normalize line endings.
+###############################################################################
+* text=auto
+
+###############################################################################
+# Set default behavior for command prompt diff.
+#
+# This is need for earlier builds of msysgit that does not have it on by
+# default for csharp files.
+# Note: This is only used by command line
+###############################################################################
+#*.cs     diff=csharp
+
+###############################################################################
+# Set the merge driver for project and solution files
+#
+# Merging from the command prompt will add diff markers to the files if there
+# are conflicts (Merging from VS is not affected by the settings below, in VS
+# the diff markers are never inserted). Diff markers may cause the following 
+# file extensions to fail to load in VS. An alternative would be to treat
+# these files as binary and thus will always conflict and require user
+# intervention with every merge. To do so, just uncomment the entries below
+###############################################################################
+#*.sln       merge=binary
+#*.csproj    merge=binary
+#*.vbproj    merge=binary
+#*.vcxproj   merge=binary
+#*.vcproj    merge=binary
+#*.dbproj    merge=binary
+#*.fsproj    merge=binary
+#*.lsproj    merge=binary
+#*.wixproj   merge=binary
+#*.modelproj merge=binary
+#*.sqlproj   merge=binary
+#*.wwaproj   merge=binary
+
+###############################################################################
+# behavior for image files
+#
+# image files are treated as binary by default.
+###############################################################################
+#*.jpg   binary
+#*.png   binary
+#*.gif   binary
+
+###############################################################################
+# diff behavior for common document formats
+# 
+# Convert binary document formats to text before diffing them. This feature
+# is only available from the command line. Turn it on by uncommenting the 
+# entries below.
+###############################################################################
+#*.doc   diff=astextplain
+#*.DOC   diff=astextplain
+#*.docx  diff=astextplain
+#*.DOCX  diff=astextplain
+#*.dot   diff=astextplain
+#*.DOT   diff=astextplain
+#*.pdf   diff=astextplain
+#*.PDF   diff=astextplain
+#*.rtf   diff=astextplain
+#*.RTF   diff=astextplain

.gitignore

+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
+
+# User-specific files
+*.rsuser
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Mono auto generated files
+mono_crash.*
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+[Ww][Ii][Nn]32/
+[Aa][Rr][Mm]/
+[Aa][Rr][Mm]64/
+bld/
+[Bb]in/
+[Oo]bj/
+[Oo]ut/
+[Ll]og/
+[Ll]ogs/
+
+# Visual Studio 2015/2017 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# Visual Studio 2017 auto generated files
+Generated\ Files/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUnit
+*.VisualState.xml
+TestResult.xml
+nunit-*.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# Benchmark Results
+BenchmarkDotNet.Artifacts/
+
+# .NET Core
+project.lock.json
+project.fragment.lock.json
+artifacts/
+
+# ASP.NET Scaffolding
+ScaffoldingReadMe.txt
+
+# StyleCop
+StyleCopReport.xml
+
+# Files built by Visual Studio
+*_i.c
+*_p.c
+*_h.h
+*.ilk
+*.meta
+*.obj
+*.iobj
+*.pch
+*.pdb
+*.ipdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*_wpftmp.csproj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# Visual Studio Trace Files
+*.e2e
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# AxoCover is a Code Coverage Tool
+.axoCover/*
+!.axoCover/settings.json
+
+# Coverlet is a free, cross platform Code Coverage Tool
+coverage*.json
+coverage*.xml
+coverage*.info
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# Note: Comment the next line if you want to checkin your web deploy settings,
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# NuGet Symbol Packages
+*.snupkg
+# The packages folder can be ignored because of Package Restore
+**/[Pp]ackages/*
+# except build/, which is used as an MSBuild target.
+!**/[Pp]ackages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/[Pp]ackages/repositories.config
+# NuGet v3's project.json files produces more ignorable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+*.appx
+*.appxbundle
+*.appxupload
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!?*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+orleans.codegen.cs
+
+# Including strong name files can present a security risk
+# (https://github.com/github/gitignore/pull/2483#issue-259490424)
+#*.snk
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+ServiceFabricBackup/
+*.rptproj.bak
+
+# SQL Server files
+*.mdf
+*.ldf
+*.ndf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+*.rptproj.rsuser
+*- [Bb]ackup.rdl
+*- [Bb]ackup ([0-9]).rdl
+*- [Bb]ackup ([0-9][0-9]).rdl
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+node_modules/
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# CodeRush personal settings
+.cr/personal
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/**
+# !tools/packages.config
+
+# Tabs Studio
+*.tss
+
+# Telerik's JustMock configuration file
+*.jmconfig
+
+# BizTalk build output
+*.btp.cs
+*.btm.cs
+*.odx.cs
+*.xsd.cs
+
+# OpenCover UI analysis results
+OpenCover/
+
+# Azure Stream Analytics local run output
+ASALocalRun/
+
+# MSBuild Binary and Structured Log
+*.binlog
+
+# NVidia Nsight GPU debugger configuration file
+*.nvuser
+
+# MFractors (Xamarin productivity tool) working folder
+.mfractor/
+
+# Local History for Visual Studio
+.localhistory/
+
+# BeatPulse healthcheck temp database
+healthchecksdb
+
+# Backup folder for Package Reference Convert tool in Visual Studio 2017
+MigrationBackup/
+
+# Ionide (cross platform F# VS Code tools) working folder
+.ionide/
+
+# Fody - auto-generated XML schema
+FodyWeavers.xsd
\ No newline at end of file

Controllers/CategoryController.cs

+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.EntityFrameworkCore;
+using News_site.Data;
+using News_site.Models;
+using News_site.Provider;
+using News_site.Services;
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace News_site.Controllers
+{
+	[Route("api/categories")]
+	[ApiController]
+	public class CategoryController : ControllerBase
+	{
+		private readonly CategoryProvider CategoryProvider = new CategoryProvider();
+		private readonly HandleExceptionAsync _handleExceptionAsync = new HandleExceptionAsync();
+
+		[HttpGet]
+		public ActionResult GetAllCategories()
+		{
+			try
+			{
+				var categories = CategoryProvider.GetAll();
+
+				if (categories == null)
+				{
+					return NotFound();
+				}
+
+				return Ok(new { Success = true, Data = categories.Result });
+			} catch (Exception error)
+			{
+				_handleExceptionAsync.WriteError(error);
+				return BadRequest();
+			}
+		}
+
+		[HttpDelete]
+		public ActionResult DeleteAllCategories()
+		{
+			try
+			{
+				CategoryProvider.DeleteAll();
+				return NoContent();
+			}
+			catch (Exception error)
+			{
+				_handleExceptionAsync.WriteError(error);
+				return BadRequest();
+			}
+		}
+	}
+}

Controllers/CrawlController.cs

+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using News_site.Provider;
+using System;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+namespace News_site.Controllers
+{
+	[Route("api/crawlData")]
+	[ApiController]
+	public class CrawlController : ControllerBase
+	{
+		private readonly CrawlDataProvider _dataProvider = new CrawlDataProvider();
+
+		[HttpGet("category")]
+		public async Task<ActionResult<string>> CrawlCategory()
+		{
+			await _dataProvider.CrawlCategories();
+
+			return Ok(new { Data = true });
+		}
+
+		[HttpGet("news")]
+		public async Task<ActionResult<string>> CrawlNews()
+		{
+			await _dataProvider.CrawlLatestNews();
+
+			return Ok(new { Data = true });
+		}
+	}
+}

Controllers/NewsController.cs

+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.EntityFrameworkCore;
+using News_site.Data;
+using News_site.ExtendModels;
+using News_site.helpers;
+using News_site.Models;
+using News_site.Provider;
+using News_site.Services;
+using Sieve.Models;
+using Sieve.Services;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace News_site.Controllers
+{
+	[Route("api/news")]
+	[ApiController]
+	public class NewsController : ControllerBase
+	{
+		private readonly NewsProvider newsProvider = new NewsProvider();
+		private readonly HandleExceptionAsync handleExceptionAsync = new HandleExceptionAsync();
+		private readonly SieveProcessor _sieveProcessor;
+
+		public NewsController(SieveProcessor sieveProcessor)
+		{
+			_sieveProcessor = sieveProcessor;
+		}
+
+		[HttpGet]
+		public ActionResult GetNews([FromQuery] SieveModel sieveModel)
+		{
+			try
+			{
+				List<Newspaper> newspapers = newsProvider.GetAll();
+
+				if (newspapers == null)
+				{
+					return NotFound();
+				}
+
+				var result = _sieveProcessor.Apply(sieveModel, newspapers.AsQueryable());
+	
+
+				return Ok(new { 
+					success = true, 
+					data = new { 
+						collection = result.Select(news => new
+						{
+							news.Id,
+							news.Image,
+							news.Title,
+							news.Categorylink,
+							news.CategorylinkNavigation,
+							news.Description,
+							news.CreatedAt
+						}).ToList(),
+						total = result.Count(),
+						pagesize = sieveModel.PageSize, 
+						pageIndex = sieveModel.Page
+					} 
+				});
+			} catch (Exception error)
+			{
+				handleExceptionAsync.WriteError(error);
+				return BadRequest(error.Message);
+			}
+		}
+
+		[HttpGet("{id:Guid}")]
+		public ActionResult GetNewsById(Guid id)
+		{
+			try
+			{
+				var newspapers = newsProvider.GetById(id);
+
+				if (newspapers == null)
+				{
+					return NotFound();
+				}
+
+				return Ok(new { success = true, data = newspapers.Result });
+			} catch (Exception error)
+			{
+				handleExceptionAsync.WriteError(error);
+				return BadRequest(error.Message);
+			}
+		}
+
+		[HttpGet("getFilterNews")]
+		public ActionResult GetFilteredNews(string category)
+		{
+			try
+			{
+				var newspapers = newsProvider.GetByCategoryLink(category);
+
+				if (newspapers == null)
+				{
+					return NotFound();
+				}
+
+				return Ok(new { success = true, data = newspapers.Result });
+			}
+			catch (Exception error)
+			{
+				handleExceptionAsync.WriteError(error);
+				return BadRequest(error.Message);
+			}
+		}
+	}
+}

Data/DataContext.cs

+// <auto-generated> This file has been auto generated by EF Core Power Tools. </auto-generated>
+#nullable disable
+using System;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata;
+using News_site.Models;
+
+namespace News_site.Data
+{
+    public partial class DataContext : DbContext
+    {
+        public DataContext()
+        {
+        }
+
+        public DataContext(DbContextOptions<DataContext> options)
+            : base(options)
+        {
+        }
+
+        public virtual DbSet<Category> Categories { get; set; }
+        public virtual DbSet<Newspaper> Newspapers { get; set; }
+
+        protected override void OnModelCreating(ModelBuilder modelBuilder)
+        {
+            modelBuilder.HasPostgresExtension("uuid-ossp");
+
+            modelBuilder.Entity<Category>(entity =>
+            {
+                entity.HasKey(e => e.Link)
+                    .HasName("category_pkey");
+            });
+
+            modelBuilder.Entity<Newspaper>(entity =>
+            {
+                entity.Property(e => e.Id).ValueGeneratedNever();
+
+                entity.HasOne(d => d.CategorylinkNavigation)
+                    .WithMany(p => p.Newspapers)
+                    .HasForeignKey(d => d.Categorylink)
+                    .OnDelete(DeleteBehavior.ClientSetNull)
+                    .HasConstraintName("newspaper_categorylink_fkey");
+            });
+
+            OnModelCreatingPartial(modelBuilder);
+        }
+
+        partial void OnModelCreatingPartial(ModelBuilder modelBuilder);
+    }
+}
\ No newline at end of file

Dockerfile

+#See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
+
+FROM mcr.microsoft.com/dotnet/aspnet:3.1 AS base
+WORKDIR /app
+EXPOSE 80
+EXPOSE 443
+
+FROM mcr.microsoft.com/dotnet/sdk:3.1 AS build
+WORKDIR /src
+COPY ["News site.csproj", "."]
+RUN dotnet restore "./News site.csproj"
+COPY . .
+WORKDIR "/src/."
+RUN dotnet build "News site.csproj" -c Release -o /app/build
+
+FROM build AS publish
+RUN dotnet publish "News site.csproj" -c Release -o /app/publish /p:UseAppHost=false
+
+FROM base AS final
+WORKDIR /app
+COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "News site.dll"]
\ No newline at end of file

ExtendModels/ExtendNewspaper.cs

+using News_site.Models;
+using System.ComponentModel.DataAnnotations.Schema;
+using System.ComponentModel.DataAnnotations;
+using System;
+
+namespace News_site.ExtendModels
+{
+	public class ExtendNewspaper : Newspaper
+	{
+		public string Label => this.CategorylinkNavigation.Label.ToString();
+	}
+}

Models/Category.cs

+// <auto-generated> This file has been auto generated by EF Core Power Tools. </auto-generated>
+#nullable disable
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+
+namespace News_site.Models
+{
+    [Table("category")]
+    public partial class Category
+    {
+        public Category()
+        {
+            Newspapers = new HashSet<Newspaper>();
+        }
+
+        [Required]
+        [Column("label")]
+        [StringLength(20)]
+        public string Label { get; set; }
+        [Key]
+        [Column("link")]
+        [StringLength(50)]
+        public string Link { get; set; }
+
+        [JsonIgnore]
+        [InverseProperty(nameof(Newspaper.CategorylinkNavigation))]
+        public virtual ICollection<Newspaper> Newspapers { get; set; }
+    }
+}
\ No newline at end of file

Models/Newspaper.cs

+// <auto-generated> This file has been auto generated by EF Core Power Tools. </auto-generated>
+#nullable disable
+using Sieve.Attributes;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+
+namespace News_site.Models
+{
+    [Table("newspaper")]
+    public partial class Newspaper
+    {
+        [Key]
+        [Column("id")]
+		public Guid Id { get; set; }
+        [Required]
+		[Sieve(CanFilter = true)]
+        [Column("categorylink")]
+        [StringLength(50)]
+        public string Categorylink { get; set; }
+        [Required]
+        [Column("title")]
+        [StringLength(200)]
+        public string Title { get; set; }
+        [Required]
+        [Column("content")]
+        public string Content { get; set; }
+        [Required]
+        [Column("description")]
+        public string Description { get; set; }
+        [Required]
+        [Column("image")]
+        public string Image { get; set; }
+		[Column("created_at")]
+		[Sieve(CanSort = true)]
+		public DateTime CreatedAt { get; set; }
+
+        [ForeignKey(nameof(Categorylink))]
+        [InverseProperty(nameof(Category.Newspapers))]
+        public virtual Category CategorylinkNavigation { get; set; }
+    }
+}
\ No newline at end of file

News site.csproj

+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <RootNamespace>News_site</RootNamespace>
+    <UserSecretsId>cdc48a06-f83e-42c9-82be-a1e1788a18c3</UserSecretsId>
+    <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
+    <DockerfileContext>.</DockerfileContext>
+    <DockerComposeProjectPath>docker-compose.dcproj</DockerComposeProjectPath>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.32" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="3.1.30" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="3.1.30">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="3.1.30">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.17.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="Npgsql" Version="4.1.9" />
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="3.1.18" />
+    <PackageReference Include="Sieve" Version="2.5.5" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+
+</Project>

News site.sln

+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32825.248
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "News site", "News site.csproj", "{16D2415D-89AF-4C8B-A1F9-EC10A64E8D2C}"
+EndProject
+Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-compose.dcproj", "{7542A12B-177D-4A8C-9F4F-45D0374A3C44}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{16D2415D-89AF-4C8B-A1F9-EC10A64E8D2C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{16D2415D-89AF-4C8B-A1F9-EC10A64E8D2C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{16D2415D-89AF-4C8B-A1F9-EC10A64E8D2C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{16D2415D-89AF-4C8B-A1F9-EC10A64E8D2C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7542A12B-177D-4A8C-9F4F-45D0374A3C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7542A12B-177D-4A8C-9F4F-45D0374A3C44}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7542A12B-177D-4A8C-9F4F-45D0374A3C44}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7542A12B-177D-4A8C-9F4F-45D0374A3C44}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {76A52E3F-9F70-49C2-BCAD-BD4BC269443F}
+	EndGlobalSection
+EndGlobal

Program.cs

+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using News_site.Services;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace News_site
+{
+	public class Program
+	{
+		public static void Main(string[] args)
+		{
+			CreateHostBuilder(args).Build().Run();
+		}
+
+		public static IHostBuilder CreateHostBuilder(string[] args) =>
+			Host.CreateDefaultBuilder(args)
+				.ConfigureWebHostDefaults(webBuilder =>
+				{
+					webBuilder.UseStartup<Startup>();
+				});
+	}
+}

Properties/launchSettings.json

+{
+  "profiles": {
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "News_site": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "applicationUrl": "https://localhost:5001;http://localhost:5000"
+    },
+    "Docker": {
+      "commandName": "Docker",
+      "launchBrowser": true,
+      "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}/swagger",
+      "publishAllPorts": true,
+      "useSSL": true
+    }
+  },
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:57242",
+      "sslPort": 44386
+    }
+  }
+}
\ No newline at end of file

Provider/BaseProvider.cs

+using Microsoft.EntityFrameworkCore;
+using News_site.Data;
+using System.Threading.Tasks;
+
+namespace News_site.Provider
+{
+	public class BaseProvider
+	{
+		public DataContext db = null;
+
+		public BaseProvider()
+		{
+			DbContextOptionsBuilder<DataContext> optionsBuilder = new DbContextOptionsBuilder<DataContext>();
+			optionsBuilder.UseNpgsql("{YOUR CONNECTION STRING}");
+			db = new DataContext(optionsBuilder.Options);
+		}
+		public async Task<bool> SaveDataAsync()
+		{
+			await db.SaveChangesAsync();
+			return true;
+
+		}
+	}
+}

Provider/CategoryProvider.cs

+using Microsoft.EntityFrameworkCore;
+using News_site.Data;
+using News_site.Models;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace News_site.Provider
+{
+	public class CategoryProvider : BaseProvider
+	{
+		public async Task<List<Category>> GetAll()
+		{
+			return await db.Categories.ToListAsync();
+		}
+
+		public async Task DeleteAll()
+		{
+			var items = await GetAll();
+			foreach (var item in items)
+			{
+				db.Categories.Remove(item);
+			}
+			await db.SaveChangesAsync();
+		}
+	}
+}

Provider/CrawlDataProvider.cs

+using System.Net.Http;
+using System;
+using System.Text.RegularExpressions;
+using System.Net;
+using News_site.Models;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using System.Linq;
+using System.Collections.Generic;
+using News_site.Services;
+using System.Globalization;
+
+namespace News_site.Provider
+{
+	public class CrawlDataProvider : BaseProvider
+	{
+		private readonly string baseUrl = "https://tuoitre.vn/";
+		private readonly HttpClient httpClient = new HttpClient();
+		private readonly HandleExceptionAsync _handleExceptionAsync = new HandleExceptionAsync();
+
+		public string CrawlDataFromUrl(string url = "", string absoluteUrl = "")
+		{
+			try
+			{
+				string targetUrl = absoluteUrl == "" ? baseUrl + url : absoluteUrl + url;
+				string html = WebUtility.HtmlDecode(httpClient.GetStringAsync(targetUrl).Result);
+
+				return html;
+			} catch (Exception error)
+			{
+				_handleExceptionAsync.WriteError(error);
+				return "";
+			}
+		}
+
+		public async Task CrawlLatestNews()
+		{
+			var newsList = new List<Newspaper>();
+			var newestSite = "tin-moi-nhat.htm";
+			string htmlData = CrawlDataFromUrl(newestSite);
+			var newsListHtml = Regex.Matches(htmlData, @"<div class=""box-category-item(.*?)</p>", RegexOptions.Singleline);
+
+			foreach (var newsItem in newsListHtml)
+			{
+				string newsUrl = Regex.Match(newsItem.ToString(), @"href=""(.*?)""", RegexOptions.Singleline).Value.Replace("href=\"", "").Replace("\"", "");
+
+				string newsTitleElem = Regex.Match(newsItem.ToString(), @"<h3(.*?)</h3>", RegexOptions.Singleline).Value;
+				string newsTitle = Regex.Match(newsTitleElem, @"title=""(.*?)""", RegexOptions.Singleline).Value.Replace("title=\"", "").Replace("\"", "");
+				string newsImgUrl = Regex.Match(newsItem.ToString(), @"src=""(.*?)""", RegexOptions.Singleline).Value.Replace("src=\"", "").Replace("\"", "");
+
+				string newsDescriptionElem = Regex.Match(newsItem.ToString(), @"<p(.*?)</p>", RegexOptions.Singleline).Value;
+				string newsDescription = Regex.Match(newsDescriptionElem, @">(.*?)<", RegexOptions.Singleline).Value.Replace(">", "").Replace("<", "");
+
+				string newsCategoryElem = Regex.Match(newsItem.ToString(), @"<a class=""box-category-category(.*?)</a>", RegexOptions.Singleline).Value;
+				string newsCategory = Regex.Match(newsCategoryElem, @">(.*?)<", RegexOptions.Singleline).Value.Replace(">", "").Replace("<", "");
+
+				var categoryInDb = await db.Categories.Where(category => category.Label.ToLower().Equals(newsCategory.ToLower())).FirstOrDefaultAsync();
+				bool isAbsoluteCategoryLink = !(categoryInDb is null) && categoryInDb.Link.Contains("https");
+				string newsData = isAbsoluteCategoryLink ? CrawlDataFromUrl(newsUrl, categoryInDb.Link) : CrawlDataFromUrl(newsUrl);
+
+				if (newsData != "")
+				{
+					string newsTime = Regex.Match(newsData, @"publishdate(.*?)G", RegexOptions.Singleline).Value.Replace("publishdate\">", "").Replace("G", "").Trim();
+					string newsContent = Regex.Match(newsData, @"<div class=""detail-content(.*?)RelatedOneNews(.*?)</div>", RegexOptions.Singleline).Value + "</div>";
+
+					string checkCategoryLinkElem = Regex.Match(newsData, @"<div class=""detail-cate(.*?)</a>", RegexOptions.Singleline).Value;
+					string checkCategoryLink = Regex.Match(checkCategoryLinkElem, @"href=""(.*?)""", RegexOptions.Singleline).Value.Replace("href=\"", "").Replace("\"", "");
+
+					bool isNewspaperCrawled = await db.Newspapers.AnyAsync(news => news.Title.Equals(newsTitle));
+					bool isCategoryExisted = await db.Categories.AnyAsync(category => category.Link.Equals(checkCategoryLink));
+
+					if (!isNewspaperCrawled && isCategoryExisted)
+					{
+						Newspaper newspaper = new Newspaper
+						{
+							Id = Guid.NewGuid(),
+							Image = newsImgUrl,
+							Title = newsTitle,
+							Description = newsDescription,
+							Categorylink = checkCategoryLink,
+							CreatedAt = DateTime.ParseExact(newsTime, "dd/MM/yyyy HH:mm", CultureInfo.CurrentCulture),
+							Content = newsContent,
+						};
+
+						newsList.Add(newspaper);
+					}
+				}
+			}
+
+			db.Newspapers.AddRange(newsList);
+			await db.SaveChangesAsync();
+		}
+
+		public async Task CrawlCategories()
+		{
+			var categoryList = new List<Category>();
+			string htmlData = CrawlDataFromUrl();
+			var categoryListHtml = Regex.Match(htmlData, @"<ul class=""menu-nav(.*?)</ul>", RegexOptions.Singleline).Value;
+			var categoryListItemHtml = Regex.Matches(categoryListHtml, @"<li(.*?)</li>", RegexOptions.Singleline);
+
+			foreach (var category in categoryListItemHtml)
+			{
+				string categoryTitle = Regex.Match(category.ToString(), @"title=""(.*?)""", RegexOptions.Singleline).Value.Replace("title=\"", "").Replace("\"", "");
+				string categoryLink = Regex.Match(category.ToString(), @"href=""(.*?)""", RegexOptions.Singleline).Value.Replace("href=\"", "").Replace("\"", "");
+
+				bool isCategoryExisted = await db.Categories.AnyAsync(item => item.Link.Equals(categoryLink));
+
+				if (!isCategoryExisted)
+				{
+					Category newCategory = new Category
+					{
+						Label = categoryTitle,
+						Link = categoryLink
+					};
+
+					categoryList.Add(newCategory);
+				}
+			}
+			db.Categories.AddRange(categoryList);
+			await db.SaveChangesAsync();
+		}
+	}
+}

Provider/NewsProvider.cs

+using Microsoft.EntityFrameworkCore;
+using News_site.ExtendModels;
+using News_site.Models;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace News_site.Provider
+{
+	public class NewsProvider : BaseProvider
+	{
+		public List<Newspaper> GetAll()
+		{
+			var news = db.Newspapers.Include(news => news.CategorylinkNavigation).ToList();
+
+			return news;
+		}
+
+		public async Task<Newspaper> GetById(Guid Id)
+		{
+			return await db.Newspapers.Where(news => news.Id == Id).SingleOrDefaultAsync();
+		}
+
+		public async Task<List<Newspaper>> GetByCategoryLink(string category)
+		{
+			return await db.Newspapers.Where(news => news.Categorylink.Equals(category)).ToListAsync();
+		}
+	}
+}

Services/Cronbot.cs

+using Microsoft.Extensions.Logging;
+using News_site.Provider;
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace News_site.Services
+{
+	public class Cronbot : ICronbot
+	{
+		private readonly ILogger<Cronbot> _logger;
+		private readonly CrawlDataProvider _dataProvider = new CrawlDataProvider();
+
+		public Cronbot(ILogger<Cronbot> logger)
+		{
+			_logger = logger;
+		}
+
+		public async Task DoCrawlData(CancellationToken cancellationToken)
+		{
+			while (!cancellationToken.IsCancellationRequested)
+			{
+				/* 30 minutes */
+				var duration = 1000 * 60 * 30;
+				var startCrawlingTime = DateTime.Now;
+				_logger.LogInformation($"Start crawling at: {startCrawlingTime:HH:mm}");
+
+				await _dataProvider.CrawlCategories();
+				await _dataProvider.CrawlLatestNews();
+
+				var stopCrawlingTime = DateTime.Now;
+				_logger.LogInformation($"Stop task, crawling in {(stopCrawlingTime - startCrawlingTime).TotalSeconds} seconds");
+
+				await Task.Delay(duration);
+			}
+		}
+	}
+}

Services/CronjobService.cs

+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace News_site.Services
+{
+	public class CronjobService : BackgroundService
+	{
+		private readonly ICronbot _cronbot;
+
+		public CronjobService(ICronbot cronbot)
+		{
+			_cronbot = cronbot;
+		}
+
+		protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+		{
+			await _cronbot.DoCrawlData(stoppingToken);
+		}
+	}
+}

Services/HandleExceptionAsync.cs

+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc.Filters;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace News_site.Services
+{
+	public class HandleExceptionAsync
+	{
+		private readonly string targetDir = @"/home/logs/";
+		private readonly string baseFilename = "log_";
+		private readonly string fileType = ".txt";
+
+		public void WriteError(Exception error)
+		{
+			var todayForFilename = DateTime.Now.ToString("ddMMyyyy");
+			var todayForFolder = DateTime.Now.ToString("dd-MM-yyyy");
+			var currentTime = DateTime.Now.ToString("HH:mm");
+			var filename = baseFilename + todayForFilename + fileType;
+
+			var folderPath = Path.Combine(targetDir, todayForFolder);
+			var filepath = Path.Combine(folderPath, filename);
+
+			if (!Directory.Exists(folderPath))
+			{
+				Directory.CreateDirectory(folderPath);
+			}
+
+			using StreamWriter streamWriter = new StreamWriter(filepath, append: true);
+			string errorMsg = currentTime + " - " + error.Message.ToString();
+
+			streamWriter.WriteLine(errorMsg);
+		}
+	}
+}

Services/ICronbot.cs

+using System.Threading;
+using System.Threading.Tasks;
+
+namespace News_site.Services
+{
+	public interface ICronbot
+	{
+		Task DoCrawlData(CancellationToken cancellationToken);
+	}
+}
\ No newline at end of file

Startup.cs

+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.OpenApi.Models;
+using News_site.Data;
+using News_site.Services;
+using Sieve.Models;
+using Sieve.Services;
+
+namespace News_site
+{
+	public class Startup
+	{
+		readonly string MyAllowSpecificOrigins = "_myAllowSpecificOrigins";
+
+		public Startup(IConfiguration configuration)
+		{
+			Configuration = configuration;
+		}
+
+		public IConfiguration Configuration { get; }
+
+		// This method gets called by the runtime. Use this method to add services to the container.
+		public void ConfigureServices(IServiceCollection services)
+		{
+			services.AddCors(options =>
+			{
+				options.AddPolicy(name: MyAllowSpecificOrigins,
+				policy =>
+				{
+					policy.AllowAnyOrigin();
+					policy.AllowAnyMethod();
+					policy.AllowAnyHeader();
+				});
+			});
+
+			services.Configure<SieveOptions>(Configuration.GetSection("Sieve"));
+
+			services.AddSingleton<ICronbot, Cronbot>();
+			services.AddHostedService<CronjobService>();
+			services.AddScoped<SieveProcessor>();
+			services.AddDbContext<DataContext>();
+
+			services.AddControllers().AddNewtonsoftJson(options => options.SerializerSettings.ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore);
+
+			ConfigureSwagger(services);
+		}
+
+		private static void ConfigureSwagger(IServiceCollection services)
+		{
+			services.AddSwaggerGen(c =>
+			{
+				c.SwaggerDoc("v1", new OpenApiInfo { Title = "My API", Version = "v1" });
+			});
+		}
+
+		// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
+		public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
+		{
+			app.UseSwagger();
+
+			app.UseSwaggerUI(c =>
+			{
+				c.SwaggerEndpoint("/swagger/v1/swagger.json", "My API V1");
+			});
+
+			if (env.IsDevelopment())
+			{
+				app.UseDeveloperExceptionPage();
+			}
+
+			app.UseHttpsRedirection();
+
+			app.UseRouting();
+
+			app.UseCors(MyAllowSpecificOrigins);
+
+			app.UseAuthorization();
+
+			app.UseEndpoints(endpoints =>
+			{
+				endpoints.MapControllers();
+			});
+		}
+	}
+}

appsettings.Development.json

+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft": "Warning",
+      "Microsoft.Hosting.Lifetime": "Information"
+    }
+  }
+}

appsettings.json

+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft": "Warning",
+      "Microsoft.Hosting.Lifetime": "Information"
+    }
+  },
+  "AllowedHosts": "*",
+  "Sieve": {
+    "DefaultPageSize": 10,
+    "MaxPageSize": 20
+  }
+}

build.bat

+type password.txt| docker login -u leekfsolo --password-stdin
+
+docker-compose up -d
+
+docker logout
\ No newline at end of file

build.bat.bak

+echo "test bat file"
\ No newline at end of file

docker-compose.dcproj

+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" Sdk="Microsoft.Docker.Sdk">
+  <PropertyGroup Label="Globals">
+    <ProjectVersion>2.1</ProjectVersion>
+    <DockerTargetOS>Linux</DockerTargetOS>
+    <ProjectGuid>7542a12b-177d-4a8c-9f4f-45d0374a3c44</ProjectGuid>
+    <DockerLaunchAction>LaunchBrowser</DockerLaunchAction>
+    <DockerServiceUrl>{Scheme}://localhost:{ServicePort}/swagger</DockerServiceUrl>
+    <DockerServiceName>newssite</DockerServiceName>
+  </PropertyGroup>
+  <ItemGroup>
+    <None Include="docker-compose.override.yml">
+      <DependentUpon>docker-compose.yml</DependentUpon>
+    </None>
+    <None Include="docker-compose.yml" />
+    <None Include=".dockerignore" />
+  </ItemGroup>
+</Project>
\ No newline at end of file

docker-compose.override.yml

+version: '3.4'
+
+services:
+  newssite:
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Development
+      - ASPNETCORE_URLS=https://+:443;http://+:80
+    ports:
+      - "80"
+      - "443"
+    volumes:
+      - ${APPDATA}/Microsoft/UserSecrets:/root/.microsoft/usersecrets:ro
+      - ${APPDATA}/ASP.NET/Https:/root/.aspnet/https:ro
\ No newline at end of file

docker-compose.yml

+version: '3.4'
+
+networks:
+  dev: 
+    driver: bridge
+
+services:
+  newssite:
+    image: leekfsolo/news_site_be
+    ports:
+      - "5000:80"
+      - "5001:443"
+    container_name: news_sites_be
+    networks:
+      - dev
+    restart: always
+    volumes:
+      - /home/logs:/home/logs
+

efpt.config.json

+{
+   "CodeGenerationMode": 1,
+   "ContextClassName": "DataContext",
+   "ContextNamespace": null,
+   "DefaultDacpacSchema": null,
+   "FilterSchemas": false,
+   "IncludeConnectionString": true,
+   "ModelNamespace": null,
+   "OutputContextPath": "Data",
+   "OutputPath": "Models",
+   "PreserveCasingWithRegex": true,
+   "ProjectRootNamespace": "News_site",
+   "Schemas": null,
+   "SelectedHandlebarsLanguage": 0,
+   "SelectedToBeGenerated": 0,
+   "Tables": [
+      {
+         "Name": "public.category",
+         "ObjectType": 0
+      },
+      {
+         "Name": "public.newspaper",
+         "ObjectType": 0
+      }
+   ],
+   "UiHint": "newspaper_connection",
+   "UseBoolPropertiesWithoutDefaultSql": false,
+   "UseDatabaseNames": false,
+   "UseDbContextSplitting": false,
+   "UseFluentApiOnly": false,
+   "UseHandleBars": false,
+   "UseHierarchyId": false,
+   "UseInflector": true,
+   "UseLegacyPluralizer": false,
+   "UseManyToManyEntity": false,
+   "UseNoConstructor": false,
+   "UseNoDefaultConstructor": false,
+   "UseNoNavigations": false,
+   "UseNoObjectFilter": false,
+   "UseNodaTime": false,
+   "UseNullableReferences": false,
+   "UseSchemaFolders": false,
+   "UseSpatial": false,
+   "UseT4": false
+}
\ No newline at end of file

helpers/ModelFactory.cs

+using System.Reflection;
+using System;
+
+namespace News_site.helpers
+{
+	public class ModelFactory<PClass, CClass>
+		where PClass : class
+		where CClass : class
+	{
+
+		public static CClass CloneObject(CClass a, PClass parent)
+		{
+			try
+			{
+				foreach (PropertyInfo prop in parent.GetType().GetProperties())
+				{
+					var propFound = a.GetType().GetProperty(prop.Name);
+					if (propFound != null)
+					{
+						propFound.SetValue(a, prop.GetValue(parent, null), null);
+					}
+
+				}
+				return a;
+			}
+			catch (Exception ex)
+			{
+				throw ex;
+			}
+
+		}
+	}
+}

password.txt

+dckr_pat_so4nqDK9kEBqY6bK-0FN-y799v4
\ No newline at end of file

run.bat

+type password.txt| docker login -u leekfsolo --password-stdin
+
+docker build -t leekfsolo/news_site_be -f ./Dockerfile .
+
+docker push leekfsolo/news_site_be:latest
+
+docker logout
\ No newline at end of file

run.bat.bak

+echo "test bat file"
\ No newline at end of file