ページ追加

GetDocumentの記述変更
This commit is contained in:
armonica committed 2026-08-23 02:32:40 +09:00
1 parent 74427c8861
commit 518f22d36e
5 files changed
+145 -28

No files matched your search

+36 -27
View File
@@ -123,42 +123,35 @@ internal class CmnUtil
/// <summary> /// <summary>
/// IDocumentの取得 /// IDocumentの取得
/// </summary> /// </summary>
/// <param name="client"></param>
/// <param name="pageURL"></param> /// <param name="pageURL"></param>
/// <returns></returns> /// <returns></returns>
public static async Task<IDocument> GetDocument(HttpClient client, string pageURL) public static async Task<IDocument> GetDocument(HttpClient client, string pageURL)
{ {
Console.WriteLine($"{pageURL}"); Console.WriteLine($"Fetching: {pageURL}");
// 非同期でHTML文字列を丸ごとダウンロード // 1. User-Agent が未設定なら付与(Bot遮断対策)
string htmlContent = await client.GetStringAsync(pageURL); if (!client.DefaultRequestHeaders.UserAgent.Any())
if (htmlContent.Contains("&lt;") || htmlContent.Contains("&gt;"))
{ {
htmlContent = WebUtility.HtmlDecode(htmlContent); client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
} }
var parser = new HtmlParser(); // 2. HTMLのダウンロード
IDocument document = await parser.ParseDocumentAsync(htmlContent); string htmlContent = await client.GetStringAsync(pageURL);
if (string.IsNullOrWhiteSpace(htmlContent))
{
Console.WriteLine("Warning: 取得したHTMLが空です。");
}
// 3. AngleSharpのコンテキスト生成(通信なし・最速設定)
var config = Configuration.Default;
var context = BrowsingContext.New(config);
// 4. アドレス(pageURL)を紐付けてパースすることで about:blank を防ぐ
IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL));
return document; return document;
// // ダウンロードしたHTMLを AngleSharp に流し込んで解析する
// // 通信はHttpClient側で終わっているため、AngleSharp側は「通信機能なし」の最速設定でOK
// var config = Configuration.Default;
// var context = BrowsingContext.New(config);
//
// // 文字列のHTMLをAngleSharpにパースさせます(URLを紐付けることで、相対パスを自動解決できます)
// return await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL));
// Console.WriteLine($"{pageURL}");
//
// // 1. AngleSharpの設定(Webからデータをダウンロードするローダーを有効にする)
// var config = Configuration.Default.WithDefaultLoader();
// var context = BrowsingContext.New(config);
//
// // 2. URLを指定して直接 IDocument を取得(内部で自動的にRequesterが動きます)
// IDocument document = await context.OpenAsync(pageURL);
//
// return document;
} }
public static async Task<IEnumerable<SyndicationItem>> GetFeedItems(HttpClient client, string url) public static async Task<IEnumerable<SyndicationItem>> GetFeedItems(HttpClient client, string url)
@@ -466,4 +459,20 @@ internal class CmnUtil
DateTime nextHour = now.AddHours(1); DateTime nextHour = now.AddHours(1);
return new DateTime(nextHour.Year, nextHour.Month, nextHour.Day, nextHour.Hour, sortedMinutes[0], 0, 0, now.Kind); return new DateTime(nextHour.Year, nextHour.Month, nextHour.Day, nextHour.Hour, sortedMinutes[0], 0, 0, now.Kind);
} }
/// <summary>
/// 相対URLの場合、絶対URLに解決する
/// </summary>
/// <param name="baseUri"></param>
/// <param name="relUri"></param>
/// <returns></returns>
public static Uri? GetAbsoluteUri(Uri? baseUri, string? relUri)
{
// 相対URLの場合、絶対URLに解決する
if (Uri.TryCreate(baseUri, relUri, out var absoluteUri))
{
return absoluteUri;
}
return null;
}
} }
+1 -1
View File
@@ -40,7 +40,7 @@ internal static class LogManager // 実際のクラス名に合わせてくだ
var log = new LogMessage var log = new LogMessage
{ {
Message = message.TrimEnd('\r', '\n'), Message = message!.TrimEnd('\r', '\n'),
Timestamp = DateTime.Now // ログが発生した瞬間の時間を保持 Timestamp = DateTime.Now // ログが発生した瞬間の時間を保持
}; };
+102
View File
@@ -0,0 +1,102 @@
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using System;
using System.Collections.Generic;
using System.Reflection.Metadata;
using System.ServiceModel.Syndication;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using WebDriverBiDi.Script;
namespace rss_generator.Class.RSS;
internal class aurahentai
{
public static async Task CreateRSSAsync(HttpClient client)
{
var baseURL = "https://aurahentai.com/language/japanese";
var query = "";
var feed = new SyndicationFeed("aurahentai.com", "", new Uri($"{baseURL}{query}"));
var items = new List<SyndicationItem>();
try
{
// 複数ページをまとめる場合に先に取得する
var docList = new List<IDocument>();
for (int pageId = 1; pageId <= 5; pageId++)
{
query = $"{pageId}";
var pageURL = $"{baseURL}?page={query}";
var document = await CmnUtil.GetDocument(client, pageURL);
var newItems = await CreateFeedItem(client, document);
foreach (var newItem in newItems)
{
items.Add(newItem);
}
}
feed.Items = items;
CmnUtil.ExportXMLFile("aurahentai.xml", feed);
}
catch (Exception ex)
{
LogManager.AddLog($"[エラー] aurahentai:{ex.Message}");
}
}
private static async Task<List<SyndicationItem>> CreateFeedItem(HttpClient client, IDocument docListPage)
{
var items = new List<SyndicationItem>();
var elements = docListPage.QuerySelectorAll("main article>a");
var linkList = new List<string>();
foreach (var linkElement in elements)
{
var item = new SyndicationItem();
var relLink = linkElement.GetAttribute("href") ?? string.Empty;
var contentsUri = CmnUtil.GetAbsoluteUri(docListPage.BaseUrl!, relLink)!;
var linkHref = contentsUri.AbsoluteUri;
var docContents = await CmnUtil.GetDocument(client, linkHref);
var divDetail = docContents.QuerySelector("main>div:nth-of-type(2)")!;
var title = "";
{
if (divDetail.QuerySelector("p:nth-of-type(1)") is IHtmlElement element)
{
title = $"{element.TextContent.Trim()}";
item.Title = new TextSyndicationContent(title);
}
}
{
if (title == "" && divDetail.QuerySelector("h1") is IHtmlElement element)
{
title = $"{element.TextContent.Trim()}";
item.Title = new TextSyndicationContent(title);
}
}
var imgUrl = "";
{
var replaceFr = "https://aurahentai.com/gallery/";
var replaceTo = "https://img.aurahentai.com/g/";
imgUrl = $"{linkHref.Replace(replaceFr, replaceTo)}/001.webp";
}
var body = new StringBuilder();
{
body.Append($@"<div><a href=""{linkHref}"">{title}</a></div>");
body.Append($@"<div><img src=""{imgUrl}"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
//body.Append(CmnUtil.GetHtmlWithoutJavaScript(divDetail));
}
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
// ID
item.Id = CmnUtil.CreateGuid($"{linkHref}");
items.Add(item);
}
return items;
}
}
+2
View File
@@ -65,7 +65,9 @@ while (true)
tasks.Add(missav123.CreateRSSAsync(client)); tasks.Add(missav123.CreateRSSAsync(client));
tasks.Add(npb_6.CreateRSSAsync(client)); tasks.Add(npb_6.CreateRSSAsync(client));
tasks.Add(ci_en.CreateRSSAsync(client)); tasks.Add(ci_en.CreateRSSAsync(client));
//tasks.Add(index_japanese.CreateRSSAsync(client)); //tasks.Add(index_japanese.CreateRSSAsync(client));
tasks.Add(aurahentai.CreateRSSAsync(client));
await Task.WhenAll(tasks); await Task.WhenAll(tasks);
+4
View File
@@ -20,6 +20,10 @@
<PackageReference Include="System.ServiceModel.Syndication" Version="10.0.11" /> <PackageReference Include="System.ServiceModel.Syndication" Version="10.0.11" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<Folder Include="Class\RSS\Pixiv\" />
</ItemGroup>
<Target Name="PostPublishDeploy" AfterTargets="Publish"> <Target Name="PostPublishDeploy" AfterTargets="Publish">
<Message Text="=== [VPS Deploy] Starting Deploy via MSBuild ===" Importance="high" /> <Message Text="=== [VPS Deploy] Starting Deploy via MSBuild ===" Importance="high" />
<Exec Command="ssh -p 22321 armonica@162.43.79.113 &quot;mkdir -p /home/armonica/docker/rss-generator/app&quot;" /> <Exec Command="ssh -p 22321 armonica@162.43.79.113 &quot;mkdir -p /home/armonica/docker/rss-generator/app&quot;" />