From 518f22d36e2dd42f0f6f23d5aefb837fbc96eb55 Mon Sep 17 00:00:00 2001 From: uduki Date: Sun, 23 Aug 2026 02:32:40 +0900 Subject: [PATCH] =?UTF-8?q?=E3=83=9A=E3=83=BC=E3=82=B8=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=20GetDocument=E3=81=AE=E8=A8=98=E8=BF=B0=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rss-generator/Class/CmnUtil.cs | 63 +++++++++------- rss-generator/Class/LogManager.cs | 2 +- rss-generator/Class/RSS/aurahentai.cs | 102 ++++++++++++++++++++++++++ rss-generator/Program.cs | 2 + rss-generator/rss-generator.csproj | 4 + 5 files changed, 145 insertions(+), 28 deletions(-) create mode 100644 rss-generator/Class/RSS/aurahentai.cs diff --git a/rss-generator/Class/CmnUtil.cs b/rss-generator/Class/CmnUtil.cs index d038fe1..04d307f 100644 --- a/rss-generator/Class/CmnUtil.cs +++ b/rss-generator/Class/CmnUtil.cs @@ -123,42 +123,35 @@ internal class CmnUtil /// /// IDocumentの取得 /// + /// /// /// public static async Task GetDocument(HttpClient client, string pageURL) { - Console.WriteLine($"{pageURL}"); + Console.WriteLine($"Fetching: {pageURL}"); - // 非同期でHTML文字列を丸ごとダウンロード - string htmlContent = await client.GetStringAsync(pageURL); - if (htmlContent.Contains("<") || htmlContent.Contains(">")) + // 1. User-Agent が未設定なら付与(Bot遮断対策) + if (!client.DefaultRequestHeaders.UserAgent.Any()) { - htmlContent = WebUtility.HtmlDecode(htmlContent); + client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); } - var parser = new HtmlParser(); - IDocument document = await parser.ParseDocumentAsync(htmlContent); + // 2. HTMLのダウンロード + string htmlContent = await client.GetStringAsync(pageURL); + + if (string.IsNullOrWhiteSpace(htmlContent)) + { + Console.WriteLine("Warning: 取得したHTMLが空です。"); + } + + // 3. AngleSharpのコンテキスト生成(通信なし・最速設定) + var config = Configuration.Default; + var context = BrowsingContext.New(config); + + // 4. アドレス(pageURL)を紐付けてパースすることで about:blank を防ぐ + IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL)); + return document; - - // // ダウンロードしたHTMLを AngleSharp に流し込んで解析する - // // 通信はHttpClient側で終わっているため、AngleSharp側は「通信機能なし」の最速設定でOK - // var config = Configuration.Default; - // var context = BrowsingContext.New(config); - // - // // 文字列のHTMLをAngleSharpにパースさせます(URLを紐付けることで、相対パスを自動解決できます) - // return await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL)); - - - // Console.WriteLine($"{pageURL}"); - // - // // 1. AngleSharpの設定(Webからデータをダウンロードするローダーを有効にする) - // var config = Configuration.Default.WithDefaultLoader(); - // var context = BrowsingContext.New(config); - // - // // 2. URLを指定して直接 IDocument を取得(内部で自動的にRequesterが動きます) - // IDocument document = await context.OpenAsync(pageURL); - // - // return document; } public static async Task> GetFeedItems(HttpClient client, string url) @@ -466,4 +459,20 @@ internal class CmnUtil DateTime nextHour = now.AddHours(1); return new DateTime(nextHour.Year, nextHour.Month, nextHour.Day, nextHour.Hour, sortedMinutes[0], 0, 0, now.Kind); } + + /// + /// 相対URLの場合、絶対URLに解決する + /// + /// + /// + /// + public static Uri? GetAbsoluteUri(Uri? baseUri, string? relUri) + { + // 相対URLの場合、絶対URLに解決する + if (Uri.TryCreate(baseUri, relUri, out var absoluteUri)) + { + return absoluteUri; + } + return null; + } } diff --git a/rss-generator/Class/LogManager.cs b/rss-generator/Class/LogManager.cs index 4247910..9e3c4e5 100644 --- a/rss-generator/Class/LogManager.cs +++ b/rss-generator/Class/LogManager.cs @@ -40,7 +40,7 @@ internal static class LogManager // 実際のクラス名に合わせてくだ var log = new LogMessage { - Message = message.TrimEnd('\r', '\n'), + Message = message!.TrimEnd('\r', '\n'), Timestamp = DateTime.Now // ログが発生した瞬間の時間を保持 }; diff --git a/rss-generator/Class/RSS/aurahentai.cs b/rss-generator/Class/RSS/aurahentai.cs new file mode 100644 index 0000000..3f00474 --- /dev/null +++ b/rss-generator/Class/RSS/aurahentai.cs @@ -0,0 +1,102 @@ +using AngleSharp.Dom; +using AngleSharp.Html.Dom; +using System; +using System.Collections.Generic; +using System.Reflection.Metadata; +using System.ServiceModel.Syndication; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml.Linq; +using WebDriverBiDi.Script; + +namespace rss_generator.Class.RSS; + +internal class aurahentai +{ + public static async Task CreateRSSAsync(HttpClient client) + { + var baseURL = "https://aurahentai.com/language/japanese"; + var query = ""; + + var feed = new SyndicationFeed("aurahentai.com", "", new Uri($"{baseURL}{query}")); + var items = new List(); + try + { + // 複数ページをまとめる場合に先に取得する + var docList = new List(); + for (int pageId = 1; pageId <= 5; pageId++) + { + query = $"{pageId}"; + var pageURL = $"{baseURL}?page={query}"; + var document = await CmnUtil.GetDocument(client, pageURL); + var newItems = await CreateFeedItem(client, document); + foreach (var newItem in newItems) + { + items.Add(newItem); + } + } + feed.Items = items; + + CmnUtil.ExportXMLFile("aurahentai.xml", feed); + } + catch (Exception ex) + { + LogManager.AddLog($"[エラー] aurahentai:{ex.Message}"); + } + } + + private static async Task> CreateFeedItem(HttpClient client, IDocument docListPage) + { + var items = new List(); + var elements = docListPage.QuerySelectorAll("main article>a"); + var linkList = new List(); + foreach (var linkElement in elements) + { + var item = new SyndicationItem(); + + var relLink = linkElement.GetAttribute("href") ?? string.Empty; + var contentsUri = CmnUtil.GetAbsoluteUri(docListPage.BaseUrl!, relLink)!; + var linkHref = contentsUri.AbsoluteUri; + + var docContents = await CmnUtil.GetDocument(client, linkHref); + var divDetail = docContents.QuerySelector("main>div:nth-of-type(2)")!; + + var title = ""; + { + if (divDetail.QuerySelector("p:nth-of-type(1)") is IHtmlElement element) + { + title = $"{element.TextContent.Trim()}"; + item.Title = new TextSyndicationContent(title); + } + } + { + if (title == "" && divDetail.QuerySelector("h1") is IHtmlElement element) + { + title = $"{element.TextContent.Trim()}"; + item.Title = new TextSyndicationContent(title); + } + } + var imgUrl = ""; + { + var replaceFr = "https://aurahentai.com/gallery/"; + var replaceTo = "https://img.aurahentai.com/g/"; + imgUrl = $"{linkHref.Replace(replaceFr, replaceTo)}/001.webp"; + } + + var body = new StringBuilder(); + { + body.Append($@""); + body.Append($@"
"); + //body.Append(CmnUtil.GetHtmlWithoutJavaScript(divDetail)); + } + + item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); + // ID + item.Id = CmnUtil.CreateGuid($"{linkHref}"); + + items.Add(item); + } + + return items; + } +} diff --git a/rss-generator/Program.cs b/rss-generator/Program.cs index 193cf9b..87323cb 100644 --- a/rss-generator/Program.cs +++ b/rss-generator/Program.cs @@ -65,7 +65,9 @@ while (true) tasks.Add(missav123.CreateRSSAsync(client)); tasks.Add(npb_6.CreateRSSAsync(client)); tasks.Add(ci_en.CreateRSSAsync(client)); + //tasks.Add(index_japanese.CreateRSSAsync(client)); + tasks.Add(aurahentai.CreateRSSAsync(client)); await Task.WhenAll(tasks); diff --git a/rss-generator/rss-generator.csproj b/rss-generator/rss-generator.csproj index f9b0e0f..933b076 100644 --- a/rss-generator/rss-generator.csproj +++ b/rss-generator/rss-generator.csproj @@ -20,6 +20,10 @@ + + + +