ページ追加
GetDocumentの記述変更
This commit is contained in:
1 parent
74427c8861
commit
518f22d36e
5 files changed
+145
-28
No files matched your search
@@ -123,42 +123,35 @@ internal class CmnUtil
|
||||
/// <summary>
|
||||
/// IDocumentの取得
|
||||
/// </summary>
|
||||
/// <param name="client"></param>
|
||||
/// <param name="pageURL"></param>
|
||||
/// <returns></returns>
|
||||
public static async Task<IDocument> GetDocument(HttpClient client, string pageURL)
|
||||
{
|
||||
Console.WriteLine($"{pageURL}");
|
||||
Console.WriteLine($"Fetching: {pageURL}");
|
||||
|
||||
// 非同期でHTML文字列を丸ごとダウンロード
|
||||
string htmlContent = await client.GetStringAsync(pageURL);
|
||||
if (htmlContent.Contains("<") || htmlContent.Contains(">"))
|
||||
// 1. User-Agent が未設定なら付与(Bot遮断対策)
|
||||
if (!client.DefaultRequestHeaders.UserAgent.Any())
|
||||
{
|
||||
htmlContent = WebUtility.HtmlDecode(htmlContent);
|
||||
client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
|
||||
}
|
||||
|
||||
var parser = new HtmlParser();
|
||||
IDocument document = await parser.ParseDocumentAsync(htmlContent);
|
||||
// 2. HTMLのダウンロード
|
||||
string htmlContent = await client.GetStringAsync(pageURL);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(htmlContent))
|
||||
{
|
||||
Console.WriteLine("Warning: 取得したHTMLが空です。");
|
||||
}
|
||||
|
||||
// 3. AngleSharpのコンテキスト生成(通信なし・最速設定)
|
||||
var config = Configuration.Default;
|
||||
var context = BrowsingContext.New(config);
|
||||
|
||||
// 4. アドレス(pageURL)を紐付けてパースすることで about:blank を防ぐ
|
||||
IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL));
|
||||
|
||||
return document;
|
||||
|
||||
// // ダウンロードしたHTMLを AngleSharp に流し込んで解析する
|
||||
// // 通信はHttpClient側で終わっているため、AngleSharp側は「通信機能なし」の最速設定でOK
|
||||
// var config = Configuration.Default;
|
||||
// var context = BrowsingContext.New(config);
|
||||
//
|
||||
// // 文字列のHTMLをAngleSharpにパースさせます(URLを紐付けることで、相対パスを自動解決できます)
|
||||
// return await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL));
|
||||
|
||||
|
||||
// Console.WriteLine($"{pageURL}");
|
||||
//
|
||||
// // 1. AngleSharpの設定(Webからデータをダウンロードするローダーを有効にする)
|
||||
// var config = Configuration.Default.WithDefaultLoader();
|
||||
// var context = BrowsingContext.New(config);
|
||||
//
|
||||
// // 2. URLを指定して直接 IDocument を取得(内部で自動的にRequesterが動きます)
|
||||
// IDocument document = await context.OpenAsync(pageURL);
|
||||
//
|
||||
// return document;
|
||||
}
|
||||
|
||||
public static async Task<IEnumerable<SyndicationItem>> GetFeedItems(HttpClient client, string url)
|
||||
@@ -466,4 +459,20 @@ internal class CmnUtil
|
||||
DateTime nextHour = now.AddHours(1);
|
||||
return new DateTime(nextHour.Year, nextHour.Month, nextHour.Day, nextHour.Hour, sortedMinutes[0], 0, 0, now.Kind);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 相対URLの場合、絶対URLに解決する
|
||||
/// </summary>
|
||||
/// <param name="baseUri"></param>
|
||||
/// <param name="relUri"></param>
|
||||
/// <returns></returns>
|
||||
public static Uri? GetAbsoluteUri(Uri? baseUri, string? relUri)
|
||||
{
|
||||
// 相対URLの場合、絶対URLに解決する
|
||||
if (Uri.TryCreate(baseUri, relUri, out var absoluteUri))
|
||||
{
|
||||
return absoluteUri;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -40,7 +40,7 @@ internal static class LogManager // 実際のクラス名に合わせてくだ
|
||||
|
||||
var log = new LogMessage
|
||||
{
|
||||
Message = message.TrimEnd('\r', '\n'),
|
||||
Message = message!.TrimEnd('\r', '\n'),
|
||||
Timestamp = DateTime.Now // ログが発生した瞬間の時間を保持
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Html.Dom;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Reflection.Metadata;
|
||||
using System.ServiceModel.Syndication;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Xml.Linq;
|
||||
using WebDriverBiDi.Script;
|
||||
|
||||
namespace rss_generator.Class.RSS;
|
||||
|
||||
internal class aurahentai
|
||||
{
|
||||
public static async Task CreateRSSAsync(HttpClient client)
|
||||
{
|
||||
var baseURL = "https://aurahentai.com/language/japanese";
|
||||
var query = "";
|
||||
|
||||
var feed = new SyndicationFeed("aurahentai.com", "", new Uri($"{baseURL}{query}"));
|
||||
var items = new List<SyndicationItem>();
|
||||
try
|
||||
{
|
||||
// 複数ページをまとめる場合に先に取得する
|
||||
var docList = new List<IDocument>();
|
||||
for (int pageId = 1; pageId <= 5; pageId++)
|
||||
{
|
||||
query = $"{pageId}";
|
||||
var pageURL = $"{baseURL}?page={query}";
|
||||
var document = await CmnUtil.GetDocument(client, pageURL);
|
||||
var newItems = await CreateFeedItem(client, document);
|
||||
foreach (var newItem in newItems)
|
||||
{
|
||||
items.Add(newItem);
|
||||
}
|
||||
}
|
||||
feed.Items = items;
|
||||
|
||||
CmnUtil.ExportXMLFile("aurahentai.xml", feed);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogManager.AddLog($"[エラー] aurahentai:{ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<List<SyndicationItem>> CreateFeedItem(HttpClient client, IDocument docListPage)
|
||||
{
|
||||
var items = new List<SyndicationItem>();
|
||||
var elements = docListPage.QuerySelectorAll("main article>a");
|
||||
var linkList = new List<string>();
|
||||
foreach (var linkElement in elements)
|
||||
{
|
||||
var item = new SyndicationItem();
|
||||
|
||||
var relLink = linkElement.GetAttribute("href") ?? string.Empty;
|
||||
var contentsUri = CmnUtil.GetAbsoluteUri(docListPage.BaseUrl!, relLink)!;
|
||||
var linkHref = contentsUri.AbsoluteUri;
|
||||
|
||||
var docContents = await CmnUtil.GetDocument(client, linkHref);
|
||||
var divDetail = docContents.QuerySelector("main>div:nth-of-type(2)")!;
|
||||
|
||||
var title = "";
|
||||
{
|
||||
if (divDetail.QuerySelector("p:nth-of-type(1)") is IHtmlElement element)
|
||||
{
|
||||
title = $"{element.TextContent.Trim()}";
|
||||
item.Title = new TextSyndicationContent(title);
|
||||
}
|
||||
}
|
||||
{
|
||||
if (title == "" && divDetail.QuerySelector("h1") is IHtmlElement element)
|
||||
{
|
||||
title = $"{element.TextContent.Trim()}";
|
||||
item.Title = new TextSyndicationContent(title);
|
||||
}
|
||||
}
|
||||
var imgUrl = "";
|
||||
{
|
||||
var replaceFr = "https://aurahentai.com/gallery/";
|
||||
var replaceTo = "https://img.aurahentai.com/g/";
|
||||
imgUrl = $"{linkHref.Replace(replaceFr, replaceTo)}/001.webp";
|
||||
}
|
||||
|
||||
var body = new StringBuilder();
|
||||
{
|
||||
body.Append($@"<div><a href=""{linkHref}"">{title}</a></div>");
|
||||
body.Append($@"<div><img src=""{imgUrl}"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
|
||||
//body.Append(CmnUtil.GetHtmlWithoutJavaScript(divDetail));
|
||||
}
|
||||
|
||||
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
|
||||
// ID
|
||||
item.Id = CmnUtil.CreateGuid($"{linkHref}");
|
||||
|
||||
items.Add(item);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
}
|
||||
@@ -65,7 +65,9 @@ while (true)
|
||||
tasks.Add(missav123.CreateRSSAsync(client));
|
||||
tasks.Add(npb_6.CreateRSSAsync(client));
|
||||
tasks.Add(ci_en.CreateRSSAsync(client));
|
||||
|
||||
//tasks.Add(index_japanese.CreateRSSAsync(client));
|
||||
tasks.Add(aurahentai.CreateRSSAsync(client));
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
|
||||
@@ -20,6 +20,10 @@
|
||||
<PackageReference Include="System.ServiceModel.Syndication" Version="10.0.11" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="Class\RSS\Pixiv\" />
|
||||
</ItemGroup>
|
||||
|
||||
<Target Name="PostPublishDeploy" AfterTargets="Publish">
|
||||
<Message Text="=== [VPS Deploy] Starting Deploy via MSBuild ===" Importance="high" />
|
||||
<Exec Command="ssh -p 22321 armonica@162.43.79.113 "mkdir -p /home/armonica/docker/rss-generator/app"" />
|
||||
|
||||
Reference in new issue
Block a user