ローカルのdocker用設定など

This commit is contained in:
armonica committed 2026-08-28 03:09:55 +09:00
1 parent 4a93843d3f
commit b01665423e
10 files changed
+558 -123

No files matched your search

+330 -33
View File
@@ -1,4 +1,4 @@
using AngleSharp; using AngleSharp;
using AngleSharp.Dom; using AngleSharp.Dom;
using AngleSharp.Html.Parser; using AngleSharp.Html.Parser;
using PuppeteerSharp; using PuppeteerSharp;
@@ -7,15 +7,18 @@ using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Net; using System.Net;
using System.Net.Http.Headers; using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.ServiceModel.Syndication; using System.ServiceModel.Syndication;
using System.Text; using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Xml; using System.Xml;
namespace rss_generator.Class; namespace rss_generator.Class;
internal class CmnUtil public class CmnUtil
{ {
/// <summary> /// <summary>
/// XMLファイルの出力先パスを取得 /// XMLファイルの出力先パスを取得
@@ -36,17 +39,72 @@ internal class CmnUtil
return filePath; return filePath;
} }
public static void AddLog(string? message) /// <summary>
/// 例外発生箇所からポストバックのイベントハンドラまでの呼び出し履歴を出力します
/// </summary>
public static void PrintCallStackToEvent(Exception ex)
{ {
var filrDir = Path.Combine(GetExportXmlPath(""), "Log"); #if DEBUG
if (Directory.Exists(filrDir) == false) // 1. 現在のスレッドの実行スタック全体を取得 (PDB情報を含む)
StackTrace st = new StackTrace(true);
StackFrame[] frames = st.GetFrames();
if (frames == null || frames.Length == 0) return;
Debug.Print("================================================================================");
Debug.Print(string.Format("[例外種別] {0}", ex != null ? ex.GetType().FullName : "Unknown Exception"));
Debug.Print(string.Format("[エラー内容] {0}", ex != null ? ex.Message.Trim() : "なし"));
Debug.Print("--------------------------------------------------------------------------------");
Debug.Print("【呼び出し経路】");
List<StackFrame> userFrames = new List<StackFrame>();
foreach (StackFrame frame in frames)
{ {
Directory.CreateDirectory(filrDir); var method = frame.GetMethod();
if (method == null) continue;
var declaringType = method.DeclaringType;
var typeFullName = (declaringType != null ? declaringType.FullName : "") ?? "";
// ASP.NET 内部のパイプラインに到達したら、それより上位(IIS内部)は除外して打ち切り
if (typeFullName.StartsWith("System.Web.") || typeFullName.StartsWith("System.Web.UI."))
{
break;
} }
var fileName = $"{DateTime.Now:yyyy-MM-dd}.log";
var filePath = Path.Combine(filrDir, fileName); userFrames.Add(frame);
using var sw = File.AppendText(filePath); }
sw.WriteLine(message);
// 時系列順に並び替える
userFrames.Reverse();
for (int i = 0; i < userFrames.Count; i++)
{
StackFrame frame = userFrames[i];
var method = frame.GetMethod();
var declaringType = method != null ? method.DeclaringType : null;
string className = declaringType != null ? declaringType.Name : "Unknown";
string methodName = method != null ? method.Name : "";
if (methodName == "PrintCallStackToEvent")
{
continue;
}
var fileName = frame.GetFileName();
int lineNumber = frame.GetFileLineNumber();
string fileInfo = "";
if (!string.IsNullOrEmpty(fileName) && lineNumber > 0)
{
fileInfo = string.Format(" ({0} : {1} 行目)", Path.GetFileName(fileName), lineNumber);
}
Debug.Print($"-> {className}.{methodName}{fileInfo}");
}
Debug.Print("================================================================================");
#endif
} }
/// <summary> /// <summary>
@@ -103,6 +161,7 @@ internal class CmnUtil
{ {
// 404の場合などのログをここに書くことも可能 // 404の場合などのログをここに書くことも可能
//AddLog($"[HTTP Error] Status Code: {response.StatusCode} for {url}"); //AddLog($"[HTTP Error] Status Code: {response.StatusCode} for {url}");
LogManager.AddLog($"[HTTP Error] Status Code: {response.StatusCode} for {url}");
return null; return null;
} }
@@ -112,7 +171,7 @@ internal class CmnUtil
catch (HttpRequestException ex) catch (HttpRequestException ex)
{ {
// ネットワーク不通やタイムアウトなどの例外処理 // ネットワーク不通やタイムアウトなどの例外処理
AddLog($"[Network Error] {ex.Message}"); LogManager.AddLog($"[Network Error] {ex.Message}");
return null; return null;
} }
catch (Exception) catch (Exception)
@@ -122,25 +181,33 @@ internal class CmnUtil
} }
/// <summary> /// <summary>
/// IDocumentの取得 /// 直接HTTP通信により対象URLのHTMLを取得し、AngleSharpのIDocumentを生成して返します。
/// ストリーム直接パース、並列アクセスセーフ、メモリリーク防止、高速化を施しています。
/// </summary> /// </summary>
/// <param name="client"></param> /// <param name="client">HttpClient インスタンス</param>
/// <param name="pageURL"></param> /// <param name="pageURL">取得対象のWebページURL</param>
/// <returns></returns> /// <param name="useCacheBuster">キャッシュ回避用パラメータ(_tutc)を付与するかどうか(デフォルト: true)</param>
public static async Task<IDocument> GetDocument(HttpClient client, string pageURL) /// <param name="cancellationToken">キャンセレーショントークン</param>
/// <returns>AngleSharpのIDocument</returns>
public static async Task<IDocument> GetDocument(
HttpClient client,
string pageURL,
bool useCacheBuster = true,
CancellationToken cancellationToken = default)
{ {
Console.WriteLine($"Fetching: {pageURL}"); Console.WriteLine($"Fetching: {pageURL}");
// 1. User-Agent が未設定なら付与 // 1. リクエストURLの構築(キャッシュ回避パラメータ付与)
if (!client.DefaultRequestHeaders.UserAgent.Any()) string requestUrl = pageURL;
if (useCacheBuster)
{ {
client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); string separator = pageURL.Contains('?') ? "&" : "?";
requestUrl = $"{pageURL}{separator}_tutc={DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
} }
// 2. キャッシュ無効化ヘッダーを付与したリクエストを生成 using var request = new HttpRequestMessage(HttpMethod.Get, requestUrl);
string separator = pageURL.Contains('?') ? "&" : "?";
string cacheBusterUrl = $"{pageURL}{separator}_tutc={DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}"; // 2. キャッシュ制御ヘッダー
using var request = new HttpRequestMessage(HttpMethod.Get, cacheBusterUrl);
request.Headers.CacheControl = new CacheControlHeaderValue request.Headers.CacheControl = new CacheControlHeaderValue
{ {
NoCache = true, NoCache = true,
@@ -149,26 +216,256 @@ internal class CmnUtil
}; };
request.Headers.Pragma.ParseAdd("no-cache"); request.Headers.Pragma.ParseAdd("no-cache");
// 3. HTMLのダウンロード // 3. User-Agent(クライアント側に未設定の場合のみリクエスト単位で設定し、共有clientの競合を防止)
using var response = await client.SendAsync(request); if (!client.DefaultRequestHeaders.UserAgent.Any() && !request.Headers.UserAgent.Any())
response.EnsureSuccessStatusCode();
string htmlContent = await response.Content.ReadAsStringAsync();
if (string.IsNullOrWhiteSpace(htmlContent))
{ {
Console.WriteLine("Warning: 取得したHTMLが空です。"); request.Headers.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
} }
// 4. AngleSharpのコンテキスト生成(メモリ内パース) // 4. Accept ヘッダー
if (!client.DefaultRequestHeaders.Accept.Any() && !request.Headers.Accept.Any())
{
request.Headers.Accept.ParseAdd("text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
}
// 5. ストリームによるレスポンス受信と直接パース(ResponseHeadersRead で中間バッファリングと巨大文字列生成を排除)
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
// Content-Type (charset 等) を取得して AngleSharp に通知
var contentType = response.Content.Headers.ContentType?.ToString();
// 6. AngleSharp の軽量コンテキスト生成とストリーム直接パース
var config = Configuration.Default; var config = Configuration.Default;
var context = BrowsingContext.New(config); var context = BrowsingContext.New(config);
// 5. アドレス(pageURL)を紐付けてパース IDocument document = await context.OpenAsync(req =>
{
req.Content(stream);
req.Address(pageURL); // 基準URLは元のpageURLに設定
if (!string.IsNullOrEmpty(contentType))
{
req.Header("Content-Type", contentType);
}
}, cancellationToken);
return document;
}
#region FlareSolverr 連携
/// <summary>
/// FlareSolverr 用の JSON シリアライズオプション(静的インスタンスとしてキャッシュしメモリ浪費を防止)
/// </summary>
private static readonly JsonSerializerOptions FlareSolverrJsonOptions = new()
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
/// <summary>
/// FlareSolverr 経由で対象 URL の HTML を取得し、AngleSharp の IDocument を生成して返します。
/// メモリ割り当ての最小化およびストリーム直接デシリアライズにより、メモリリーク防止と高速化を行っています。
/// </summary>
/// <param name="client">HttpClient インスタンス</param>
/// <param name="pageURL">取得対象のWebページURL</param>
/// <param name="flareSolverrEndpoint">FlareSolverrのエンドポイントURL(デフォルト: https://udukipc-flaresolverr.armonica-vps.work/v1)</param>
/// <param name="maxTimeout">FlareSolverr側の最大タイムアウトミリ秒(デフォルト: 60000ms)</param>
/// <param name="session">セッションID(セッションを再利用して高速化する場合に指定)</param>
/// <returns>AngleSharpのIDocument</returns>
public static async Task<IDocument> GetDocumentFlareSolverr(HttpClient client, string pageURL, string flareSolverrEndpoint = "https://udukipc-flaresolverr.armonica-vps.work/v1", int maxTimeout = 60000, string? session = null)
{
Console.WriteLine($"Fetching via FlareSolverr: {pageURL}");
var payload = new FlareSolverrRequest
{
Cmd = "request.get",
Url = pageURL,
MaxTimeout = maxTimeout,
Session = session
};
using var request = new HttpRequestMessage(HttpMethod.Post, flareSolverrEndpoint)
{
Content = JsonContent.Create(payload, options: FlareSolverrJsonOptions)
};
// レスポンスヘッダー受信時点で制御を戻し、ストリームから直接デシリアライズしてメモリ消費を抑制
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
try
{
response.EnsureSuccessStatusCode();
}
catch (Exception ex)
{
LogManager.AddLog($"[エラー] FlareSolverr {pageURL} {ex.Message}");
PrintCallStackToEvent(ex);
throw;
}
using var responseStream = await response.Content.ReadAsStreamAsync();
var flareResponse = await JsonSerializer.DeserializeAsync<FlareSolverrResponse>(responseStream, FlareSolverrJsonOptions);
if (flareResponse == null)
{
throw new InvalidOperationException($"FlareSolverr からのレスポンスのデシリアライズに失敗しました: {pageURL}");
}
if (!string.Equals(flareResponse.Status, "ok", StringComparison.OrdinalIgnoreCase))
{
string errorMsg = $"FlareSolverr エラー (Status: {flareResponse.Status}, Message: {flareResponse.Message}): {pageURL}";
LogManager.AddLog($"[エラー] {errorMsg}");
throw new InvalidOperationException(errorMsg);
}
string htmlContent = flareResponse.Solution?.Response ?? string.Empty;
if (string.IsNullOrWhiteSpace(htmlContent))
{
Console.WriteLine($"Warning: FlareSolverr から取得した HTML が空です ({pageURL})");
}
// AngleSharp の軽量コンテキスト生成とメモリ内パース
var config = Configuration.Default;
var context = BrowsingContext.New(config);
IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL)); IDocument document = await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL));
return document; return document;
} }
/// <summary>
/// FlareSolverr の新規セッションを作成します。セッションを再利用することで2回目以降のリクエストが大幅に高速化します。
/// </summary>
/// <param name="client">HttpClient インスタンス</param>
/// <param name="session">セッションID(省略時はFlareSolverr側で自動生成)</param>
/// <param name="flareSolverrEndpoint">FlareSolverrのエンドポイントURL</param>
/// <returns>作成されたセッションID</returns>
public static async Task<string?> CreateFlareSolverrSession(HttpClient client, string? session = null, string flareSolverrEndpoint = "https://udukipc-flaresolverr.armonica-vps.work/v1")
{
var payload = new FlareSolverrRequest
{
Cmd = "sessions.create",
Session = session
};
using var request = new HttpRequestMessage(HttpMethod.Post, flareSolverrEndpoint)
{
Content = JsonContent.Create(payload, options: FlareSolverrJsonOptions)
};
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
response.EnsureSuccessStatusCode();
using var responseStream = await response.Content.ReadAsStreamAsync();
var flareResponse = await JsonSerializer.DeserializeAsync<FlareSolverrResponse>(responseStream, FlareSolverrJsonOptions);
return flareResponse?.Session;
}
/// <summary>
/// FlareSolverr のセッションを破棄します。
/// </summary>
/// <param name="client">HttpClient インスタンス</param>
/// <param name="session">破棄対象のセッションID</param>
/// <param name="flareSolverrEndpoint">FlareSolverrのエンドポイントURL</param>
/// <returns>破棄が成功したかどうか</returns>
public static async Task<bool> DestroyFlareSolverrSession(
HttpClient client,
string session,
string flareSolverrEndpoint = "https://udukipc-flaresolverr.armonica-vps.work/v1")
{
if (string.IsNullOrWhiteSpace(session)) return false;
var payload = new FlareSolverrRequest
{
Cmd = "sessions.destroy",
Session = session
};
using var request = new HttpRequestMessage(HttpMethod.Post, flareSolverrEndpoint)
{
Content = JsonContent.Create(payload, options: FlareSolverrJsonOptions)
};
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
response.EnsureSuccessStatusCode();
using var responseStream = await response.Content.ReadAsStreamAsync();
var flareResponse = await JsonSerializer.DeserializeAsync<FlareSolverrResponse>(responseStream, FlareSolverrJsonOptions);
return string.Equals(flareResponse?.Status, "ok", StringComparison.OrdinalIgnoreCase);
}
/// <summary>
/// FlareSolverr リクエスト用 DTO
/// </summary>
private sealed class FlareSolverrRequest
{
[JsonPropertyName("cmd")]
public string Cmd { get; set; } = "request.get";
[JsonPropertyName("url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Url { get; set; }
[JsonPropertyName("maxTimeout")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? MaxTimeout { get; set; }
[JsonPropertyName("session")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Session { get; set; }
}
/// <summary>
/// FlareSolverr レスポンス用 DTO
/// </summary>
private sealed class FlareSolverrResponse
{
[JsonPropertyName("status")]
public string Status { get; set; } = string.Empty;
[JsonPropertyName("message")]
public string Message { get; set; } = string.Empty;
[JsonPropertyName("session")]
public string? Session { get; set; }
[JsonPropertyName("solution")]
public FlareSolverrSolution? Solution { get; set; }
[JsonPropertyName("startTimestamp")]
public long? StartTimestamp { get; set; }
[JsonPropertyName("endTimestamp")]
public long? EndTimestamp { get; set; }
[JsonPropertyName("version")]
public string? Version { get; set; }
}
/// <summary>
/// FlareSolverr 解決結果用 DTO
/// </summary>
private sealed class FlareSolverrSolution
{
[JsonPropertyName("url")]
public string? Url { get; set; }
[JsonPropertyName("status")]
public int Status { get; set; }
[JsonPropertyName("response")]
public string? Response { get; set; }
[JsonPropertyName("userAgent")]
public string? UserAgent { get; set; }
}
#endregion
public static async Task<IEnumerable<SyndicationItem>> GetFeedItems(HttpClient client, string url) public static async Task<IEnumerable<SyndicationItem>> GetFeedItems(HttpClient client, string url)
{ {
try try
@@ -181,7 +478,7 @@ internal class CmnUtil
} }
catch (Exception ex) catch (Exception ex)
{ {
AddLog($"[エラー] {url} の取得に失敗: {ex.Message}"); LogManager.AddLog($"[エラー] {url} の取得に失敗: {ex.Message}");
return Enumerable.Empty<SyndicationItem>(); return Enumerable.Empty<SyndicationItem>();
} }
} }
+2 -4
View File
@@ -1,4 +1,4 @@
using AngleSharp.Dom; using AngleSharp.Dom;
using AngleSharp.Html.Parser; using AngleSharp.Html.Parser;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
@@ -147,9 +147,7 @@ internal class rank
body.Append(originalSummary); body.Append(originalSummary);
if (link != null) if (link != null)
{ {
var task = CmnUtil.GetDocument(client, link.ToString()); var docContents = await CmnUtil.GetDocument(client, link.ToString());
task.Wait();
var docContents = task.Result;
var imgElements = docContents.QuerySelectorAll("#post-hentai img"); var imgElements = docContents.QuerySelectorAll("#post-hentai img");
foreach (var img in imgElements) foreach (var img in imgElements)
{ {
+110 -7
View File
@@ -1,10 +1,11 @@
using AngleSharp.Dom; using AngleSharp.Dom;
using AngleSharp.Html.Dom; using AngleSharp.Html.Dom;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Reflection.Metadata; using System.Reflection.Metadata;
using System.ServiceModel.Syndication; using System.ServiceModel.Syndication;
using System.Text; using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Xml.Linq; using System.Xml.Linq;
using WebDriverBiDi.Script; using WebDriverBiDi.Script;
@@ -22,19 +23,29 @@ internal class aurahentai
var items = new List<SyndicationItem>(); var items = new List<SyndicationItem>();
try try
{ {
// 複数ページをまとめる場合に先に取得する // セッションを作成して複数ページおよび詳細ページ取得を高速化
var docList = new List<IDocument>(); string? sessionId = await CmnUtil.CreateFlareSolverrSession(client);
try
{
for (int pageId = 1; pageId <= 5; pageId++) for (int pageId = 1; pageId <= 5; pageId++)
{ {
query = $"{pageId}"; query = $"{pageId}";
var pageURL = $"{baseURL}?page={query}"; var pageURL = $"{baseURL}?page={query}";
var document = await CmnUtil.GetDocument(client, pageURL); var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId);
var newItems = await CreateFeedItem(client, document); var newItems = await CreateFeedItem(client, document, sessionId);
foreach (var newItem in newItems) foreach (var newItem in newItems)
{ {
items.Add(newItem); items.Add(newItem);
} }
} }
}
finally
{
if (sessionId != null)
{
await CmnUtil.DestroyFlareSolverrSession(client, sessionId);
}
}
feed.Items = items; feed.Items = items;
CmnUtil.ExportXMLFile("aurahentai.xml", feed); CmnUtil.ExportXMLFile("aurahentai.xml", feed);
@@ -45,7 +56,7 @@ internal class aurahentai
} }
} }
private static async Task<List<SyndicationItem>> CreateFeedItem(HttpClient client, IDocument docListPage) private static async Task<List<SyndicationItem>> CreateFeedItem(HttpClient client, IDocument docListPage, string? sessionId = null)
{ {
var items = new List<SyndicationItem>(); var items = new List<SyndicationItem>();
var elements = docListPage.QuerySelectorAll("main article>a"); var elements = docListPage.QuerySelectorAll("main article>a");
@@ -58,7 +69,7 @@ internal class aurahentai
var contentsUri = CmnUtil.GetAbsoluteUri(docListPage.BaseUrl!, relLink)!; var contentsUri = CmnUtil.GetAbsoluteUri(docListPage.BaseUrl!, relLink)!;
var linkHref = contentsUri.AbsoluteUri; var linkHref = contentsUri.AbsoluteUri;
var docContents = await CmnUtil.GetDocument(client, linkHref); var docContents = await CmnUtil.GetDocumentFlareSolverr(client, linkHref, session: sessionId);
var divDetail = docContents.QuerySelector("main>div:nth-of-type(2)")!; var divDetail = docContents.QuerySelector("main>div:nth-of-type(2)")!;
var title = ""; var title = "";
@@ -82,6 +93,20 @@ internal class aurahentai
var replaceTo = "https://img.aurahentai.com/g/"; var replaceTo = "https://img.aurahentai.com/g/";
imgUrl = $"{linkHref.Replace(replaceFr, replaceTo)}/001.webp"; imgUrl = $"{linkHref.Replace(replaceFr, replaceTo)}/001.webp";
} }
var pubDate = new DateTime();
{
var pubDateStr = GetDatePublished(docContents);
// 3. DateTimeOffset型としてパース(時差情報を維持するため)
if (DateTimeOffset.TryParse(pubDateStr, out DateTimeOffset parsedOffset))
{
// 4. 日本時間(UTC+9時間)のオフセットに変換
DateTimeOffset jstOffset = parsedOffset.ToOffset(TimeSpan.FromHours(9));
// 5. DateTime型(KindはUnspecifiedまたはLocal扱い)として戻す
pubDate = jstOffset.DateTime;
}
}
var body = new StringBuilder(); var body = new StringBuilder();
{ {
@@ -93,10 +118,88 @@ internal class aurahentai
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
// ID // ID
item.Id = CmnUtil.CreateGuid($"{linkHref}"); item.Id = CmnUtil.CreateGuid($"{linkHref}");
// Date
item.PublishDate = pubDate;
item.LastUpdatedTime = pubDate;
items.Add(item); items.Add(item);
} }
return items; return items;
} }
/// <summary>
/// IDocument 内の複数ある JSON-LD から、@type が "ImageGallery" の datePublished を取得します。
/// </summary>
/// <param name="document">AngleSharp の IDocument</param>
/// <returns>取得した datePublished 文字列(見つからない場合は null)</returns>
private static string? GetDatePublished(IDocument document)
{
// type="application/ld+json" を持つすべての script タグを取得
var scriptElements = document.QuerySelectorAll("script[type='application/ld+json']");
foreach (var script in scriptElements)
{
string jsonText = script.TextContent?.Trim() ?? string.Empty;
if (string.IsNullOrEmpty(jsonText))
{
continue;
}
try
{
// メモリ割り当てを最小限に抑えるため JsonDocument を利用
using var jsonDoc = JsonDocument.Parse(jsonText);
var root = jsonDoc.RootElement;
// 単一オブジェクトの場合
if (root.ValueKind == JsonValueKind.Object)
{
if (TryExtractDate(root, out var date))
{
return date;
}
}
// JSON-LD が配列([{...}, {...}])で定義されている場合
else if (root.ValueKind == JsonValueKind.Array)
{
foreach (var item in root.EnumerateArray())
{
if (item.ValueKind == JsonValueKind.Object && TryExtractDate(item, out var date))
{
return date;
}
}
}
}
catch (JsonException)
{
// 不正な JSON 文字列が含まれていた場合は次のタグへスキップ
continue;
}
}
return null;
}
/// <summary>
/// @type が ImageGallery か判定し、datePublished の値を取り出します。
/// </summary>
private static bool TryExtractDate(JsonElement element, out string? datePublished)
{
datePublished = null;
// @type が "ImageGallery" かどうか確認
if (element.TryGetProperty("@type", out var typeProp) &&
string.Equals(typeProp.GetString(), "ImageGallery", StringComparison.OrdinalIgnoreCase))
{
if (element.TryGetProperty("datePublished", out var dateProp))
{
datePublished = dateProp.GetString();
return true;
}
}
return false;
}
} }
@@ -1,4 +1,4 @@
using AngleSharp.Html.Dom; using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser; using AngleSharp.Html.Parser;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
@@ -27,8 +27,8 @@ internal class index_japanese
var results = await Task.WhenAll(tasks); var results = await Task.WhenAll(tasks);
var allItems = results.SelectMany(items => items).ToList(); var allItems = results.SelectMany(items => items).ToList();
// 簡潔なRSSに整形 // 簡潔なRSSに非同期で整形
var simplifiedItems = allItems.OrderByDescending(x => x.PublishDate).Select(originalItem => var itemTasks = allItems.OrderByDescending(x => x.PublishDate).Select(async originalItem =>
{ {
List<SyndicationCategory> getAddCategory() List<SyndicationCategory> getAddCategory()
{ {
@@ -134,9 +134,7 @@ internal class index_japanese
body.Append(originalSummary); body.Append(originalSummary);
if (link != null) if (link != null)
{ {
var task = CmnUtil.GetDocument(client, link.ToString()); var docContents = await CmnUtil.GetDocument(client, link.ToString());
task.Wait();
var docContents = task.Result;
var imgElements = docContents.QuerySelectorAll("#post-hentai img"); var imgElements = docContents.QuerySelectorAll("#post-hentai img");
foreach (var img in imgElements) foreach (var img in imgElements)
{ {
@@ -155,12 +153,15 @@ internal class index_japanese
newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
return newItem; return newItem;
}); }).ToList();
var simplifiedItems = await Task.WhenAll(itemTasks);
// 新しい統合フィードを作成 // 新しい統合フィードを作成
var mergedFeed = new SyndicationFeed( var mergedFeed = new SyndicationFeed(
"momon-ga.com", "hitomi.la",
"", "",
new Uri("https://momon-ga.com/magazine/"), new Uri("https://hitomi.la/index-japanese.html"),
simplifiedItems simplifiedItems
); );
+14 -4
View File
@@ -1,4 +1,4 @@
using AngleSharp; using AngleSharp;
using AngleSharp.Dom; using AngleSharp.Dom;
using NpgsqlTypes; using NpgsqlTypes;
using System; using System;
@@ -18,15 +18,25 @@ internal static class missav
var items = new List<SyndicationItem>(); var items = new List<SyndicationItem>();
try try
{ {
// 複数ページをまとめる場合に先に取得する // セッションを作成して複数ページ取得を高速化
string? sessionId = await CmnUtil.CreateFlareSolverrSession(client);
var docList = new List<IDocument>(); var docList = new List<IDocument>();
try
{
for (int pageId = 1; pageId <= 2; pageId++) for (int pageId = 1; pageId <= 2; pageId++)
{ {
var pageURL = $"https://missav.ai/ja/new{(pageId > 1 ? $"?page={pageId}" : "")}"; var pageURL = $"https://missav.ai/ja/new{(pageId > 1 ? $"?page={pageId}" : "")}";
var rssUrl = $"https://freshrss.armonica-vps.work/api/misc.php/FreshRss FlareSolverr?raw=1&feed={pageURL}"; var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId);
var document = await CmnUtil.GetDocument(client, rssUrl);
docList.Add(document); docList.Add(document);
} }
}
finally
{
if (sessionId != null)
{
await CmnUtil.DestroyFlareSolverrSession(client, sessionId);
}
}
var linkList = new List<string>(); var linkList = new List<string>();
foreach (var document in docList) foreach (var document in docList)
+1 -2
View File
@@ -1,4 +1,4 @@
using AngleSharp.Dom; using AngleSharp.Dom;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ServiceModel.Syndication; using System.ServiceModel.Syndication;
@@ -21,7 +21,6 @@ internal class missav123
for (int pageId = 1; pageId <= 2; pageId++) for (int pageId = 1; pageId <= 2; pageId++)
{ {
var pageURL = $"{pageMaster}/ja/recent{(pageId > 1 ? $"?page={pageId}" : "")}"; var pageURL = $"{pageMaster}/ja/recent{(pageId > 1 ? $"?page={pageId}" : "")}";
var rssUrl = $"https://freshrss.armonica-vps.work/api/misc.php/FreshRss FlareSolverr?raw=1&feed={pageURL}";
var document = await CmnUtil.GetDocument(client, pageURL); var document = await CmnUtil.GetDocument(client, pageURL);
docList.Add(document); docList.Add(document);
} }
+15 -28
View File
@@ -1,4 +1,4 @@
using AngleSharp.Html.Parser; using AngleSharp.Html.Parser;
using System.ServiceModel.Syndication; using System.ServiceModel.Syndication;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
@@ -24,8 +24,8 @@ internal class momonga_fanzine
var results = await Task.WhenAll(tasks); var results = await Task.WhenAll(tasks);
var allItems = results.SelectMany(items => items).ToList(); var allItems = results.SelectMany(items => items).ToList();
// 簡潔なRSSに整形 // 簡潔なRSSに非同期で整形
var simplifiedItems = allItems.OrderByDescending(x => x.PublishDate).Select(originalItem => var itemTasks = allItems.OrderByDescending(x => x.PublishDate).Select(async originalItem =>
{ {
List<SyndicationCategory> getAddCategory() List<SyndicationCategory> getAddCategory()
{ {
@@ -48,38 +48,26 @@ internal class momonga_fanzine
// ---------------------------------------------------- // ----------------------------------------------------
// 正規表現で各項目の値を抽出する // 正規表現で各項目の値を抽出する
// ---------------------------------------------------- // ----------------------------------------------------
// パターン解説:
// 【[^】]+】 : 「【」で始まり、「】以外の文字」が続き、「】」で終わる文字列 (例: 【パロディ】)
// ([^【]*) : その後、「【」以外の文字が続く限り取得する (Group1としてキャプチャ)
string pattern = @"【[^】]+】([^【]*)"; string pattern = @"【[^】]+】([^【]*)";
var matches = Regex.Matches(targetText, pattern); var matches = Regex.Matches(targetText, pattern);
foreach (Match match in matches) foreach (Match match in matches)
{ {
// match.Groups[1].Value には "パロ1, パロ 2 " などの値が入ります
// (項目の中身が空の場合は空文字になります)
string valuesStr = match.Groups[1].Value; string valuesStr = match.Groups[1].Value;
if (string.IsNullOrWhiteSpace(valuesStr)) if (string.IsNullOrWhiteSpace(valuesStr))
{ {
continue; // 中身が空の場合はスキップ continue;
} }
// ----------------------------------------------------
// 3. カンマで分割し、前後の空白を除去してカテゴリに追加
// ----------------------------------------------------
// ※もし全角カンマ「、」も考慮したい場合は new[] { ',', '、' } と記述します
string[] splitValues = valuesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); string[] splitValues = valuesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var val in splitValues) foreach (var val in splitValues)
{ {
// 前後の半角・全角スペースを取り除く
string categoryName = val.Trim(); string categoryName = val.Trim();
// 空でなければカテゴリとして追加
if (!string.IsNullOrEmpty(categoryName)) if (!string.IsNullOrEmpty(categoryName))
{ {
// domain属性なし(前回回答の手法)でカテゴリを追加
result.Add(new SyndicationCategory(categoryName)); result.Add(new SyndicationCategory(categoryName));
} }
} }
@@ -90,14 +78,9 @@ internal class momonga_fanzine
var newItem = new SyndicationItem() var newItem = new SyndicationItem()
{ {
// タイトルは TextSyndicationContent でラップする必要があります Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title"),
Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title") PublishDate = originalItem.PublishDate,
, Id = CmnUtil.CreateGuid(originalItem.Id) LastUpdatedTime = originalItem.LastUpdatedTime
, PublishDate = originalItem.PublishDate
, LastUpdatedTime = originalItem.LastUpdatedTime
// 元のコンテンツをそのまま引き継ぐ
// , Content = originalItem.Content
// , Summary = originalItem.Summary
}; };
// リンクを引き継ぐ // リンクを引き継ぐ
@@ -122,14 +105,15 @@ internal class momonga_fanzine
newItem.Authors.Add(author); newItem.Authors.Add(author);
} }
// ID
newItem.Id = CmnUtil.CreateGuid($"{link}");
var body = new StringBuilder(); var body = new StringBuilder();
var originalSummary = originalItem.Summary.Text; var originalSummary = originalItem.Summary.Text;
body.Append(originalSummary); body.Append(originalSummary);
if (link != null) if (link != null)
{ {
var task = CmnUtil.GetDocument(client, link.ToString()); var docContents = await CmnUtil.GetDocument(client, link.ToString());
task.Wait();
var docContents = task.Result;
var imgElements = docContents.QuerySelectorAll("#post-hentai img"); var imgElements = docContents.QuerySelectorAll("#post-hentai img");
foreach (var img in imgElements) foreach (var img in imgElements)
{ {
@@ -156,7 +140,10 @@ internal class momonga_fanzine
newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
return newItem; return newItem;
}); }).ToList();
var simplifiedItems = await Task.WhenAll(itemTasks);
// 新しい統合フィードを作成 // 新しい統合フィードを作成
var mergedFeed = new SyndicationFeed( var mergedFeed = new SyndicationFeed(
"momon-ga.com", "momon-ga.com",
+36 -29
View File
@@ -1,4 +1,4 @@
using AngleSharp.Html.Parser; using AngleSharp.Html.Parser;
using System.ServiceModel.Syndication; using System.ServiceModel.Syndication;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
@@ -24,8 +24,8 @@ internal class momonga_magazine
var results = await Task.WhenAll(tasks); var results = await Task.WhenAll(tasks);
var allItems = results.SelectMany(items => items).ToList(); var allItems = results.SelectMany(items => items).ToList();
// 簡潔なRSSに整形 // 簡潔なRSSに非同期で整形
var simplifiedItems = allItems.OrderByDescending(x => x.PublishDate).Select(originalItem => var itemTasks = allItems.OrderByDescending(x => x.PublishDate).Select(async originalItem =>
{ {
List<SyndicationCategory> getAddCategory() List<SyndicationCategory> getAddCategory()
{ {
@@ -48,38 +48,26 @@ internal class momonga_magazine
// ---------------------------------------------------- // ----------------------------------------------------
// 正規表現で各項目の値を抽出する // 正規表現で各項目の値を抽出する
// ---------------------------------------------------- // ----------------------------------------------------
// パターン解説:
// 【[^】]+】 : 「【」で始まり、「】以外の文字」が続き、「】」で終わる文字列 (例: 【パロディ】)
// ([^【]*) : その後、「【」以外の文字が続く限り取得する (Group1としてキャプチャ)
string pattern = @"【[^】]+】([^【]*)"; string pattern = @"【[^】]+】([^【]*)";
var matches = Regex.Matches(targetText, pattern); var matches = Regex.Matches(targetText, pattern);
foreach (Match match in matches) foreach (Match match in matches)
{ {
// match.Groups[1].Value には "パロ1, パロ 2 " などの値が入ります
// (項目の中身が空の場合は空文字になります)
string valuesStr = match.Groups[1].Value; string valuesStr = match.Groups[1].Value;
if (string.IsNullOrWhiteSpace(valuesStr)) if (string.IsNullOrWhiteSpace(valuesStr))
{ {
continue; // 中身が空の場合はスキップ continue;
} }
// ----------------------------------------------------
// 3. カンマで分割し、前後の空白を除去してカテゴリに追加
// ----------------------------------------------------
// ※もし全角カンマ「、」も考慮したい場合は new[] { ',', '、' } と記述します
string[] splitValues = valuesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); string[] splitValues = valuesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var val in splitValues) foreach (var val in splitValues)
{ {
// 前後の半角・全角スペースを取り除く
string categoryName = val.Trim(); string categoryName = val.Trim();
// 空でなければカテゴリとして追加
if (!string.IsNullOrEmpty(categoryName)) if (!string.IsNullOrEmpty(categoryName))
{ {
// domain属性なし(前回回答の手法)でカテゴリを追加
result.Add(new SyndicationCategory(categoryName)); result.Add(new SyndicationCategory(categoryName));
} }
} }
@@ -90,15 +78,9 @@ internal class momonga_magazine
var newItem = new SyndicationItem() var newItem = new SyndicationItem()
{ {
// タイトルは TextSyndicationContent でラップする必要があります Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title"),
Title = new TextSyndicationContent(originalItem.Title?.Text ?? "No Title") PublishDate = originalItem.PublishDate,
, Id = CmnUtil.CreateGuid(originalItem.Id) LastUpdatedTime = originalItem.LastUpdatedTime
, PublishDate = originalItem.PublishDate
, LastUpdatedTime = originalItem.LastUpdatedTime
// 元のコンテンツをそのまま引き継ぐ
// , Content = originalItem.Content
// , Summary = originalItem.Summary
}; };
// リンクを引き継ぐ // リンクを引き継ぐ
@@ -123,14 +105,15 @@ internal class momonga_magazine
newItem.Authors.Add(author); newItem.Authors.Add(author);
} }
// ID
newItem.Id = CmnUtil.CreateGuid($"{link}");
var body = new StringBuilder(); var body = new StringBuilder();
var originalSummary = originalItem.Summary.Text; var originalSummary = originalItem.Summary.Text;
body.Append(originalSummary); body.Append(originalSummary);
if (link != null) if (link != null)
{ {
var task = CmnUtil.GetDocument(client, link.ToString()); var docContents = await CmnUtil.GetDocument(client, link.ToString());
task.Wait();
var docContents = task.Result;
var imgElements = docContents.QuerySelectorAll("#post-hentai img"); var imgElements = docContents.QuerySelectorAll("#post-hentai img");
foreach (var img in imgElements) foreach (var img in imgElements)
{ {
@@ -140,7 +123,15 @@ internal class momonga_magazine
// 相対URLの場合も考慮して、絶対URLに解決する // 相対URLの場合も考慮して、絶対URLに解決する
if (Uri.TryCreate(link, src, out var absoluteUri)) if (Uri.TryCreate(link, src, out var absoluteUri))
{ {
string encoded = ToBase64Url(link.AbsoluteUri);
string downloadUrl = $"https://apicien.armonica-vps.work/fanzine/{encoded}";
body.Append($"<div>");
body.Append($"<a href=\"{downloadUrl}\">[Download]</a>");
body.Append($"</div>");
body.Append($"<div>");
body.Append($"<img src=\"{absoluteUri}\" />"); body.Append($"<img src=\"{absoluteUri}\" />");
body.Append($"</div>");
break; break;
} }
} }
@@ -149,7 +140,10 @@ internal class momonga_magazine
newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html); newItem.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
return newItem; return newItem;
}); }).ToList();
var simplifiedItems = await Task.WhenAll(itemTasks);
// 新しい統合フィードを作成 // 新しい統合フィードを作成
var mergedFeed = new SyndicationFeed( var mergedFeed = new SyndicationFeed(
"momon-ga.com", "momon-ga.com",
@@ -165,4 +159,17 @@ internal class momonga_magazine
LogManager.AddLog($"[エラー] momonga-magazine:{ex.Message}"); LogManager.AddLog($"[エラー] momonga-magazine:{ex.Message}");
} }
} }
/// <summary>
/// 文字列をURLセーフなBase64(Base64URL)に変換します (RFC 4648 準拠)
/// </summary>
static string ToBase64Url(string plainText)
{
byte[] bytes = Encoding.UTF8.GetBytes(plainText);
return Convert.ToBase64String(bytes)
.Replace('+', '-') // '+' を '-' に置換
.Replace('/', '_') // '/' を '_' に置換
.TrimEnd('='); // 末尾のパディング '=' を削除
}
} }
+15 -3
View File
@@ -1,4 +1,5 @@
using AngleSharp.Dom; using AngleSharp.Dom;
using PuppeteerSharp;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ServiceModel.Syndication; using System.ServiceModel.Syndication;
@@ -15,8 +16,19 @@ internal class npb_6
try try
{ {
var pageURL = $"https://baseball.yahoo.co.jp/npb/teams/6/top"; var pageURL = $"https://baseball.yahoo.co.jp/npb/teams/6/top";
var rssUrl = $"https://freshrss.armonica-vps.work/api/misc.php/FreshRss FlareSolverr?raw=1&feed={pageURL}"; IDocument document;
var document = await CmnUtil.GetDocument(client, rssUrl); string? sessionId = await CmnUtil.CreateFlareSolverrSession(client);
try
{
document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId);
}
finally
{
if (sessionId != null)
{
await CmnUtil.DestroyFlareSolverrSession(client, sessionId);
}
}
var linkList = new List<string>(); var linkList = new List<string>();
if (document != null) if (document != null)
+25 -4
View File
@@ -6,6 +6,27 @@ using rss_generator.Class.RSS.hitomi_la;
using rss_generator.Class.RSS.X; using rss_generator.Class.RSS.X;
using System.Diagnostics; using System.Diagnostics;
// 1. 同期スレッド等の未処理例外
AppDomain.CurrentDomain.UnhandledException += (sender, e) =>
{
var ex = e.ExceptionObject as Exception;
if (ex != null)
{
CmnUtil.PrintCallStackToEvent(ex);
}
};
// 2. async/await や Task 内で放置された未処理例外 (GC回収時に発火)
TaskScheduler.UnobservedTaskException += (sender, e) =>
{
if (e != null)
{
CmnUtil.PrintCallStackToEvent(e.Exception);
e.SetObserved(); // クラッシュを防止する場合に指定
}
};
var services = new ServiceCollection(); var services = new ServiceCollection();
// 2. IHttpClientFactory の登録とクッキーの有効化設定(名前付きクライアント) // 2. IHttpClientFactory の登録とクッキーの有効化設定(名前付きクライアント)
services.AddHttpClient("webClient"); services.AddHttpClient("webClient");
@@ -57,7 +78,7 @@ while (true)
tasks.Add(momonga_fanzine.CreateRSSAsync(client)); tasks.Add(momonga_fanzine.CreateRSSAsync(client));
tasks.Add(momonga_magazine.CreateRSSAsync(client)); tasks.Add(momonga_magazine.CreateRSSAsync(client));
} }
tasks.Add(Youtube.CreateRSSAsync(client)); //tasks.Add(Youtube.CreateRSSAsync(client));
tasks.Add(x_list.CreateRSSAsync(client)); tasks.Add(x_list.CreateRSSAsync(client));
tasks.Add(javtwi.CreateRSSAsync(client)); tasks.Add(javtwi.CreateRSSAsync(client));
tasks.Add(kimochiinfo.CreateRSSAsync(client)); tasks.Add(kimochiinfo.CreateRSSAsync(client));
@@ -72,13 +93,13 @@ while (true)
await Task.WhenAll(tasks); await Task.WhenAll(tasks);
sw.Stop(); sw.Stop();
LogManager.AddLog($"[情報] RSS生成完了 実行時間:{sw.Elapsed.TotalSeconds}秒"); LogManager.AddLog($"[情報] RSS生成完了 実行時間:{sw.Elapsed.TotalSeconds:0.################}秒");
sw.Restart(); sw.Restart();
// /bin/bash /usr/local/bin/update_rss.sh // /bin/bash /usr/local/bin/update_rss.sh
#if DEBUG #if DEBUG
sw.Stop(); sw.Stop();
LogManager.AddLog($"[情報] 実行完了 実行時間:{sw.Elapsed.TotalSeconds}秒"); LogManager.AddLog($"[情報] デバッグ完了");
await LogManager.FlushAndCloseAsync(); await LogManager.FlushAndCloseAsync();
break; break;
@@ -87,7 +108,7 @@ while (true)
await CmnUtil.RunCommandAsync("/bin/bash", "/usr/local/bin/update_rss.sh", ""); await CmnUtil.RunCommandAsync("/bin/bash", "/usr/local/bin/update_rss.sh", "");
sw.Stop(); sw.Stop();
LogManager.AddLog($"[情報] FreshRSS 更新完了 実行時間:{sw.Elapsed.TotalSeconds}秒"); LogManager.AddLog($"[情報] FreshRSS 更新完了 実行時間:{sw.Elapsed.TotalSeconds:0.################}秒");
#endif #endif
} }