fix(hitomi-npb): hitomi.laおよびスポーツナビの既存XMLキャッシュ引き継ぎによる未読化防止対応
This commit is contained in:
1 parent
0fe359cf17
commit
4800d98223
2 files changed
+252
-24
No files matched your search
@@ -61,15 +61,69 @@ internal class index_japanese
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 重複除外(IDベース)
|
// 既存フィードを読み込み、過去の公開日時・更新日時・本文HTMLを引き継ぐ
|
||||||
var uniqueItems = items
|
var existingEntries = LoadExistingEntries("hitomi-japanese.xml");
|
||||||
.GroupBy(x => x.Id)
|
|
||||||
.Select(g => g.First())
|
// 既存エントリの引き継ぎとマージ
|
||||||
|
var mergedDict = new Dictionary<string, SyndicationItem>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
// まず既存フィードの全アイテムを登録
|
||||||
|
foreach (var kvp in existingEntries)
|
||||||
|
{
|
||||||
|
var entry = kvp.Value;
|
||||||
|
if (!mergedDict.ContainsKey(entry.Guid))
|
||||||
|
{
|
||||||
|
var synItem = new SyndicationItem(
|
||||||
|
entry.Title,
|
||||||
|
(string?)null,
|
||||||
|
!string.IsNullOrEmpty(entry.Link) ? new Uri(entry.Link) : null,
|
||||||
|
entry.Guid,
|
||||||
|
entry.PublishDate != DateTimeOffset.MinValue ? entry.PublishDate : entry.LastUpdatedTime
|
||||||
|
)
|
||||||
|
{
|
||||||
|
PublishDate = entry.PublishDate != DateTimeOffset.MinValue ? entry.PublishDate : entry.LastUpdatedTime,
|
||||||
|
LastUpdatedTime = entry.LastUpdatedTime != DateTimeOffset.MinValue ? entry.LastUpdatedTime : entry.PublishDate,
|
||||||
|
Summary = new TextSyndicationContent(entry.SummaryHtml, TextSyndicationContentKind.Html)
|
||||||
|
};
|
||||||
|
foreach (var author in entry.Authors)
|
||||||
|
{
|
||||||
|
synItem.Authors.Add(new SyndicationPerson { Name = author });
|
||||||
|
}
|
||||||
|
foreach (var category in entry.Categories)
|
||||||
|
{
|
||||||
|
synItem.Categories.Add(new SyndicationCategory(category));
|
||||||
|
}
|
||||||
|
mergedDict[entry.Guid] = synItem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 今回取得したアイテムを反映(既存がある場合は前回の公開日時・本文HTMLを最優先で維持)
|
||||||
|
foreach (var newItem in items)
|
||||||
|
{
|
||||||
|
if (!string.IsNullOrEmpty(newItem.Id))
|
||||||
|
{
|
||||||
|
if (existingEntries.TryGetValue(newItem.Id, out var existing))
|
||||||
|
{
|
||||||
|
newItem.PublishDate = existing.PublishDate != DateTimeOffset.MinValue ? existing.PublishDate : newItem.PublishDate;
|
||||||
|
newItem.LastUpdatedTime = existing.LastUpdatedTime != DateTimeOffset.MinValue ? existing.LastUpdatedTime : newItem.LastUpdatedTime;
|
||||||
|
if (!string.IsNullOrWhiteSpace(existing.SummaryHtml))
|
||||||
|
{
|
||||||
|
newItem.Summary = new TextSyndicationContent(existing.SummaryHtml, TextSyndicationContentKind.Html);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mergedDict[newItem.Id] = newItem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 重複除外とソート(新しい順)
|
||||||
|
var finalItems = mergedDict.Values
|
||||||
|
.OrderByDescending(x => x.PublishDate != DateTimeOffset.MinValue ? x.PublishDate : x.LastUpdatedTime)
|
||||||
|
.Take(100)
|
||||||
.ToList();
|
.ToList();
|
||||||
|
|
||||||
feed.Items = uniqueItems;
|
feed.Items = finalItems;
|
||||||
|
|
||||||
await CmnUtil.ExportXMLFileAsync("hitomi-japanese.xml", feed, client);
|
await CmnUtil.ExportXMLFileAsync("hitomi-japanese.xml", feed, client, filter24Hours: false);
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
@@ -229,4 +283,50 @@ internal class index_japanese
|
|||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 既存の hitomi-japanese.xml を読み込み、GUIDおよびURLをキーとするキャッシュ辞書を返します。
|
||||||
|
/// </summary>
|
||||||
|
private static Dictionary<string, (string Guid, string Link, string Title, string SummaryHtml, DateTimeOffset PublishDate, DateTimeOffset LastUpdatedTime, List<string> Authors, List<string> Categories)> LoadExistingEntries(string fileName)
|
||||||
|
{
|
||||||
|
var dict = new Dictionary<string, (string Guid, string Link, string Title, string SummaryHtml, DateTimeOffset PublishDate, DateTimeOffset LastUpdatedTime, List<string> Authors, List<string> Categories)>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
var filePath = CmnUtil.GetExportXmlPath(fileName);
|
||||||
|
if (!System.IO.File.Exists(filePath)) return dict;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var reader = System.Xml.XmlReader.Create(filePath);
|
||||||
|
var feed = SyndicationFeed.Load(reader);
|
||||||
|
if (feed?.Items != null)
|
||||||
|
{
|
||||||
|
foreach (var item in feed.Items)
|
||||||
|
{
|
||||||
|
string guid = item.Id ?? string.Empty;
|
||||||
|
string link = item.Links.FirstOrDefault()?.Uri?.ToString() ?? string.Empty;
|
||||||
|
string title = item.Title?.Text ?? string.Empty;
|
||||||
|
string summary = item.Summary?.Text ?? string.Empty;
|
||||||
|
DateTimeOffset pubDate = item.PublishDate;
|
||||||
|
DateTimeOffset lastUpdated = item.LastUpdatedTime;
|
||||||
|
var authors = item.Authors.Select(a => a.Name).Where(n => !string.IsNullOrEmpty(n)).ToList();
|
||||||
|
var categories = item.Categories.Select(c => c.Name).Where(n => !string.IsNullOrEmpty(n)).ToList();
|
||||||
|
|
||||||
|
var entry = (guid, link, title, summary, pubDate, lastUpdated, authors, categories);
|
||||||
|
|
||||||
|
if (!string.IsNullOrEmpty(guid) && !dict.ContainsKey(guid))
|
||||||
|
{
|
||||||
|
dict[guid] = entry;
|
||||||
|
}
|
||||||
|
if (!string.IsNullOrEmpty(link) && !dict.ContainsKey(link))
|
||||||
|
{
|
||||||
|
dict[link] = entry;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
LogManager.AddLog($"[警告] 【hitomi-japanese】 既存フィード読み込み失敗 ({fileName}): {ex.Message}");
|
||||||
|
}
|
||||||
|
return dict;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,26 +1,69 @@
|
|||||||
using AngleSharp.Dom;
|
using AngleSharp.Dom;
|
||||||
using PuppeteerSharp;
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Net.Http;
|
||||||
using System.ServiceModel.Syndication;
|
using System.ServiceModel.Syndication;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Xml;
|
||||||
using rss_generator.Class.Common;
|
using rss_generator.Class.Common;
|
||||||
|
|
||||||
namespace rss_generator.Class.YahooBaseball;
|
namespace rss_generator.Class.YahooBaseball;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 広島東洋カープ(スポーツナビ)のニュースタイムラインからRSSフィードを生成するクラス
|
||||||
|
/// </summary>
|
||||||
internal class npb_6
|
internal class npb_6
|
||||||
{
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// 広島東洋カープの最新情報を取得し、npb_6.xml として出力します。
|
||||||
|
/// 既存フィードを読み込み、過去の公開日時および本文HTMLを維持することで、FreshRSSにおける不要な未読化を防ぎます。
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="client">HttpClient インスタンス</param>
|
||||||
|
/// <param name="sessionId">FlareSolverr セッションID(任意)</param>
|
||||||
public static async Task CreateRSSAsync(HttpClient client, string? sessionId = null)
|
public static async Task CreateRSSAsync(HttpClient client, string? sessionId = null)
|
||||||
{
|
{
|
||||||
var feed = new SyndicationFeed("広島東洋カープ - プロ野球 - スポーツナビ", "", new Uri("https://baseball.yahoo.co.jp/npb/teams/6/top"));
|
var feed = new SyndicationFeed("広島東洋カープ - プロ野球 - スポーツナビ", "広島東洋カープの最新ニュース・タイムライン", new Uri("https://baseball.yahoo.co.jp/npb/teams/6/top"));
|
||||||
var items = new List<SyndicationItem>();
|
var items = new List<SyndicationItem>();
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var pageURL = $"https://baseball.yahoo.co.jp/npb/teams/6/top";
|
var pageURL = "https://baseball.yahoo.co.jp/npb/teams/6/top";
|
||||||
sessionId ??= CmnUtil.SharedFlareSolverrSessionId;
|
sessionId ??= CmnUtil.SharedFlareSolverrSessionId;
|
||||||
var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId);
|
var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId);
|
||||||
|
|
||||||
var linkList = new List<string>();
|
// 既存フィードを読み込み、過去の公開日時・更新日時・本文HTMLを引き継ぐ
|
||||||
|
var existingEntries = LoadExistingEntries("npb_6.xml");
|
||||||
|
var mergedDict = new Dictionary<string, SyndicationItem>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
// 既存フィードの全アイテムを登録
|
||||||
|
foreach (var kvp in existingEntries)
|
||||||
|
{
|
||||||
|
var entry = kvp.Value;
|
||||||
|
if (!mergedDict.ContainsKey(entry.Guid))
|
||||||
|
{
|
||||||
|
var synItem = new SyndicationItem(
|
||||||
|
entry.Title,
|
||||||
|
(string?)null,
|
||||||
|
!string.IsNullOrEmpty(entry.Link) ? new Uri(entry.Link) : null,
|
||||||
|
entry.Guid,
|
||||||
|
entry.PublishDate != DateTimeOffset.MinValue ? entry.PublishDate : entry.LastUpdatedTime
|
||||||
|
)
|
||||||
|
{
|
||||||
|
PublishDate = entry.PublishDate != DateTimeOffset.MinValue ? entry.PublishDate : entry.LastUpdatedTime,
|
||||||
|
LastUpdatedTime = entry.LastUpdatedTime != DateTimeOffset.MinValue ? entry.LastUpdatedTime : entry.PublishDate,
|
||||||
|
Summary = new TextSyndicationContent(entry.SummaryHtml, TextSyndicationContentKind.Html)
|
||||||
|
};
|
||||||
|
foreach (var author in entry.Authors)
|
||||||
|
{
|
||||||
|
synItem.Authors.Add(new SyndicationPerson { Name = author });
|
||||||
|
}
|
||||||
|
mergedDict[entry.Guid] = synItem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (document != null)
|
if (document != null)
|
||||||
{
|
{
|
||||||
var elements = document.QuerySelectorAll("li.sn-timeLine__item");
|
var elements = document.QuerySelectorAll("li.sn-timeLine__item");
|
||||||
@@ -42,36 +85,121 @@ internal class npb_6
|
|||||||
|
|
||||||
var itemUri = !string.IsNullOrEmpty(linkHref) ? new Uri(linkHref) : null;
|
var itemUri = !string.IsNullOrEmpty(linkHref) ? new Uri(linkHref) : null;
|
||||||
var imageUri = !string.IsNullOrEmpty(imageSrc) ? new Uri(imageSrc) : null;
|
var imageUri = !string.IsNullOrEmpty(imageSrc) ? new Uri(imageSrc) : null;
|
||||||
|
string guid = CmnUtil.CreateGuid(linkHref);
|
||||||
|
|
||||||
var body = new StringBuilder();
|
var body = new StringBuilder();
|
||||||
body.Append($@"<div><a href=""{itemUri}"" target=""_blank"" rel=""noopener noreferrer"" referrerpolicy=""no-referrer"">{title}</a></div>");
|
body.Append($@"<div><a href=""{itemUri}"" target=""_blank"" rel=""noopener noreferrer"" referrerpolicy=""no-referrer"">{title}</a></div>");
|
||||||
body.Append($@"<div><img src=""{imageUri}"" referrerpolicy=""no-referrer"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
|
if (imageUri != null)
|
||||||
|
|
||||||
var updateTime = DateTime.Now;
|
|
||||||
if (DateTime.TryParse(elemTime?.TextContent?.Trim(), out DateTime itemTime))
|
|
||||||
{
|
{
|
||||||
updateTime = itemTime;
|
body.Append($@"<div><img src=""{imageUri}"" referrerpolicy=""no-referrer"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 既存エントリが存在する場合は前回の公開日時・本文HTMLを最優先で維持
|
||||||
|
DateTimeOffset itemDate;
|
||||||
|
string summaryHtml = body.ToString();
|
||||||
|
|
||||||
|
if (existingEntries.TryGetValue(guid, out var existing))
|
||||||
|
{
|
||||||
|
itemDate = existing.PublishDate != DateTimeOffset.MinValue ? existing.PublishDate : existing.LastUpdatedTime;
|
||||||
|
if (!string.IsNullOrWhiteSpace(existing.SummaryHtml))
|
||||||
|
{
|
||||||
|
summaryHtml = existing.SummaryHtml;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (DateTime.TryParse(elemTime?.TextContent?.Trim(), out DateTime parsedTime))
|
||||||
|
{
|
||||||
|
itemDate = new DateTimeOffset(parsedTime);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
itemDate = DateTimeOffset.Now;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var item = new SyndicationItem(
|
var item = new SyndicationItem(
|
||||||
title,
|
title,
|
||||||
string.Empty,
|
(string?)null,
|
||||||
itemUri,
|
itemUri,
|
||||||
CmnUtil.CreateGuid(linkHref),
|
guid,
|
||||||
updateTime
|
itemDate
|
||||||
);
|
)
|
||||||
item.Authors.Add(new SyndicationPerson() { Name = elemCredit?.TextContent?.Trim() ?? "" });
|
{
|
||||||
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
|
PublishDate = itemDate,
|
||||||
|
LastUpdatedTime = itemDate,
|
||||||
|
Summary = new TextSyndicationContent(summaryHtml, TextSyndicationContentKind.Html)
|
||||||
|
};
|
||||||
|
|
||||||
items.Add(item);
|
var credit = elemCredit?.TextContent?.Trim();
|
||||||
|
if (!string.IsNullOrEmpty(credit))
|
||||||
|
{
|
||||||
|
item.Authors.Add(new SyndicationPerson { Name = credit });
|
||||||
|
}
|
||||||
|
|
||||||
|
mergedDict[guid] = item;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
feed.Items = items;
|
|
||||||
|
|
||||||
await CmnUtil.ExportXMLFileAsync("npb_6.xml", feed, client);
|
// 重複除外とソート(新しい順)
|
||||||
|
var finalItems = mergedDict.Values
|
||||||
|
.OrderByDescending(x => x.PublishDate != DateTimeOffset.MinValue ? x.PublishDate : x.LastUpdatedTime)
|
||||||
|
.Take(100)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
feed.Items = finalItems;
|
||||||
|
|
||||||
|
await CmnUtil.ExportXMLFileAsync("npb_6.xml", feed, client, filter24Hours: false);
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
LogManager.AddLog($"[エラー] 【npb_6】 {ex.Message}");
|
LogManager.AddLog($"[エラー] 【npb_6】 {ex.Message}");
|
||||||
|
CmnUtil.PrintCallStackToEvent(ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 既存の npb_6.xml を読み込み、GUIDおよびURLをキーとするキャッシュ辞書を返します。
|
||||||
|
/// </summary>
|
||||||
|
private static Dictionary<string, (string Guid, string Link, string Title, string SummaryHtml, DateTimeOffset PublishDate, DateTimeOffset LastUpdatedTime, List<string> Authors)> LoadExistingEntries(string fileName)
|
||||||
|
{
|
||||||
|
var dict = new Dictionary<string, (string Guid, string Link, string Title, string SummaryHtml, DateTimeOffset PublishDate, DateTimeOffset LastUpdatedTime, List<string> Authors)>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
var filePath = CmnUtil.GetExportXmlPath(fileName);
|
||||||
|
if (!File.Exists(filePath)) return dict;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var reader = XmlReader.Create(filePath);
|
||||||
|
var feed = SyndicationFeed.Load(reader);
|
||||||
|
if (feed?.Items != null)
|
||||||
|
{
|
||||||
|
foreach (var item in feed.Items)
|
||||||
|
{
|
||||||
|
string guid = item.Id ?? string.Empty;
|
||||||
|
string link = item.Links.FirstOrDefault()?.Uri?.ToString() ?? string.Empty;
|
||||||
|
string title = item.Title?.Text ?? string.Empty;
|
||||||
|
string summary = item.Summary?.Text ?? string.Empty;
|
||||||
|
DateTimeOffset pubDate = item.PublishDate;
|
||||||
|
DateTimeOffset lastUpdated = item.LastUpdatedTime;
|
||||||
|
var authors = item.Authors.Select(a => a.Name).Where(n => !string.IsNullOrEmpty(n)).ToList();
|
||||||
|
|
||||||
|
var entry = (guid, link, title, summary, pubDate, lastUpdated, authors);
|
||||||
|
|
||||||
|
if (!string.IsNullOrEmpty(guid) && !dict.ContainsKey(guid))
|
||||||
|
{
|
||||||
|
dict[guid] = entry;
|
||||||
|
}
|
||||||
|
if (!string.IsNullOrEmpty(link) && !dict.ContainsKey(link))
|
||||||
|
{
|
||||||
|
dict[link] = entry;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
LogManager.AddLog($"[警告] 【npb_6】 既存フィード読み込み失敗 ({fileName}): {ex.Message}");
|
||||||
|
}
|
||||||
|
return dict;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user