fix(hitomi-npb): hitomi.laおよびスポーツナビの既存XMLキャッシュ引き継ぎによる未読化防止対応
This commit is contained in:
1 parent
0fe359cf17
commit
4800d98223
2 files changed
+254
-26
No files matched your search
@@ -61,15 +61,69 @@ internal class index_japanese
|
||||
}
|
||||
}
|
||||
|
||||
// 重複除外(IDベース)
|
||||
var uniqueItems = items
|
||||
.GroupBy(x => x.Id)
|
||||
.Select(g => g.First())
|
||||
// 既存フィードを読み込み、過去の公開日時・更新日時・本文HTMLを引き継ぐ
|
||||
var existingEntries = LoadExistingEntries("hitomi-japanese.xml");
|
||||
|
||||
// 既存エントリの引き継ぎとマージ
|
||||
var mergedDict = new Dictionary<string, SyndicationItem>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// まず既存フィードの全アイテムを登録
|
||||
foreach (var kvp in existingEntries)
|
||||
{
|
||||
var entry = kvp.Value;
|
||||
if (!mergedDict.ContainsKey(entry.Guid))
|
||||
{
|
||||
var synItem = new SyndicationItem(
|
||||
entry.Title,
|
||||
(string?)null,
|
||||
!string.IsNullOrEmpty(entry.Link) ? new Uri(entry.Link) : null,
|
||||
entry.Guid,
|
||||
entry.PublishDate != DateTimeOffset.MinValue ? entry.PublishDate : entry.LastUpdatedTime
|
||||
)
|
||||
{
|
||||
PublishDate = entry.PublishDate != DateTimeOffset.MinValue ? entry.PublishDate : entry.LastUpdatedTime,
|
||||
LastUpdatedTime = entry.LastUpdatedTime != DateTimeOffset.MinValue ? entry.LastUpdatedTime : entry.PublishDate,
|
||||
Summary = new TextSyndicationContent(entry.SummaryHtml, TextSyndicationContentKind.Html)
|
||||
};
|
||||
foreach (var author in entry.Authors)
|
||||
{
|
||||
synItem.Authors.Add(new SyndicationPerson { Name = author });
|
||||
}
|
||||
foreach (var category in entry.Categories)
|
||||
{
|
||||
synItem.Categories.Add(new SyndicationCategory(category));
|
||||
}
|
||||
mergedDict[entry.Guid] = synItem;
|
||||
}
|
||||
}
|
||||
|
||||
// 今回取得したアイテムを反映(既存がある場合は前回の公開日時・本文HTMLを最優先で維持)
|
||||
foreach (var newItem in items)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(newItem.Id))
|
||||
{
|
||||
if (existingEntries.TryGetValue(newItem.Id, out var existing))
|
||||
{
|
||||
newItem.PublishDate = existing.PublishDate != DateTimeOffset.MinValue ? existing.PublishDate : newItem.PublishDate;
|
||||
newItem.LastUpdatedTime = existing.LastUpdatedTime != DateTimeOffset.MinValue ? existing.LastUpdatedTime : newItem.LastUpdatedTime;
|
||||
if (!string.IsNullOrWhiteSpace(existing.SummaryHtml))
|
||||
{
|
||||
newItem.Summary = new TextSyndicationContent(existing.SummaryHtml, TextSyndicationContentKind.Html);
|
||||
}
|
||||
}
|
||||
mergedDict[newItem.Id] = newItem;
|
||||
}
|
||||
}
|
||||
|
||||
// 重複除外とソート(新しい順)
|
||||
var finalItems = mergedDict.Values
|
||||
.OrderByDescending(x => x.PublishDate != DateTimeOffset.MinValue ? x.PublishDate : x.LastUpdatedTime)
|
||||
.Take(100)
|
||||
.ToList();
|
||||
|
||||
feed.Items = uniqueItems;
|
||||
feed.Items = finalItems;
|
||||
|
||||
await CmnUtil.ExportXMLFileAsync("hitomi-japanese.xml", feed, client);
|
||||
await CmnUtil.ExportXMLFileAsync("hitomi-japanese.xml", feed, client, filter24Hours: false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -229,4 +283,50 @@ internal class index_japanese
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 既存の hitomi-japanese.xml を読み込み、GUIDおよびURLをキーとするキャッシュ辞書を返します。
|
||||
/// </summary>
|
||||
private static Dictionary<string, (string Guid, string Link, string Title, string SummaryHtml, DateTimeOffset PublishDate, DateTimeOffset LastUpdatedTime, List<string> Authors, List<string> Categories)> LoadExistingEntries(string fileName)
|
||||
{
|
||||
var dict = new Dictionary<string, (string Guid, string Link, string Title, string SummaryHtml, DateTimeOffset PublishDate, DateTimeOffset LastUpdatedTime, List<string> Authors, List<string> Categories)>(StringComparer.OrdinalIgnoreCase);
|
||||
var filePath = CmnUtil.GetExportXmlPath(fileName);
|
||||
if (!System.IO.File.Exists(filePath)) return dict;
|
||||
|
||||
try
|
||||
{
|
||||
using var reader = System.Xml.XmlReader.Create(filePath);
|
||||
var feed = SyndicationFeed.Load(reader);
|
||||
if (feed?.Items != null)
|
||||
{
|
||||
foreach (var item in feed.Items)
|
||||
{
|
||||
string guid = item.Id ?? string.Empty;
|
||||
string link = item.Links.FirstOrDefault()?.Uri?.ToString() ?? string.Empty;
|
||||
string title = item.Title?.Text ?? string.Empty;
|
||||
string summary = item.Summary?.Text ?? string.Empty;
|
||||
DateTimeOffset pubDate = item.PublishDate;
|
||||
DateTimeOffset lastUpdated = item.LastUpdatedTime;
|
||||
var authors = item.Authors.Select(a => a.Name).Where(n => !string.IsNullOrEmpty(n)).ToList();
|
||||
var categories = item.Categories.Select(c => c.Name).Where(n => !string.IsNullOrEmpty(n)).ToList();
|
||||
|
||||
var entry = (guid, link, title, summary, pubDate, lastUpdated, authors, categories);
|
||||
|
||||
if (!string.IsNullOrEmpty(guid) && !dict.ContainsKey(guid))
|
||||
{
|
||||
dict[guid] = entry;
|
||||
}
|
||||
if (!string.IsNullOrEmpty(link) && !dict.ContainsKey(link))
|
||||
{
|
||||
dict[link] = entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogManager.AddLog($"[警告] 【hitomi-japanese】 既存フィード読み込み失敗 ({fileName}): {ex.Message}");
|
||||
}
|
||||
return dict;
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,69 @@
|
||||
using AngleSharp.Dom;
|
||||
using PuppeteerSharp;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.ServiceModel.Syndication;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml;
|
||||
using rss_generator.Class.Common;
|
||||
|
||||
namespace rss_generator.Class.YahooBaseball;
|
||||
|
||||
/// <summary>
|
||||
/// 広島東洋カープ(スポーツナビ)のニュースタイムラインからRSSフィードを生成するクラス
|
||||
/// </summary>
|
||||
internal class npb_6
|
||||
{
|
||||
/// <summary>
|
||||
/// 広島東洋カープの最新情報を取得し、npb_6.xml として出力します。
|
||||
/// 既存フィードを読み込み、過去の公開日時および本文HTMLを維持することで、FreshRSSにおける不要な未読化を防ぎます。
|
||||
/// </summary>
|
||||
/// <param name="client">HttpClient インスタンス</param>
|
||||
/// <param name="sessionId">FlareSolverr セッションID(任意)</param>
|
||||
public static async Task CreateRSSAsync(HttpClient client, string? sessionId = null)
|
||||
{
|
||||
var feed = new SyndicationFeed("広島東洋カープ - プロ野球 - スポーツナビ", "", new Uri("https://baseball.yahoo.co.jp/npb/teams/6/top"));
|
||||
var feed = new SyndicationFeed("広島東洋カープ - プロ野球 - スポーツナビ", "広島東洋カープの最新ニュース・タイムライン", new Uri("https://baseball.yahoo.co.jp/npb/teams/6/top"));
|
||||
var items = new List<SyndicationItem>();
|
||||
|
||||
try
|
||||
{
|
||||
var pageURL = $"https://baseball.yahoo.co.jp/npb/teams/6/top";
|
||||
var pageURL = "https://baseball.yahoo.co.jp/npb/teams/6/top";
|
||||
sessionId ??= CmnUtil.SharedFlareSolverrSessionId;
|
||||
var document = await CmnUtil.GetDocumentFlareSolverr(client, pageURL, session: sessionId);
|
||||
|
||||
var linkList = new List<string>();
|
||||
// 既存フィードを読み込み、過去の公開日時・更新日時・本文HTMLを引き継ぐ
|
||||
var existingEntries = LoadExistingEntries("npb_6.xml");
|
||||
var mergedDict = new Dictionary<string, SyndicationItem>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// 既存フィードの全アイテムを登録
|
||||
foreach (var kvp in existingEntries)
|
||||
{
|
||||
var entry = kvp.Value;
|
||||
if (!mergedDict.ContainsKey(entry.Guid))
|
||||
{
|
||||
var synItem = new SyndicationItem(
|
||||
entry.Title,
|
||||
(string?)null,
|
||||
!string.IsNullOrEmpty(entry.Link) ? new Uri(entry.Link) : null,
|
||||
entry.Guid,
|
||||
entry.PublishDate != DateTimeOffset.MinValue ? entry.PublishDate : entry.LastUpdatedTime
|
||||
)
|
||||
{
|
||||
PublishDate = entry.PublishDate != DateTimeOffset.MinValue ? entry.PublishDate : entry.LastUpdatedTime,
|
||||
LastUpdatedTime = entry.LastUpdatedTime != DateTimeOffset.MinValue ? entry.LastUpdatedTime : entry.PublishDate,
|
||||
Summary = new TextSyndicationContent(entry.SummaryHtml, TextSyndicationContentKind.Html)
|
||||
};
|
||||
foreach (var author in entry.Authors)
|
||||
{
|
||||
synItem.Authors.Add(new SyndicationPerson { Name = author });
|
||||
}
|
||||
mergedDict[entry.Guid] = synItem;
|
||||
}
|
||||
}
|
||||
|
||||
if (document != null)
|
||||
{
|
||||
var elements = document.QuerySelectorAll("li.sn-timeLine__item");
|
||||
@@ -42,36 +85,121 @@ internal class npb_6
|
||||
|
||||
var itemUri = !string.IsNullOrEmpty(linkHref) ? new Uri(linkHref) : null;
|
||||
var imageUri = !string.IsNullOrEmpty(imageSrc) ? new Uri(imageSrc) : null;
|
||||
string guid = CmnUtil.CreateGuid(linkHref);
|
||||
|
||||
var body = new StringBuilder();
|
||||
body.Append($@"<div><a href=""{itemUri}"" target=""_blank"" rel=""noopener noreferrer"" referrerpolicy=""no-referrer"">{title}</a></div>");
|
||||
body.Append($@"<div><img src=""{imageUri}"" referrerpolicy=""no-referrer"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
|
||||
|
||||
var updateTime = DateTime.Now;
|
||||
if (DateTime.TryParse(elemTime?.TextContent?.Trim(), out DateTime itemTime))
|
||||
if (imageUri != null)
|
||||
{
|
||||
updateTime = itemTime;
|
||||
body.Append($@"<div><img src=""{imageUri}"" referrerpolicy=""no-referrer"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
|
||||
}
|
||||
|
||||
// 既存エントリが存在する場合は前回の公開日時・本文HTMLを最優先で維持
|
||||
DateTimeOffset itemDate;
|
||||
string summaryHtml = body.ToString();
|
||||
|
||||
if (existingEntries.TryGetValue(guid, out var existing))
|
||||
{
|
||||
itemDate = existing.PublishDate != DateTimeOffset.MinValue ? existing.PublishDate : existing.LastUpdatedTime;
|
||||
if (!string.IsNullOrWhiteSpace(existing.SummaryHtml))
|
||||
{
|
||||
summaryHtml = existing.SummaryHtml;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (DateTime.TryParse(elemTime?.TextContent?.Trim(), out DateTime parsedTime))
|
||||
{
|
||||
itemDate = new DateTimeOffset(parsedTime);
|
||||
}
|
||||
else
|
||||
{
|
||||
itemDate = DateTimeOffset.Now;
|
||||
}
|
||||
}
|
||||
|
||||
var item = new SyndicationItem(
|
||||
title,
|
||||
string.Empty,
|
||||
(string?)null,
|
||||
itemUri,
|
||||
CmnUtil.CreateGuid(linkHref),
|
||||
updateTime
|
||||
);
|
||||
item.Authors.Add(new SyndicationPerson() { Name = elemCredit?.TextContent?.Trim() ?? "" });
|
||||
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
|
||||
guid,
|
||||
itemDate
|
||||
)
|
||||
{
|
||||
PublishDate = itemDate,
|
||||
LastUpdatedTime = itemDate,
|
||||
Summary = new TextSyndicationContent(summaryHtml, TextSyndicationContentKind.Html)
|
||||
};
|
||||
|
||||
items.Add(item);
|
||||
var credit = elemCredit?.TextContent?.Trim();
|
||||
if (!string.IsNullOrEmpty(credit))
|
||||
{
|
||||
item.Authors.Add(new SyndicationPerson { Name = credit });
|
||||
}
|
||||
|
||||
mergedDict[guid] = item;
|
||||
}
|
||||
}
|
||||
feed.Items = items;
|
||||
|
||||
await CmnUtil.ExportXMLFileAsync("npb_6.xml", feed, client);
|
||||
// 重複除外とソート(新しい順)
|
||||
var finalItems = mergedDict.Values
|
||||
.OrderByDescending(x => x.PublishDate != DateTimeOffset.MinValue ? x.PublishDate : x.LastUpdatedTime)
|
||||
.Take(100)
|
||||
.ToList();
|
||||
|
||||
feed.Items = finalItems;
|
||||
|
||||
await CmnUtil.ExportXMLFileAsync("npb_6.xml", feed, client, filter24Hours: false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogManager.AddLog($"[エラー] 【npb_6】 {ex.Message}");
|
||||
CmnUtil.PrintCallStackToEvent(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 既存の npb_6.xml を読み込み、GUIDおよびURLをキーとするキャッシュ辞書を返します。
|
||||
/// </summary>
|
||||
private static Dictionary<string, (string Guid, string Link, string Title, string SummaryHtml, DateTimeOffset PublishDate, DateTimeOffset LastUpdatedTime, List<string> Authors)> LoadExistingEntries(string fileName)
|
||||
{
|
||||
var dict = new Dictionary<string, (string Guid, string Link, string Title, string SummaryHtml, DateTimeOffset PublishDate, DateTimeOffset LastUpdatedTime, List<string> Authors)>(StringComparer.OrdinalIgnoreCase);
|
||||
var filePath = CmnUtil.GetExportXmlPath(fileName);
|
||||
if (!File.Exists(filePath)) return dict;
|
||||
|
||||
try
|
||||
{
|
||||
using var reader = XmlReader.Create(filePath);
|
||||
var feed = SyndicationFeed.Load(reader);
|
||||
if (feed?.Items != null)
|
||||
{
|
||||
foreach (var item in feed.Items)
|
||||
{
|
||||
string guid = item.Id ?? string.Empty;
|
||||
string link = item.Links.FirstOrDefault()?.Uri?.ToString() ?? string.Empty;
|
||||
string title = item.Title?.Text ?? string.Empty;
|
||||
string summary = item.Summary?.Text ?? string.Empty;
|
||||
DateTimeOffset pubDate = item.PublishDate;
|
||||
DateTimeOffset lastUpdated = item.LastUpdatedTime;
|
||||
var authors = item.Authors.Select(a => a.Name).Where(n => !string.IsNullOrEmpty(n)).ToList();
|
||||
|
||||
var entry = (guid, link, title, summary, pubDate, lastUpdated, authors);
|
||||
|
||||
if (!string.IsNullOrEmpty(guid) && !dict.ContainsKey(guid))
|
||||
{
|
||||
dict[guid] = entry;
|
||||
}
|
||||
if (!string.IsNullOrEmpty(link) && !dict.ContainsKey(link))
|
||||
{
|
||||
dict[link] = entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogManager.AddLog($"[警告] 【npb_6】 既存フィード読み込み失敗 ({fileName}): {ex.Message}");
|
||||
}
|
||||
return dict;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user