feat: movie.eroterest.net の記事公開日および元動画公開日の取得とRSSへの反映
This commit is contained in:
1 parent
740c57624c
commit
0e095c8fa3
3 files changed
+193
-27
No files matched your search
@@ -55,4 +55,9 @@ internal static class MovieEroterestConfig
|
|||||||
/// 記事詳細およびブログ取得時の並列度
|
/// 記事詳細およびブログ取得時の並列度
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public const int MaxParallelTasks = 3;
|
public const int MaxParallelTasks = 3;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// RSS の pubDate に元動画の公開日を優先するかどうか(false の場合は記事の実際の公開日を優先し、元動画公開日は本文に記載)
|
||||||
|
/// </summary>
|
||||||
|
public const bool PreferOriginalVideoDateOverArticleDate = false;
|
||||||
}
|
}
|
||||||
@@ -127,8 +127,9 @@ internal static class MovieEroterestFeedManager
|
|||||||
new Uri(pageUrl)
|
new Uri(pageUrl)
|
||||||
);
|
);
|
||||||
|
|
||||||
var preliminaryItems = new List<(string ArticleUrl, string Title, string? Thumbnail, string? Duration, string? Provider, string? SiteName)>();
|
var preliminaryItems = new List<(string ArticleUrl, string Title, string? Thumbnail, string? Duration, string? Provider, string? SiteName, string? ItemTime, int Order)>();
|
||||||
var seenUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
var seenUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
int orderIndex = 0;
|
||||||
|
|
||||||
foreach (var itemEl in itemElements)
|
foreach (var itemEl in itemElements)
|
||||||
{
|
{
|
||||||
@@ -170,7 +171,10 @@ internal static class MovieEroterestFeedManager
|
|||||||
// まとめブログ名
|
// まとめブログ名
|
||||||
var siteName = itemEl.QuerySelector(".itemSiteName")?.TextContent?.Trim();
|
var siteName = itemEl.QuerySelector(".itemSiteName")?.TextContent?.Trim();
|
||||||
|
|
||||||
preliminaryItems.Add((fullUrl, title, thumbSrc, duration, provider, siteName));
|
// 記事公開時間(相対時間表記: 例 "4分前", "1時間前")
|
||||||
|
var itemTime = itemEl.QuerySelector(".itemTime")?.TextContent?.Trim();
|
||||||
|
|
||||||
|
preliminaryItems.Add((fullUrl, title, thumbSrc, duration, provider, siteName, itemTime, orderIndex++));
|
||||||
|
|
||||||
if (preliminaryItems.Count >= MovieEroterestConfig.MaxArticlesPerFeed)
|
if (preliminaryItems.Count >= MovieEroterestConfig.MaxArticlesPerFeed)
|
||||||
{
|
{
|
||||||
@@ -178,10 +182,12 @@ internal static class MovieEroterestFeedManager
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 基準時刻
|
||||||
|
var baseTime = DateTimeOffset.Now;
|
||||||
|
|
||||||
// 同時並行数を制御して各記事の詳細と動画探索を実行
|
// 同時並行数を制御して各記事の詳細と動画探索を実行
|
||||||
using var semaphore = new SemaphoreSlim(MovieEroterestConfig.MaxParallelTasks);
|
using var semaphore = new SemaphoreSlim(MovieEroterestConfig.MaxParallelTasks);
|
||||||
var syndicationItems = new List<SyndicationItem>();
|
var syndicationEntries = new List<(SyndicationItem Item, int Order)>();
|
||||||
int itemOrder = 0;
|
|
||||||
|
|
||||||
var tasks = preliminaryItems.Select(async item =>
|
var tasks = preliminaryItems.Select(async item =>
|
||||||
{
|
{
|
||||||
@@ -195,17 +201,16 @@ internal static class MovieEroterestFeedManager
|
|||||||
item.Thumbnail,
|
item.Thumbnail,
|
||||||
item.Duration,
|
item.Duration,
|
||||||
item.Provider,
|
item.Provider,
|
||||||
item.SiteName);
|
item.SiteName,
|
||||||
|
item.ItemTime,
|
||||||
|
baseTime);
|
||||||
|
|
||||||
if (articleInfo != null)
|
if (articleInfo != null)
|
||||||
{
|
{
|
||||||
// 新着順を保持するため秒単位で降順設定
|
|
||||||
articleInfo.PublishDate = DateTimeOffset.Now.AddSeconds(-Interlocked.Increment(ref itemOrder));
|
|
||||||
|
|
||||||
var synItem = MovieEroterestVideoParser.CreateSyndicationItem(articleInfo);
|
var synItem = MovieEroterestVideoParser.CreateSyndicationItem(articleInfo);
|
||||||
lock (syndicationItems)
|
lock (syndicationEntries)
|
||||||
{
|
{
|
||||||
syndicationItems.Add(synItem);
|
syndicationEntries.Add((synItem, item.Order));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -221,8 +226,12 @@ internal static class MovieEroterestFeedManager
|
|||||||
|
|
||||||
await Task.WhenAll(tasks);
|
await Task.WhenAll(tasks);
|
||||||
|
|
||||||
// 新着順にソート
|
// 公開日時順(同日時の場合は一覧の掲載順を維持)にソート
|
||||||
feed.Items = syndicationItems.OrderByDescending(i => i.PublishDate).ToList();
|
feed.Items = syndicationEntries
|
||||||
|
.OrderByDescending(e => e.Item.PublishDate)
|
||||||
|
.ThenBy(e => e.Order)
|
||||||
|
.Select(e => e.Item)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
// XML ファイルへ出力 (filter24Hours: false で直近取得分を確実に出力)
|
// XML ファイルへ出力 (filter24Hours: false で直近取得分を確実に出力)
|
||||||
await CmnUtil.ExportXMLFileAsync(outputFileName, feed, client, addDir: subDir, embedBase64: false, filter24Hours: false);
|
await CmnUtil.ExportXMLFileAsync(outputFileName, feed, client, addDir: subDir, embedBase64: false, filter24Hours: false);
|
||||||
|
|||||||
@@ -34,6 +34,11 @@ internal static class MovieEroterestVideoParser
|
|||||||
public List<string> Actresses { get; set; } = new();
|
public List<string> Actresses { get; set; } = new();
|
||||||
public DateTimeOffset PublishDate { get; set; } = DateTimeOffset.Now;
|
public DateTimeOffset PublishDate { get; set; } = DateTimeOffset.Now;
|
||||||
|
|
||||||
|
// 日付・公開日情報
|
||||||
|
public string? RelativeTimeText { get; set; }
|
||||||
|
public DateTimeOffset? ArticlePublishDate { get; set; }
|
||||||
|
public DateTimeOffset? VideoPublishDate { get; set; }
|
||||||
|
|
||||||
// 探索結果
|
// 探索結果
|
||||||
public string? DestinationBlogUrl { get; set; }
|
public string? DestinationBlogUrl { get; set; }
|
||||||
public string? DestinationBlogSiteName { get; set; }
|
public string? DestinationBlogSiteName { get; set; }
|
||||||
@@ -55,6 +60,8 @@ internal static class MovieEroterestVideoParser
|
|||||||
string? preliminaryDuration = null,
|
string? preliminaryDuration = null,
|
||||||
string? preliminaryProvider = null,
|
string? preliminaryProvider = null,
|
||||||
string? preliminarySiteName = null,
|
string? preliminarySiteName = null,
|
||||||
|
string? preliminaryItemTime = null,
|
||||||
|
DateTimeOffset? referenceTime = null,
|
||||||
CancellationToken ct = default)
|
CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
var info = new EroterestArticleInfo
|
var info = new EroterestArticleInfo
|
||||||
@@ -64,9 +71,16 @@ internal static class MovieEroterestVideoParser
|
|||||||
ThumbnailUrl = preliminaryThumbnail,
|
ThumbnailUrl = preliminaryThumbnail,
|
||||||
Duration = preliminaryDuration,
|
Duration = preliminaryDuration,
|
||||||
Provider = preliminaryProvider,
|
Provider = preliminaryProvider,
|
||||||
DestinationBlogSiteName = preliminarySiteName
|
DestinationBlogSiteName = preliminarySiteName,
|
||||||
|
RelativeTimeText = preliminaryItemTime
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// 一覧から渡された相対時間(例: "4分前")から記事公開日を算出
|
||||||
|
if (!string.IsNullOrWhiteSpace(preliminaryItemTime))
|
||||||
|
{
|
||||||
|
info.ArticlePublishDate = ParseRelativeOrAbsoluteDate(preliminaryItemTime, referenceTime);
|
||||||
|
}
|
||||||
|
|
||||||
// 記事IDの抽出 (例: /page/44918986/)
|
// 記事IDの抽出 (例: /page/44918986/)
|
||||||
var idMatch = Regex.Match(articleUrl, @"/page/([0-9]+)");
|
var idMatch = Regex.Match(articleUrl, @"/page/([0-9]+)");
|
||||||
if (idMatch.Success)
|
if (idMatch.Success)
|
||||||
@@ -129,6 +143,17 @@ internal static class MovieEroterestVideoParser
|
|||||||
info.ClickCount = clickEl;
|
info.ClickCount = clickEl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 記事公開時間(詳細ページからフォールバック取得)
|
||||||
|
if (!info.ArticlePublishDate.HasValue)
|
||||||
|
{
|
||||||
|
var detailTime = articleDoc.QuerySelector(".itemFoot .itemTime, .itemTime")?.TextContent?.Trim();
|
||||||
|
if (!string.IsNullOrWhiteSpace(detailTime))
|
||||||
|
{
|
||||||
|
info.RelativeTimeText ??= detailTime;
|
||||||
|
info.ArticlePublishDate = ParseRelativeOrAbsoluteDate(detailTime, referenceTime);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// タグ
|
// タグ
|
||||||
var tagElements = articleDoc.QuerySelectorAll(".itemTag a");
|
var tagElements = articleDoc.QuerySelectorAll(".itemTag a");
|
||||||
foreach (var t in tagElements)
|
foreach (var t in tagElements)
|
||||||
@@ -200,6 +225,9 @@ internal static class MovieEroterestVideoParser
|
|||||||
// 3. WebImageViewer の再生URLを決定
|
// 3. WebImageViewer の再生URLを決定
|
||||||
BuildViewerUrl(info);
|
BuildViewerUrl(info);
|
||||||
|
|
||||||
|
// 4. RSS アイテムの日付を決定(設定・優先順位に基づく)
|
||||||
|
ResolvePublishDate(info);
|
||||||
|
|
||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -401,32 +429,56 @@ internal static class MovieEroterestVideoParser
|
|||||||
var content = script.TextContent;
|
var content = script.TextContent;
|
||||||
if (string.IsNullOrWhiteSpace(content)) continue;
|
if (string.IsNullOrWhiteSpace(content)) continue;
|
||||||
|
|
||||||
if (content.Contains("contentUrl", StringComparison.OrdinalIgnoreCase))
|
try
|
||||||
{
|
{
|
||||||
try
|
using var doc = JsonDocument.Parse(content);
|
||||||
|
var root = doc.RootElement;
|
||||||
|
|
||||||
|
// contentUrl (h264.m3u8)
|
||||||
|
if (string.IsNullOrWhiteSpace(info.StreamUrl) && root.TryGetProperty("contentUrl", out var contentUrlProp))
|
||||||
{
|
{
|
||||||
using var doc = JsonDocument.Parse(content);
|
var cUrl = contentUrlProp.GetString();
|
||||||
var root = doc.RootElement;
|
if (!string.IsNullOrWhiteSpace(cUrl))
|
||||||
if (root.TryGetProperty("contentUrl", out var contentUrlProp))
|
|
||||||
{
|
{
|
||||||
var cUrl = contentUrlProp.GetString();
|
info.StreamUrl = cUrl.Trim();
|
||||||
if (!string.IsNullOrWhiteSpace(cUrl))
|
|
||||||
{
|
|
||||||
info.StreamUrl = cUrl.Trim();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch
|
|
||||||
|
// uploadDate (元動画の公開日)
|
||||||
|
if (!info.VideoPublishDate.HasValue && root.TryGetProperty("uploadDate", out var uploadDateProp))
|
||||||
|
{
|
||||||
|
var uDateStr = uploadDateProp.GetString();
|
||||||
|
if (!string.IsNullOrWhiteSpace(uDateStr) && DateTimeOffset.TryParse(uDateStr, out var parsedUploadDate))
|
||||||
|
{
|
||||||
|
info.VideoPublishDate = parsedUploadDate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// JSONパースに失敗した場合は正規表現でフォールバック
|
||||||
|
if (string.IsNullOrWhiteSpace(info.StreamUrl))
|
||||||
{
|
{
|
||||||
// JSONパースに失敗した場合は正規表現でフォールバック
|
|
||||||
var match = Regex.Match(content, @"""contentUrl""\s*:\s*""([^""]+)""");
|
var match = Regex.Match(content, @"""contentUrl""\s*:\s*""([^""]+)""");
|
||||||
if (match.Success)
|
if (match.Success)
|
||||||
{
|
{
|
||||||
info.StreamUrl = match.Groups[1].Value.Trim();
|
info.StreamUrl = match.Groups[1].Value.Trim();
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!info.VideoPublishDate.HasValue)
|
||||||
|
{
|
||||||
|
var dateMatch = Regex.Match(content, @"""uploadDate""\s*:\s*""([^""]+)""");
|
||||||
|
if (dateMatch.Success && DateTimeOffset.TryParse(dateMatch.Groups[1].Value, out var parsedDate))
|
||||||
|
{
|
||||||
|
info.VideoPublishDate = parsedDate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(info.StreamUrl) && info.VideoPublishDate.HasValue)
|
||||||
|
{
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -562,6 +614,15 @@ internal static class MovieEroterestVideoParser
|
|||||||
|
|
||||||
// 4. メタ情報リスト
|
// 4. メタ情報リスト
|
||||||
sb.Append("<ul style=\"font-size: 0.9em; color: #475569; list-style-type: none; padding-left: 0; margin: 8px 0; line-height: 1.6;\">");
|
sb.Append("<ul style=\"font-size: 0.9em; color: #475569; list-style-type: none; padding-left: 0; margin: 8px 0; line-height: 1.6;\">");
|
||||||
|
if (info.ArticlePublishDate.HasValue)
|
||||||
|
{
|
||||||
|
var relSuffix = !string.IsNullOrWhiteSpace(info.RelativeTimeText) ? $" ({WebUtility.HtmlEncode(info.RelativeTimeText)})" : "";
|
||||||
|
sb.Append($"<li>📅 <strong>記事公開日:</strong> {info.ArticlePublishDate.Value:yyyy/MM/dd HH:mm}{relSuffix}</li>");
|
||||||
|
}
|
||||||
|
if (info.VideoPublishDate.HasValue)
|
||||||
|
{
|
||||||
|
sb.Append($"<li>🎬 <strong>元動画公開日:</strong> {info.VideoPublishDate.Value:yyyy/MM/dd HH:mm}</li>");
|
||||||
|
}
|
||||||
if (!string.IsNullOrWhiteSpace(info.Provider))
|
if (!string.IsNullOrWhiteSpace(info.Provider))
|
||||||
{
|
{
|
||||||
sb.Append($"<li>🏷 <strong>配信元・プロバイダ:</strong> {WebUtility.HtmlEncode(info.Provider)}</li>");
|
sb.Append($"<li>🏷 <strong>配信元・プロバイダ:</strong> {WebUtility.HtmlEncode(info.Provider)}</li>");
|
||||||
@@ -662,4 +723,95 @@ internal static class MovieEroterestVideoParser
|
|||||||
}
|
}
|
||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 設定および優先順位に基づいて RSS アイテムの公開日 (PublishDate) を決定します。
|
||||||
|
/// </summary>
|
||||||
|
public static void ResolvePublishDate(EroterestArticleInfo info)
|
||||||
|
{
|
||||||
|
if (MovieEroterestConfig.PreferOriginalVideoDateOverArticleDate && info.VideoPublishDate.HasValue)
|
||||||
|
{
|
||||||
|
info.PublishDate = info.VideoPublishDate.Value;
|
||||||
|
}
|
||||||
|
else if (info.ArticlePublishDate.HasValue)
|
||||||
|
{
|
||||||
|
info.PublishDate = info.ArticlePublishDate.Value;
|
||||||
|
}
|
||||||
|
else if (info.VideoPublishDate.HasValue)
|
||||||
|
{
|
||||||
|
info.PublishDate = info.VideoPublishDate.Value;
|
||||||
|
}
|
||||||
|
// フォールバック: 初期値 (DateTimeOffset.Now)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 相対時間表記(例: "4秒前", "4分前", "11時間前", "1日前", "2か月前", "1年前")または絶対日付文字列から DateTimeOffset を算出します。
|
||||||
|
/// </summary>
|
||||||
|
public static DateTimeOffset? ParseRelativeOrAbsoluteDate(string? dateText, DateTimeOffset? referenceTime = null)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(dateText)) return null;
|
||||||
|
|
||||||
|
var text = dateText.Trim();
|
||||||
|
var now = referenceTime ?? DateTimeOffset.Now;
|
||||||
|
|
||||||
|
// 1. 秒前
|
||||||
|
var secMatch = Regex.Match(text, @"^(\d+)\s*秒前");
|
||||||
|
if (secMatch.Success && int.TryParse(secMatch.Groups[1].Value, out int sec))
|
||||||
|
{
|
||||||
|
return now.AddSeconds(-sec);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. 分前
|
||||||
|
var minMatch = Regex.Match(text, @"^(\d+)\s*分前");
|
||||||
|
if (minMatch.Success && int.TryParse(minMatch.Groups[1].Value, out int min))
|
||||||
|
{
|
||||||
|
return now.AddMinutes(-min);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. 時間前
|
||||||
|
var hourMatch = Regex.Match(text, @"^(\d+)\s*時間前");
|
||||||
|
if (hourMatch.Success && int.TryParse(hourMatch.Groups[1].Value, out int hour))
|
||||||
|
{
|
||||||
|
return now.AddHours(-hour);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. 日前
|
||||||
|
var dayMatch = Regex.Match(text, @"^(\d+)\s*日前");
|
||||||
|
if (dayMatch.Success && int.TryParse(dayMatch.Groups[1].Value, out int day))
|
||||||
|
{
|
||||||
|
return now.AddDays(-day);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. か月前 / ヶ月前 / カ月前
|
||||||
|
var monthMatch = Regex.Match(text, @"^(\d+)\s*(?:か月|ヶ月|カ月)前");
|
||||||
|
if (monthMatch.Success && int.TryParse(monthMatch.Groups[1].Value, out int month))
|
||||||
|
{
|
||||||
|
return now.AddMonths(-month);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. 年前
|
||||||
|
var yearMatch = Regex.Match(text, @"^(\d+)\s*年前");
|
||||||
|
if (yearMatch.Success && int.TryParse(yearMatch.Groups[1].Value, out int year))
|
||||||
|
{
|
||||||
|
return now.AddYears(-year);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 7. 昨日・一昨日
|
||||||
|
if (text.StartsWith("昨日"))
|
||||||
|
{
|
||||||
|
return now.AddDays(-1);
|
||||||
|
}
|
||||||
|
if (text.StartsWith("一昨日"))
|
||||||
|
{
|
||||||
|
return now.AddDays(-2);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 8. ISO 8601 または標準日時形式 (YYYY-MM-DD, YYYY/MM/DD 等)
|
||||||
|
if (DateTimeOffset.TryParse(text, out var parsedDto))
|
||||||
|
{
|
||||||
|
return parsedDto;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user