ページ移転対応
javtwiの追加
This commit is contained in:
1 parent
e91f2ffdb4
commit
36c4c02013
2 files changed
+123
-9
No files matched your search
+119
-8
@@ -1,8 +1,10 @@
|
|||||||
using AngleSharp;
|
using AngleSharp;
|
||||||
using AngleSharp.Dom;
|
using AngleSharp.Dom;
|
||||||
|
using AngleSharp.Html.Dom;
|
||||||
using AngleSharp.Html.Parser;
|
using AngleSharp.Html.Parser;
|
||||||
using AngleSharp.Xml.Parser;
|
using AngleSharp.Xml.Parser;
|
||||||
using Microsoft.AspNetCore.Html;
|
using Microsoft.AspNetCore.Html;
|
||||||
|
using Microsoft.AspNetCore.Mvc.RazorPages;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.ServiceModel.Syndication;
|
using System.ServiceModel.Syndication;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
@@ -235,7 +237,7 @@ app.MapGet("/missav", async () =>
|
|||||||
// --- 2. APIエンドポイントの定義 ---
|
// --- 2. APIエンドポイントの定義 ---
|
||||||
app.MapGet("/missav123", async () =>
|
app.MapGet("/missav123", async () =>
|
||||||
{
|
{
|
||||||
var feed = new SyndicationFeed("missav123.to", "", new Uri("https://missav123.to/ja/recent"));
|
var feed = new SyndicationFeed("123av.com", "", new Uri("https://123av.com/ja/recent"));
|
||||||
var items = new List<SyndicationItem>();
|
var items = new List<SyndicationItem>();
|
||||||
|
|
||||||
try
|
try
|
||||||
@@ -243,7 +245,7 @@ app.MapGet("/missav123", async () =>
|
|||||||
var linkList = new List<string>();
|
var linkList = new List<string>();
|
||||||
for (int pageId = 1; pageId <= 10; pageId++)
|
for (int pageId = 1; pageId <= 10; pageId++)
|
||||||
{
|
{
|
||||||
var pageMaster = "https://missav123.to";
|
var pageMaster = "https://123av.com";
|
||||||
var pageURL = string.Empty;
|
var pageURL = string.Empty;
|
||||||
if (pageId > 1)
|
if (pageId > 1)
|
||||||
{
|
{
|
||||||
@@ -258,13 +260,12 @@ app.MapGet("/missav123", async () =>
|
|||||||
// --- 3. HTMLの取得と解析 ---
|
// --- 3. HTMLの取得と解析 ---
|
||||||
var document = await context.OpenAsync(rssUrl);
|
var document = await context.OpenAsync(rssUrl);
|
||||||
var pageTitle = document.QuerySelector("title")?.TextContent;
|
var pageTitle = document.QuerySelector("title")?.TextContent;
|
||||||
|
var elements = document.QuerySelectorAll("div.card");
|
||||||
var elements = document.QuerySelectorAll("div.item");
|
|
||||||
var targetDivs = elements.Where(e => e.ClassList.Length == 1).ToList();
|
var targetDivs = elements.Where(e => e.ClassList.Length == 1).ToList();
|
||||||
foreach (var div in targetDivs)
|
foreach (var div in targetDivs)
|
||||||
{
|
{
|
||||||
var titleElement = div.QuerySelector("a.title");
|
var titleElement = div.QuerySelector("a.card__link");
|
||||||
var linkElement = div.QuerySelector("a.title");
|
var linkElement = div.QuerySelector("a.card__link");
|
||||||
var imgElement = div.QuerySelector("img");
|
var imgElement = div.QuerySelector("img");
|
||||||
|
|
||||||
var title = titleElement?.TextContent?.Trim() ?? "タイトルなし";
|
var title = titleElement?.TextContent?.Trim() ?? "タイトルなし";
|
||||||
@@ -286,8 +287,6 @@ app.MapGet("/missav123", async () =>
|
|||||||
body.Append($@"<div><a href=""{itemUri}"">{title}</a></div>");
|
body.Append($@"<div><a href=""{itemUri}"">{title}</a></div>");
|
||||||
body.Append($@"<div><img src=""{imageUri}"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
|
body.Append($@"<div><img src=""{imageUri}"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
|
||||||
|
|
||||||
Debug.WriteLine(body.ToString());
|
|
||||||
|
|
||||||
var item = new SyndicationItem(
|
var item = new SyndicationItem(
|
||||||
title,
|
title,
|
||||||
string.Empty,
|
string.Empty,
|
||||||
@@ -706,4 +705,116 @@ app.MapGet("/momonga/magazine", async (HttpContext context, IHttpClientFactory h
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// --- 2. APIエンドポイントの定義 ---
|
||||||
|
app.MapGet("/javtwi", async () =>
|
||||||
|
{
|
||||||
|
var baseURL = "https://javtwi.com/new.html";
|
||||||
|
|
||||||
|
|
||||||
|
var feed = new SyndicationFeed("javtwi", "", new Uri(baseURL));
|
||||||
|
var items = new List<SyndicationItem>();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var linkList = new List<string>();
|
||||||
|
|
||||||
|
// --- 3. HTMLの取得と解析 ---
|
||||||
|
var document = await context.OpenAsync(baseURL);
|
||||||
|
var pageTitle = document.QuerySelector("title")?.TextContent;
|
||||||
|
|
||||||
|
var elements = document.QuerySelectorAll("div.center_contents a.center_contents");
|
||||||
|
foreach (var linkElement in elements)
|
||||||
|
{
|
||||||
|
var imgElement = linkElement.QuerySelector("a img");
|
||||||
|
|
||||||
|
var linkHref = linkElement?.GetAttribute("href") ?? string.Empty;
|
||||||
|
var imageSrc = (imgElement as IHtmlImageElement)!.Source;
|
||||||
|
if (linkHref != string.Empty && !linkList.Contains(linkHref))
|
||||||
|
{
|
||||||
|
linkList.Add(linkHref);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var itemUri = !string.IsNullOrEmpty(linkHref) ? new Uri(linkHref) : null;
|
||||||
|
var imageUri = !string.IsNullOrEmpty(imageSrc) ? new Uri(imageSrc) : null;
|
||||||
|
if (itemUri == null)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (itemUri.AbsolutePath.Contains("404."))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var title = cmnUtil.GetLastPathPart(linkHref);
|
||||||
|
|
||||||
|
var body = new StringBuilder();
|
||||||
|
body.Append($@"<div><a href=""{itemUri}"">{title}</a></div>");
|
||||||
|
body.Append($@"<div><img src=""{imageUri}"" alt=""{title}"" style=""max-width: 100%; height: auto;"" /></div>");
|
||||||
|
|
||||||
|
var item = new SyndicationItem(
|
||||||
|
title,
|
||||||
|
string.Empty,
|
||||||
|
itemUri,
|
||||||
|
itemUri?.ToString() ?? Guid.NewGuid().ToString(), // 一意なID (URLを使うことが多い)
|
||||||
|
DateTimeOffset.Now // 取得できた公開日時があればそれをセット
|
||||||
|
);
|
||||||
|
item.Content = SyndicationContent.CreateHtmlContent(body.ToString());
|
||||||
|
// ※ RSS 2.0の場合は Content よりも Summary(<description>タグ)が使われることが多いため、
|
||||||
|
// 両方に設定しておくか、Summary側にHTMLを設定すると大半のリーダーで確実です。
|
||||||
|
item.Summary = new TextSyndicationContent(body.ToString(), TextSyndicationContentKind.Html);
|
||||||
|
|
||||||
|
items.Add(item);
|
||||||
|
}
|
||||||
|
feed.Items = items;
|
||||||
|
|
||||||
|
// 3. XMLとしてMemoryStreamに書き込み
|
||||||
|
using var memoryStream = new MemoryStream();
|
||||||
|
// 文字コードをUTF-8に指定(重要)
|
||||||
|
var settings = new XmlWriterSettings
|
||||||
|
{
|
||||||
|
Encoding = new UTF8Encoding(false), // BOMなしUTF-8
|
||||||
|
Indent = true
|
||||||
|
};
|
||||||
|
using (var xmlWriter = XmlWriter.Create(memoryStream, settings))
|
||||||
|
{
|
||||||
|
// RSS 2.0 形式で出力
|
||||||
|
var formatter = new Rss20FeedFormatter(feed);
|
||||||
|
// ※ Atom形式にしたい場合は、上の1行を以下に変えるだけです。
|
||||||
|
// var formatter = new Atom10FeedFormatter(feed);
|
||||||
|
formatter.WriteTo(xmlWriter);
|
||||||
|
}
|
||||||
|
// 4. クライアントに返却
|
||||||
|
// Atomに変更した場合は contentType を "application/atom+xml; charset=utf-8" にしてください
|
||||||
|
return Results.File(
|
||||||
|
memoryStream.ToArray(),
|
||||||
|
contentType: "application/rss+xml; charset=utf-8"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
return Results.Problem(detail: ex.Message, statusCode: 500, title: "Scraping Error");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
app.Run();
|
app.Run();
|
||||||
|
|
||||||
|
public class cmnUtil
|
||||||
|
{
|
||||||
|
public static string GetLastPathPart(string urlStr)
|
||||||
|
{
|
||||||
|
if (Uri.TryCreate(urlStr, UriKind.Absolute, out Uri? uri))
|
||||||
|
{
|
||||||
|
// uri.AbsolutePath は "/path/to/page.html" を返す
|
||||||
|
string absolutePath = uri!.AbsolutePath;
|
||||||
|
|
||||||
|
// 末尾のスラッシュを除去した上で、スラッシュで分割して最後の要素を取得
|
||||||
|
string lastPart = absolutePath.TrimEnd('/').Split('/').LastOrDefault()!;
|
||||||
|
return lastPart ?? string.Empty;
|
||||||
|
}
|
||||||
|
return string.Empty;
|
||||||
|
}
|
||||||
|
}
|
||||||
+4
-1
@@ -1,6 +1,9 @@
|
|||||||
@RSSAPI_HostAddress = http://localhost:5185
|
@RSSAPI_HostAddress = http://localhost:5185
|
||||||
|
|
||||||
GET {{RSSAPI_HostAddress}}/momonga/fanzine
|
# GET {{RSSAPI_HostAddress}}/momonga/fanzine
|
||||||
|
# Accept: application/json
|
||||||
|
|
||||||
|
GET {{RSSAPI_HostAddress}}/missav123
|
||||||
Accept: application/json
|
Accept: application/json
|
||||||
|
|
||||||
###
|
###
|
||||||
Reference in new issue
Block a user