From 900f643da0a02337ecc8eb64ea267df51981e6ed Mon Sep 17 00:00:00 2001 From: uduki Date: Tue, 18 Aug 2026 08:53:06 +0900 Subject: [PATCH] =?UTF-8?q?2026-08-18=20=E6=99=82=E7=82=B9=E3=81=AE?= =?UTF-8?q?=E3=82=BD=E3=83=BC=E3=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .editorconfig | 7 + .gitignore | 7 + rss-generator.sln | 30 ++ rss-generator/Class/CmnDB.cs | 90 ++++ rss-generator/Class/CmnDefine.cs | 90 ++++ rss-generator/Class/CmnUtil.cs | 469 ++++++++++++++++++ rss-generator/Class/LogManager.cs | 101 ++++ rss-generator/Class/RSS/X/x_list.cs | 89 ++++ rss-generator/Class/RSS/Youtube.cs | 104 ++++ rss-generator/Class/RSS/ci_en.cs | 97 ++++ .../Class/RSS/hitomi_la/index_japanese.cs | 174 +++++++ rss-generator/Class/RSS/javtwi.cs | 94 ++++ rss-generator/Class/RSS/kimochiinfo.cs | 125 +++++ rss-generator/Class/RSS/missav.cs | 90 ++++ rss-generator/Class/RSS/missav123.cs | 93 ++++ rss-generator/Class/RSS/momonga_fanzine.cs | 188 +++++++ rss-generator/Class/RSS/momonga_magazine.cs | 168 +++++++ rss-generator/Class/RSS/npb_6.cs | 75 +++ rss-generator/Program.cs | 99 ++++ .../PublishProfiles/FolderProfile.pubxml | 11 + rss-generator/rss-generator.csproj | 30 ++ 21 files changed, 2231 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 rss-generator.sln create mode 100644 rss-generator/Class/CmnDB.cs create mode 100644 rss-generator/Class/CmnDefine.cs create mode 100644 rss-generator/Class/CmnUtil.cs create mode 100644 rss-generator/Class/LogManager.cs create mode 100644 rss-generator/Class/RSS/X/x_list.cs create mode 100644 rss-generator/Class/RSS/Youtube.cs create mode 100644 rss-generator/Class/RSS/ci_en.cs create mode 100644 rss-generator/Class/RSS/hitomi_la/index_japanese.cs create mode 100644 rss-generator/Class/RSS/javtwi.cs create mode 100644 rss-generator/Class/RSS/kimochiinfo.cs create mode 100644 rss-generator/Class/RSS/missav.cs create mode 100644 rss-generator/Class/RSS/missav123.cs create mode 100644 rss-generator/Class/RSS/momonga_fanzine.cs create mode 100644 rss-generator/Class/RSS/momonga_magazine.cs create mode 100644 rss-generator/Class/RSS/npb_6.cs create mode 100644 rss-generator/Program.cs create mode 100644 rss-generator/Properties/PublishProfiles/FolderProfile.pubxml create mode 100644 rss-generator/rss-generator.csproj diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..5503bfa --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +[*.cs] + +# IDE1006: 命名スタイル +dotnet_diagnostic.IDE1006.severity = none + +# CS8981: 型名には、小文字の ASCII 文字のみが含まれています。このような名前は、プログラミング言語用に予約されている可能性があります。 +dotnet_diagnostic.CS8981.severity = none diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dd87710 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +################################################################################ +# この .gitignore ファイルは Microsoft(R) Visual Studio によって自動的に作成されました。 +################################################################################ + +/.vs +/rss-generator/bin +/rss-generator/obj diff --git a/rss-generator.sln b/rss-generator.sln new file mode 100644 index 0000000..918a095 --- /dev/null +++ b/rss-generator.sln @@ -0,0 +1,30 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 18 +VisualStudioVersion = 18.7.11925.98 stable +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "rss-generator", "rss-generator\rss-generator.csproj", "{6F617B69-AC7A-4413-873A-590B3D274842}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{291B6CE0-3738-4248-AAE7-82A0606D5DE6}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {6F617B69-AC7A-4413-873A-590B3D274842}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6F617B69-AC7A-4413-873A-590B3D274842}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6F617B69-AC7A-4413-873A-590B3D274842}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6F617B69-AC7A-4413-873A-590B3D274842}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {6E2A6FC2-F51E-41EE-8EFE-24996723C902} + EndGlobalSection +EndGlobal diff --git a/rss-generator/Class/CmnDB.cs b/rss-generator/Class/CmnDB.cs new file mode 100644 index 0000000..8d63c01 --- /dev/null +++ b/rss-generator/Class/CmnDB.cs @@ -0,0 +1,90 @@ +using Dapper; +using Npgsql; +#if DEBUG +using Renci.SshNet; +#endif + +namespace rss_generator.Class; + +internal class CmnDB : IDisposable +{ + private readonly string _connectionString; + +#if DEBUG + private SshClient _sshClient; + private ForwardedPortLocal _forwardedPort; +#endif + + public CmnDB() + { + // データベースの接続情報 + string dbName = "freshrss"; + string dbUser = "armonica"; + string dbPass = "mashiro0403"; + +#if DEBUG + // ========================================== + // 【Debug環境】SSHトンネル(秘密鍵認証)を構築して接続 + // ========================================== + + // SSH接続情報 + string sshHost = "162.43.79.113"; + string sshUser = "armonica"; + string keyFilePath = @"C:\Users\uduki\Documents\_MEGA同期\VPS認証キー\シンVPS\armonica.pem"; + + // トンネリングの設定 + string dbHostInVps = "127.0.0.1"; + uint dbPortInVps = 5432; + uint localPort = 15432; + + // 秘密鍵ファイルを用いた認証メソッドを作成 + var keyFile = new PrivateKeyFile(keyFilePath); + var authMethod = new PrivateKeyAuthenticationMethod(sshUser, keyFile); + var connectionInfo = new ConnectionInfo(sshHost, sshUser, authMethod); + + _sshClient = new SshClient(connectionInfo); + _sshClient.Connect(); + + _forwardedPort = new ForwardedPortLocal("127.0.0.1", localPort, dbHostInVps, dbPortInVps); + _sshClient.AddForwardedPort(_forwardedPort); + _forwardedPort.Start(); + + // ローカルに構築したSSHトンネルのポートへ繋ぐ + _connectionString = $"Host=127.0.0.1;Port={localPort};Database={dbName};Username={dbUser};Password={dbPass}"; +#else + // ========================================== + // 【Release環境】VPS内部のネットワークで直接接続 + // ========================================== + // VPS上からDockerコンテナ(postgres-common)へ繋ぐため、127.0.0.1の5432ポートを指定 + _connectionString = $"Host=postgres;Port=5432;Database={dbName};Username={dbUser};Password={dbPass}"; +#endif + } + + public async Task> QueryAsync(string sql, object? parameters = null) + { + await using var connection = new NpgsqlConnection(_connectionString); + return await connection.QueryAsync(sql, parameters); + } + + public async Task ExecuteAsync(string sql, object? parameters = null) + { + await using var connection = new NpgsqlConnection(_connectionString); + return await connection.ExecuteAsync(sql, parameters); + } + + public void Dispose() + { +#if DEBUG + if (_forwardedPort != null && _forwardedPort.IsStarted) + { + _forwardedPort.Stop(); + _forwardedPort.Dispose(); + } + if (_sshClient != null && _sshClient.IsConnected) + { + _sshClient.Disconnect(); + _sshClient.Dispose(); + } +#endif + } +} diff --git a/rss-generator/Class/CmnDefine.cs b/rss-generator/Class/CmnDefine.cs new file mode 100644 index 0000000..1b0b546 --- /dev/null +++ b/rss-generator/Class/CmnDefine.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace rss_generator.Class; + +internal class CmnDefine +{ + public class ArmonicaFeed + { + public int Id { get; set; } + + public string Url { get; set; } = string.Empty; + + // smallintはC#のshort(Int16)に対応 + public short Kind { get; set; } + + public int Category { get; set; } + + public string Name { get; set; } = string.Empty; + + // NULL許容型(Nullable) + public string? Website { get; set; } + + public string? Description { get; set; } + + // bigintはC#のlong(Int64)に対応 + // SQL側がダブルクォーテーションで囲まれているため、 + // Dapperの自動マッピングを効かせるためにプロパティ名も大文字小文字を合わせます + public long LastUpdate { get; set; } + + public short Priority { get; set; } + + public string? PathEntries { get; set; } + + public string? HttpAuth { get; set; } + + public short Error { get; set; } + + public int Ttl { get; set; } + + public string? Attributes { get; set; } + + public int CacheNbEntries { get; set; } + + public int CacheNbUnreads { get; set; } + } + + public class ArmonicaEntry + { + // bigintはC#のlong(Int64)に対応 + public long Id { get; set; } + + // not nullカラムは string.Empty で初期化 + public string Guid { get; set; } = string.Empty; + + public string Title { get; set; } = string.Empty; + + // NULL許容カラムは string? (Nullable) + public string? Author { get; set; } + + public string? Content { get; set; } + + public string LinK { get; set; } = string.Empty; + + // NULL許容のbigintは long? に対応 + public long? Date { get; set; } + + public long LastSeen { get; set; } + + public long LastUserModified { get; set; } + + // bytea(バイナリデータ)は byte[] にマッピングされます + public byte[]? Hash { get; set; } + + // smallintは short(Int16) に対応 + public short IsRead { get; set; } + + public short IsFavorite { get; set; } + + // NULL許容のintegerは int? に対応 + public int? IdFeed { get; set; } + + public string? Tags { get; set; } + + public string? Attributes { get; set; } + + public long? LastModified { get; set; } + } +} diff --git a/rss-generator/Class/CmnUtil.cs b/rss-generator/Class/CmnUtil.cs new file mode 100644 index 0000000..d038fe1 --- /dev/null +++ b/rss-generator/Class/CmnUtil.cs @@ -0,0 +1,469 @@ +using AngleSharp; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; +using PuppeteerSharp; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Net; +using System.Security.Cryptography; +using System.ServiceModel.Syndication; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; + +namespace rss_generator.Class; + +internal class CmnUtil +{ + /// + /// XMLファイルの出力先パスを取得 + /// + /// + /// + public static string GetExportXmlPath(string fileName = "", string addDir = "") + { + string outputDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "output", addDir); + // 出力先フォルダがなければ自動作成する(ローカル・VPS両方の対策) + if (!Directory.Exists(outputDir)) + { + Directory.CreateDirectory(outputDir); + } + // 出力するファイル名を決定 + string filePath = fileName != "" ? Path.Combine(outputDir, fileName) : outputDir; + + return filePath; + } + + public static void AddLog(string? message) + { + var filrDir = Path.Combine(GetExportXmlPath(""), "Log"); + if (Directory.Exists(filrDir) == false) + { + Directory.CreateDirectory(filrDir); + } + var fileName = $"{DateTime.Now:yyyy-MM-dd}.log"; + var filePath = Path.Combine(filrDir, fileName); + using var sw = File.AppendText(filePath); + sw.WriteLine(message); + } + + /// + /// ファイル出力 + /// + /// + /// + public static void ExportFile(string fileName, string? contents) + { + var filePath = GetExportXmlPath(fileName); + // ファイル書き出し + File.WriteAllText(filePath, contents); + } + + /// + /// Feedをファイルとして出力 + /// + /// + /// + public static void ExportXMLFile(string fileName, SyndicationFeed feed, string addDir = "") + { + var filePath = GetExportXmlPath(fileName, addDir); + // 文字コードをUTF-8に指定 + var settings = new XmlWriterSettings + { + Encoding = new UTF8Encoding(false), // BOMなしUTF-8 + Indent = true + }; + + using var xmlWriter = XmlWriter.Create(filePath, settings); + + // RSS 2.0 形式でファイルに直接出力 + var formatter = new Rss20FeedFormatter(feed); + formatter.WriteTo(xmlWriter); + } + + /// + /// 指定したURLのストリームを安全に取得します。404などのエラー時はnullを返します。 + /// + public static async Task GetStreamSafeAsync(HttpClient client, string url) + { + if (string.IsNullOrWhiteSpace(url)) return null; + + try + { + // HttpRequestMessageを作成 + var request = new HttpRequestMessage(HttpMethod.Get, url); + + // ResponseHeadersRead を指定することで、ヘッダーを読み込んだ直後に制御が戻る(一瞬で判定できる) + var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + + // 404 や 500 などのエラーではないか(成功ステータス 200~299 か)チェック + if (!response.IsSuccessStatusCode) + { + // 404の場合などのログをここに書くことも可能 + //AddLog($"[HTTP Error] Status Code: {response.StatusCode} for {url}"); + return null; + } + + // 成功していれば、ストリームを読み込んで返す + return await response.Content.ReadAsStreamAsync(); + } + catch (HttpRequestException ex) + { + // ネットワーク不通やタイムアウトなどの例外処理 + AddLog($"[Network Error] {ex.Message}"); + return null; + } + catch (Exception) + { + return null; + } + } + + /// + /// IDocumentの取得 + /// + /// + /// + public static async Task GetDocument(HttpClient client, string pageURL) + { + Console.WriteLine($"{pageURL}"); + + // 非同期でHTML文字列を丸ごとダウンロード + string htmlContent = await client.GetStringAsync(pageURL); + if (htmlContent.Contains("<") || htmlContent.Contains(">")) + { + htmlContent = WebUtility.HtmlDecode(htmlContent); + } + + var parser = new HtmlParser(); + IDocument document = await parser.ParseDocumentAsync(htmlContent); + return document; + + // // ダウンロードしたHTMLを AngleSharp に流し込んで解析する + // // 通信はHttpClient側で終わっているため、AngleSharp側は「通信機能なし」の最速設定でOK + // var config = Configuration.Default; + // var context = BrowsingContext.New(config); + // + // // 文字列のHTMLをAngleSharpにパースさせます(URLを紐付けることで、相対パスを自動解決できます) + // return await context.OpenAsync(req => req.Content(htmlContent).Address(pageURL)); + + + // Console.WriteLine($"{pageURL}"); + // + // // 1. AngleSharpの設定(Webからデータをダウンロードするローダーを有効にする) + // var config = Configuration.Default.WithDefaultLoader(); + // var context = BrowsingContext.New(config); + // + // // 2. URLを指定して直接 IDocument を取得(内部で自動的にRequesterが動きます) + // IDocument document = await context.OpenAsync(pageURL); + // + // return document; + } + + public static async Task> GetFeedItems(HttpClient client, string url) + { + try + { + using var stream = await client.GetStreamAsync(url); + using var reader = XmlReader.Create(stream); + var feed = SyndicationFeed.Load(reader); + + return feed?.Items ?? Enumerable.Empty(); + } + catch (Exception ex) + { + AddLog($"[エラー] {url} の取得に失敗: {ex.Message}"); + return Enumerable.Empty(); + } + } + + /// + /// IDocumentの取得 + /// + /// + /// + public static async Task GetDocumentRunJS(HttpClient client, string pageURL) + { +#if DEBUG + Console.WriteLine($"{pageURL}"); +#endif + string wsEndpoint = "ws://browserless:3000"; + + // 1. Browserless に WebSocket 接続 + var options = new ConnectOptions { BrowserWSEndpoint = wsEndpoint }; + using var browser = await Puppeteer.ConnectAsync(options); + using var page = await browser.NewPageAsync(); + + // 2. リクエスト遮断(画像・CSS・フォントを無効化して高速化・省メモリ化) + await page.SetRequestInterceptionAsync(true); + page.Request += (sender, e) => + { + var type = e.Request.ResourceType; + if (type == ResourceType.Image || + type == ResourceType.StyleSheet || + type == ResourceType.Font || + type == ResourceType.Media) + { + e.Request.AbortAsync(); + } + else + { + e.Request.ContinueAsync(); + } + }; + + // 3. 対象ページへ移動 + string targetUrl = "https://example.com/article/123"; + await page.GoToAsync(targetUrl, new NavigationOptions + { + WaitUntil = new[] { WaitUntilNavigation.DOMContentLoaded } + }); + + // 4.
タグの出現を待機して取得 + try + { + await page.WaitForSelectorAsync("article", new WaitForSelectorOptions { Timeout = 5000 }); + } + catch (WaitTaskTimeoutException) { /* タイムアウト処理 */ } + + string rawArticleHtml = await page.EvaluateExpressionAsync( + "document.querySelector('article') ? document.querySelector('article').outerHTML : ''" + ); + + var parser = new HtmlParser(); + var document = parser.ParseDocument(rawArticleHtml); + + return document; + } + + public static string GetLastPathPart(string urlStr) + { + if (Uri.TryCreate(urlStr, UriKind.Absolute, out Uri? uri)) + { + // uri.AbsolutePath は "/path/to/page.html" を返す + string absolutePath = uri!.AbsolutePath; + + // 末尾のスラッシュを除去した上で、スラッシュで分割して最後の要素を取得 + string lastPart = absolutePath.TrimEnd('/').Split('/').LastOrDefault()!; + return lastPart ?? string.Empty; + } + return string.Empty; + } + + public static string GetHtmlWithoutJavaScript(IElement originalElement) + { + if (originalElement == null) return string.Empty; + + // 元のDOMツリーに影響を与えないよう、要素をディープコピー(複製)します + var element = (IElement)originalElement.Clone(true); + + // クラス名の削除 + foreach (var el in element.QuerySelectorAll("[class]")) + { + el.RemoveAttribute("class"); + } + + // 1.