Files
GameLauncher/Services/NtfsMftScanService.cs
armonica b0fdea2411 最適化: NTFS MFT走査およびディスク読み取りの並列化と限界高速化
- 複数ドライブ(C:, Z:等)のMFTスキャンを完全並列実行するように改善
- MFTのI/Oバッファを2MBに拡張しシステムコールを削減
- ターゲット外ファイルの無駄な文字列化・オブジェクト生成を完全スキップする2パス抽出アルゴリズムを導入
- 既存ゲーム実在検証処理(VerifyAndEnrichGamesAsync)をParallel.ForEachで並列化
- 新規フォルダ走査の並列度引き上げと複数ドライブ同時走査に対応
2026-10-20 15:02:14 +09:00

400 lines
17 KiB
C#

using System;
using System.Buffers.Binary;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Win32.SafeHandles;
namespace GameLauncher.Services
{
/// <summary>
/// NTFS の MFT (USN Journal: FSCTL_ENUM_USN_DATA) を直接読み取り、
/// ドライブ内のファイルおよびディレクトリツリーを爆速で一括抽出するサービス
/// </summary>
public class NtfsMftScanService
{
#region □定数(Private)
private const uint GENERIC_READ = 0x80000000;
private const uint FILE_SHARE_READ = 0x00000001;
private const uint FILE_SHARE_WRITE = 0x00000002;
private const uint OPEN_EXISTING = 3;
private const uint FSCTL_QUERY_USN_JOURNAL = 0x000900f4;
private const uint FSCTL_ENUM_USN_DATA = 0x000900b3;
private const uint FILE_ATTRIBUTE_DIRECTORY = 0x00000010;
/// <summary>
/// MFT読み込みバッファサイズ(2MB)。1回のDeviceIoControl呼び出しで最大効率を引き出します。
/// </summary>
private const int BufferSize = 2048 * 1024;
#endregion
#region □構造体(Private)
[StructLayout(LayoutKind.Sequential)]
private struct USN_JOURNAL_DATA
{
public ulong UsnJournalID;
public long FirstUsn;
public long NextUsn;
public long LowestValidUsn;
public long MaxUsn;
public ulong MaximumSize;
public ulong AllocationDelta;
}
[StructLayout(LayoutKind.Sequential)]
private struct MFT_ENUM_DATA_V0
{
public ulong StartFileReferenceNumber;
public long LowUsn;
public long HighUsn;
}
/// <summary>
/// MFTディレクトリ情報ノード
/// </summary>
private struct DirNode
{
public ulong Id;
public ulong ParentId;
public string Name;
}
#endregion
#region □メソッド(Public)
/// <summary>
/// 指定されたパス群(例: C:\..., Z:\...)に対して、NTFS MFT 直接読み取りを用いて高速スナップショットを生成します。
/// 複数ドライブが存在する場合は、各ドライブを並列処理してディスクアクセス速度を極限まで高めます。
/// </summary>
/// <param name="targetDirectories">走査対象のディレクトリルート一覧</param>
/// <returns>生成されたスナップショット。失敗または非NTFS時は null</returns>
public static FastDirectorySnapshot? TryScanTargetsFast(IEnumerable<string> targetDirectories)
{
var targetsByDrive = new Dictionary<char, List<string>>();
foreach (var dir in targetDirectories)
{
if (string.IsNullOrWhiteSpace(dir)) continue;
var full = Path.GetFullPath(dir).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
if (full.Length >= 2 && full[1] == ':')
{
char driveLetter = char.ToUpperInvariant(full[0]);
if (!targetsByDrive.TryGetValue(driveLetter, out var list))
{
list = new List<string>();
targetsByDrive[driveLetter] = list;
}
list.Add(full);
}
}
if (targetsByDrive.Count == 0) return null;
var snapshots = new ConcurrentBag<FastDirectorySnapshot>();
// ドライブごとに並列処理を実行(物理/論理ドライブ間のI/OおよびCPUを限界まで同時並行化)
Parallel.ForEach(targetsByDrive, kvp =>
{
char driveLetter = kvp.Key;
var targetPaths = kvp.Value;
var driveSnapshot = new FastDirectorySnapshot();
try
{
bool ok = ScanVolumeMft(driveLetter, targetPaths, driveSnapshot);
if (ok && (driveSnapshot.TotalDirectoryCount > 0 || driveSnapshot.TotalFileCount > 0))
{
snapshots.Add(driveSnapshot);
}
}
catch
{
// ドライブスキャン失敗時はスキップ
}
});
if (snapshots.IsEmpty) return null;
var merged = new FastDirectorySnapshot();
foreach (var snap in snapshots)
{
merged.MergeFrom(snap);
}
return merged;
}
#endregion
#region □メソッド(Private)
/// <summary>
/// 1つのドライブボリュームのMFTを読み込み、指定されたターゲットパス配下のファイルをスナップショットに登録します。
/// </summary>
private static bool ScanVolumeMft(char driveLetter, List<string> targetPaths, FastDirectorySnapshot snapshot)
{
string volumePath = @"\\.\" + driveLetter + ":";
using var hVolume = CreateFile(
volumePath,
GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE,
IntPtr.Zero,
OPEN_EXISTING,
0,
IntPtr.Zero);
if (hVolume.IsInvalid)
{
return false;
}
// 1. USNジャーナル情報の取得
var journalData = new USN_JOURNAL_DATA();
int journalDataSize = Marshal.SizeOf(typeof(USN_JOURNAL_DATA));
if (!DeviceIoControl(hVolume, FSCTL_QUERY_USN_JOURNAL, IntPtr.Zero, 0, out journalData, journalDataSize, out _, IntPtr.Zero))
{
return false;
}
// 2. MFT列挙用パラメータの準備
var enumData = new MFT_ENUM_DATA_V0
{
StartFileReferenceNumber = 0,
LowUsn = 0,
HighUsn = journalData.NextUsn
};
int enumDataSize = Marshal.SizeOf(typeof(MFT_ENUM_DATA_V0));
IntPtr enumDataPtr = Marshal.AllocHGlobal(enumDataSize);
Marshal.StructureToPtr(enumData, enumDataPtr, false);
byte[] buffer = new byte[BufferSize];
GCHandle bufferHandle = GCHandle.Alloc(buffer, GCHandleType.Pinned);
IntPtr bufferPtr = bufferHandle.AddrOfPinnedObject();
// 読み込んだ全MFTデータチャンクを保持(I/Oを最優先で一括連続完了させる)
var chunks = new List<(byte[] Buffer, int Length)>();
try
{
while (DeviceIoControl(hVolume, FSCTL_ENUM_USN_DATA, enumDataPtr, enumDataSize, bufferPtr, BufferSize, out uint bytesReturned, IntPtr.Zero))
{
if (bytesReturned <= 8) break;
// 次の開始参照番号を取得
ulong nextId = (ulong)Marshal.ReadInt64(bufferPtr);
Marshal.WriteInt64(enumDataPtr, (long)nextId);
byte[] chunkCopy = new byte[bytesReturned];
Buffer.BlockCopy(buffer, 0, chunkCopy, 0, (int)bytesReturned);
chunks.Add((chunkCopy, (int)bytesReturned));
}
}
finally
{
if (bufferHandle.IsAllocated) bufferHandle.Free();
if (enumDataPtr != IntPtr.Zero) Marshal.FreeHGlobal(enumDataPtr);
}
if (chunks.Count == 0) return false;
// 3. 【第1パス】ディレクトリのみを収集(ディレクトリは全体の数%に過ぎないため極めて高速)
var dirNodes = new Dictionary<ulong, DirNode>(65536);
var childrenByParent = new Dictionary<ulong, List<ulong>>(65536);
foreach (var chunk in chunks)
{
ReadOnlySpan<byte> span = chunk.Buffer.AsSpan(0, chunk.Length);
int offset = 8;
while (offset < chunk.Length)
{
if (offset + 60 > chunk.Length) break;
ReadOnlySpan<byte> record = span.Slice(offset);
int recordLength = BinaryPrimitives.ReadInt32LittleEndian(record);
if (recordLength <= 0 || offset + recordLength > chunk.Length) break;
uint fileAttributes = BinaryPrimitives.ReadUInt32LittleEndian(record.Slice(52));
if ((fileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0)
{
ulong fileRef = BinaryPrimitives.ReadUInt64LittleEndian(record.Slice(8)) & 0x0000FFFFFFFFFFFFUL;
ulong parentRef = BinaryPrimitives.ReadUInt64LittleEndian(record.Slice(16)) & 0x0000FFFFFFFFFFFFUL;
ushort fileNameLength = BinaryPrimitives.ReadUInt16LittleEndian(record.Slice(56));
ushort fileNameOffset = BinaryPrimitives.ReadUInt16LittleEndian(record.Slice(58));
if (fileNameLength > 0 && fileNameOffset + fileNameLength <= recordLength)
{
string dirName = Encoding.Unicode.GetString(record.Slice(fileNameOffset, fileNameLength));
dirNodes[fileRef] = new DirNode { Id = fileRef, ParentId = parentRef, Name = dirName };
if (!childrenByParent.TryGetValue(parentRef, out var childList))
{
childList = new List<ulong>();
childrenByParent[parentRef] = childList;
}
childList.Add(fileRef);
}
}
offset += recordLength;
}
}
// 4. ターゲットパスのNodeId特定と、配下の子孫ディレクトリパスの確定
var targetDirIds = new HashSet<ulong>();
var dirIdToFullPath = new Dictionary<ulong, string>();
foreach (var targetPath in targetPaths)
{
ulong? targetNodeId = FindDirNodeIdByPath(driveLetter, targetPath, dirNodes, childrenByParent);
if (targetNodeId.HasValue)
{
CollectDescendantDirectories(targetNodeId.Value, targetPath, dirNodes, childrenByParent, targetDirIds, dirIdToFullPath, snapshot);
}
}
// 該当するターゲットディレクトリが存在しなければ終了
if (targetDirIds.Count == 0) return true;
// 5. 【第2パス】ターゲット配下のファイルのみを高速抽出
// (ターゲット外の数百万個の無関係なファイルは名前取得やオブジェクト生成を完全にスキップ!)
foreach (var chunk in chunks)
{
ReadOnlySpan<byte> span = chunk.Buffer.AsSpan(0, chunk.Length);
int offset = 8;
while (offset < chunk.Length)
{
if (offset + 60 > chunk.Length) break;
ReadOnlySpan<byte> record = span.Slice(offset);
int recordLength = BinaryPrimitives.ReadInt32LittleEndian(record);
if (recordLength <= 0 || offset + recordLength > chunk.Length) break;
uint fileAttributes = BinaryPrimitives.ReadUInt32LittleEndian(record.Slice(52));
// ディレクトリは第1パスですでに登録済みのためスキップ
if ((fileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0)
{
ulong parentRef = BinaryPrimitives.ReadUInt64LittleEndian(record.Slice(16)) & 0x0000FFFFFFFFFFFFUL;
// ターゲット配下のディレクトリに属するファイルのみを抽出
if (targetDirIds.Contains(parentRef) && dirIdToFullPath.TryGetValue(parentRef, out string? parentDirPath))
{
ushort fileNameLength = BinaryPrimitives.ReadUInt16LittleEndian(record.Slice(56));
ushort fileNameOffset = BinaryPrimitives.ReadUInt16LittleEndian(record.Slice(58));
if (fileNameLength > 0 && fileNameOffset + fileNameLength <= recordLength)
{
string fileName = Encoding.Unicode.GetString(record.Slice(fileNameOffset, fileNameLength));
snapshot.AddFile(parentDirPath, fileName);
}
}
}
offset += recordLength;
}
}
return true;
}
/// <summary>
/// パス文字列から対応するディレクトリのノードIDを特定します。
/// </summary>
private static ulong? FindDirNodeIdByPath(
char driveLetter,
string path,
Dictionary<ulong, DirNode> dirNodes,
Dictionary<ulong, List<ulong>> childrenByParent)
{
string rootStr = driveLetter + @":\";
if (path.Equals(rootStr.TrimEnd('\\'), StringComparison.OrdinalIgnoreCase) || path.Equals(rootStr, StringComparison.OrdinalIgnoreCase))
{
return 5UL; // ルートディレクトリID
}
var parts = path.Substring(3).Split(new[] { '\\', '/' }, StringSplitOptions.RemoveEmptyEntries);
ulong currentParent = 5UL;
foreach (var part in parts)
{
bool found = false;
if (childrenByParent.TryGetValue(currentParent, out var childIds))
{
foreach (var childId in childIds)
{
if (dirNodes.TryGetValue(childId, out var node) && string.Equals(node.Name, part, StringComparison.OrdinalIgnoreCase))
{
currentParent = node.Id;
found = true;
break;
}
}
}
if (!found) return null;
}
return currentParent;
}
/// <summary>
/// ターゲットディレクトリ配下の子孫ディレクトリIDおよびフルパスを再帰的に収集し、スナップショットに登録します。
/// </summary>
private static void CollectDescendantDirectories(
ulong parentNodeId,
string currentPath,
Dictionary<ulong, DirNode> dirNodes,
Dictionary<ulong, List<ulong>> childrenByParent,
HashSet<ulong> targetDirIds,
Dictionary<ulong, string> dirIdToFullPath,
FastDirectorySnapshot snapshot)
{
targetDirIds.Add(parentNodeId);
dirIdToFullPath[parentNodeId] = currentPath;
if (!childrenByParent.TryGetValue(parentNodeId, out var childIds)) return;
foreach (var childId in childIds)
{
if (!dirNodes.TryGetValue(childId, out var childNode)) continue;
string childPath = Path.Combine(currentPath, childNode.Name);
snapshot.AddDirectory(currentPath, childPath);
CollectDescendantDirectories(childId, childPath, dirNodes, childrenByParent, targetDirIds, dirIdToFullPath, snapshot);
}
}
#endregion
#region □P/Invoke(Private)
[DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)]
private static extern SafeFileHandle CreateFile(
string lpFileName,
uint dwDesiredAccess,
uint dwShareMode,
IntPtr lpSecurityAttributes,
uint dwCreationDisposition,
uint dwFlagsAndAttributes,
IntPtr hTemplateFile);
[DllImport("kernel32.dll", ExactSpelling = true, SetLastError = true, CharSet = CharSet.Auto)]
private static extern bool DeviceIoControl(
SafeFileHandle hDevice,
uint dwIoControlCode,
IntPtr lpInBuffer,
int nInBufferSize,
out USN_JOURNAL_DATA lpOutBuffer,
int nOutBufferSize,
out uint lpBytesReturned,
IntPtr lpOverlapped);
[DllImport("kernel32.dll", ExactSpelling = true, SetLastError = true, CharSet = CharSet.Auto)]
private static extern bool DeviceIoControl(
SafeFileHandle hDevice,
uint dwIoControlCode,
IntPtr lpInBuffer,
int nInBufferSize,
IntPtr lpOutBuffer,
int nOutBufferSize,
out uint lpBytesReturned,
IntPtr lpOverlapped);
#endregion
}
}