最適化: NTFS MFT走査およびディスク読み取りの並列化と限界高速化
- 複数ドライブ(C:, Z:等)のMFTスキャンを完全並列実行するように改善 - MFTのI/Oバッファを2MBに拡張しシステムコールを削減 - ターゲット外ファイルの無駄な文字列化・オブジェクト生成を完全スキップする2パス抽出アルゴリズムを導入 - 既存ゲーム実在検証処理(VerifyAndEnrichGamesAsync)をParallel.ForEachで並列化 - 新規フォルダ走査の並列度引き上げと複数ドライブ同時走査に対応
This commit is contained in:
1 parent
dd00c96e20
commit
b0fdea2411
3 files changed
+252
-125
No files matched your search
+142
-77
@@ -1,8 +1,11 @@
|
||||
using System;
|
||||
using System.Buffers.Binary;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Win32.SafeHandles;
|
||||
|
||||
namespace GameLauncher.Services
|
||||
@@ -21,6 +24,11 @@ namespace GameLauncher.Services
|
||||
private const uint FSCTL_QUERY_USN_JOURNAL = 0x000900f4;
|
||||
private const uint FSCTL_ENUM_USN_DATA = 0x000900b3;
|
||||
private const uint FILE_ATTRIBUTE_DIRECTORY = 0x00000010;
|
||||
|
||||
/// <summary>
|
||||
/// MFT読み込みバッファサイズ(2MB)。1回のDeviceIoControl呼び出しで最大効率を引き出します。
|
||||
/// </summary>
|
||||
private const int BufferSize = 2048 * 1024;
|
||||
#endregion
|
||||
|
||||
#region □構造体(Private)
|
||||
@@ -44,24 +52,26 @@ namespace GameLauncher.Services
|
||||
public long HighUsn;
|
||||
}
|
||||
|
||||
private class MftNode
|
||||
/// <summary>
|
||||
/// MFTディレクトリ情報ノード
|
||||
/// </summary>
|
||||
private struct DirNode
|
||||
{
|
||||
public ulong Id;
|
||||
public ulong ParentId;
|
||||
public string Name = string.Empty;
|
||||
public bool IsDirectory;
|
||||
public string Name;
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region □メソッド(Public)
|
||||
/// <summary>
|
||||
/// 指定されたパス群(例: C:\..., Z:\...)に対して、NTFS MFT 直接読み取りを用いて高速スナップショットを生成します。
|
||||
/// 複数ドライブが存在する場合は、各ドライブを並列処理してディスクアクセス速度を極限まで高めます。
|
||||
/// </summary>
|
||||
/// <param name="targetDirectories">走査対象のディレクトリルート一覧</param>
|
||||
/// <returns>生成されたスナップショット。失敗または非NTFS時は null</returns>
|
||||
public static FastDirectorySnapshot? TryScanTargetsFast(IEnumerable<string> targetDirectories)
|
||||
{
|
||||
var snapshot = new FastDirectorySnapshot();
|
||||
var targetsByDrive = new Dictionary<char, List<string>>();
|
||||
|
||||
foreach (var dir in targetDirectories)
|
||||
@@ -82,34 +92,44 @@ namespace GameLauncher.Services
|
||||
|
||||
if (targetsByDrive.Count == 0) return null;
|
||||
|
||||
bool anySuccess = false;
|
||||
var snapshots = new ConcurrentBag<FastDirectorySnapshot>();
|
||||
|
||||
foreach (var kvp in targetsByDrive)
|
||||
// ドライブごとに並列処理を実行(物理/論理ドライブ間のI/OおよびCPUを限界まで同時並行化)
|
||||
Parallel.ForEach(targetsByDrive, kvp =>
|
||||
{
|
||||
char driveLetter = kvp.Key;
|
||||
var targetPaths = kvp.Value;
|
||||
var driveSnapshot = new FastDirectorySnapshot();
|
||||
|
||||
try
|
||||
{
|
||||
bool ok = ScanVolumeMft(driveLetter, targetPaths, snapshot);
|
||||
if (ok)
|
||||
bool ok = ScanVolumeMft(driveLetter, targetPaths, driveSnapshot);
|
||||
if (ok && (driveSnapshot.TotalDirectoryCount > 0 || driveSnapshot.TotalFileCount > 0))
|
||||
{
|
||||
anySuccess = true;
|
||||
snapshots.Add(driveSnapshot);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ドライブスキャン失敗時はスキップ
|
||||
}
|
||||
});
|
||||
|
||||
if (snapshots.IsEmpty) return null;
|
||||
|
||||
var merged = new FastDirectorySnapshot();
|
||||
foreach (var snap in snapshots)
|
||||
{
|
||||
merged.MergeFrom(snap);
|
||||
}
|
||||
|
||||
return anySuccess ? snapshot : null;
|
||||
return merged;
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region □メソッド(Private)
|
||||
/// <summary>
|
||||
/// 1つのドライブボリュームのMFTを読み込み、指定されたパス配下のファイルをスナップショットに登録します。
|
||||
/// 1つのドライブボリュームのMFTを読み込み、指定されたターゲットパス配下のファイルをスナップショットに登録します。
|
||||
/// </summary>
|
||||
private static bool ScanVolumeMft(char driveLetter, List<string> targetPaths, FastDirectorySnapshot snapshot)
|
||||
{
|
||||
@@ -148,17 +168,16 @@ namespace GameLauncher.Services
|
||||
IntPtr enumDataPtr = Marshal.AllocHGlobal(enumDataSize);
|
||||
Marshal.StructureToPtr(enumData, enumDataPtr, false);
|
||||
|
||||
const int bufferSize = 1024 * 512; // 512KBバッファ
|
||||
byte[] buffer = new byte[bufferSize];
|
||||
byte[] buffer = new byte[BufferSize];
|
||||
GCHandle bufferHandle = GCHandle.Alloc(buffer, GCHandleType.Pinned);
|
||||
IntPtr bufferPtr = bufferHandle.AddrOfPinnedObject();
|
||||
|
||||
// ID -> MftNode のマッピングを保持
|
||||
var nodes = new Dictionary<ulong, MftNode>(500000);
|
||||
// 読み込んだ全MFTデータチャンクを保持(I/Oを最優先で一括連続完了させる)
|
||||
var chunks = new List<(byte[] Buffer, int Length)>();
|
||||
|
||||
try
|
||||
{
|
||||
while (DeviceIoControl(hVolume, FSCTL_ENUM_USN_DATA, enumDataPtr, enumDataSize, bufferPtr, bufferSize, out uint bytesReturned, IntPtr.Zero))
|
||||
while (DeviceIoControl(hVolume, FSCTL_ENUM_USN_DATA, enumDataPtr, enumDataSize, bufferPtr, BufferSize, out uint bytesReturned, IntPtr.Zero))
|
||||
{
|
||||
if (bytesReturned <= 8) break;
|
||||
|
||||
@@ -166,36 +185,9 @@ namespace GameLauncher.Services
|
||||
ulong nextId = (ulong)Marshal.ReadInt64(bufferPtr);
|
||||
Marshal.WriteInt64(enumDataPtr, (long)nextId);
|
||||
|
||||
int offset = 8;
|
||||
while (offset < bytesReturned)
|
||||
{
|
||||
IntPtr recordPtr = IntPtr.Add(bufferPtr, offset);
|
||||
int recordLength = Marshal.ReadInt32(recordPtr);
|
||||
if (recordLength <= 0) break;
|
||||
|
||||
ulong fileRef = (ulong)Marshal.ReadInt64(recordPtr, 8) & 0x0000FFFFFFFFFFFFUL;
|
||||
ulong parentRef = (ulong)Marshal.ReadInt64(recordPtr, 16) & 0x0000FFFFFFFFFFFFUL;
|
||||
uint fileAttributes = (uint)Marshal.ReadInt32(recordPtr, 52);
|
||||
int fileNameLength = Marshal.ReadInt16(recordPtr, 56);
|
||||
int fileNameOffset = Marshal.ReadInt16(recordPtr, 58);
|
||||
|
||||
if (fileNameLength > 0)
|
||||
{
|
||||
string fileName = Marshal.PtrToStringUni(IntPtr.Add(recordPtr, fileNameOffset), fileNameLength / 2);
|
||||
bool isDir = (fileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
|
||||
|
||||
var node = new MftNode
|
||||
{
|
||||
Id = fileRef,
|
||||
ParentId = parentRef,
|
||||
Name = fileName,
|
||||
IsDirectory = isDir
|
||||
};
|
||||
nodes[fileRef] = node;
|
||||
}
|
||||
|
||||
offset += recordLength;
|
||||
}
|
||||
byte[] chunkCopy = new byte[bytesReturned];
|
||||
Buffer.BlockCopy(buffer, 0, chunkCopy, 0, (int)bytesReturned);
|
||||
chunks.Add((chunkCopy, (int)bytesReturned));
|
||||
}
|
||||
}
|
||||
finally
|
||||
@@ -204,27 +196,99 @@ namespace GameLauncher.Services
|
||||
if (enumDataPtr != IntPtr.Zero) Marshal.FreeHGlobal(enumDataPtr);
|
||||
}
|
||||
|
||||
if (nodes.Count == 0) return false;
|
||||
if (chunks.Count == 0) return false;
|
||||
|
||||
// 3. 親IDから子IDのツリーを構築
|
||||
var childrenByParent = new Dictionary<ulong, List<ulong>>(nodes.Count);
|
||||
foreach (var node in nodes.Values)
|
||||
// 3. 【第1パス】ディレクトリのみを収集(ディレクトリは全体の数%に過ぎないため極めて高速)
|
||||
var dirNodes = new Dictionary<ulong, DirNode>(65536);
|
||||
var childrenByParent = new Dictionary<ulong, List<ulong>>(65536);
|
||||
|
||||
foreach (var chunk in chunks)
|
||||
{
|
||||
if (!childrenByParent.TryGetValue(node.ParentId, out var list))
|
||||
ReadOnlySpan<byte> span = chunk.Buffer.AsSpan(0, chunk.Length);
|
||||
int offset = 8;
|
||||
while (offset < chunk.Length)
|
||||
{
|
||||
list = new List<ulong>();
|
||||
childrenByParent[node.ParentId] = list;
|
||||
if (offset + 60 > chunk.Length) break;
|
||||
ReadOnlySpan<byte> record = span.Slice(offset);
|
||||
int recordLength = BinaryPrimitives.ReadInt32LittleEndian(record);
|
||||
if (recordLength <= 0 || offset + recordLength > chunk.Length) break;
|
||||
|
||||
uint fileAttributes = BinaryPrimitives.ReadUInt32LittleEndian(record.Slice(52));
|
||||
if ((fileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0)
|
||||
{
|
||||
ulong fileRef = BinaryPrimitives.ReadUInt64LittleEndian(record.Slice(8)) & 0x0000FFFFFFFFFFFFUL;
|
||||
ulong parentRef = BinaryPrimitives.ReadUInt64LittleEndian(record.Slice(16)) & 0x0000FFFFFFFFFFFFUL;
|
||||
ushort fileNameLength = BinaryPrimitives.ReadUInt16LittleEndian(record.Slice(56));
|
||||
ushort fileNameOffset = BinaryPrimitives.ReadUInt16LittleEndian(record.Slice(58));
|
||||
|
||||
if (fileNameLength > 0 && fileNameOffset + fileNameLength <= recordLength)
|
||||
{
|
||||
string dirName = Encoding.Unicode.GetString(record.Slice(fileNameOffset, fileNameLength));
|
||||
dirNodes[fileRef] = new DirNode { Id = fileRef, ParentId = parentRef, Name = dirName };
|
||||
|
||||
if (!childrenByParent.TryGetValue(parentRef, out var childList))
|
||||
{
|
||||
childList = new List<ulong>();
|
||||
childrenByParent[parentRef] = childList;
|
||||
}
|
||||
childList.Add(fileRef);
|
||||
}
|
||||
}
|
||||
|
||||
offset += recordLength;
|
||||
}
|
||||
list.Add(node.Id);
|
||||
}
|
||||
|
||||
// 4. 各ターゲットパスのノードIDを特定し、その配下のエントリをスナップショットに格納
|
||||
// 4. ターゲットパスのNodeId特定と、配下の子孫ディレクトリパスの確定
|
||||
var targetDirIds = new HashSet<ulong>();
|
||||
var dirIdToFullPath = new Dictionary<ulong, string>();
|
||||
|
||||
foreach (var targetPath in targetPaths)
|
||||
{
|
||||
var targetNodeId = FindNodeIdByPath(driveLetter, targetPath, nodes, childrenByParent);
|
||||
ulong? targetNodeId = FindDirNodeIdByPath(driveLetter, targetPath, dirNodes, childrenByParent);
|
||||
if (targetNodeId.HasValue)
|
||||
{
|
||||
PopulateSnapshotFromNode(targetNodeId.Value, targetPath, nodes, childrenByParent, snapshot);
|
||||
CollectDescendantDirectories(targetNodeId.Value, targetPath, dirNodes, childrenByParent, targetDirIds, dirIdToFullPath, snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
// 該当するターゲットディレクトリが存在しなければ終了
|
||||
if (targetDirIds.Count == 0) return true;
|
||||
|
||||
// 5. 【第2パス】ターゲット配下のファイルのみを高速抽出
|
||||
// (ターゲット外の数百万個の無関係なファイルは名前取得やオブジェクト生成を完全にスキップ!)
|
||||
foreach (var chunk in chunks)
|
||||
{
|
||||
ReadOnlySpan<byte> span = chunk.Buffer.AsSpan(0, chunk.Length);
|
||||
int offset = 8;
|
||||
while (offset < chunk.Length)
|
||||
{
|
||||
if (offset + 60 > chunk.Length) break;
|
||||
ReadOnlySpan<byte> record = span.Slice(offset);
|
||||
int recordLength = BinaryPrimitives.ReadInt32LittleEndian(record);
|
||||
if (recordLength <= 0 || offset + recordLength > chunk.Length) break;
|
||||
|
||||
uint fileAttributes = BinaryPrimitives.ReadUInt32LittleEndian(record.Slice(52));
|
||||
// ディレクトリは第1パスですでに登録済みのためスキップ
|
||||
if ((fileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0)
|
||||
{
|
||||
ulong parentRef = BinaryPrimitives.ReadUInt64LittleEndian(record.Slice(16)) & 0x0000FFFFFFFFFFFFUL;
|
||||
|
||||
// ターゲット配下のディレクトリに属するファイルのみを抽出
|
||||
if (targetDirIds.Contains(parentRef) && dirIdToFullPath.TryGetValue(parentRef, out string? parentDirPath))
|
||||
{
|
||||
ushort fileNameLength = BinaryPrimitives.ReadUInt16LittleEndian(record.Slice(56));
|
||||
ushort fileNameOffset = BinaryPrimitives.ReadUInt16LittleEndian(record.Slice(58));
|
||||
|
||||
if (fileNameLength > 0 && fileNameOffset + fileNameLength <= recordLength)
|
||||
{
|
||||
string fileName = Encoding.Unicode.GetString(record.Slice(fileNameOffset, fileNameLength));
|
||||
snapshot.AddFile(parentDirPath, fileName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
offset += recordLength;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -232,19 +296,22 @@ namespace GameLauncher.Services
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// パス文字列から対応するノードIDを検索します。
|
||||
/// パス文字列から対応するディレクトリのノードIDを特定します。
|
||||
/// </summary>
|
||||
private static ulong? FindNodeIdByPath(char driveLetter, string path, Dictionary<ulong, MftNode> nodes, Dictionary<ulong, List<ulong>> childrenByParent)
|
||||
private static ulong? FindDirNodeIdByPath(
|
||||
char driveLetter,
|
||||
string path,
|
||||
Dictionary<ulong, DirNode> dirNodes,
|
||||
Dictionary<ulong, List<ulong>> childrenByParent)
|
||||
{
|
||||
string rootStr = driveLetter + @":\";
|
||||
if (path.Equals(rootStr.TrimEnd('\\'), StringComparison.OrdinalIgnoreCase) || path.Equals(rootStr, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// ドライブ直下のルートディレクトリ(通常IDは 5 / 0x0005)
|
||||
return 5UL;
|
||||
return 5UL; // ルートディレクトリID
|
||||
}
|
||||
|
||||
var parts = path.Substring(3).Split(new[] { '\\', '/' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
ulong currentParent = 5UL; // ルートID
|
||||
ulong currentParent = 5UL;
|
||||
|
||||
foreach (var part in parts)
|
||||
{
|
||||
@@ -253,7 +320,7 @@ namespace GameLauncher.Services
|
||||
{
|
||||
foreach (var childId in childIds)
|
||||
{
|
||||
if (nodes.TryGetValue(childId, out var node) && node.IsDirectory && string.Equals(node.Name, part, StringComparison.OrdinalIgnoreCase))
|
||||
if (dirNodes.TryGetValue(childId, out var node) && string.Equals(node.Name, part, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
currentParent = node.Id;
|
||||
found = true;
|
||||
@@ -268,31 +335,29 @@ namespace GameLauncher.Services
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ノードIDから配下を再帰的に巡回し、スナップショットに登録します。
|
||||
/// ターゲットディレクトリ配下の子孫ディレクトリIDおよびフルパスを再帰的に収集し、スナップショットに登録します。
|
||||
/// </summary>
|
||||
private static void PopulateSnapshotFromNode(
|
||||
private static void CollectDescendantDirectories(
|
||||
ulong parentNodeId,
|
||||
string currentPath,
|
||||
Dictionary<ulong, MftNode> nodes,
|
||||
Dictionary<ulong, DirNode> dirNodes,
|
||||
Dictionary<ulong, List<ulong>> childrenByParent,
|
||||
HashSet<ulong> targetDirIds,
|
||||
Dictionary<ulong, string> dirIdToFullPath,
|
||||
FastDirectorySnapshot snapshot)
|
||||
{
|
||||
targetDirIds.Add(parentNodeId);
|
||||
dirIdToFullPath[parentNodeId] = currentPath;
|
||||
|
||||
if (!childrenByParent.TryGetValue(parentNodeId, out var childIds)) return;
|
||||
|
||||
foreach (var childId in childIds)
|
||||
{
|
||||
if (!nodes.TryGetValue(childId, out var childNode)) continue;
|
||||
if (!dirNodes.TryGetValue(childId, out var childNode)) continue;
|
||||
|
||||
if (childNode.IsDirectory)
|
||||
{
|
||||
string childPath = Path.Combine(currentPath, childNode.Name);
|
||||
snapshot.AddDirectory(currentPath, childPath);
|
||||
PopulateSnapshotFromNode(childId, childPath, nodes, childrenByParent, snapshot);
|
||||
}
|
||||
else
|
||||
{
|
||||
snapshot.AddFile(currentPath, childNode.Name);
|
||||
}
|
||||
string childPath = Path.Combine(currentPath, childNode.Name);
|
||||
snapshot.AddDirectory(currentPath, childPath);
|
||||
CollectDescendantDirectories(childId, childPath, dirNodes, childrenByParent, targetDirIds, dirIdToFullPath, snapshot);
|
||||
}
|
||||
}
|
||||
#endregion
|
||||
|
||||
Reference in new issue
Block a user