575 lines
23 KiB
C#
575 lines
23 KiB
C#
using Minio;
|
|
using Minio.DataModel;
|
|
using Minio.DataModel.Args;
|
|
|
|
namespace RobotNet10.StorageManager;
|
|
|
|
public class StorageManager : IStorageManager
|
|
{
|
|
private readonly IMinioClient? MinioClient;
|
|
private readonly StorageConfig StorageConfig;
|
|
|
|
public StorageManager(StorageConfig config)
|
|
{
|
|
StorageConfig = config ?? throw new ArgumentNullException(nameof(config));
|
|
|
|
if (!StorageConfig.UsingLocal)
|
|
{
|
|
if (StorageConfig.MinioConfig == null)
|
|
throw new ArgumentException("MinioConfig is required when UsingLocal is false", nameof(config));
|
|
|
|
if (string.IsNullOrWhiteSpace(StorageConfig.MinioConfig.Endpoint))
|
|
throw new ArgumentException("MinioConfig.Endpoint cannot be null or empty", nameof(config));
|
|
|
|
if (string.IsNullOrWhiteSpace(StorageConfig.Bucket))
|
|
throw new ArgumentException("Bucket cannot be null or empty when UsingLocal is false", nameof(config));
|
|
|
|
MinioClient = new MinioClient()
|
|
.WithEndpoint(StorageConfig.MinioConfig.Endpoint)
|
|
.WithCredentials(StorageConfig.MinioConfig.User, StorageConfig.MinioConfig.Password)
|
|
.WithSSL(StorageConfig.MinioConfig.EnableSSL)
|
|
.Build();
|
|
}
|
|
}
|
|
|
|
private static void ValidatePath(string path)
|
|
{
|
|
if (path.Contains(".."))
|
|
throw new ArgumentException("Path cannot contain '..'", nameof(path));
|
|
|
|
var invalidChars = Path.GetInvalidPathChars();
|
|
if (path.Any(c => invalidChars.Contains(c)))
|
|
throw new ArgumentException("Path contains invalid characters", nameof(path));
|
|
}
|
|
|
|
private static void ValidateObjectName(string objectName)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(objectName))
|
|
throw new ArgumentException("ObjectName cannot be null or empty", nameof(objectName));
|
|
|
|
var invalidChars = Path.GetInvalidFileNameChars();
|
|
if (objectName.Any(c => invalidChars.Contains(c)))
|
|
throw new ArgumentException("ObjectName contains invalid characters", nameof(objectName));
|
|
}
|
|
|
|
private static string GetFileExtension(string contentType)
|
|
{
|
|
return contentType.ToLowerInvariant() switch
|
|
{
|
|
"image/png" => ".png",
|
|
"image/jpeg" => ".jpg",
|
|
"image/jpg" => ".jpg",
|
|
"image/gif" => ".gif",
|
|
"image/bmp" => ".bmp",
|
|
"image/webp" => ".webp",
|
|
"application/pdf" => ".pdf",
|
|
"application/json" => ".json",
|
|
"text/plain" => ".txt",
|
|
"text/csv" => ".csv",
|
|
"application/xml" => ".xml",
|
|
"application/zip" => ".zip",
|
|
_ => ".bin"
|
|
};
|
|
}
|
|
|
|
private async Task<T> ExecuteWithRetryAsync<T>(Func<Task<T>> operation, CancellationToken cancellationToken)
|
|
{
|
|
int maxRetries = StorageConfig.RetryCount;
|
|
for (int i = 0; i < maxRetries; i++)
|
|
{
|
|
try
|
|
{
|
|
return await operation();
|
|
}
|
|
catch (Exception) when (i < maxRetries - 1)
|
|
{
|
|
await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, i)), cancellationToken);
|
|
}
|
|
}
|
|
throw new InvalidOperationException($"Operation failed after {maxRetries} retries");
|
|
}
|
|
|
|
private async Task UploadLocal(string path, string objectName, Stream data, string contentType, CancellationToken cancellationToken)
|
|
{
|
|
var extension = GetFileExtension(contentType);
|
|
var localFolder = StorageConfig.LocalFolder;
|
|
if (!Directory.Exists(localFolder)) Directory.CreateDirectory(localFolder);
|
|
var folderPath = Path.Combine(localFolder, path);
|
|
if (!Directory.Exists(folderPath)) Directory.CreateDirectory(folderPath);
|
|
var pathLocal = Path.Combine(folderPath, $"{objectName}{extension}");
|
|
if (File.Exists($"{pathLocal}.bk")) File.Delete($"{pathLocal}.bk");
|
|
if (File.Exists(pathLocal)) File.Move(pathLocal, $"{pathLocal}.bk");
|
|
using Stream fileStream = new FileStream(pathLocal, FileMode.Create);
|
|
await data.CopyToAsync(fileStream, cancellationToken);
|
|
}
|
|
|
|
private async Task UploadMinio(IMinioClient minioClient, string path, string objectName, Stream data, long size, string contentType, CancellationToken cancellationToken)
|
|
{
|
|
await ExecuteWithRetryAsync(async () =>
|
|
{
|
|
var beArgs = new BucketExistsArgs().WithBucket(StorageConfig.Bucket);
|
|
bool found = await minioClient.BucketExistsAsync(beArgs, cancellationToken).ConfigureAwait(false);
|
|
if (!found)
|
|
{
|
|
var mbArgs = new MakeBucketArgs()
|
|
.WithBucket(StorageConfig.Bucket);
|
|
await minioClient.MakeBucketAsync(mbArgs, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
var putObjectArgs = new PutObjectArgs()
|
|
.WithBucket(StorageConfig.Bucket)
|
|
.WithObject($"{path}/{objectName}")
|
|
.WithObjectSize(size)
|
|
.WithStreamData(data)
|
|
.WithContentType(contentType);
|
|
await minioClient.PutObjectAsync(putObjectArgs, cancellationToken).ConfigureAwait(false);
|
|
return Task.CompletedTask;
|
|
}, cancellationToken);
|
|
|
|
}
|
|
|
|
public async Task UploadAsync(string path, string objectName, Stream data, long size, string contentType, CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(path))
|
|
throw new ArgumentException("Path cannot be null or empty", nameof(path));
|
|
ValidatePath(path);
|
|
ValidateObjectName(objectName);
|
|
if (data != null)
|
|
{
|
|
if (size < 0)
|
|
throw new ArgumentException("Size cannot be negative", nameof(size));
|
|
if (string.IsNullOrWhiteSpace(contentType))
|
|
throw new ArgumentException("ContentType cannot be null or empty", nameof(contentType));
|
|
|
|
if (!StorageConfig.UsingLocal && MinioClient != null)
|
|
await UploadMinio(MinioClient, path, objectName, data, size, contentType, cancellationToken);
|
|
else
|
|
await UploadLocal(path, objectName, data, contentType, cancellationToken);
|
|
}
|
|
else throw new ArgumentNullException(nameof(data));
|
|
}
|
|
|
|
private async Task<string> GetUrlLocal(string path, string objectName)
|
|
{
|
|
var localFolder = StorageConfig.LocalFolder;
|
|
if (Directory.Exists(localFolder))
|
|
{
|
|
var folderPath = Path.Combine(localFolder, path);
|
|
if (Directory.Exists(folderPath))
|
|
{
|
|
// Try common extensions
|
|
var extensions = new[] { ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".pdf", ".json", ".txt", ".bin" };
|
|
foreach (var ext in extensions)
|
|
{
|
|
var pathLocal = Path.Combine(folderPath, $"{objectName}{ext}");
|
|
if (File.Exists(pathLocal))
|
|
{
|
|
return pathLocal;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return string.Empty;
|
|
}
|
|
|
|
private async Task<string> GetUrlMinio(IMinioClient minioClient, string path, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
return await ExecuteWithRetryAsync(async () =>
|
|
{
|
|
var presignedGetObjectArgs = new PresignedGetObjectArgs()
|
|
.WithBucket(StorageConfig.Bucket)
|
|
.WithObject($"{path}/{objectName}")
|
|
.WithExpiry(60 * 60 * 24);
|
|
return await minioClient.PresignedGetObjectAsync(presignedGetObjectArgs);
|
|
}, cancellationToken);
|
|
}
|
|
|
|
public async Task<string> GetUrlAsync(string path, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(path))
|
|
throw new ArgumentException("Path cannot be null or empty", nameof(path));
|
|
ValidatePath(path);
|
|
ValidateObjectName(objectName);
|
|
|
|
if (!StorageConfig.UsingLocal && MinioClient != null)
|
|
return await GetUrlMinio(MinioClient, path, objectName, cancellationToken);
|
|
else
|
|
return await GetUrlLocal(path, objectName);
|
|
}
|
|
|
|
private async Task<Stream> GetFileLocal(string path, string objectName)
|
|
{
|
|
var localFolder = StorageConfig.LocalFolder;
|
|
if (Directory.Exists(localFolder))
|
|
{
|
|
var folderPath = Path.Combine(localFolder, path);
|
|
if (Directory.Exists(folderPath))
|
|
{
|
|
// Try common extensions
|
|
var extensions = new[] { ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".pdf", ".json", ".txt", ".bin" };
|
|
foreach (var ext in extensions)
|
|
{
|
|
var pathLocal = Path.Combine(folderPath, $"{objectName}{ext}");
|
|
if (File.Exists(pathLocal))
|
|
{
|
|
return new FileStream(pathLocal, FileMode.Open, FileAccess.Read, FileShare.Read);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
throw new FileNotFoundException($"File not found: {path}/{objectName}");
|
|
}
|
|
|
|
private async Task<Stream> GetFileMinio(IMinioClient minioClient, string path, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
return await ExecuteWithRetryAsync(async () =>
|
|
{
|
|
var memoryStream = new MemoryStream();
|
|
|
|
var getObjectArgs = new GetObjectArgs()
|
|
.WithBucket(StorageConfig.Bucket)
|
|
.WithObject($"{path}/{objectName}")
|
|
.WithCallbackStream(async (stream, token) =>
|
|
{
|
|
await stream.CopyToAsync(memoryStream, token);
|
|
});
|
|
|
|
await minioClient.GetObjectAsync(getObjectArgs, cancellationToken);
|
|
|
|
memoryStream.Position = 0;
|
|
return memoryStream;
|
|
}, cancellationToken);
|
|
}
|
|
|
|
public async Task<Stream> GetFileAsync(string path, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(path))
|
|
throw new ArgumentException("Path cannot be null or empty", nameof(path));
|
|
ValidatePath(path);
|
|
ValidateObjectName(objectName);
|
|
|
|
if (!StorageConfig.UsingLocal && MinioClient != null)
|
|
return await GetFileMinio(MinioClient, path, objectName, cancellationToken);
|
|
else
|
|
return await GetFileLocal(path, objectName);
|
|
}
|
|
|
|
private async Task DeleteLocal(string path, string objectName)
|
|
{
|
|
var localFolder = StorageConfig.LocalFolder;
|
|
if (Directory.Exists(localFolder))
|
|
{
|
|
var folderPath = Path.Combine(localFolder, path);
|
|
if (Directory.Exists(folderPath))
|
|
{
|
|
// Try common extensions
|
|
var extensions = new[] { ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".pdf", ".json", ".txt", ".bin" };
|
|
foreach (var ext in extensions)
|
|
{
|
|
var pathLocal = Path.Combine(folderPath, $"{objectName}{ext}");
|
|
if (File.Exists(pathLocal))
|
|
{
|
|
File.Delete(pathLocal);
|
|
if (File.Exists($"{pathLocal}.bk"))
|
|
File.Delete($"{pathLocal}.bk");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
await Task.CompletedTask;
|
|
}
|
|
|
|
private async Task DeleteMinio(IMinioClient minioClient, string path, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
await ExecuteWithRetryAsync(async () =>
|
|
{
|
|
var removeObjectArgs = new RemoveObjectArgs()
|
|
.WithBucket(StorageConfig.Bucket)
|
|
.WithObject($"{path}/{objectName}")
|
|
.WithVersionId("null");
|
|
await minioClient.RemoveObjectAsync(removeObjectArgs, cancellationToken);
|
|
return Task.CompletedTask;
|
|
}, cancellationToken);
|
|
}
|
|
|
|
public async Task DeleteAsync(string path, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(path))
|
|
throw new ArgumentException("Path cannot be null or empty", nameof(path));
|
|
ValidatePath(path);
|
|
ValidateObjectName(objectName);
|
|
|
|
if (!StorageConfig.UsingLocal && MinioClient != null)
|
|
await DeleteMinio(MinioClient, path, objectName, cancellationToken);
|
|
else
|
|
await DeleteLocal(path, objectName);
|
|
}
|
|
|
|
public async Task<bool> ExistsAsync(string path, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(path))
|
|
throw new ArgumentException("Path cannot be null or empty", nameof(path));
|
|
ValidatePath(path);
|
|
ValidateObjectName(objectName);
|
|
|
|
if (!StorageConfig.UsingLocal && MinioClient != null)
|
|
return await ExistsMinio(MinioClient, path, objectName, cancellationToken);
|
|
else
|
|
return await ExistsLocal(path, objectName);
|
|
}
|
|
|
|
private async Task<bool> ExistsLocal(string path, string objectName)
|
|
{
|
|
var localFolder = StorageConfig.LocalFolder;
|
|
if (Directory.Exists(localFolder))
|
|
{
|
|
var folderPath = Path.Combine(localFolder, path);
|
|
if (Directory.Exists(folderPath))
|
|
{
|
|
var extensions = new[] { ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".pdf", ".json", ".txt", ".bin" };
|
|
foreach (var ext in extensions)
|
|
{
|
|
var pathLocal = Path.Combine(folderPath, $"{objectName}{ext}");
|
|
if (File.Exists(pathLocal))
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private async Task<bool> ExistsMinio(IMinioClient minioClient, string path, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
return await ExecuteWithRetryAsync(async () =>
|
|
{
|
|
try
|
|
{
|
|
var statObjectArgs = new StatObjectArgs()
|
|
.WithBucket(StorageConfig.Bucket)
|
|
.WithObject($"{path}/{objectName}");
|
|
await minioClient.StatObjectAsync(statObjectArgs, cancellationToken);
|
|
return true;
|
|
}
|
|
catch (Minio.Exceptions.ObjectNotFoundException)
|
|
{
|
|
return false;
|
|
}
|
|
}, cancellationToken);
|
|
}
|
|
|
|
public async Task<List<string>> ListAsync(string path, bool recursive, CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(path))
|
|
throw new ArgumentException("Path cannot be null or empty", nameof(path));
|
|
ValidatePath(path);
|
|
|
|
if (!StorageConfig.UsingLocal && MinioClient != null)
|
|
return await ListMinio(MinioClient, path, recursive, cancellationToken);
|
|
else
|
|
return await ListLocal(path, recursive);
|
|
}
|
|
|
|
private async Task<List<string>> ListLocal(string path, bool recursive)
|
|
{
|
|
var result = new List<string>();
|
|
var localFolder = StorageConfig.LocalFolder;
|
|
var folderPath = Path.Combine(localFolder, path);
|
|
|
|
if (!Directory.Exists(folderPath))
|
|
return result;
|
|
|
|
if (recursive)
|
|
{
|
|
var allFiles = Directory.GetFiles(folderPath, "*", SearchOption.AllDirectories);
|
|
var basePath = Path.GetFullPath(folderPath);
|
|
foreach (var file in allFiles)
|
|
{
|
|
var relativePath = Path.GetRelativePath(basePath, file);
|
|
var relativePathNormalized = relativePath.Replace('\\', '/');
|
|
result.Add(relativePathNormalized);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
var files = Directory.GetFiles(folderPath);
|
|
foreach (var file in files)
|
|
{
|
|
result.Add(Path.GetFileName(file));
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private async Task<List<string>> ListMinio(IMinioClient minioClient, string path, bool recursive, CancellationToken cancellationToken)
|
|
{
|
|
return await ExecuteWithRetryAsync(async () =>
|
|
{
|
|
var result = new List<string>();
|
|
var listArgs = new ListObjectsArgs()
|
|
.WithBucket(StorageConfig.Bucket)
|
|
.WithPrefix($"{path}/")
|
|
.WithRecursive(recursive);
|
|
|
|
await foreach (var item in minioClient.ListObjectsEnumAsync(listArgs, cancellationToken))
|
|
{
|
|
if (!string.IsNullOrEmpty(item.Key))
|
|
{
|
|
var relativePath = item.Key.StartsWith($"{path}/")
|
|
? item.Key[(path.Length + 1)..]
|
|
: item.Key;
|
|
result.Add(relativePath);
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}, cancellationToken);
|
|
}
|
|
|
|
public async Task CopyAsync(string sourcePath, string destPath, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(sourcePath))
|
|
throw new ArgumentException("SourcePath cannot be null or empty", nameof(sourcePath));
|
|
if (string.IsNullOrWhiteSpace(destPath))
|
|
throw new ArgumentException("DestPath cannot be null or empty", nameof(destPath));
|
|
ValidatePath(sourcePath);
|
|
ValidatePath(destPath);
|
|
ValidateObjectName(objectName);
|
|
|
|
if (!StorageConfig.UsingLocal && MinioClient != null)
|
|
await CopyMinio(MinioClient, sourcePath, destPath, objectName, cancellationToken);
|
|
else
|
|
await CopyLocal(sourcePath, destPath, objectName);
|
|
}
|
|
|
|
private async Task CopyLocal(string sourcePath, string destPath, string objectName)
|
|
{
|
|
var localFolder = StorageConfig.LocalFolder;
|
|
var sourceFolder = Path.Combine(localFolder, sourcePath);
|
|
var destFolder = Path.Combine(localFolder, destPath);
|
|
|
|
if (!Directory.Exists(destFolder))
|
|
Directory.CreateDirectory(destFolder);
|
|
|
|
var extensions = new[] { ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".pdf", ".json", ".txt", ".bin" };
|
|
foreach (var ext in extensions)
|
|
{
|
|
var sourceFile = Path.Combine(sourceFolder, $"{objectName}{ext}");
|
|
if (File.Exists(sourceFile))
|
|
{
|
|
var destFile = Path.Combine(destFolder, $"{objectName}{ext}");
|
|
File.Copy(sourceFile, destFile, true);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task CopyMinio(IMinioClient minioClient, string sourcePath, string destPath, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
await ExecuteWithRetryAsync(async () =>
|
|
{
|
|
var copySourceArgs = new CopySourceObjectArgs()
|
|
.WithBucket(StorageConfig.Bucket)
|
|
.WithObject($"{sourcePath}/{objectName}");
|
|
|
|
var copyObjectArgs = new CopyObjectArgs()
|
|
.WithBucket(StorageConfig.Bucket)
|
|
.WithObject($"{destPath}/{objectName}")
|
|
.WithCopyObjectSource(copySourceArgs);
|
|
|
|
await minioClient.CopyObjectAsync(copyObjectArgs, cancellationToken);
|
|
return Task.CompletedTask;
|
|
}, cancellationToken);
|
|
}
|
|
|
|
public async Task<StorageMetadata> GetMetadataAsync(string path, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(path))
|
|
throw new ArgumentException("Path cannot be null or empty", nameof(path));
|
|
ValidatePath(path);
|
|
ValidateObjectName(objectName);
|
|
|
|
if (!StorageConfig.UsingLocal && MinioClient != null)
|
|
return await GetMetadataMinio(MinioClient, path, objectName, cancellationToken);
|
|
else
|
|
return await GetMetadataLocal(path, objectName);
|
|
}
|
|
|
|
private async Task<StorageMetadata> GetMetadataLocal(string path, string objectName)
|
|
{
|
|
var localFolder = StorageConfig.LocalFolder;
|
|
var folderPath = Path.Combine(localFolder, path);
|
|
|
|
var extensions = new[] { ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".pdf", ".json", ".txt", ".bin" };
|
|
foreach (var ext in extensions)
|
|
{
|
|
var filePath = Path.Combine(folderPath, $"{objectName}{ext}");
|
|
if (File.Exists(filePath))
|
|
{
|
|
var fileInfo = new FileInfo(filePath);
|
|
return new StorageMetadata
|
|
{
|
|
Path = path,
|
|
ObjectName = objectName,
|
|
Size = fileInfo.Length,
|
|
ContentType = GetContentTypeFromExtension(ext),
|
|
LastModified = fileInfo.LastWriteTime,
|
|
ETag = null
|
|
};
|
|
}
|
|
}
|
|
|
|
throw new FileNotFoundException($"File not found: {path}/{objectName}");
|
|
}
|
|
|
|
private async Task<StorageMetadata> GetMetadataMinio(IMinioClient minioClient, string path, string objectName, CancellationToken cancellationToken)
|
|
{
|
|
return await ExecuteWithRetryAsync(async () =>
|
|
{
|
|
var statObjectArgs = new StatObjectArgs()
|
|
.WithBucket(StorageConfig.Bucket)
|
|
.WithObject($"{path}/{objectName}");
|
|
|
|
var stat = await minioClient.StatObjectAsync(statObjectArgs, cancellationToken);
|
|
|
|
return new StorageMetadata
|
|
{
|
|
Path = path,
|
|
ObjectName = objectName,
|
|
Size = stat.Size,
|
|
ContentType = stat.ContentType ?? string.Empty,
|
|
LastModified = stat.LastModified,
|
|
ETag = stat.ETag
|
|
};
|
|
}, cancellationToken);
|
|
}
|
|
|
|
private static string GetContentTypeFromExtension(string extension)
|
|
{
|
|
return extension.ToLowerInvariant() switch
|
|
{
|
|
".png" => "image/png",
|
|
".jpg" => "image/jpeg",
|
|
".jpeg" => "image/jpeg",
|
|
".gif" => "image/gif",
|
|
".bmp" => "image/bmp",
|
|
".webp" => "image/webp",
|
|
".pdf" => "application/pdf",
|
|
".json" => "application/json",
|
|
".txt" => "text/plain",
|
|
".csv" => "text/csv",
|
|
".xml" => "application/xml",
|
|
".zip" => "application/zip",
|
|
_ => "application/octet-stream"
|
|
};
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
if (MinioClient is IDisposable disposable)
|
|
{
|
|
disposable.Dispose();
|
|
}
|
|
GC.SuppressFinalize(this);
|
|
}
|
|
} |