我有一段 c# 代码，是关于阿里云OSS函数计算打包的，我想边下载边打包，但是基于内存有限，对于大文件只能分片下载分片上传，我遇到了分片上传文件格式损坏的问题，只能实现打包一个完整文件才上传，这里可能需要换个打包方式才能实现。如何才能分片上传并且保证文件格式完好
#region 下载-边打包zip到内存边上传
/// <summary>
/// 下载-边打包zip到内存边上传
/// </summary>
/// <param name="saveAsPathAndFileName">保存路径，以及oss的key</param>
/// <param name="sourceResourcePathsArray">打包资源列表</param>
/// <returns></returns>
private void Testtesttttttt(string saveAsPathAndFileName, string[] sourceResourcePathsArray)
{
OSSFileWriter writer = new OSSFileWriter(_clientAliyunOss, saveAsPathAndFileName);
byte[] byteArray;
var partSize = (_IsTest ? 1 * 512 * 1024 : MAX_PART_SIZE);
using (var zipStream = new MemoryStream())
{
//打包文件的大小
long currentMemoryStreamSize = 0;
// 使用ZipArchive创建ZIP存档
using ZipArchive archive = new ZipArchive(zipStream, ZipArchiveMode.Create, leaveOpen: true);
foreach (string objPath in sourceResourcePathsArray)
{
string objPathDecode = objPath.ConvertBase64String();
OSSFileReader fileReadeer = new OSSFileReader(_clientAliyunOss, partSize);
var zipEntry = archive.CreateEntry(Path.GetFileName(objPathDecode), CompressionLevel.Optimal);
using var entryStream = zipEntry.Open();
foreach (var ossObject in fileReadeer.GetObject(objPathDecode))
{
//// 如果内存流大小即将超过阈值，则上传ZIP文件并清空内存流(TODO: 暂时行不通，文件损坏,只能在打包好单个文件之后上传能用)
//if (currentMemoryStreamSize > 0 && (currentMemoryStreamSize + ossObject.ContentLength) > 1 * 1024 * 1024)
//{
// zipStream.Flush();
// entryStream.Flush();
// writer.Write(zipStream);
// result.OssPartlyUpload = writer.Write(zipStream);
// result.ProcessFileCount = tempProcessFileCount;
// result.LastProcessFileName = objPath;
// result.Update();
// currentMemoryStreamSize = 0;
//}
// 将文件内容写入 Zip 文件
using var objectStream = ossObject.Content;
objectStream.CopyTo(entryStream);
currentMemoryStreamSize += ossObject.ContentLength;
}
zipStream.Flush();
entryStream.Flush();
//只有做了dispose操作后的压缩文件才是完整的
entryStream.Dispose();
// 如果内存流大小即将超过阈值，则上传ZIP文件并清空内存流（TODO:这里是打包单个文件再上传，如遇到单个文件过大会造成内存过大）
if (currentMemoryStreamSize > partSize)
{
writer.Write(zipStream);
currentMemoryStreamSize = 0;
}
}
//只有做了dispose操作后的压缩包才是完整的
archive.Dispose();
zipStream.Flush();
byteArray = zipStream.ToArray();
zipStream.Dispose();
}
writer.Close(byteArray);
if (!FcEnvironment.OnlyLogError)
_Logger?.LogInformation($"Zip file created: {saveAsPathAndFileName}");
}
#endregion 下载-边打包zip到内存边上传
分片下载的代码
public class OSSFileReader
{
private readonly ClientAliyunOss _clientAliyunOss;
private readonly long _partSize;
public OSSFileReader(ClientAliyunOss clientAliyunOss, int partSize)
{
_clientAliyunOss = clientAliyunOss;
_partSize = partSize;
}
/// <summary>
/// 迭代器分片获取文件
/// </summary>
/// <param name="key"></param>
/// <returns></returns>
public IEnumerable<OssObject> GetObject(string key)
{
var objectMetadata = _clientAliyunOss.GetObjectMetaData(key);
var fileLength = objectMetadata.ContentLength;
if (fileLength > _partSize)
{
var partCount = CalPartCount(fileLength, _partSize);
for (var i = 0; i < partCount; i++)
{
var startPos = _partSize * i;
var endPos = _partSize * i + (_partSize < (fileLength - startPos) ? _partSize : (fileLength - startPos)) - 1;
yield return GetObjectPartly(key, startPos, endPos);
}
}
else
{
var ossObject = _clientAliyunOss.OssGet(key);
yield return ossObject;
}
}
private int CalPartCount(long fileLength, long partSize)
{
var partCount = (int)(fileLength / partSize);
if (fileLength % partSize != 0)
{
partCount++;
}
return partCount;
}
private OssObject GetObjectPartly(string key, long startPos, long endPos)
{
var ossObject = _clientAliyunOss.OssGet(key, startPos, endPos);
return ossObject;
}
}
分片上传的代码
public class OSSFileWriter
{
private readonly ClientAliyunOss _clientAliyunOss;
private OssPartlyUpload ossPartlyUpload;
public OSSFileWriter(ClientAliyunOss clientAliyunOss, string ossKey)
{
_clientAliyunOss = clientAliyunOss;
ossPartlyUpload = new OssPartlyUpload()
{
OssKey = ossKey,
Finished = false
};
}
/// <summary>
/// 设置断点续传信息
/// </summary>
/// <param name="partlyUpload"></param>
public void setPartInfo(OssPartlyUpload partlyUpload)
{
ossPartlyUpload = partlyUpload;
}
private string GetUploadId()
{
if (string.IsNullOrEmpty(ossPartlyUpload.UploadId))
{
ossPartlyUpload.UploadId = _clientAliyunOss.InitiateMultipartUpload(ossPartlyUpload.OssKey);
ossPartlyUpload.PartNumber = 0;
ossPartlyUpload.PartList = new List<OssPartETag>();
}
return ossPartlyUpload.UploadId;
}
public OssPartlyUpload Write(Stream data)
{
SendBuffer(data);
data.SetLength(0);
data.Position = 0;
return ossPartlyUpload;
}
private string SendBuffer(Stream data)
{
var id = GetUploadId();
if (string.IsNullOrEmpty(id))
{
//log 上传ID为空
return null;
}
else
{
UploadPart(data);
return id;
}
}
private void UploadPart(Stream data)
{
data.Seek(0, SeekOrigin.Begin);
ossPartlyUpload.PartNumber++;
var part = _clientAliyunOss.UploadPart(ossPartlyUpload.OssKey, ossPartlyUpload.UploadId, data, ossPartlyUpload.PartNumber);
ossPartlyUpload.PartList.Add(new OssPartETag
{
PartNumber = part.PartNumber,
ETag = part.ETag,
Crc64 = part.Crc64,
Length = part.Length
});
}
/// <summary>
/// 结束分片上传
/// </summary>
/// <param name="data"></param>
private void Close(Stream data)
{
if (ossPartlyUpload.Finished) return;
if (string.IsNullOrEmpty(ossPartlyUpload.UploadId))
{
if (data.Length > 0)
{
data.Seek(0, SeekOrigin.Begin);
_clientAliyunOss.OssSave(new OssAliyunSaveRequest
{
OssKey = ossPartlyUpload.OssKey,
Content = data,
ContentMime = "application/zip"
});
}
}
else
{
if (data.Length > 0)
{
UploadPart(data);
}
_clientAliyunOss.CompleteMultipartUpload(ossPartlyUpload.OssKey, ossPartlyUpload.UploadId, ossPartlyUpload.PartList);
}
ossPartlyUpload.Finished = true;
}
public OssPartlyUpload Close(byte[] bytes)
{
Stream stream = new MemoryStream(bytes);
Close(stream);
return ossPartlyUpload;
}
}
