赞
踩
unity-多线程断点下载HttpWebRequest
在 unity-多线程异步下载HttpWebRequest 的基础上, 增加断点下载功能 (代码有稍作修改, 适配断点下载), 应用场景是下载一个大文件时, 将文件分割成多个片段进行下载, 即使中间断网后, 重新下载时重已累积的下载大小的基础上, 继续剩余未下载完内容.
效果如下, 最终下载完大小是 31,715KB (中间鼠标框选时, 进程已经杀了, 中断了下载, 后面重启进程继续累计下载)
断点下载 MultiResumeMgr.cs
using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Text; using LuaInterface; using UnityEngine; // 多线程断点下载 public class MultiResumeMgr : MonoBehaviour { private static MultiResumeMgr _instance; public static MultiResumeMgr Instance { get { return _instance; } } private List<MultiResumeObj> mObjList = new List<MultiResumeObj>(); public MultiDownMgr multiDownIns; public int chunkMinSize = 1024 * 1024; // 分块最小值 public int mergeBufferSize = 1024 * 1024; void Awake() { _instance = this; } void Start() { if (multiDownIns == null) { multiDownIns = gameObject.GetComponent<MultiDownMgr>(); if (multiDownIns == null) { multiDownIns = gameObject.AddComponent<MultiDownMgr>(); multiDownIns.bufferSize = chunkMinSize; } } } long GetContentLength(string url) { try { var req = (HttpWebRequest) WebRequest.CreateDefault(new Uri(url)); req.Method = "HEAD"; req.Timeout = 5000; var res = (HttpWebResponse) req.GetResponse(); long contentLen = 0; if (res.StatusCode == HttpStatusCode.OK) { contentLen = res.ContentLength; } res.Close(); req.Abort(); return contentLen; } catch (System.Exception ex) { return 0; } } public void Request(string url, string path, LuaFunction completeFn, LuaFunction progressFn, int chunkNum = 10) { MultiResumeObj msObj = GetMultiResumeObj(url, path, chunkNum); msObj.luaFn = completeFn; msObj.progressFn = progressFn; RequestObj(msObj); } [NoToLua] public void RequestObj(MultiResumeObj msObj) { mObjList.Add(msObj); StartCoroutine(StartTask(msObj)); } // 获取分块信息 private MultiResumeObj GetMultiResumeObj(string url, string path, int chunkNum) { long contentLen = GetContentLength(url); // LogUtil.D("--- contentLen: {0}", contentLen); MultiResumeObj msObj = new MultiResumeObj(); msObj.url = url; msObj.path = path; msObj.contentLen = contentLen; msObj.lenFile = string.Format("{0}.len", path);; msObj.chunkLst = new List<MultiDownObj>(); // 小于 最小值 if (contentLen < chunkMinSize || chunkNum < 2) { MultiDownObj mdObj = new MultiDownObj(); mdObj.url = url; mdObj.path = path; mdObj.from = 0; mdObj.to = contentLen - 1; mdObj.order = 0; msObj.chunkLst.Add(mdObj); } else { // 计算分块大小 long chunkSize = contentLen / chunkNum + 1; // LogUtil.D("--- chunkSize: {0}", chunkSize); long offset = 0; for (int i = 0; i < chunkNum; i++) { long from = offset; long to = i == chunkNum - 1 ? contentLen - 1 : offset + chunkSize; // 最后一块大小直接读到 end offset = to + 1; // 下一片段的开始值偏移量 string partFile = string.Format("{0}.part{1:00}", path, i + 1); // LogUtil.D("--- idx: {0}, from: {1}, to: {2}, path: {3}", i, from, to, partFile); MultiDownObj mdObj = new MultiDownObj(); mdObj.url = url; mdObj.path = partFile; mdObj.from = from; mdObj.to = to; mdObj.order = i; msObj.chunkLst.Add(mdObj); } } return msObj; } // 开始任务 IEnumerator StartTask(MultiResumeObj msObj) { yield return null; // 判断要下载的文件以缓存文件的记录长度是否一致, 不一致要清楚 part CacheDiffDeal(msObj); SortedList<int, MultiDownObj> sortLst = new SortedList<int, MultiDownObj>(); foreach (MultiDownObj mdObj in msObj.chunkLst) { sortLst.Add(mdObj.order, mdObj); } int cnt = 0; // 此回调已经是主线程, 可以打 log MultiDownDlg onCompleteFn = (MultiDownObj mdObj) => { cnt++; // 下载进度通知 lua if (msObj.progressFn != null) { msObj.progressFn.Call(cnt, msObj.chunkLst.Count); } // 只有一块, 不需要合并文件 if (msObj.chunkLst.Count == 1) { Close(msObj); msObj.isDone = true; return; } if (cnt == sortLst.Count) { // 全部下载完成 // 是否全部 success bool isAllSucc = true; string partErr = ""; foreach (var item in sortLst) { if (!(item.Value.code == (int) HttpStatusCode.OK || item.Value.code == (int) HttpStatusCode.PartialContent)) { isAllSucc = false; partErr += string.Format("\n order: {0}, code: {1}, msg: {1}", item.Value.order, item.Value.code, item.Value.path); } } if (isAllSucc) { // LogUtil.D("--- all success"); msObj.code = (int) HttpStatusCode.PartialContent; List<string> fileLst = sortLst.Select((item) => { return item.Value.path; }).ToList(); Close(msObj); // 必须先关闭文件流, 才能进行读取合并 string cbErr = Combine(fileLst, msObj.path); if (cbErr != null) { // 合并文件失败 LogUtil.E("--- combine err: {0}", cbErr); msObj.code = (int) EMSErr.Combine; msObj.path = partErr; } RemoveParts(msObj); } else { LogUtil.E("--- some part err: {0}", partErr); msObj.code = (int) EMSErr.DownPart; msObj.path = partErr; } msObj.isDone = true; } }; foreach (MultiDownObj mdObj in msObj.chunkLst) { FileStream fs = new FileStream(mdObj.path, FileMode.Append, FileAccess.Write); mdObj.fs = fs; mdObj.csFn = onCompleteFn; long rest = (mdObj.to - mdObj.from + 1) - fs.Length; // LogUtil.D("--- order: {0}, rest: {1}, path: {2}", mdObj.order, rest, mdObj.path); if (rest > 0) { mdObj.from = mdObj.to - rest + 1; // 重新计算下载开始值 multiDownIns.RequestObj(mdObj); } else { // 无需再下载, 直接回调 mdObj.code = (int) HttpStatusCode.PartialContent; mdObj.csFn(mdObj); } } } string Combine(List<string> fileLst, string outPath) { Utils.DeleteFile(outPath); byte[] buffer = new byte[mergeBufferSize]; using(FileStream outStream = new FileStream(outPath, FileMode.Create)) { int readLen = 0; FileStream srcStream = null; try { for (int i = 0; i < fileLst.Count; i++) { srcStream = new FileStream(fileLst[i], FileMode.Open); while ((readLen = srcStream.Read(buffer, 0, mergeBufferSize)) > 0) { outStream.Write(buffer, 0, readLen); outStream.Flush(); } srcStream.Close(); } } catch (System.Exception ex) { if (srcStream != null) { srcStream.Close(); } return ex.Message; } } return null; } /* // 合并文件流, fsLst 必须是顺序的, 没必要写成异步 // FileMode.Append 模式必须要要求是 FileAccess.Write 权限, 而 FileAccess.Write 权限又不能 read, 所以没法直接把 fileStream 合并, 所以这个方法不能使用 void Combine(List<FileStream> fsLst, string outPath) { Utils.DeleteFile(outPath); byte[] buffer = new byte[mergeBufferSize]; using(FileStream outStream = new FileStream(outPath, FileMode.Create)) { int readLen = 0; for (int i = 0; i < fsLst.Count; i++) { FileStream srcStream = fsLst[i]; while ((readLen = srcStream.Read(buffer, 0, mergeBufferSize)) > 0) { outStream.Write(buffer, 0, readLen); outStream.Flush(); } } } } */ // 缓存文件长度记录不一致, 删掉 void CacheDiffDeal(MultiResumeObj msObj) { if (msObj.lenFile != null) { string txt = Utils.ReadAllTextFromFile(msObj.lenFile); if (txt != null) { long cacheLen = long.Parse(txt); if (msObj.contentLen != cacheLen) { // LogUtil.D("--- cache len diff, dstLen: {0}, cacheLen: {1}", msObj.contentLen, cacheLen); RemoveParts(msObj); } } } Utils.WriteFileUTF8(msObj.lenFile, msObj.contentLen.ToString()); } // 删除临时文件 void RemoveParts(MultiResumeObj msObj) { Utils.DeleteFile(msObj.lenFile); if (msObj != null && msObj.chunkLst != null) { foreach (MultiDownObj mdObj in msObj.chunkLst) { Utils.DeleteFile(mdObj.path); } } } // 释放资源 void Close(MultiResumeObj msObj) { if (msObj == null) return; msObj.isActive = false; // 释放文件流 if (msObj.chunkLst != null) { foreach (var mdObj in msObj.chunkLst) { mdObj.isActive = false; if (mdObj.fs != null) { mdObj.fs.Close(); mdObj.fs = null; } } } if (msObj.progressFn != null) { msObj.progressFn.Dispose(); msObj.progressFn = null; } } public void StopDown(string url) { for (int i = 0; i < mObjList.Count; ++i) { MultiResumeObj msObj = mObjList[i]; Close(msObj); if (msObj.url == url) { mObjList.RemoveAt(i); i -= 1; } } } public void StopAll() { for (int i = 0; i < mObjList.Count; ++i) { MultiResumeObj msObj = mObjList[i]; StopDown(msObj.url); } mObjList.Clear(); } void OnDestroy() { StopAll(); } void Update() { for (int i = 0; i < mObjList.Count; ++i) { MultiResumeObj msObj = mObjList[i]; if (msObj.isDone) { // LogUtil.D("--- msObj done, code: {0}, path: {1}", msObj.code, msObj.path); if (msObj.luaFn != null) { msObj.luaFn.Call(msObj.code, msObj.url, msObj.path); msObj.luaFn.Dispose(); msObj.luaFn = null; } if (msObj.csFn != null) { msObj.csFn(msObj); msObj.csFn = null; } Close(msObj); mObjList.RemoveAt(i); i -= 1; } } } [NoToLua] public void Test() { string url = "https://www.rmgstation.com/download/rvv3_1053-3-2_v0.301.11.68_vc30_rummy_luxury_20220608T161228_00c1c5e6697b674a4b8db7409be645d0.apk"; string path01 = string.Format("{0}/apks/aaa.apk", Application.persistentDataPath); string path02 = string.Format("{0}/apks/bbb.apk", Application.persistentDataPath); Request(url, path01, null, null, 10); Request(url, path02, null, null, 10); } }
多线程下载 MultiDownMgr.cs
using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Net; using System.Text; using System.Threading; using LuaInterface; using UnityEngine; // 参考: https://docs.microsoft.com/zh-cn/dotnet/framework/network-programming/making-asynchronous-requests // 多线程异步下载 public class MultiDownMgr : MonoBehaviour { private static MultiDownMgr _instance; public static MultiDownMgr Instance { get { return _instance; } } private AsyncCallback rspCb = null; private AsyncCallback readCb = null; private List<MultiDownObj> mObjList = new List<MultiDownObj>(); public int timeout = 1000 * 10; // 10s 超时 public int bufferSize = 1024 * 4; public bool keepAlive = true; public int connectLimit { set { ServicePointManager.DefaultConnectionLimit = value; } // 并发线程数量 } void Awake() { _instance = this; rspCb = new AsyncCallback(ResponseCb); readCb = new AsyncCallback(ReadDataCb); connectLimit = 32; // 默认 32 个并发 } // lua 接口 public void Request(string url, string path, LuaFunction fn) { // LogUtil.D("--- Request, url: {0}, path: {1}", url, path); MultiDownObj mdObj = new MultiDownObj(); mdObj.url = url; mdObj.path = path; mdObj.luaFn = fn; RequestObj(mdObj); } // cs 接口 [NoToLua] public void Request(string url, string path, MultiDownDlg fn, long from = 0, long to = 0, int order = 0) { // LogUtil.D("--- Request, url: {0}, path: {1}", url, path); MultiDownObj mdObj = new MultiDownObj(); mdObj.url = url; mdObj.path = path; mdObj.csFn = fn; mdObj.from = from; mdObj.to = to; mdObj.order = order; RequestObj(mdObj); } [NoToLua] public void RequestObj(MultiDownObj mdObj) { mObjList.Add(mdObj); try { mdObj.httpReq = WebRequest.Create(mdObj.url) as HttpWebRequest; mdObj.httpReq.Method = "GET"; mdObj.httpReq.Timeout = timeout; mdObj.httpReq.KeepAlive = keepAlive; // 设置为 false 会导致中断下载, 报错: Remote prematurely closed connection. // 下载片段 if (IsResume(mdObj)) mdObj.httpReq.AddRange(mdObj.from, mdObj.to); mdObj.httpReq.BeginGetResponse(rspCb, mdObj); } catch (System.Exception ex) { Close(mdObj); mdObj.code = (int) EMDErr.CreateRequest; mdObj.path = ex.Message; mdObj.isDone = true; } } void ResponseCb(IAsyncResult ar) { MultiDownObj mdObj = ar.AsyncState as MultiDownObj; try { HttpWebResponse response = mdObj.httpReq.EndGetResponse(ar) as HttpWebResponse; if (response == null) { Close(mdObj); mdObj.code = (int) EMDErr.NullResponse; mdObj.isDone = true; return; } mdObj.httpRsp = response; mdObj.code = (int) response.StatusCode; mdObj.rspStream = response.GetResponseStream(); if (!(response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.PartialContent)) { // 206 返回值为 断点下载的正常状态码 Close(mdObj); mdObj.isDone = true; return; } // 创建父目录 string dirPath = System.IO.Path.GetDirectoryName(mdObj.path); if (!Utils.IsDirectoryExist(dirPath)) { Utils.CreateDirectory(dirPath); } if (mdObj.fs == null) { mdObj.fs = new FileStream(mdObj.path, FileMode.Create); } mdObj.buffer = new byte[bufferSize]; if (mdObj.isActive) { mdObj.rspStream.BeginRead(mdObj.buffer, 0, bufferSize, readCb, mdObj); } else { Close(mdObj); mdObj.code = (int) EMDErr.DeActive; mdObj.path = "DeActive"; mdObj.isDone = true; } } catch (System.Exception ex) { Close(mdObj); mdObj.code = (int) EMDErr.GetResponse; mdObj.path = ex.Message; mdObj.isDone = true; } } void ReadDataCb(IAsyncResult ar) { // Thread.Sleep(50); // TODO: 测试 MultiDownObj mdObj = ar.AsyncState as MultiDownObj; try { int read = mdObj.rspStream.EndRead(ar); if (read > 0) { mdObj.fs.Write(mdObj.buffer, 0, read); mdObj.fs.Flush(); if (mdObj.isActive) { mdObj.rspStream.BeginRead(mdObj.buffer, 0, bufferSize, readCb, mdObj); } else { Close(mdObj); mdObj.code = (int) EMDErr.DeActive; mdObj.path = "DeActive"; mdObj.isDone = true; } } else { Close(mdObj); mdObj.isDone = true; } } catch (System.Exception ex) { Close(mdObj); mdObj.code = (int) EMDErr.ReadStream; mdObj.path = ex.Message; mdObj.isDone = true; return; } } // 释放资源 void Close(MultiDownObj mdObj) { if (mdObj == null) return; mdObj.isActive = false; // 断点下载不能释放文件流 if (mdObj.fs != null) { mdObj.fs.Close(); mdObj.fs = null; } if (mdObj.rspStream != null) { mdObj.rspStream.Close(); mdObj.rspStream = null; } if (mdObj.httpRsp != null) { mdObj.httpRsp.Close(); mdObj.httpRsp = null; } if (mdObj.httpReq != null) { mdObj.httpReq.Abort(); mdObj.httpReq = null; } mdObj.buffer = null; } // 是否断点下载 bool IsResume(MultiDownObj mdObj) { return mdObj.from >= 0 && mdObj.to >= 0; } public void StopDown(string url) { for (int i = 0; i < mObjList.Count; ++i) { MultiDownObj mdObj = mObjList[i]; mdObj.isActive = false; if (mdObj.url == url) { mObjList.RemoveAt(i); i -= 1; } } } public void StopAll() { for (int i = 0; i < mObjList.Count; ++i) { MultiDownObj mdObj = mObjList[i]; mdObj.isActive = false; } mObjList.Clear(); } void OnDestroy() { StopAll(); } void Update() { for (int i = 0; i < mObjList.Count; ++i) { MultiDownObj mdObj = mObjList[i]; if (mdObj.isDone) { // LogUtil.D("--- mdObj done, code: {0}, path: {1}", mdObj.code, mdObj.path); if (mdObj.luaFn != null) { mdObj.luaFn.Call(mdObj.code, mdObj.url, mdObj.path); mdObj.luaFn.Dispose(); mdObj.luaFn = null; } if (mdObj.csFn != null) { mdObj.csFn(mdObj); mdObj.csFn = null; } mObjList.RemoveAt(i); i -= 1; } } } }
透传参数对象的封装 MultiDownObj.cs
using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Net; using System.Text; using LuaInterface; using UnityEngine; public delegate void MultiDownDlg(MultiDownObj mdObj); public enum EMDErr : int { CreateRequest = -10001, GetResponse = -10002, NullResponse = -10003, ReadStream = -10004, DeActive = -10005, } public enum EMSErr : int { DownPart = -20001, Combine = -20002, } public class MultiDownObj { public string url; public string path; public LuaFunction luaFn; public MultiDownDlg csFn; public bool isActive = true; // 断点信息 public long from; public long to; public int order; // 透传参数 public bool isDone = false; public int code; public HttpWebRequest httpReq; public HttpWebResponse httpRsp; public byte[] buffer; public Stream rspStream; public FileStream fs; } public class MultiResumeObj : MultiDownObj { public List<MultiDownObj> chunkLst; public long contentLen; public string lenFile = null; public LuaFunction progressFn; }
lua 断点下载工具的封装
--====================================================================== -- descrip: 多线程断点下载 --====================================================================== local CMultiResumeManager = class() CMultiResumeManager.__name = "CMultiResumeManager" function CMultiResumeManager.Init(self) self._dlIns = nil end function CMultiResumeManager.StartDownCnt(self, url, savePath, completeFn, progressFn, cnt, interval, chunkNum) local wrapFn if completeFn then cnt = cnt or 1 interval = interval or 1 wrapFn = function(isSucc, ...) if not isSucc and cnt > 1 then cnt = cnt - 1 gTimeMgr:SetTimeOut(interval, function() -- 跳 n 秒再尝试下载 self:StartDown(url, savePath, wrapFn, progressFn) end) else completeFn(isSucc, ...) end end end self:StartDown(url, savePath, wrapFn, progressFn, chunkNum) end function CMultiResumeManager.StartDown(self, url, savePath, completeFn, progressFn, chunkNum) chunkNum = chunkNum or 5 -- 默认分块 local completeWrapFn if completeFn then -- 跳一帧在回调回去 completeWrapFn = function(code02, url02, path02) gTimeMgr:SetTimeOut(0, function() completeFn(code02 == 206, 100, 100, url02, path02) end) end end self._dlIns:Request(url, savePath, completeWrapFn, progressFn, chunkNum) end function CMultiResumeManager.InitDownloader(self) if not Feature.IsSupportCs(Feature.Cs.MultiResume) then return end if not IsNull(self._dlIns) then return end self._dlIns = gGame:GetGameMgrGo():GetComponent(typeof(MultiResumeMgr)) if not IsNull(self._dlIns) then return end gLog("--- CMultiDownManager.InitDownloader") self._dlIns = gGame:GetGameMgrGo():AddComponent(typeof(MultiResumeMgr)) self._dlIns.chunkMinSize = 1024 * 1024 -- 分块最小值 self._dlIns.mergeBufferSize = 1024 * 1024 -- 合并文件缓冲大小 -- 尝试获取已有的异步下载 local multiDownIns = gGame:GetGameMgrGo():GetComponent(typeof(MultiDownMgr)) if not IsNull(multiDownIns) then gLog("--- CMultiDownManager set multiDownIns") multiDownIns.connectLimit = 16 -- 并发数量 multiDownIns.timeout = 1000*10 -- 10s 超时 multiDownIns.bufferSize = 1024 * 1024 self._dlIns.multiDownIns = multiDownIns end end return CMultiResumeManager
lua 测试代码
function gDebugCustom.MultiResume() local progressFn = function(downLen, totalLen) gLog("--- progressFn, downLen: {0}, totalLen: {1}", downLen, totalLen) end local completeFn = function(isSucc, downLen, totalLen, url, savePath) gLog("--- completeFn, isSucc: {0}, downLen: {1}, totalLen: {2}, url: {3}, savePath: {4} ", isSucc, downLen, totalLen, url, savePath) end local url = "https://www.aaa.com/aaa.apk" local path = gTool.PathJoin(Application.persistentDataPath, string.formatExt("apks/aaa.apk")) gLog("--- start down") -- 多线程断点 gMultiResumeMgr:InitDownloader() gMultiResumeMgr:StartDown(url, path, completeFn, nil, 10) -- gMultiResumeMgr:StartDownCnt(url, path, completeFn, nil, 3, 1, 10) end
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。