From 9bf8505c776883b5551767e327172ff9756fd5c1 Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 23 Dec 2018 21:05:13 +0100 Subject: [PATCH] First Commit of reworked updater --- .../Scripts/Updater/HttpHandler.cs | 2 + .../Scripts/Updater/UpdaterBehaviour.cs | 538 +++++++++--------- 2 files changed, 262 insertions(+), 278 deletions(-) diff --git a/Assets/RothenburgAR/Scripts/Updater/HttpHandler.cs b/Assets/RothenburgAR/Scripts/Updater/HttpHandler.cs index 4b2865d..b27b857 100644 --- a/Assets/RothenburgAR/Scripts/Updater/HttpHandler.cs +++ b/Assets/RothenburgAR/Scripts/Updater/HttpHandler.cs @@ -66,6 +66,7 @@ namespace RothenburgAR.Updater public UnityWebRequestAsyncOperation Operation { get; set; } private bool isDone = false; + public bool IsError { get; set; } public bool IsDone { @@ -85,6 +86,7 @@ namespace RothenburgAR.Updater this.Operation.completed += (_) => { this.isDone = true; + this.IsError = request.isNetworkError || request.isHttpError || download.data.Length == 0; }; } } diff --git a/Assets/RothenburgAR/Scripts/Updater/UpdaterBehaviour.cs b/Assets/RothenburgAR/Scripts/Updater/UpdaterBehaviour.cs index 500e9f4..06ee324 100644 --- a/Assets/RothenburgAR/Scripts/Updater/UpdaterBehaviour.cs +++ b/Assets/RothenburgAR/Scripts/Updater/UpdaterBehaviour.cs @@ -6,8 +6,8 @@ using System; using Newtonsoft.Json; using RothenburgAR.Common; using System.IO; -using System.IO.Compression; using System.Security.Cryptography; +using System.Text.RegularExpressions; namespace RothenburgAR.Updater { @@ -20,13 +20,13 @@ namespace RothenburgAR.Updater public GameObject UpdateFailedDialog; private readonly float afterDownloadWaitTime = 10f; + private readonly int retriesUntilFailure = 3; + private readonly string trackerMainFile = "tracker.dat"; public ApiVersioncheckAnswer VersionAnswer { get; set; } - private List httpHandlers = new List(); + Dictionary> ExhibitMetas = new Dictionary>(); private LogFileHandler logFileHandler; - private readonly List downloadingMedia = new List(); - void Start() { #if !UNITY_EDITOR @@ -60,26 +60,36 @@ namespace RothenburgAR.Updater { // check for updates and ask for permission to download if there are any string versionMap = GenerateVersionMap(); - HttpHandler http = new HttpRequest(ApiInfo.VersionCheckEndpoint, HttpVerb.POST, versionMap).Send(); - httpHandlers.Add(http); + CheckForUpdates(versionMap, 0); + } + private void CheckForUpdates(string versionMap, int retries) + { + HttpHandler http = new HttpRequest(ApiInfo.VersionCheckEndpoint, HttpVerb.POST, versionMap).Send(); http.Operation.completed += ar => { - if (CheckNetworkErrors(http)) + Debug.Log("VersionAnswer: " + http.Download.text); + var answer = JsonConvert.DeserializeObject(http.Download.text); + if (http.IsError) { - return; + if (retries <= retriesUntilFailure) + { + CheckForUpdates(versionMap, retries + 1); + return; + } + else + { + UpdateFailed(http); + return; + } } - Debug.Log(http.Download.text); - - VersionAnswer = JsonConvert.DeserializeObject(http.Download.text); Debug.Log(string.Format("{1}-DONE with {0}", ApiInfo.VersionCheckEndpoint, Time.realtimeSinceStartup)); + VersionAnswer = answer; - var answer = VersionAnswer; - if (VersionAnswer != null && VersionAnswer.Data.TrueForAll(d => d.Meta.Status == VersionStatus.ok && d.Tracker.Status == VersionStatus.ok)) + if (VersionAnswer.Data.TrueForAll(d => d.Meta.Status == VersionStatus.ok && d.Tracker.Status == VersionStatus.ok)) { - // no updates required, continue to app - LoadMainScene(); + StartCoroutine(CheckFiles()); return; } @@ -165,9 +175,6 @@ namespace RothenburgAR.Updater UpdateConfirmationDialog.SetActive(false); UpdateFailedDialog.SetActive(false); - this.downloadingMedia.Clear(); - this.httpHandlers.ForEach(h => h.Request.Abort()); - this.httpHandlers.Clear(); if (VersionAnswer == null) { @@ -178,9 +185,9 @@ namespace RothenburgAR.Updater //TODO write languages to file the app can read (so that the languagemanager can decide which languages the user can choose from) - var updatedMeta = VersionAnswer.Data.Where(d => d.Meta.Status == VersionStatus.updated).ToList(); + var updatedExhibitions = VersionAnswer.Data.Where(d => d.Meta.Status == VersionStatus.updated).ToList(); var updatedTracker = VersionAnswer.Data.Where(d => d.Tracker.Status == VersionStatus.updated).ToList(); - updatedMeta.Union(updatedTracker).ToList().ForEach(updatedExhibition => + updatedExhibitions.Union(updatedTracker).Distinct().ToList().ForEach(updatedExhibition => { // create exhibition directories var path = Path.Combine(PathHelper.ExhibitionPath, updatedExhibition.Id); @@ -190,191 +197,242 @@ namespace RothenburgAR.Updater } }); - updatedMeta.ForEach(d => UpdateMeta(d)); - updatedTracker.ForEach(d => UpdateTracker(d)); - - var deletedData = VersionAnswer.Data.Where(d => - d.Meta.Status == VersionStatus.deleted - || d.Tracker.Status == VersionStatus.deleted).ToList(); - deletedData.ForEach(d => DeleteExhibition(d)); + updatedExhibitions.ForEach(d => UpdateExhibition(d, updatedExhibitions)); + StartCoroutine(UpdateExhibitionsCoroutine(updatedExhibitions)); } - private void DeleteExhibition(ApiExhibitionVersion exhibition) + private IEnumerator UpdateExhibitionsCoroutine(List updatedExhibitions) { - //TODO delete media as well if not used elsewhere? - var path = Path.Combine(PathHelper.ExhibitionPath, exhibition.Id); - Directory.Delete(path, true); + yield return new WaitUntil(() => updatedExhibitions.Count == 0); + StartCoroutine(CheckFiles()); } - private void UpdateMeta(ApiExhibitionVersion exhibition) + private void UpdateExhibition(ApiExhibitionVersion exhibition, List updatedExhibitions) { - var currentHandlers = new List(); + var exhibitionDownloads = new List(); foreach (var lang in VersionAnswer.Languages) { var path = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibition.Id, lang); - var url = exhibition.Meta.UpdateUrl.Replace("{lang}", lang); - var http = new HttpRequest(url, HttpVerb.GET).Send(); - httpHandlers.Add(http); - currentHandlers.Add(http); - - http.Operation.completed += ar => - { - if (CheckNetworkErrors(http)) - { - return; - } - - if (!Directory.Exists(path)) - { - Directory.CreateDirectory(path); - } - - File.WriteAllText(Path.Combine(path, "meta.json"), http.Download.text, System.Text.Encoding.UTF8); - Debug.Log(string.Format("{1}-DONE with {0}", url, Time.realtimeSinceStartup)); - - var exhibits = JsonConvert.DeserializeObject>(http.Download.text); - if (exhibits == null) - { - Debug.LogError(string.Format("Exhibit could not be parsed from Json:\nurl: {0}\njson: {1}", url, http.Download.text)); - ResetUpdateOnError(); - return; - } - else - { - foreach (var exhibit in exhibits) - { - UpdateMedia(exhibit, currentHandlers); - } - } - }; - } - - var versionPath = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibition.Id, "version.txt"); - var version = exhibition.Meta.UpdateVersion.ToString(); - StartCoroutine(UpdateVersionFileCoroutine(currentHandlers, versionPath, version)); - } - - private IEnumerator UpdateVersionFileCoroutine(List currentHandlers, string versionPath, string version) - { - // updates version file only if all sub files were downloaded successfully - yield return new WaitUntil(() => currentHandlers.All(h => h.IsDone)); - if (currentHandlers.Any(http => http.Request.isNetworkError || http.Request.isHttpError)) yield break; - - File.WriteAllText(versionPath, version); - } - - private void UpdateMedia(ApiExhibit exhibit, List batchHandlers) - { - var mediaIDs = exhibit.Pois.Select(p => p.MediaId).Except(new List { null }).ToList(); - - foreach (var mediaId in mediaIDs) - { - if (downloadingMedia.Contains(mediaId)) continue; - downloadingMedia.Add(mediaId); - - var path = PathHelper.CombinePaths(PathHelper.MediaPath, mediaId); - var filepath = Path.Combine(path, mediaId + ".obj"); - - // create dir if nonexistent - if (!Directory.Exists(path)) Directory.CreateDirectory(path); - - var eTag = GenerateETag(filepath); - - var url = ApiInfo.FileEndpoint.Replace("{id}", mediaId); - var http = new HttpRequest(url, HttpVerb.GET) - .WithHeader("If-None-Match", eTag) - .WithHeader("Accept-Encoding", "gzip,deflate") - .Send(); - - httpHandlers.Add(http); - batchHandlers.Add(http); - - http.Operation.completed += ar => - { - if (CheckNetworkErrors(http)) - { - return; - } - - // do nothing if unchanged - if (http.Request.responseCode == 304) return; - - if (http.Download.data.Length == 0) - { - Debug.LogError(string.Format("Downloaded Media is 0 bytes long:\nurl: {0}", url)); - ResetUpdateOnError(); - return; - } - - // gzipped files are un-gzipped automagically - File.WriteAllBytes(filepath, http.Download.data); - - var subfilesHeader = http.Request.GetResponseHeader("subfiles"); - if (subfilesHeader != null) - { - string[] separator = new string[1] { ";;" }; - var subfiles = subfilesHeader.Split(separator, StringSplitOptions.RemoveEmptyEntries); - foreach (var subfile in subfiles) - { - UpdateSubfile(path, url, subfile, batchHandlers); - } - } - - Debug.Log(string.Format("{1}-DONE with {0}", url, Time.realtimeSinceStartup)); - }; - } - } - - private void UpdateTracker(ApiExhibitionVersion exhibition) - { - var path = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibition.Id); - var filepath = Path.Combine(path, "tracker.dat"); //TODO test if it's the xml instead - - // create dir if nonexistent - if (!Directory.Exists(path)) Directory.CreateDirectory(path); - - var eTag = GenerateETag(filepath); - - var url = exhibition.Tracker.UpdateUrl; - var http = new HttpRequest(url, HttpVerb.GET) - .WithHeader("If-None-Match", eTag) - .Send(); - httpHandlers.Add(http); - - http.Operation.completed += ar => - { - if (CheckNetworkErrors(http)) - { - return; - } - if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } - // do nothing if unchanged - if (http.Request.responseCode == 304) return; - - // gzipped files are un-gzipped automagically - File.WriteAllBytes(filepath, http.Download.data); + UpdateMetaFile(exhibition, path, lang, exhibitionDownloads); + } - var subfilesHeader = http.Request.GetResponseHeader("subfiles"); - if (subfilesHeader != null) + var versionPath = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibition.Id, "version.txt"); + var version = exhibition.Meta.UpdateVersion.ToString(); + StartCoroutine(UpdateVersionFileCoroutine(exhibitionDownloads, versionPath, version, exhibition, updatedExhibitions)); + } + + private IEnumerator UpdateVersionFileCoroutine(List exhibitionDownloads, string versionPath, string version, ApiExhibitionVersion exhibition, List updatedExhibitions) + { + // updates version file only if all sub files were downloaded successfully + yield return new WaitUntil(() => exhibitionDownloads.All(h => h.IsDone)); + if (exhibitionDownloads.Any(http => http.IsError)) yield break; + + File.WriteAllText(versionPath, version); + updatedExhibitions.Remove(exhibition); + } + + private void UpdateMetaFile(ApiExhibitionVersion exhibition, string path, string lang, List exhibitionDownloads, int retries = 0) + { + var url = exhibition.Meta.UpdateUrl.Replace("{lang}", lang); + var http = new HttpRequest(url, HttpVerb.GET).Send(); + exhibitionDownloads.Add(http); + + http.Operation.completed += ar => + { + if (http.IsError) { - string[] separator = new string[1] { ";;" }; - var subfiles = subfilesHeader.Split(separator, StringSplitOptions.RemoveEmptyEntries); - foreach (var subfile in subfiles) + if (retries <= retriesUntilFailure) { - UpdateSubfile(path, url, subfile, new List()); + exhibitionDownloads.Remove(http); + UpdateMetaFile(exhibition, path, lang, exhibitionDownloads, retries + 1); + return; + } + else + { + UpdateFailed(http); + return; } } + File.WriteAllText(Path.Combine(path, "meta.json"), http.Download.text, System.Text.Encoding.UTF8); Debug.Log(string.Format("{1}-DONE with {0}", url, Time.realtimeSinceStartup)); + + var exhibits = JsonConvert.DeserializeObject>(http.Download.text); + if (exhibits == null) + { + Debug.LogError(string.Format("Exhibit could not be parsed from Json:\nurl: {0}\njson: {1}", url, http.Download.text)); + http.IsError = true; + UpdateFailed(http); + } }; } + private IEnumerator CheckFiles() + { + //+ create file list + //+ run updatemedia sequentially + // write to cache until all subfiles are here, then to filesystem + // use progressbar + + var usedFileList = new Dictionary(); + GenerateUsedFileList(usedFileList); + + int progress = 0; + foreach (var file in usedFileList) + { + Dictionary data = new Dictionary(); + yield return UpdateFile(file.Value, data); + + foreach (var item in data) + { + File.WriteAllBytes(item.Key, item.Value); + } + + float downloadProgress = progress / usedFileList.Count; + ProgressBar.value = ProgressBar.value < downloadProgress ? downloadProgress : ProgressBar.value; + progress++; + } + + var deletedData = VersionAnswer.Data.Where(d => + d.Meta.Status == VersionStatus.deleted + || d.Tracker.Status == VersionStatus.deleted).ToList(); + deletedData.ForEach(d => DeleteExhibition(d)); + + CleanupMediaFiles(); + UpdateCompletedDialog.SetActive(true); + StartCoroutine(LoadMainScene()); + } + + private void GenerateUsedFileList(Dictionary downloadList) + { + var rootDir = new DirectoryInfo(PathHelper.ExhibitionPath); + var exhibitionDirs = rootDir.GetDirectories().ToList(); + foreach (var dir in exhibitionDirs) + { + var versionFilePath = Path.Combine(dir.FullName, "version.txt"); + if (!File.Exists(versionFilePath)) continue; + + var exhibitionId = dir.Name; + foreach (var lang in VersionAnswer.Languages) + { + var metaFilePath = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibitionId, lang, "meta.json"); + var metaFile = File.ReadAllText(metaFilePath); + var exhibits = JsonConvert.DeserializeObject>(metaFile); + ExhibitMetas.Add(exhibitionId + "/" + lang, exhibits); + } + } + + //var rootDir = new DirectoryInfo(PathHelper.ExhibitionPath); + //var exhibitionDirs = rootDir.GetDirectories().ToList(); + //foreach (var dir in exhibitionDirs) + //{ + // var versionFilePath = Path.Combine(dir.FullName, "version.txt"); + // if (!File.Exists(versionFilePath)) continue; + + // var trackerFilePath = Path.Combine(dir.FullName, trackerMainFile); + // TODO I dont know the URL of trackers + // downloadList.Add(trackerFilePath); + //} + + var updatedTracker = VersionAnswer.Data.Where(d => d.Tracker.Status == VersionStatus.updated).ToList(); + foreach (var exhibition in updatedTracker) + { + var url = exhibition.Tracker.UpdateUrl; + var directory = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibition.Id); + var filepath = PathHelper.CombinePaths(directory, trackerMainFile); + + downloadList.Add(url, new FileDownloadInfo(url, filepath, directory)); + } + + var mediaList = ExhibitMetas.Values.ToList().SelectMany(m => m.SelectMany(i => i.Pois.Select(p => p.MediaId))).Where(m => m != null).Distinct().ToList(); + foreach (var mediaId in mediaList) + { + var url = ApiInfo.FileEndpoint.Replace("{id}", mediaId); + var directory = PathHelper.CombinePaths(PathHelper.MediaPath, mediaId); + var filepath = PathHelper.CombinePaths(directory, mediaId + ".obj"); + + downloadList.Add(url, new FileDownloadInfo(url, filepath, directory)); + } + } + + private class FileDownloadInfo + { + public string url; + public string filepath; + public string directory; + + public FileDownloadInfo(string url, string filepath, string directory) + { + this.url = url; + this.filepath = filepath; + this.directory = directory; + } + } + + private IEnumerator UpdateFile(FileDownloadInfo info, Dictionary data) + { + // create dir if nonexistent + if (!Directory.Exists(info.directory)) Directory.CreateDirectory(info.directory); + + var eTag = GenerateETag(info.filepath); + yield return UpdateFile(info, eTag, data); + } + + private IEnumerator UpdateFile(FileDownloadInfo info, string eTag, Dictionary data, int retries = 0) + { + var url = info.url; + var http = new HttpRequest(url, HttpVerb.GET) + .WithHeader("If-None-Match", eTag) + .WithHeader("Accept-Encoding", "gzip,deflate") + .Send(); + + yield return new WaitUntil(() => http.IsDone); + + if (http.IsError) + { + if (retries <= retriesUntilFailure) + { + yield return UpdateFile(info, eTag, data, retries + 1); + } + else + { + UpdateFailed(http); + yield return null; + } + } + + if (http.Request.responseCode != 304) + { + // gzipped files are un-gzipped automagically + data.Add(info.filepath, http.Download.data); + } + + var subfilesHeader = http.Request.GetResponseHeader("subfiles"); + if (subfilesHeader != null) + { + string[] separator = new string[1] { ";;" }; + var subfiles = subfilesHeader.Split(separator, StringSplitOptions.RemoveEmptyEntries); + foreach (var subfileName in subfiles) + { + var subfileInfo = new FileDownloadInfo( + info.url + "/" + subfileName, + Path.Combine(info.directory, subfileName), + info.directory); + + var subfileETag = GenerateETag(info.filepath); + yield return UpdateFile(subfileInfo, subfileETag, data); + } + } + + Debug.Log(string.Format("{1}-DONE with {0}", url, Time.realtimeSinceStartup)); + } + private string GenerateETag(string filepath) { string eTag = ""; @@ -392,121 +450,45 @@ namespace RothenburgAR.Updater return eTag; } - private void UpdateSubfile(string path, string parentUrl, string subfile, List batchHandlers) + private void DeleteExhibition(ApiExhibitionVersion exhibition) { - var filepath = Path.Combine(path, subfile); - var eTag = GenerateETag(filepath); - - var url = parentUrl + "/" + subfile; - var http = new HttpRequest(url, HttpVerb.GET) - .WithHeader("If-None-Match", eTag) - .Send(); - - httpHandlers.Add(http); - batchHandlers.Add(http); - - http.Operation.completed += ar => - { - if (CheckNetworkErrors(http)) - { - return; - } - - // do nothing if unchanged - if (http.Request.responseCode == 304) return; - - if (http.Download.data.Length == 0) - { - Debug.LogError(string.Format("Downloaded Subfile is 0 bytes long:\nurl: {0}", url)); - ResetUpdateOnError(); - return; - } - - // gzipped files are un-gzipped automagically - File.WriteAllBytes(filepath, http.Download.data); - - Debug.Log(string.Format("{1}-DONE with {0}", url, Time.realtimeSinceStartup)); - }; + //TODO delete media as well if not used elsewhere? + var path = Path.Combine(PathHelper.ExhibitionPath, exhibition.Id); + Directory.Delete(path, true); } - private bool CheckNetworkErrors(HttpHandler http) + private void CleanupMediaFiles() { - if (http.Request.isNetworkError || http.Request.isHttpError) + var requiredMedia = ExhibitMetas.Values.ToList().SelectMany(m => m.SelectMany(i => i.Pois.Select(p => p.MediaId))).Where(m => m != null).Distinct().ToList(); + + var rootDir = new DirectoryInfo(PathHelper.MediaPath); + var exhibitionDirs = rootDir.GetDirectories().ToList(); + foreach (var dir in exhibitionDirs) { - Debug.LogError(String.Format("Error while downloading\nurl: {0}\nNetwork Error: {1}\nHttp Error: {2}\nHttp Response Code: {3}", + if (!requiredMedia.Contains(dir.Name) && Regex.IsMatch(dir.Name, "/[0-9]/")) + { + Directory.Delete(dir.FullName, true); + } + } + } + + private void UpdateFailed(HttpHandler http) + { + Debug.LogError(String.Format("Error while downloading\nurl: {0}\nNetwork Error: {1}\nHttp Error: {2}\nHttp Response Code: {3}", http.Request.url, http.Request.isNetworkError, http.Request.isHttpError, http.Request.responseCode)); - ResetUpdateOnError(); - - //TODO decide on level of detail for user notification - //var all = UpdateFailedDialog.GetComponentsInChildren().ToList(); - //var errorText = all.First(c => c.name == "ErrorText"); - //errorText.text = string.Format("Fehlercode: {0}", http.Request.responseCode.ToString()); - - return true; - } - return false; - } - - private void ResetUpdateOnError() - { UpdateDialog.SetActive(true); UpdateFailedDialog.SetActive(true); - - this.httpHandlers.ForEach(h => h.Request.Abort()); } - public void LoadMainScene() + public IEnumerator LoadMainScene() { + yield return new WaitForSecondsRealtime(10f); Debug.Log("Loading mainScene"); UnityEngine.SceneManagement.SceneManager.LoadScene("mainScene"); } - - private DateTime downloadEndedTime = DateTime.Now.AddYears(1); - - void Update() - { - // Update the download progress bar - if (httpHandlers.Count > 0) - { - float downloadProgress = 0f; - httpHandlers.ForEach(h => - { - downloadProgress += h.Operation.progress; - }); - - downloadProgress /= httpHandlers.Count; - - // operation.progress is bugged and goes all over the place: - // jumps from 0 to 1 and back despite not being done - // steadily increases from 0,5 to 0,99 and resets to 0,5 multiple times (5-15x) during a single download - - // update progress bar and ignore all negative changes - ProgressBar.value = ProgressBar.value < downloadProgress ? downloadProgress : ProgressBar.value; - } - - // Continue to Main Scene after all downloads are done and this.afterDownloadWaitTime seconds have passed without any new downloads triggering. - if (httpHandlers.All(h => h.IsDone) && !UpdateFailedDialog.activeInHierarchy) - { - if (downloadEndedTime > DateTime.Now) - { - Debug.Log("Done Updating"); - UpdateCompletedDialog.SetActive(true); - downloadEndedTime = DateTime.Now; - } - if (downloadEndedTime.AddSeconds(afterDownloadWaitTime) < DateTime.Now) - { - LoadMainScene(); - } - } - else - { - UpdateCompletedDialog.SetActive(false); - downloadEndedTime = DateTime.Now.AddYears(1); - } - } } } \ No newline at end of file