First Commit of reworked updater

This commit is contained in:
2018-12-23 21:05:13 +01:00
parent b0b1b5b9e2
commit 9bf8505c77
2 changed files with 262 additions and 278 deletions

View File

@@ -66,6 +66,7 @@ namespace RothenburgAR.Updater
public UnityWebRequestAsyncOperation Operation { get; set; }
private bool isDone = false;
public bool IsError { get; set; }
public bool IsDone
{
@@ -85,6 +86,7 @@ namespace RothenburgAR.Updater
this.Operation.completed += (_) =>
{
this.isDone = true;
this.IsError = request.isNetworkError || request.isHttpError || download.data.Length == 0;
};
}
}

View File

@@ -6,8 +6,8 @@ using System;
using Newtonsoft.Json;
using RothenburgAR.Common;
using System.IO;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text.RegularExpressions;
namespace RothenburgAR.Updater
{
@@ -20,13 +20,13 @@ namespace RothenburgAR.Updater
public GameObject UpdateFailedDialog;
private readonly float afterDownloadWaitTime = 10f;
private readonly int retriesUntilFailure = 3;
private readonly string trackerMainFile = "tracker.dat";
public ApiVersioncheckAnswer VersionAnswer { get; set; }
private List<HttpHandler> httpHandlers = new List<HttpHandler>();
Dictionary<string, List<ApiExhibit>> ExhibitMetas = new Dictionary<string, List<ApiExhibit>>();
private LogFileHandler logFileHandler;
private readonly List<string> downloadingMedia = new List<string>();
void Start()
{
#if !UNITY_EDITOR
@@ -60,26 +60,36 @@ namespace RothenburgAR.Updater
{
// check for updates and ask for permission to download if there are any
string versionMap = GenerateVersionMap();
HttpHandler http = new HttpRequest(ApiInfo.VersionCheckEndpoint, HttpVerb.POST, versionMap).Send();
httpHandlers.Add(http);
CheckForUpdates(versionMap, 0);
}
private void CheckForUpdates(string versionMap, int retries)
{
HttpHandler http = new HttpRequest(ApiInfo.VersionCheckEndpoint, HttpVerb.POST, versionMap).Send();
http.Operation.completed += ar =>
{
if (CheckNetworkErrors(http))
Debug.Log("VersionAnswer: " + http.Download.text);
var answer = JsonConvert.DeserializeObject<ApiVersioncheckAnswer>(http.Download.text);
if (http.IsError)
{
return;
if (retries <= retriesUntilFailure)
{
CheckForUpdates(versionMap, retries + 1);
return;
}
else
{
UpdateFailed(http);
return;
}
}
Debug.Log(http.Download.text);
VersionAnswer = JsonConvert.DeserializeObject<ApiVersioncheckAnswer>(http.Download.text);
Debug.Log(string.Format("{1}-DONE with {0}", ApiInfo.VersionCheckEndpoint, Time.realtimeSinceStartup));
VersionAnswer = answer;
var answer = VersionAnswer;
if (VersionAnswer != null && VersionAnswer.Data.TrueForAll(d => d.Meta.Status == VersionStatus.ok && d.Tracker.Status == VersionStatus.ok))
if (VersionAnswer.Data.TrueForAll(d => d.Meta.Status == VersionStatus.ok && d.Tracker.Status == VersionStatus.ok))
{
// no updates required, continue to app
LoadMainScene();
StartCoroutine(CheckFiles());
return;
}
@@ -165,9 +175,6 @@ namespace RothenburgAR.Updater
UpdateConfirmationDialog.SetActive(false);
UpdateFailedDialog.SetActive(false);
this.downloadingMedia.Clear();
this.httpHandlers.ForEach(h => h.Request.Abort());
this.httpHandlers.Clear();
if (VersionAnswer == null)
{
@@ -178,9 +185,9 @@ namespace RothenburgAR.Updater
//TODO write languages to file the app can read (so that the languagemanager can decide which languages the user can choose from)
var updatedMeta = VersionAnswer.Data.Where(d => d.Meta.Status == VersionStatus.updated).ToList();
var updatedExhibitions = VersionAnswer.Data.Where(d => d.Meta.Status == VersionStatus.updated).ToList();
var updatedTracker = VersionAnswer.Data.Where(d => d.Tracker.Status == VersionStatus.updated).ToList();
updatedMeta.Union(updatedTracker).ToList().ForEach(updatedExhibition =>
updatedExhibitions.Union(updatedTracker).Distinct().ToList().ForEach(updatedExhibition =>
{
// create exhibition directories
var path = Path.Combine(PathHelper.ExhibitionPath, updatedExhibition.Id);
@@ -190,191 +197,242 @@ namespace RothenburgAR.Updater
}
});
updatedMeta.ForEach(d => UpdateMeta(d));
updatedTracker.ForEach(d => UpdateTracker(d));
var deletedData = VersionAnswer.Data.Where(d =>
d.Meta.Status == VersionStatus.deleted
|| d.Tracker.Status == VersionStatus.deleted).ToList();
deletedData.ForEach(d => DeleteExhibition(d));
updatedExhibitions.ForEach(d => UpdateExhibition(d, updatedExhibitions));
StartCoroutine(UpdateExhibitionsCoroutine(updatedExhibitions));
}
private void DeleteExhibition(ApiExhibitionVersion exhibition)
private IEnumerator UpdateExhibitionsCoroutine(List<ApiExhibitionVersion> updatedExhibitions)
{
//TODO delete media as well if not used elsewhere?
var path = Path.Combine(PathHelper.ExhibitionPath, exhibition.Id);
Directory.Delete(path, true);
yield return new WaitUntil(() => updatedExhibitions.Count == 0);
StartCoroutine(CheckFiles());
}
private void UpdateMeta(ApiExhibitionVersion exhibition)
private void UpdateExhibition(ApiExhibitionVersion exhibition, List<ApiExhibitionVersion> updatedExhibitions)
{
var currentHandlers = new List<HttpHandler>();
var exhibitionDownloads = new List<HttpHandler>();
foreach (var lang in VersionAnswer.Languages)
{
var path = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibition.Id, lang);
var url = exhibition.Meta.UpdateUrl.Replace("{lang}", lang);
var http = new HttpRequest(url, HttpVerb.GET).Send();
httpHandlers.Add(http);
currentHandlers.Add(http);
http.Operation.completed += ar =>
{
if (CheckNetworkErrors(http))
{
return;
}
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
File.WriteAllText(Path.Combine(path, "meta.json"), http.Download.text, System.Text.Encoding.UTF8);
Debug.Log(string.Format("{1}-DONE with {0}", url, Time.realtimeSinceStartup));
var exhibits = JsonConvert.DeserializeObject<List<ApiExhibit>>(http.Download.text);
if (exhibits == null)
{
Debug.LogError(string.Format("Exhibit could not be parsed from Json:\nurl: {0}\njson: {1}", url, http.Download.text));
ResetUpdateOnError();
return;
}
else
{
foreach (var exhibit in exhibits)
{
UpdateMedia(exhibit, currentHandlers);
}
}
};
}
var versionPath = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibition.Id, "version.txt");
var version = exhibition.Meta.UpdateVersion.ToString();
StartCoroutine(UpdateVersionFileCoroutine(currentHandlers, versionPath, version));
}
private IEnumerator UpdateVersionFileCoroutine(List<HttpHandler> currentHandlers, string versionPath, string version)
{
// updates version file only if all sub files were downloaded successfully
yield return new WaitUntil(() => currentHandlers.All(h => h.IsDone));
if (currentHandlers.Any(http => http.Request.isNetworkError || http.Request.isHttpError)) yield break;
File.WriteAllText(versionPath, version);
}
private void UpdateMedia(ApiExhibit exhibit, List<HttpHandler> batchHandlers)
{
var mediaIDs = exhibit.Pois.Select(p => p.MediaId).Except(new List<string> { null }).ToList();
foreach (var mediaId in mediaIDs)
{
if (downloadingMedia.Contains(mediaId)) continue;
downloadingMedia.Add(mediaId);
var path = PathHelper.CombinePaths(PathHelper.MediaPath, mediaId);
var filepath = Path.Combine(path, mediaId + ".obj");
// create dir if nonexistent
if (!Directory.Exists(path)) Directory.CreateDirectory(path);
var eTag = GenerateETag(filepath);
var url = ApiInfo.FileEndpoint.Replace("{id}", mediaId);
var http = new HttpRequest(url, HttpVerb.GET)
.WithHeader("If-None-Match", eTag)
.WithHeader("Accept-Encoding", "gzip,deflate")
.Send();
httpHandlers.Add(http);
batchHandlers.Add(http);
http.Operation.completed += ar =>
{
if (CheckNetworkErrors(http))
{
return;
}
// do nothing if unchanged
if (http.Request.responseCode == 304) return;
if (http.Download.data.Length == 0)
{
Debug.LogError(string.Format("Downloaded Media is 0 bytes long:\nurl: {0}", url));
ResetUpdateOnError();
return;
}
// gzipped files are un-gzipped automagically
File.WriteAllBytes(filepath, http.Download.data);
var subfilesHeader = http.Request.GetResponseHeader("subfiles");
if (subfilesHeader != null)
{
string[] separator = new string[1] { ";;" };
var subfiles = subfilesHeader.Split(separator, StringSplitOptions.RemoveEmptyEntries);
foreach (var subfile in subfiles)
{
UpdateSubfile(path, url, subfile, batchHandlers);
}
}
Debug.Log(string.Format("{1}-DONE with {0}", url, Time.realtimeSinceStartup));
};
}
}
private void UpdateTracker(ApiExhibitionVersion exhibition)
{
var path = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibition.Id);
var filepath = Path.Combine(path, "tracker.dat"); //TODO test if it's the xml instead
// create dir if nonexistent
if (!Directory.Exists(path)) Directory.CreateDirectory(path);
var eTag = GenerateETag(filepath);
var url = exhibition.Tracker.UpdateUrl;
var http = new HttpRequest(url, HttpVerb.GET)
.WithHeader("If-None-Match", eTag)
.Send();
httpHandlers.Add(http);
http.Operation.completed += ar =>
{
if (CheckNetworkErrors(http))
{
return;
}
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
// do nothing if unchanged
if (http.Request.responseCode == 304) return;
// gzipped files are un-gzipped automagically
File.WriteAllBytes(filepath, http.Download.data);
UpdateMetaFile(exhibition, path, lang, exhibitionDownloads);
}
var subfilesHeader = http.Request.GetResponseHeader("subfiles");
if (subfilesHeader != null)
var versionPath = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibition.Id, "version.txt");
var version = exhibition.Meta.UpdateVersion.ToString();
StartCoroutine(UpdateVersionFileCoroutine(exhibitionDownloads, versionPath, version, exhibition, updatedExhibitions));
}
private IEnumerator UpdateVersionFileCoroutine(List<HttpHandler> exhibitionDownloads, string versionPath, string version, ApiExhibitionVersion exhibition, List<ApiExhibitionVersion> updatedExhibitions)
{
// updates version file only if all sub files were downloaded successfully
yield return new WaitUntil(() => exhibitionDownloads.All(h => h.IsDone));
if (exhibitionDownloads.Any(http => http.IsError)) yield break;
File.WriteAllText(versionPath, version);
updatedExhibitions.Remove(exhibition);
}
private void UpdateMetaFile(ApiExhibitionVersion exhibition, string path, string lang, List<HttpHandler> exhibitionDownloads, int retries = 0)
{
var url = exhibition.Meta.UpdateUrl.Replace("{lang}", lang);
var http = new HttpRequest(url, HttpVerb.GET).Send();
exhibitionDownloads.Add(http);
http.Operation.completed += ar =>
{
if (http.IsError)
{
string[] separator = new string[1] { ";;" };
var subfiles = subfilesHeader.Split(separator, StringSplitOptions.RemoveEmptyEntries);
foreach (var subfile in subfiles)
if (retries <= retriesUntilFailure)
{
UpdateSubfile(path, url, subfile, new List<HttpHandler>());
exhibitionDownloads.Remove(http);
UpdateMetaFile(exhibition, path, lang, exhibitionDownloads, retries + 1);
return;
}
else
{
UpdateFailed(http);
return;
}
}
File.WriteAllText(Path.Combine(path, "meta.json"), http.Download.text, System.Text.Encoding.UTF8);
Debug.Log(string.Format("{1}-DONE with {0}", url, Time.realtimeSinceStartup));
var exhibits = JsonConvert.DeserializeObject<List<ApiExhibit>>(http.Download.text);
if (exhibits == null)
{
Debug.LogError(string.Format("Exhibit could not be parsed from Json:\nurl: {0}\njson: {1}", url, http.Download.text));
http.IsError = true;
UpdateFailed(http);
}
};
}
private IEnumerator CheckFiles()
{
//+ create file list
//+ run updatemedia sequentially
// write to cache until all subfiles are here, then to filesystem
// use progressbar
var usedFileList = new Dictionary<string, FileDownloadInfo>();
GenerateUsedFileList(usedFileList);
int progress = 0;
foreach (var file in usedFileList)
{
Dictionary<string, byte[]> data = new Dictionary<string, byte[]>();
yield return UpdateFile(file.Value, data);
foreach (var item in data)
{
File.WriteAllBytes(item.Key, item.Value);
}
float downloadProgress = progress / usedFileList.Count;
ProgressBar.value = ProgressBar.value < downloadProgress ? downloadProgress : ProgressBar.value;
progress++;
}
var deletedData = VersionAnswer.Data.Where(d =>
d.Meta.Status == VersionStatus.deleted
|| d.Tracker.Status == VersionStatus.deleted).ToList();
deletedData.ForEach(d => DeleteExhibition(d));
CleanupMediaFiles();
UpdateCompletedDialog.SetActive(true);
StartCoroutine(LoadMainScene());
}
private void GenerateUsedFileList(Dictionary<string, FileDownloadInfo> downloadList)
{
var rootDir = new DirectoryInfo(PathHelper.ExhibitionPath);
var exhibitionDirs = rootDir.GetDirectories().ToList();
foreach (var dir in exhibitionDirs)
{
var versionFilePath = Path.Combine(dir.FullName, "version.txt");
if (!File.Exists(versionFilePath)) continue;
var exhibitionId = dir.Name;
foreach (var lang in VersionAnswer.Languages)
{
var metaFilePath = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibitionId, lang, "meta.json");
var metaFile = File.ReadAllText(metaFilePath);
var exhibits = JsonConvert.DeserializeObject<List<ApiExhibit>>(metaFile);
ExhibitMetas.Add(exhibitionId + "/" + lang, exhibits);
}
}
//var rootDir = new DirectoryInfo(PathHelper.ExhibitionPath);
//var exhibitionDirs = rootDir.GetDirectories().ToList();
//foreach (var dir in exhibitionDirs)
//{
// var versionFilePath = Path.Combine(dir.FullName, "version.txt");
// if (!File.Exists(versionFilePath)) continue;
// var trackerFilePath = Path.Combine(dir.FullName, trackerMainFile);
// TODO I dont know the URL of trackers
// downloadList.Add(trackerFilePath);
//}
var updatedTracker = VersionAnswer.Data.Where(d => d.Tracker.Status == VersionStatus.updated).ToList();
foreach (var exhibition in updatedTracker)
{
var url = exhibition.Tracker.UpdateUrl;
var directory = PathHelper.CombinePaths(PathHelper.ExhibitionPath, exhibition.Id);
var filepath = PathHelper.CombinePaths(directory, trackerMainFile);
downloadList.Add(url, new FileDownloadInfo(url, filepath, directory));
}
var mediaList = ExhibitMetas.Values.ToList().SelectMany(m => m.SelectMany(i => i.Pois.Select(p => p.MediaId))).Where(m => m != null).Distinct().ToList();
foreach (var mediaId in mediaList)
{
var url = ApiInfo.FileEndpoint.Replace("{id}", mediaId);
var directory = PathHelper.CombinePaths(PathHelper.MediaPath, mediaId);
var filepath = PathHelper.CombinePaths(directory, mediaId + ".obj");
downloadList.Add(url, new FileDownloadInfo(url, filepath, directory));
}
}
private class FileDownloadInfo
{
public string url;
public string filepath;
public string directory;
public FileDownloadInfo(string url, string filepath, string directory)
{
this.url = url;
this.filepath = filepath;
this.directory = directory;
}
}
private IEnumerator UpdateFile(FileDownloadInfo info, Dictionary<string, byte[]> data)
{
// create dir if nonexistent
if (!Directory.Exists(info.directory)) Directory.CreateDirectory(info.directory);
var eTag = GenerateETag(info.filepath);
yield return UpdateFile(info, eTag, data);
}
private IEnumerator UpdateFile(FileDownloadInfo info, string eTag, Dictionary<string, byte[]> data, int retries = 0)
{
var url = info.url;
var http = new HttpRequest(url, HttpVerb.GET)
.WithHeader("If-None-Match", eTag)
.WithHeader("Accept-Encoding", "gzip,deflate")
.Send();
yield return new WaitUntil(() => http.IsDone);
if (http.IsError)
{
if (retries <= retriesUntilFailure)
{
yield return UpdateFile(info, eTag, data, retries + 1);
}
else
{
UpdateFailed(http);
yield return null;
}
}
if (http.Request.responseCode != 304)
{
// gzipped files are un-gzipped automagically
data.Add(info.filepath, http.Download.data);
}
var subfilesHeader = http.Request.GetResponseHeader("subfiles");
if (subfilesHeader != null)
{
string[] separator = new string[1] { ";;" };
var subfiles = subfilesHeader.Split(separator, StringSplitOptions.RemoveEmptyEntries);
foreach (var subfileName in subfiles)
{
var subfileInfo = new FileDownloadInfo(
info.url + "/" + subfileName,
Path.Combine(info.directory, subfileName),
info.directory);
var subfileETag = GenerateETag(info.filepath);
yield return UpdateFile(subfileInfo, subfileETag, data);
}
}
Debug.Log(string.Format("{1}-DONE with {0}", url, Time.realtimeSinceStartup));
}
private string GenerateETag(string filepath)
{
string eTag = "";
@@ -392,121 +450,45 @@ namespace RothenburgAR.Updater
return eTag;
}
private void UpdateSubfile(string path, string parentUrl, string subfile, List<HttpHandler> batchHandlers)
private void DeleteExhibition(ApiExhibitionVersion exhibition)
{
var filepath = Path.Combine(path, subfile);
var eTag = GenerateETag(filepath);
var url = parentUrl + "/" + subfile;
var http = new HttpRequest(url, HttpVerb.GET)
.WithHeader("If-None-Match", eTag)
.Send();
httpHandlers.Add(http);
batchHandlers.Add(http);
http.Operation.completed += ar =>
{
if (CheckNetworkErrors(http))
{
return;
}
// do nothing if unchanged
if (http.Request.responseCode == 304) return;
if (http.Download.data.Length == 0)
{
Debug.LogError(string.Format("Downloaded Subfile is 0 bytes long:\nurl: {0}", url));
ResetUpdateOnError();
return;
}
// gzipped files are un-gzipped automagically
File.WriteAllBytes(filepath, http.Download.data);
Debug.Log(string.Format("{1}-DONE with {0}", url, Time.realtimeSinceStartup));
};
//TODO delete media as well if not used elsewhere?
var path = Path.Combine(PathHelper.ExhibitionPath, exhibition.Id);
Directory.Delete(path, true);
}
private bool CheckNetworkErrors(HttpHandler http)
private void CleanupMediaFiles()
{
if (http.Request.isNetworkError || http.Request.isHttpError)
var requiredMedia = ExhibitMetas.Values.ToList().SelectMany(m => m.SelectMany(i => i.Pois.Select(p => p.MediaId))).Where(m => m != null).Distinct().ToList();
var rootDir = new DirectoryInfo(PathHelper.MediaPath);
var exhibitionDirs = rootDir.GetDirectories().ToList();
foreach (var dir in exhibitionDirs)
{
Debug.LogError(String.Format("Error while downloading\nurl: {0}\nNetwork Error: {1}\nHttp Error: {2}\nHttp Response Code: {3}",
if (!requiredMedia.Contains(dir.Name) && Regex.IsMatch(dir.Name, "/[0-9]/"))
{
Directory.Delete(dir.FullName, true);
}
}
}
private void UpdateFailed(HttpHandler http)
{
Debug.LogError(String.Format("Error while downloading\nurl: {0}\nNetwork Error: {1}\nHttp Error: {2}\nHttp Response Code: {3}",
http.Request.url,
http.Request.isNetworkError,
http.Request.isHttpError,
http.Request.responseCode));
ResetUpdateOnError();
//TODO decide on level of detail for user notification
//var all = UpdateFailedDialog.GetComponentsInChildren<TextMeshProUGUI>().ToList();
//var errorText = all.First(c => c.name == "ErrorText");
//errorText.text = string.Format("Fehlercode: {0}", http.Request.responseCode.ToString());
return true;
}
return false;
}
private void ResetUpdateOnError()
{
UpdateDialog.SetActive(true);
UpdateFailedDialog.SetActive(true);
this.httpHandlers.ForEach(h => h.Request.Abort());
}
public void LoadMainScene()
public IEnumerator LoadMainScene()
{
yield return new WaitForSecondsRealtime(10f);
Debug.Log("Loading mainScene");
UnityEngine.SceneManagement.SceneManager.LoadScene("mainScene");
}
private DateTime downloadEndedTime = DateTime.Now.AddYears(1);
void Update()
{
// Update the download progress bar
if (httpHandlers.Count > 0)
{
float downloadProgress = 0f;
httpHandlers.ForEach(h =>
{
downloadProgress += h.Operation.progress;
});
downloadProgress /= httpHandlers.Count;
// operation.progress is bugged and goes all over the place:
// jumps from 0 to 1 and back despite not being done
// steadily increases from 0,5 to 0,99 and resets to 0,5 multiple times (5-15x) during a single download
// update progress bar and ignore all negative changes
ProgressBar.value = ProgressBar.value < downloadProgress ? downloadProgress : ProgressBar.value;
}
// Continue to Main Scene after all downloads are done and this.afterDownloadWaitTime seconds have passed without any new downloads triggering.
if (httpHandlers.All(h => h.IsDone) && !UpdateFailedDialog.activeInHierarchy)
{
if (downloadEndedTime > DateTime.Now)
{
Debug.Log("Done Updating");
UpdateCompletedDialog.SetActive(true);
downloadEndedTime = DateTime.Now;
}
if (downloadEndedTime.AddSeconds(afterDownloadWaitTime) < DateTime.Now)
{
LoadMainScene();
}
}
else
{
UpdateCompletedDialog.SetActive(false);
downloadEndedTime = DateTime.Now.AddYears(1);
}
}
}
}