From 0aad3f545af854619c641dc8b57ac0cf12971c8e Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 19:36:56 -0500 Subject: [PATCH 01/64] update Content Patcher schema --- docs/release-notes.md | 5 ++++ .../wwwroot/schemas/content-patcher.json | 23 +++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 1c1065427..9c3b8e3e6 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,6 +1,11 @@ ← [README](README.md) # Release notes +## Upcoming release + +* For the web UI: + * Updated the JSON validator for Content Patcher 1.10.0. + ## 3.0.1 Released 02 December 2019 for Stardew Valley 1.4.0.1. diff --git a/src/SMAPI.Web/wwwroot/schemas/content-patcher.json b/src/SMAPI.Web/wwwroot/schemas/content-patcher.json index 61a633cb9..c0236f1ee 100644 --- a/src/SMAPI.Web/wwwroot/schemas/content-patcher.json +++ b/src/SMAPI.Web/wwwroot/schemas/content-patcher.json @@ -11,9 +11,9 @@ "title": "Format version", "description": "The format version. You should always use the latest version to enable the latest features and avoid obsolete behavior.", "type": "string", - "const": "1.9", + "const": "1.10.0", "@errorMessages": { - "const": "Incorrect value '@value'. This should be set to the latest format version, currently '1.9'." + "const": "Incorrect value '@value'. This should be set to the latest format version, currently '1.10.0'." } }, "ConfigSchema": { @@ -194,6 +194,8 @@ } }, "MoveEntries": { + "title": "Move entries", + "description": "Change the entry order in a list asset like Data/MoviesReactions. (Using this with a non-list asset will cause an error, since those have no order.)", "type": "array", "items": { "type": "object", @@ -259,6 +261,14 @@ } } }, + "MapProperties": { + "title": "Map properties", + "description": "The map properties (not tile properties) to add, replace, or delete. To add an property, just specify a key that doesn't exist; to delete an entry, set the value to null (like \"some key\": null). This field supports tokens in property keys and values.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "When": { "title": "When", "description": "Only apply the patch if the given conditions match.", @@ -300,7 +310,7 @@ }, "then": { "propertyNames": { - "enum": [ "Action", "Target", "LogName", "Enabled", "When", "Fields", "Entries", "MoveEntries" ] + "enum": [ "Action", "Target", "LogName", "Enabled", "When", "FromFile", "Fields", "Entries", "MoveEntries" ] } } }, @@ -313,7 +323,7 @@ "then": { "properties": { "FromFile": { - "description": "The relative path to the map in your content pack folder from which to copy (like assets/town.tbin). This can be a .tbin or .xnb file. This field supports tokens and capitalization doesn't matter.\nContent Patcher will handle tilesheets referenced by the FromFile map for you if it's a .tbin file:\n - If a tilesheet isn't referenced by the target map, Content Patcher will add it for you (with a z_ ID prefix to avoid conflicts with hardcoded game logic). If the source map has a custom version of a tilesheet that's already referenced, it'll be added as a separate tilesheet only used by your tiles.\n - If you include the tilesheet file in your mod folder, Content Patcher will use that one automatically; otherwise it will be loaded from the game's Content/Maps folder." + "description": "The relative path to the map in your content pack folder from which to copy (like assets/town.tbin). This can be a .tbin or .xnb file. This field supports tokens and capitalization doesn't matter.\nContent Patcher will handle tilesheets referenced by the FromFile map for you:\n - If a tilesheet isn't referenced by the target map, Content Patcher will add it for you (with a z_ ID prefix to avoid conflicts with hardcoded game logic). If the source map has a custom version of a tilesheet that's already referenced, it'll be added as a separate tilesheet only used by your tiles.\n - If you include the tilesheet file in your mod folder, Content Patcher will use that one automatically; otherwise it will be loaded from the game's Content/Maps folder." }, "FromArea": { "description": "The part of the source map to copy. Defaults to the whole source map." @@ -323,9 +333,8 @@ } }, "propertyNames": { - "enum": [ "Action", "Target", "LogName", "Enabled", "When", "FromFile", "FromArea", "ToArea" ] - }, - "required": [ "FromFile", "ToArea" ] + "enum": [ "Action", "Target", "LogName", "Enabled", "When", "FromFile", "FromArea", "ToArea", "MapProperties" ] + } } } ], From 2b1f607d41b3d4d071c0db0671dbc99b6982909f Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 21:21:28 -0500 Subject: [PATCH 02/64] encapsulate file storage, also handle Pastebin rate limits in JSON validator --- docs/release-notes.md | 1 + .../Controllers/JsonValidatorController.cs | 58 +++--- .../Controllers/LogParserController.cs | 178 ++---------------- .../Framework/Clients/Pastebin/PasteInfo.cs | 6 - .../ConfigModels/ApiClientsConfig.cs | 2 +- .../Framework/Storage/IStorageProvider.cs | 19 ++ .../Framework/Storage/StorageProvider.cs | 147 +++++++++++++++ .../Framework/Storage/StoredFileInfo.cs | 23 +++ .../Framework/Storage/UploadResult.cs | 33 ++++ src/SMAPI.Web/Startup.cs | 10 +- .../JsonValidator/JsonValidatorModel.cs | 19 +- .../Views/JsonValidator/Index.cshtml | 15 +- src/SMAPI.Web/appsettings.json | 2 +- .../wwwroot/Content/css/json-validator.css | 6 + 14 files changed, 307 insertions(+), 212 deletions(-) create mode 100644 src/SMAPI.Web/Framework/Storage/IStorageProvider.cs create mode 100644 src/SMAPI.Web/Framework/Storage/StorageProvider.cs create mode 100644 src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs create mode 100644 src/SMAPI.Web/Framework/Storage/UploadResult.cs diff --git a/docs/release-notes.md b/docs/release-notes.md index 9c3b8e3e6..0b0a0f9e1 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -4,6 +4,7 @@ ## Upcoming release * For the web UI: + * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. * Updated the JSON validator for Content Patcher 1.10.0. ## 3.0.1 diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs index 40599abc8..830fe8394 100644 --- a/src/SMAPI.Web/Controllers/JsonValidatorController.cs +++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs @@ -9,8 +9,7 @@ using Newtonsoft.Json.Linq; using Newtonsoft.Json.Schema; using StardewModdingAPI.Web.Framework; -using StardewModdingAPI.Web.Framework.Clients.Pastebin; -using StardewModdingAPI.Web.Framework.Compression; +using StardewModdingAPI.Web.Framework.Storage; using StardewModdingAPI.Web.ViewModels.JsonValidator; namespace StardewModdingAPI.Web.Controllers @@ -21,11 +20,8 @@ internal class JsonValidatorController : Controller /********* ** Fields *********/ - /// The underlying Pastebin client. - private readonly IPastebinClient Pastebin; - - /// The underlying text compression helper. - private readonly IGzipHelper GzipHelper; + /// Provides access to raw data storage. + private readonly IStorageProvider Storage; /// The supported JSON schemas (names indexed by ID). private readonly IDictionary SchemaFormats = new Dictionary @@ -49,12 +45,10 @@ internal class JsonValidatorController : Controller ** Constructor ***/ /// Construct an instance. - /// The Pastebin API client. - /// The underlying text compression helper. - public JsonValidatorController(IPastebinClient pastebin, IGzipHelper gzipHelper) + /// Provides access to raw data storage. + public JsonValidatorController(IStorageProvider storage) { - this.Pastebin = pastebin; - this.GzipHelper = gzipHelper; + this.Storage = storage; } /*** @@ -62,7 +56,7 @@ public JsonValidatorController(IPastebinClient pastebin, IGzipHelper gzipHelper) ***/ /// Render the schema validator UI. /// The schema name with which to validate the JSON. - /// The paste ID. + /// The stored file ID. [HttpGet] [Route("json")] [Route("json/{schemaName}")] @@ -76,16 +70,16 @@ public async Task Index(string schemaName = null, string id = null) return this.View("Index", result); // fetch raw JSON - PasteInfo paste = await this.GetAsync(id); - if (string.IsNullOrWhiteSpace(paste.Content)) + StoredFileInfo file = await this.Storage.GetAsync(id); + if (string.IsNullOrWhiteSpace(file.Content)) return this.View("Index", result.SetUploadError("The JSON file seems to be empty.")); - result.SetContent(paste.Content); + result.SetContent(file.Content, expiry: file.Expiry, uploadWarning: file.Warning); // parse JSON JToken parsed; try { - parsed = JToken.Parse(paste.Content, new JsonLoadSettings + parsed = JToken.Parse(file.Content, new JsonLoadSettings { DuplicatePropertyNameHandling = DuplicatePropertyNameHandling.Error, CommentHandling = CommentHandling.Load @@ -97,7 +91,7 @@ public async Task Index(string schemaName = null, string id = null) } // format JSON - result.SetContent(parsed.ToString(Formatting.Indented)); + result.SetContent(parsed.ToString(Formatting.Indented), expiry: file.Expiry, uploadWarning: file.Warning); // skip if no schema selected if (schemaName == "none") @@ -132,23 +126,20 @@ public async Task Index(string schemaName = null, string id = null) public async Task PostAsync(JsonValidatorRequestModel request) { if (request == null) - return this.View("Index", new JsonValidatorModel(null, null, this.SchemaFormats).SetUploadError("The request seems to be invalid.")); + return this.View("Index", this.GetModel(null, null).SetUploadError("The request seems to be invalid.")); // normalize schema name string schemaName = this.NormalizeSchemaName(request.SchemaName); - // get raw log text + // get raw text string input = request.Content; if (string.IsNullOrWhiteSpace(input)) - return this.View("Index", new JsonValidatorModel(null, schemaName, this.SchemaFormats).SetUploadError("The JSON file seems to be empty.")); - - // upload log - input = this.GzipHelper.CompressString(input); - SavePasteResult result = await this.Pastebin.PostAsync($"JSON validator {DateTime.UtcNow:s}", input); + return this.View("Index", this.GetModel(null, schemaName).SetUploadError("The JSON file seems to be empty.")); - // handle errors - if (!result.Success) - return this.View("Index", new JsonValidatorModel(result.ID, schemaName, this.SchemaFormats).SetUploadError($"Pastebin error: {result.Error ?? "unknown error"}")); + // upload file + var result = await this.Storage.SaveAsync(title: $"JSON validator {DateTime.UtcNow:s}", content: input, compress: true); + if (!result.Succeeded) + return this.View("Index", this.GetModel(result.ID, schemaName).SetUploadError(result.UploadError)); // redirect to view return this.Redirect(this.Url.PlainAction("Index", "JsonValidator", new { schemaName = schemaName, id = result.ID })); @@ -158,13 +149,12 @@ public async Task PostAsync(JsonValidatorRequestModel request) /********* ** Private methods *********/ - /// Fetch raw text from Pastebin. - /// The Pastebin paste ID. - private async Task GetAsync(string id) + /// Build a JSON validator model. + /// The stored file ID. + /// The schema name with which the JSON was validated. + private JsonValidatorModel GetModel(string pasteID, string schemaName) { - PasteInfo response = await this.Pastebin.GetAsync(id); - response.Content = this.GzipHelper.DecompressString(response.Content); - return response; + return new JsonValidatorModel(pasteID, schemaName, this.SchemaFormats); } /// Get a normalized schema name, or the if blank. diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index 318b34d03..e270ae0a6 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -1,22 +1,12 @@ using System; -using System.IO; using System.Linq; -using System.Text; using System.Threading.Tasks; -using Amazon; -using Amazon.Runtime; -using Amazon.S3; -using Amazon.S3.Model; -using Amazon.S3.Transfer; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit.Utilities; using StardewModdingAPI.Web.Framework; -using StardewModdingAPI.Web.Framework.Clients.Pastebin; -using StardewModdingAPI.Web.Framework.Compression; -using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.Framework.LogParsing; using StardewModdingAPI.Web.Framework.LogParsing.Models; +using StardewModdingAPI.Web.Framework.Storage; using StardewModdingAPI.Web.ViewModels; namespace StardewModdingAPI.Web.Controllers @@ -27,14 +17,8 @@ internal class LogParserController : Controller /********* ** Fields *********/ - /// The API client settings. - private readonly ApiClientsConfig ClientsConfig; - - /// The underlying Pastebin client. - private readonly IPastebinClient Pastebin; - - /// The underlying text compression helper. - private readonly IGzipHelper GzipHelper; + /// Provides access to raw data storage. + private readonly IStorageProvider Storage; /********* @@ -44,21 +28,17 @@ internal class LogParserController : Controller ** Constructor ***/ /// Construct an instance. - /// The API client settings. - /// The Pastebin API client. - /// The underlying text compression helper. - public LogParserController(IOptions clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + /// Provides access to raw data storage. + public LogParserController(IStorageProvider storage) { - this.ClientsConfig = clientsConfig.Value; - this.Pastebin = pastebin; - this.GzipHelper = gzipHelper; + this.Storage = storage; } /*** ** Web UI ***/ /// Render the log parser UI. - /// The paste ID. + /// The stored file ID. /// Whether to display the raw unparsed log. [HttpGet] [Route("log")] @@ -70,12 +50,12 @@ public async Task Index(string id = null, bool raw = false) return this.View("Index", this.GetModel(id)); // log page - PasteInfo paste = await this.GetAsync(id); - ParsedLog log = paste.Success - ? new LogParser().Parse(paste.Content) - : new ParsedLog { IsValid = false, Error = paste.Error }; + StoredFileInfo file = await this.Storage.GetAsync(id); + ParsedLog log = file.Success + ? new LogParser().Parse(file.Content) + : new ParsedLog { IsValid = false, Error = file.Error }; - return this.View("Index", this.GetModel(id, uploadWarning: paste.Warning, expiry: paste.Expiry).SetResult(log, raw)); + return this.View("Index", this.GetModel(id, uploadWarning: file.Warning, expiry: file.Expiry).SetResult(log, raw)); } /*** @@ -92,8 +72,7 @@ public async Task PostAsync() return this.View("Index", this.GetModel(null, uploadError: "The log file seems to be empty.")); // upload log - input = this.GzipHelper.CompressString(input); - var uploadResult = await this.TrySaveLog(input); + UploadResult uploadResult = await this.Storage.SaveAsync(title: $"SMAPI log {DateTime.UtcNow:s}", content: input, compress: true); if (!uploadResult.Succeeded) return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError)); @@ -105,106 +84,8 @@ public async Task PostAsync() /********* ** Private methods *********/ - /// Fetch raw text from Pastebin. - /// The Pastebin paste ID. - private async Task GetAsync(string id) - { - // get from Amazon S3 - if (Guid.TryParseExact(id, "N", out Guid _)) - { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - - using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) - { - try - { - using (GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonLogBucket, $"logs/{id}")) - using (Stream responseStream = response.ResponseStream) - using (StreamReader reader = new StreamReader(responseStream)) - { - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; - string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); - - return new PasteInfo - { - Success = true, - Content = content, - Expiry = expiry, - Warning = pastebinError - }; - } - } - catch (AmazonServiceException ex) - { - return ex.ErrorCode == "NoSuchKey" - ? new PasteInfo { Error = "There's no log with that ID." } - : new PasteInfo { Error = $"Could not fetch that log from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; - } - } - } - - // get from PasteBin - else - { - PasteInfo response = await this.Pastebin.GetAsync(id); - response.Content = this.GzipHelper.DecompressString(response.Content); - return response; - } - } - - /// Save a log to Pastebin or Amazon S3, if available. - /// The content to upload. - /// Returns metadata about the save attempt. - private async Task TrySaveLog(string content) - { - // save to PasteBin - string uploadError; - { - SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", content); - if (result.Success) - return new UploadResult(true, result.ID, null); - - uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; - } - - // fallback to S3 - try - { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - - using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content))) - using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) - using (TransferUtility uploader = new TransferUtility(s3)) - { - string id = Guid.NewGuid().ToString("N"); - - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = this.ClientsConfig.AmazonLogBucket, - Key = $"logs/{id}", - InputStream = stream, - Metadata = - { - // note: AWS will lowercase keys and prefix 'x-amz-meta-' - ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), - ["pastebin-error"] = uploadError - } - }; - - await uploader.UploadAsync(uploadRequest); - - return new UploadResult(true, id, uploadError); - } - } - catch (Exception ex) - { - return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); - } - } - /// Build a log parser model. - /// The paste ID. + /// The stored file ID. /// When the uploaded file will no longer be available. /// A non-blocking warning while uploading the log. /// An error which occurred while uploading the log. @@ -243,36 +124,5 @@ private LogParserModel GetModel(string pasteID, DateTime? expiry = null, string return null; } } - - /// The result of an attempt to upload a file. - private class UploadResult - { - /********* - ** Accessors - *********/ - /// Whether the file upload succeeded. - public bool Succeeded { get; } - - /// The file ID, if applicable. - public string ID { get; } - - /// The upload error, if any. - public string UploadError { get; } - - - /********* - ** Public methods - *********/ - /// Construct an instance. - /// Whether the file upload succeeded. - /// The file ID, if applicable. - /// The upload error, if any. - public UploadResult(bool succeeded, string id, string uploadError) - { - this.Succeeded = succeeded; - this.ID = id; - this.UploadError = uploadError; - } - } } } diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs index bb2de3568..1ef3ef122 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs @@ -11,12 +11,6 @@ internal class PasteInfo /// The fetched paste content (if is true). public string Content { get; set; } - /// When the file will no longer be available. - public DateTime? Expiry { get; set; } - - /// The error message if saving succeeded, but a non-blocking issue was encountered. - public string Warning { get; set; } - /// The error message if saving failed. public string Error { get; set; } } diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index 7119ef032..1e020840a 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -26,7 +26,7 @@ internal class ApiClientsConfig public string AmazonRegion { get; set; } /// The AWS bucket in which to store temporary uploaded logs. - public string AmazonLogBucket { get; set; } + public string AmazonTempBucket { get; set; } /**** diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs new file mode 100644 index 000000000..e222a235e --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -0,0 +1,19 @@ +using System.Threading.Tasks; + +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// Provides access to raw data storage. + internal interface IStorageProvider + { + /// Save a text file to Pastebin or Amazon S3, if available. + /// The display title, if applicable. + /// The content to upload. + /// Whether to gzip the text. + /// Returns metadata about the save attempt. + Task SaveAsync(string title, string content, bool compress = true); + + /// Fetch raw text from storage. + /// The storage ID returned by . + Task GetAsync(string id); + } +} diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs new file mode 100644 index 000000000..bbb6e06b8 --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -0,0 +1,147 @@ +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Amazon; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Microsoft.Extensions.Options; +using StardewModdingAPI.Web.Framework.Clients.Pastebin; +using StardewModdingAPI.Web.Framework.Compression; +using StardewModdingAPI.Web.Framework.ConfigModels; + +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// Provides access to raw data storage. + internal class StorageProvider : IStorageProvider + { + /********* + ** Fields + *********/ + /// The API client settings. + private readonly ApiClientsConfig ClientsConfig; + + /// The underlying Pastebin client. + private readonly IPastebinClient Pastebin; + + /// The underlying text compression helper. + private readonly IGzipHelper GzipHelper; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The API client settings. + /// The underlying Pastebin client. + /// The underlying text compression helper. + public StorageProvider(IOptions clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + { + this.ClientsConfig = clientsConfig.Value; + this.Pastebin = pastebin; + this.GzipHelper = gzipHelper; + } + + /// Save a text file to Pastebin or Amazon S3, if available. + /// The display title, if applicable. + /// The content to upload. + /// Whether to gzip the text. + /// Returns metadata about the save attempt. + public async Task SaveAsync(string title, string content, bool compress = true) + { + // save to PasteBin + string uploadError; + { + SavePasteResult result = await this.Pastebin.PostAsync(title, content); + if (result.Success) + return new UploadResult(true, result.ID, null); + + uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; + } + + // fallback to S3 + try + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + using TransferUtility uploader = new TransferUtility(s3); + + string id = Guid.NewGuid().ToString("N"); + + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = this.ClientsConfig.AmazonTempBucket, + Key = $"uploads/{id}", + InputStream = stream, + Metadata = + { + // note: AWS will lowercase keys and prefix 'x-amz-meta-' + ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), + ["pastebin-error"] = uploadError + } + }; + + await uploader.UploadAsync(uploadRequest); + + return new UploadResult(true, id, uploadError); + } + catch (Exception ex) + { + return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + } + } + + /// Fetch raw text from storage. + /// The storage ID returned by . + public async Task GetAsync(string id) + { + // get from Amazon S3 + if (Guid.TryParseExact(id, "N", out Guid _)) + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + + try + { + using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); + using Stream responseStream = response.ResponseStream; + using StreamReader reader = new StreamReader(responseStream); + + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new StoredFileInfo { Error = "There's no file with that ID." } + : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } + } + + // get from PasteBin + else + { + PasteInfo response = await this.Pastebin.GetAsync(id); + response.Content = this.GzipHelper.DecompressString(response.Content); + return new StoredFileInfo + { + Success = response.Success, + Content = response.Content, + Error = response.Error + }; + } + } + } +} diff --git a/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs b/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs new file mode 100644 index 000000000..30676c882 --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs @@ -0,0 +1,23 @@ +using System; + +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// The response for a get-file request. + internal class StoredFileInfo + { + /// Whether the file was successfully fetched. + public bool Success { get; set; } + + /// The fetched file content (if is true). + public string Content { get; set; } + + /// When the file will no longer be available. + public DateTime? Expiry { get; set; } + + /// The error message if saving succeeded, but a non-blocking issue was encountered. + public string Warning { get; set; } + + /// The error message if saving failed. + public string Error { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/Storage/UploadResult.cs b/src/SMAPI.Web/Framework/Storage/UploadResult.cs new file mode 100644 index 000000000..483c1769c --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/UploadResult.cs @@ -0,0 +1,33 @@ +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// The result of an attempt to upload a file. + internal class UploadResult + { + /********* + ** Accessors + *********/ + /// Whether the file upload succeeded. + public bool Succeeded { get; } + + /// The file ID, if applicable. + public string ID { get; } + + /// The upload error, if any. + public string UploadError { get; } + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// Whether the file upload succeeded. + /// The file ID, if applicable. + /// The upload error, if any. + public UploadResult(bool succeeded, string id, string uploadError) + { + this.Succeeded = succeeded; + this.ID = id; + this.UploadError = uploadError; + } + } +} diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 53823771e..31b5e61db 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -7,6 +7,7 @@ using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using MongoDB.Bson.Serialization; using MongoDB.Driver; using Newtonsoft.Json; @@ -24,6 +25,7 @@ using StardewModdingAPI.Web.Framework.Compression; using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.Framework.RewriteRules; +using StardewModdingAPI.Web.Framework.Storage; namespace StardewModdingAPI.Web { @@ -158,7 +160,13 @@ public void ConfigureServices(IServiceCollection services) } // init helpers - services.AddSingleton(new GzipHelper()); + services + .AddSingleton(new GzipHelper()) + .AddSingleton(serv => new StorageProvider( + serv.GetRequiredService>(), + serv.GetRequiredService(), + serv.GetRequiredService() + )); } /// The method called by the runtime to configure the HTTP request pipeline. diff --git a/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs b/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs index 5b18331f7..c0dd7184d 100644 --- a/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs +++ b/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; @@ -24,7 +25,13 @@ public class JsonValidatorModel /// The schema validation errors, if any. public JsonValidatorErrorModel[] Errors { get; set; } = new JsonValidatorErrorModel[0]; - /// An error which occurred while uploading the JSON to Pastebin. + /// A non-blocking warning while uploading the file. + public string UploadWarning { get; set; } + + /// When the uploaded file will no longer be available. + public DateTime? Expiry { get; set; } + + /// An error which occurred while uploading the JSON. public string UploadError { get; set; } /// An error which occurred while parsing the JSON. @@ -41,7 +48,7 @@ public class JsonValidatorModel public JsonValidatorModel() { } /// Construct an instance. - /// The paste ID. + /// The stored file ID. /// The schema name with which the JSON was validated. /// The supported JSON schemas (names indexed by ID). public JsonValidatorModel(string pasteID, string schemaName, IDictionary schemaFormats) @@ -53,14 +60,18 @@ public JsonValidatorModel(string pasteID, string schemaName, IDictionarySet the validated content. /// The validated content. - public JsonValidatorModel SetContent(string content) + /// When the uploaded file will no longer be available. + /// A non-blocking warning while uploading the log. + public JsonValidatorModel SetContent(string content, DateTime? expiry, string uploadWarning = null) { this.Content = content; + this.Expiry = expiry; + this.UploadWarning = uploadWarning; return this; } - /// Set the error which occurred while uploading the log to Pastebin. + /// Set the error which occurred while uploading the JSON. /// The error message. public JsonValidatorModel SetUploadError(string error) { diff --git a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml index de6b06a23..a5a134ac8 100644 --- a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml +++ b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml @@ -1,3 +1,4 @@ +@using Humanizer @using StardewModdingAPI.Web.Framework @using StardewModdingAPI.Web.ViewModels.JsonValidator @model JsonValidatorModel @@ -26,7 +27,7 @@ { } - + @@ -67,6 +68,18 @@ else if (Model.PasteID != null) } +@* save warnings *@ +@if (Model.UploadWarning != null || Model.Expiry != null) +{ +
+ @if (Model.Expiry != null) + { + This log will expire in @((DateTime.UtcNow - Model.Expiry.Value).Humanize()). + } + +
+} + @* upload new file *@ @if (Model.Content == null) { diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index f81587ef7..b3567469d 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -26,7 +26,7 @@ "AmazonAccessKey": null, "AmazonSecretKey": null, "AmazonRegion": "us-east-1", - "AmazonLogBucket": "smapi-log-parser", + "AmazonTempBucket": "smapi-web-temp", "ChucklefishBaseUrl": "https://community.playstarbound.com", "ChucklefishModPageUrlFormat": "resources/{0}", diff --git a/src/SMAPI.Web/wwwroot/Content/css/json-validator.css b/src/SMAPI.Web/wwwroot/Content/css/json-validator.css index cd1176942..181950982 100644 --- a/src/SMAPI.Web/wwwroot/Content/css/json-validator.css +++ b/src/SMAPI.Web/wwwroot/Content/css/json-validator.css @@ -41,6 +41,12 @@ background: #FCC; } +.save-metadata { + margin-top: 1em; + font-size: 0.8em; + opacity: 0.3; +} + /********* ** Validation results *********/ From 8ddb60cee636cc17291100c316df4786eb3bb448 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 23:06:42 -0500 Subject: [PATCH 03/64] move supporter list into environment config --- src/SMAPI.Web/Controllers/IndexController.cs | 2 +- .../Framework/ConfigModels/SiteConfig.cs | 3 +++ src/SMAPI.Web/ViewModels/IndexModel.cs | 7 +++++- src/SMAPI.Web/Views/Index/Index.cshtml | 24 +++++++------------ src/SMAPI.Web/appsettings.Development.json | 5 ---- src/SMAPI.Web/appsettings.json | 5 ++-- 6 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/SMAPI.Web/Controllers/IndexController.cs b/src/SMAPI.Web/Controllers/IndexController.cs index 4e3602d5f..a887f14aa 100644 --- a/src/SMAPI.Web/Controllers/IndexController.cs +++ b/src/SMAPI.Web/Controllers/IndexController.cs @@ -72,7 +72,7 @@ public async Task Index() : null; // render view - var model = new IndexModel(stableVersionModel, betaVersionModel, this.SiteConfig.BetaBlurb); + var model = new IndexModel(stableVersionModel, betaVersionModel, this.SiteConfig.BetaBlurb, this.SiteConfig.SupporterList); return this.View(model); } diff --git a/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs index d379c423a..43969f51a 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs @@ -11,5 +11,8 @@ public class SiteConfig // must be public to pass into views /// A short sentence shown under the beta download button, if any. public string BetaBlurb { get; set; } + + /// A list of supports to credit on the main page, in Markdown format. + public string SupporterList { get; set; } } } diff --git a/src/SMAPI.Web/ViewModels/IndexModel.cs b/src/SMAPI.Web/ViewModels/IndexModel.cs index 82c4e06f9..450fdc0ea 100644 --- a/src/SMAPI.Web/ViewModels/IndexModel.cs +++ b/src/SMAPI.Web/ViewModels/IndexModel.cs @@ -15,6 +15,9 @@ public class IndexModel /// A short sentence shown under the beta download button, if any. public string BetaBlurb { get; set; } + /// A list of supports to credit on the main page, in Markdown format. + public string SupporterList { get; set; } + /********* ** Public methods @@ -26,11 +29,13 @@ public IndexModel() { } /// The latest stable SMAPI version. /// The latest prerelease SMAPI version (if newer than ). /// A short sentence shown under the beta download button, if any. - internal IndexModel(IndexVersionModel stableVersion, IndexVersionModel betaVersion, string betaBlurb) + /// A list of supports to credit on the main page, in Markdown format. + internal IndexModel(IndexVersionModel stableVersion, IndexVersionModel betaVersion, string betaBlurb, string supporterList) { this.StableVersion = stableVersion; this.BetaVersion = betaVersion; this.BetaBlurb = betaBlurb; + this.SupporterList = supporterList; } } } diff --git a/src/SMAPI.Web/Views/Index/Index.cshtml b/src/SMAPI.Web/Views/Index/Index.cshtml index ec9cfafe3..5d91dc842 100644 --- a/src/SMAPI.Web/Views/Index/Index.cshtml +++ b/src/SMAPI.Web/Views/Index/Index.cshtml @@ -1,3 +1,4 @@ +@using Markdig @using Microsoft.Extensions.Options @using StardewModdingAPI.Web.Framework @using StardewModdingAPI.Web.Framework.ConfigModels @@ -94,29 +95,22 @@ else
    • -

    - Special thanks to - bwdy, - hawkfalcon, - iKeychain, - jwdred, - Karmylla, - minervamaga, - Pucklynn, - Renorien, - Robby LaFarge, - and a few anonymous users for their ongoing support on Patreon; you're awesome! -

    +@if (!string.IsNullOrWhiteSpace(Model.SupporterList)) +{ + @Html.Raw(Markdig.Markdown.ToHtml( + $"Special thanks to {Model.SupporterList}, and a few anonymous users for their ongoing support on Patreon; you're awesome!" + )) +}

    For mod creators

      diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index 6b32f4ab5..74ded25d1 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -8,11 +8,6 @@ */ { - "Site": { - "BetaEnabled": false, - "BetaBlurb": null - }, - "ApiClients": { "AmazonAccessKey": null, "AmazonSecretKey": null, diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index b3567469d..2e20b2994 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -16,8 +16,9 @@ }, "Site": { - "BetaEnabled": null, - "BetaBlurb": null + "BetaEnabled": false, + "BetaBlurb": null, + "SupporterList": null }, "ApiClients": { From 9465628effad6bdb1994599031a8f60c3af2452e Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 4 Dec 2019 20:52:40 -0500 Subject: [PATCH 04/64] fix JSON validator format selector no longer working since URL changes --- docs/release-notes.md | 1 + src/SMAPI.Web/Views/JsonValidator/Index.cshtml | 6 +++--- src/SMAPI.Web/wwwroot/Content/js/json-validator.js | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 0b0a0f9e1..dec552d66 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -6,6 +6,7 @@ * For the web UI: * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. * Updated the JSON validator for Content Patcher 1.10.0. + * Fixed JSON validator no longer letting you change format when viewing results. ## 3.0.1 Released 02 December 2019 for Stardew Valley 1.4.0.1. diff --git a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml index a5a134ac8..a042f0240 100644 --- a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml +++ b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml @@ -27,17 +27,17 @@ { } - + - + } diff --git a/src/SMAPI.Web/wwwroot/Content/js/json-validator.js b/src/SMAPI.Web/wwwroot/Content/js/json-validator.js index 76b5f6d47..401efbee1 100644 --- a/src/SMAPI.Web/wwwroot/Content/js/json-validator.js +++ b/src/SMAPI.Web/wwwroot/Content/js/json-validator.js @@ -70,10 +70,10 @@ smapi.LineNumberRange = function (maxLines) { /** * UI logic for the JSON validator page. - * @param {any} sectionUrl The base JSON validator page URL. - * @param {any} pasteID The Pastebin paste ID for the content being viewed, if any. + * @param {string} urlFormat The URL format for a file, with $schemaName and $id placeholders. + * @param {string} pasteID The Pastebin paste ID for the content being viewed, if any. */ -smapi.jsonValidator = function (sectionUrl, pasteID) { +smapi.jsonValidator = function (urlFormat, pasteID) { /** * The original content element. */ @@ -138,7 +138,7 @@ smapi.jsonValidator = function (sectionUrl, pasteID) { // change format $("#output #format").on("change", function() { var schemaName = $(this).val(); - location.href = new URL(schemaName + "/" + pasteID, sectionUrl).toString(); + location.href = urlFormat.replace("$schemaName", schemaName).replace("$id", pasteID); }); // upload form From 9c9a0a41b041a1799904e78596fdf1d77451e1c4 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Thu, 5 Dec 2019 22:10:57 -0500 Subject: [PATCH 05/64] update for 'force off' gamepad option added in Stardew Valley 1.4.0.1 --- docs/release-notes.md | 3 +++ src/SMAPI/Framework/Input/SInputState.cs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/release-notes.md b/docs/release-notes.md index dec552d66..c4607ef01 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -3,6 +3,9 @@ # Release notes ## Upcoming release +* For players: + * Updated for the 'Force Off' gamepad mode added in Stardew Valley 1.4.0.1. + * For the web UI: * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. * Updated the JSON validator for Content Patcher 1.10.0. diff --git a/src/SMAPI/Framework/Input/SInputState.cs b/src/SMAPI/Framework/Input/SInputState.cs index d69e5604d..84cea36ce 100644 --- a/src/SMAPI/Framework/Input/SInputState.cs +++ b/src/SMAPI/Framework/Input/SInputState.cs @@ -129,6 +129,9 @@ public void UpdateSuppression() [Obsolete("This method should only be called by the game itself.")] public override GamePadState GetGamePadState() { + if (Game1.options.gamepadMode == Options.GamepadModes.ForceOff) + return base.GetGamePadState(); + return this.ShouldSuppressNow() ? this.SuppressedController : this.RealController; From 49080501d3929be4f954c5c93483a6254005f435 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 7 Dec 2019 10:24:01 -0500 Subject: [PATCH 06/64] fix link in package readme (#677) --- docs/technical/mod-package.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/technical/mod-package.md b/docs/technical/mod-package.md index a33480ade..5b971f96d 100644 --- a/docs/technical/mod-package.md +++ b/docs/technical/mod-package.md @@ -40,7 +40,7 @@ property | description `$(GamePath)` | The absolute path to the detected game folder. `$(GameExecutableName)` | The game's executable name for the current OS (`Stardew Valley` on Windows, or `StardewValley` on Linux/Mac). -If you get a build error saying it can't find your game, see [_set the game path_](#set-the-game-path). +If you get a build error saying it can't find your game, see [_custom game path_](#custom-game-path). ### Add assembly references The package adds assembly references to SMAPI, Stardew Valley, xTile, and MonoGame (Linux/Mac) or XNA @@ -228,7 +228,7 @@ or you have multiple installs, you can specify the path yourself. There's two wa ``` - 4. Replace `PATH_HERE` with your game path. + 4. Replace `PATH_HERE` with your game's folder path. * **Option 2: path in the project file.** _You'll need to do this for each project that uses the package._ From 47beb2f5345670be5fc1ba5ec109835f6a67e7a0 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 7 Dec 2019 19:24:27 -0500 Subject: [PATCH 07/64] fix launcher compatibility on Arch Linux Arch Linux sets the $TERMINAL variable, which makes SMAPI think the terminal is being overridden for testing and bypass the terminal selection logic. Since it's only used for testing, we can re-add it locally when needed. --- docs/release-notes.md | 1 + src/SMAPI.Installer/unix-launcher.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index c4607ef01..bab7409b4 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -5,6 +5,7 @@ * For players: * Updated for the 'Force Off' gamepad mode added in Stardew Valley 1.4.0.1. + * Fixed compatibility issue with Arch Linux. * For the web UI: * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. diff --git a/src/SMAPI.Installer/unix-launcher.sh b/src/SMAPI.Installer/unix-launcher.sh index bebba9fed..1422d888e 100644 --- a/src/SMAPI.Installer/unix-launcher.sh +++ b/src/SMAPI.Installer/unix-launcher.sh @@ -61,8 +61,8 @@ else COMMAND="type" fi - # select terminal (prefer $TERMINAL for overrides and testing, then xterm for best compatibility, then known supported terminals) - for terminal in "$TERMINAL" xterm gnome-terminal kitty terminator xfce4-terminal konsole terminal termite alacritty x-terminal-emulator; do + # select terminal (prefer xterm for best compatibility, then known supported terminals) + for terminal in xterm gnome-terminal kitty terminator xfce4-terminal konsole terminal termite alacritty x-terminal-emulator; do if $COMMAND "$terminal" 2>/dev/null; then # Find the true shell behind x-terminal-emulator if [ "$(basename "$(readlink -f $(which "$terminal"))")" != "x-terminal-emulator" ]; then From 04b9a810dde93ff790e356f0af3510c7d20bebfc Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 8 Dec 2019 11:27:23 -0500 Subject: [PATCH 08/64] add asset propagation for grass textures --- docs/release-notes.md | 3 ++ src/SMAPI/Metadata/CoreAssetPropagator.cs | 35 ++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index bab7409b4..8754e7775 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -7,6 +7,9 @@ * Updated for the 'Force Off' gamepad mode added in Stardew Valley 1.4.0.1. * Fixed compatibility issue with Arch Linux. +* For modders: + * Added asset propagation for grass textures. + * For the web UI: * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. * Updated the JSON validator for Content Patcher 1.10.0. diff --git a/src/SMAPI/Metadata/CoreAssetPropagator.cs b/src/SMAPI/Metadata/CoreAssetPropagator.cs index 1c0a04f0c..985e4e1bb 100644 --- a/src/SMAPI/Metadata/CoreAssetPropagator.cs +++ b/src/SMAPI/Metadata/CoreAssetPropagator.cs @@ -474,10 +474,14 @@ private bool PropagateOther(LocalizedContentManager content, string key, Type ty /**** ** Content\TerrainFeatures ****/ - case "terrainfeatures\\flooring": // Flooring + case "terrainfeatures\\flooring": // from Flooring Flooring.floorsTexture = content.Load(key); return true; + case "terrainfeatures\\grass": // from Grass + this.ReloadGrassTextures(content, key); + return true; + case "terrainfeatures\\hoedirt": // from HoeDirt HoeDirt.lightTexture = content.Load(key); return true; @@ -694,6 +698,35 @@ select fence return true; } + /// Reload tree textures. + /// The content manager through which to reload the asset. + /// The asset key to reload. + /// Returns whether any textures were reloaded. + private bool ReloadGrassTextures(LocalizedContentManager content, string key) + { + Grass[] grasses = + ( + from location in Game1.locations + from grass in location.terrainFeatures.Values.OfType() + let textureName = this.NormalizeAssetNameIgnoringEmpty( + this.Reflection.GetMethod(grass, "textureName").Invoke() + ) + where textureName == key + select grass + ) + .ToArray(); + + if (grasses.Any()) + { + Lazy texture = new Lazy(() => content.Load(key)); + foreach (Grass grass in grasses) + this.Reflection.GetField>(grass, "texture").SetValue(texture); + return true; + } + + return false; + } + /// Reload the disposition data for matching NPCs. /// The content manager through which to reload the asset. /// The asset key to reload. From 194b96a79c335fa098a6cf55c2be75c7f2e9c6ad Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 8 Dec 2019 11:31:20 -0500 Subject: [PATCH 09/64] use GetLocations logic more consistently in asset propagation --- src/SMAPI/Metadata/CoreAssetPropagator.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/SMAPI/Metadata/CoreAssetPropagator.cs b/src/SMAPI/Metadata/CoreAssetPropagator.cs index 985e4e1bb..8b00d893e 100644 --- a/src/SMAPI/Metadata/CoreAssetPropagator.cs +++ b/src/SMAPI/Metadata/CoreAssetPropagator.cs @@ -611,7 +611,7 @@ private bool ReloadBuildings(LocalizedContentManager content, string key) { // get buildings string type = Path.GetFileName(key); - Building[] buildings = Game1.locations + Building[] buildings = this.GetLocations(buildingInteriors: false) .OfType() .SelectMany(p => p.buildings) .Where(p => p.buildingType.Value == type) @@ -706,7 +706,7 @@ private bool ReloadGrassTextures(LocalizedContentManager content, string key) { Grass[] grasses = ( - from location in Game1.locations + from location in this.GetLocations() from grass in location.terrainFeatures.Values.OfType() let textureName = this.NormalizeAssetNameIgnoringEmpty( this.Reflection.GetMethod(grass, "textureName").Invoke() @@ -804,7 +804,7 @@ private int ReloadNpcPortraits(LocalizedContentManager content, IEnumerableReturns whether any textures were reloaded. private bool ReloadTreeTextures(LocalizedContentManager content, string key, int type) { - Tree[] trees = Game1.locations + Tree[] trees = this.GetLocations() .SelectMany(p => p.terrainFeatures.Values.OfType()) .Where(tree => tree.treeType.Value == type) .ToArray(); @@ -909,7 +909,8 @@ private IEnumerable GetFarmAnimals() } /// Get all locations in the game. - private IEnumerable GetLocations() + /// Whether to also get the interior locations for constructable buildings. + private IEnumerable GetLocations(bool buildingInteriors = true) { // get available root locations IEnumerable rootLocations = Game1.locations; @@ -921,7 +922,7 @@ private IEnumerable GetLocations() { yield return location; - if (location is BuildableGameLocation buildableLocation) + if (buildingInteriors && location is BuildableGameLocation buildableLocation) { foreach (Building building in buildableLocation.buildings) { From 238fbfe5698fb1791d47e8772ba1c5a86f9300ca Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 8 Dec 2019 12:20:59 -0500 Subject: [PATCH 10/64] let mods use Read/WriteSaveData while a save is being loaded --- docs/release-notes.md | 1 + src/SMAPI/Framework/ModHelpers/DataHelper.cs | 48 +++++++++++++++----- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 8754e7775..fc3cd4f7c 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -9,6 +9,7 @@ * For modders: * Added asset propagation for grass textures. + * `helper.Read/WriteSaveData` can now be used while a save is being loaded (e.g. within a `Specialized.LoadStageChanged` event). * For the web UI: * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. diff --git a/src/SMAPI/Framework/ModHelpers/DataHelper.cs b/src/SMAPI/Framework/ModHelpers/DataHelper.cs index cc08c42bf..3d43c539f 100644 --- a/src/SMAPI/Framework/ModHelpers/DataHelper.cs +++ b/src/SMAPI/Framework/ModHelpers/DataHelper.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.IO; using Newtonsoft.Json; +using StardewModdingAPI.Enums; using StardewModdingAPI.Toolkit.Serialization; using StardewModdingAPI.Toolkit.Utilities; using StardewValley; @@ -77,33 +79,45 @@ public void WriteJsonFile(string path, TModel data) where TModel : class /// The player hasn't loaded a save file yet or isn't the main player. public TModel ReadSaveData(string key) where TModel : class { - if (!Game1.hasLoadedGame) + if (Context.LoadStage == LoadStage.None) throw new InvalidOperationException($"Can't use {nameof(IMod.Helper)}.{nameof(IModHelper.Data)}.{nameof(this.ReadSaveData)} when a save file isn't loaded."); if (!Game1.IsMasterGame) throw new InvalidOperationException($"Can't use {nameof(IMod.Helper)}.{nameof(IModHelper.Data)}.{nameof(this.ReadSaveData)} because this isn't the main player. (Save files are stored on the main player's computer.)"); - return Game1.CustomData.TryGetValue(this.GetSaveFileKey(key), out string value) - ? this.JsonHelper.Deserialize(value) - : null; + + string internalKey = this.GetSaveFileKey(key); + foreach (IDictionary dataField in this.GetDataFields(Context.LoadStage)) + { + if (dataField.TryGetValue(internalKey, out string value)) + return this.JsonHelper.Deserialize(value); + } + return null; } /// Save arbitrary data to the current save slot. This is only possible if a save has been loaded, and the data will be lost if the player exits without saving the current day. /// The model type. This should be a plain class that has public properties for the data you want. The properties can be complex types. /// The unique key identifying the data. - /// The arbitrary data to save. + /// The arbitrary data to save. /// The player hasn't loaded a save file yet or isn't the main player. - public void WriteSaveData(string key, TModel data) where TModel : class + public void WriteSaveData(string key, TModel model) where TModel : class { - if (!Game1.hasLoadedGame) + if (Context.LoadStage == LoadStage.None) throw new InvalidOperationException($"Can't use {nameof(IMod.Helper)}.{nameof(IModHelper.Data)}.{nameof(this.WriteSaveData)} when a save file isn't loaded."); if (!Game1.IsMasterGame) throw new InvalidOperationException($"Can't use {nameof(IMod.Helper)}.{nameof(IModHelper.Data)}.{nameof(this.WriteSaveData)} because this isn't the main player. (Save files are stored on the main player's computer.)"); string internalKey = this.GetSaveFileKey(key); - if (data != null) - Game1.CustomData[internalKey] = this.JsonHelper.Serialize(data, Formatting.None); - else - Game1.CustomData.Remove(internalKey); + string data = model != null + ? this.JsonHelper.Serialize(model, Formatting.None) + : null; + + foreach (IDictionary dataField in this.GetDataFields(Context.LoadStage)) + { + if (data != null) + dataField[internalKey] = data; + else + dataField.Remove(internalKey); + } } /**** @@ -146,6 +160,18 @@ private string GetSaveFileKey(string key) return $"smapi/mod-data/{this.ModID}/{key}".ToLower(); } + /// Get the data fields to read/write for save data. + /// The current load stage. + private IEnumerable> GetDataFields(LoadStage stage) + { + if (stage == LoadStage.None) + yield break; + + yield return Game1.CustomData; + if (SaveGame.loaded != null) + yield return SaveGame.loaded.CustomData; + } + /// Get the absolute path for a global data file. /// The unique key identifying the data. private string GetGlobalDataPath(string key) From 0454d7dad91b8963bad65014df500ef8081f889c Mon Sep 17 00:00:00 2001 From: Alexander Paetzelt Date: Thu, 12 Dec 2019 20:52:20 +0100 Subject: [PATCH 11/64] Add mate-terminal to the known-to-work terminals --- src/SMAPI.Installer/unix-launcher.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SMAPI.Installer/unix-launcher.sh b/src/SMAPI.Installer/unix-launcher.sh index 1422d888e..b72eed227 100644 --- a/src/SMAPI.Installer/unix-launcher.sh +++ b/src/SMAPI.Installer/unix-launcher.sh @@ -62,7 +62,7 @@ else fi # select terminal (prefer xterm for best compatibility, then known supported terminals) - for terminal in xterm gnome-terminal kitty terminator xfce4-terminal konsole terminal termite alacritty x-terminal-emulator; do + for terminal in xterm gnome-terminal kitty terminator xfce4-terminal konsole terminal termite alacritty mate-terminal x-terminal-emulator; do if $COMMAND "$terminal" 2>/dev/null; then # Find the true shell behind x-terminal-emulator if [ "$(basename "$(readlink -f $(which "$terminal"))")" != "x-terminal-emulator" ]; then @@ -108,7 +108,7 @@ else alacritty -e sh -c 'TERM=xterm ./StardewModdingAPI.bin.x86 $*' fi ;; - xterm|xfce4-terminal|gnome-terminal|terminal|termite) + xterm|xfce4-terminal|gnome-terminal|terminal|termite|mate-terminal) $LAUNCHTERM -e "sh -c 'TERM=xterm $LAUNCHER'" ;; konsole) From e4a7ca5826ae0cb372aec529c4f21a53c98079da Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Thu, 12 Dec 2019 23:22:19 -0500 Subject: [PATCH 12/64] batch asset editor/loader changes --- docs/release-notes.md | 1 + .../Content/AssetInterceptorChange.cs | 91 +++++++++++++++++++ src/SMAPI/Framework/ContentCoordinator.cs | 82 ----------------- src/SMAPI/Framework/SCore.cs | 27 +----- src/SMAPI/Framework/SGame.cs | 50 +++++++++- 5 files changed, 143 insertions(+), 108 deletions(-) create mode 100644 src/SMAPI/Framework/Content/AssetInterceptorChange.cs diff --git a/docs/release-notes.md b/docs/release-notes.md index fc3cd4f7c..690c64429 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -6,6 +6,7 @@ * For players: * Updated for the 'Force Off' gamepad mode added in Stardew Valley 1.4.0.1. * Fixed compatibility issue with Arch Linux. + * Internal optimizations. * For modders: * Added asset propagation for grass textures. diff --git a/src/SMAPI/Framework/Content/AssetInterceptorChange.cs b/src/SMAPI/Framework/Content/AssetInterceptorChange.cs new file mode 100644 index 000000000..498afe365 --- /dev/null +++ b/src/SMAPI/Framework/Content/AssetInterceptorChange.cs @@ -0,0 +1,91 @@ +using System; +using System.Reflection; + +namespace StardewModdingAPI.Framework.Content +{ + /// A wrapper for and for internal cache invalidation. + internal class AssetInterceptorChange + { + /********* + ** Accessors + *********/ + /// The mod which registered the interceptor. + public IModMetadata Mod { get; } + + /// The interceptor instance. + public object Instance { get; } + + /// Whether the asset interceptor was added since the last tick. Mutually exclusive with . + public bool WasAdded { get; } + + /// Whether the asset interceptor was removed since the last tick. Mutually exclusive with . + public bool WasRemoved => this.WasAdded; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The mod registering the interceptor. + /// The interceptor. This must be an or instance. + /// Whether the asset interceptor was added since the last tick; else removed. + public AssetInterceptorChange(IModMetadata mod, object instance, bool wasAdded) + { + this.Mod = mod ?? throw new ArgumentNullException(nameof(mod)); + this.Instance = instance ?? throw new ArgumentNullException(nameof(instance)); + this.WasAdded = wasAdded; + + if (!(instance is IAssetEditor) && !(instance is IAssetLoader)) + throw new InvalidCastException($"The provided {nameof(instance)} value must be an {nameof(IAssetEditor)} or {nameof(IAssetLoader)} instance."); + } + + /// Get whether this instance can intercept the given asset. + /// Basic metadata about the asset being loaded. + public bool CanIntercept(IAssetInfo asset) + { + MethodInfo canIntercept = this.GetType().GetMethod(nameof(this.CanInterceptImpl), BindingFlags.Instance | BindingFlags.NonPublic); + if (canIntercept == null) + throw new InvalidOperationException($"SMAPI couldn't access the {nameof(AssetInterceptorChange)}.{nameof(this.CanInterceptImpl)} implementation."); + + return (bool)canIntercept.MakeGenericMethod(asset.DataType).Invoke(this, new object[] { asset }); + } + + + /********* + ** Private methods + *********/ + /// Get whether this instance can intercept the given asset. + /// The asset type. + /// Basic metadata about the asset being loaded. + private bool CanInterceptImpl(IAssetInfo asset) + { + // check edit + if (this.Instance is IAssetEditor editor) + { + try + { + return editor.CanEdit(asset); + } + catch (Exception ex) + { + this.Mod.LogAsMod($"Mod failed when checking whether it could edit asset '{asset.AssetName}'. Error details:\n{ex.GetLogSummary()}", LogLevel.Error); + } + } + + // check load + if (this.Instance is IAssetLoader loader) + { + try + { + return loader.CanLoad(asset); + } + catch (Exception ex) + { + this.Mod.LogAsMod($"Mod failed when checking whether it could load asset '{asset.AssetName}'. Error details:\n{ex.GetLogSummary()}", LogLevel.Error); + } + } + + return false; + } + } +} diff --git a/src/SMAPI/Framework/ContentCoordinator.cs b/src/SMAPI/Framework/ContentCoordinator.cs index 08ebe6a5a..97b54c5be 100644 --- a/src/SMAPI/Framework/ContentCoordinator.cs +++ b/src/SMAPI/Framework/ContentCoordinator.cs @@ -3,7 +3,6 @@ using System.Globalization; using System.IO; using System.Linq; -using System.Reflection; using Microsoft.Xna.Framework.Content; using StardewModdingAPI.Framework.Content; using StardewModdingAPI.Framework.ContentManagers; @@ -188,59 +187,6 @@ public T LoadManagedAsset(string contentManagerID, string relativePath) return contentManager.Load(relativePath, this.DefaultLanguage, useCache: false); } - /// Purge assets from the cache that match one of the interceptors. - /// The asset editors for which to purge matching assets. - /// The asset loaders for which to purge matching assets. - /// Returns the invalidated asset names. - public IEnumerable InvalidateCacheFor(IAssetEditor[] editors, IAssetLoader[] loaders) - { - if (!editors.Any() && !loaders.Any()) - return new string[0]; - - // get CanEdit/Load methods - MethodInfo canEdit = typeof(IAssetEditor).GetMethod(nameof(IAssetEditor.CanEdit)); - MethodInfo canLoad = typeof(IAssetLoader).GetMethod(nameof(IAssetLoader.CanLoad)); - if (canEdit == null || canLoad == null) - throw new InvalidOperationException("SMAPI could not access the interceptor methods."); // should never happen - - // invalidate matching keys - return this.InvalidateCache(asset => - { - // check loaders - MethodInfo canLoadGeneric = canLoad.MakeGenericMethod(asset.DataType); - foreach (IAssetLoader loader in loaders) - { - try - { - if ((bool)canLoadGeneric.Invoke(loader, new object[] { asset })) - return true; - } - catch (Exception ex) - { - this.GetModFor(loader).LogAsMod($"Mod failed when checking whether it could load asset '{asset.AssetName}'. Error details:\n{ex.GetLogSummary()}", LogLevel.Error); - } - } - - // check editors - MethodInfo canEditGeneric = canEdit.MakeGenericMethod(asset.DataType); - foreach (IAssetEditor editor in editors) - { - try - { - if ((bool)canEditGeneric.Invoke(editor, new object[] { asset })) - return true; - } - catch (Exception ex) - { - this.GetModFor(editor).LogAsMod($"Mod failed when checking whether it could edit asset '{asset.AssetName}'. Error details:\n{ex.GetLogSummary()}", LogLevel.Error); - } - } - - // asset not affected by a loader or editor - return false; - }); - } - /// Purge matched assets from the cache. /// Matches the asset keys to invalidate. /// Whether to dispose invalidated assets. This should only be true when they're being invalidated as part of a dispose, to avoid crashing the game. @@ -308,33 +254,5 @@ private void OnDisposing(IContentManager contentManager) this.ContentManagers.Remove(contentManager); } - - /// Get the mod which registered an asset loader. - /// The asset loader. - /// The given loader couldn't be matched to a mod. - private IModMetadata GetModFor(IAssetLoader loader) - { - foreach (var pair in this.Loaders) - { - if (pair.Value.Contains(loader)) - return pair.Key; - } - - throw new KeyNotFoundException("This loader isn't associated with a known mod."); - } - - /// Get the mod which registered an asset editor. - /// The asset editor. - /// The given editor couldn't be matched to a mod. - private IModMetadata GetModFor(IAssetEditor editor) - { - foreach (var pair in this.Editors) - { - if (pair.Value.Contains(editor)) - return pair.Key; - } - - throw new KeyNotFoundException("This editor isn't associated with a known mod."); - } } } diff --git a/src/SMAPI/Framework/SCore.cs b/src/SMAPI/Framework/SCore.cs index f18733912..fb3506b4b 100644 --- a/src/SMAPI/Framework/SCore.cs +++ b/src/SMAPI/Framework/SCore.cs @@ -842,34 +842,11 @@ void LogSkip(IModMetadata mod, string errorPhrase, string errorDetails) { if (metadata.Mod.Helper.Content is ContentHelper helper) { - helper.ObservableAssetEditors.CollectionChanged += (sender, e) => - { - if (e.NewItems?.Count > 0) - { - this.Monitor.Log("Invalidating cache entries for new asset editors...", LogLevel.Trace); - this.ContentCore.InvalidateCacheFor(e.NewItems.Cast().ToArray(), new IAssetLoader[0]); - } - }; - helper.ObservableAssetLoaders.CollectionChanged += (sender, e) => - { - if (e.NewItems?.Count > 0) - { - this.Monitor.Log("Invalidating cache entries for new asset loaders...", LogLevel.Trace); - this.ContentCore.InvalidateCacheFor(new IAssetEditor[0], e.NewItems.Cast().ToArray()); - } - }; + helper.ObservableAssetEditors.CollectionChanged += (sender, e) => this.GameInstance.OnAssetInterceptorsChanged(metadata, e.NewItems, e.OldItems); + helper.ObservableAssetLoaders.CollectionChanged += (sender, e) => this.GameInstance.OnAssetInterceptorsChanged(metadata, e.NewItems, e.OldItems); } } - // reset cache now if any editors or loaders were added during entry - IAssetEditor[] editors = loadedMods.SelectMany(p => p.Mod.Helper.Content.AssetEditors).ToArray(); - IAssetLoader[] loaders = loadedMods.SelectMany(p => p.Mod.Helper.Content.AssetLoaders).ToArray(); - if (editors.Any() || loaders.Any()) - { - this.Monitor.Log("Invalidating cached assets for new editors & loaders...", LogLevel.Trace); - this.ContentCore.InvalidateCacheFor(editors, loaders); - } - // unlock mod integrations this.ModRegistry.AreAllModsInitialized = true; } diff --git a/src/SMAPI/Framework/SGame.cs b/src/SMAPI/Framework/SGame.cs index 47261862d..4774233e1 100644 --- a/src/SMAPI/Framework/SGame.cs +++ b/src/SMAPI/Framework/SGame.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.ObjectModel; @@ -12,6 +13,7 @@ using Netcode; using StardewModdingAPI.Enums; using StardewModdingAPI.Events; +using StardewModdingAPI.Framework.Content; using StardewModdingAPI.Framework.Events; using StardewModdingAPI.Framework.Input; using StardewModdingAPI.Framework.Networking; @@ -99,7 +101,7 @@ internal class SGame : Game1 private WatcherCore Watchers; /// A snapshot of the current state. - private WatcherSnapshot WatcherSnapshot = new WatcherSnapshot(); + private readonly WatcherSnapshot WatcherSnapshot = new WatcherSnapshot(); /// Whether post-game-startup initialization has been performed. private bool IsInitialized; @@ -133,6 +135,9 @@ internal class SGame : Game1 /// This property must be threadsafe, since it's accessed from a separate console input thread. public ConcurrentQueue CommandQueue { get; } = new ConcurrentQueue(); + /// Asset interceptors added or removed since the last tick. + private readonly List ReloadAssetInterceptorsQueue = new List(); + /********* ** Protected methods @@ -249,6 +254,24 @@ internal void OnLoadStageChanged(LoadStage newStage) this.Events.ReturnedToTitle.RaiseEmpty(); } + /// A callback invoked when a mod adds or removes an asset interceptor. + /// The mod which added or removed interceptors. + /// The added interceptors. + /// The removed interceptors. + internal void OnAssetInterceptorsChanged(IModMetadata mod, IEnumerable added, IEnumerable removed) + { + if (added != null) + { + foreach (object instance in added) + this.ReloadAssetInterceptorsQueue.Add(new AssetInterceptorChange(mod, instance, wasAdded: true)); + } + if (removed != null) + { + foreach (object instance in removed) + this.ReloadAssetInterceptorsQueue.Add(new AssetInterceptorChange(mod, instance, wasAdded: false)); + } + } + /// Constructor a content manager to read XNB files. /// The service provider to use to locate services. /// The root directory to search for content. @@ -404,6 +427,31 @@ protected override void Update(GameTime gameTime) return; } + + /********* + ** Reload assets when interceptors are added/removed + *********/ + if (this.ReloadAssetInterceptorsQueue.Any()) + { + this.Monitor.Log("Invalidating cached assets for new editors & loaders...", LogLevel.Trace); + this.Monitor.Log( + "changed: " + + string.Join(", ", + this.ReloadAssetInterceptorsQueue + .GroupBy(p => p.Mod) + .OrderBy(p => p.Key.DisplayName) + .Select(modGroup => + $"{modGroup.Key.DisplayName} (" + + string.Join(", ", modGroup.GroupBy(p => p.WasAdded).ToDictionary(p => p.Key, p => p.Count()).Select(p => $"{(p.Key ? "added" : "removed")} {p.Value}")) + + ")" + ) + ) + ); + + this.ContentCore.InvalidateCache(asset => this.ReloadAssetInterceptorsQueue.Any(p => p.CanIntercept(asset))); + this.ReloadAssetInterceptorsQueue.Clear(); + } + /********* ** Execute commands *********/ From ff94a8149ed5a0f597500bfb2b1896bdb2f1fff3 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Thu, 12 Dec 2019 23:46:32 -0500 Subject: [PATCH 13/64] fix assets not being disposed when a content manager is disposed --- src/SMAPI/Framework/Content/ContentCache.cs | 2 +- src/SMAPI/Framework/ContentManagers/BaseContentManager.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SMAPI/Framework/Content/ContentCache.cs b/src/SMAPI/Framework/Content/ContentCache.cs index 4178b6639..c252b7b67 100644 --- a/src/SMAPI/Framework/Content/ContentCache.cs +++ b/src/SMAPI/Framework/Content/ContentCache.cs @@ -119,7 +119,7 @@ public bool Remove(string key, bool dispose) /// Matches the asset keys to invalidate. /// Whether to dispose invalidated assets. This should only be true when they're being invalidated as part of a dispose, to avoid crashing the game. /// Returns the removed keys (if any). - public IEnumerable Remove(Func predicate, bool dispose = false) + public IEnumerable Remove(Func predicate, bool dispose) { List removed = new List(); foreach (string key in this.Cache.Keys.ToArray()) diff --git a/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs b/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs index 5283340ea..93fd729ba 100644 --- a/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs +++ b/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs @@ -200,7 +200,7 @@ public IEnumerable> InvalidateCache(Func return true; } return false; - }); + }, dispose); return removeAssetNames.Select(p => Tuple.Create(p.Key, p.Value)); } From 3ba718749c258e48d83d7c2fe6b2dc08f165a29a Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 14 Dec 2019 10:35:08 -0500 Subject: [PATCH 14/64] don't keep a reference to uncached assets --- .../ContentManagers/BaseContentManager.cs | 12 +++-- .../ContentManagers/GameContentManager.cs | 47 ++++++++++--------- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs b/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs index 93fd729ba..4cfeeebab 100644 --- a/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs +++ b/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs @@ -258,20 +258,24 @@ protected virtual T RawLoad(string assetName, bool useCache) : base.ReadAsset(assetName, disposable => this.Disposables.Add(new WeakReference(disposable))); } - /// Inject an asset into the cache. + /// Add tracking data to an asset and add it to the cache. /// The type of asset to inject. /// The asset path relative to the loader root directory, not including the .xnb extension. /// The asset value. /// The language code for which to inject the asset. - protected virtual void Inject(string assetName, T value, LanguageCode language) + /// Whether to save the asset to the asset cache. + protected virtual void TrackAsset(string assetName, T value, LanguageCode language, bool useCache) { // track asset key if (value is Texture2D texture) texture.Name = assetName; // cache asset - assetName = this.AssertAndNormalizeAssetName(assetName); - this.Cache[assetName] = value; + if (useCache) + { + assetName = this.AssertAndNormalizeAssetName(assetName); + this.Cache[assetName] = value; + } } /// Parse a cache key into its component parts. diff --git a/src/SMAPI/Framework/ContentManagers/GameContentManager.cs b/src/SMAPI/Framework/ContentManagers/GameContentManager.cs index 0b5635554..04c4564f1 100644 --- a/src/SMAPI/Framework/ContentManagers/GameContentManager.cs +++ b/src/SMAPI/Framework/ContentManagers/GameContentManager.cs @@ -83,8 +83,7 @@ public override T Load(string assetName, LocalizedContentManager.LanguageCode if (this.Coordinator.TryParseManagedAssetKey(assetName, out string contentManagerID, out string relativePath)) { T managedAsset = this.Coordinator.LoadManagedAsset(contentManagerID, relativePath); - if (useCache) - this.Inject(assetName, managedAsset, language); + this.TrackAsset(assetName, managedAsset, language, useCache); return managedAsset; } @@ -111,7 +110,7 @@ public override T Load(string assetName, LocalizedContentManager.LanguageCode } // update cache & return data - this.Inject(assetName, data, language); + this.TrackAsset(assetName, data, language, useCache); return data; } @@ -169,18 +168,19 @@ protected override bool IsNormalizedKeyLoaded(string normalizedAssetName) return false; } - /// Inject an asset into the cache. + /// Add tracking data to an asset and add it to the cache. /// The type of asset to inject. /// The asset path relative to the loader root directory, not including the .xnb extension. /// The asset value. /// The language code for which to inject the asset. - protected override void Inject(string assetName, T value, LanguageCode language) + /// Whether to save the asset to the asset cache. + protected override void TrackAsset(string assetName, T value, LanguageCode language, bool useCache) { // handle explicit language in asset name { if (this.TryParseExplicitLanguageAssetKey(assetName, out string newAssetName, out LanguageCode newLanguage)) { - this.Inject(newAssetName, value, newLanguage); + this.TrackAsset(newAssetName, value, newLanguage, useCache); return; } } @@ -192,24 +192,27 @@ protected override void Inject(string assetName, T value, LanguageCode langua // only caches by the most specific key). // 2. Because a mod asset loader/editor may have changed the asset in a way that // doesn't change the instance stored in the cache, e.g. using `asset.ReplaceWith`. - string keyWithLocale = $"{assetName}.{this.GetLocale(language)}"; - base.Inject(assetName, value, language); - if (this.Cache.ContainsKey(keyWithLocale)) - base.Inject(keyWithLocale, value, language); - - // track whether the injected asset is translatable for is-loaded lookups - if (this.Cache.ContainsKey(keyWithLocale)) - { - this.IsLocalizableLookup[assetName] = true; - this.IsLocalizableLookup[keyWithLocale] = true; - } - else if (this.Cache.ContainsKey(assetName)) + if (useCache) { - this.IsLocalizableLookup[assetName] = false; - this.IsLocalizableLookup[keyWithLocale] = false; + string keyWithLocale = $"{assetName}.{this.GetLocale(language)}"; + base.TrackAsset(assetName, value, language, useCache: true); + if (this.Cache.ContainsKey(keyWithLocale)) + base.TrackAsset(keyWithLocale, value, language, useCache: true); + + // track whether the injected asset is translatable for is-loaded lookups + if (this.Cache.ContainsKey(keyWithLocale)) + { + this.IsLocalizableLookup[assetName] = true; + this.IsLocalizableLookup[keyWithLocale] = true; + } + else if (this.Cache.ContainsKey(assetName)) + { + this.IsLocalizableLookup[assetName] = false; + this.IsLocalizableLookup[keyWithLocale] = false; + } + else + this.Monitor.Log($"Asset '{assetName}' could not be found in the cache immediately after injection.", LogLevel.Error); } - else - this.Monitor.Log($"Asset '{assetName}' could not be found in the cache immediately after injection.", LogLevel.Error); } /// Load an asset file directly from the underlying content manager. From 6dc442803fe4fbe2a38b9fb287990cc8692c17eb Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 14 Dec 2019 10:38:17 -0500 Subject: [PATCH 15/64] fix private assets from content packs not having tracking info --- docs/release-notes.md | 1 + .../ContentManagers/ModContentManager.cs | 30 +++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 690c64429..6f06d3d23 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -11,6 +11,7 @@ * For modders: * Added asset propagation for grass textures. * `helper.Read/WriteSaveData` can now be used while a save is being loaded (e.g. within a `Specialized.LoadStageChanged` event). + * Fixed private textures loaded from content packs not having their `Name` field set. * For the web UI: * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. diff --git a/src/SMAPI/Framework/ContentManagers/ModContentManager.cs b/src/SMAPI/Framework/ContentManagers/ModContentManager.cs index 90b861793..fdf76b24d 100644 --- a/src/SMAPI/Framework/ContentManagers/ModContentManager.cs +++ b/src/SMAPI/Framework/ContentManagers/ModContentManager.cs @@ -105,6 +105,7 @@ public override T Load(string assetName, LanguageCode language, bool useCache // get local asset SContentLoadException GetContentError(string reasonPhrase) => new SContentLoadException($"Failed loading asset '{assetName}' from {this.Name}: {reasonPhrase}"); + T asset; try { // get file @@ -118,22 +119,22 @@ public override T Load(string assetName, LanguageCode language, bool useCache // XNB file case ".xnb": { - T data = this.RawLoad(assetName, useCache: false); - if (data is Map map) + asset = this.RawLoad(assetName, useCache: false); + if (asset is Map map) { this.NormalizeTilesheetPaths(map); this.FixCustomTilesheetPaths(map, relativeMapPath: assetName); } - return data; } + break; // unpacked data case ".json": { - if (!this.JsonHelper.ReadJsonFileIfExists(file.FullName, out T data)) + if (!this.JsonHelper.ReadJsonFileIfExists(file.FullName, out asset)) throw GetContentError("the JSON file is invalid."); // should never happen since we check for file existence above - return data; } + break; // unpacked image case ".png": @@ -143,13 +144,13 @@ public override T Load(string assetName, LanguageCode language, bool useCache throw GetContentError($"can't read file with extension '{file.Extension}' as type '{typeof(T)}'; must be type '{typeof(Texture2D)}'."); // fetch & cache - using (FileStream stream = File.OpenRead(file.FullName)) - { - Texture2D texture = Texture2D.FromStream(Game1.graphics.GraphicsDevice, stream); - texture = this.PremultiplyTransparency(texture); - return (T)(object)texture; - } + using FileStream stream = File.OpenRead(file.FullName); + + Texture2D texture = Texture2D.FromStream(Game1.graphics.GraphicsDevice, stream); + texture = this.PremultiplyTransparency(texture); + asset = (T)(object)texture; } + break; // unpacked map case ".tbin": @@ -163,8 +164,9 @@ public override T Load(string assetName, LanguageCode language, bool useCache Map map = formatManager.LoadMap(file.FullName); this.NormalizeTilesheetPaths(map); this.FixCustomTilesheetPaths(map, relativeMapPath: assetName); - return (T)(object)map; + asset = (T)(object)map; } + break; default: throw GetContentError($"unknown file extension '{file.Extension}'; must be one of '.json', '.png', '.tbin', or '.xnb'."); @@ -176,6 +178,10 @@ public override T Load(string assetName, LanguageCode language, bool useCache throw GetContentError("couldn't find libgdiplus, which is needed to load mod images. Make sure Mono is installed and you're running the game through the normal launcher."); throw new SContentLoadException($"The content manager failed loading content asset '{assetName}' from {this.Name}.", ex); } + + // track & return asset + this.TrackAsset(assetName, asset, language, useCache); + return asset; } /// Create a new content manager for temporary use. From 16f986c51b9c87c2253a39fd771dcc24f7c43db4 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 14 Dec 2019 21:31:34 -0500 Subject: [PATCH 16/64] refactor cache invalidation & propagation to allow for future optimizations --- src/SMAPI/Framework/Content/ContentCache.cs | 5 +- src/SMAPI/Framework/ContentCoordinator.cs | 25 ++++--- .../ContentManagers/BaseContentManager.cs | 16 ++--- .../ContentManagers/GameContentManager.cs | 2 +- .../ContentManagers/IContentManager.cs | 4 +- src/SMAPI/Metadata/CoreAssetPropagator.cs | 67 ++++++++++--------- 6 files changed, 65 insertions(+), 54 deletions(-) diff --git a/src/SMAPI/Framework/Content/ContentCache.cs b/src/SMAPI/Framework/Content/ContentCache.cs index c252b7b67..f33ff84dd 100644 --- a/src/SMAPI/Framework/Content/ContentCache.cs +++ b/src/SMAPI/Framework/Content/ContentCache.cs @@ -119,13 +119,12 @@ public bool Remove(string key, bool dispose) /// Matches the asset keys to invalidate. /// Whether to dispose invalidated assets. This should only be true when they're being invalidated as part of a dispose, to avoid crashing the game. /// Returns the removed keys (if any). - public IEnumerable Remove(Func predicate, bool dispose) + public IEnumerable Remove(Func predicate, bool dispose) { List removed = new List(); foreach (string key in this.Cache.Keys.ToArray()) { - Type type = this.Cache[key].GetType(); - if (predicate(key, type)) + if (predicate(key, this.Cache[key])) { this.Remove(key, dispose); removed.Add(key); diff --git a/src/SMAPI/Framework/ContentCoordinator.cs b/src/SMAPI/Framework/ContentCoordinator.cs index 97b54c5be..82d3805b4 100644 --- a/src/SMAPI/Framework/ContentCoordinator.cs +++ b/src/SMAPI/Framework/ContentCoordinator.cs @@ -7,6 +7,7 @@ using StardewModdingAPI.Framework.Content; using StardewModdingAPI.Framework.ContentManagers; using StardewModdingAPI.Framework.Reflection; +using StardewModdingAPI.Framework.StateTracking.Comparers; using StardewModdingAPI.Metadata; using StardewModdingAPI.Toolkit.Serialization; using StardewModdingAPI.Toolkit.Utilities; @@ -207,24 +208,28 @@ public IEnumerable InvalidateCache(Func predicate, boo /// Returns the invalidated asset names. public IEnumerable InvalidateCache(Func predicate, bool dispose = false) { - // invalidate cache - IDictionary removedAssetNames = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + // invalidate cache & track removed assets + IDictionary> removedAssets = new Dictionary>(StringComparer.InvariantCultureIgnoreCase); foreach (IContentManager contentManager in this.ContentManagers) { - foreach (Tuple asset in contentManager.InvalidateCache(predicate, dispose)) - removedAssetNames[asset.Item1] = asset.Item2; + foreach (var entry in contentManager.InvalidateCache(predicate, dispose)) + { + if (!removedAssets.TryGetValue(entry.Key, out ISet assets)) + removedAssets[entry.Key] = assets = new HashSet(new ObjectReferenceComparer()); + assets.Add(entry.Value); + } } // reload core game assets - int reloaded = this.CoreAssets.Propagate(this.MainContentManager, removedAssetNames); // use an intercepted content manager - - // report result - if (removedAssetNames.Any()) - this.Monitor.Log($"Invalidated {removedAssetNames.Count} asset names: {string.Join(", ", removedAssetNames.Keys.OrderBy(p => p, StringComparer.InvariantCultureIgnoreCase))}. Reloaded {reloaded} core assets.", LogLevel.Trace); + if (removedAssets.Any()) + { + IDictionary propagated = this.CoreAssets.Propagate(this.MainContentManager, removedAssets.ToDictionary(p => p.Key, p => p.Value.First().GetType())); // use an intercepted content manager + this.Monitor.Log($"Invalidated {removedAssets.Count} asset names ({string.Join(", ", removedAssets.Keys.OrderBy(p => p, StringComparer.InvariantCultureIgnoreCase))}); propagated {propagated.Count(p => p.Value)} core assets.", LogLevel.Trace); + } else this.Monitor.Log("Invalidated 0 cache entries.", LogLevel.Trace); - return removedAssetNames.Keys; + return removedAssets.Keys; } /// Dispose held resources. diff --git a/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs b/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs index 4cfeeebab..41ce7c37f 100644 --- a/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs +++ b/src/SMAPI/Framework/ContentManagers/BaseContentManager.cs @@ -184,25 +184,25 @@ public IEnumerable GetAssetKeys() /// Purge matched assets from the cache. /// Matches the asset keys to invalidate. /// Whether to dispose invalidated assets. This should only be true when they're being invalidated as part of a dispose, to avoid crashing the game. - /// Returns the invalidated asset names and types. - public IEnumerable> InvalidateCache(Func predicate, bool dispose = false) + /// Returns the invalidated asset names and instances. + public IDictionary InvalidateCache(Func predicate, bool dispose = false) { - Dictionary removeAssetNames = new Dictionary(StringComparer.InvariantCultureIgnoreCase); - this.Cache.Remove((key, type) => + IDictionary removeAssets = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + this.Cache.Remove((key, asset) => { this.ParseCacheKey(key, out string assetName, out _); - if (removeAssetNames.ContainsKey(assetName)) + if (removeAssets.ContainsKey(assetName)) return true; - if (predicate(assetName, type)) + if (predicate(assetName, asset.GetType())) { - removeAssetNames[assetName] = type; + removeAssets[assetName] = asset; return true; } return false; }, dispose); - return removeAssetNames.Select(p => Tuple.Create(p.Key, p.Value)); + return removeAssets; } /// Dispose held resources. diff --git a/src/SMAPI/Framework/ContentManagers/GameContentManager.cs b/src/SMAPI/Framework/ContentManagers/GameContentManager.cs index 04c4564f1..8930267d3 100644 --- a/src/SMAPI/Framework/ContentManagers/GameContentManager.cs +++ b/src/SMAPI/Framework/ContentManagers/GameContentManager.cs @@ -130,7 +130,7 @@ public override void OnLocaleChanged() removeAssetNames.Contains(key) || (this.TryParseExplicitLanguageAssetKey(key, out string assetName, out _) && removeAssetNames.Contains(assetName)) ) - .Select(p => p.Item1) + .Select(p => p.Key) .OrderBy(p => p, StringComparer.InvariantCultureIgnoreCase) .ToArray(); if (invalidated.Any()) diff --git a/src/SMAPI/Framework/ContentManagers/IContentManager.cs b/src/SMAPI/Framework/ContentManagers/IContentManager.cs index 12c013529..8da9a7773 100644 --- a/src/SMAPI/Framework/ContentManagers/IContentManager.cs +++ b/src/SMAPI/Framework/ContentManagers/IContentManager.cs @@ -66,7 +66,7 @@ internal interface IContentManager : IDisposable /// Purge matched assets from the cache. /// Matches the asset keys to invalidate. /// Whether to dispose invalidated assets. This should only be true when they're being invalidated as part of a dispose, to avoid crashing the game. - /// Returns the invalidated asset names and types. - IEnumerable> InvalidateCache(Func predicate, bool dispose = false); + /// Returns the invalidated asset names and instances. + IDictionary InvalidateCache(Func predicate, bool dispose = false); } } diff --git a/src/SMAPI/Metadata/CoreAssetPropagator.cs b/src/SMAPI/Metadata/CoreAssetPropagator.cs index 8b00d893e..84102828a 100644 --- a/src/SMAPI/Metadata/CoreAssetPropagator.cs +++ b/src/SMAPI/Metadata/CoreAssetPropagator.cs @@ -65,8 +65,8 @@ public CoreAssetPropagator(Func assertAndNormalizeAssetName, Ref /// Reload one of the game's core assets (if applicable). /// The content manager through which to reload the asset. /// The asset keys and types to reload. - /// Returns the number of reloaded assets. - public int Propagate(LocalizedContentManager content, IDictionary assets) + /// Returns a lookup of asset names to whether they've been propagated. + public IDictionary Propagate(LocalizedContentManager content, IDictionary assets) { // group into optimized lists var buckets = assets.GroupBy(p => @@ -81,25 +81,26 @@ public int Propagate(LocalizedContentManager content, IDictionary }); // reload assets - int reloaded = 0; + IDictionary propagated = assets.ToDictionary(p => p.Key, p => false, StringComparer.InvariantCultureIgnoreCase); foreach (var bucket in buckets) { switch (bucket.Key) { case AssetBucket.Sprite: - reloaded += this.ReloadNpcSprites(content, bucket.Select(p => p.Key)); + this.ReloadNpcSprites(content, bucket.Select(p => p.Key), propagated); break; case AssetBucket.Portrait: - reloaded += this.ReloadNpcPortraits(content, bucket.Select(p => p.Key)); + this.ReloadNpcPortraits(content, bucket.Select(p => p.Key), propagated); break; default: - reloaded += bucket.Count(p => this.PropagateOther(content, p.Key, p.Value)); + foreach (var entry in bucket) + propagated[entry.Key] = this.PropagateOther(content, entry.Key, entry.Value); break; } } - return reloaded; + return propagated; } @@ -750,51 +751,57 @@ private bool ReloadNpcDispositions(LocalizedContentManager content, string key) /// Reload the sprites for matching NPCs. /// The content manager through which to reload the asset. /// The asset keys to reload. - /// Returns the number of reloaded assets. - private int ReloadNpcSprites(LocalizedContentManager content, IEnumerable keys) + /// The asset keys which have been propagated. + private void ReloadNpcSprites(LocalizedContentManager content, IEnumerable keys, IDictionary propagated) { // get NPCs HashSet lookup = new HashSet(keys, StringComparer.InvariantCultureIgnoreCase); - NPC[] characters = this.GetCharacters() - .Where(npc => npc.Sprite != null && lookup.Contains(this.NormalizeAssetNameIgnoringEmpty(npc.Sprite?.Texture?.Name))) + var characters = + ( + from npc in this.GetCharacters() + let key = this.NormalizeAssetNameIgnoringEmpty(npc.Sprite?.Texture?.Name) + where key != null && lookup.Contains(key) + select new { Npc = npc, Key = key } + ) .ToArray(); if (!characters.Any()) - return 0; + return; // update sprite - int reloaded = 0; - foreach (NPC npc in characters) + foreach (var target in characters) { - this.SetSpriteTexture(npc.Sprite, content.Load(npc.Sprite.textureName.Value)); - reloaded++; + this.SetSpriteTexture(target.Npc.Sprite, content.Load(target.Key)); + propagated[target.Key] = true; } - - return reloaded; } /// Reload the portraits for matching NPCs. /// The content manager through which to reload the asset. /// The asset key to reload. - /// Returns the number of reloaded assets. - private int ReloadNpcPortraits(LocalizedContentManager content, IEnumerable keys) + /// The asset keys which have been propagated. + private void ReloadNpcPortraits(LocalizedContentManager content, IEnumerable keys, IDictionary propagated) { // get NPCs HashSet lookup = new HashSet(keys, StringComparer.InvariantCultureIgnoreCase); - var villagers = this - .GetCharacters() - .Where(npc => npc.isVillager() && lookup.Contains(this.NormalizeAssetNameIgnoringEmpty(npc.Portrait?.Name))) + var characters = + ( + from npc in this.GetCharacters() + where npc.isVillager() + + let key = this.NormalizeAssetNameIgnoringEmpty(npc.Portrait?.Name) + where key != null && lookup.Contains(key) + select new { Npc = npc, Key = key } + ) .ToArray(); - if (!villagers.Any()) - return 0; + if (!characters.Any()) + return; // update portrait - int reloaded = 0; - foreach (NPC npc in villagers) + foreach (var target in characters) { - npc.Portrait = content.Load(npc.Portrait.Name); - reloaded++; + target.Npc.Portrait = content.Load(target.Key); + propagated[target.Key] = true; } - return reloaded; } /// Reload tree textures. From 5ea5932661316e2504833951188eae4118f460f3 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 14 Dec 2019 22:11:25 -0500 Subject: [PATCH 17/64] add asset propagation for bundles --- docs/release-notes.md | 5 +++-- src/SMAPI/Metadata/CoreAssetPropagator.cs | 25 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 6f06d3d23..9ea7bfce6 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -4,12 +4,13 @@ ## Upcoming release * For players: - * Updated for the 'Force Off' gamepad mode added in Stardew Valley 1.4.0.1. - * Fixed compatibility issue with Arch Linux. + * Updated for the 'Force Off' gamepad mode added in Stardew Valley 1.4.1. + * Fixed compatibility with Arch Linux. * Internal optimizations. * For modders: * Added asset propagation for grass textures. + * Added asset propagation for `Data\Bundles` changes (for added bundles only). * `helper.Read/WriteSaveData` can now be used while a save is being loaded (e.g. within a `Specialized.LoadStageChanged` event). * Fixed private textures loaded from content packs not having their `Name` field set. diff --git a/src/SMAPI/Metadata/CoreAssetPropagator.cs b/src/SMAPI/Metadata/CoreAssetPropagator.cs index 84102828a..970936362 100644 --- a/src/SMAPI/Metadata/CoreAssetPropagator.cs +++ b/src/SMAPI/Metadata/CoreAssetPropagator.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using Microsoft.Xna.Framework.Graphics; +using Netcode; using StardewModdingAPI.Framework.Reflection; using StardewValley; using StardewValley.BellsAndWhistles; @@ -11,6 +12,7 @@ using StardewValley.GameData.Movies; using StardewValley.Locations; using StardewValley.Menus; +using StardewValley.Network; using StardewValley.Objects; using StardewValley.Projectiles; using StardewValley.TerrainFeatures; @@ -227,6 +229,29 @@ private bool PropagateOther(LocalizedContentManager content, string key, Type ty Game1.bigCraftablesInformation = content.Load>(key); return true; + case "data\\bundles": // NetWorldState constructor + { + var bundles = this.Reflection.GetField(Game1.netWorldState.Value, "bundles").GetValue(); + var rewards = this.Reflection.GetField>(Game1.netWorldState.Value, "bundleRewards").GetValue(); + foreach (var pair in content.Load>(key)) + { + int bundleKey = int.Parse(pair.Key.Split('/')[1]); + int rewardsCount = pair.Value.Split('/')[2].Split(' ').Length; + + // add bundles + if (bundles.TryGetValue(bundleKey, out bool[] values)) + bundles.Remove(bundleKey); + else + values = new bool[0]; + bundles[bundleKey] = values.Concat(Enumerable.Repeat(false, rewardsCount - values.Length)).ToArray(); + + // add bundle rewards + if (!rewards.ContainsKey(bundleKey)) + rewards[bundleKey] = false; + } + } + break; + case "data\\clothinginformation": // Game1.LoadContent Game1.clothingInformation = content.Load>(key); return true; From 4aa2c0c3ec429c26da247089cf7fac631db30a0b Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 14 Dec 2019 22:22:10 -0500 Subject: [PATCH 18/64] update release notes (#676, #678) --- docs/release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 9ea7bfce6..0185d1ece 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -5,7 +5,7 @@ * For players: * Updated for the 'Force Off' gamepad mode added in Stardew Valley 1.4.1. - * Fixed compatibility with Arch Linux. + * Fixed compatibility with Linux Mint 18 (thanks to techge!) and Arch Linux. * Internal optimizations. * For modders: From 18a5b07c5ba277e4ea424228a9148e498e0361fa Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 15 Dec 2019 00:04:00 -0500 Subject: [PATCH 19/64] fix overeager asset propagation for bundles --- src/SMAPI/Metadata/CoreAssetPropagator.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/SMAPI/Metadata/CoreAssetPropagator.cs b/src/SMAPI/Metadata/CoreAssetPropagator.cs index 970936362..a684b4730 100644 --- a/src/SMAPI/Metadata/CoreAssetPropagator.cs +++ b/src/SMAPI/Metadata/CoreAssetPropagator.cs @@ -239,11 +239,13 @@ private bool PropagateOther(LocalizedContentManager content, string key, Type ty int rewardsCount = pair.Value.Split('/')[2].Split(' ').Length; // add bundles - if (bundles.TryGetValue(bundleKey, out bool[] values)) + if (!bundles.TryGetValue(bundleKey, out bool[] values) || values.Length < rewardsCount) + { + values ??= new bool[0]; + bundles.Remove(bundleKey); - else - values = new bool[0]; - bundles[bundleKey] = values.Concat(Enumerable.Repeat(false, rewardsCount - values.Length)).ToArray(); + bundles[bundleKey] = values.Concat(Enumerable.Repeat(false, rewardsCount - values.Length)).ToArray(); + } // add bundle rewards if (!rewards.ContainsKey(bundleKey)) From d662ea858c4914eefe5a0b0f911d1f41086b0424 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 15 Dec 2019 00:33:08 -0500 Subject: [PATCH 20/64] improve error message for TargetParameterCountException in the reflection API --- docs/release-notes.md | 1 + src/SMAPI/Framework/Reflection/ReflectedMethod.cs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/release-notes.md b/docs/release-notes.md index 0185d1ece..dc38710af 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -11,6 +11,7 @@ * For modders: * Added asset propagation for grass textures. * Added asset propagation for `Data\Bundles` changes (for added bundles only). + * Improved error messages for `TargetParameterCountException` when using the reflection API. * `helper.Read/WriteSaveData` can now be used while a save is being loaded (e.g. within a `Specialized.LoadStageChanged` event). * Fixed private textures loaded from content packs not having their `Name` field set. diff --git a/src/SMAPI/Framework/Reflection/ReflectedMethod.cs b/src/SMAPI/Framework/Reflection/ReflectedMethod.cs index 039f27c38..82737a7f9 100644 --- a/src/SMAPI/Framework/Reflection/ReflectedMethod.cs +++ b/src/SMAPI/Framework/Reflection/ReflectedMethod.cs @@ -65,6 +65,10 @@ public TValue Invoke(params object[] arguments) { result = this.MethodInfo.Invoke(this.Parent, arguments); } + catch (TargetParameterCountException) + { + throw new Exception($"Couldn't invoke the {this.DisplayName} method: it expects {this.MethodInfo.GetParameters().Length} parameters, but {arguments.Length} were provided."); + } catch (Exception ex) { throw new Exception($"Couldn't invoke the {this.DisplayName} method", ex); From 6275821288aec6a5178f660eda951e6343f5e381 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 15 Dec 2019 01:08:35 -0500 Subject: [PATCH 21/64] add friendly log message for save file-not-found errors --- docs/release-notes.md | 1 + src/SMAPI/Framework/SCore.cs | 59 ++++++++++++++++++++++++++++++------ 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index dc38710af..217b0f342 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -4,6 +4,7 @@ ## Upcoming release * For players: + * Added friendly log message for save file-not-found errors. * Updated for the 'Force Off' gamepad mode added in Stardew Valley 1.4.1. * Fixed compatibility with Linux Mint 18 (thanks to techge!) and Arch Linux. * Internal optimizations. diff --git a/src/SMAPI/Framework/SCore.cs b/src/SMAPI/Framework/SCore.cs index fb3506b4b..2c6c0e769 100644 --- a/src/SMAPI/Framework/SCore.cs +++ b/src/SMAPI/Framework/SCore.cs @@ -97,16 +97,25 @@ internal class SCore : IDisposable }; /// Regex patterns which match console messages to show a more friendly error for. - private readonly Tuple[] ReplaceConsolePatterns = + private readonly ReplaceLogPattern[] ReplaceConsolePatterns = { - Tuple.Create( - new Regex(@"^System\.InvalidOperationException: Steamworks is not initialized\.", RegexOptions.Compiled | RegexOptions.CultureInvariant), + // Steam not loaded + new ReplaceLogPattern( + search: new Regex(@"^System\.InvalidOperationException: Steamworks is not initialized\.[\s\S]+$", RegexOptions.Compiled | RegexOptions.CultureInvariant), + replacement: #if SMAPI_FOR_WINDOWS - "Oops! Steam achievements won't work because Steam isn't loaded. You can launch the game through Steam to fix that (see 'Part 2: Configure Steam' in the install guide for more info: https://smapi.io/install).", + "Oops! Steam achievements won't work because Steam isn't loaded. You can launch the game through Steam to fix that (see 'Part 2: Configure Steam' in the install guide for more info: https://smapi.io/install).", #else - "Oops! Steam achievements won't work because Steam isn't loaded. You can launch the game through Steam to fix that.", + "Oops! Steam achievements won't work because Steam isn't loaded. You can launch the game through Steam to fix that.", #endif - LogLevel.Error + logLevel: LogLevel.Error + ), + + // save file not found error + new ReplaceLogPattern( + search: new Regex(@"^System\.IO\.FileNotFoundException: [^\n]+\n[^:]+: '[^\n]+[/\\]Saves[/\\]([^'\r\n]+)[/\\]([^'\r\n]+)'[\s\S]+LoadGameMenu\.FindSaveGames[\s\S]+$", RegexOptions.Compiled | RegexOptions.CultureInvariant), + replacement: "The game can't find the '$2' file for your '$1' save. See https://stardewvalleywiki.com/Saves#Troubleshooting for help.", + logLevel: LogLevel.Error ) }; @@ -1294,11 +1303,12 @@ private void HandleConsoleMessage(IMonitor gameMonitor, string message) return; // show friendly error if applicable - foreach (var entry in this.ReplaceConsolePatterns) + foreach (ReplaceLogPattern entry in this.ReplaceConsolePatterns) { - if (entry.Item1.IsMatch(message)) + string newMessage = entry.Search.Replace(message, entry.Replacement); + if (message != newMessage) { - this.Monitor.Log(entry.Item2, entry.Item3); + gameMonitor.Log(newMessage, entry.LogLevel); gameMonitor.Log(message, LogLevel.Trace); return; } @@ -1388,5 +1398,36 @@ private void PurgeNormalLogs() } } } + + /// A console log pattern to replace with a different message. + private class ReplaceLogPattern + { + /********* + ** Accessors + *********/ + /// The regex pattern matching the portion of the message to replace. + public Regex Search { get; } + + /// The replacement string. + public string Replacement { get; } + + /// The log level for the new message. + public LogLevel LogLevel { get; } + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The regex pattern matching the portion of the message to replace. + /// The replacement string. + /// The log level for the new message. + public ReplaceLogPattern(Regex search, string replacement, LogLevel logLevel) + { + this.Search = search; + this.Replacement = replacement; + this.LogLevel = logLevel; + } + } } } From 4711d19b3e3fa71c304100209450c530a0e5c51a Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 15 Dec 2019 10:50:05 -0500 Subject: [PATCH 22/64] fix .gitignore and line endings for Linux --- .gitignore | 3 +++ src/SMAPI/i18n/zh.json | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 656952112..6f7a0096b 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user +# Rider +.idea/ + # NuGet packages *.nupkg **/packages/* diff --git a/src/SMAPI/i18n/zh.json b/src/SMAPI/i18n/zh.json index bbd6a5745..9c0e0c21c 100644 --- a/src/SMAPI/i18n/zh.json +++ b/src/SMAPI/i18n/zh.json @@ -1,3 +1,3 @@ -{ - "warn.invalid-content-removed": "非法内容已移除以防游戏闪退（查看SMAPI控制台获得更多信息）" -} +{ + "warn.invalid-content-removed": "非法内容已移除以防游戏闪退（查看SMAPI控制台获得更多信息）" +} From 9018750eb326c666b5424749ddd51b9a18470265 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 15 Dec 2019 11:27:46 -0500 Subject: [PATCH 23/64] fix Linux systems with libhybris-utils installed incorrectly detected as Android (#668) --- docs/release-notes.md | 1 + .../Utilities/EnvironmentUtility.cs | 32 +++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 217b0f342..8e1b9f1c7 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -7,6 +7,7 @@ * Added friendly log message for save file-not-found errors. * Updated for the 'Force Off' gamepad mode added in Stardew Valley 1.4.1. * Fixed compatibility with Linux Mint 18 (thanks to techge!) and Arch Linux. + * Fixed compatibility with Linux systems which have libhybris-utils installed. * Internal optimizations. * For modders: diff --git a/src/SMAPI.Toolkit/Utilities/EnvironmentUtility.cs b/src/SMAPI.Toolkit/Utilities/EnvironmentUtility.cs index 6dce5da53..2a01fe4b4 100644 --- a/src/SMAPI.Toolkit/Utilities/EnvironmentUtility.cs +++ b/src/SMAPI.Toolkit/Utilities/EnvironmentUtility.cs @@ -105,23 +105,27 @@ private static Platform DetectPlatformImpl() /// private static bool IsRunningAndroid() { - using (Process process = new Process()) + using Process process = new Process { - process.StartInfo.FileName = "getprop"; - process.StartInfo.Arguments = "ro.build.user"; - process.StartInfo.RedirectStandardOutput = true; - process.StartInfo.UseShellExecute = false; - process.StartInfo.CreateNoWindow = true; - try + StartInfo = { - process.Start(); - string output = process.StandardOutput.ReadToEnd(); - return !string.IsNullOrEmpty(output); - } - catch - { - return false; + FileName = "getprop", + Arguments = "ro.build.user", + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true } + }; + + try + { + process.Start(); + string output = process.StandardOutput.ReadToEnd(); + return !string.IsNullOrWhiteSpace(output); + } + catch + { + return false; } } From c7426a191afe1a0b61e109d7bdcd5e1f6a5c98df Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 15 Dec 2019 21:47:42 -0500 Subject: [PATCH 24/64] add Spanish translations Thanks to PlussRolf! --- docs/README.md | 2 +- docs/release-notes.md | 1 + src/SMAPI/i18n/es.json | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 src/SMAPI/i18n/es.json diff --git a/docs/README.md b/docs/README.md index 386259a94..49cf28e49 100644 --- a/docs/README.md +++ b/docs/README.md @@ -72,5 +72,5 @@ Japanese | ❑ not translated Korean | ❑ not translated Portuguese | ❑ not translated Russian | ✓ [fully translated](../src/SMAPI/i18n/ru.json) -Spanish | ❑ not translated +Spanish | ✓ [fully translated](../src/SMAPI/i18n/es.json) Turkish | ✓ [fully translated](../src/SMAPI/i18n/tr.json) diff --git a/docs/release-notes.md b/docs/release-notes.md index 8e1b9f1c7..bd377d0b5 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -9,6 +9,7 @@ * Fixed compatibility with Linux Mint 18 (thanks to techge!) and Arch Linux. * Fixed compatibility with Linux systems which have libhybris-utils installed. * Internal optimizations. + * Updated translations. Thanks to PlussRolf (added Spanish)! * For modders: * Added asset propagation for grass textures. diff --git a/src/SMAPI/i18n/es.json b/src/SMAPI/i18n/es.json new file mode 100644 index 000000000..f5a74dfeb --- /dev/null +++ b/src/SMAPI/i18n/es.json @@ -0,0 +1,3 @@ +{ + "warn.invalid-content-removed": "Se ha quitado contenido inválido para evitar un cierre forzoso (revisa la consola de SMAPI para más información)." +} From f692af269c46b05b06e717c98121ade34a374cb9 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Mon, 16 Dec 2019 20:37:41 -0500 Subject: [PATCH 25/64] ignore deployment slot profiles --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6f7a0096b..5450a2f54 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,4 @@ appsettings.Development.json src/SMAPI.Web.LegacyRedirects/aws-beanstalk-tools-defaults.json # Azure generated files -src/SMAPI.Web/Properties/PublishProfiles/smapi-web-release - Web Deploy.pubxml +src/SMAPI.Web/Properties/PublishProfiles/*.pubxml From c4e2e94eed30f6e04312d5973322e4696ea672ea Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Mon, 16 Dec 2019 21:39:37 -0500 Subject: [PATCH 26/64] add option to edit & reupload in the JSON validator --- docs/release-notes.md | 11 ++-- .../Controllers/JsonValidatorController.cs | 6 ++- .../Views/JsonValidator/Index.cshtml | 50 ++++++++++--------- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index bd377d0b5..75438aaa7 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -11,6 +11,12 @@ * Internal optimizations. * Updated translations. Thanks to PlussRolf (added Spanish)! +* For the web UI: + * Added option to edit & reupload in the JSON validator. + * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. + * Updated the JSON validator for Content Patcher 1.10.0. + * Fixed JSON validator no longer letting you change format when viewing results. + * For modders: * Added asset propagation for grass textures. * Added asset propagation for `Data\Bundles` changes (for added bundles only). @@ -18,11 +24,6 @@ * `helper.Read/WriteSaveData` can now be used while a save is being loaded (e.g. within a `Specialized.LoadStageChanged` event). * Fixed private textures loaded from content packs not having their `Name` field set. -* For the web UI: - * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. - * Updated the JSON validator for Content Patcher 1.10.0. - * Fixed JSON validator no longer letting you change format when viewing results. - ## 3.0.1 Released 02 December 2019 for Stardew Valley 1.4.0.1. diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs index 830fe8394..e4eff0f4e 100644 --- a/src/SMAPI.Web/Controllers/JsonValidatorController.cs +++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs @@ -55,7 +55,7 @@ public JsonValidatorController(IStorageProvider storage) ** Web UI ***/ /// Render the schema validator UI. - /// The schema name with which to validate the JSON. + /// The schema name with which to validate the JSON, or 'edit' to return to the edit screen. /// The stored file ID. [HttpGet] [Route("json")] @@ -75,6 +75,10 @@ public async Task Index(string schemaName = null, string id = null) return this.View("Index", result.SetUploadError("The JSON file seems to be empty.")); result.SetContent(file.Content, expiry: file.Expiry, uploadWarning: file.Warning); + // skip parsing if we're going to the edit screen + if (schemaName?.ToLower() == "edit") + return this.View("Index", result); + // parse JSON JToken parsed; try diff --git a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml index a042f0240..fb43823ab 100644 --- a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml +++ b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml @@ -8,7 +8,8 @@ string curPageUrl = this.Url.PlainAction("Index", "JsonValidator", new { schemaName = Model.SchemaName, id = Model.PasteID }); string newUploadUrl = this.Url.PlainAction("Index", "JsonValidator", new { schemaName = Model.SchemaName }); string schemaDisplayName = null; - bool isValidSchema = Model.SchemaName != null && Model.SchemaFormats.TryGetValue(Model.SchemaName, out schemaDisplayName) && schemaDisplayName != "None"; + bool isValidSchema = Model.SchemaName != null && Model.SchemaFormats.TryGetValue(Model.SchemaName, out schemaDisplayName) && schemaDisplayName?.ToLower() != "none"; + bool isEditView = Model.Content == null || Model.SchemaName?.ToLower() == "edit"; // build title ViewData["Title"] = "JSON validator"; @@ -60,7 +61,7 @@ else if (Model.ParseError != null) Error details: @Model.ParseError } -else if (Model.PasteID != null) +else if (!isEditView && Model.PasteID != null) {