Compress save file (#986)

* Adding gzip compression for improving storage use and backward compatibility. (#984)

* Basic gzip an gunzip on load and save.

* refactor file save type to .gz and update the data in ui.

---------

Co-authored-by: Azgaar <maxganiev@yandex.com>

* refactor: cleanup, change wording

* feat: streamline saving options

* fix: fixes

---------

Co-authored-by: Efruz Yıldırır <30903352+yldrefruz@users.noreply.github.com>
This commit is contained in:
Azgaar 2023-08-15 16:50:28 +04:00 committed by GitHub
parent 5fba7d60f4
commit 26f48a48fd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 197 additions and 104 deletions

View file

@ -1775,6 +1775,18 @@
</td>
</tr>
<tr data-tip="Set what Generator should do on load">
<td></td>
<td>Onload behavior</td>
<td>
<select id="onloadBehavior" data-stored="onloadBehavior">
<option value="random" selected>Generate random map</option>
<option value="lastSaved">Open last saved map</option>
</select>
</td>
<td></td>
</tr>
<tr data-tip="Select speech synthesis voice to pronounce generated names">
<td></td>
<td>Speaker voice</td>
@ -2329,8 +2341,8 @@
<div id="sticked">
<button id="newMapButton" data-tip="Generate a new map based on options" data-shortcut="F2">New Map</button>
<button id="exportButton" data-tip="Select format to download image or export map data">Export</button>
<button id="saveButton" data-tip="Save fully-functional map in .map format">Save</button>
<button id="loadButton" data-tip="Load fully-functional map in .map format">Load</button>
<button id="saveButton" data-tip="Save fully-functional map in .gz format">Save</button>
<button id="loadButton" data-tip="Load fully-functional map (.gz or .map formats)">Load</button>
<button id="zoomReset" data-tip="Reset map zoom" data-shortcut="0 (zero)">Reset Zoom</button>
</div>
</div>
@ -5884,37 +5896,42 @@
<div id="saveMapData" style="display: none" class="dialog">
<div style="margin-top: 0.3em">
<strong>Save map to</strong>
<button onclick="dowloadMap()" data-tip="Download .map file to your local disk" data-shortcut="Ctrl + S">
<button onclick="saveMap('machine')" data-tip="Download map file to your local disk" data-shortcut="Ctrl + S">
machine
</button>
<button onclick="saveToDropbox()" data-tip="Save .map file to your Dropbox" data-shortcut="Ctrl + C">
<button onclick="saveMap('dropbox')" data-tip="Save map file to your Dropbox" data-shortcut="Ctrl + C">
dropbox
</button>
<button
onclick="quickSave()"
data-tip="Save the project to browser storage. It will overwrite the latest autosave"
data-shortcut="F6"
>
<button onclick="saveMap('storage')" data-tip="Save the project to browser storage only" data-shortcut="F6">
browser
</button>
</div>
<p>
Maps are saved in <i>.map</i> format, that can be loaded back via the <i>Load</i> in menu. There is no way to
restore the progress if file is lost. Please keep old <i>.map</i> files on your machine or cloud storage as
backups.
Maps are saved in <i>.gz</i> format, that can be loaded back via the <i>Load</i> in menu. There is no way to
restore the progress if file is lost. Please keep old save files on your machine or cloud storage as backups.
</p>
</div>
<div id="loadMapData" style="display: none" class="dialog">
<div>
<strong>Load map from</strong>
<button onclick="mapToLoad.click()" data-tip="Load .map file from your local disk">machine</button>
<button onclick="loadURL()" data-tip="Load .map file from URL (server should allow CORS)">URL</button>
<button onclick="mapToLoad.click()" data-tip="Load map file (.gz or .map) from your local disk">
machine
</button>
<button
onclick="loadURL()"
data-tip="Load map file (.gz or .map) file from URL. Note that the server should allow CORS"
>
URL
</button>
<button onclick="quickLoad()" data-tip="Load map from browser storage (if saved before)">storage</button>
</div>
<p>Click on <i>storage</i> to open the last saved map.</p>
<div id="loadFromDropbox">
<p style="margin-bottom: 0.3em">
From your Dropbox account
Or load from your Dropbox account
<button
id="dropboxConnectButton"
onclick="connectToDropbox()"
@ -5926,7 +5943,7 @@
<select id="loadFromDropboxSelect" style="width: 22em"></select>
<div id="loadFromDropboxButtons" style="margin-bottom: 0.6em">
<button onclick="loadFromDropbox()" data-tip="Load .map file from your Dropbox">Load</button>
<button onclick="loadFromDropbox()" data-tip="Load map file (.gz or .map) from your Dropbox">Load</button>
<button
onclick="createSharableDropboxLink()"
data-tip="Select file and create a link to share with your friends"
@ -5995,7 +6012,7 @@
<div id="resampleDialog" style="display: none" class="dialog">
<div style="width: 34em; max-width: 80vw; font-weight: bold; padding: 6px">
This operation is destructive and irreversible. It will create a completely new map based on the current one.
Don't forget to save the current project as a .map file first!
Don't forget to save the current project to your machine first!
</div>
<div
@ -6116,10 +6133,10 @@
data-main="Сlick the arrow button for options. Zoom in to see the map in details"
></div>
<div id="mapOverlay" style="display: none">Drop a .map file to open</div>
<div id="mapOverlay" style="display: none">Drop a map file to open</div>
<div id="fileInputs" style="display: none">
<input type="file" accept=".map" id="mapToLoad" />
<input type="file" accept=".map,.gz" id="mapToLoad" />
<input type="file" accept=".txt,.csv" id="burgsListToLoad" />
<input type="file" accept=".txt" id="legendsToLoad" />
<input type="file" accept="image/*" id="imageToLoad" />
@ -7930,7 +7947,7 @@
<script src="utils/graphUtils.js?v=1.90.01"></script>
<script src="utils/nodeUtils.js"></script>
<script src="utils/numberUtils.js?v=1.89.08"></script>
<script src="utils/polyfills.js"></script>
<script src="utils/polyfills.js?v=1.93.00"></script>
<script src="utils/probabilityUtils.js?v=1.88.00"></script>
<script src="utils/stringUtils.js"></script>
<script src="utils/languageUtils.js"></script>
@ -7963,15 +7980,15 @@
<script src="modules/ui/stylePresets.js?v=1.89.11"></script>
<script src="modules/ui/general.js?v=1.87.03"></script>
<script src="modules/ui/options.js?v=1.91.00"></script>
<script src="main.js?v=1.92.00"></script>
<script src="modules/ui/options.js?v=1.93.00"></script>
<script src="main.js?v=1.93.00"></script>
<script defer src="modules/relief-icons.js"></script>
<script defer src="modules/ui/style.js"></script>
<script defer src="modules/ui/editors.js?v=1.92.00"></script>
<script defer src="modules/ui/tools.js?v=1.92.00"></script>
<script defer src="modules/ui/world-configurator.js?v=1.91.05"></script>
<script defer src="modules/ui/heightmap-editor.js?v=1.92.00"></script>
<script defer src="modules/ui/heightmap-editor.js?v=1.93.00"></script>
<script defer src="modules/ui/provinces-editor.js?v=1.92.00"></script>
<script defer src="modules/ui/biomes-editor.js?v=1.91.05"></script>
<script defer src="modules/ui/namesbase-editor.js?v=1.89.26"></script>
@ -8001,13 +8018,12 @@
<script defer src="modules/ui/markers-editor.js"></script>
<script defer src="modules/ui/3d.js?v=1.89.36"></script>
<script defer src="modules/ui/submap.js?v=1.92.00"></script>
<script defer src="modules/ui/hotkeys.js?v=1.88.00"></script>
<script defer src="modules/ui/hotkeys.js?v=1.93.00"></script>
<script defer src="modules/coa-renderer.js?v=1.91.00"></script>
<script defer src="libs/rgbquant.min.js"></script>
<script defer src="libs/jquery.ui.touch-punch.min.js"></script>
<script defer src="modules/io/save.js?v=1.91.04"></script>
<script defer src="modules/io/load.js?v=1.92.05"></script>
<script defer src="modules/io/save.js?v=1.93.00"></script>
<script defer src="modules/io/load.js?v=1.93.00"></script>
<script defer src="modules/io/cloud.js"></script>
<script defer src="modules/io/export.js?v=1.89.36"></script>
<script defer src="modules/io/formats.js"></script>

30
main.js
View file

@ -270,7 +270,7 @@ async function checkLoadParameters() {
const url = new URL(window.location.href);
const params = url.searchParams;
// of there is a valid maplink, try to load .map file from URL
// of there is a valid maplink, try to load .gz/.map file from URL
if (params.get("maplink")) {
WARN && console.warn("Load map from URL");
const maplink = params.get("maplink");
@ -292,17 +292,20 @@ async function checkLoadParameters() {
}
// check if there is a map saved to indexedDB
try {
const blob = await ldb.get("lastMap");
if (blob) {
WARN && console.warn("Loading last stored map");
uploadMap(blob);
return;
if (byId("onloadBehavior").value === "lastSaved") {
try {
const blob = await ldb.get("lastMap");
if (blob) {
WARN && console.warn("Loading last stored map");
uploadMap(blob);
return;
}
} catch (error) {
ERROR && console.error(error);
}
} catch (error) {
console.error(error);
}
// else generate random map
WARN && console.warn("Generate random map");
generateMapOnLoad();
}
@ -574,9 +577,10 @@ void (function addDragToUpload() {
overlay.style.display = "none";
if (e.dataTransfer.items == null || e.dataTransfer.items.length !== 1) return; // no files or more than one
const file = e.dataTransfer.items[0].getAsFile();
if (file.name.indexOf(".map") == -1) {
// not a .map file
alertMessage.innerHTML = "Please upload a <b>.map</b> file you have previously downloaded";
if (!file.name.endsWith(".map") && !file.name.endsWith(".gz")) {
alertMessage.innerHTML =
"Please upload a map file (<i>.gz</i> or <i>.map</i> formats) you have previously downloaded";
$("#alert").dialog({
resizable: false,
title: "Invalid file format",
@ -596,7 +600,7 @@ void (function addDragToUpload() {
if (closeDialogs) closeDialogs();
uploadMap(file, () => {
overlay.style.display = "none";
overlay.innerHTML = "Drop a .map file to open";
overlay.innerHTML = "Drop a map file to open";
});
});
})();

View file

@ -1,6 +1,6 @@
"use strict";
// update old .map version to the current one
// update old map file to the current version
export function resolveVersionConflicts(version) {
if (version < 1) {
// v1.0 added a new religions layer

View file

@ -1,6 +1,5 @@
"use strict";
// Functions to load and parse .map files
// Functions to load and parse .gz/.map files
async function quickLoad() {
const blob = await ldb.get("lastMap");
if (blob) loadMapPrompt(blob);
@ -112,11 +111,11 @@ function uploadMap(file, callback) {
const currentVersion = parseFloat(version);
const fileReader = new FileReader();
fileReader.onload = function (fileLoadedEvent) {
fileReader.onloadend = async function (fileLoadedEvent) {
if (callback) callback();
document.getElementById("coas").innerHTML = ""; // remove auto-generated emblems
const result = fileLoadedEvent.target.result;
const [mapData, mapVersion] = parseLoadedResult(result);
const [mapData, mapVersion] = await parseLoadedResult(result);
const isInvalid = !mapData || isNaN(mapVersion) || mapData.length < 26 || !mapData[5];
const isUpdated = mapVersion === currentVersion;
@ -131,18 +130,40 @@ function uploadMap(file, callback) {
if (isOutdated) return showUploadMessage("outdated", mapData, mapVersion);
};
fileReader.readAsText(file, "UTF-8");
fileReader.readAsArrayBuffer(file);
}
function parseLoadedResult(result) {
async function uncompress(compressedData) {
try {
const uncompressedStream = new Blob([compressedData]).stream().pipeThrough(new DecompressionStream("gzip"));
let uncompressedData = [];
for await (const chunk of uncompressedStream) {
uncompressedData = uncompressedData.concat(Array.from(chunk));
}
return new Uint8Array(uncompressedData);
} catch (error) {
ERROR && console.error(error);
return null;
}
}
async function parseLoadedResult(result) {
try {
const resultAsString = new TextDecoder().decode(result);
// data can be in FMG internal format or base64 encoded
const isDelimited = result.substr(0, 10).includes("|");
const decoded = isDelimited ? result : decodeURIComponent(atob(result));
const isDelimited = resultAsString.substring(0, 10).includes("|");
const decoded = isDelimited ? resultAsString : decodeURIComponent(atob(resultAsString));
const mapData = decoded.split("\r\n");
const mapVersion = parseFloat(mapData[0].split("|")[0] || mapData[0]);
return [mapData, mapVersion];
} catch (error) {
// map file can be compressed with gzip
const uncompressedData = await uncompress(result);
if (uncompressedData) return parseLoadedResult(uncompressedData);
ERROR && console.error(error);
return [null, null];
}
@ -153,7 +174,7 @@ function showUploadMessage(type, mapData, mapVersion) {
let message, title, canBeLoaded;
if (type === "invalid") {
message = `The file does not look like a valid <i>.map</i> file.<br>Please check the data format`;
message = `The file does not look like a valid save file.<br>Please check the data format`;
title = "Invalid file";
canBeLoaded = false;
} else if (type === "ancient") {
@ -165,7 +186,7 @@ function showUploadMessage(type, mapData, mapVersion) {
title = "Newer file";
canBeLoaded = false;
} else if (type === "outdated") {
message = `The map version (${mapVersion}) does not match the Generator version (${version}).<br>Click OK to get map <b>auto-updated</b>.<br>In case of issues please keep using an ${archive} of the Generator`;
message = `The map version (${mapVersion}) does not match the Generator version (${version}).<br>That is fine, click OK to the get map <b style="color: #005000">auto-updated</b>.<br>In case of issues please keep using an ${archive} of the Generator`;
title = "Outdated file";
canBeLoaded = true;
}
@ -435,7 +456,7 @@ async function parseLoadedData(data) {
{
// dynamically import and run auto-udpdate script
const versionNumber = parseFloat(params[0]);
const {resolveVersionConflicts} = await import("../dynamic/auto-update.js?v=1.92.05");
const {resolveVersionConflicts} = await import("../dynamic/auto-update.js?v=1.93.00");
resolveVersionConflicts(versionNumber);
}

View file

@ -1,8 +1,43 @@
"use strict";
// functions to save project as .map file
// prepare map data for saving
function getMapData() {
// functions to save the project to a file
async function saveMap(method) {
if (customization) return tip("Map cannot be saved in EDIT mode, please complete the edit and retry", false, "error");
closeDialogs("#alert");
try {
const compressedMapData = await compressData(prepareMapData());
const filename = getFileName() + ".gz";
saveToStorage(compressedMapData, method === "storage"); // any method saves to indexedDB
if (method === "machine") saveToMachine(compressedMapData, filename);
if (method === "dropbox") saveToDropbox(compressedMapData, filename);
} catch (error) {
ERROR && console.error(error);
alertMessage.innerHTML = /* html */ `An error is occured on map saving. If the issue persists, please copy the message below and report it on ${link(
"https://github.com/Azgaar/Fantasy-Map-Generator/issues",
"GitHub"
)}. <p id="errorBox">${parseError(error)}</p>`;
$("#alert").dialog({
resizable: false,
title: "Saving error",
width: "28em",
buttons: {
Retry: function () {
$(this).dialog("close");
saveMap(method);
},
Close: function () {
$(this).dialog("close");
}
},
position: {my: "center", at: "center", of: "svg"}
});
}
}
function prepareMapData() {
const date = new Date();
const dateString = date.getFullYear() + "-" + (date.getMonth() + 1) + "-" + date.getDate();
const license = "File can be loaded in azgaar.github.io/Fantasy-Map-Generator";
@ -117,36 +152,30 @@ function getMapData() {
return mapData;
}
// Download .map file
function dowloadMap() {
if (customization)
return tip("Map cannot be saved when edit mode is active, please exit the mode and retry", false, "error");
closeDialogs("#alert");
// save map file to indexedDB
async function saveToStorage(compressedMapData, showTip = false) {
const blob = new Blob([compressedMapData], {type: "text/plain"});
await ldb.set("lastMap", blob);
showTip && tip("Map is saved to the browser storage", false, "success");
}
const mapData = getMapData();
const blob = new Blob([mapData], {type: "text/plain"});
// download .gz file
function saveToMachine(compressedMapData, filename) {
const blob = new Blob([compressedMapData], {type: "text/plain"});
const URL = window.URL.createObjectURL(blob);
const link = document.createElement("a");
link.download = getFileName() + ".map";
link.download = filename;
link.href = URL;
link.click();
tip(`${link.download} is saved. Open "Downloads" screen (CTRL + J) to check`, true, "success", 7000);
tip('Map is saved to the "Downloads" folder (CTRL + J to open)', true, "success", 8000);
window.URL.revokeObjectURL(URL);
}
async function saveToDropbox() {
if (customization)
return tip("Map cannot be saved when edit mode is active, please exit the mode and retry", false, "error");
closeDialogs("#alert");
const mapData = getMapData();
const filename = getFileName() + ".map";
try {
await Cloud.providers.dropbox.save(filename, mapData);
tip("Map is saved to your Dropbox", true, "success", 8000);
} catch (msg) {
ERROR && console.error(msg);
tip("Cannot save .map to your Dropbox", true, "error", 8000);
}
async function saveToDropbox(compressedMapData, filename) {
await Cloud.providers.dropbox.save(filename, compressedMapData);
tip("Map is saved to your Dropbox", true, "success", 8000);
}
async function initiateAutosave() {
@ -161,38 +190,43 @@ async function initiateAutosave() {
if (diffInMinutes < timeoutMinutes) return;
if (customization) return tip("Autosave: map cannot be saved in edit mode", false, "warning", 2000);
tip("Autosave: saving map...", false, "warning", 3000);
const mapData = getMapData();
const blob = new Blob([mapData], {type: "text/plain"});
await ldb.set("lastMap", blob);
INFO && console.log("Autosaved at", new Date().toLocaleTimeString());
lastSavedAt = Date.now();
try {
tip("Autosave: saving map...", false, "warning", 3000);
const compressedMapData = await compressData(prepareMapData());
await saveToStorage(compressedMapData);
tip("Autosave: map is saved", false, "success", 2000);
lastSavedAt = Date.now();
} catch (error) {
ERROR && console.error(error);
}
}
setInterval(autosave, MINUTE / 2);
}
async function quickSave() {
if (customization)
return tip("Map cannot be saved when edit mode is active, please exit the mode first", false, "error");
async function compressData(uncompressedData) {
const compressedStream = new Blob([uncompressedData]).stream().pipeThrough(new CompressionStream("gzip"));
const mapData = getMapData();
const blob = new Blob([mapData], {type: "text/plain"});
await ldb.set("lastMap", blob); // auto-save map
tip("Map is saved to browser memory. Please also save as .map file to secure progress", true, "success", 2000);
let compressedData = [];
for await (const chunk of compressedStream) {
compressedData = compressedData.concat(Array.from(chunk));
}
return new Uint8Array(compressedData);
}
const saveReminder = function () {
if (localStorage.getItem("noReminder")) return;
const message = [
"Please don't forget to save your work as a .map file",
"Please remember to save work as a .map file",
"Saving in .map format will ensure your data won't be lost in case of issues",
"Please don't forget to save the project to desktop from time to time",
"Please remember to save the map to your desktop",
"Saving will ensure your data won't be lost in case of issues",
"Safety is number one priority. Please save the map",
"Don't forget to save your map on a regular basis!",
"Just a gentle reminder for you to save the map",
"Please don't forget to save your progress (saving as .map is the best option)",
"Don't want to be reminded about need to save? Press CTRL+Q"
"Please don't forget to save your progress (saving to desktop is the best option)",
"Don't want to get reminded about need to save? Press CTRL+Q"
];
const interval = 15 * 60 * 1000; // remind every 15 minutes

View file

@ -28,7 +28,7 @@ function editHeightmap(options) {
<p><i>Erase</i> mode also allows you Convert an Image into a heightmap or use Template Editor.</p>
<p>You can <i>keep</i> the data, but you won't be able to change the coastline.</p>
<p>Try <i>risk</i> mode to change the coastline and keep the data. The data will be restored as much as possible, but it can cause unpredictable errors.</p>
<p>Please <span class="pseudoLink" onclick="dowloadMap();">save the map</span> before editing the heightmap!</p>
<p>Please <span class="pseudoLink" onclick="saveMap('machine')">save the map</span> before editing the heightmap!</p>
<p style="margin-bottom: 0">Check out ${link(
"https://github.com/Azgaar/Fantasy-Map-Generator/wiki/Heightmap-customization",
"wiki"

View file

@ -25,15 +25,15 @@ function handleKeyup(event) {
if (code === "F1") showInfo();
else if (code === "F2") regeneratePrompt();
else if (code === "F6") quickSave();
else if (code === "F6") saveMap("storage");
else if (code === "F9") quickLoad();
else if (code === "Tab") toggleOptions(event);
else if (code === "Escape") closeAllDialogs();
else if (code === "Delete") removeElementOnKey();
else if (code === "KeyO" && document.getElementById("canvas3d")) toggle3dOptions();
else if (ctrl && code === "KeyQ") toggleSaveReminder();
else if (ctrl && code === "KeyS") dowloadMap();
else if (ctrl && code === "KeyC") saveToDropbox();
else if (ctrl && code === "KeyS") saveMap("machine");
else if (ctrl && code === "KeyC") saveMap("dropbox");
else if (ctrl && code === "KeyZ" && undo?.offsetParent) undo.click();
else if (ctrl && code === "KeyY" && redo?.offsetParent) redo.click();
else if (shift && code === "KeyH") editHeightmap();

View file

@ -793,7 +793,7 @@ async function showLoadPane() {
$("#loadMapData").dialog({
title: "Load map",
resizable: false,
width: "24em",
width: "auto",
position: {my: "center", at: "center", of: "svg"},
buttons: {
Close: function () {
@ -844,8 +844,8 @@ async function connectToDropbox() {
function loadURL() {
const pattern = /(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/;
const inner = `Provide URL to a .map file:
<input id="mapURL" type="url" style="width: 24em" placeholder="https://e-cloud.com/test.map">
const inner = `Provide URL to map file:
<input id="mapURL" type="url" style="width: 24em" placeholder="https://e-cloud.com/test.gz">
<br><i>Please note server should allow CORS for file to be loaded. If CORS is not allowed, save file to Dropbox and provide a direct link</i>`;
alertMessage.innerHTML = inner;
$("#alert").dialog({

View file

@ -14,3 +14,19 @@ if (Array.prototype.flat === undefined) {
return this.reduce((acc, val) => (Array.isArray(val) ? acc.concat(val.flat()) : acc.concat(val)), []);
};
}
// readable stream iterator: https://bugs.chromium.org/p/chromium/issues/detail?id=929585#c10
if (ReadableStream.prototype[Symbol.asyncIterator] === undefined) {
ReadableStream.prototype[Symbol.asyncIterator] = async function* () {
const reader = this.getReader();
try {
while (true) {
const {done, value} = await reader.read();
if (done) return;
yield value;
}
} finally {
reader.releaseLock();
}
};
}

View file

@ -1,7 +1,7 @@
"use strict";
// version and caching control
const version = "1.92.05"; // generator version, update each time
const version = "1.93.00"; // generator version, update each time
{
document.title += " v" + version;
@ -23,11 +23,13 @@ const version = "1.92.05"; // generator version, update each time
const discord = "https://discordapp.com/invite/X7E84HU";
const patreon = "https://www.patreon.com/azgaar";
alertMessage.innerHTML = /* html */ `The Fantasy Map Generator is updated up to version <strong>${version}</strong>. This version is compatible with <a href="${changelog}" target="_blank">previous versions</a>, loaded <i>.map</i> files will be auto-updated.
alertMessage.innerHTML = /* html */ `The Fantasy Map Generator is updated up to version <strong>${version}</strong>. This version is compatible with <a href="${changelog}" target="_blank">previous versions</a>, loaded save files will be auto-updated.
${storedVersion ? "<span>Reload the page to fetch fresh code.</span>" : ""}
<ul>
<strong>Latest changes:</strong>
<li>Auto-load of the last saved map is now optional (see <i>Onload behavior</i> in Options)</li>
<li>Save files compression (file extension is changed to <i>.gz</i>). Old <i>.map</i> files are still supported</li>
<li>New label placement algorithm for states</li>
<li>North and South Poles temperature can be set independently</li>
<li>More than 70 new heraldic charges</li>