Adding gzip compression for improving storage use and backward compatibility. (#984)

* Basic gzip an gunzip on load and save.

* refactor file save type to .gz and update the data in ui.

---------

Co-authored-by: Azgaar <maxganiev@yandex.com>
This commit is contained in:
Efruz Yıldırır 2023-08-15 11:26:01 +03:00 committed by GitHub
parent 5fba7d60f4
commit ef24e3ea1a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 111 additions and 68 deletions

View file

@ -2329,8 +2329,8 @@
<div id="sticked">
<button id="newMapButton" data-tip="Generate a new map based on options" data-shortcut="F2">New Map</button>
<button id="exportButton" data-tip="Select format to download image or export map data">Export</button>
<button id="saveButton" data-tip="Save fully-functional map in .map format">Save</button>
<button id="loadButton" data-tip="Load fully-functional map in .map format">Load</button>
<button id="saveButton" data-tip="Save fully-functional map in .gz format">Save</button>
<button id="loadButton" data-tip="Load fully-functional map in .gz or .map format">Load</button>
<button id="zoomReset" data-tip="Reset map zoom" data-shortcut="0 (zero)">Reset Zoom</button>
</div>
</div>
@ -5884,10 +5884,10 @@
<div id="saveMapData" style="display: none" class="dialog">
<div style="margin-top: 0.3em">
<strong>Save map to</strong>
<button onclick="dowloadMap()" data-tip="Download .map file to your local disk" data-shortcut="Ctrl + S">
<button onclick="downloadMap()" data-tip="Download .gz file to your local disk" data-shortcut="Ctrl + S">
machine
</button>
<button onclick="saveToDropbox()" data-tip="Save .map file to your Dropbox" data-shortcut="Ctrl + C">
<button onclick="saveToDropbox()" data-tip="Save .gz file to your Dropbox" data-shortcut="Ctrl + C">
dropbox
</button>
<button
@ -5899,8 +5899,8 @@
</button>
</div>
<p>
Maps are saved in <i>.map</i> format, that can be loaded back via the <i>Load</i> in menu. There is no way to
restore the progress if file is lost. Please keep old <i>.map</i> files on your machine or cloud storage as
Maps are saved in <i>.gz</i> format, that can be loaded back via the <i>Load</i> in menu. There is no way to
restore the progress if file is lost. Please keep old <i>.gz or .map</i> files on your machine or cloud storage as
backups.
</p>
</div>
@ -5908,8 +5908,8 @@
<div id="loadMapData" style="display: none" class="dialog">
<div>
<strong>Load map from</strong>
<button onclick="mapToLoad.click()" data-tip="Load .map file from your local disk">machine</button>
<button onclick="loadURL()" data-tip="Load .map file from URL (server should allow CORS)">URL</button>
<button onclick="mapToLoad.click()" data-tip="Load .gz or .map file from your local disk">machine</button>
<button onclick="loadURL()" data-tip="Load .gz or .map file from URL (server should allow CORS)">URL</button>
<button onclick="quickLoad()" data-tip="Load map from browser storage (if saved before)">storage</button>
</div>
<div id="loadFromDropbox">
@ -5926,7 +5926,7 @@
<select id="loadFromDropboxSelect" style="width: 22em"></select>
<div id="loadFromDropboxButtons" style="margin-bottom: 0.6em">
<button onclick="loadFromDropbox()" data-tip="Load .map file from your Dropbox">Load</button>
<button onclick="loadFromDropbox()" data-tip="Load .gz or .map file from your Dropbox">Load</button>
<button
onclick="createSharableDropboxLink()"
data-tip="Select file and create a link to share with your friends"
@ -5995,7 +5995,7 @@
<div id="resampleDialog" style="display: none" class="dialog">
<div style="width: 34em; max-width: 80vw; font-weight: bold; padding: 6px">
This operation is destructive and irreversible. It will create a completely new map based on the current one.
Don't forget to save the current project as a .map file first!
Don't forget to save the current project as a .gz file first!
</div>
<div
@ -6116,10 +6116,10 @@
data-main="Сlick the arrow button for options. Zoom in to see the map in details"
></div>
<div id="mapOverlay" style="display: none">Drop a .map file to open</div>
<div id="mapOverlay" style="display: none">Drop a .gz or .map file to open</div>
<div id="fileInputs" style="display: none">
<input type="file" accept=".map" id="mapToLoad" />
<input type="file" accept=".map,.gz" id="mapToLoad" />
<input type="file" accept=".txt,.csv" id="burgsListToLoad" />
<input type="file" accept=".txt" id="legendsToLoad" />
<input type="file" accept="image/*" id="imageToLoad" />
@ -7930,7 +7930,7 @@
<script src="utils/graphUtils.js?v=1.90.01"></script>
<script src="utils/nodeUtils.js"></script>
<script src="utils/numberUtils.js?v=1.89.08"></script>
<script src="utils/polyfills.js"></script>
<script src="utils/polyfills.js?v=1.93.00"></script>
<script src="utils/probabilityUtils.js?v=1.88.00"></script>
<script src="utils/stringUtils.js"></script>
<script src="utils/languageUtils.js"></script>
@ -7963,15 +7963,15 @@
<script src="modules/ui/stylePresets.js?v=1.89.11"></script>
<script src="modules/ui/general.js?v=1.87.03"></script>
<script src="modules/ui/options.js?v=1.91.00"></script>
<script src="main.js?v=1.92.00"></script>
<script src="modules/ui/options.js?v=1.93.00"></script>
<script src="main.js?v=1.93.00"></script>
<script defer src="modules/relief-icons.js"></script>
<script defer src="modules/ui/style.js"></script>
<script defer src="modules/ui/editors.js?v=1.92.00"></script>
<script defer src="modules/ui/tools.js?v=1.92.00"></script>
<script defer src="modules/ui/world-configurator.js?v=1.91.05"></script>
<script defer src="modules/ui/heightmap-editor.js?v=1.92.00"></script>
<script defer src="modules/ui/heightmap-editor.js?v=1.93.00"></script>
<script defer src="modules/ui/provinces-editor.js?v=1.92.00"></script>
<script defer src="modules/ui/biomes-editor.js?v=1.91.05"></script>
<script defer src="modules/ui/namesbase-editor.js?v=1.89.26"></script>
@ -8001,13 +8001,12 @@
<script defer src="modules/ui/markers-editor.js"></script>
<script defer src="modules/ui/3d.js?v=1.89.36"></script>
<script defer src="modules/ui/submap.js?v=1.92.00"></script>
<script defer src="modules/ui/hotkeys.js?v=1.88.00"></script>
<script defer src="modules/ui/hotkeys.js?v=1.93.00"></script>
<script defer src="modules/coa-renderer.js?v=1.91.00"></script>
<script defer src="libs/rgbquant.min.js"></script>
<script defer src="libs/jquery.ui.touch-punch.min.js"></script>
<script defer src="modules/io/save.js?v=1.91.04"></script>
<script defer src="modules/io/load.js?v=1.92.05"></script>
<script defer src="modules/io/save.js?v=1.93.00"></script>
<script defer src="modules/io/load.js?v=1.93.00"></script>
<script defer src="modules/io/cloud.js"></script>
<script defer src="modules/io/export.js?v=1.89.36"></script>
<script defer src="modules/io/formats.js"></script>

10
main.js
View file

@ -270,7 +270,7 @@ async function checkLoadParameters() {
const url = new URL(window.location.href);
const params = url.searchParams;
// of there is a valid maplink, try to load .map file from URL
// of there is a valid maplink, try to load .gz/.map file from URL
if (params.get("maplink")) {
WARN && console.warn("Load map from URL");
const maplink = params.get("maplink");
@ -574,9 +574,9 @@ void (function addDragToUpload() {
overlay.style.display = "none";
if (e.dataTransfer.items == null || e.dataTransfer.items.length !== 1) return; // no files or more than one
const file = e.dataTransfer.items[0].getAsFile();
if (file.name.indexOf(".map") == -1) {
// not a .map file
alertMessage.innerHTML = "Please upload a <b>.map</b> file you have previously downloaded";
if (!file.name.endsWith(".map") && !file.name.endsWith(".gz")) {
// not a .gz/.map file
alertMessage.innerHTML = "Please upload a <b>.gz or .map</b> file you have previously downloaded";
$("#alert").dialog({
resizable: false,
title: "Invalid file format",
@ -596,7 +596,7 @@ void (function addDragToUpload() {
if (closeDialogs) closeDialogs();
uploadMap(file, () => {
overlay.style.display = "none";
overlay.innerHTML = "Drop a .map file to open";
overlay.innerHTML = "Drop a .gz or .map file to open";
});
});
})();

View file

@ -1,6 +1,6 @@
"use strict";
// update old .map version to the current one
// update old .gz/.map version to the current one
export function resolveVersionConflicts(version) {
if (version < 1) {
// v1.0 added a new religions layer

View file

@ -1,6 +1,5 @@
"use strict";
// Functions to load and parse .map files
// Functions to load and parse .gz/.map files
async function quickLoad() {
const blob = await ldb.get("lastMap");
if (blob) loadMapPrompt(blob);
@ -91,8 +90,7 @@ function loadMapFromURL(maplink, random) {
function showUploadErrorMessage(error, URL, random) {
ERROR && console.error(error);
alertMessage.innerHTML = /* html */ `Cannot load map from the ${link(URL, "link provided")}. ${
random ? `A new random map is generated. ` : ""
alertMessage.innerHTML = /* html */ `Cannot load map from the ${link(URL, "link provided")}. ${random ? `A new random map is generated. ` : ""
} Please ensure the
linked file is reachable and CORS is allowed on server side`;
$("#alert").dialog({
@ -112,11 +110,11 @@ function uploadMap(file, callback) {
const currentVersion = parseFloat(version);
const fileReader = new FileReader();
fileReader.onload = function (fileLoadedEvent) {
fileReader.onloadend = async function (fileLoadedEvent) {
if (callback) callback();
document.getElementById("coas").innerHTML = ""; // remove auto-generated emblems
const result = fileLoadedEvent.target.result;
const [mapData, mapVersion] = parseLoadedResult(result);
const [mapData, mapVersion] = await parseLoadedResult(result);
const isInvalid = !mapData || isNaN(mapVersion) || mapData.length < 26 || !mapData[5];
const isUpdated = mapVersion === currentVersion;
@ -131,18 +129,37 @@ function uploadMap(file, callback) {
if (isOutdated) return showUploadMessage("outdated", mapData, mapVersion);
};
fileReader.readAsText(file, "UTF-8");
fileReader.readAsArrayBuffer(file);
}
function parseLoadedResult(result) {
async function uncompressMapData(compressedMapData) {
console.log("trying to uncompress:", compressedMapData);
try {
const uncompressedStream = new Blob([compressedMapData]).stream().pipeThrough(new DecompressionStream("gzip"));
let uncompressedData = [];
for await (const chunk of uncompressedStream) {
uncompressedData = uncompressedData.concat(Array.from(chunk));
}
return new Uint8Array(uncompressedData);
}
catch (error) {
console.error(error);
return null;
}
}
async function parseLoadedResult(result) {
try {
const resultAsString = new TextDecoder().decode(result);
// data can be in FMG internal format or base64 encoded
const isDelimited = result.substr(0, 10).includes("|");
const decoded = isDelimited ? result : decodeURIComponent(atob(result));
const isDelimited = resultAsString.substring(0, 10).includes("|");
const decoded = isDelimited ? resultAsString : decodeURIComponent(atob(resultAsString));
const mapData = decoded.split("\r\n");
const mapVersion = parseFloat(mapData[0].split("|")[0] || mapData[0]);
return [mapData, mapVersion];
} catch (error) {
const uncompressedData = await uncompressMapData(result);
if (uncompressedData !== null) {
return parseLoadedResult(uncompressedData);
}
ERROR && console.error(error);
return [null, null];
}
@ -153,7 +170,7 @@ function showUploadMessage(type, mapData, mapVersion) {
let message, title, canBeLoaded;
if (type === "invalid") {
message = `The file does not look like a valid <i>.map</i> file.<br>Please check the data format`;
message = `The file does not look like a valid <i>.gz or .map</i> file.<br>Please check the data format`;
title = "Invalid file";
canBeLoaded = false;
} else if (type === "ancient") {

View file

@ -1,5 +1,5 @@
"use strict";
// functions to save project as .map file
// functions to save project as .gz file
// prepare map data for saving
function getMapData() {
@ -116,18 +116,27 @@ function getMapData() {
].join("\r\n");
return mapData;
}
async function compressMapData(mapData){
const compressedStream = new Blob([mapData]).stream().pipeThrough(new CompressionStream("gzip"));
let compressedData = [];
for await (const chunk of compressedStream){
compressedData = compressedData.concat(Array.from(chunk));
}
return new Uint8Array(compressedData);
}
// Download .map file
function dowloadMap() {
// Download .gz file
async function downloadMap() {
if (customization)
return tip("Map cannot be saved when edit mode is active, please exit the mode and retry", false, "error");
closeDialogs("#alert");
const mapData = getMapData();
const mapData = await compressMapData(getMapData());
const blob = new Blob([mapData], {type: "text/plain"});
const URL = window.URL.createObjectURL(blob);
const link = document.createElement("a");
link.download = getFileName() + ".map";
link.download = getFileName() + ".gz";
link.href = URL;
link.click();
tip(`${link.download} is saved. Open "Downloads" screen (CTRL + J) to check`, true, "success", 7000);
@ -138,14 +147,14 @@ async function saveToDropbox() {
if (customization)
return tip("Map cannot be saved when edit mode is active, please exit the mode and retry", false, "error");
closeDialogs("#alert");
const mapData = getMapData();
const filename = getFileName() + ".map";
const mapData = await compressMapData(getMapData());
const filename = getFileName() + ".gz";
try {
await Cloud.providers.dropbox.save(filename, mapData);
tip("Map is saved to your Dropbox", true, "success", 8000);
} catch (msg) {
ERROR && console.error(msg);
tip("Cannot save .map to your Dropbox", true, "error", 8000);
tip("Cannot save .gz to your Dropbox", true, "error", 8000);
}
}
@ -162,7 +171,7 @@ async function initiateAutosave() {
if (customization) return tip("Autosave: map cannot be saved in edit mode", false, "warning", 2000);
tip("Autosave: saving map...", false, "warning", 3000);
const mapData = getMapData();
const mapData = await compressMapData(getMapData());
const blob = new Blob([mapData], {type: "text/plain"});
await ldb.set("lastMap", blob);
INFO && console.log("Autosaved at", new Date().toLocaleTimeString());
@ -176,22 +185,22 @@ async function quickSave() {
if (customization)
return tip("Map cannot be saved when edit mode is active, please exit the mode first", false, "error");
const mapData = getMapData();
const mapData = await compressMapData(getMapData());
const blob = new Blob([mapData], {type: "text/plain"});
await ldb.set("lastMap", blob); // auto-save map
tip("Map is saved to browser memory. Please also save as .map file to secure progress", true, "success", 2000);
tip("Map is saved to browser memory. Please also save as .gz file to secure progress", true, "success", 2000);
}
const saveReminder = function () {
if (localStorage.getItem("noReminder")) return;
const message = [
"Please don't forget to save your work as a .map file",
"Please remember to save work as a .map file",
"Saving in .map format will ensure your data won't be lost in case of issues",
"Please don't forget to save your work as a .gz file",
"Please remember to save work as a .gz file",
"Saving in .gz format will ensure your data won't be lost in case of issues",
"Safety is number one priority. Please save the map",
"Don't forget to save your map on a regular basis!",
"Just a gentle reminder for you to save the map",
"Please don't forget to save your progress (saving as .map is the best option)",
"Please don't forget to save your progress (saving as .gz is the best option)",
"Don't want to be reminded about need to save? Press CTRL+Q"
];
const interval = 15 * 60 * 1000; // remind every 15 minutes

View file

@ -28,7 +28,7 @@ function editHeightmap(options) {
<p><i>Erase</i> mode also allows you Convert an Image into a heightmap or use Template Editor.</p>
<p>You can <i>keep</i> the data, but you won't be able to change the coastline.</p>
<p>Try <i>risk</i> mode to change the coastline and keep the data. The data will be restored as much as possible, but it can cause unpredictable errors.</p>
<p>Please <span class="pseudoLink" onclick="dowloadMap();">save the map</span> before editing the heightmap!</p>
<p>Please <span class="pseudoLink" onclick="downloadMap();">save the map</span> before editing the heightmap!</p>
<p style="margin-bottom: 0">Check out ${link(
"https://github.com/Azgaar/Fantasy-Map-Generator/wiki/Heightmap-customization",
"wiki"

View file

@ -32,7 +32,7 @@ function handleKeyup(event) {
else if (code === "Delete") removeElementOnKey();
else if (code === "KeyO" && document.getElementById("canvas3d")) toggle3dOptions();
else if (ctrl && code === "KeyQ") toggleSaveReminder();
else if (ctrl && code === "KeyS") dowloadMap();
else if (ctrl && code === "KeyS") downloadMap();
else if (ctrl && code === "KeyC") saveToDropbox();
else if (ctrl && code === "KeyZ" && undo?.offsetParent) undo.click();
else if (ctrl && code === "KeyY" && redo?.offsetParent) redo.click();

View file

@ -844,8 +844,8 @@ async function connectToDropbox() {
function loadURL() {
const pattern = /(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/;
const inner = `Provide URL to a .map file:
<input id="mapURL" type="url" style="width: 24em" placeholder="https://e-cloud.com/test.map">
const inner = `Provide URL to a .gz or .map file:
<input id="mapURL" type="url" style="width: 24em" placeholder="https://e-cloud.com/test.gz">
<br><i>Please note server should allow CORS for file to be loaded. If CORS is not allowed, save file to Dropbox and provide a direct link</i>`;
alertMessage.innerHTML = inner;
$("#alert").dialog({

View file

@ -14,3 +14,20 @@ if (Array.prototype.flat === undefined) {
return this.reduce((acc, val) => (Array.isArray(val) ? acc.concat(val.flat()) : acc.concat(val)), []);
};
}
// polyfill readable stream iterator: https://bugs.chromium.org/p/chromium/issues/detail?id=929585#c10
if(ReadableStream.prototype[Symbol.asyncIterator] === undefined){
ReadableStream.prototype[Symbol.asyncIterator] = async function* () {
const reader = this.getReader()
try {
while (true) {
const {done, value} = await reader.read()
if (done) return
yield value
}
}
finally {
reader.releaseLock()
}
}
}

View file

@ -1,7 +1,7 @@
"use strict";
// version and caching control
const version = "1.92.05"; // generator version, update each time
const version = "1.93.00"; // generator version, update each time
{
document.title += " v" + version;
@ -23,11 +23,12 @@ const version = "1.92.05"; // generator version, update each time
const discord = "https://discordapp.com/invite/X7E84HU";
const patreon = "https://www.patreon.com/azgaar";
alertMessage.innerHTML = /* html */ `The Fantasy Map Generator is updated up to version <strong>${version}</strong>. This version is compatible with <a href="${changelog}" target="_blank">previous versions</a>, loaded <i>.map</i> files will be auto-updated.
alertMessage.innerHTML = /* html */ `The Fantasy Map Generator is updated up to version <strong>${version}</strong>. This version is compatible with <a href="${changelog}" target="_blank">previous versions</a>, loaded <i>.gz or .map</i> files will be auto-updated.
${storedVersion ? "<span>Reload the page to fetch fresh code.</span>" : ""}
<ul>
<strong>Latest changes:</strong>
<li>Map save file extension is changed to .gz, .map files are still loadable</li>
<li>New label placement algorithm for states</li>
<li>North and South Poles temperature can be set independently</li>
<li>More than 70 new heraldic charges</li>