From 3f205cd499358a5397fac157b295a3b4a5a76bb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Efruz=20Y=C4=B1ld=C4=B1r=C4=B1r?=
 <30903352+yldrefruz@users.noreply.github.com>
Date: Mon, 14 Aug 2023 12:06:55 +0300
Subject: [PATCH] Basic gzip an gunzip on load and save.

---
 index.html         |  8 +++----
 modules/io/load.js | 57 ++++++++++++++++++++++++++++++----------------
 modules/io/save.js | 19 ++++++++++++----
 utils/polyfills.js | 17 ++++++++++++++
 versioning.js      |  2 +-
 5 files changed, 73 insertions(+), 30 deletions(-)

diff --git a/index.html b/index.html
index 9ab2fc63..b435fab2 100644
--- a/index.html
+++ b/index.html
@@ -5884,7 +5884,7 @@
       <div id="saveMapData" style="display: none" class="dialog">
         <div style="margin-top: 0.3em">
           <strong>Save map to</strong>
-          <button onclick="dowloadMap()" data-tip="Download .map file to your local disk" data-shortcut="Ctrl + S">
+          <button onclick="downloadMap()" data-tip="Download .map file to your local disk" data-shortcut="Ctrl + S">
             machine
           </button>
           <button onclick="saveToDropbox()" data-tip="Save .map file to your Dropbox" data-shortcut="Ctrl + C">
@@ -7930,7 +7930,7 @@
     <script src="utils/graphUtils.js?v=1.90.01"></script>
     <script src="utils/nodeUtils.js"></script>
     <script src="utils/numberUtils.js?v=1.89.08"></script>
-    <script src="utils/polyfills.js"></script>
+    <script src="utils/polyfills.js?v=1.93.00"></script>
     <script src="utils/probabilityUtils.js?v=1.88.00"></script>
     <script src="utils/stringUtils.js"></script>
     <script src="utils/languageUtils.js"></script>
@@ -8006,8 +8006,8 @@
     <script defer src="libs/rgbquant.min.js"></script>
     <script defer src="libs/jquery.ui.touch-punch.min.js"></script>
 
-    <script defer src="modules/io/save.js?v=1.91.04"></script>
-    <script defer src="modules/io/load.js?v=1.92.02"></script>
+    <script defer src="modules/io/save.js?v=1.93.00"></script>
+    <script defer src="modules/io/load.js?v=1.93.00"></script>
     <script defer src="modules/io/cloud.js"></script>
     <script defer src="modules/io/export.js?v=1.89.36"></script>
     <script defer src="modules/io/formats.js"></script>
diff --git a/modules/io/load.js b/modules/io/load.js
index 1f8366d2..4618c93c 100644
--- a/modules/io/load.js
+++ b/modules/io/load.js
@@ -1,6 +1,5 @@
 "use strict";
 // Functions to load and parse .map files
-
 async function quickLoad() {
   const blob = await ldb.get("lastMap");
   if (blob) loadMapPrompt(blob);
@@ -77,7 +76,7 @@ function loadMapPrompt(blob) {
 function loadMapFromURL(maplink, random) {
   const URL = decodeURIComponent(maplink);
 
-  fetch(URL, {method: "GET", mode: "cors"})
+  fetch(URL, { method: "GET", mode: "cors" })
     .then(response => {
       if (response.ok) return response.blob();
       throw new Error("Cannot load map from URL");
@@ -91,9 +90,8 @@ function loadMapFromURL(maplink, random) {
 
 function showUploadErrorMessage(error, URL, random) {
   ERROR && console.error(error);
-  alertMessage.innerHTML = /* html */ `Cannot load map from the ${link(URL, "link provided")}. ${
-    random ? `A new random map is generated. ` : ""
-  } Please ensure the
+  alertMessage.innerHTML = /* html */ `Cannot load map from the ${link(URL, "link provided")}. ${random ? `A new random map is generated. ` : ""
+    } Please ensure the
   linked file is reachable and CORS is allowed on server side`;
   $("#alert").dialog({
     title: "Loading error",
@@ -112,11 +110,11 @@ function uploadMap(file, callback) {
   const currentVersion = parseFloat(version);
 
   const fileReader = new FileReader();
-  fileReader.onload = function (fileLoadedEvent) {
+  fileReader.onloadend = async function (fileLoadedEvent) {
     if (callback) callback();
     document.getElementById("coas").innerHTML = ""; // remove auto-generated emblems
     const result = fileLoadedEvent.target.result;
-    const [mapData, mapVersion] = parseLoadedResult(result);
+    const [mapData, mapVersion] = await parseLoadedResult(result);
 
     const isInvalid = !mapData || isNaN(mapVersion) || mapData.length < 26 || !mapData[5];
     const isUpdated = mapVersion === currentVersion;
@@ -131,18 +129,37 @@ function uploadMap(file, callback) {
     if (isOutdated) return showUploadMessage("outdated", mapData, mapVersion);
   };
 
-  fileReader.readAsText(file, "UTF-8");
+  fileReader.readAsArrayBuffer(file);
 }
-
-function parseLoadedResult(result) {
+async function uncompressMapData(compressedMapData) {
+  console.log("trying to uncompress:", compressedMapData);
   try {
+    const uncompressedStream = new Blob([compressedMapData]).stream().pipeThrough(new DecompressionStream("gzip"));
+    let uncompressedData = [];
+    for await (const chunk of uncompressedStream) {
+      uncompressedData = uncompressedData.concat(Array.from(chunk));
+    }
+    return new Uint8Array(uncompressedData);
+  }
+  catch (error) {
+    console.error(error);
+    return null;
+  }
+}
+async function parseLoadedResult(result) {
+  try {
+    const resultAsString = new TextDecoder().decode(result);
     // data can be in FMG internal format or base64 encoded
-    const isDelimited = result.substr(0, 10).includes("|");
-    const decoded = isDelimited ? result : decodeURIComponent(atob(result));
+    const isDelimited = resultAsString.substring(0, 10).includes("|");
+    const decoded = isDelimited ? resultAsString : decodeURIComponent(atob(resultAsString));
     const mapData = decoded.split("\r\n");
     const mapVersion = parseFloat(mapData[0].split("|")[0] || mapData[0]);
     return [mapData, mapVersion];
   } catch (error) {
+    const uncompressedData = await uncompressMapData(result);
+    if (uncompressedData !== null) {
+      return parseLoadedResult(uncompressedData);
+    }
     ERROR && console.error(error);
     return [null, null];
   }
@@ -177,7 +194,7 @@ function showUploadMessage(type, mapData, mapVersion) {
       if (canBeLoaded) parseLoadedData(mapData);
     }
   };
-  $("#alert").dialog({title, buttons});
+  $("#alert").dialog({ title, buttons });
 }
 
 async function parseLoadedData(data) {
@@ -246,9 +263,9 @@ async function parseLoadedData(data) {
       if (data[34]) {
         const usedFonts = JSON.parse(data[34]);
         usedFonts.forEach(usedFont => {
-          const {family: usedFamily, unicodeRange: usedRange, variant: usedVariant} = usedFont;
+          const { family: usedFamily, unicodeRange: usedRange, variant: usedVariant } = usedFont;
           const defaultFont = fonts.find(
-            ({family, unicodeRange, variant}) =>
+            ({ family, unicodeRange, variant }) =>
               family === usedFamily && unicodeRange === usedRange && variant === usedVariant
           );
           if (!defaultFont) fonts.push(usedFont);
@@ -331,7 +348,7 @@ async function parseLoadedData(data) {
     void (function parseGridData() {
       grid = JSON.parse(data[6]);
 
-      const {cells, vertices} = calculateVoronoi(grid.points, grid.boundary);
+      const { cells, vertices } = calculateVoronoi(grid.points, grid.boundary);
       grid.cells = cells;
       grid.vertices = vertices;
 
@@ -349,7 +366,7 @@ async function parseLoadedData(data) {
       pack.cultures = JSON.parse(data[13]);
       pack.states = JSON.parse(data[14]);
       pack.burgs = JSON.parse(data[15]);
-      pack.religions = data[29] ? JSON.parse(data[29]) : [{i: 0, name: "No religion"}];
+      pack.religions = data[29] ? JSON.parse(data[29]) : [{ i: 0, name: "No religion" }];
       pack.provinces = data[30] ? JSON.parse(data[30]) : [0];
       pack.rivers = data[32] ? JSON.parse(data[32]) : [];
       pack.markers = data[35] ? JSON.parse(data[35]) : [];
@@ -375,7 +392,7 @@ async function parseLoadedData(data) {
           const e = d.split("|");
           if (!e.length) return;
           const b = e[5].split(",").length > 2 || !nameBases[i] ? e[5] : nameBases[i].b;
-          nameBases[i] = {name: e[0], min: e[1], max: e[2], d: e[3], m: e[4], b};
+          nameBases[i] = { name: e[0], min: e[1], max: e[2], d: e[3], m: e[4], b };
         });
       }
     })();
@@ -435,7 +452,7 @@ async function parseLoadedData(data) {
     {
       // dynamically import and run auto-udpdate script
       const versionNumber = parseFloat(params[0]);
-      const {resolveVersionConflicts} = await import("../dynamic/auto-update.js?v=1.92.02");
+      const { resolveVersionConflicts } = await import("../dynamic/auto-update.js?v=1.92.02");
       resolveVersionConflicts(versionNumber);
     }
 
@@ -624,7 +641,7 @@ async function parseLoadedData(data) {
           $(this).dialog("close");
         }
       },
-      position: {my: "center", at: "center", of: "svg"}
+      position: { my: "center", at: "center", of: "svg" }
     });
   }
 }
diff --git a/modules/io/save.js b/modules/io/save.js
index 97cf7c09..5a47d84f 100644
--- a/modules/io/save.js
+++ b/modules/io/save.js
@@ -116,14 +116,23 @@ function getMapData() {
   ].join("\r\n");
   return mapData;
 }
+async function compressMapData(mapData){
+  const compressedStream = new Blob([mapData]).stream().pipeThrough(new CompressionStream("gzip"));
+  let compressedData = [];
+  for await (const chunk of compressedStream){
+    compressedData = compressedData.concat(Array.from(chunk));
+  }
+  return new Uint8Array(compressedData);
+}
+
 
 // Download .map file
-function dowloadMap() {
+async function downloadMap() {
   if (customization)
     return tip("Map cannot be saved when edit mode is active, please exit the mode and retry", false, "error");
   closeDialogs("#alert");
 
-  const mapData = getMapData();
+  const mapData = await compressMapData(getMapData());
   const blob = new Blob([mapData], {type: "text/plain"});
   const URL = window.URL.createObjectURL(blob);
   const link = document.createElement("a");
@@ -138,7 +147,7 @@ async function saveToDropbox() {
   if (customization)
     return tip("Map cannot be saved when edit mode is active, please exit the mode and retry", false, "error");
   closeDialogs("#alert");
-  const mapData = getMapData();
+  const mapData = await compressMapData(getMapData());
   const filename = getFileName() + ".map";
   try {
     await Cloud.providers.dropbox.save(filename, mapData);
@@ -162,7 +171,7 @@ async function initiateAutosave() {
     if (customization) return tip("Autosave: map cannot be saved in edit mode", false, "warning", 2000);
 
     tip("Autosave: saving map...", false, "warning", 3000);
-    const mapData = getMapData();
+    const mapData = await compressMapData(getMapData());
     const blob = new Blob([mapData], {type: "text/plain"});
     await ldb.set("lastMap", blob);
     INFO && console.log("Autosaved at", new Date().toLocaleTimeString());
@@ -176,7 +185,7 @@ async function quickSave() {
   if (customization)
     return tip("Map cannot be saved when edit mode is active, please exit the mode first", false, "error");
 
-  const mapData = getMapData();
+  const mapData = await compressMapData(getMapData());
   const blob = new Blob([mapData], {type: "text/plain"});
   await ldb.set("lastMap", blob); // auto-save map
   tip("Map is saved to browser memory. Please also save as .map file to secure progress", true, "success", 2000);
diff --git a/utils/polyfills.js b/utils/polyfills.js
index 369e647f..fc567c1e 100644
--- a/utils/polyfills.js
+++ b/utils/polyfills.js
@@ -14,3 +14,20 @@ if (Array.prototype.flat === undefined) {
     return this.reduce((acc, val) => (Array.isArray(val) ? acc.concat(val.flat()) : acc.concat(val)), []);
   };
 }
+
+// polyfill readable stream iterator: https://bugs.chromium.org/p/chromium/issues/detail?id=929585#c10
+if(ReadableStream.prototype[Symbol.asyncIterator] === undefined){
+  ReadableStream.prototype[Symbol.asyncIterator] = async function* () {
+    const reader = this.getReader()
+    try {
+      while (true) {
+        const {done, value} = await reader.read()
+        if (done) return
+        yield value
+      }
+    }
+    finally {
+      reader.releaseLock()
+    }
+  }
+}
\ No newline at end of file
diff --git a/versioning.js b/versioning.js
index ed883322..5582a17e 100644
--- a/versioning.js
+++ b/versioning.js
@@ -1,7 +1,7 @@
 "use strict";
 
 // version and caching control
-const version = "1.92.02"; // generator version, update each time
+const version = "1.93.00"; // generator version, update each time
 
 {
   document.title += " v" + version;