Remove all data that could be retrieved from ESO API from the save (title, body, medium, showTitle)

Orionik [05-27-22 - 15:37]
Remove all data that could be retrieved from ESO API from the save (title, body, medium, showTitle)
Now that we have the bookId, we can retrieve them easily.
It lightens the save (we now are around 2MB for one character with all books) so it will help avoiding data corruption.
It also allow to fully support other language.
It has a small impact when loading the addon but nothing more.
Filename
CHANGELOG
Librarian.lua
lang/fr.lua
diff --git a/CHANGELOG b/CHANGELOG
index 9ec5e8b..b2073c6 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -9,7 +9,8 @@ Librarian v3.0 2022-05-13
 - Remove Import from before patch option which was meant for save created back in 2015
 - Remove some deprecation code for save before 2015
 - Add unreadPerCollection in save in order to know which collection should have an icon in lore library
-- Deprecate saves to use bookId as identifiers instead of the title (several books had the same name), characterBooks now reference the bookId instead of the title and fill at the same time the unreadPerCollection list
+- Deprecate global saves to use bookId as identifiers instead of the title (several books had the same name) and remove all data that could be retrieved thanks to ESO API now that we have the bookId (title, body, medium, showTitle, wordCount). It lightens the save (~2MB for all the books on one character) and allow the player to switch language.
+- Deprecate characterBooks save, it now references the bookId instead of the title and fill at the same time the unreadPerCollection list
 - Optimize ImportFromLoreLibrary by removing the Refresh every frame (without it, past 1000 books, the game slow down a lot and at 4000 you end up with less than 1fps)
 - Activate back auto importation from init now that it is quick enough
 - Add full localization and support for french language
diff --git a/Librarian.lua b/Librarian.lua
index f2ed37d..36f6e82 100644
--- a/Librarian.lua
+++ b/Librarian.lua
@@ -98,13 +98,19 @@ function Librarian:AddLoreReaderUnreadToggle()
         readerKeybinds = LORE_READER.PCKeybindStripDescriptor
     end

+    local function IsSameBook(book, title)
+        local categoryIndex, collectionIndex, bookIndex = GetLoreBookIndicesFromBookId(book.bookId)
+        local bookTitle = GetLoreBookInfo(categoryIndex, collectionIndex, bookIndex)
+        return bookTitle == title
+    end
+
     if readerKeybinds then
         local toggleKeybind =
         {
             alignment = KEYBIND_STRIP_ALIGN_RIGHT,
             name = function()
                 local book = self:FindBook(self.lastShownBookId)
-                if not book or (book.unread and book.title == LORE_READER.titleText) then
+                if not book or (book.unread and IsSameBook(book, LORE_READER.titleText)) then
                     if self.settings.showUnreadIndicatorInReader then
                         self.loreReaderUnreadIndicator:SetHidden(false)
                     else
@@ -119,10 +125,11 @@ function Librarian:AddLoreReaderUnreadToggle()
             keybind = "UI_SHORTCUT_SECONDARY",
             callback = function()
                 local book = self:FindBook(self.lastShownBookId)
-                if not book or book.title ~= LORE_READER.titleText then return end
-                self:ToggleReadBook(book)
-                KEYBIND_STRIP:UpdateKeybindButtonGroup(readerKeybinds)
-                self:RefreshAllData()
+                if book and IsSameBook(book, LORE_READER.titleText) then
+                    self:ToggleReadBook(book)
+                    KEYBIND_STRIP:UpdateKeybindButtonGroup(readerKeybinds)
+                    self:RefreshAllData()
+                end
             end
         }
         table.insert(readerKeybinds, toggleKeybind)
@@ -231,10 +238,8 @@ function Librarian:AddLoreLibraryIcons()
             if selectedRow then
                 local book = self:FindBook(selectedRow.bookId)
                 if not book then
-                    local bookId = select(4, GetLoreBookInfo(selectedRow.categoryIndex, selectedRow.collectionIndex, selectedRow.bookIndex))
-                    local body, medium, showTitle = ReadLoreBook(selectedRow.categoryIndex, selectedRow.collectionIndex, selectedRow.bookIndex)
-                    book = { title = selectedRow.text:GetText(), body = body, medium = medium, showTitle = showTitle, bookId = bookId }
-                    self:AddBook(book, true)
+                    self:AddBook(selectedRow.bookId, selectedRow.text:GetText(), true)
+                    book = self:FindBook(selectedRow.bookId)
                 end

                 if book then
@@ -330,26 +335,21 @@ function Librarian:UpdateSavedVariables()
                 if book then
                     if not book.bookId then
                         book.bookId = bookId
-                    else -- there is a collision (2 books with same name)
-                        local body, medium, showTitle = ReadLoreBook(bookIndexes.categoryIndex, bookIndexes.collectionIndex, bookIndexes.bookIndex)
-
-                        local bookBody = table.concat(book.body)
-                        if bookBody == body then
-                            -- the current book is the one corresponding so we will update the bookId and add a book for the previous book which had the same title
-                            local previousCategoryIndex, previousCollectionIndex, previousBookIndex = GetLoreBookIndicesFromBookId(book.bookId)
-                            local previousBody, previousMedium, previousShowTitle = ReadLoreBook(previousCategoryIndex, previousCollectionIndex, previousBookIndex)
-                            local newBook = { title = title, body = previousBody, medium = previousMedium, showTitle = previousShowTitle, bookId = book.bookId }
-                            self:AddBookToGlobalSave(newBook, false)
-
-                            book.bookId = bookId
-                        else
-                            -- the current book has a different body, so let's create an entry for it
-                            local newBook = { title = title, body = body, medium = medium, showTitle = showTitle, bookId = bookId }
-                            self:AddBookToGlobalSave(newBook, false)
-                        end
+                    else -- there is a collision (2 books with same name) so we add an entry for the new one
+                        self:AddBookToGlobalSave(bookId)
                     end
                 end
             end
+
+            -- now we can erase all the field that are useless (ESO API is performant enough to get these data only when needed)
+            -- it will also help keeping the save small and avoid issues with corrupted save
+            for _,book in pairs(self.books) do
+                book.title = nil
+                book.body = nil
+                book.medium = nil
+                book.showTitle = nil
+                book.wordCount = nil
+            end
         end
     end

@@ -509,12 +509,10 @@ function Librarian:ImportFromLoreLibrary()
             bookIndex = 0
         end
         bookIndex = bookIndex + 1
-        local title, icon, known, bookId = GetLoreBookInfo(categoryIndex, collectionIndex, bookIndex)
+        local title, _, known, bookId = GetLoreBookInfo(categoryIndex, collectionIndex, bookIndex)
         if known then
             if not self:FindCharacterBook(bookId) then
-                local body, medium, showTitle = ReadLoreBook(categoryIndex, collectionIndex, bookIndex)
-                local book = { title = title, body = body, medium = medium, showTitle = showTitle, bookId = bookId }
-                self:AddBook(book, false)
+                self:AddBook(bookId, title, false)
                 hasImportedBooks = true
             end
         end
@@ -534,6 +532,7 @@ end

 function Librarian:BuildMasterList()
     local GetLoreBookIndicesFromBookId, GetLoreCollectionInfo = GetLoreBookIndicesFromBookId, GetLoreCollectionInfo
+    local GetLoreBookInfo = GetLoreBookInfo

     local function ShouldDisplayBook(book)
         if not self.settings.showHiddenBook then
@@ -548,14 +547,30 @@ function Librarian:BuildMasterList()

     for i, book in ipairs(self.books) do
         if ShouldDisplayBook(book) then
-            local data = {}
+            if not self.masterList[i] then
+                self.masterList[i] = {}
+            end
+
+            local data = self.masterList[i]
             for k,v in pairs(book) do
-                if k == "body" then
-                    data[k] = table.concat(book.body)
-                else
-                    data[k] = v
+                data[k] = v
+            end
+
+            -- because we reuse the same list between 2 refresh, the data may already be filled up so no need to recompute it, these won't change
+            if not data.categoryIndex or not data.collectionIndex or not data.bookIndex or not data.title or not data.body or not data.wordCount then
+                data.categoryIndex, data.collectionIndex, data.bookIndex = GetLoreBookIndicesFromBookId(book.bookId)
+                data.title = GetLoreBookInfo(data.categoryIndex, data.collectionIndex, data.bookIndex)
+                data.body = ReadLoreBook(data.categoryIndex, data.collectionIndex, data.bookIndex)
+
+                local wordCount = 0
+                if data.body then
+                    for w in data.body:gmatch("%S+") do
+                        wordCount = wordCount + 1
+                    end
                 end
+                data.wordCount = wordCount
             end
+
             data.type = LIBRARIAN_SEARCH
             local characterBook = self:FindCharacterBook(book.bookId)
             if characterBook then
@@ -565,7 +580,8 @@ function Librarian:BuildMasterList()
                 data.seenByCurrentCharacter = false
                 data.timeStamp = book.timeStamp
             end
-            self.masterList[i] = data
+        else
+            self.masterList[i] = nil
         end
     end
 end
@@ -577,8 +593,7 @@ function Librarian:FilterScrollList()
     local bookCount = 0
     local unreadCount = 0
     local searchTerm = self.searchBox:GetText()
-    for i = 1, #self.masterList do
-        local data = self.masterList[i]
+    for _, data in pairs(self.masterList) do
         if self.settings.showAllBooks or data.seenByCurrentCharacter then
             if(searchTerm == "" or self.search:IsMatch(searchTerm, data)) then
                 table.insert(scrollData, ZO_ScrollList_CreateDataEntry(LIBRARIAN_DATA, data))
@@ -661,35 +676,24 @@ function Librarian:FindBook(bookId)
     end
 end

-function Librarian:AddBookToGlobalSave(book)
-    book.timeStamp = GetTimeStamp()
-    book.unread = true
-    local wordCount = 0
-    for w in book.body:gmatch("%S+") do wordCount = wordCount + 1 end
-    book.wordCount = wordCount
-
-    local newBody = book.body
-    book.body = {}
-    while string.len(newBody) > 1024 do
-        table.insert(book.body, string.sub(newBody, 0, 1024))
-        newBody = string.sub(newBody, 1025)
-    end
-    table.insert(book.body, newBody)
-
+function Librarian:AddBookToGlobalSave(bookId)
+    local book = { bookId = bookId, timeStamp = GetTimeStamp(), unread = true }
     table.insert(self.books, book)
+    return book
 end

-function Librarian:AddBook(book, refreshDataRightAway)
-    if not self:FindCharacterBook(book.bookId) then
-        if not self:FindBook(book.bookId) then
-            self:AddBookToGlobalSave(book)
+function Librarian:AddBook(bookId, bookTitle, refreshDataRightAway)
+    if not self:FindCharacterBook(bookId) then
+        local book = self:FindBook(bookId)
+        if not book then
+            book = self:AddBookToGlobalSave(bookId)
         end

-        local characterBook = { bookId = book.bookId, timeStamp = GetTimeStamp() }
+        local characterBook = { bookId = bookId, timeStamp = GetTimeStamp() }
         table.insert(self.characterBooks, characterBook)

         if book.unread then
-            local categoryIndex, collectionIndex = GetLoreBookIndicesFromBookId(book.bookId)
+            local categoryIndex, collectionIndex = GetLoreBookIndicesFromBookId(bookId)
             self:AddUnreadBookInCollection(categoryIndex, collectionIndex)
         end

@@ -705,7 +709,7 @@ function Librarian:AddBook(book, refreshDataRightAway)
             CENTER_SCREEN_ANNOUNCE:AddMessageWithParams(params)
         end
         if self.settings.chatEnabled then
-            d(string.format(GetString(LIBRARIAN_NEW_BOOK_FOUND_WITH_TITLE), book.title))
+            d(string.format(GetString(LIBRARIAN_NEW_BOOK_FOUND_WITH_TITLE), bookTitle))
         end

         self.newBookCount = self.newBookCount + 1
@@ -713,6 +717,7 @@ function Librarian:AddBook(book, refreshDataRightAway)
             d(GetString(LIBRARIAN_RELOAD_REMINDER))
         end
     end
+
 end

 function Librarian:AddUnreadBookInCollection(categoryIndex, collectionIndex)
@@ -747,7 +752,8 @@ end
 function Librarian:ReadBook(data)
     self.lastShownBookId = data.bookId

-    LORE_READER:SetupBook(data.title, data.body, data.medium, data.showTitle)
+    local body, medium, showTitle = ReadLoreBook(data.categoryIndex, data.collectionIndex, data.bookIndex)
+    LORE_READER:SetupBook(data.title, body, medium, showTitle)
     SCENE_MANAGER:Push("loreReaderInteraction")

     -- PlaySound(LORE_READER.OpenSound)
@@ -798,8 +804,7 @@ function Librarian.SlashCommand(args)
 end

 function Librarian.OnShowBook(eventCode, title, body, medium, showTitle, bookId)
-    local book = { title = title, body = body, medium = medium, showTitle = showTitle, bookId = bookId }
-    LIBRARIAN:AddBook(book, true)
+    LIBRARIAN:AddBook(bookId, title, true)

     LIBRARIAN.lastShownBookId = bookId
 end
diff --git a/lang/fr.lua b/lang/fr.lua
index aa8c72b..abc15a5 100644
--- a/lang/fr.lua
+++ b/lang/fr.lua
@@ -16,7 +16,7 @@ SafeAddString(LIBRARIAN_NEW_BOOK_FOUND_WITH_TITLE,					"Livre ajouté à Librari
 SafeAddString(LIBRARIAN_FULLTEXT_SEARCH,							"Recherche:", 1)
 SafeAddString(LIBRARIAN_SEARCH_HINT,								"Texte à rechercher.", 1)
 SafeAddString(LIBRARIAN_RELOAD_REMINDER,							"ReloadUI conseillé pour mettre à jour les données de Librarian.", 1)
-SafeAddString(LIBRARIAN_BACKUP_REMINDER,							"Pensez à sauvegader les SaveVariables de Librarian régulièrement.  Look up Librarian on ESOUI for instructions.", 1)
+SafeAddString(LIBRARIAN_BACKUP_REMINDER,							"Pensez à sauvegader les SaveVariables de Librarian régulièrement. Cherchez 'Librarian' sur ESOUI pour plus d'explications (en anglais).", 1)

 SafeAddString(LIBRARIAN_SETTINGS_DISPLAY_NAME,						"Librarian (Gestionaire de livres)", 1)
 SafeAddString(LIBRARIAN_SETTINGS_TIME,								"Format d'heure", 1)
@@ -24,19 +24,19 @@ SafeAddString(LIBRARIAN_SETTINGS_TIME_TOOLTIP,						"Quel format d'heure préfé
 SafeAddString(LIBRARIAN_SETTINGS_ALERT,								"Options d'alert", 1)
 SafeAddString(LIBRARIAN_SETTINGS_ALERT_TOOLTIP,						"Comment souhaitez-vous être alerté ?", 1)
 SafeAddString(LIBRARIAN_SETTINGS_RELOADUI_REMINDER,					"Rappel 'ReloadUI' après", 1)
-SafeAddString(LIBRARIAN_SETTINGS_RELOADUI_REMINDER_TOOLTIP,			"Rappel pour lancer la commande /reloadui après que ce nombre de livre ait été découverts.", 1)
+SafeAddString(LIBRARIAN_SETTINGS_RELOADUI_REMINDER_TOOLTIP,			"Rappel pour lancer la commande /reloadui après que ce nombre de livre ait été découvert.", 1)
 SafeAddString(LIBRARIAN_SETTINGS_SHOW_HIDDEN_BOOK, 					"Montrer les livres cachés", 1)
 SafeAddString(LIBRARIAN_SETTINGS_SHOW_HIDDEN_BOOK_TOOLTIP, 			"TESO cache certaines collections de livres dans la bibliothèque. Par example les livres contenant le motif complet (au lieu des 14 pages) font parti de ces livres cachés. Donc, si vous souhaitez que le nombre de livre affiché dans Librarian corresponde à celui de la Bibliothèque, vous devez décocher cette case.", 1)
 SafeAddString(LIBRARIAN_SETTINGS_UNREAD_INDICATOR_READER,			"Icone 'Non-lu' (Liseuse)", 1)
 SafeAddString(LIBRARIAN_SETTINGS_UNREAD_INDICATOR_READER_TOOLTIP,	"Affiche une icone 'Non-lu' à côté du titre lors de la lecture d'un livre.", 1)
 SafeAddString(LIBRARIAN_SETTINGS_ICON_TRANSPARENCY,					"Transparence de l'icone", 1)
 SafeAddString(LIBRARIAN_SETTINGS_ICON_TRANSPARENCY_TOOLTIP,			"A quel point souhaitez-vous que l'icone 'Non-lu' soit transparente (100 complètement visible, 0 complètement transparente).", 1)
-SafeAddString(LIBRARIAN_SETTINGS_UNREAD_INDICATOR_LIBRARY,			"Icone 'Non-lu' (Bibliothèque)",
-SafeAddString(LIBRARIAN_SETTINGS_UNREAD_INDICATOR_LIBRARY_TOOLTIP,	"Affiche une icone 'Non-lu' dans la bibliothèque à côté des collections contenant un livre 'Non-lu' et à côté de chaque livre 'Non-lu'.",
+SafeAddString(LIBRARIAN_SETTINGS_UNREAD_INDICATOR_LIBRARY,			"Icone 'Non-lu' (Bibliothèque)", 1)
+SafeAddString(LIBRARIAN_SETTINGS_UNREAD_INDICATOR_LIBRARY_TOOLTIP,	"Affiche une icone 'Non-lu' dans la Bibliothèque à côté des collections contenant au moins un livre 'Non-lu' et à côté de chaque livre 'Non-lu'.", 1)
 SafeAddString(LIBRARIAN_SETTINGS_CHARACTER_SPIN,					"Tourner le personnage", 1)
 SafeAddString(LIBRARIAN_SETTINGS_CHARACTER_SPIN_TOOLTIP,			"Tourne le personnage pour qu'il soit face à la camera quand Librarian est ouvert.", 1)
-SafeAddString(LIBRARIAN_SETTINGS_IMPORT,							"Importer de Lore Library", 1)
-SafeAddString(LIBRARIAN_SETTINGS_IMPORT_TOOLTIP,					"Importer tous les livres de la Lore Library.  Fonctionne avec tous les livres lorsque la mémoire eidétique est débloquée.", 1)
+SafeAddString(LIBRARIAN_SETTINGS_IMPORT,							"Importer de la Bibliothèque", 1)
+SafeAddString(LIBRARIAN_SETTINGS_IMPORT_TOOLTIP,					"Importer tous les livres de la Bibliothèque.  Fonctionne avec tous les livres lorsque la mémoire eidétique est débloquée.", 1)

 SafeAddString(LIBRARIAN_SETTINGS_TIME_12,							"12 heures", 1)
 SafeAddString(LIBRARIAN_SETTINGS_TIME_24,							"24 heures", 1)