feat(Logger): Use IndexedDB to store logs in order to store more

Signed-off-by: Marcel Klehr <mklehr@gmx.net>

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "floccus",
-  "version": "5.4.4",
+  "version": "5.4.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "floccus",
-      "version": "5.4.4",
+      "version": "5.4.5",
       "license": "MPL-2.0",
       "dependencies": {
         "@byteowls/capacitor-oauth2": "5.x",
@@ -30,6 +30,7 @@
         "buffer": "^6.0.3",
         "cheerio": "^1.0.0-rc.12",
         "core-js": "3.x",
+        "dexie": "^4.0.11",
         "fast-xml-parser": "^4.2.7",
         "humanize-duration": "^3.25.1",
         "intl-messageformat": "^9.9.1",
@@ -6410,6 +6411,12 @@
         "node": ">=0.10"
       }
     },
+    "node_modules/dexie": {
+      "version": "4.0.11",
+      "resolved": "https://registry.npmjs.org/dexie/-/dexie-4.0.11.tgz",
+      "integrity": "sha512-SOKO002EqlvBYYKQSew3iymBoN2EQ4BDw/3yprjh7kAfFzjBYkaMNa/pZvcA7HSWlcKSQb9XhPe3wKyQ0x4A8A==",
+      "license": "Apache-2.0"
+    },
     "node_modules/didyoumean": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz",

package.json

@@ -97,6 +97,7 @@
     "buffer": "^6.0.3",
     "cheerio": "^1.0.0-rc.12",
     "core-js": "3.x",
+    "dexie": "^4.0.11",
     "fast-xml-parser": "^4.2.7",
     "humanize-duration": "^3.25.1",
     "intl-messageformat": "^9.9.1",

src/lib/Account.ts

@@ -198,7 +198,6 @@ export default class Account {
               Logger.log(
                 'Resource is locked, trying again soon'
               )
-              await Logger.persist()
               return
             }
           } else {
@@ -348,7 +347,6 @@ export default class Account {
         await this.init()
       }
     }
-    await Logger.persist()
   }
 
   static async stringifyError(er:any):Promise<string> {

src/lib/IndexedDB.ts

@@ -0,0 +1,43 @@
+import Dexie, { type EntityTable } from 'dexie'
+
+interface LogMessage {
+  id: number;
+  dateTime: number;
+  message: string;
+}
+
+const db = new Dexie('floccus') as Dexie & {
+  logs: EntityTable<
+    LogMessage,
+    'id' // primary key "id" (for the typings only)
+  >;
+}
+
+db.version(1).stores({
+  logs: '++id, dateTime, message'
+})
+
+export { db }
+export { LogMessage }
+
+export async function freeStorageIfNecessary() {
+  if (navigator.storage && navigator.storage.estimate) {
+    let {usage, quota} = await navigator.storage.estimate()
+    if (usage / quota > 0.9) {
+      const oneWeekAgo = Date.now() - 60 * 60 * 1000 * 24 * 7
+
+      await db.logs
+        .where('dateTime').below(oneWeekAgo)
+        .delete()
+    }
+
+    ({usage, quota} = await navigator.storage.estimate())
+    if (usage / quota > 0.6) {
+      const oneDayAgo = Date.now() - 60 * 60 * 1000 * 24
+
+      await db.logs
+        .where('dateTime').below(oneDayAgo)
+        .delete()
+    }
+  }
+}

src/lib/Logger.js

@@ -6,38 +6,31 @@ import Crypto from './Crypto'
 import { Share } from '@capacitor/share'
 import { Filesystem, Directory, Encoding } from '@capacitor/filesystem'
 import { Capacitor } from '@capacitor/core'
+import { db } from './IndexedDB'
 
 export default class Logger {
   static log() {
-    const logMsg = [new Date().toISOString(), ...arguments]
+    const dateTime = Date.now()
+    const logMsg = [...arguments]
+    const message = util.format.apply(util, logMsg)
 
     // log to console
     DEBUG && console.log(util.format.apply(util, logMsg))
-    this.messages.push(util.format.apply(util, logMsg)) // TODO: Use a linked list here to get O(n)
-  }
-
-  static async persist() {
-    const Storage = (Capacitor.getPlatform() === 'web') ? await import('./browser/BrowserAccountStorage') : await import('./native/NativeAccountStorage')
-    await Storage.default.changeEntry(
-      'logs',
-      log => {
-        const messages = this.messages
-        this.messages = []
-        return messages // only save the last sync run
-      },
-      []
-    )
+    db.logs.add({dateTime, message})
+      .catch(e => {
+        console.error('Failed to log to IndexedDB: ', e)
+        console.error(e)
+      })
   }
 
   static async getLogs() {
-    const Storage = (Capacitor.getPlatform() === 'web') ? await import('./browser/BrowserAccountStorage') : await import('./native/NativeAccountStorage')
-    return Storage.default.getEntry('logs', [])
+    return db.logs.orderBy('dateTime').toArray()
   }
 
   static async anonymizeLogs(logs) {
     const regex = /\[(.*?)\]\((.*?)\)|\[(.*?)\]/g
-    const newLogs = await Parallel.map(logs, async(entry) => {
-      return Logger.replaceAsync(entry, regex, async(match, p1, p2, p3) => {
+    await Parallel.map(logs, async(logMessage) => {
+      logMessage.message = await Logger.replaceAsync(logMessage.message, regex, async(match, p1, p2, p3) => {
         if (p1 && p2) {
           const hash1 = await Crypto.sha256(p1)
           const hash2 = await Crypto.sha256(p2)
@@ -50,8 +43,11 @@ export default class Logger {
     }, 1)
     const regex2 = /url=https?%3A%2F%2F.*$|url=https?%3A%2F%2F[^ ]*/
     const regex3 = /https?:\/\/[^ /]*\//
-    return newLogs
-      .map(line => line.replace(regex2, '###url###').replace(regex3, '###server###'))
+    logs
+      .forEach(logMessage => {
+        logMessage.message = logMessage.message.replace(regex2, '###url###').replace(regex3, '###server###')
+      })
+    return logs
   }
 
   static async replaceAsync(str, regex, asyncFn) {
@@ -75,7 +71,12 @@ export default class Logger {
     if (anonymous) {
       logs = await Logger.anonymizeLogs(logs)
     }
-    let blob = new Blob([logs.join('\n')], {
+    logs = logs
+      .map(logMessage => {
+        return new Date(logMessage.dateTime).toISOString() + ' ' + logMessage.message
+      })
+      .join('\n')
+    let blob = new Blob([logs], {
       type: 'text/plain',
       endings: 'native'
     })
@@ -121,4 +122,3 @@ export default class Logger {
     }
   }
 }
-Logger.messages = []

src/lib/browser/BrowserController.js

@@ -9,6 +9,7 @@ import uniqBy from 'lodash/uniqBy'
 import Account from '../Account'
 import { STATUS_ALLGOOD, STATUS_DISABLED, STATUS_ERROR, STATUS_SYNCING } from '../interfaces/Controller'
 import * as Sentry from '@sentry/browser'
+import { freeStorageIfNecessary } from '../IndexedDB'
 
 const INACTIVITY_TIMEOUT = 7 * 1000 // 7 seconds
 const MAX_BACKOFF_INTERVAL = 1000 * 60 * 60 // 1 hour
@@ -121,6 +122,17 @@ export default class BrowserController {
       }
     })
 
+    // Remove old logs
+
+    BrowserAccountStorage.changeEntry(
+      'logs',
+      log => {
+        return []
+      },
+      []
+    )
+    freeStorageIfNecessary()
+
     // do some cleaning if this is a new version
 
     browser.storage.local.get(['currentVersion', 'lastInterventionAt']).then(async d => {

src/lib/native/NativeController.js

@@ -4,6 +4,7 @@ import Cryptography from '../Crypto'
 import NativeAccountStorage from './NativeAccountStorage'
 import Account from '../Account'
 import { STATUS_ALLGOOD, STATUS_DISABLED, STATUS_ERROR, STATUS_SYNCING } from '../interfaces/Controller'
+import { freeStorageIfNecessary } from '../IndexedDB'
 
 const INACTIVITY_TIMEOUT = 1000 * 7
 const MAX_BACKOFF_INTERVAL = 1000 * 60 * 60 // 1 hour
@@ -83,6 +84,17 @@ export default class NativeController {
 
     this.alarms = new AlarmManager(this)
 
+    // Remove old logs
+
+    NativeAccountStorage.changeEntry(
+      'logs',
+      log => {
+        return []
+      },
+      []
+    )
+    freeStorageIfNecessary()
+
     // lock accounts when locking is enabled
 
     Storage.get({key: 'accountsLocked' }).then(async({value: accountsLocked}) => {