Add configurable chunked storage limits

2026-07-27 14:06:14 +00:00
parent dca86ae393
commit 8f21874492
5 changed files with 306 additions and 95 deletions
@@ -16,6 +16,8 @@ import se.ajpanton.notificationlog.data.EncryptedNotificationLogStore
 import se.ajpanton.notificationlog.data.EncryptedImageStore
 import se.ajpanton.notificationlog.capture.NotificationCaptureService
 import se.ajpanton.notificationlog.export.LogExporter
+import se.ajpanton.notificationlog.settings.StorageLimits
+import se.ajpanton.notificationlog.settings.StorageLimitsStore
 import java.util.zip.ZipOutputStream

 class SettingsFragment : Fragment(R.layout.fragment_settings) {
@@ -33,6 +35,7 @@ class SettingsFragment : Fragment(R.layout.fragment_settings) {
        val lockStore = AppLockStore(requireContext())
        binding!!.appLock.isChecked = lockStore.enabled
        binding!!.appLock.setOnCheckedChangeListener { _, enabled -> lockStore.enabled = enabled }
+        bindStorageLimits()
    }
    override fun onDestroyView() { binding = null; super.onDestroyView() }

@@ -44,6 +47,27 @@ class SettingsFragment : Fragment(R.layout.fragment_settings) {
    private fun isListenerEnabled(): Boolean = requireContext().getSystemService(android.app.NotificationManager::class.java)
        .isNotificationListenerAccessGranted(ComponentName(requireContext(), NotificationCaptureService::class.java))

+    private fun bindStorageLimits() {
+        val store = StorageLimitsStore(requireContext())
+        val limits = store.load()
+        binding!!.logLimitMib.setText((limits.logBytes / StorageLimitsStore.MEBIBYTE).toString())
+        binding!!.imageLimitMib.setText((limits.imageBytes / StorageLimitsStore.MEBIBYTE).toString())
+        binding!!.saveStorageLimits.setOnClickListener {
+            val logLimit = binding!!.logLimitMib.text.toString().toLongOrNull()
+            val imageLimit = binding!!.imageLimitMib.text.toString().toLongOrNull()
+            if (logLimit == null || imageLimit == null ||
+                logLimit !in 1..MAXIMUM_LIMIT_MIB || imageLimit !in 1..MAXIMUM_LIMIT_MIB
+            ) {
+                binding!!.logLimitMib.error = "Enter 1 to $MAXIMUM_LIMIT_MIB MiB"
+                binding!!.imageLimitMib.error = "Enter 1 to $MAXIMUM_LIMIT_MIB MiB"
+                return@setOnClickListener
+            }
+            store.save(StorageLimits(logLimit * StorageLimitsStore.MEBIBYTE, imageLimit * StorageLimitsStore.MEBIBYTE))
+            val appContext = requireContext().applicationContext
+            Thread { EncryptedNotificationLogStore(appContext).enforceLimits() }.start()
+        }
+    }
+
    private fun confirmClearLogs() {
        MaterialAlertDialogBuilder(requireContext())
            .setTitle("Clear logs?")
@@ -104,4 +128,8 @@ class SettingsFragment : Fragment(R.layout.fragment_settings) {
    }

    private enum class ExportFormat(val extension: String) { CSV("csv"), FORMATTED("txt"), HTML_ZIP("zip") }
+
+    private companion object {
+        const val MAXIMUM_LIMIT_MIB = 1024L * 1024L
+    }
 }
@@ -25,23 +25,32 @@ import androidx.fragment.app.Fragment
 class ViewLogsFragment : Fragment(R.layout.fragment_view_logs) {
    private var binding: FragmentViewLogsBinding? = null
    private val expandedIds = mutableSetOf<String>()
+    private val rows = mutableListOf<LogRow>()
+    private var loadedEntries = 0
+    private var loading = false
+    private var noMoreRows = false
+    private var loadGeneration = 0

    override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
        super.onViewCreated(view, savedInstanceState)
        binding = FragmentViewLogsBinding.bind(view)
-        binding!!.logsRefresh.setOnRefreshListener(::loadLogs)
+        binding!!.logsRefresh.setOnRefreshListener { loadLogs(reset = true) }
        binding!!.logScroll.setOnScrollChangeListener { _, _, scrollY, _, _ ->
            binding?.scrollToTop?.visibility = if (scrollY > dp(SCROLL_TO_TOP_THRESHOLD_DP)) View.VISIBLE else View.GONE
+            val scroll = binding?.logScroll ?: return@setOnScrollChangeListener
+            if (!noMoreRows && !loading && scrollY >= scroll.getChildAt(0).height - scroll.height - dp(LOAD_MORE_THRESHOLD_DP)) {
+                loadLogs(reset = false)
+            }
        }
        binding!!.scrollToTop.setOnClickListener {
            binding?.logScroll?.smoothScrollTo(0, 0)
        }
-        loadLogs()
+        loadLogs(reset = true)
    }

    override fun onResume() {
        super.onResume()
-        loadLogs()
+        loadLogs(reset = true)
    }

    override fun onDestroyView() {
@@ -49,26 +58,40 @@ class ViewLogsFragment : Fragment(R.layout.fragment_view_logs) {
        super.onDestroyView()
    }

-    private fun loadLogs() {
+    private fun loadLogs(reset: Boolean) {
+        if (loading) return
+        if (!reset && noMoreRows) return
        val context = requireContext().applicationContext
+        if (reset) {
+            rows.clear()
+            loadedEntries = 0
+            noMoreRows = false
+            loadGeneration++
+        }
+        val generation = loadGeneration
+        loading = true
        binding?.logsRefresh?.isRefreshing = true
        Thread {
-            val imageStore = EncryptedImageStore(context)
-            val rows = EncryptedNotificationLogStore(context).readAll()
-                .sortedByDescending { it.recordedAtEpochMillis }
-                .map { entry -> LogRow(entry, entry.imageId?.let(imageStore::read)) }
+            val entries = EncryptedNotificationLogStore(context).readNewest(loadedEntries, PAGE_SIZE)
            activity?.runOnUiThread {
-                binding?.let { render(it, rows) }
+                if (generation == loadGeneration) {
+                    loadedEntries += entries.size
+                    noMoreRows = entries.size < PAGE_SIZE
+                    val addedRows = entries.map(::LogRow)
+                    rows += addedRows
+                    binding?.let { render(it, addedRows, reset) }
+                }
+                loading = false
                binding?.logsRefresh?.isRefreshing = false
            }
        }.start()
    }

-    private fun render(view: FragmentViewLogsBinding, rows: List<LogRow>) {
-        view.logRows.removeAllViews()
+    private fun render(view: FragmentViewLogsBinding, addedRows: List<LogRow>, reset: Boolean) {
+        if (reset) view.logRows.removeAllViews()
        view.emptyView.visibility = if (rows.isEmpty()) View.VISIBLE else View.GONE
        val settings = LogViewSettingsStore(requireContext()).load()
-        rows.filter { eventFor(it.entry.action) in settings.visibleEvents }
+        addedRows.filter { eventFor(it.entry.action) in settings.visibleEvents }
            .forEach { row -> view.logRows.addView(logRowView(row, settings)) }
    }

@@ -159,18 +182,23 @@ class ViewLogsFragment : Fragment(R.layout.fragment_view_logs) {
                setPadding(0, dp(4), 0, 0)
            })
        }
-        if (expanded) {
-            row.imageBytes?.let { bytes ->
-                BitmapFactory.decodeByteArray(bytes, 0, bytes.size)?.let { bitmap ->
-                    addView(ImageView(context).apply {
-                        setImageBitmap(bitmap)
-                        adjustViewBounds = true
-                        maxHeight = dp(240)
-                        contentDescription = "Notification image"
-                        setPadding(0, dp(4), 0, 0)
-                    })
-                }
+        if (expanded && row.entry.imageId != null) {
+            val image = ImageView(context).apply {
+                adjustViewBounds = true
+                maxHeight = dp(240)
+                contentDescription = "Notification image"
+                setPadding(0, dp(4), 0, 0)
            }
+            addView(image)
+            val imageId = row.entry.imageId
+            val appContext = requireContext().applicationContext
+            Thread {
+                EncryptedImageStore(appContext).read(imageId)?.let { bytes ->
+                    BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
+                }?.let { bitmap ->
+                    activity?.runOnUiThread { if (image.isAttachedToWindow) image.setImageBitmap(bitmap) }
+                }
+            }.start()
        }
        setOnClickListener { onToggleExpanded() }
        setOnLongClickListener { showDeleteDialog(row.entry.id) }
@@ -225,7 +253,7 @@ class ViewLogsFragment : Fragment(R.layout.fragment_view_logs) {
            .setPositiveButton("Delete") { _, _ ->
                Thread {
                    EncryptedNotificationLogStore(requireContext().applicationContext).delete(entryId)
-                    activity?.runOnUiThread(::loadLogs)
+                    activity?.runOnUiThread { loadLogs(reset = true) }
                }.start()
            }
            .show()
@@ -244,12 +272,14 @@ class ViewLogsFragment : Fragment(R.layout.fragment_view_logs) {

    private fun dp(value: Int): Int = (value * resources.displayMetrics.density).toInt()

-    private data class LogRow(val entry: NotificationLogEntry, val imageBytes: ByteArray?)
+    private data class LogRow(val entry: NotificationLogEntry)
    private data class MetadataValue(val text: String, val textSize: Float, val indented: Boolean = false, val bold: Boolean = false)

    private companion object {
        const val LEFT_PILL_WEIGHT = 0.42f
        const val RIGHT_PILL_WEIGHT = 0.58f
        const val SCROLL_TO_TOP_THRESHOLD_DP = 120
+        const val LOAD_MORE_THRESHOLD_DP = 480
+        const val PAGE_SIZE = 80
    }
 }
@@ -3,101 +3,152 @@ package se.ajpanton.notificationlog.data
 import android.content.Context
 import android.util.AtomicFile
 import se.ajpanton.notificationlog.model.NotificationLogEntry
+import se.ajpanton.notificationlog.settings.StorageLimitsStore
 import java.io.BufferedInputStream
 import java.io.BufferedOutputStream
 import java.io.DataInputStream
 import java.io.DataOutputStream
-import java.io.FileNotFoundException
 import java.io.File
+import java.io.FileNotFoundException

 /**
- * An atomically replaced encrypted event log. It keeps every persisted log field
- * encrypted, including timestamps and package names, while the app is at rest.
+ * Independently encrypted log chunks. Reading or appending a record needs only
+ * one bounded chunk; older chunks stay on disk until requested by the viewer.
 */
 class EncryptedNotificationLogStore(context: Context) {
-    private val lock = Any()
-    private val file = AtomicFile(context.filesDir.resolve(FILE_NAME))
-    private val imageDirectory = File(context.filesDir, IMAGE_DIRECTORY_NAME)
+    private val appContext = context.applicationContext
+    private val directory = File(appContext.filesDir, DIRECTORY_NAME).also(File::mkdirs)
+    private val imageDirectory = File(appContext.filesDir, IMAGE_DIRECTORY_NAME)
    private val cipher = AesGcmCipher(LogEncryptionKeyProvider().getOrCreate())

+    init {
+        // This development build deliberately does not retain the former monolithic format.
+        File(appContext.filesDir, LEGACY_FILE_NAME).delete()
+    }
+
    fun readAll(): List<NotificationLogEntry> = synchronized(lock) {
-        val payload = readPayloadOrNull() ?: return emptyList()
-        NotificationLogEntryJson.decode(cipher.decrypt(payload))
+        chunkFiles().flatMap(::readChunk)
+    }
+
+    fun readNewest(offset: Int, limit: Int): List<NotificationLogEntry> = synchronized(lock) {
+        require(offset >= 0)
+        require(limit > 0)
+        val result = ArrayList<NotificationLogEntry>(limit)
+        var skipped = 0
+        for (chunk in chunkFiles().asReversed()) {
+            readChunk(chunk).asReversed().forEach { entry ->
+                if (skipped < offset) {
+                    skipped++
+                } else if (result.size < limit) {
+                    result += entry
+                }
+            }
+            if (result.size == limit) break
+        }
+        result.sortedByDescending { it.recordedAtEpochMillis }
    }

    fun append(entry: NotificationLogEntry) = synchronized(lock) {
-        val entries = (readPayloadOrNull()?.let { NotificationLogEntryJson.decode(cipher.decrypt(it)) } ?: emptyList())
-        val retained = retain(entries + entry)
-        write(retained)
-        (entries.mapNotNull { it.imageId } - retained.mapNotNull { it.imageId }.toSet()).forEach(::deleteImage)
+        val chunks = chunkFiles()
+        val newest = chunks.lastOrNull()
+        val current = newest?.let(::readChunk).orEmpty()
+        if (newest == null || encodedSize(current + entry) > MAX_CHUNK_PLAINTEXT_BYTES) {
+            writeChunk(nextChunkFile(chunks), listOf(entry))
+        } else {
+            writeChunk(newest, current + entry)
+        }
+        enforceLimits()
    }

    fun replaceAll(entries: List<NotificationLogEntry>) = synchronized(lock) {
-        val existing = readPayloadOrNull()?.let { NotificationLogEntryJson.decode(cipher.decrypt(it)) } ?: emptyList()
-        write(entries)
-        removeUnreferencedImages(entries, existing)
+        clearChunksOnly()
+        entries.chunked(MAX_ENTRIES_PER_CHUNK).forEachIndexed { index, chunk ->
+            writeChunk(chunkFile(index.toLong()), chunk)
+        }
+        enforceLimits()
    }

    fun clear() = synchronized(lock) {
-        file.delete()
+        clearChunksOnly()
        imageDirectory.listFiles()?.forEach(File::delete)
    }

    fun delete(id: String): Boolean = synchronized(lock) {
-        val entries = readPayloadOrNull()?.let { NotificationLogEntryJson.decode(cipher.decrypt(it)) } ?: return false
-        val entry = entries.firstOrNull { it.id == id } ?: return false
-        write(entries.filterNot { it.id == id })
-        entry.imageId?.let(::deleteImage)
-        true
+        chunkFiles().forEach { chunk ->
+            val entries = readChunk(chunk)
+            val entry = entries.firstOrNull { it.id == id } ?: return@forEach
+            val retained = entries.filterNot { it.id == id }
+            if (retained.isEmpty()) chunk.delete() else writeChunk(chunk, retained)
+            entry.imageId?.let(::deleteImage)
+            return true
+        }
+        false
    }

-    /** Removes failed-write remnants and images no remaining log entry references. */
+    /** Removes failed-write remnants and images no current log entry references. */
    fun removeOrphanedImages() = synchronized(lock) {
-        val entries = readPayloadOrNull()?.let { NotificationLogEntryJson.decode(cipher.decrypt(it)) } ?: emptyList()
-        removeUnreferencedImages(entries)
+        if (imageDirectory.listFiles().isNullOrEmpty()) return
+        val referenced = mutableSetOf<String>()
+        chunkFiles().forEach { chunk ->
+            readChunk(chunk).mapNotNullTo(referenced) { it.imageId }
+        }
+        removeUnreferencedImages(referenced)
    }

-    private fun write(entries: List<NotificationLogEntry>) {
-        val payload = cipher.encrypt(NotificationLogEntryJson.encode(entries))
-        val output = file.startWrite()
-        try {
-            val stream = DataOutputStream(BufferedOutputStream(output))
-            stream.writeInt(FILE_VERSION)
-            stream.writeInt(payload.initializationVector.size)
-            stream.write(payload.initializationVector)
-            stream.writeInt(payload.cipherText.size)
-            stream.write(payload.cipherText)
-            stream.flush()
-            file.finishWrite(output)
-        } catch (error: Exception) {
-            file.failWrite(output)
-            throw error
+    fun enforceLimits() = synchronized(lock) {
+        val limits = StorageLimitsStore(appContext).load()
+        trimLogBytes(limits.logBytes)
+        trimImageBytes(limits.imageBytes)
+    }
+
+    private fun trimLogBytes(limit: Long) {
+        var chunks = chunkFiles()
+        var totalBytes = chunks.sumOf(File::length)
+        while (totalBytes > limit && chunks.isNotEmpty()) {
+            val oldest = chunks.first()
+            val entries = readChunk(oldest)
+            if (entries.isEmpty()) {
+                oldest.delete()
+            } else if (chunks.size > 1) {
+                entries.mapNotNull { it.imageId }.forEach(::deleteImage)
+                oldest.delete()
+            } else {
+                var retained = entries
+                while (totalBytes > limit && retained.isNotEmpty()) {
+                    retained.first().imageId?.let(::deleteImage)
+                    retained = retained.drop(1)
+                    if (retained.isEmpty()) oldest.delete() else writeChunk(oldest, retained)
+                    totalBytes = chunkFiles().sumOf(File::length)
+                }
+            }
+            chunks = chunkFiles()
+            totalBytes = chunks.sumOf(File::length)
        }
    }

-    private fun retain(entries: List<NotificationLogEntry>): List<NotificationLogEntry> {
-        var retained = entries.takeLast(MAX_ENTRIES)
-        while (retained.size > 1 && NotificationLogEntryJson.encode(retained).size > MAX_PLAINTEXT_BYTES) {
-            retained = retained.drop(1)
+    private fun trimImageBytes(limit: Long) {
+        var imageBytes = imageDirectory.listFiles()?.filter(File::isFile)?.sumOf(File::length) ?: 0L
+        if (imageBytes <= limit) return
+        chunkFiles().forEach { chunk ->
+            if (imageBytes <= limit) return@forEach
+            val entries = readChunk(chunk)
+            var changed = false
+            val retained = entries.map { entry ->
+                if (imageBytes > limit && entry.imageId != null) {
+                    val image = File(imageDirectory, "${entry.imageId}.bin")
+                    imageBytes -= image.length()
+                    image.delete()
+                    changed = true
+                    entry.copy(imageId = null)
+                } else {
+                    entry
+                }
+            }
+            if (changed) writeChunk(chunk, retained)
        }
-        require(NotificationLogEntryJson.encode(retained).size <= MAX_PLAINTEXT_BYTES) {
-            "A notification event exceeds the encrypted log retention limit."
-        }
-        return retained
    }

-    private fun deleteImage(id: String) {
-        if (id.matches(IMAGE_ID_PATTERN)) File(imageDirectory, "$id.bin").delete()
-    }
-
-    private fun removeUnreferencedImages(
-        entries: List<NotificationLogEntry>,
-        previousEntries: List<NotificationLogEntry> = emptyList(),
-    ) {
-        val referenced = entries.mapNotNull { it.imageId }.toSet()
-        previousEntries.mapNotNull { it.imageId }
-            .filterNot(referenced::contains)
-            .forEach(::deleteImage)
+    private fun removeUnreferencedImages(referenced: Set<String>) {
        imageDirectory.listFiles()?.forEach { image ->
            val imageId = image.name.removeSuffix(".bin")
            if (image.name.endsWith(".new") ||
@@ -110,17 +161,48 @@ class EncryptedNotificationLogStore(context: Context) {
        }
    }

-    private fun readPayloadOrNull(): EncryptedPayload? {
+    private fun chunkFiles(): List<File> = directory.listFiles()
+        ?.filter { it.isFile && it.name.startsWith(CHUNK_PREFIX) && it.name.endsWith(CHUNK_SUFFIX) }
+        ?.sortedBy { it.name }
+        ?: emptyList()
+
+    private fun nextChunkFile(chunks: List<File>): File {
+        val next = chunks.lastOrNull()?.name?.removePrefix(CHUNK_PREFIX)?.removeSuffix(CHUNK_SUFFIX)?.toLongOrNull()?.plus(1) ?: 0L
+        return chunkFile(next)
+    }
+
+    private fun chunkFile(sequence: Long): File = File(directory, "$CHUNK_PREFIX${sequence.toString().padStart(CHUNK_NAME_WIDTH, '0')}$CHUNK_SUFFIX")
+
+    private fun readChunk(file: File): List<NotificationLogEntry> {
        val input = try {
-            file.openRead()
+            AtomicFile(file).openRead()
        } catch (_: FileNotFoundException) {
-            return null
+            return emptyList()
        }
        DataInputStream(BufferedInputStream(input)).use { stream ->
-            require(stream.readInt() == FILE_VERSION) { "Unsupported encrypted notification-log version." }
+            require(stream.readInt() == FILE_VERSION) { "Unsupported encrypted notification-log chunk." }
            val initializationVector = readByteArray(stream, MAX_IV_BYTES)
-            val cipherText = readByteArray(stream, MAX_CIPHER_TEXT_BYTES)
-            return EncryptedPayload(initializationVector, cipherText)
+            val cipherText = readByteArray(stream, MAX_CHUNK_CIPHER_TEXT_BYTES)
+            return NotificationLogEntryJson.decode(cipher.decrypt(EncryptedPayload(initializationVector, cipherText)))
+        }
+    }
+
+    private fun writeChunk(file: File, entries: List<NotificationLogEntry>) {
+        val payload = cipher.encrypt(NotificationLogEntryJson.encode(entries))
+        val atomicFile = AtomicFile(file)
+        val output = atomicFile.startWrite()
+        try {
+            val stream = DataOutputStream(BufferedOutputStream(output))
+            stream.writeInt(FILE_VERSION)
+            stream.writeInt(payload.initializationVector.size)
+            stream.write(payload.initializationVector)
+            stream.writeInt(payload.cipherText.size)
+            stream.write(payload.cipherText)
+            stream.flush()
+            atomicFile.finishWrite(output)
+        } catch (error: Exception) {
+            atomicFile.failWrite(output)
+            throw error
        }
    }

@@ -130,14 +212,29 @@ class EncryptedNotificationLogStore(context: Context) {
        return ByteArray(length).also(stream::readFully)
    }

+    private fun encodedSize(entries: List<NotificationLogEntry>) = NotificationLogEntryJson.encode(entries).size
+
+    private fun clearChunksOnly() {
+        chunkFiles().forEach(File::delete)
+    }
+
+    private fun deleteImage(id: String) {
+        if (id.matches(IMAGE_ID_PATTERN)) File(imageDirectory, "$id.bin").delete()
+    }
+
    private companion object {
-        const val FILE_NAME = "notification-log.v1"
+        val lock = Any()
+        const val DIRECTORY_NAME = "notification-log"
+        const val LEGACY_FILE_NAME = "notification-log.v1"
+        const val IMAGE_DIRECTORY_NAME = "notification-images"
+        const val CHUNK_PREFIX = "chunk-"
+        const val CHUNK_SUFFIX = ".bin"
+        const val CHUNK_NAME_WIDTH = 12
        const val FILE_VERSION = 1
        const val MAX_IV_BYTES = 32
-        const val MAX_CIPHER_TEXT_BYTES = 50 * 1024 * 1024
-        const val MAX_PLAINTEXT_BYTES = 20 * 1024 * 1024
-        const val MAX_ENTRIES = 10_000
-        const val IMAGE_DIRECTORY_NAME = "notification-images"
+        const val MAX_CHUNK_CIPHER_TEXT_BYTES = 1024 * 1024
+        const val MAX_CHUNK_PLAINTEXT_BYTES = 256 * 1024
+        const val MAX_ENTRIES_PER_CHUNK = 500
        val IMAGE_ID_PATTERN = Regex("[0-9a-f-]{36}")
    }
 }
@@ -0,0 +1,30 @@
+package se.ajpanton.notificationlog.settings
+
+import android.content.Context
+import androidx.core.content.edit
+
+data class StorageLimits(
+    val logBytes: Long = 5 * 1024L * 1024L,
+    val imageBytes: Long = 100 * 1024L * 1024L,
+)
+
+class StorageLimitsStore(context: Context) {
+    private val preferences = context.applicationContext.getSharedPreferences(FILE_NAME, Context.MODE_PRIVATE)
+
+    fun load() = StorageLimits(
+        logBytes = preferences.getLong(LOG_BYTES, StorageLimits().logBytes),
+        imageBytes = preferences.getLong(IMAGE_BYTES, StorageLimits().imageBytes),
+    )
+
+    fun save(limits: StorageLimits) = preferences.edit {
+        putLong(LOG_BYTES, limits.logBytes)
+        putLong(IMAGE_BYTES, limits.imageBytes)
+    }
+
+    companion object {
+        const val MEBIBYTE = 1024L * 1024L
+        const val FILE_NAME = "storage-limits"
+        const val LOG_BYTES = "log_bytes"
+        const val IMAGE_BYTES = "image_bytes"
+    }
+}