Stream notification log exports

app/src/main/java/se/ajpanton/notificationlog/SettingsFragment.kt

@@ -99,34 +99,80 @@ class SettingsFragment : Fragment(R.layout.fragment_settings) {
         val format = pendingExport ?: return
         val context = requireContext().applicationContext
         Thread {
-            val entries = EncryptedNotificationLogStore(context).readAll()
+            val logStore = EncryptedNotificationLogStore(context)
             val settings = LogViewSettingsStore(context).load()
             context.contentResolver.openOutputStream(uri)?.use { output ->
                 when (format) {
-                    ExportFormat.CSV -> output.write(LogExporter.csv(entries, settings).encodeToByteArray())
+                    ExportFormat.CSV -> writeCsv(output.bufferedWriter(Charsets.UTF_8), logStore, settings)
-                    ExportFormat.FORMATTED -> output.write(LogExporter.formatted(entries, settings).encodeToByteArray())
+                    ExportFormat.FORMATTED -> writeFormatted(output.bufferedWriter(Charsets.UTF_8), logStore, settings)
-                    ExportFormat.HTML_ZIP -> ZipOutputStream(output).use { zip ->
+                    ExportFormat.HTML_ZIP -> writeHtmlZip(ZipOutputStream(output), logStore, settings, context)
-                        zip.putNextEntry(java.util.zip.ZipEntry("notification-log.html"))
-                        val imageStore = EncryptedImageStore(context)
-                        val imageEntries = entries.filter { it.imageId != null && LogField.CONTENTS in settings.visibleFields }
-                        val exportedImages = imageEntries.mapNotNull { entry ->
-                            imageStore.read(entry.imageId!!)?.let { entry.imageId to it }
-                        }.toMap()
-                        zip.write(LogExporter.html(entries, settings) { entry ->
-                            entry.imageId?.takeIf(exportedImages::containsKey)?.let { "images/$it.png" }
-                        }.encodeToByteArray())
-                        zip.closeEntry()
-                        exportedImages.forEach { (imageId, image) ->
-                            zip.putNextEntry(java.util.zip.ZipEntry("images/$imageId.png"))
-                            zip.write(image)
-                            zip.closeEntry()
-                        }
-                    }
                 }
             }
         }.start()
     }
 
+    private fun writeCsv(
+        writer: java.io.BufferedWriter,
+        logStore: EncryptedNotificationLogStore,
+        settings: se.ajpanton.notificationlog.settings.LogViewSettings,
+    ) = writer.use {
+        it.write(LogExporter.csvHeader(settings))
+        logStore.forEachNewest { entry ->
+            it.newLine()
+            it.write(LogExporter.csvRow(entry, settings))
+        }
+    }
+
+    private fun writeFormatted(
+        writer: java.io.BufferedWriter,
+        logStore: EncryptedNotificationLogStore,
+        settings: se.ajpanton.notificationlog.settings.LogViewSettings,
+    ) = writer.use {
+        val headers = LogExporter.headers(settings)
+        val widths = headers.map { header -> header.length }.toMutableList()
+        logStore.forEachNewest { entry ->
+            LogExporter.values(entry, settings).forEachIndexed { index, value ->
+                widths[index] = maxOf(widths[index], value.length)
+            }
+        }
+        it.write(LogExporter.formattedRow(headers, widths))
+        logStore.forEachNewest { entry ->
+            it.newLine()
+            it.write(LogExporter.formattedRow(LogExporter.values(entry, settings), widths))
+        }
+    }
+
+    private fun writeHtmlZip(
+        zip: ZipOutputStream,
+        logStore: EncryptedNotificationLogStore,
+        settings: se.ajpanton.notificationlog.settings.LogViewSettings,
+        context: android.content.Context,
+    ) = zip.use { zipOutput ->
+        val imageStore = EncryptedImageStore(context)
+        zipOutput.putNextEntry(java.util.zip.ZipEntry("notification-log.html"))
+        zipOutput.write(LogExporter.htmlStart(settings).encodeToByteArray())
+        logStore.forEachNewest { entry ->
+            val imagePath = entry.imageId
+                ?.takeIf { LogField.CONTENTS in settings.visibleFields && imageStore.exists(it) }
+                ?.let { imageId -> "images/$imageId.png" }
+            zipOutput.write(LogExporter.htmlRow(entry, settings, imagePath).encodeToByteArray())
+        }
+        zipOutput.write(LogExporter.htmlEnd().encodeToByteArray())
+        zipOutput.closeEntry()
+
+        if (LogField.CONTENTS in settings.visibleFields) {
+            logStore.forEachNewest { entry ->
+                entry.imageId?.let { imageId ->
+                    imageStore.read(imageId)?.let { image ->
+                        zipOutput.putNextEntry(java.util.zip.ZipEntry("images/$imageId.png"))
+                        zipOutput.write(image)
+                        zipOutput.closeEntry()
+                    }
+                }
+            }
+        }
+    }
+
     private enum class ExportFormat(val extension: String) { CSV("csv"), FORMATTED("txt"), HTML_ZIP("zip") }
 
     private companion object {

app/src/main/java/se/ajpanton/notificationlog/data/EncryptedImageStore.kt

@@ -47,6 +47,11 @@ class EncryptedImageStore(context: Context) {
         }
     }
 
+    fun exists(id: String): Boolean {
+        require(id.matches(IMAGE_ID_PATTERN)) { "Invalid notification image ID." }
+        return File(directory, "$id.bin").isFile
+    }
+
     fun delete(id: String) {
         require(id.matches(IMAGE_ID_PATTERN)) { "Invalid notification image ID." }
         File(directory, "$id.bin").delete()

app/src/main/java/se/ajpanton/notificationlog/data/EncryptedNotificationLogStore.kt

@@ -48,6 +48,22 @@ class EncryptedNotificationLogStore(context: Context) {
         result.sortedByDescending { it.recordedAtEpochMillis }
     }
 
+    /**
+     * Visits the current history newest first without collecting it in memory.
+     * Each encrypted chunk is decrypted while holding the store lock, then its
+     * entries are passed to [action] after releasing the lock so capture is not
+     * held up by a slow export destination.
+     */
+    fun forEachNewest(action: (NotificationLogEntry) -> Unit) {
+        val chunks = synchronized(lock) { chunkFiles().asReversed() }
+        chunks.forEach { chunk ->
+            val entries = synchronized(lock) {
+                if (chunk.exists()) readChunk(chunk).asReversed() else emptyList()
+            }
+            entries.forEach(action)
+        }
+    }
+
     fun append(entry: NotificationLogEntry) = synchronized(lock) {
         val chunks = chunkFiles()
         val newest = chunks.lastOrNull()

app/src/main/java/se/ajpanton/notificationlog/export/LogExporter.kt

@@ -9,43 +9,65 @@ import java.util.Date
 import java.util.TimeZone
 
 object LogExporter {
-    fun csv(entries: List<NotificationLogEntry>, settings: LogViewSettings): String =
+    fun csv(entries: List<NotificationLogEntry>, settings: LogViewSettings): String = (
-        rows(entries, settings).joinToString("\n") { row -> row.joinToString(",") { csvValue(it) } }
+        listOf(csvHeader(settings)) + entries.sortedByDescending { it.recordedAtEpochMillis }.map { csvRow(it, settings) }
+    ).joinToString("\n")
 
     fun formatted(entries: List<NotificationLogEntry>, settings: LogViewSettings): String {
-        val rows = rows(entries, settings)
+        val rows = listOf(headers(settings)) + entries.sortedByDescending { it.recordedAtEpochMillis }.map { values(it, settings) }
         val widths = rows.fold(emptyList<Int>()) { current, row -> row.mapIndexed { i, value -> maxOf(current.getOrElse(i) { 0 }, value.length) } }
-        return rows.joinToString("\n") { row -> row.mapIndexed { i, value -> value.padEnd(widths[i]) }.joinToString("  ") }
+        return rows.joinToString("\n") { formattedRow(it, widths) }
     }
 
+    fun headers(settings: LogViewSettings): List<String> = fields(settings).map(::header)
+
+    fun values(entry: NotificationLogEntry, settings: LogViewSettings): List<String> = fields(settings).map { field -> when (field) {
+        LogField.TIMESTAMP -> timestamp(entry.recordedAtEpochMillis, settings.timestampZone, entry.eventTimeZoneId)
+        LogField.APP_NAME -> entry.appName
+        LogField.PACKAGE_NAME -> entry.packageName
+        LogField.ACTION -> entry.action.name.lowercase().replace('_', ' ')
+        LogField.CONTENTS -> entry.contents ?: ""
+    } }
+
+    fun csvHeader(settings: LogViewSettings): String = csvLine(headers(settings))
+
+    fun csvRow(entry: NotificationLogEntry, settings: LogViewSettings): String = csvLine(values(entry, settings))
+
+    fun formattedRow(values: List<String>, widths: List<Int>): String =
+        values.mapIndexed { index, value -> value.padEnd(widths[index]) }.joinToString("  ")
+
+    fun htmlStart(settings: LogViewSettings): String = buildString {
+        append("<!doctype html><meta charset=\"utf-8\"><table><thead><tr>")
+        fields(settings).forEach { append("<th>${escape(header(it))}</th>") }
+        append("</tr></thead><tbody>")
+    }
+
+    fun htmlRow(
+        entry: NotificationLogEntry,
+        settings: LogViewSettings,
+        imagePath: String? = null,
+    ): String = buildString {
+        append("<tr>")
+        fields(settings).forEach { field -> append("<td>${htmlValue(field, entry, settings, imagePath)}</td>") }
+        append("</tr>")
+    }
+
+    fun htmlEnd(): String = "</tbody></table>"
+
     fun html(
         entries: List<NotificationLogEntry>,
         settings: LogViewSettings,
         imagePath: (NotificationLogEntry) -> String? = { null },
     ): String = buildString {
-        append("<!doctype html><meta charset=\"utf-8\"><table><thead><tr>")
+        append(htmlStart(settings))
-        settings.order.filter { it in settings.visibleFields }.forEach { append("<th>${escape(header(it))}</th>") }
-        append("</tr></thead><tbody>")
         entries.sortedByDescending { it.recordedAtEpochMillis }.forEach { entry ->
-            append("<tr>")
+            append(htmlRow(entry, settings, imagePath(entry)))
-            settings.order.filter { it in settings.visibleFields }.forEach { field ->
-                append("<td>${htmlValue(field, entry, settings, imagePath(entry))}</td>")
-            }
-            append("</tr>")
         }
-        append("</tbody></table>")
+        append(htmlEnd())
     }
 
-    private fun rows(entries: List<NotificationLogEntry>, settings: LogViewSettings) = listOf(
+    private fun fields(settings: LogViewSettings): List<LogField> = settings.order.filter { it in settings.visibleFields }
-        settings.order.filter { it in settings.visibleFields }.map(::header),
+
-    ) + entries.sortedByDescending { it.recordedAtEpochMillis }.map { entry ->
-        settings.order.filter { it in settings.visibleFields }.map { field -> when (field) {
-            LogField.TIMESTAMP -> timestamp(entry.recordedAtEpochMillis, settings.timestampZone, entry.eventTimeZoneId)
-            LogField.APP_NAME -> entry.appName; LogField.PACKAGE_NAME -> entry.packageName
-            LogField.ACTION -> entry.action.name.lowercase().replace('_', ' ')
-            LogField.CONTENTS -> entry.contents ?: ""
-        } }
-    }
     private fun header(field: LogField) = field.name.lowercase().replace('_', ' ')
     private fun htmlValue(field: LogField, entry: NotificationLogEntry, settings: LogViewSettings, imagePath: String?): String {
         val value = when (field) {
@@ -69,6 +91,7 @@ object LogExporter {
             TimestampZone.EVENT_LOCAL -> eventTimeZoneId?.let(TimeZone::getTimeZone) ?: TimeZone.getDefault()
         }
     }.format(Date(value))
+    private fun csvLine(values: List<String>): String = values.joinToString(",") { csvValue(it) }
     private fun csvValue(value: String) = "\"${value.replace("\"", "\"\"")}\""
     private fun escape(value: String) = value.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;")
 }

app/src/test/java/se/ajpanton/notificationlog/export/LogExporterTest.kt

@@ -1,6 +1,7 @@
 package se.ajpanton.notificationlog.export
 
 import org.junit.Assert.assertFalse
+import org.junit.Assert.assertEquals
 import org.junit.Assert.assertTrue
 import org.junit.Test
 import se.ajpanton.notificationlog.model.NotificationAction
@@ -30,4 +31,24 @@ class LogExporterTest {
         assertFalse(withoutImage.contains("<img"))
         assertTrue(withoutImage.contains("[image]"))
     }
+
+    @Test fun `row writers produce the same export as the convenience methods`() {
+        val settings = LogViewSettings()
+        val csv = listOf(LogExporter.csvHeader(settings), LogExporter.csvRow(imageEntry, settings)).joinToString("\n")
+        val widths = LogExporter.headers(settings).map { it.length }.toMutableList()
+        LogExporter.values(imageEntry, settings).forEachIndexed { index, value ->
+            widths[index] = maxOf(widths[index], value.length)
+        }
+        val formatted = listOf(
+            LogExporter.formattedRow(LogExporter.headers(settings), widths),
+            LogExporter.formattedRow(LogExporter.values(imageEntry, settings), widths),
+        ).joinToString("\n")
+        val html = LogExporter.htmlStart(settings) +
+            LogExporter.htmlRow(imageEntry, settings, "images/${imageEntry.imageId}.png") +
+            LogExporter.htmlEnd()
+
+        assertEquals(LogExporter.csv(listOf(imageEntry), settings), csv)
+        assertEquals(LogExporter.formatted(listOf(imageEntry), settings), formatted)
+        assertEquals(LogExporter.html(listOf(imageEntry), settings) { "images/${it.imageId}.png" }, html)
+    }
 }