/* * Copyright (c) 2025 Vitor Pamplona * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to use, * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the * Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package com.vitorpamplona.relaybench.bench import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.relaybench.BenchJson import com.vitorpamplona.relaybench.NostrSocket import com.vitorpamplona.relaybench.Percentiles import com.vitorpamplona.relaybench.asBoolean import com.vitorpamplona.relaybench.asText import com.vitorpamplona.relaybench.get import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeout import kotlinx.serialization.Serializable import okhttp3.OkHttpClient import java.util.concurrent.atomic.AtomicInteger /** * Write-path metrics. * * **Visibility latency** answers the exact question "how long from the * relay *receiving* an EVENT until a REQ can return it": one connection * publishes, and from the same send-instant a second connection * hammer-polls `REQ {ids:[id]}` until the event comes back. The polled * REQ round-trip on loopback (~sub-ms) bounds the probe's resolution. * The OK-ack latency and whether the event was readable no later than * its OK (i.e. does the ack imply queryable — NIP-20 semantics) are * recorded alongside. * * **Throughput** replays the corpus over N connections with a bounded * number of unacked EVENTs in flight per connection, timing wall clock * from first send to last OK. */ object IngestBenchmark { @Serializable data class VisibilityResult( val okLatency: Percentiles, val visibleLatency: Percentiles, val visibleByOkTime: Double, val samples: Int, val rejected: Int, ) @Serializable data class ThroughputResult( val events: Int, val accepted: Int, val rejected: Int, val wallMs: Double, val eventsPerSec: Double, val connections: Int, ) /** (okNanos, visibleNanos, visibleBeforeOk) for one accepted event, or null if rejected. */ private class Sample( val okNanos: Long, val visibleNanos: Long, val visibleByOk: Boolean, ) suspend fun visibility( wsUrl: String, samples: List, http: OkHttpClient, log: (String) -> Unit, ): VisibilityResult { val publisher = NostrSocket.connect(http, wsUrl) val prober = NostrSocket.connect(http, wsUrl) val okNanos = ArrayList(samples.size) val visibleNanos = ArrayList(samples.size) var byOkHits = 0 var rejected = 0 var timeouts = 0 try { for ((i, event) in samples.withIndex()) { if (timeouts >= 5) { log(" ! aborting visibility probe after $timeouts timeouts") break } val sample = try { probeOne(publisher, prober, event, i) } catch (_: TimeoutCancellationException) { timeouts++ continue } if (sample == null) { rejected++ } else { okNanos += sample.okNanos visibleNanos += sample.visibleNanos if (sample.visibleByOk) byOkHits++ } } } finally { publisher.disconnect() prober.disconnect() } if (rejected > 0) log(" ! $rejected visibility samples were rejected by the relay") return VisibilityResult( okLatency = Percentiles.ofNanos(okNanos), visibleLatency = Percentiles.ofNanos(visibleNanos), visibleByOkTime = if (visibleNanos.isEmpty()) 0.0 else byOkHits.toDouble() / visibleNanos.size, samples = visibleNanos.size, rejected = rejected, ) } private suspend fun probeOne( publisher: NostrSocket, prober: NostrSocket, event: Event, index: Int, ): Sample? = withTimeout(30_000) { coroutineScope { val start = System.nanoTime() val okAt = async { for (raw in publisher.incoming) { val node = runCatching { BenchJson.parse(raw) }.getOrNull() ?: continue if (node[0]?.asText() == "OK" && node[1]?.asText() == event.id) { return@async Pair(System.nanoTime(), node[2]?.asBoolean() == true) } } Pair(0L, false) } val visibleAt = async { var polls = 0 while (true) { val subId = "v$index-${polls++}" prober.req(subId, """{"ids":["${event.id}"]}""") var foundAt = 0L for (raw in prober.incoming) { val node = runCatching { BenchJson.parse(raw) }.getOrNull() ?: continue if (node[1]?.asText() != subId) continue when (node[0]?.asText()) { "EVENT" -> foundAt = System.nanoTime() "EOSE", "CLOSED" -> { prober.close(subId) if (foundAt != 0L) return@async foundAt break } } } } @Suppress("UNREACHABLE_CODE") 0L } check(publisher.publish(OptimizedJsonMapper.toJson(event))) { "publish failed" } val (okTime, accepted) = okAt.await() if (!accepted) { visibleAt.cancelAndJoin() return@coroutineScope null } val visibleTime = visibleAt.await() Sample(okTime - start, visibleTime - start, visibleTime <= okTime) } } suspend fun throughput( wsUrl: String, events: List, connections: Int, window: Int, http: OkHttpClient, ): ThroughputResult { val accepted = AtomicInteger() val rejected = AtomicInteger() val slices = List(connections) { c -> events.filterIndexed { i, _ -> i % connections == c } } val start = System.nanoTime() withContext(Dispatchers.IO) { slices .filter { it.isNotEmpty() } .map { slice -> async { publishSlice(wsUrl, slice, window, http, accepted, rejected) } } .awaitAll() } val wallMs = (System.nanoTime() - start) / 1_000_000.0 return ThroughputResult( events = events.size, accepted = accepted.get(), rejected = rejected.get(), wallMs = wallMs, eventsPerSec = events.size / (wallMs / 1000.0), connections = connections, ) } private suspend fun publishSlice( wsUrl: String, slice: List, window: Int, http: OkHttpClient, accepted: AtomicInteger, rejected: AtomicInteger, ) { val socket = NostrSocket.connect(http, wsUrl) try { coroutineScope { val inFlight = Semaphore(window) val reader = launch { var acked = 0 for (raw in socket.incoming) { // Fast-path: only OK frames matter here. if (!raw.startsWith("[\"OK\"")) continue val node = runCatching { BenchJson.parse(raw) }.getOrNull() ?: continue if (node[2]?.asBoolean() == true) accepted.incrementAndGet() else rejected.incrementAndGet() inFlight.release() if (++acked == slice.size) break } } for (event in slice) { inFlight.acquire() check(socket.publish(OptimizedJsonMapper.toJson(event))) { "publish failed (socket closed?)" } } reader.join() } } finally { socket.disconnect() } } }