From 5d958873562afe9147190c7af85a14a675506fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Weber?= Date: Wed, 8 Jan 2025 10:25:52 +0100 Subject: [PATCH] #625 refactored forwarding metrics to use timer only --- .../playground/server/admin/v1/routing/rules | 4 + gateleen-routing/README_routing.md | 8 +- .../gateleen/routing/AbstractForwarder.java | 2 - .../swisspush/gateleen/routing/Forwarder.java | 12 +-- .../gateleen/routing/NullForwarder.java | 17 +-- .../gateleen/routing/StorageForwarder.java | 8 -- .../gateleen/routing/RouterTest.java | 100 ++++++++++++------ 7 files changed, 85 insertions(+), 66 deletions(-) diff --git a/gateleen-playground/src/main/resources/playground/server/admin/v1/routing/rules b/gateleen-playground/src/main/resources/playground/server/admin/v1/routing/rules index 351a62d7..07bc2048 100755 --- a/gateleen-playground/src/main/resources/playground/server/admin/v1/routing/rules +++ b/gateleen-playground/src/main/resources/playground/server/admin/v1/routing/rules @@ -9,18 +9,22 @@ }, "/playground/cb/1/(.*)": { "description": "CircuitBreaker Test", + "metricName": "cb_1", "url": "http://localhost:1234/playground/server/tests/exp/$1" }, "/playground/cb/2/(.*)": { "description": "CircuitBreaker Test", + "metricName": "cb_2", "url": "http://localhost:1234/playground/server/tests/exp/$1" }, "/playground/cb/3/(.*)": { "description": "CircuitBreaker Test", + "metricName": "cb_3", "url": "http://localhost:1234/playground/server/tests/exp/$1" }, "/playground/cb/4/(.*)": { "description": "CircuitBreaker Test", + "metricName": "cb_4", "url": "http://localhost:1234/playground/server/tests/exp/$1" }, "/playground/": { diff --git a/gateleen-routing/README_routing.md b/gateleen-routing/README_routing.md index 3a50831a..f5e69eed 100644 --- a/gateleen-routing/README_routing.md +++ b/gateleen-routing/README_routing.md @@ -138,7 +138,6 @@ Each request header entry is validated in the format `: `, so you ar ## Micrometer metrics The routing feature is monitored with micrometer. The following metrics are available: -* gateleen_forwarded_total * gateleen_forwarded_seconds * gateleen_forwarded_seconds_max * gateleen_forwarded_seconds_count @@ -149,18 +148,13 @@ Additional tags are provided to split the forward count into sub counts. | tag | description | |------------|-------------------------------------------------------------------------------------------------------------------| | metricName | The `metricName` property from the corresponding routing rule. With this, you are able to count requests per rule | -| type | Describes where the request was forwarded to. Possible values are `local`, `external` and `null` | +| type | Describes where the request was forwarded to. Possible values are `storage`, `local`, `external` and `null` | | quantile | Values of `0.75` and `0.95` for percentile durations of requests | Example metrics: ``` -# HELP gateleen_forwarded_total Amount of forwarded requests -# TYPE gateleen_forwarded_total counter -gateleen_forwarded_total{metricName="storage-resources",type="storage",} 67565.0 -gateleen_forwarded_total{metricName="infotool_v1_informations",type="external",} 655.0 -gateleen_forwarded_total{metricName="infotool-v1",type="storage",} 4320.0 # HELP gateleen_forwarded_seconds_max Durations of forwarded requests # TYPE gateleen_forwarded_seconds_max gauge gateleen_forwarded_seconds_max{metricName="storage-resources",type="storage",} 8.5515 diff --git a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/AbstractForwarder.java b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/AbstractForwarder.java index dccf549b..55c5ee1f 100644 --- a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/AbstractForwarder.java +++ b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/AbstractForwarder.java @@ -23,8 +23,6 @@ public abstract class AbstractForwarder implements Handler { protected final MonitoringHandler monitoringHandler; protected final String metricNameTag; - public static final String FORWARDER_COUNT_METRIC_NAME = "gateleen.forwarded"; - public static final String FORWARDER_COUNT_METRIC_DESCRIPTION = "Amount of forwarded requests"; public static final String FORWARDS_METRIC_NAME = "gateleen.forwarded.seconds"; public static final String FORWARDS_METRIC_DESCRIPTION = "Durations of forwarded requests"; public static final String FORWARDER_METRIC_TAG_TYPE = "type"; diff --git a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/Forwarder.java b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/Forwarder.java index e9ceccff..0a0233af 100755 --- a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/Forwarder.java +++ b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/Forwarder.java @@ -1,6 +1,5 @@ package org.swisspush.gateleen.routing; -import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; import io.netty.channel.ConnectTimeoutException; @@ -67,7 +66,6 @@ public class Forwarder extends AbstractForwarder { private static final int STATUS_CODE_2XX = 2; private static final Logger LOG = LoggerFactory.getLogger(Forwarder.class); - private Counter forwardCounter; private Timer forwardTimer; private MeterRegistry meterRegistry; @@ -99,11 +97,6 @@ public Forwarder(Vertx vertx, HttpClient client, Rule rule, final ResourceStorag public void setMeterRegistry(MeterRegistry meterRegistry) { this.meterRegistry = meterRegistry; if (meterRegistry != null) { - forwardCounter = Counter.builder(FORWARDER_COUNT_METRIC_NAME) - .description(FORWARDER_COUNT_METRIC_DESCRIPTION) - .tag(FORWARDER_METRIC_TAG_TYPE, getRequestTarget(target)) - .tag(FORWARDER_METRIC_TAG_METRICNAME, metricNameTag) - .register(meterRegistry); forwardTimer = Timer.builder(FORWARDS_METRIC_NAME) .description(FORWARDS_METRIC_DESCRIPTION) .publishPercentiles(0.75, 0.95) @@ -180,10 +173,6 @@ public void handle(final RoutingContext ctx, final Buffer bodyData, @Nullable fi } target = rule.getHost() + ":" + port; - if (forwardCounter != null) { - forwardCounter.increment(); - } - if (monitoringHandler != null) { monitoringHandler.updateRequestsMeter(target, req.uri()); monitoringHandler.updateRequestPerRuleMonitoring(req, rule.getMetricName()); @@ -305,6 +294,7 @@ public void handle(AsyncResult event) { if (event.failed()) { log.warn("Problem to request {}: {}", targetUri, event.cause()); + handleForwardDurationMetrics(finalTimerSample); final HttpServerResponse response = req.response(); response.setStatusCode(StatusCode.SERVICE_UNAVAILABLE.getStatusCode()); response.setStatusMessage(StatusCode.SERVICE_UNAVAILABLE.getStatusMessage()); diff --git a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/NullForwarder.java b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/NullForwarder.java index bed0d498..d658994d 100755 --- a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/NullForwarder.java +++ b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/NullForwarder.java @@ -1,7 +1,7 @@ package org.swisspush.gateleen.routing; -import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import io.vertx.core.MultiMap; import io.vertx.core.buffer.Buffer; import io.vertx.core.eventbus.EventBus; @@ -29,7 +29,8 @@ public class NullForwarder extends AbstractForwarder { private EventBus eventBus; - private Counter forwardCounter; + private MeterRegistry meterRegistry; + private Timer forwardTimer; public NullForwarder(Rule rule, LoggingResourceManager loggingResourceManager, LogAppenderRepository logAppenderRepository, @Nullable MonitoringHandler monitoringHandler, EventBus eventBus) { super(rule, loggingResourceManager, logAppenderRepository, monitoringHandler); @@ -45,9 +46,11 @@ public NullForwarder(Rule rule, LoggingResourceManager loggingResourceManager, L */ @Override public void setMeterRegistry(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; if(meterRegistry != null) { - forwardCounter = Counter.builder(FORWARDER_COUNT_METRIC_NAME) - .description(FORWARDER_COUNT_METRIC_DESCRIPTION) + forwardTimer = Timer.builder(FORWARDS_METRIC_NAME) + .description(FORWARDS_METRIC_DESCRIPTION) + .publishPercentiles(0.75, 0.95) .tag(FORWARDER_METRIC_TAG_METRICNAME, metricNameTag) .tag(FORWARDER_METRIC_TAG_TYPE, "null") .register(meterRegistry); @@ -63,8 +66,10 @@ public void handle(final RoutingContext ctx) { return; } - if(forwardCounter != null) { - forwardCounter.increment(); + Timer.Sample timerSample = null; + if(meterRegistry != null) { + timerSample = Timer.start(meterRegistry); + timerSample.stop(forwardTimer); } if(monitoringHandler != null) { diff --git a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/StorageForwarder.java b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/StorageForwarder.java index 32a0f38a..4de2e34d 100755 --- a/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/StorageForwarder.java +++ b/gateleen-routing/src/main/java/org/swisspush/gateleen/routing/StorageForwarder.java @@ -47,7 +47,6 @@ public class StorageForwarder extends AbstractForwarder { private CORSHandler corsHandler; private GateleenExceptionFactory gateleenExceptionFactory; - private Counter forwardCounter; private Timer forwardTimer; private MeterRegistry meterRegistry; @@ -75,12 +74,6 @@ public StorageForwarder(EventBus eventBus, Rule rule, LoggingResourceManager log public void setMeterRegistry(MeterRegistry meterRegistry) { this.meterRegistry = meterRegistry; if (meterRegistry != null) { - forwardCounter = Counter.builder(FORWARDER_COUNT_METRIC_NAME) - .description(FORWARDER_COUNT_METRIC_DESCRIPTION) - .tag(FORWARDER_METRIC_TAG_METRICNAME, metricNameTag) - .tag(FORWARDER_METRIC_TAG_TYPE, TYPE_STORAGE) - .register(meterRegistry); - forwardTimer = Timer.builder(FORWARDS_METRIC_NAME) .description(FORWARDS_METRIC_DESCRIPTION) .publishPercentiles(0.75, 0.95) @@ -106,7 +99,6 @@ public void handle(final RoutingContext ctx) { Timer.Sample timerSample = null; if(meterRegistry != null) { timerSample = Timer.start(meterRegistry); - forwardCounter.increment(); } if (monitoringHandler != null) { diff --git a/gateleen-routing/src/test/java/org/swisspush/gateleen/routing/RouterTest.java b/gateleen-routing/src/test/java/org/swisspush/gateleen/routing/RouterTest.java index 2c90dfbc..826a7596 100755 --- a/gateleen-routing/src/test/java/org/swisspush/gateleen/routing/RouterTest.java +++ b/gateleen-routing/src/test/java/org/swisspush/gateleen/routing/RouterTest.java @@ -1,18 +1,14 @@ package org.swisspush.gateleen.routing; import com.google.common.collect.ImmutableMap; -import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Timer; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; -import io.vertx.core.Handler; -import io.vertx.core.MultiMap; -import io.vertx.core.Vertx; +import io.vertx.core.*; import io.vertx.core.buffer.Buffer; import io.vertx.core.eventbus.EventBus; -import io.vertx.core.http.HttpClient; -import io.vertx.core.http.HttpMethod; -import io.vertx.core.http.HttpServerRequest; -import io.vertx.core.http.HttpServerResponse; +import io.vertx.core.eventbus.Message; +import io.vertx.core.http.*; import io.vertx.core.http.impl.headers.HeadersMultiMap; import io.vertx.core.json.JsonObject; import io.vertx.ext.unit.TestContext; @@ -33,6 +29,8 @@ import java.util.*; +import static org.mockito.Mockito.doAnswer; + /** * Tests for the Router class * @@ -42,6 +40,9 @@ public class RouterTest { private Vertx vertx; + private EventBus eventBus; + private Message message; + private Map properties; private LoggingResourceManager loggingResourceManager; private MonitoringHandler monitoringHandler; @@ -130,11 +131,26 @@ public class RouterTest { + " \"randomkey2\": 456\n" + "}"; + private byte[] eventBusMessageAsByteArray = {0, 0, 0, 52, 123, 34, 115, 116, 97, 116, 117, 115, 67, 111, 100, 101, 34, + 58, 50, 48, 48, 44, 34, 115, 116, 97, 116, 117, 115, 77, 101, 115, 115, 97, 103, 101, 34, 58, 34, 79, 75, 34, + 44, 34, 104, 101, 97, 100, 101, 114, 115, 34, 58, 91, 93, 125}; + @Before public void setUp() { // setup vertx = Mockito.mock(Vertx.class); - Mockito.when(vertx.eventBus()).thenReturn(Mockito.mock(EventBus.class)); + eventBus = Mockito.mock(EventBus.class); + message = Mockito.mock(Message.class); + + Mockito.when(message.body()).thenReturn(Buffer.buffer(eventBusMessageAsByteArray)); + + doAnswer(invocation -> { + Handler>> handler = (Handler>>) invocation.getArguments()[3]; + handler.handle(Future.succeededFuture(message)); + return null; + }).when(eventBus).request(Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any()); + + Mockito.when(vertx.eventBus()).thenReturn(eventBus); Mockito.when(vertx.createHttpClient()).thenReturn(Mockito.mock(HttpClient.class)); Mockito.when(vertx.sharedData()).thenReturn(Vertx.vertx().sharedData()); @@ -172,8 +188,8 @@ private RouterBuilder routerBuilder() { } private void assertNoCountersIncremented(TestContext context) { - for (Counter counter : meterRegistry.get(AbstractForwarder.FORWARDER_COUNT_METRIC_NAME).counters()) { - context.assertEquals(0.0, counter.count(), "No counter should have been incremented"); + for (Timer timer : meterRegistry.get(AbstractForwarder.FORWARDS_METRIC_NAME).timers()) { + context.assertEquals(0L, timer.count(), "No timer should have been incremented"); } } @@ -191,9 +207,15 @@ public void testRequestHopValidationLimitNotYetReached(TestContext context) { // change the hops limit to 5 router.resourceChanged(serverUrl + "/admin/v1/routing/config", Buffer.buffer("{\"request.hops.limit\":5}")); - final DummyHttpServerResponse response = new DummyHttpServerResponse(); + DummyHttpServerResponse response = new DummyHttpServerResponse() { + @Override + public Future write(Buffer data) { + return Future.succeededFuture(); + } + }; response.setStatusCode(StatusCode.OK.getStatusCode()); response.setStatusMessage(StatusCode.OK.getStatusMessage()); + class GETRandomResourceRequest extends DummyHttpServerRequest { MultiMap headers = MultiMap.caseInsensitiveMultiMap(); @@ -249,8 +271,8 @@ public DummyHttpServerResponse response() { GETRandomResourceRequest request = new GETRandomResourceRequest(); router.route(request); - Counter counter = meterRegistry.get(AbstractForwarder.FORWARDER_COUNT_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "loop_4").counter(); - context.assertEquals(1.0, counter.count(), "Counter for `loop_4` rule should have been incremented by 1"); + Timer timer = meterRegistry.get(AbstractForwarder.FORWARDS_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "loop_4").timer(); + context.assertEquals(1L, timer.count(), "Timer for `loop_4` rule should have been incremented by 1"); context.assertEquals("1", request.headers().get("x-hops"), "x-hops header should have value 1"); context.assertEquals(StatusCode.OK.getStatusCode(), request.response().getStatusCode(), "StatusCode should be 200"); @@ -350,7 +372,12 @@ public void testRequestHopValidationWithLimit5(TestContext context) { // change the hops limit to 5 router.resourceChanged(serverUrl + "/admin/v1/routing/config", Buffer.buffer("{\"request.hops.limit\":5}")); - final DummyHttpServerResponse response = new DummyHttpServerResponse(); + final DummyHttpServerResponse response = new DummyHttpServerResponse() { + @Override + public Future write(Buffer data) { + return Future.succeededFuture(); + } + }; response.setStatusCode(StatusCode.OK.getStatusCode()); response.setStatusMessage(StatusCode.OK.getStatusMessage()); class GETRandomResourceRequest extends DummyHttpServerRequest { @@ -419,8 +446,8 @@ public DummyHttpServerResponse response() { context.assertEquals(StatusCode.INTERNAL_SERVER_ERROR.getStatusCode(), request.response().getStatusCode(), "StatusCode should be 500"); context.assertEquals("Request hops limit exceeded", request.response().getStatusMessage(), "StatusMessage should be 'Request hops limit exceeded'"); - Counter counter = meterRegistry.get(AbstractForwarder.FORWARDER_COUNT_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "loop_4").counter(); - context.assertEquals(5.0, counter.count(), "Counter for `loop_4` rule should have been incremented by 5"); + Timer timer = meterRegistry.get(AbstractForwarder.FORWARDS_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "loop_4").timer(); + context.assertEquals(5L, timer.count(), "Timer for `loop_4` rule should have been incremented by 5"); } @Test @@ -431,7 +458,12 @@ public void testRequestHopValidationNoLimitConfiguration(TestContext context) { context.assertFalse(router.isRoutingBroken(), "Routing should not be broken"); context.assertNull(router.getRoutingBrokenMessage(), "RoutingBrokenMessage should be null"); - final DummyHttpServerResponse response = new DummyHttpServerResponse(); + final DummyHttpServerResponse response = new DummyHttpServerResponse() { + @Override + public Future write(Buffer data) { + return Future.succeededFuture(); + } + }; response.setStatusCode(StatusCode.OK.getStatusCode()); response.setStatusMessage(StatusCode.OK.getStatusMessage()); class GETRandomResourceRequest extends DummyHttpServerRequest { @@ -496,8 +528,8 @@ public DummyHttpServerResponse response() { context.assertEquals(StatusCode.OK.getStatusCode(), request.response().getStatusCode(), "StatusCode should be 200"); context.assertEquals(StatusCode.OK.getStatusMessage(), request.response().getStatusMessage(), "StatusMessage should be OK"); - Counter counter = meterRegistry.get(AbstractForwarder.FORWARDER_COUNT_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "loop_4").counter(); - context.assertEquals(20.0, counter.count(), "Counter for `loop_4` rule should have been incremented by 20"); + Timer timer = meterRegistry.get(AbstractForwarder.FORWARDS_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "loop_4").timer(); + context.assertEquals(20L, timer.count(), "Timer for `loop_4` rule should have been incremented by 20"); } @@ -772,9 +804,6 @@ public HttpServerRequest pause() { router.route(requestRandomResource); context.assertFalse(router.isRoutingBroken(), "Routing should not be broken anymore"); context.assertNull(router.getRoutingBrokenMessage(), "RoutingBrokenMessage should be null"); - - Counter counter = meterRegistry.get(AbstractForwarder.FORWARDER_COUNT_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_TYPE, "local").counter(); - context.assertEquals(1.0, counter.count(), "Counter should have been incremented by 1"); } @Test @@ -896,7 +925,12 @@ public void testStorageRequestWithHeadersFilterPresent(TestContext context) { context.assertFalse(router.isRoutingBroken(), "Routing should not be broken"); context.assertNull(router.getRoutingBrokenMessage(), "RoutingBrokenMessage should be null"); - final DummyHttpServerResponse response = new DummyHttpServerResponse(); + final DummyHttpServerResponse response = new DummyHttpServerResponse() { + @Override + public Future write(Buffer data) { + return Future.succeededFuture(); + } + }; response.setStatusCode(StatusCode.OK.getStatusCode()); response.setStatusMessage(StatusCode.OK.getStatusMessage()); @@ -908,8 +942,8 @@ public void testStorageRequestWithHeadersFilterPresent(TestContext context) { context.assertEquals(StatusCode.OK.getStatusCode(), request.response().getStatusCode(), "StatusCode should be 200"); context.assertEquals(StatusCode.OK.getStatusMessage(), request.response().getStatusMessage(), "StatusMessage should be OK"); - Counter counter = meterRegistry.get(AbstractForwarder.FORWARDER_COUNT_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "forward_storage").counter(); - context.assertEquals(1.0, counter.count(), "Counter for `forward_storage` rule should have been incremented by 1"); + Timer timer = meterRegistry.get(AbstractForwarder.FORWARDS_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "forward_storage").timer(); + context.assertEquals(1L, timer.count(), "Timer for `forward_storage` rule should have been incremented by 1"); } @Test @@ -970,7 +1004,12 @@ public void testNullForwarderRequestWithHeadersFilterPresent(TestContext context context.assertFalse(router.isRoutingBroken(), "Routing should not be broken"); context.assertNull(router.getRoutingBrokenMessage(), "RoutingBrokenMessage should be null"); - final DummyHttpServerResponse response = new DummyHttpServerResponse(); + final DummyHttpServerResponse response = new DummyHttpServerResponse() { + @Override + public Future write(Buffer data) { + return Future.succeededFuture(); + } + }; response.setStatusCode(StatusCode.OK.getStatusCode()); response.setStatusMessage(StatusCode.OK.getStatusMessage()); @@ -982,8 +1021,8 @@ public void testNullForwarderRequestWithHeadersFilterPresent(TestContext context context.assertEquals(StatusCode.OK.getStatusCode(), request.response().getStatusCode(), "StatusCode should be 200"); context.assertEquals(StatusCode.OK.getStatusMessage(), request.response().getStatusMessage(), "StatusMessage should be OK"); - Counter counter = meterRegistry.get(AbstractForwarder.FORWARDER_COUNT_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "forward_null").counter(); - context.assertEquals(1.0, counter.count(), "Counter for `forward_null` rule should have been incremented by 1"); + Timer timer = meterRegistry.get(AbstractForwarder.FORWARDS_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "forward_null").timer(); + context.assertEquals(1L, timer.count(), "Timer for `forward_null` rule should have been incremented by 1"); } @Test @@ -1009,9 +1048,6 @@ public void testForwarderRequestWithHeadersFilterPresent(TestContext context) { // we expect a status code 500 because of a NullPointerException in the test setup // however, this means that the headersFilter evaluation did not return a 400 Bad Request context.assertEquals(StatusCode.INTERNAL_SERVER_ERROR.getStatusCode(), request.response().getStatusCode(), "StatusCode should be 500"); - - Counter counter = meterRegistry.get(AbstractForwarder.FORWARDER_COUNT_METRIC_NAME).tag(AbstractForwarder.FORWARDER_METRIC_TAG_METRICNAME, "forward_backend").counter(); - context.assertEquals(1.0, counter.count(), "Counter for `forward_backend` rule should have been incremented by 1"); } @Test