diff --git a/src/main/java/io/apitally/common/RequestLogger.java b/src/main/java/io/apitally/common/RequestLogger.java index 0180b2c..5ca5f7c 100644 --- a/src/main/java/io/apitally/common/RequestLogger.java +++ b/src/main/java/io/apitally/common/RequestLogger.java @@ -8,7 +8,6 @@ import java.util.Arrays; import java.util.Deque; import java.util.List; -import java.util.UUID; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -21,12 +20,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.apitally.common.dto.ExceptionDto; import io.apitally.common.dto.Header; import io.apitally.common.dto.Request; +import io.apitally.common.dto.RequestLogItem; import io.apitally.common.dto.Response; public class RequestLogger { @@ -40,6 +41,7 @@ public class RequestLogger { private static final byte[] BODY_MASKED = "".getBytes(StandardCharsets.UTF_8); private static final String MASKED = "******"; public static final List ALLOWED_CONTENT_TYPES = Arrays.asList("application/json", "text/plain"); + private static final Pattern JSON_CONTENT_TYPE_PATTERN = Pattern.compile("\\bjson\\b", Pattern.CASE_INSENSITIVE); private static final List EXCLUDE_PATH_PATTERNS = Arrays.asList( "/_?healthz?$", "/_?health[_-]?checks?$", @@ -65,12 +67,21 @@ public class RequestLogger { "secret", "token", "cookie"); + private static final List MASK_BODY_FIELD_PATTERNS = Arrays.asList( + "password", + "pwd", + "token", + "secret", + "auth", + "card[-_ ]?number", + "ccv", + "ssn"); private static final int MAINTAIN_INTERVAL_SECONDS = 1; private final RequestLoggingConfig config; private final ObjectMapper objectMapper; private final ReentrantLock lock; - private final Deque pendingWrites; + private final Deque pendingWrites; private final Deque files; private TempGzipFile currentFile; private boolean enabled; @@ -82,6 +93,7 @@ public class RequestLogger { private final List compiledUserAgentExcludePatterns; private final List compiledQueryParamMaskPatterns; private final List compiledHeaderMaskPatterns; + private final List compiledBodyFieldMaskPatterns; public RequestLogger(RequestLoggingConfig config) { this.config = config; @@ -96,6 +108,8 @@ public RequestLogger(RequestLoggingConfig config) { this.compiledQueryParamMaskPatterns = compilePatterns(MASK_QUERY_PARAM_PATTERNS, config.getQueryParamMaskPatterns()); this.compiledHeaderMaskPatterns = compilePatterns(MASK_HEADER_PATTERNS, config.getHeaderMaskPatterns()); + this.compiledBodyFieldMaskPatterns = compilePatterns(MASK_BODY_FIELD_PATTERNS, + config.getBodyFieldMaskPatterns()); if (enabled) { startMaintenance(); @@ -131,80 +145,27 @@ public void logRequest(Request request, Response response, Exception exception) try { String userAgent = findHeader(request.getHeaders(), "user-agent"); - if (shouldExcludePath(request.getPath()) || shouldExcludeUserAgent(userAgent) - || (config.getCallbacks() != null && config.getCallbacks().shouldExclude(request, response))) { + if (shouldExcludePath(request.getPath()) || shouldExcludeUserAgent(userAgent)) { return; } - - // Process query params and URL - if (request.getUrl() != null) { - try { - URL url = new URL(request.getUrl()); - String query = url.getQuery(); - if (!config.isQueryParamsIncluded()) { - query = null; - } else if (query != null) { - query = maskQueryParams(query); - } - request.setUrl(new java.net.URL(url.getProtocol(), url.getHost(), url.getPort(), - url.getPath() + (query != null ? "?" + query : "")).toString()); - } catch (MalformedURLException e) { - return; - } + if (config.getCallbacks() != null && config.getCallbacks().shouldExclude(request, response)) { + return; } - // Process request body if (!config.isRequestBodyIncluded() || !hasSupportedContentType(request.getHeaders())) { request.setBody(null); - } else if (request.getBody() != null) { - if (request.getBody().length > MAX_BODY_SIZE) { - request.setBody(BODY_TOO_LARGE); - } else if (config.getCallbacks() != null) { - byte[] maskedBody = config.getCallbacks().maskRequestBody(request); - request.setBody(maskedBody != null ? maskedBody : BODY_MASKED); - if (request.getBody().length > MAX_BODY_SIZE) { - request.setBody(BODY_TOO_LARGE); - } - } } - - // Process response body if (!config.isResponseBodyIncluded() || !hasSupportedContentType(response.getHeaders())) { response.setBody(null); - } else if (response.getBody() != null) { - if (response.getBody().length > MAX_BODY_SIZE) { - response.setBody(BODY_TOO_LARGE); - } else if (config.getCallbacks() != null) { - byte[] maskedBody = config.getCallbacks().maskResponseBody(request, response); - response.setBody(maskedBody != null ? maskedBody : BODY_MASKED); - if (response.getBody().length > MAX_BODY_SIZE) { - response.setBody(BODY_TOO_LARGE); - } - } } - // Process headers - request.setHeaders( - config.isRequestHeadersIncluded() - ? maskHeaders(request.getHeaders()).toArray(new Header[0]) - : new Header[0]); - response.setHeaders( - config.isResponseHeadersIncluded() - ? maskHeaders(response.getHeaders()).toArray(new Header[0]) - : new Header[0]); - - // Create log item - ObjectNode item = objectMapper.createObjectNode(); - item.put("uuid", UUID.randomUUID().toString()); - item.set("request", skipEmptyValues(objectMapper.valueToTree(request))); - item.set("response", skipEmptyValues(objectMapper.valueToTree(response))); + ExceptionDto exceptionDto = null; if (exception != null && config.isExceptionIncluded()) { - ExceptionDto exceptionDto = new ExceptionDto(exception); - item.set("exception", objectMapper.valueToTree(exceptionDto)); + exceptionDto = new ExceptionDto(exception); } - String serializedItem = objectMapper.writeValueAsString(item); - pendingWrites.add(serializedItem); + RequestLogItem item = new RequestLogItem(request, response, exceptionDto); + pendingWrites.add(item); if (pendingWrites.size() > MAX_PENDING_WRITES) { pendingWrites.poll(); @@ -214,6 +175,74 @@ public void logRequest(Request request, Response response, Exception exception) } } + private void applyMasking(RequestLogItem item) { + Request request = item.getRequest(); + Response response = item.getResponse(); + + if (request.getBody() != null) { + // Apply user-provided masking callback for request body + if (config.getCallbacks() != null) { + byte[] maskedBody = config.getCallbacks().maskRequestBody(request); + request.setBody(maskedBody != null ? maskedBody : BODY_MASKED); + } + + if (request.getBody().length > MAX_BODY_SIZE) { + request.setBody(BODY_TOO_LARGE); + } + + // Mask request body fields (if JSON) + if (!Arrays.equals(request.getBody(), BODY_TOO_LARGE) && !Arrays.equals(request.getBody(), BODY_MASKED) + && hasJsonContentType(request.getHeaders())) { + request.setBody(maskJsonBody(request.getBody())); + } + } + + if (response.getBody() != null) { + // Apply user-provided masking callback for response body + if (config.getCallbacks() != null) { + byte[] maskedBody = config.getCallbacks().maskResponseBody(request, response); + response.setBody(maskedBody != null ? maskedBody : BODY_MASKED); + } + + if (response.getBody().length > MAX_BODY_SIZE) { + response.setBody(BODY_TOO_LARGE); + } + + // Mask response body fields (if JSON) + if (!Arrays.equals(response.getBody(), BODY_TOO_LARGE) && !Arrays.equals(response.getBody(), BODY_MASKED) + && hasJsonContentType(response.getHeaders())) { + response.setBody(maskJsonBody(response.getBody())); + } + } + + // Process headers + request.setHeaders( + config.isRequestHeadersIncluded() + ? maskHeaders(request.getHeaders()).toArray(new Header[0]) + : new Header[0]); + response.setHeaders( + config.isResponseHeadersIncluded() + ? maskHeaders(response.getHeaders()).toArray(new Header[0]) + : new Header[0]); + + // Process query params and URL + if (request.getUrl() != null) { + try { + URL url = new URL(request.getUrl()); + String query = url.getQuery(); + if (!config.isQueryParamsIncluded()) { + query = null; + } else if (query != null) { + query = maskQueryParams(query); + } + request.setUrl(new java.net.URL(url.getProtocol(), url.getHost(), url.getPort(), + url.getPath() + (query != null ? "?" + query : "")).toString()); + } catch (MalformedURLException e) { + // Keep original URL if malformed + } + } + } + public void writeToFile() throws IOException { if (!enabled || pendingWrites.isEmpty()) { return; @@ -223,9 +252,20 @@ public void writeToFile() throws IOException { if (currentFile == null) { currentFile = new TempGzipFile(); } - String item; + RequestLogItem item; while ((item = pendingWrites.poll()) != null) { - currentFile.writeLine(item.getBytes(StandardCharsets.UTF_8)); + applyMasking(item); + + ObjectNode itemNode = objectMapper.createObjectNode(); + itemNode.put("uuid", item.getUuid()); + itemNode.set("request", skipEmptyValues(objectMapper.valueToTree(item.getRequest()))); + itemNode.set("response", skipEmptyValues(objectMapper.valueToTree(item.getResponse()))); + if (item.getException() != null) { + itemNode.set("exception", objectMapper.valueToTree(item.getException())); + } + + String serializedItem = objectMapper.writeValueAsString(itemNode); + currentFile.writeLine(serializedItem.getBytes(StandardCharsets.UTF_8)); } } finally { lock.unlock(); @@ -344,6 +384,11 @@ private boolean shouldMaskHeader(String name) { .anyMatch(p -> p.matcher(name).find()); } + private boolean shouldMaskBodyField(String name) { + return compiledBodyFieldMaskPatterns.stream() + .anyMatch(p -> p.matcher(name).find()); + } + private String maskQueryParams(String query) { if (query == null || query.isEmpty()) { return query; @@ -371,12 +416,43 @@ private List
maskHeaders(Header[] headers) { .collect(Collectors.toList()); } + private byte[] maskJsonBody(byte[] body) { + try { + String json = new String(body, StandardCharsets.UTF_8); + JsonNode node = objectMapper.readTree(json); + maskJsonNode(node); + return objectMapper.writeValueAsString(node).getBytes(StandardCharsets.UTF_8); + } catch (Exception e) { + return body; + } + } + + private void maskJsonNode(JsonNode node) { + if (node.isObject()) { + ObjectNode objectNode = (ObjectNode) node; + objectNode.fields().forEachRemaining(entry -> { + if (entry.getValue().isTextual() && shouldMaskBodyField(entry.getKey())) { + objectNode.put(entry.getKey(), MASKED); + } else { + maskJsonNode(entry.getValue()); + } + }); + } else if (node.isArray()) { + node.forEach(this::maskJsonNode); + } + } + private boolean hasSupportedContentType(Header[] headers) { String contentType = findHeader(headers, "content-type"); return contentType != null && ALLOWED_CONTENT_TYPES.stream() .anyMatch(contentType::startsWith); } + private boolean hasJsonContentType(Header[] headers) { + String contentType = findHeader(headers, "content-type"); + return contentType != null && JSON_CONTENT_TYPE_PATTERN.matcher(contentType).find(); + } + private String findHeader(Header[] headers, String name) { return Arrays.stream(headers) .filter(h -> h.getName().toLowerCase().equals(name)) diff --git a/src/main/java/io/apitally/common/RequestLoggingConfig.java b/src/main/java/io/apitally/common/RequestLoggingConfig.java index 2ecb29b..4b38646 100644 --- a/src/main/java/io/apitally/common/RequestLoggingConfig.java +++ b/src/main/java/io/apitally/common/RequestLoggingConfig.java @@ -13,6 +13,7 @@ public class RequestLoggingConfig { private boolean exceptionIncluded = true; private List queryParamMaskPatterns = new ArrayList<>(); private List headerMaskPatterns = new ArrayList<>(); + private List bodyFieldMaskPatterns = new ArrayList<>(); private List pathExcludePatterns = new ArrayList<>(); private RequestLoggingCallbacks callbacks; @@ -88,6 +89,14 @@ public void setHeaderMaskPatterns(List headerMaskPatterns) { this.headerMaskPatterns = headerMaskPatterns; } + public List getBodyFieldMaskPatterns() { + return bodyFieldMaskPatterns; + } + + public void setBodyFieldMaskPatterns(List bodyFieldMaskPatterns) { + this.bodyFieldMaskPatterns = bodyFieldMaskPatterns; + } + public List getPathExcludePatterns() { return pathExcludePatterns; } diff --git a/src/main/java/io/apitally/common/dto/RequestLogItem.java b/src/main/java/io/apitally/common/dto/RequestLogItem.java new file mode 100644 index 0000000..fe03ba9 --- /dev/null +++ b/src/main/java/io/apitally/common/dto/RequestLogItem.java @@ -0,0 +1,39 @@ +package io.apitally.common.dto; + +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class RequestLogItem extends BaseDto { + private final String uuid; + private final Request request; + private final Response response; + private final ExceptionDto exception; + + public RequestLogItem(Request request, Response response, ExceptionDto exception) { + this.uuid = UUID.randomUUID().toString(); + this.request = request; + this.response = response; + this.exception = exception; + } + + @JsonProperty("uuid") + public String getUuid() { + return uuid; + } + + @JsonProperty("request") + public Request getRequest() { + return request; + } + + @JsonProperty("response") + public Response getResponse() { + return response; + } + + @JsonProperty("exception") + public ExceptionDto getException() { + return exception; + } +} diff --git a/src/test/java/io/apitally/common/RequestLoggerTest.java b/src/test/java/io/apitally/common/RequestLoggerTest.java index 2379dbd..3c2a2e1 100644 --- a/src/test/java/io/apitally/common/RequestLoggerTest.java +++ b/src/test/java/io/apitally/common/RequestLoggerTest.java @@ -3,7 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; @@ -68,70 +67,99 @@ void testEndToEnd() { "{\"items\": []}".getBytes()); Exception exception = new Exception("test"); requestLogger.logRequest(request, response, exception); - requestLogger.maintain(); - requestLogger.rotateFile(); - TempGzipFile logFile = requestLogger.getFile(); - assertNotNull(logFile); - assertTrue(logFile.getSize() > 0); + JsonNode[] items = getLoggedItems(requestLogger); + assertEquals(1, items.length); + + JsonNode jsonNode = items[0]; + assertEquals("GET", jsonNode.get("request").get("method").asText()); + assertEquals("/items", jsonNode.get("request").get("path").asText()); + assertEquals("http://test/items", jsonNode.get("request").get("url").asText()); + assertFalse(jsonNode.get("request").has("body")); + assertEquals(200, jsonNode.get("response").get("statusCode").asInt()); + assertEquals(0.123, jsonNode.get("response").get("responseTime").asDouble(), 0.001); + assertEquals("{\"items\":[]}", + new String(Base64.getDecoder() + .decode(jsonNode.get("response").get("body").asText()))); + + JsonNode requestHeadersNode = jsonNode.get("request").get("headers"); + assertTrue(requestHeadersNode.isArray()); + assertEquals(1, requestHeadersNode.size()); + assertEquals("User-Agent", requestHeadersNode.get(0).get(0).asText()); + assertEquals("Test", requestHeadersNode.get(0).get(1).asText()); + + JsonNode responseHeadersNode = jsonNode.get("response").get("headers"); + assertTrue(responseHeadersNode.isArray()); + assertEquals(1, responseHeadersNode.size()); + assertEquals("Content-Type", responseHeadersNode.get(0).get(0).asText()); + assertEquals("application/json", responseHeadersNode.get(0).get(1).asText()); + + JsonNode exceptionNode = jsonNode.get("exception"); + assertNotNull(exceptionNode); + assertEquals("Exception", exceptionNode.get("type").asText()); + assertEquals("test", exceptionNode.get("message").asText()); + assertTrue(exceptionNode.get("stackTrace").asText().contains("test")); - try { - List lines = logFile.readDecompressedLines(); - assertEquals(1, lines.size()); + requestLogger.clear(); - // Parse the line as JSON - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonNode = objectMapper.readTree(lines.get(0)); - - assertEquals("GET", jsonNode.get("request").get("method").asText()); - assertEquals("/items", jsonNode.get("request").get("path").asText()); - assertEquals("http://test/items", jsonNode.get("request").get("url").asText()); - assertFalse(jsonNode.get("request").has("body")); - assertEquals(200, jsonNode.get("response").get("statusCode").asInt()); - assertEquals(0.123, jsonNode.get("response").get("responseTime").asDouble(), 0.001); - assertEquals("{\"items\": []}", - new String(Base64.getDecoder() - .decode(jsonNode.get("response").get("body").asText()))); - - JsonNode requestHeadersNode = jsonNode.get("request").get("headers"); - assertTrue(requestHeadersNode.isArray()); - assertEquals(1, requestHeadersNode.size()); - assertEquals("User-Agent", requestHeadersNode.get(0).get(0).asText()); - assertEquals("Test", requestHeadersNode.get(0).get(1).asText()); - - JsonNode responseHeadersNode = jsonNode.get("response").get("headers"); - assertTrue(responseHeadersNode.isArray()); - assertEquals(1, responseHeadersNode.size()); - assertEquals("Content-Type", responseHeadersNode.get(0).get(0).asText()); - assertEquals("application/json", responseHeadersNode.get(0).get(1).asText()); - - JsonNode exceptionNode = jsonNode.get("exception"); - assertNotNull(exceptionNode); - assertEquals("Exception", exceptionNode.get("type").asText()); - assertEquals("test", exceptionNode.get("message").asText()); - assertTrue(exceptionNode.get("stackTrace").asText().contains("test")); - } catch (IOException e) { - throw new AssertionError("Failed to read gzipped file", e); - } + items = getLoggedItems(requestLogger); + assertEquals(0, items.length); + } - requestLogger.clear(); - requestLogger.maintain(); - requestLogger.rotateFile(); + @Test + void testExcludeBasedOnOptions() { + requestLoggingConfig.setEnabled(true); + requestLoggingConfig.setQueryParamsIncluded(false); + requestLoggingConfig.setRequestHeadersIncluded(false); + requestLoggingConfig.setRequestBodyIncluded(false); + requestLoggingConfig.setResponseHeadersIncluded(false); + requestLoggingConfig.setResponseBodyIncluded(false); + requestLogger = new RequestLogger(requestLoggingConfig); + + Header[] requestHeaders = new Header[] { + new Header("Content-Type", "application/json"), + }; + Header[] responseHeaders = new Header[] { + new Header("Content-Type", "application/json"), + }; + Request request = new Request( + System.currentTimeMillis() / 1000.0, + "tester", + "POST", + "/items", + "http://test/items?token=my-secret-token", + requestHeaders, + 16L, + "{\"key\": \"value\"}".getBytes()); + Response response = new Response( + 200, + 0.123, + responseHeaders, + 16L, + "{\"key\": \"value\"}".getBytes()); - logFile = requestLogger.getFile(); - assertNull(logFile); + requestLogger.logRequest(request, response, null); + + JsonNode[] items = getLoggedItems(requestLogger); + assertEquals(1, items.length); + assertEquals("http://test/items", items[0].get("request").get("url").asText()); + assertFalse(items[0].get("request").has("headers")); + assertFalse(items[0].get("request").has("body")); + assertFalse(items[0].get("response").has("headers")); + assertFalse(items[0].get("response").has("body")); } @Test - void testExclusion() { + void testExcludeBasedOnCallback() { + requestLoggingConfig.setEnabled(true); requestLoggingConfig.setCallbacks(new RequestLoggingCallbacks() { @Override public boolean shouldExclude(Request request, Response response) { - return request.getConsumer().contains("tester"); + return request.getConsumer() != null && request.getConsumer().contains("tester"); } }); + requestLogger = new RequestLogger(requestLoggingConfig); - // Exclude based on shouldExclude() callback Request request = new Request( System.currentTimeMillis() / 1000.0, "tester", @@ -147,10 +175,16 @@ public boolean shouldExclude(Request request, Response response) { new Header[0], 13L, "{\"items\": []}".getBytes()); + requestLogger.logRequest(request, response, null); - // Exclude health check requests - request = new Request( + JsonNode[] items = getLoggedItems(requestLogger); + assertEquals(0, items.length); + } + + @Test + void testExcludeBasedOnPath() { + Request request = new Request( System.currentTimeMillis() / 1000.0, null, "GET", @@ -159,93 +193,259 @@ public boolean shouldExclude(Request request, Response response) { new Header[0], 0L, new byte[0]); - response = new Response( + Response response = new Response( 200, 0.123, new Header[0], 17L, "{\"healthy\": true}".getBytes()); + requestLogger.logRequest(request, response, null); - request = new Request( + JsonNode[] items = getLoggedItems(requestLogger); + assertEquals(0, items.length); + } + + @Test + void testExcludeBasedOnUserAgent() { + Header[] requestHeaders = new Header[] { + new Header("User-Agent", "ELB-HealthChecker/2.0"), + }; + Request request = new Request( System.currentTimeMillis() / 1000.0, null, "GET", "/", "http://test/", - new Header[] { - new Header("User-Agent", "ELB-HealthChecker/2.0"), - }, + requestHeaders, + 0L, + new byte[0]); + Response response = new Response( + 200, + 0, + new Header[0], 0L, new byte[0]); + requestLogger.logRequest(request, response, null); - requestLogger.maintain(); - requestLogger.rotateFile(); - TempGzipFile logFile = requestLogger.getFile(); - assertNull(logFile); + JsonNode[] items = getLoggedItems(requestLogger); + assertEquals(0, items.length); } @Test - void testMasking() { + void testMaskHeaders() { + requestLoggingConfig.setEnabled(true); + requestLoggingConfig.setRequestHeadersIncluded(true); + requestLoggingConfig.setHeaderMaskPatterns(List.of("(?i)test")); + requestLogger = new RequestLogger(requestLoggingConfig); + + Header[] requestHeaders = new Header[] { + new Header("Accept", "text/plain"), + new Header("Authorization", "Bearer 123456"), + new Header("X-Test", "123456"), + }; + Request request = new Request( + System.currentTimeMillis() / 1000.0, + null, + "GET", + "/test", + "http://localhost:8000/test?foo=bar", + requestHeaders, + 0L, + new byte[0]); + Response response = new Response(200, 0, new Header[0], 0L, new byte[0]); + + requestLogger.logRequest(request, response, null); + + JsonNode[] items = getLoggedItems(requestLogger); + assertEquals(1, items.length); + JsonNode requestHeadersNode = items[0].get("request").get("headers"); + + // Convert headers array to a map for easier testing + java.util.Map headers = new java.util.HashMap<>(); + for (JsonNode header : requestHeadersNode) { + headers.put(header.get(0).asText(), header.get(1).asText()); + } + + assertEquals("text/plain", headers.get("Accept")); + assertEquals("******", headers.get("Authorization")); + assertEquals("******", headers.get("X-Test")); + } + + @Test + void testMaskQueryParams() { + requestLoggingConfig.setEnabled(true); + requestLoggingConfig.setQueryParamsIncluded(true); + requestLoggingConfig.setQueryParamMaskPatterns(List.of("(?i)test")); + requestLogger = new RequestLogger(requestLoggingConfig); + + Request request = new Request( + System.currentTimeMillis() / 1000.0, + null, + "GET", + "/test", + "http://localhost/test?secret=123456&test=123456&other=abcdef", + new Header[0], + 0L, + new byte[0]); + Response response = new Response(200, 0, new Header[0], 0L, new byte[0]); + + requestLogger.logRequest(request, response, null); + + JsonNode[] items = getLoggedItems(requestLogger); + assertEquals(1, items.length); + String url = items[0].get("request").get("url").asText(); + assertTrue(url.contains("secret=******")); + assertTrue(url.contains("test=******")); + assertTrue(url.contains("other=abcdef")); + } + + @Test + void testMaskBodyUsingCallback() { + requestLoggingConfig.setEnabled(true); + requestLoggingConfig.setRequestBodyIncluded(true); + requestLoggingConfig.setResponseBodyIncluded(true); requestLoggingConfig.setCallbacks(new RequestLoggingCallbacks() { @Override public byte[] maskRequestBody(Request request) { - return null; + if ("/test".equals(request.getPath())) { + return null; + } + return request.getBody(); } @Override public byte[] maskResponseBody(Request request, Response response) { - return null; + if ("/test".equals(request.getPath())) { + return null; + } + return response.getBody(); } }); + requestLogger = new RequestLogger(requestLoggingConfig); Header[] requestHeaders = new Header[] { - new Header("Authorization", "Bearer 1234567890"), new Header("Content-Type", "application/json"), }; - Header[] responseHeaders = new Header[] { + Request request = new Request( + System.currentTimeMillis() / 1000.0, + null, + "GET", + "/test", + "http://test/test", + requestHeaders, + 4L, + "test".getBytes()); + Response response = new Response( + 200, + 0, + new Header[] { new Header("Content-Type", "application/json") }, + 4L, + "test".getBytes()); + + requestLogger.logRequest(request, response, null); + + JsonNode[] items = getLoggedItems(requestLogger); + assertEquals(1, items.length); + String requestBody = new String(Base64.getDecoder() + .decode(items[0].get("request").get("body").asText())); + assertEquals("", requestBody); + + String responseBody = new String(Base64.getDecoder() + .decode(items[0].get("response").get("body").asText())); + assertEquals("", responseBody); + } + + @Test + void testMaskBodyFields() { + requestLoggingConfig.setEnabled(true); + requestLoggingConfig.setRequestBodyIncluded(true); + requestLoggingConfig.setResponseBodyIncluded(true); + requestLoggingConfig.setBodyFieldMaskPatterns(List.of("(?i)custom")); + requestLogger = new RequestLogger(requestLoggingConfig); + + String requestBodyJson = "{\"username\":\"john_doe\",\"password\":\"secret123\",\"token\":\"abc123\",\"custom\":\"xyz789\",\"user_id\":42,\"api_key\":123,\"normal_field\":\"value\",\"nested\":{\"password\":\"nested_secret\",\"count\":5,\"deeper\":{\"auth\":\"deep_token\"}},\"array\":[{\"password\":\"array_secret\",\"id\":1},{\"normal\":\"text\",\"token\":\"array_token\"}]}"; + String responseBodyJson = "{\"status\":\"success\",\"secret\":\"response_secret\",\"data\":{\"pwd\":\"response_pwd\"}}"; + + Header[] requestHeaders = new Header[] { new Header("Content-Type", "application/json"), }; Request request = new Request( System.currentTimeMillis() / 1000.0, - "tester", + null, "POST", - "/items", - "http://test/items?token=my-secret-token", + "/test", + "http://localhost:8000/test?foo=bar", requestHeaders, - 16L, - "{\"key\": \"value\"}".getBytes()); + (long) requestBodyJson.getBytes().length, + requestBodyJson.getBytes()); Response response = new Response( 200, - 0.123, - responseHeaders, - 16L, - "{\"key\": \"value\"}".getBytes()); + 0.1, + new Header[] { new Header("Content-Type", "application/json") }, + (long) responseBodyJson.getBytes().length, + responseBodyJson.getBytes()); + requestLogger.logRequest(request, response, null); + JsonNode[] items = getLoggedItems(requestLogger); + assertEquals(1, items.length); + + String reqBodyBase64 = items[0].get("request").get("body").asText(); + JsonNode reqBody; + String respBodyBase64 = items[0].get("response").get("body").asText(); + JsonNode respBody; + + try { + ObjectMapper mapper = new ObjectMapper(); + reqBody = mapper.readTree(new String(Base64.getDecoder().decode(reqBodyBase64))); + respBody = mapper.readTree(new String(Base64.getDecoder().decode(respBodyBase64))); + } catch (Exception e) { + throw new AssertionError("Failed to parse JSON bodies", e); + } + + // Test fields that should be masked + assertEquals("******", reqBody.get("password").asText()); + assertEquals("******", reqBody.get("token").asText()); + assertEquals("******", reqBody.get("custom").asText()); + assertEquals("******", reqBody.get("nested").get("password").asText()); + assertEquals("******", reqBody.get("nested").get("deeper").get("auth").asText()); + assertEquals("******", reqBody.get("array").get(0).get("password").asText()); + assertEquals("******", reqBody.get("array").get(1).get("token").asText()); + assertEquals("******", respBody.get("secret").asText()); + assertEquals("******", respBody.get("data").get("pwd").asText()); + + // Test fields that should NOT be masked + assertEquals("john_doe", reqBody.get("username").asText()); + assertEquals(42, reqBody.get("user_id").asInt()); + assertEquals(123, reqBody.get("api_key").asInt()); + assertEquals("value", reqBody.get("normal_field").asText()); + assertEquals(5, reqBody.get("nested").get("count").asInt()); + assertEquals(1, reqBody.get("array").get(0).get("id").asInt()); + assertEquals("text", reqBody.get("array").get(1).get("normal").asText()); + assertEquals("success", respBody.get("status").asText()); + } + + private JsonNode[] getLoggedItems(RequestLogger requestLogger) { requestLogger.maintain(); requestLogger.rotateFile(); + TempGzipFile logFile = requestLogger.getFile(); - assertNotNull(logFile); - assertTrue(logFile.getSize() > 0); + if (logFile == null) { + return new JsonNode[0]; + } try { List lines = logFile.readDecompressedLines(); - assertEquals(1, lines.size()); - + JsonNode[] items = new JsonNode[lines.size()]; ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonNode = objectMapper.readTree(lines.get(0)); - assertEquals("http://test/items?token=******", jsonNode.get("request").get("url").asText()); - assertEquals("", - new String(Base64.getDecoder().decode(jsonNode.get("response").get("body").asText()))); + for (int i = 0; i < lines.size(); i++) { + items[i] = objectMapper.readTree(lines.get(i)); + } - JsonNode requestHeadersNode = jsonNode.get("request").get("headers"); - assertTrue(requestHeadersNode.isArray()); - assertEquals(2, requestHeadersNode.size()); - assertEquals("Authorization", requestHeadersNode.get(0).get(0).asText()); - assertEquals("******", requestHeadersNode.get(0).get(1).asText()); + return items; } catch (IOException e) { throw new AssertionError("Failed to read gzipped file", e); }