diff --git a/.gitignore b/.gitignore index d7749a2f..1e16f692 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + # dep directory vendor @@ -5,4 +10,13 @@ vendor prebid-cache # ide -.vscode/ \ No newline at end of file +.vscode/ + +# autogenerated mac file + +.DS_Store + +# Autogenerated Vim swap files +*~ +*.swp +*.swo diff --git a/config.yaml b/config.yaml index 729d419b..277cee95 100644 --- a/config.yaml +++ b/config.yaml @@ -10,6 +10,8 @@ request_limits: max_size_bytes: 10240 # 10K max_num_values: 10 max_ttl_seconds: 3600 +request_logging: + referer_sampling_rate: 0.0 backend: type: "memory" # Switch to be "aerospike", "cassandra", "memcache", "ignite" or "redis" for production. # aerospike: diff --git a/config/config.go b/config/config.go index 18f41dd5..6e714a7e 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( "net/http" + "strconv" "strings" "time" @@ -97,6 +98,7 @@ func setConfigDefaults(v *viper.Viper) { v.SetDefault("request_limits.max_num_values", utils.REQUEST_MAX_NUM_VALUES) v.SetDefault("request_limits.max_ttl_seconds", utils.REQUEST_MAX_TTL_SECONDS) v.SetDefault("request_limits.max_header_size_bytes", http.DefaultMaxHeaderBytes) + v.SetDefault("request_logging.referer_sampling_rate", 0.0) v.SetDefault("routes.allow_public_write", true) } @@ -114,17 +116,19 @@ func setEnvVarsLookup(v *viper.Viper) { } type Configuration struct { - Port int `mapstructure:"port"` - AdminPort int `mapstructure:"admin_port"` - IndexResponse string `mapstructure:"index_response"` - Log Log `mapstructure:"log"` - RateLimiting RateLimiting `mapstructure:"rate_limiter"` - RequestLimits RequestLimits `mapstructure:"request_limits"` - StatusResponse string `mapstructure:"status_response"` - Backend Backend `mapstructure:"backend"` - Compression Compression `mapstructure:"compression"` - Metrics Metrics `mapstructure:"metrics"` - Routes Routes `mapstructure:"routes"` + Port int `mapstructure:"port"` + AdminPort int `mapstructure:"admin_port"` + IndexResponse string `mapstructure:"index_response"` + Log Log `mapstructure:"log"` + RateLimiting RateLimiting `mapstructure:"rate_limiter"` + RequestLimits RequestLimits `mapstructure:"request_limits"` + RequestLogging RequestLogging `mapstructure:"request_logging"` + + StatusResponse string `mapstructure:"status_response"` + Backend Backend `mapstructure:"backend"` + Compression Compression `mapstructure:"compression"` + Metrics Metrics `mapstructure:"metrics"` + Routes Routes `mapstructure:"routes"` } // ValidateAndLog validates the config, terminating the program on any errors. @@ -136,6 +140,7 @@ func (cfg *Configuration) ValidateAndLog() { cfg.Log.validateAndLog() cfg.RateLimiting.validateAndLog() cfg.RequestLimits.validateAndLog() + cfg.RequestLogging.validateAndLog() if err := cfg.Backend.validateAndLog(); err != nil { log.Fatalf("%s", err.Error()) @@ -176,6 +181,21 @@ func (cfg *RateLimiting) validateAndLog() { log.Infof("config.rate_limiter.num_requests: %d", cfg.MaxRequestsPerSecond) } +type RequestLogging struct { + // RefererSamplingRate represents the probability of Prebid Cache loging the incoming request referer header + // chance = 1.0 => always log, + // chance = 0.0 => never log + RefererSamplingRate float64 `mapstructure:"referer_sampling_rate"` +} + +func (cfg *RequestLogging) validateAndLog() { + if cfg.RefererSamplingRate >= 0.0 && cfg.RefererSamplingRate <= 1.0 { + log.Infof("config.request_logging.referer_sampling_rate: %s", strconv.FormatFloat(cfg.RefererSamplingRate, 'f', -1, 64)) + } else { + log.Fatalf("invalid config.request_logging.referer_sampling_rate: value must be positive and not greater than 1.0. Got %s", strconv.FormatFloat(cfg.RefererSamplingRate, 'f', -1, 64)) + } +} + type RequestLimits struct { MaxSize int `mapstructure:"max_size_bytes"` MaxNumValues int `mapstructure:"max_num_values"` diff --git a/config/config_test.go b/config/config_test.go index 1c12cdb1..7e1a5477 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,7 +1,6 @@ package config import ( - "fmt" "os" "path/filepath" "strings" @@ -13,6 +12,7 @@ import ( testLogrus "github.com/sirupsen/logrus/hooks/test" "github.com/spf13/viper" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDefaults(t *testing.T) { @@ -915,6 +915,89 @@ func TestRequestLimitsValidateAndLog(t *testing.T) { } } +func TestRequestLogging(t *testing.T) { + hook := testLogrus.NewGlobal() + + type logComponents struct { + msg string + lvl logrus.Level + } + + testCases := []struct { + name string + inRequestLoggingCfg *RequestLogging + expectedLogInfo []logComponents + }{ + { + name: "invalid_negative", // must be greater or equal to zero. Expect fatal log + inRequestLoggingCfg: &RequestLogging{ + RefererSamplingRate: -0.1, + }, + expectedLogInfo: []logComponents{ + {msg: `invalid config.request_logging.referer_sampling_rate: value must be positive and not greater than 1.0. Got -0.1`, lvl: logrus.FatalLevel}, + }, + }, + { + name: "invalid_high", // must be less than or equal to 1. expect fatal log. + inRequestLoggingCfg: &RequestLogging{ + RefererSamplingRate: 1.1, + }, + expectedLogInfo: []logComponents{ + {msg: `invalid config.request_logging.referer_sampling_rate: value must be positive and not greater than 1.0. Got 1.1`, lvl: logrus.FatalLevel}, + }, + }, + { + name: "valid_one", // sampling rate of 1.0 is between the acceptable threshold. Expect info log" + inRequestLoggingCfg: &RequestLogging{ + RefererSamplingRate: 1.0, + }, + expectedLogInfo: []logComponents{ + {msg: `config.request_logging.referer_sampling_rate: 1`, lvl: logrus.InfoLevel}, + }, + }, + { + name: "valid_zero", // sampling rate of 0.0 is between the acceptable threshold. Expect info log. + inRequestLoggingCfg: &RequestLogging{ + RefererSamplingRate: 0.0, + }, + expectedLogInfo: []logComponents{ + {msg: `config.request_logging.referer_sampling_rate: 0`, lvl: logrus.InfoLevel}, + }, + }, + { + name: "valid", + inRequestLoggingCfg: &RequestLogging{ + RefererSamplingRate: 0.1111, + }, + expectedLogInfo: []logComponents{ + {msg: `config.request_logging.referer_sampling_rate: 0.1111`, lvl: logrus.InfoLevel}, + }, + }, + } + + //substitute logger exit function so execution doesn't get interrupted + defer func() { logrus.StandardLogger().ExitFunc = nil }() + logrus.StandardLogger().ExitFunc = func(int) {} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.inRequestLoggingCfg.validateAndLog() + + // assertions + require.Len(t, hook.Entries, len(tc.expectedLogInfo), tc.name+":log_entries") + for i := 0; i < len(tc.expectedLogInfo); i++ { + assert.Equal(t, tc.expectedLogInfo[i].msg, hook.Entries[i].Message, tc.name+":message") + assert.Equal(t, tc.expectedLogInfo[i].lvl, hook.Entries[i].Level, tc.name+":log_level") + } + + //Reset log after every test and assert successful reset + hook.Reset() + assert.Nil(t, hook.LastEntry()) + + }) + } +} + func TestCompressionValidateAndLog(t *testing.T) { // logrus entries will be recorded to this `hook` object so we can compare and assert them @@ -1081,19 +1164,20 @@ func TestConfigurationValidateAndLog(t *testing.T) { expectedConfig := getExpectedDefaultConfig() expectedLogInfo := []logComponents{ - {msg: fmt.Sprintf("config.port: %d", expectedConfig.Port), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.admin_port: %d", expectedConfig.AdminPort), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.log.level: %s", expectedConfig.Log.Level), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.rate_limiter.enabled: %t", expectedConfig.RateLimiting.Enabled), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.rate_limiter.num_requests: %d", expectedConfig.RateLimiting.MaxRequestsPerSecond), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.request_limits.allow_setting_keys: %v", expectedConfig.RequestLimits.AllowSettingKeys), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.request_limits.max_ttl_seconds: %d", expectedConfig.RequestLimits.MaxTTLSeconds), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.request_limits.max_size_bytes: %d", expectedConfig.RequestLimits.MaxSize), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.request_limits.max_num_values: %d", expectedConfig.RequestLimits.MaxNumValues), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.request_limits.max_header_size_bytes: %d", expectedConfig.RequestLimits.MaxHeaderSize), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.backend.type: %s", expectedConfig.Backend.Type), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("config.compression.type: %s", expectedConfig.Compression.Type), lvl: logrus.InfoLevel}, - {msg: fmt.Sprintf("Prebid Cache will run without metrics"), lvl: logrus.InfoLevel}, + {msg: "config.port: 2424", lvl: logrus.InfoLevel}, + {msg: "config.admin_port: 2525", lvl: logrus.InfoLevel}, + {msg: "config.log.level: info", lvl: logrus.InfoLevel}, + {msg: "config.rate_limiter.enabled: true", lvl: logrus.InfoLevel}, + {msg: "config.rate_limiter.num_requests: 100", lvl: logrus.InfoLevel}, + {msg: "config.request_limits.allow_setting_keys: false", lvl: logrus.InfoLevel}, + {msg: "config.request_limits.max_ttl_seconds: 3600", lvl: logrus.InfoLevel}, + {msg: "config.request_limits.max_size_bytes: 10240", lvl: logrus.InfoLevel}, + {msg: "config.request_limits.max_num_values: 10", lvl: logrus.InfoLevel}, + {msg: "config.request_limits.max_header_size_bytes: 1048576", lvl: logrus.InfoLevel}, + {msg: "config.request_logging.referer_sampling_rate: 0", lvl: logrus.InfoLevel}, + {msg: "config.backend.type: memory", lvl: logrus.InfoLevel}, + {msg: "config.compression.type: snappy", lvl: logrus.InfoLevel}, + {msg: "Prebid Cache will run without metrics", lvl: logrus.InfoLevel}, } // Run test @@ -1102,7 +1186,7 @@ func TestConfigurationValidateAndLog(t *testing.T) { // Assertions if assert.Len(t, hook.Entries, len(expectedLogInfo)) { for i := 0; i < len(expectedLogInfo); i++ { - assert.True(t, strings.HasPrefix(hook.Entries[i].Message, expectedLogInfo[i].msg), "Wrong message") + assert.Equal(t, expectedLogInfo[i].msg, hook.Entries[i].Message, "Wrong message") assert.Equal(t, expectedLogInfo[i].lvl, hook.Entries[i].Level, "Wrong log level") } } @@ -1228,6 +1312,9 @@ func getExpectedDefaultConfig() Configuration { Enabled: true, MaxRequestsPerSecond: 100, }, + RequestLogging: RequestLogging{ + RefererSamplingRate: 0.00, + }, RequestLimits: RequestLimits{ MaxSize: 10240, MaxNumValues: 10, diff --git a/endpoints/get.go b/endpoints/get.go index ebd00e54..cd225ca6 100644 --- a/endpoints/get.go +++ b/endpoints/get.go @@ -16,20 +16,28 @@ import ( // GetHandler serves "GET /cache" requests. type GetHandler struct { - backend backends.Backend - metrics *metrics.Metrics + backend backends.Backend + metrics *metrics.Metrics + cfg getHandlerConfig +} + +type getHandlerConfig struct { allowCustomKeys bool + refererLogRate float64 } // NewGetHandler returns the handle function for the "/cache" endpoint when it receives a GET request -func NewGetHandler(storage backends.Backend, metrics *metrics.Metrics, allowCustomKeys bool) func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { +func NewGetHandler(storage backends.Backend, metrics *metrics.Metrics, allowCustomKeys bool, refererSamplingRate float64) func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { getHandler := &GetHandler{ // Assign storage client to get endpoint backend: storage, // pass metrics engine metrics: metrics, - // Pass configuration value - allowCustomKeys: allowCustomKeys, + // Pass configuration values + cfg: getHandlerConfig{ + allowCustomKeys: allowCustomKeys, + refererLogRate: refererSamplingRate, + }, } // Return handle function @@ -38,9 +46,16 @@ func NewGetHandler(storage backends.Backend, metrics *metrics.Metrics, allowCust func (e *GetHandler) handle(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { e.metrics.RecordGetTotal() + + // If incoming request comes with a referer header, there's a e.cfg.refererLogRate percent chance + // getting it logged + if referer := r.Referer(); referer != "" && utils.RandomPick(e.cfg.refererLogRate) { + log.Info("GET request Referer header: " + referer) + } + start := time.Now() - uuid, parseErr := parseUUID(r, e.allowCustomKeys) + uuid, parseErr := parseUUID(r, e.cfg.allowCustomKeys) if parseErr != nil { // parseUUID either returns http.StatusBadRequest or http.StatusNotFound. Both should be // accounted using RecordGetBadRequest() diff --git a/endpoints/get_test.go b/endpoints/get_test.go index 910c90f6..53df79b7 100644 --- a/endpoints/get_test.go +++ b/endpoints/get_test.go @@ -33,19 +33,28 @@ func TestGetJsonTests(t *testing.T) { } router := httprouter.New() - router.GET("/cache", NewGetHandler(backend, m, tc.HostConfig.AllowSettingKeys)) - request, err := http.NewRequest("GET", "/cache?"+tc.Query, nil) + router.GET("/cache", NewGetHandler(backend, m, tc.HostConfig.AllowSettingKeys, tc.HostConfig.RefererLogRate)) + request, err := http.NewRequest("GET", "/cache?"+tc.Request.Query, nil) if !assert.NoError(t, err, "Failed to create a GET request: %v", err) { hook.Reset() assert.Nil(t, hook.LastEntry()) continue } + + if len(tc.Request.Headers) > 0 { + for header, values := range tc.Request.Headers { + for _, v := range values { + request.Header.Set(header, v) + } + } + } + rr := httptest.NewRecorder() - // RUN TEST + // Run test router.ServeHTTP(rr, request) - // ASSERTIONS + // Assertions assert.Equal(t, tc.ExpectedOutput.Code, rr.Code, testFile) // Assert this is a valid test that expects either an error or a GetResponse @@ -82,7 +91,7 @@ func TestGetInvalidUUIDs(t *testing.T) { }, } - router.GET("/cache", NewGetHandler(backend, m, false)) + router.GET("/cache", NewGetHandler(backend, m, false, 0.0)) getResults := doMockGet(t, router, "fdd9405b-ef2b-46da-a55a-2f526d338e16") if getResults.Code != http.StatusNotFound { @@ -108,9 +117,14 @@ func TestGetHandler(t *testing.T) { msg string lvl logrus.Level } + type testConfig struct { + allowKeys bool + refererSamplingRate float64 + } type testInput struct { - uuid string - allowKeys bool + uuid string + cfg testConfig + reqHeaders map[string]string } type testOutput struct { responseCode int @@ -127,8 +141,7 @@ func TestGetHandler(t *testing.T) { { "Missing UUID. Return http error but don't interrupt server's execution", testInput{ - uuid: "", - allowKeys: false, + uuid: "", }, testOutput{ responseCode: http.StatusBadRequest, @@ -148,8 +161,7 @@ func TestGetHandler(t *testing.T) { { "Prebid Cache wasn't configured to allow custom keys therefore, it doesn't allow for keys different than 36 char long. Respond with http error and don't interrupt server's execution", testInput{ - uuid: "non-36-char-key-maps-to-json", - allowKeys: false, + uuid: "non-36-char-key-maps-to-json", }, testOutput{ responseCode: http.StatusNotFound, @@ -169,8 +181,8 @@ func TestGetHandler(t *testing.T) { { "Configuration that allows custom keys. These are not required to be 36 char long. Since the uuid maps to a value, return it along a 200 status code", testInput{ - uuid: "non-36-char-key-maps-to-json", - allowKeys: true, + uuid: "non-36-char-key-maps-to-json", + cfg: testConfig{allowKeys: true}, }, testOutput{ responseCode: http.StatusOK, @@ -231,6 +243,48 @@ func TestGetHandler(t *testing.T) { }, }, }, + { + "Sampling rate is set to 100% but request comes with no referer header. No logs expected.", + testInput{ + uuid: "36-char-key-maps-to-actual-xml-value", + cfg: testConfig{refererSamplingRate: 1.0}, + reqHeaders: map[string]string{"OtherHeader": "headervalue"}, + }, + testOutput{ + responseCode: http.StatusOK, + responseBody: "xml data here", + logEntries: []logEntry{}, + expectedMetrics: []string{ + "RecordGetTotal", + "RecordGetDuration", + }, + }, + }, + { + "Sampling rate is set to 100%. Expect request referer header to be logged.", + testInput{ + uuid: "36-char-key-maps-to-actual-xml-value", + cfg: testConfig{refererSamplingRate: 1.0}, + reqHeaders: map[string]string{ + "Referer": "anyreferer", + "OtherHeader": "headervalue", + }, + }, + testOutput{ + responseCode: http.StatusOK, + responseBody: "xml data here", + logEntries: []logEntry{ + { + msg: "GET request Referer header: anyreferer", + lvl: logrus.InfoLevel, + }, + }, + expectedMetrics: []string{ + "RecordGetTotal", + "RecordGetDuration", + }, + }, + }, } // Lower Log Treshold so we can see DebugLevel entries in our mock logrus log @@ -260,13 +314,20 @@ func TestGetHandler(t *testing.T) { &mockMetrics, }, } - router.GET("/cache", NewGetHandler(backend, m, test.in.allowKeys)) + router.GET("/cache", NewGetHandler(backend, m, test.in.cfg.allowKeys, test.in.cfg.refererSamplingRate)) // Run test getResults := httptest.NewRecorder() body := new(bytes.Buffer) getReq, err := http.NewRequest("GET", "/cache"+"?uuid="+test.in.uuid, body) + + if len(test.in.reqHeaders) > 0 { + for k, v := range test.in.reqHeaders { + getReq.Header.Set(k, v) + } + } + if !assert.NoError(t, err, "Failed to create a GET request: %v", err) { hook.Reset() continue diff --git a/endpoints/put.go b/endpoints/put.go index 13f0771c..702cdbe4 100644 --- a/endpoints/put.go +++ b/endpoints/put.go @@ -26,8 +26,9 @@ type PutHandler struct { } type putHandlerConfig struct { - maxNumValues int - allowKeys bool + maxNumValues int + allowKeys bool + refererLogRate float64 } type syncPools struct { @@ -36,7 +37,7 @@ type syncPools struct { } // NewPutHandler returns the handle function for the "/cache" endpoint when it receives a POST request -func NewPutHandler(storage backends.Backend, metrics *metrics.Metrics, maxNumValues int, allowKeys bool) func(http.ResponseWriter, *http.Request, httprouter.Params) { +func NewPutHandler(storage backends.Backend, metrics *metrics.Metrics, maxNumValues int, allowKeys bool, refererLogRate float64) func(http.ResponseWriter, *http.Request, httprouter.Params) { putHandler := &PutHandler{} // Assign storage client to put endpoint @@ -47,8 +48,9 @@ func NewPutHandler(storage backends.Backend, metrics *metrics.Metrics, maxNumVal // Pass configuration values putHandler.cfg = putHandlerConfig{ - maxNumValues: maxNumValues, - allowKeys: allowKeys, + maxNumValues: maxNumValues, + allowKeys: allowKeys, + refererLogRate: refererLogRate, } // Instantiate thread-safe memory pools @@ -184,6 +186,12 @@ func logBackendError(err error) { func (e *PutHandler) handle(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { e.metrics.RecordPutTotal() + // If incoming request comes with a referer header, there's a e.cfg.refererLogRate percent chance + // getting it logged + if referer := r.Referer(); referer != "" && utils.RandomPick(e.cfg.refererLogRate) { + logrus.Info("POST request Referer header: " + referer) + } + start := time.Now() bytes, err := e.processPutRequest(r) diff --git a/endpoints/put_test.go b/endpoints/put_test.go index abc143f0..428c052c 100644 --- a/endpoints/put_test.go +++ b/endpoints/put_test.go @@ -50,12 +50,21 @@ func TestPutJsonTests(t *testing.T) { } router := httprouter.New() - router.POST("/cache", NewPutHandler(backend, m, tc.HostConfig.MaxNumValues, tc.HostConfig.AllowSettingKeys)) - request, err := http.NewRequest("POST", "/cache", strings.NewReader(string(tc.PutRequest))) + router.POST("/cache", NewPutHandler(backend, m, tc.HostConfig.MaxNumValues, tc.HostConfig.AllowSettingKeys, tc.HostConfig.RefererLogRate)) + request, err := http.NewRequest("POST", "/cache", strings.NewReader(string(tc.Request.Body))) if !assert.NoError(t, err, "Failed to create a POST request. Test file: %s Error: %v", testFile, err) { hook.Reset() continue } + + if len(tc.Request.Headers) > 0 { + for header, values := range tc.Request.Headers { + for _, v := range values { + request.Header.Set(header, v) + } + } + } + rr := httptest.NewRecorder() // RUN TEST @@ -96,12 +105,17 @@ func TestPutJsonTests(t *testing.T) { } type testData struct { - HostConfig hostConfig `json:"config"` - PutRequest json.RawMessage `json:"put_request"` - ExpectedOutput expectedOut `json:"expected_output"` - ExpectedLogEntries []logEntry `json:"expected_log_entries"` - ExpectedMetrics []string `json:"expected_metrics"` - Query string `json:"get_request_query"` + HostConfig hostConfig `json:"config"` + Request testRequest `json:"request"` + ExpectedOutput expectedOut `json:"expected_output"` + ExpectedLogEntries []logEntry `json:"expected_log_entries"` + ExpectedMetrics []string `json:"expected_metrics"` +} + +type testRequest struct { + Body json.RawMessage `json:"body"` + Headers map[string][]string `json:"headers"` + Query string `json:"query"` } type expectedOut struct { @@ -122,6 +136,7 @@ type hostConfig struct { MaxNumValues int `json:"max_num_values"` MaxTTLSeconds int `json:"max_ttl_seconds"` FakeBackend fakeBackend `json:"fake_backend"` + RefererLogRate float64 `json:"referer_sampling_rate"` } type fakeBackend struct { @@ -551,8 +566,8 @@ func TestSuccessfulPut(t *testing.T) { }, } - router.POST("/cache", NewPutHandler(backend, m, 10, true)) - router.GET("/cache", NewGetHandler(backend, m, true)) + router.POST("/cache", NewPutHandler(backend, m, 10, true, 0.0)) + router.GET("/cache", NewGetHandler(backend, m, true, 0.0)) // Feed the tests input put request to the endpoint's handle putResponse := doPut(t, router, tc.inPutBody) @@ -641,7 +656,7 @@ func TestMalformedOrInvalidValue(t *testing.T) { }, } - router.POST("/cache", NewPutHandler(backend, m, 10, true)) + router.POST("/cache", NewPutHandler(backend, m, 10, true, 0.0)) // Run test putResponse := doPut(t, router, tc.inPutBody) @@ -677,7 +692,7 @@ func TestNonSupportedType(t *testing.T) { &mockMetrics, }, } - router.POST("/cache", NewPutHandler(backend, m, 10, true)) + router.POST("/cache", NewPutHandler(backend, m, 10, true, 0.0)) putResponse := doPut(t, router, requestBody) @@ -713,7 +728,7 @@ func TestPutNegativeTTL(t *testing.T) { }, } - testRouter.POST("/cache", NewPutHandler(testBackend, m, 10, true)) + testRouter.POST("/cache", NewPutHandler(testBackend, m, 10, true, 0.0)) recorder := httptest.NewRecorder() @@ -815,7 +830,7 @@ func TestCustomKey(t *testing.T) { } router := httprouter.New() - putEndpointHandler := NewPutHandler(mockBackendWithValues, m, 10, tgroup.allowSettingKeys) + putEndpointHandler := NewPutHandler(mockBackendWithValues, m, 10, tgroup.allowSettingKeys, 0.0) router.POST("/cache", putEndpointHandler) recorder := httptest.NewRecorder() @@ -852,7 +867,7 @@ func TestRequestReadError(t *testing.T) { &mockMetrics, }, } - putEndpointHandler := NewPutHandler(mockBackendWithValues, m, 10, false) + putEndpointHandler := NewPutHandler(mockBackendWithValues, m, 10, false, 0.0) router := httprouter.New() router.POST("/cache", putEndpointHandler) @@ -897,7 +912,7 @@ func TestTooManyPutElements(t *testing.T) { &mockMetrics, }, } - router.POST("/cache", NewPutHandler(backend, m, len(putElements)-1, true)) + router.POST("/cache", NewPutHandler(backend, m, len(putElements)-1, true, 0.0)) putResponse := doPut(t, router, reqBody) @@ -952,8 +967,8 @@ func TestMultiPutRequest(t *testing.T) { }, } - router.POST("/cache", NewPutHandler(backend, m, 10, true)) - router.GET("/cache", NewGetHandler(backend, m, true)) + router.POST("/cache", NewPutHandler(backend, m, 10, true, 0.0)) + router.GET("/cache", NewGetHandler(backend, m, true, 0.0)) rr := httptest.NewRecorder() @@ -1003,7 +1018,7 @@ func TestBadPayloadSizePutError(t *testing.T) { &mockMetrics, }, } - router.POST("/cache", NewPutHandler(backend, m, 10, true)) + router.POST("/cache", NewPutHandler(backend, m, 10, true, 0.0)) putResponse := doPut(t, router, reqBody) @@ -1040,7 +1055,7 @@ func TestInternalPutClientError(t *testing.T) { // Use mock client that will return an error backendWithMetrics := decorators.LogMetrics(newErrorReturningBackend(), m) - router.POST("/cache", NewPutHandler(backendWithMetrics, m, 10, true)) + router.POST("/cache", NewPutHandler(backendWithMetrics, m, 10, true, 0.0)) // Run test putResponse := doPut(t, router, reqBody) @@ -1114,7 +1129,7 @@ func TestEmptyPutRequests(t *testing.T) { }, } router := httprouter.New() - router.POST("/cache", NewPutHandler(backend, m, 10, true)) + router.POST("/cache", NewPutHandler(backend, m, 10, true, 0.0)) rr := httptest.NewRecorder() // Create request everytime @@ -1157,7 +1172,7 @@ func TestPutClientDeadlineExceeded(t *testing.T) { &mockMetrics, }, } - router.POST("/cache", NewPutHandler(backend, m, 10, true)) + router.POST("/cache", NewPutHandler(backend, m, 10, true, 0.0)) putResponse := doPut(t, router, reqBody) @@ -1453,8 +1468,8 @@ func benchmarkPutHandler(b *testing.B, testCase string) { }, } - router.POST("/cache", NewPutHandler(backend, m, 10, true)) - router.GET("/cache", NewGetHandler(backend, m, true)) + router.POST("/cache", NewPutHandler(backend, m, 10, true, 0.0)) + router.GET("/cache", NewGetHandler(backend, m, true, 0.0)) rr := httptest.NewRecorder() diff --git a/endpoints/routing/handler.go b/endpoints/routing/handler.go index 477b543f..c5141ef5 100644 --- a/endpoints/routing/handler.go +++ b/endpoints/routing/handler.go @@ -37,12 +37,12 @@ func NewPublicHandler(cfg config.Configuration, dataStore backends.Backend, appM func addReadRoutes(cfg config.Configuration, dataStore backends.Backend, appMetrics *metrics.Metrics, router *httprouter.Router) { router.GET("/", endpoints.NewIndexHandler(cfg.IndexResponse)) // Default route handler router.GET("/status", endpoints.NewStatusEndpoint(cfg.StatusResponse)) // Determines whether the server is ready for more traffic. - router.GET("/cache", endpoints.NewGetHandler(dataStore, appMetrics, cfg.RequestLimits.AllowSettingKeys)) + router.GET("/cache", endpoints.NewGetHandler(dataStore, appMetrics, cfg.RequestLimits.AllowSettingKeys, cfg.RequestLogging.RefererSamplingRate)) router.GET("/version", endpoints.NewVersionEndpoint(version.Ver, version.Rev)) } func addWriteRoutes(cfg config.Configuration, dataStore backends.Backend, appMetrics *metrics.Metrics, router *httprouter.Router) { - router.POST("/cache", endpoints.NewPutHandler(dataStore, appMetrics, cfg.RequestLimits.MaxNumValues, cfg.RequestLimits.AllowSettingKeys)) + router.POST("/cache", endpoints.NewPutHandler(dataStore, appMetrics, cfg.RequestLimits.MaxNumValues, cfg.RequestLimits.AllowSettingKeys, cfg.RequestLogging.RefererSamplingRate)) } func handleCors(handler http.Handler) http.Handler { diff --git a/endpoints/sample-requests/get-endpoint/invalid/data-corrupted.json b/endpoints/sample-requests/get-endpoint/invalid/data-corrupted.json index 9092350a..ec3a048a 100644 --- a/endpoints/sample-requests/get-endpoint/invalid/data-corrupted.json +++ b/endpoints/sample-requests/get-endpoint/invalid/data-corrupted.json @@ -10,7 +10,9 @@ ] } }, - "get_request_query": "uuid=36-char-uid-maps-to-actual-xml-value", + "request": { + "query": "uuid=36-char-uid-maps-to-actual-xml-value" + }, "expected_log_entries": [ { "message": "GET /cache uuid=36-char-uid-maps-to-actual-xml-value: Cache data was corrupted. Cannot determine type.", diff --git a/endpoints/sample-requests/get-endpoint/invalid/key-not-found.json b/endpoints/sample-requests/get-endpoint/invalid/key-not-found.json index 7bf3c9a1..0d83ad0e 100644 --- a/endpoints/sample-requests/get-endpoint/invalid/key-not-found.json +++ b/endpoints/sample-requests/get-endpoint/invalid/key-not-found.json @@ -10,7 +10,9 @@ ] } }, - "get_request_query": "uuid=36-char-uuid-is-not-found-in-backend", + "request": { + "query": "uuid=36-char-uuid-is-not-found-in-backend" + }, "expected_metrics": [ "RecordGetTotal", "RecordGetBackendError", diff --git a/endpoints/sample-requests/get-endpoint/invalid/missing-uuid.json b/endpoints/sample-requests/get-endpoint/invalid/missing-uuid.json index c8ac3f41..5b8723a9 100644 --- a/endpoints/sample-requests/get-endpoint/invalid/missing-uuid.json +++ b/endpoints/sample-requests/get-endpoint/invalid/missing-uuid.json @@ -10,7 +10,9 @@ ] } }, - "get_request_query": "uuid=", + "request": { + "query": "uuid=" + }, "expected_log_entries": [ { "message": "GET /cache: Missing required parameter uuid", diff --git a/endpoints/sample-requests/get-endpoint/invalid/uuid-length.json b/endpoints/sample-requests/get-endpoint/invalid/uuid-length.json index c83ed226..b7b71f84 100644 --- a/endpoints/sample-requests/get-endpoint/invalid/uuid-length.json +++ b/endpoints/sample-requests/get-endpoint/invalid/uuid-length.json @@ -1,6 +1,8 @@ { "description": "Gut request doesn't come with a UUID value in the URL query, expect MISSING_KEY error", - "get_request_query": "uuid=non-36-char-uuid", + "request": { + "query": "uuid=non-36-char-uuid" + }, "expected_log_entries": [ { "message": "GET /cache uuid=non-36-char-uuid: invalid uuid length", diff --git a/endpoints/sample-requests/get-endpoint/valid/element-found.json b/endpoints/sample-requests/get-endpoint/valid/element-found.json index 763dc3f0..c63451bb 100644 --- a/endpoints/sample-requests/get-endpoint/valid/element-found.json +++ b/endpoints/sample-requests/get-endpoint/valid/element-found.json @@ -10,7 +10,9 @@ ] } }, - "get_request_query": "uuid=36-char-uid-maps-to-stored-xml-value", + "request": { + "query": "uuid=36-char-uid-maps-to-stored-xml-value" + }, "expected_metrics": [ "RecordGetBackendTotal", "RecordGetDuration", @@ -21,4 +23,4 @@ "code": 200, "get_response": "stored xml value" } -} \ No newline at end of file +} diff --git a/endpoints/sample-requests/get-endpoint/valid/log-referrer-header.json b/endpoints/sample-requests/get-endpoint/valid/log-referrer-header.json new file mode 100644 index 00000000..ebc5637a --- /dev/null +++ b/endpoints/sample-requests/get-endpoint/valid/log-referrer-header.json @@ -0,0 +1,36 @@ +{ + "description": "Prebid Cache configured to log the referer header of 100% of incoming requests. Referer successfully logged.", + "config": { + "fake_backend": { + "stored_data": [ + { + "key": "36-char-uid-maps-to-stored-xml-value", + "value": "xmlstored xml value" + } + ] + }, + "referer_sampling_rate": 1.0 + }, + "request": { + "query": "uuid=36-char-uid-maps-to-stored-xml-value", + "headers": { + "Referer": [ "anyreferer" ] + } + }, + "expected_log_entries": [ + { + "message": "GET request Referer header: anyreferer", + "level": 4 + } + ], + "expected_metrics": [ + "RecordGetBackendTotal", + "RecordGetDuration", + "RecordGetBackendDuration", + "RecordGetTotal" + ], + "expected_output": { + "code": 200, + "get_response": "stored xml value" + } +} diff --git a/endpoints/sample-requests/put-endpoint/backends/ignite/record-exists-error.json b/endpoints/sample-requests/put-endpoint/backends/ignite/record-exists-error.json index 840cd15c..68035f67 100644 --- a/endpoints/sample-requests/put-endpoint/backends/ignite/record-exists-error.json +++ b/endpoints/sample-requests/put-endpoint/backends/ignite/record-exists-error.json @@ -6,13 +6,15 @@ "server_response": "{\"successStatus\":0,\"error\":\"\",\"response\":false}" } }, - "put_request": { - "puts": [ - { - "type": "xml", - "value": "xmlanother_XML" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "xmlanother_XML" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", diff --git a/endpoints/sample-requests/put-endpoint/backends/redis/invalid-redis-server-error.json b/endpoints/sample-requests/put-endpoint/backends/redis/invalid-redis-server-error.json index c6fbd27d..bea87afd 100644 --- a/endpoints/sample-requests/put-endpoint/backends/redis/invalid-redis-server-error.json +++ b/endpoints/sample-requests/put-endpoint/backends/redis/invalid-redis-server-error.json @@ -6,13 +6,15 @@ "throw_error_message": "Redis server side error" } }, - "put_request": { - "puts": [ - { - "type": "xml", - "value": "xmlanother_XML" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "xmlanother_XML" + } + ] + } }, "expected_log_entries": [ { diff --git a/endpoints/sample-requests/put-endpoint/backends/redis/valid-overwrite.json b/endpoints/sample-requests/put-endpoint/backends/redis/valid-overwrite.json index 71bfec62..dc29e502 100644 --- a/endpoints/sample-requests/put-endpoint/backends/redis/valid-overwrite.json +++ b/endpoints/sample-requests/put-endpoint/backends/redis/valid-overwrite.json @@ -12,14 +12,16 @@ ] } }, - "put_request": { - "puts": [ - { - "key": "the-custom-thirty-six-character-uuid", - "type": "xml", - "value": "NEW_XML" - } - ] + "request": { + "body": { + "puts": [ + { + "key": "the-custom-thirty-six-character-uuid", + "type": "xml", + "value": "NEW_XML" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -34,7 +36,9 @@ "code": 200, "put_response": { "responses": [ - {"uuid": ""} + { + "uuid": "" + } ] } } diff --git a/endpoints/sample-requests/put-endpoint/backends/redis/valid-success-true.json b/endpoints/sample-requests/put-endpoint/backends/redis/valid-success-true.json index fbc4f998..bcd4acdb 100644 --- a/endpoints/sample-requests/put-endpoint/backends/redis/valid-success-true.json +++ b/endpoints/sample-requests/put-endpoint/backends/redis/valid-success-true.json @@ -7,13 +7,15 @@ "throw_error_message": "redis: nil" } }, - "put_request": { - "puts": [ - { - "type": "xml", - "value": "info<\\/VAST>\r\n" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "info<\\/VAST>\r\n" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -27,7 +29,9 @@ "code": 200, "put_response": { "responses": [ - {"uuid": "random"} + { + "uuid": "random" + } ] } } diff --git a/endpoints/sample-requests/put-endpoint/custom-keys/allowed/key-field-included.json b/endpoints/sample-requests/put-endpoint/custom-keys/allowed/key-field-included.json index 3d3f8e9d..7381079a 100644 --- a/endpoints/sample-requests/put-endpoint/custom-keys/allowed/key-field-included.json +++ b/endpoints/sample-requests/put-endpoint/custom-keys/allowed/key-field-included.json @@ -3,14 +3,16 @@ "config": { "allow_setting_keys": true }, - "put_request": { - "puts": [ - { - "type": "xml", - "value": "other_XML_content", - "key": "the-custom-thirty-six-character-uuid" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "other_XML_content", + "key": "the-custom-thirty-six-character-uuid" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", diff --git a/endpoints/sample-requests/put-endpoint/custom-keys/allowed/key-field-missing.json b/endpoints/sample-requests/put-endpoint/custom-keys/allowed/key-field-missing.json index 907dd49a..b34f6583 100644 --- a/endpoints/sample-requests/put-endpoint/custom-keys/allowed/key-field-missing.json +++ b/endpoints/sample-requests/put-endpoint/custom-keys/allowed/key-field-missing.json @@ -3,13 +3,15 @@ "config": { "allow_setting_keys": true }, - "put_request": { - "puts": [ - { - "type": "xml", - "value": "__video_info__<\\/VAST>\r\n" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "__video_info__<\\/VAST>\r\n" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -23,7 +25,9 @@ "code": 200, "put_response": { "responses": [ - {"uuid": "random"} + { + "uuid": "random" + } ] } } diff --git a/endpoints/sample-requests/put-endpoint/custom-keys/not-allowed/key-field-included.json b/endpoints/sample-requests/put-endpoint/custom-keys/not-allowed/key-field-included.json index 923a53fb..d9f4b3e1 100644 --- a/endpoints/sample-requests/put-endpoint/custom-keys/not-allowed/key-field-included.json +++ b/endpoints/sample-requests/put-endpoint/custom-keys/not-allowed/key-field-included.json @@ -1,13 +1,15 @@ { "description": "Put request wants to store element under a custom key but custom keys are not allowed in Prebid Cache's config. Store under a random UUID", - "put_request": { - "puts":[ - { - "type":"xml", - "value":"other_XML_content", - "key":"the-custom-thirty-six-character-uuid" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "other_XML_content", + "key": "the-custom-thirty-six-character-uuid" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -21,7 +23,9 @@ "code": 200, "put_response": { "responses": [ - {"uuid": "random"} + { + "uuid": "random" + } ] } } diff --git a/endpoints/sample-requests/put-endpoint/invalid-number-of-elements/puts-max-num-values.json b/endpoints/sample-requests/put-endpoint/invalid-number-of-elements/puts-max-num-values.json index c63313ae..0907c834 100644 --- a/endpoints/sample-requests/put-endpoint/invalid-number-of-elements/puts-max-num-values.json +++ b/endpoints/sample-requests/put-endpoint/invalid-number-of-elements/puts-max-num-values.json @@ -1,13 +1,23 @@ { "description": "Put request wants to store more elements than allowed in the 'max_num_values' configuration. Don't store and return error", "config": { - "max_num_values": 1 + "max_num_values": 1 }, - "put_request": { - "puts": [ - {"type":"xml","ttlseconds":5,"value":"__video_info__<\\/VAST>\r\n"}, - {"type":"json","ttlseconds":5,"value":"{\"field\":100}"} - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "ttlseconds": 5, + "value": "__video_info__<\\/VAST>\r\n" + }, + { + "type": "json", + "ttlseconds": 5, + "value": "{\"field\":100}" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", diff --git a/endpoints/sample-requests/put-endpoint/invalid-types/type-missing.json b/endpoints/sample-requests/put-endpoint/invalid-types/type-missing.json index 9dda7a29..95f931a3 100644 --- a/endpoints/sample-requests/put-endpoint/invalid-types/type-missing.json +++ b/endpoints/sample-requests/put-endpoint/invalid-types/type-missing.json @@ -1,11 +1,13 @@ { "description": "Request is missing the 'type' field. Respond with error", - "put_request": { - "puts": [ - { - "value": "__video_info__<\\/VAST>\r\n" - } - ] + "request": { + "body": { + "puts": [ + { + "value": "__video_info__<\\/VAST>\r\n" + } + ] + } }, "expected_log_entries": [ { diff --git a/endpoints/sample-requests/put-endpoint/invalid-types/type-unknown.json b/endpoints/sample-requests/put-endpoint/invalid-types/type-unknown.json index 7dc52430..79a13ca6 100644 --- a/endpoints/sample-requests/put-endpoint/invalid-types/type-unknown.json +++ b/endpoints/sample-requests/put-endpoint/invalid-types/type-unknown.json @@ -1,12 +1,14 @@ { "description": "Prebid Cache only allows to store JSON or XML types and the type 'unknown' is not supported. Respond with error", - "put_request": { - "puts": [ - { - "type": "unknown", - "value": "some-value" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "unknown", + "value": "some-value" + } + ] + } }, "expected_log_entries": [ { diff --git a/endpoints/sample-requests/put-endpoint/invalid-value/value-greater-than-max.json b/endpoints/sample-requests/put-endpoint/invalid-value/value-greater-than-max.json index cfd70be7..abbe9b5b 100644 --- a/endpoints/sample-requests/put-endpoint/invalid-value/value-greater-than-max.json +++ b/endpoints/sample-requests/put-endpoint/invalid-value/value-greater-than-max.json @@ -3,13 +3,15 @@ "config": { "max_size_bytes": 1 }, - "put_request": { - "puts": [ - { - "type": "xml", - "value": "\r\n<\\/VAST>\r\n" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "\r\n<\\/VAST>\r\n" + } + ] + } }, "expected_log_entries": [ { diff --git a/endpoints/sample-requests/put-endpoint/invalid-value/value-missing.json b/endpoints/sample-requests/put-endpoint/invalid-value/value-missing.json index 3343b843..8325a6a6 100644 --- a/endpoints/sample-requests/put-endpoint/invalid-value/value-missing.json +++ b/endpoints/sample-requests/put-endpoint/invalid-value/value-missing.json @@ -1,11 +1,13 @@ { "description": "Prebid Cache returns an error if a request doesn't come with a 'value' field.", - "put_request": { - "puts": [ - { - "type": "xml" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml" + } + ] + } }, "expected_log_entries": [ { diff --git a/endpoints/sample-requests/put-endpoint/valid-whole/log-referrer-header.json b/endpoints/sample-requests/put-endpoint/valid-whole/log-referrer-header.json new file mode 100644 index 00000000..e1c679cb --- /dev/null +++ b/endpoints/sample-requests/put-endpoint/valid-whole/log-referrer-header.json @@ -0,0 +1,43 @@ +{ + "description": "Prebid Cache configured to log the referer header of 100% of incoming requests. Referer successfully logged.", + "config": { + "referer_sampling_rate": 1.0 + }, + "request": { + "body": { + "puts": [ + { + "type": "json", + "value": "{\"field\":100}" + } + ] + }, + "headers": { + "Referer": [ "anyreferer" ] + } + }, + "expected_log_entries": [ + { + "message": "POST request Referer header: anyreferer", + "level": 4 + } + ], + "expected_metrics": [ + "RecordPutTotal", + "RecordPutBackendJson", + "RecordPutBackendSize", + "RecordPutBackendTTLSeconds", + "RecordPutBackendDuration", + "RecordPutDuration" + ], + "expected_output": { + "code": 200, + "put_response": { + "responses": [ + { + "uuid": "random" + } + ] + } + } +} diff --git a/endpoints/sample-requests/put-endpoint/valid-whole/multiple-elements-to-store.json b/endpoints/sample-requests/put-endpoint/valid-whole/multiple-elements-to-store.json index f37d3f71..c0b9ad12 100644 --- a/endpoints/sample-requests/put-endpoint/valid-whole/multiple-elements-to-store.json +++ b/endpoints/sample-requests/put-endpoint/valid-whole/multiple-elements-to-store.json @@ -3,19 +3,21 @@ "config": { "max_num_values": 2 }, - "put_request": { - "puts": [ - { - "type": "xml", - "ttlseconds": 60, - "value": "__video_info__<\\/VAST>\r\n" - }, - { - "type": "json", - "ttlseconds": 60, - "value": "{\"an_int_field\": 1}" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "ttlseconds": 60, + "value": "__video_info__<\\/VAST>\r\n" + }, + { + "type": "json", + "ttlseconds": 60, + "value": "{\"an_int_field\": 1}" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -30,8 +32,12 @@ "code": 200, "put_response": { "responses": [ - {"uuid": "random"}, - {"uuid": "random"} + { + "uuid": "random" + }, + { + "uuid": "random" + } ] } } diff --git a/endpoints/sample-requests/put-endpoint/valid-whole/no-elements-to-store.json b/endpoints/sample-requests/put-endpoint/valid-whole/no-elements-to-store.json index af392319..bf2600f9 100644 --- a/endpoints/sample-requests/put-endpoint/valid-whole/no-elements-to-store.json +++ b/endpoints/sample-requests/put-endpoint/valid-whole/no-elements-to-store.json @@ -1,7 +1,9 @@ { "description": "Put request with empty 'puts' array does not return an error, we simply respond with an emtpy 'responses' array.", - "put_request": { - "puts": [] + "request": { + "body": { + "puts": [] + } }, "expected_metrics": [ "RecordPutTotal", @@ -10,7 +12,7 @@ "expected_output": { "code": 200, "put_response": { - "responses": [ ] + "responses": [] } } } diff --git a/endpoints/sample-requests/put-endpoint/valid-whole/record-exists.json b/endpoints/sample-requests/put-endpoint/valid-whole/record-exists.json index 0fd8e8bf..8130ec87 100644 --- a/endpoints/sample-requests/put-endpoint/valid-whole/record-exists.json +++ b/endpoints/sample-requests/put-endpoint/valid-whole/record-exists.json @@ -11,14 +11,16 @@ ] } }, - "put_request": { - "puts": [ - { - "type": "xml", - "value": "updated_XML", - "key": "the-custom-thirty-six-character-uuid" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "updated_XML", + "key": "the-custom-thirty-six-character-uuid" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", diff --git a/endpoints/sample-requests/put-endpoint/valid-whole/single-element-to-store.json b/endpoints/sample-requests/put-endpoint/valid-whole/single-element-to-store.json index aaea2b28..a103dbbc 100644 --- a/endpoints/sample-requests/put-endpoint/valid-whole/single-element-to-store.json +++ b/endpoints/sample-requests/put-endpoint/valid-whole/single-element-to-store.json @@ -1,12 +1,14 @@ { "description": "Put request wants to store a single element of valid type no larger than the maximum size allowed. Store under a random UUID", - "put_request": { - "puts": [ - { - "type": "xml", - "value": "__video_info__<\\/VAST>\r\n" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "__video_info__<\\/VAST>\r\n" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -20,7 +22,9 @@ "code": 200, "put_response": { "responses": [ - {"uuid": "random"} + { + "uuid": "random" + } ] } } diff --git a/endpoints/sample-requests/put-endpoint/valid-whole/ttl-missing.json b/endpoints/sample-requests/put-endpoint/valid-whole/ttl-missing.json index 7d352533..f5cfd4d8 100644 --- a/endpoints/sample-requests/put-endpoint/valid-whole/ttl-missing.json +++ b/endpoints/sample-requests/put-endpoint/valid-whole/ttl-missing.json @@ -1,12 +1,14 @@ { "description": "Object to store doesn't come with a time-to-live value. Prebid Cache allows for a zero time-to-live value and responds with a random UUID.", - "put_request": { - "puts": [ - { - "type": "xml", - "value": "__video_info__<\\/VAST>\r\n" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "__video_info__<\\/VAST>\r\n" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -26,4 +28,4 @@ ] } } -} \ No newline at end of file +} diff --git a/endpoints/sample-requests/put-endpoint/valid-whole/ttl-more-than-max.json b/endpoints/sample-requests/put-endpoint/valid-whole/ttl-more-than-max.json index 5815a3be..5ecca6f9 100644 --- a/endpoints/sample-requests/put-endpoint/valid-whole/ttl-more-than-max.json +++ b/endpoints/sample-requests/put-endpoint/valid-whole/ttl-more-than-max.json @@ -3,14 +3,16 @@ "config": { "max_ttl_seconds": 5 }, - "put_request": { - "puts": [ - { - "type": "xml", - "ttlseconds": 6, - "value": "__video_info__<\\/VAST>\r\n" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "ttlseconds": 6, + "value": "__video_info__<\\/VAST>\r\n" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -30,4 +32,4 @@ ] } } -} \ No newline at end of file +} diff --git a/endpoints/sample-requests/put-endpoint/valid-whole/uuid-scenarios-in-response.json b/endpoints/sample-requests/put-endpoint/valid-whole/uuid-scenarios-in-response.json index cb64bd3d..b1d9a5a7 100644 --- a/endpoints/sample-requests/put-endpoint/valid-whole/uuid-scenarios-in-response.json +++ b/endpoints/sample-requests/put-endpoint/valid-whole/uuid-scenarios-in-response.json @@ -1,7 +1,6 @@ { "description": "Put request wants to store multiple elements but no more than the maximum allowed by the 'max_num_values' config. Store them under a random UUIDs", "config": { - "max_num_values": 3, "max_num_values": 3, "max_size_bytes": 100, "fake_backend": { @@ -16,26 +15,28 @@ }, "allow_setting_keys": true }, - "put_request": { - "puts": [ - { - "type": "json", - "ttlseconds": 60, - "value": "{\"description\": \"value will be stored under random UUID\"}" - }, - { - "key": "the-custom-thirty-six-character-uuid", - "type": "json", - "ttlseconds": 60, - "value": "{\"description\": \"value will be stored under custom UUID\"}" - }, - { - "key": "uuid-stored-value-we-want-overwriten", - "value": "XML meant to overwrite data already stored under this key", - "type": "xml", - "ttlseconds": 60 - } - ] + "request": { + "body": { + "puts": [ + { + "type": "json", + "ttlseconds": 60, + "value": "{\"description\": \"value will be stored under random UUID\"}" + }, + { + "key": "the-custom-thirty-six-character-uuid", + "type": "json", + "ttlseconds": 60, + "value": "{\"description\": \"value will be stored under custom UUID\"}" + }, + { + "key": "uuid-stored-value-we-want-overwriten", + "value": "XML meant to overwrite data already stored under this key", + "type": "xml", + "ttlseconds": 60 + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -52,9 +53,15 @@ "code": 200, "put_response": { "responses": [ - {"uuid": "random"}, - {"uuid": "the-custom-thirty-six-character-uuid"}, - {"uuid": ""} + { + "uuid": "random" + }, + { + "uuid": "the-custom-thirty-six-character-uuid" + }, + { + "uuid": "" + } ] } } diff --git a/endpoints/sample-requests/put-endpoint/valid-whole/valid-type-json.json b/endpoints/sample-requests/put-endpoint/valid-whole/valid-type-json.json index d66cc35b..1faae71d 100644 --- a/endpoints/sample-requests/put-endpoint/valid-whole/valid-type-json.json +++ b/endpoints/sample-requests/put-endpoint/valid-whole/valid-type-json.json @@ -1,12 +1,14 @@ { "description": "Store JSON type value, which Prebid Cache allows. Store under a random UUID", - "put_request": { - "puts": [ - { - "type": "json", - "value": "{\"field\":100}" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "json", + "value": "{\"field\":100}" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -20,7 +22,9 @@ "code": 200, "put_response": { "responses": [ - {"uuid": "random"} + { + "uuid": "random" + } ] } } diff --git a/endpoints/sample-requests/put-endpoint/valid-whole/valid-type-xml.json b/endpoints/sample-requests/put-endpoint/valid-whole/valid-type-xml.json index 43a67b94..9b218e5c 100644 --- a/endpoints/sample-requests/put-endpoint/valid-whole/valid-type-xml.json +++ b/endpoints/sample-requests/put-endpoint/valid-whole/valid-type-xml.json @@ -1,12 +1,14 @@ { "description": "Prebid Cache allows the storage of XML type values. Store under a random UUID because the 'key' field missing and custom keys are not allowed anyways", - "put_request": { - "puts": [ - { - "type": "xml", - "value": "__video_info__<\\/VAST>\r\n" - } - ] + "request": { + "body": { + "puts": [ + { + "type": "xml", + "value": "__video_info__<\\/VAST>\r\n" + } + ] + } }, "expected_metrics": [ "RecordPutTotal", @@ -20,7 +22,9 @@ "code": 200, "put_response": { "responses": [ - {"uuid": "random"} + { + "uuid": "random" + } ] } } diff --git a/utils/utils.go b/utils/utils.go index 55b51a94..f9da4d0d 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,6 +1,8 @@ package utils import ( + "math/rand" + "github.com/gofrs/uuid" ) @@ -9,3 +11,13 @@ func GenerateRandomID() (string, error) { u2, err := uuid.NewV4() return u2.String(), err } + +func RandomPick(pickProbability float64) bool { + if pickProbability == 0.0 { + return false + } + if pickProbability == 1.0 { + return true + } + return rand.Float64() < pickProbability +} diff --git a/utils/utils_test.go b/utils/utils_test.go new file mode 100644 index 00000000..1d8d0a50 --- /dev/null +++ b/utils/utils_test.go @@ -0,0 +1,31 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRandomPick(t *testing.T) { + testCases := []struct { + name string + inPickProbability float64 + expected bool + }{ + { + name: "zero", // Zero probablity of true, expect false + inPickProbability: 0.00, + expected: false, + }, + { + name: "one", // 100% probability of true, expect true + inPickProbability: 1.00, + expected: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, RandomPick(tc.inPickProbability)) + }) + } +}