diff --git a/api/storeapi/store_api.proto b/api/storeapi/store_api.proto index e9ffd179..8d34fc23 100644 --- a/api/storeapi/store_api.proto +++ b/api/storeapi/store_api.proto @@ -144,6 +144,7 @@ enum SearchErrorCode { TOO_MANY_GROUP_TOKENS = 5; TOO_MANY_FRACTION_TOKENS = 6; TOO_MANY_FIELD_VALUES = 7; + MEMORY_LIMIT_EXCEEDED = 8; } message StartAsyncSearchRequest { diff --git a/cmd/seq-db/seq-db.go b/cmd/seq-db/seq-db.go index b73b9b27..5c123609 100644 --- a/cmd/seq-db/seq-db.go +++ b/cmd/seq-db/seq-db.go @@ -297,6 +297,7 @@ func startStore( WorkersCount: cfg.Resources.SearchWorkers, MaxFractionHits: cfg.Limits.FractionHits, FractionsPerIteration: config.NumCPU, + MaxQprMemory: uint64(cfg.Limits.QprMemoryUsage), RequestsLimit: uint64(cfg.Limits.SearchRequests), LogThreshold: cfg.SlowLogs.SearchThreshold, Async: asyncsearcher.AsyncSearcherConfig{ diff --git a/config/config.go b/config/config.go index 0e9d4a03..00df0d94 100644 --- a/config/config.go +++ b/config/config.go @@ -131,6 +131,9 @@ type Config struct { // DocSize specifies maximum possible size for single document. // Document larger than this threshold will be skipped. DocSize Bytes `config:"doc_size" default:"128KiB"` + // QprMemoryUsage specifies maximum heap memory which a single QPR (query partial result) + // can use in either store or proxy. + QprMemoryUsage Bytes `config:"qpr_memory_usage" default:"0B"` Aggregation struct { // FieldTokens specifies maximum amount of unique field tokens diff --git a/consts/consts.go b/consts/consts.go index aea96645..d44f0a23 100644 --- a/consts/consts.go +++ b/consts/consts.go @@ -88,4 +88,5 @@ var ( ErrTooManyGroupTokens = errors.New("aggregation has too many group tokens") ErrTooManyFractionTokens = errors.New("aggregation has too many fraction tokens") ErrTooManyFractionsHit = errors.New("too many fractions hit") + ErrMemoryLimitExceeded = errors.New("memory limit exceeded") ) diff --git a/fracmanager/searcher.go b/fracmanager/searcher.go index d69277fb..1248fd0a 100644 --- a/fracmanager/searcher.go +++ b/fracmanager/searcher.go @@ -20,6 +20,7 @@ import ( type SearcherCfg struct { MaxFractionHits int // the maximum number of fractions used in the search FractionsPerIteration int + MaxQprMemory uint64 // max heap memory a single QPR can use. 0 if no limit set } type Searcher struct { @@ -59,6 +60,7 @@ func (s *Searcher) SearchDocs(ctx context.Context, fracs []frac.Fraction, params var totalSearchTimeNanos int64 var totalWaitTimeNanos int64 + var totalMemUsage uint64 for len(remainingFracs) > 0 && (scanAll || params.Limit > 0) { subQPRs, searchTimeNanos, waitTimeNanos, err := s.searchDocsAsync(ctx, remainingFracs.Shift(fracsChunkSize), params) @@ -71,6 +73,12 @@ func (s *Searcher) SearchDocs(ctx context.Context, fracs []frac.Fraction, params seq.MergeQPRs(total, subQPRs, origLimit, seq.MillisToMID(params.HistInterval), params.Order) + totalMemUsage = total.MemUsage() + + if s.cfg.MaxQprMemory > 0 && totalMemUsage > s.cfg.MaxQprMemory { + return nil, fmt.Errorf("%w: used %d bytes, limit %d", consts.ErrMemoryLimitExceeded, total.MemUsage(), s.cfg.MaxQprMemory) + } + // reduce the limit on the number of ensured docs in response params.Limit = origLimit - calcEnsuredIDsCount(total.IDs, remainingFracs, params.Order) diff --git a/metric/store.go b/metric/store.go index 8abf03a9..54e26b54 100644 --- a/metric/store.go +++ b/metric/store.go @@ -60,6 +60,13 @@ var ( Help: "Search request duration time (only successful searches)", Buckets: SecondsBuckets, }) + SearchQprMemSize = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "seq_db_store", + Subsystem: "search", + Name: "qpr_mem_size_bytes", + Help: "QPR heap memory size in bytes", + Buckets: prometheus.ExponentialBuckets(1024, 3, 21), + }) SearchRangesSeconds = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: "seq_db_store", diff --git a/pkg/storeapi/store_api.pb.go b/pkg/storeapi/store_api.pb.go index ba3873f3..8fbb52c1 100644 --- a/pkg/storeapi/store_api.pb.go +++ b/pkg/storeapi/store_api.pb.go @@ -144,6 +144,7 @@ const ( SearchErrorCode_TOO_MANY_GROUP_TOKENS SearchErrorCode = 5 SearchErrorCode_TOO_MANY_FRACTION_TOKENS SearchErrorCode = 6 SearchErrorCode_TOO_MANY_FIELD_VALUES SearchErrorCode = 7 + SearchErrorCode_MEMORY_LIMIT_EXCEEDED SearchErrorCode = 8 ) // Enum value maps for SearchErrorCode. @@ -156,6 +157,7 @@ var ( 5: "TOO_MANY_GROUP_TOKENS", 6: "TOO_MANY_FRACTION_TOKENS", 7: "TOO_MANY_FIELD_VALUES", + 8: "MEMORY_LIMIT_EXCEEDED", } SearchErrorCode_value = map[string]int32{ "NO_ERROR": 0, @@ -165,6 +167,7 @@ var ( "TOO_MANY_GROUP_TOKENS": 5, "TOO_MANY_FRACTION_TOKENS": 6, "TOO_MANY_FIELD_VALUES": 7, + "MEMORY_LIMIT_EXCEEDED": 8, } ) @@ -2454,7 +2457,7 @@ var file_storeapi_store_api_proto_rawDesc = string([]byte{ 0x45, 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x10, 0x07, 0x2a, 0x26, 0x0a, 0x05, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x0a, 0x4f, 0x52, 0x44, 0x45, 0x52, 0x5f, 0x44, 0x45, 0x53, 0x43, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x4f, 0x52, 0x44, 0x45, 0x52, 0x5f, 0x41, 0x53, 0x43, 0x10, - 0x01, 0x2a, 0xcd, 0x01, 0x0a, 0x0f, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x45, 0x72, 0x72, 0x6f, + 0x01, 0x2a, 0xe8, 0x01, 0x0a, 0x0f, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x21, 0x0a, 0x1d, 0x49, 0x4e, 0x47, 0x45, 0x53, 0x54, 0x4f, 0x52, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x57, 0x41, 0x4e, 0x54, 0x53, 0x5f, 0x4f, 0x4c, 0x44, 0x5f, @@ -2467,61 +2470,63 @@ var file_storeapi_store_api_proto_rawDesc = string([]byte{ 0x4d, 0x41, 0x4e, 0x59, 0x5f, 0x46, 0x52, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x4f, 0x4b, 0x45, 0x4e, 0x53, 0x10, 0x06, 0x12, 0x19, 0x0a, 0x15, 0x54, 0x4f, 0x4f, 0x5f, 0x4d, 0x41, 0x4e, 0x59, 0x5f, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x53, 0x10, - 0x07, 0x2a, 0x8a, 0x01, 0x0a, 0x11, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x0a, 0x1b, 0x41, 0x73, 0x79, 0x6e, 0x63, - 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x50, 0x72, - 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x41, 0x73, 0x79, 0x6e, - 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x44, 0x6f, 0x6e, - 0x65, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, - 0x63, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x65, 0x64, - 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x03, 0x32, 0x9c, - 0x05, 0x0a, 0x08, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x41, 0x70, 0x69, 0x12, 0x32, 0x0a, 0x04, 0x42, - 0x75, 0x6c, 0x6b, 0x12, 0x10, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x42, 0x75, 0x6c, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, - 0x33, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x10, 0x53, 0x74, 0x61, 0x72, 0x74, 0x41, 0x73, 0x79, - 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x63, 0x0a, 0x16, 0x46, 0x65, 0x74, 0x63, 0x68, - 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x12, 0x22, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x73, 0x79, - 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x46, 0x65, 0x74, 0x63, - 0x68, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x11, - 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x41, 0x73, - 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x41, 0x73, 0x79, + 0x07, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x45, 0x4d, 0x4f, 0x52, 0x59, 0x5f, 0x4c, 0x49, 0x4d, 0x49, + 0x54, 0x5f, 0x45, 0x58, 0x43, 0x45, 0x45, 0x44, 0x45, 0x44, 0x10, 0x08, 0x2a, 0x8a, 0x01, 0x0a, + 0x11, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x1f, 0x0a, 0x1b, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x44, 0x6f, 0x6e, 0x65, 0x10, 0x01, 0x12, 0x1d, + 0x0a, 0x19, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x65, 0x64, 0x10, 0x02, 0x12, 0x1a, 0x0a, + 0x16, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x03, 0x32, 0x9c, 0x05, 0x0a, 0x08, 0x53, 0x74, + 0x6f, 0x72, 0x65, 0x41, 0x70, 0x69, 0x12, 0x32, 0x0a, 0x04, 0x42, 0x75, 0x6c, 0x6b, 0x12, 0x10, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x42, 0x75, 0x6c, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x06, 0x53, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x12, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x51, 0x0a, 0x10, 0x53, 0x74, 0x61, 0x72, 0x74, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x12, 0x1c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x41, + 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x54, 0x0a, 0x11, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, 0x73, 0x79, 0x6e, - 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x41, - 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x65, 0x73, 0x4c, 0x69, 0x73, 0x74, - 0x12, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, - 0x65, 0x61, 0x72, 0x63, 0x68, 0x65, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x73, 0x79, 0x6e, - 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x65, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2f, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68, - 0x12, 0x11, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, - 0x44, 0x61, 0x74, 0x61, 0x22, 0x00, 0x30, 0x01, 0x12, 0x33, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, - 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x7a, 0x6f, 0x6e, - 0x74, 0x65, 0x63, 0x68, 0x2f, 0x73, 0x65, 0x71, 0x2d, 0x64, 0x62, 0x2f, 0x70, 0x6b, 0x67, 0x2f, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x61, 0x70, 0x69, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x61, 0x70, - 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x22, 0x00, 0x12, 0x63, 0x0a, 0x16, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x73, 0x79, 0x6e, 0x63, + 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x22, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x73, 0x79, 0x6e, + 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x11, 0x43, 0x61, 0x6e, 0x63, 0x65, + 0x6c, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x1d, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, + 0x11, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, + 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, 0x73, + 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x65, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x65, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, + 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x65, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x2f, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68, 0x12, 0x11, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x22, + 0x00, 0x30, 0x01, 0x12, 0x33, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x2e, + 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x13, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x7a, 0x6f, 0x6e, 0x74, 0x65, 0x63, 0x68, 0x2f, + 0x73, 0x65, 0x71, 0x2d, 0x64, 0x62, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, + 0x61, 0x70, 0x69, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, }) var ( diff --git a/proxy/search/ingestor.go b/proxy/search/ingestor.go index b8e01784..ca2684a7 100644 --- a/proxy/search/ingestor.go +++ b/proxy/search/ingestor.go @@ -601,6 +601,9 @@ func (si *Ingestor) searchShard( if errMessage == consts.ErrTooManyFractionTokens.Error() { return nil, source, fmt.Errorf("store forbids aggregation request: %w", consts.ErrTooManyFractionTokens) } + if errMessage == consts.ErrMemoryLimitExceeded.Error() { + return nil, source, fmt.Errorf("store forbids search request: %w", consts.ErrMemoryLimitExceeded) + } errs = append(errs, err) continue } @@ -620,6 +623,8 @@ func (si *Ingestor) searchShard( return nil, source, fmt.Errorf("store forbids aggregation request: %w", consts.ErrTooManyGroupTokens) case storeapi.SearchErrorCode_TOO_MANY_FRACTION_TOKENS: return nil, source, fmt.Errorf("store forbids aggregation request: %w", consts.ErrTooManyFractionTokens) + case storeapi.SearchErrorCode_MEMORY_LIMIT_EXCEEDED: + return nil, source, fmt.Errorf("store forbids search request: %w", consts.ErrMemoryLimitExceeded) case storeapi.SearchErrorCode_TOO_MANY_FRACTIONS_HIT: return nil, source, fmt.Errorf("store forbids request: %w", consts.ErrTooManyFractionsHit) } diff --git a/seq/qpr.go b/seq/qpr.go index a7246233..e4274443 100644 --- a/seq/qpr.go +++ b/seq/qpr.go @@ -7,11 +7,13 @@ import ( "math" "slices" "sort" + "unsafe" "github.com/valyala/fastrand" "github.com/ozontech/seq-db/consts" "github.com/ozontech/seq-db/metric" + "github.com/ozontech/seq-db/util" ) type DocsOrder uint8 @@ -29,6 +31,15 @@ func (o DocsOrder) IsReverse() bool { return o == DocsOrderAsc } +const ( + SizeOfIDSource = uint64(unsafe.Sizeof(*new(IDSource))) + SizeOfErrorSource = uint64(unsafe.Sizeof(*new(ErrorSource))) + SizeOfAggBin = uint64(unsafe.Sizeof(*new(AggBin))) + SizeOfSamplesContainer = uint64(unsafe.Sizeof(*new(SamplesContainer))) + SizeOfAggregatableSamples = uint64(unsafe.Sizeof(*new(AggregatableSamples))) + SizeOfQPR = uint64(unsafe.Sizeof(*new(QPR))) +) + type IDSource struct { ID ID Source uint64 @@ -39,6 +50,13 @@ func (id *IDSource) Equal(check IDSource) bool { return id.ID.Equal(check.ID) && id.Source == check.Source } +func (id *IDSource) MemUsage() uint64 { + if id == nil { + return 0 + } + return SizeOfIDSource + uint64(len(id.Hint)) +} + type IDSources []IDSource func (p IDSources) Len() int { return len(p) } @@ -64,6 +82,13 @@ type ErrorSource struct { Source uint64 } +func (e *ErrorSource) MemUsage() uint64 { + if e == nil { + return 0 + } + return SizeOfErrorSource + uint64(len(e.ErrStr)) +} + // QPR query partial result, stores intermediate result of running query e.g. result from only one fraction or particular store // TODO: remove single Agg when n-agg support in proxy is deployed type QPR struct { @@ -74,6 +99,29 @@ type QPR struct { Errors []ErrorSource } +// MemUsage returns estimated total memory used by the QPR and its nested structs. +func (q *QPR) MemUsage() uint64 { + if q == nil { + return 0 + } + total := SizeOfQPR + + for i := range q.IDs { + total += q.IDs[i].MemUsage() + } + for range q.Histogram { + total += util.SizeOfUint64 + util.SizeOfUint64 + } + for i := range q.Aggs { + total += q.Aggs[i].MemUsage() + } + for i := range q.Errors { + total += q.Errors[i].MemUsage() + } + + return total +} + func (q *QPR) Aggregate(args []AggregateArgs) []AggregationResult { allAggregations := make([]AggregationResult, len(q.Aggs)) for i, agg := range q.Aggs { @@ -112,6 +160,13 @@ type AggBin struct { Token string } +func (b *AggBin) MemUsage() uint64 { + if b == nil { + return 0 + } + return SizeOfAggBin + uint64(len(b.Token)) +} + type AggregatableSamples struct { SamplesByBin map[AggBin]*SamplesContainer NotExists int64 @@ -291,6 +346,21 @@ func (q *AggregatableSamples) Merge(agg AggregatableSamples) { q.NotExists += agg.NotExists } +func (q *AggregatableSamples) MemUsage() uint64 { + if q == nil { + return 0 + } + + total := SizeOfAggregatableSamples + for _, s := range q.ValuesPool { + total += util.SizeOfString + uint64(len(s)) + } + for bin, samples := range q.SamplesByBin { + total += bin.MemUsage() + util.SizeOfPointer + samples.MemUsage() + } + return total +} + // SamplesContainer is a container that is used for aggregations. // Implements reservoir sampling algorithm. type SamplesContainer struct { @@ -318,6 +388,20 @@ func NewSamplesContainers() *SamplesContainer { return h } +func (h *SamplesContainer) MemUsage() uint64 { + if h == nil { + return 0 + } + total := SizeOfSamplesContainer + total += uint64(len(h.Samples)) * util.SizeOfFloat64 + if h.Values != nil { + for range h.Values { + total += util.SizeOfUint32 + } + } + return total +} + // Quantile calculates the quantile value of the histogram. // The argument should be in [0, 1] range. // diff --git a/storeapi/grpc_search.go b/storeapi/grpc_search.go index 4acd3bb7..876636dd 100644 --- a/storeapi/grpc_search.go +++ b/storeapi/grpc_search.go @@ -205,6 +205,7 @@ func (g *GrpcV1) doSearch( } metric.SearchDurationSeconds.Observe(time.Since(start).Seconds()) + metric.SearchQprMemSize.Observe(float64(qpr.MemUsage())) if req.Explain { if req.Interval > 0 { diff --git a/storeapi/grpc_v1.go b/storeapi/grpc_v1.go index 2ed55ea5..4f3e7676 100644 --- a/storeapi/grpc_v1.go +++ b/storeapi/grpc_v1.go @@ -29,6 +29,7 @@ type SearchConfig struct { WorkersCount int MaxFractionHits int FractionsPerIteration int + MaxQprMemory uint64 // max heap memory a single QPR can use (bytes). 0 if no limit set. RequestsLimit uint64 LogThreshold time.Duration Async asyncsearcher.AsyncSearcherConfig @@ -106,6 +107,7 @@ func NewGrpcV1(cfg APIConfig, fracManager *fracmanager.FracManager, mappingProvi searcher: fracmanager.NewSearcher(cfg.Search.WorkersCount, fracmanager.SearcherCfg{ MaxFractionHits: cfg.Search.MaxFractionHits, FractionsPerIteration: cfg.Search.FractionsPerIteration, + MaxQprMemory: cfg.Search.MaxQprMemory, }), }, fetchData: fetchData{ @@ -150,6 +152,9 @@ func parseStoreError(e error) (storeapi.SearchErrorCode, bool) { if errors.Is(e, consts.ErrTooManyFractionTokens) { return storeapi.SearchErrorCode_TOO_MANY_FRACTION_TOKENS, true } + if errors.Is(e, consts.ErrMemoryLimitExceeded) { + return storeapi.SearchErrorCode_MEMORY_LIMIT_EXCEEDED, true + } if errors.Is(e, consts.ErrTooManyFractionsHit) { metric.RejectedRequests.WithLabelValues("search", "fracs_exceeding").Inc() return storeapi.SearchErrorCode_TOO_MANY_FRACTIONS_HIT, true diff --git a/util/size.go b/util/size.go new file mode 100644 index 00000000..5421315a --- /dev/null +++ b/util/size.go @@ -0,0 +1,11 @@ +package util + +import "unsafe" + +const ( + SizeOfString = uint64(unsafe.Sizeof(*new(string))) + SizeOfUint32 = uint64(unsafe.Sizeof(*new(uint32))) + SizeOfUint64 = uint64(unsafe.Sizeof(*new(uint64))) + SizeOfPointer = uint64(unsafe.Sizeof(new(int))) + SizeOfFloat64 = uint64(unsafe.Sizeof(*new(float64))) +)