diff --git a/sstables/map_key_index.go b/sstables/map_key_index.go index b0f24c2..31d20c7 100644 --- a/sstables/map_key_index.go +++ b/sstables/map_key_index.go @@ -3,10 +3,11 @@ package sstables import ( "errors" "fmt" + "io" + rProto "github.com/thomasjungblut/go-sstables/recordio/proto" "github.com/thomasjungblut/go-sstables/skiplist" "github.com/thomasjungblut/go-sstables/sstables/proto" - "io" ) type ByteKeyMapper[T comparable] interface { @@ -46,9 +47,10 @@ type MapKeyIndex[T comparable] struct { } func (s *MapKeyIndex[T]) Contains(key []byte) (bool, error) { - _, found := s.index[s.mapper.MapBytes(key)] - return found, nil + idxval, found := s.index[s.mapper.MapBytes(key)] + return found && !idxval.Tombstoned, nil } + func (s *MapKeyIndex[T]) Get(key []byte) (IndexVal, error) { val, found := s.index[s.mapper.MapBytes(key)] if found { @@ -107,8 +109,8 @@ func (s *MapKeyIndexLoader[T]) Load(indexPath string, metadata *proto.MetaData) } kBytes := s.Mapper.MapBytes(record.Key) - smap[kBytes] = IndexVal{Offset: record.ValueOffset, Checksum: record.Checksum} - sx = append(sx, sliceKey{IndexVal{Offset: record.ValueOffset, Checksum: record.Checksum}, record.Key}) + smap[kBytes] = IndexVal{Offset: record.ValueOffset, Checksum: record.Checksum, Tombstoned: record.Tombstoned} + sx = append(sx, sliceKey{smap[kBytes], record.Key}) i++ } diff --git a/sstables/proto/sstable.pb.go b/sstables/proto/sstable.pb.go index 81da2ba..823f0b9 100644 --- a/sstables/proto/sstable.pb.go +++ b/sstables/proto/sstable.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.5 -// protoc v5.29.3 +// protoc-gen-go v1.32.0 +// protoc v4.25.1 // source: sstables/proto/sstable.proto package proto @@ -11,7 +11,6 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" - unsafe "unsafe" ) const ( @@ -22,19 +21,23 @@ const ( ) type IndexEntry struct { - state protoimpl.MessageState `protogen:"open.v1"` - Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - ValueOffset uint64 `protobuf:"varint,2,opt,name=valueOffset,proto3" json:"valueOffset,omitempty"` - Checksum uint64 `protobuf:"varint,3,opt,name=checksum,proto3" json:"checksum,omitempty"` // a golang crc-64 checksum of the respective dataEntry - unknownFields protoimpl.UnknownFields + state protoimpl.MessageState sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + ValueOffset uint64 `protobuf:"varint,2,opt,name=valueOffset,proto3" json:"valueOffset,omitempty"` + Checksum uint64 `protobuf:"varint,3,opt,name=checksum,proto3" json:"checksum,omitempty"` // a golang crc-64 checksum of the respective dataEntry + Tombstoned bool `protobuf:"varint,4,opt,name=tombstoned,proto3" json:"tombstoned,omitempty"` } func (x *IndexEntry) Reset() { *x = IndexEntry{} - mi := &file_sstables_proto_sstable_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_sstables_proto_sstable_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *IndexEntry) String() string { @@ -45,7 +48,7 @@ func (*IndexEntry) ProtoMessage() {} func (x *IndexEntry) ProtoReflect() protoreflect.Message { mi := &file_sstables_proto_sstable_proto_msgTypes[0] - if x != nil { + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -81,19 +84,29 @@ func (x *IndexEntry) GetChecksum() uint64 { return 0 } +func (x *IndexEntry) GetTombstoned() bool { + if x != nil { + return x.Tombstoned + } + return false +} + // deprecated, it's unnecessary overhead to marshal the bytes once more type DataEntry struct { - state protoimpl.MessageState `protogen:"open.v1"` - Value []byte `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` - unknownFields protoimpl.UnknownFields + state protoimpl.MessageState sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Value []byte `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` } func (x *DataEntry) Reset() { *x = DataEntry{} - mi := &file_sstables_proto_sstable_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_sstables_proto_sstable_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *DataEntry) String() string { @@ -104,7 +117,7 @@ func (*DataEntry) ProtoMessage() {} func (x *DataEntry) ProtoReflect() protoreflect.Message { mi := &file_sstables_proto_sstable_proto_msgTypes[1] - if x != nil { + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -127,25 +140,28 @@ func (x *DataEntry) GetValue() []byte { } type MetaData struct { - state protoimpl.MessageState `protogen:"open.v1"` - NumRecords uint64 `protobuf:"varint,1,opt,name=numRecords,proto3" json:"numRecords,omitempty"` - MinKey []byte `protobuf:"bytes,2,opt,name=minKey,proto3" json:"minKey,omitempty"` - MaxKey []byte `protobuf:"bytes,3,opt,name=maxKey,proto3" json:"maxKey,omitempty"` - DataBytes uint64 `protobuf:"varint,4,opt,name=dataBytes,proto3" json:"dataBytes,omitempty"` - IndexBytes uint64 `protobuf:"varint,5,opt,name=indexBytes,proto3" json:"indexBytes,omitempty"` - TotalBytes uint64 `protobuf:"varint,6,opt,name=totalBytes,proto3" json:"totalBytes,omitempty"` - Version uint32 `protobuf:"varint,7,opt,name=version,proto3" json:"version,omitempty"` // currently version 1, the default is version 0 with protos as values - SkippedRecords uint64 `protobuf:"varint,8,opt,name=skippedRecords,proto3" json:"skippedRecords,omitempty"` - NullValues uint64 `protobuf:"varint,9,opt,name=nullValues,proto3" json:"nullValues,omitempty"` // in simpleDB that corresponds to the number of tombstones - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NumRecords uint64 `protobuf:"varint,1,opt,name=numRecords,proto3" json:"numRecords,omitempty"` + MinKey []byte `protobuf:"bytes,2,opt,name=minKey,proto3" json:"minKey,omitempty"` + MaxKey []byte `protobuf:"bytes,3,opt,name=maxKey,proto3" json:"maxKey,omitempty"` + DataBytes uint64 `protobuf:"varint,4,opt,name=dataBytes,proto3" json:"dataBytes,omitempty"` + IndexBytes uint64 `protobuf:"varint,5,opt,name=indexBytes,proto3" json:"indexBytes,omitempty"` + TotalBytes uint64 `protobuf:"varint,6,opt,name=totalBytes,proto3" json:"totalBytes,omitempty"` + Version uint32 `protobuf:"varint,7,opt,name=version,proto3" json:"version,omitempty"` // currently version 1, the default is version 0 with protos as values + SkippedRecords uint64 `protobuf:"varint,8,opt,name=skippedRecords,proto3" json:"skippedRecords,omitempty"` + NullValues uint64 `protobuf:"varint,9,opt,name=nullValues,proto3" json:"nullValues,omitempty"` // in simpleDB that corresponds to the number of tombstones } func (x *MetaData) Reset() { *x = MetaData{} - mi := &file_sstables_proto_sstable_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_sstables_proto_sstable_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *MetaData) String() string { @@ -156,7 +172,7 @@ func (*MetaData) ProtoMessage() {} func (x *MetaData) ProtoReflect() protoreflect.Message { mi := &file_sstables_proto_sstable_proto_msgTypes[2] - if x != nil { + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -236,16 +252,18 @@ func (x *MetaData) GetNullValues() uint64 { var File_sstables_proto_sstable_proto protoreflect.FileDescriptor -var file_sstables_proto_sstable_proto_rawDesc = string([]byte{ +var file_sstables_proto_sstable_proto_rawDesc = []byte{ 0x0a, 0x1c, 0x73, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x5c, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x45, 0x6e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x7c, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x63, 0x68, 0x65, 0x63, 0x6b, - 0x73, 0x75, 0x6d, 0x22, 0x21, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x73, 0x75, 0x6d, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x6f, 0x6d, 0x62, 0x73, 0x74, 0x6f, 0x6e, 0x65, + 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x74, 0x6f, 0x6d, 0x62, 0x73, 0x74, 0x6f, + 0x6e, 0x65, 0x64, 0x22, 0x21, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x9a, 0x02, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, @@ -270,22 +288,22 @@ var file_sstables_proto_sstable_proto_rawDesc = string([]byte{ 0x2f, 0x67, 0x6f, 0x2d, 0x73, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2f, 0x73, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -}) +} var ( file_sstables_proto_sstable_proto_rawDescOnce sync.Once - file_sstables_proto_sstable_proto_rawDescData []byte + file_sstables_proto_sstable_proto_rawDescData = file_sstables_proto_sstable_proto_rawDesc ) func file_sstables_proto_sstable_proto_rawDescGZIP() []byte { file_sstables_proto_sstable_proto_rawDescOnce.Do(func() { - file_sstables_proto_sstable_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sstables_proto_sstable_proto_rawDesc), len(file_sstables_proto_sstable_proto_rawDesc))) + file_sstables_proto_sstable_proto_rawDescData = protoimpl.X.CompressGZIP(file_sstables_proto_sstable_proto_rawDescData) }) return file_sstables_proto_sstable_proto_rawDescData } var file_sstables_proto_sstable_proto_msgTypes = make([]protoimpl.MessageInfo, 3) -var file_sstables_proto_sstable_proto_goTypes = []any{ +var file_sstables_proto_sstable_proto_goTypes = []interface{}{ (*IndexEntry)(nil), // 0: proto.IndexEntry (*DataEntry)(nil), // 1: proto.DataEntry (*MetaData)(nil), // 2: proto.MetaData @@ -303,11 +321,49 @@ func file_sstables_proto_sstable_proto_init() { if File_sstables_proto_sstable_proto != nil { return } + if !protoimpl.UnsafeEnabled { + file_sstables_proto_sstable_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IndexEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sstables_proto_sstable_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DataEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sstables_proto_sstable_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MetaData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_sstables_proto_sstable_proto_rawDesc), len(file_sstables_proto_sstable_proto_rawDesc)), + RawDescriptor: file_sstables_proto_sstable_proto_rawDesc, NumEnums: 0, NumMessages: 3, NumExtensions: 0, @@ -318,6 +374,7 @@ func file_sstables_proto_sstable_proto_init() { MessageInfos: file_sstables_proto_sstable_proto_msgTypes, }.Build() File_sstables_proto_sstable_proto = out.File + file_sstables_proto_sstable_proto_rawDesc = nil file_sstables_proto_sstable_proto_goTypes = nil file_sstables_proto_sstable_proto_depIdxs = nil } diff --git a/sstables/proto/sstable.proto b/sstables/proto/sstable.proto index 3d933db..7d23536 100644 --- a/sstables/proto/sstable.proto +++ b/sstables/proto/sstable.proto @@ -6,6 +6,7 @@ message IndexEntry { bytes key = 1; uint64 valueOffset = 2; uint64 checksum = 3; // a golang crc-64 checksum of the respective dataEntry + bool tombstoned = 4; } // deprecated, it's unnecessary overhead to marshal the bytes once more diff --git a/sstables/slice_key_index.go b/sstables/slice_key_index.go index 46225ff..b7adab2 100644 --- a/sstables/slice_key_index.go +++ b/sstables/slice_key_index.go @@ -4,11 +4,12 @@ import ( "bytes" "errors" "fmt" + "io" + rProto "github.com/thomasjungblut/go-sstables/recordio/proto" "github.com/thomasjungblut/go-sstables/skiplist" "github.com/thomasjungblut/go-sstables/sstables/proto" "golang.org/x/exp/slices" - "io" ) type sliceKey struct { @@ -119,7 +120,7 @@ func (s *SliceKeyIndexLoader) Load(indexPath string, metadata *proto.MetaData) ( return nil, fmt.Errorf("error while reading index records of sstable in '%s': %w", indexPath, err) } - sx = append(sx, sliceKey{IndexVal{Offset: record.ValueOffset, Checksum: record.Checksum}, record.Key}) + sx = append(sx, sliceKey{IndexVal{Offset: record.ValueOffset, Checksum: record.Checksum, Tombstoned: record.Tombstoned}, record.Key}) } return &SliceKeyIndex{NoOpOpenClose{}, sx}, nil diff --git a/sstables/sstable_index.go b/sstables/sstable_index.go index b351373..9556575 100644 --- a/sstables/sstable_index.go +++ b/sstables/sstable_index.go @@ -7,8 +7,9 @@ import ( ) type IndexVal struct { - Offset uint64 - Checksum uint64 + Offset uint64 + Checksum uint64 + Tombstoned bool } type NoOpOpenClose struct { diff --git a/sstables/super_sstable_reader.go b/sstables/super_sstable_reader.go index 0b6e57a..8e6b97c 100644 --- a/sstables/super_sstable_reader.go +++ b/sstables/super_sstable_reader.go @@ -23,13 +23,9 @@ func (s SuperSSTableReader) Contains(key []byte) (bool, error) { if err != nil { return false, err } - if !keyExist { - continue + if keyExist { + return true, nil } - // we have to check if the value is not tombstoned - // maybe had to be implemented in an IsTombstoned in sstableReader - res, err := s.readers[i].Get(key) - return res != nil, err } return false, nil