db: interleave range deletion boundaries in levelIter

Previously, levelIter would sometimes interleave fake keys at file boundaries
or user-provided iteration boundaries when a file contains range deletions.
This commit reworks the levelIter to instead interleave the individual bounds
of the range deletions themselves, using an interleaving iterator. This
simplifies the levelIter. Because range deletions are now read in two places:
once for interleaving bounds and once to expose a rangeDelIter to the
mergingIter, this commit requires opening a file's range deletion iterator
twice. This is an intermediary state until #2863 is completed when the
mergingIter will consult the levelIter's interleaving iterator to determine
which range deletions overlap the current iterator position.

Informs #2863.

Author: jbowens

internal/keyspan/interleaving_iter.go

@@ -1036,9 +1036,10 @@ func (i *InterleavingIter) verify(kv *base.InternalKV) *base.InternalKV {
 		switch {
 		case i.dir == -1 && i.spanMarkerTruncated:
 			panic("pebble: invariant violation: truncated span key in reverse iteration")
-		case kv != nil && i.opts.LowerBound != nil && i.cmp(kv.K.UserKey, i.opts.LowerBound) < 0:
+		case kv != nil && i.opts.LowerBound != nil && !kv.K.IsExclusiveSentinel() &&
+			i.cmp(kv.K.UserKey, i.opts.LowerBound) < 0:
 			panic("pebble: invariant violation: key < lower bound")
-		case kv != nil && i.opts.UpperBound != nil &&
+		case kv != nil && i.opts.UpperBound != nil && !kv.K.IsExclusiveSentinel() &&
 			!base.UserKeyExclusive(i.opts.UpperBound).IsUpperBoundForInternalKey(i.comparer.Compare, kv.K):
 			panic("pebble: invariant violation: key ≥ upper bound")
 		case i.err != nil && kv != nil:

level_checker_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invalidating"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/internal/private"
@@ -85,6 +86,10 @@ func (f *failMerger) Close() error {
 }
 
 func TestCheckLevelsCornerCases(t *testing.T) {
+	if invariants.Enabled {
+		t.Skip("disabled under invariants; relies on violating invariants to detect them")
+	}
+
 	memFS := vfs.NewMem()
 	var levels [][]*fileMetadata
 	formatKey := testkeys.Comparer.FormatKey