diff --git a/src/Cassandra.IntegrationTests/Core/ZeroTokenNodeTests.cs b/src/Cassandra.IntegrationTests/Core/ZeroTokenNodeTests.cs new file mode 100644 index 000000000..bee3c18f6 --- /dev/null +++ b/src/Cassandra.IntegrationTests/Core/ZeroTokenNodeTests.cs @@ -0,0 +1,373 @@ +// +// Copyright (C) DataStax Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System.Collections.Generic; +using System.Linq; +using Cassandra.IntegrationTests.TestBase; +using Cassandra.IntegrationTests.TestClusterManagement; +using Cassandra.Tests; +using NUnit.Framework; +using Assert = NUnit.Framework.Legacy.ClassicAssert; +using CollectionAssert = NUnit.Framework.Legacy.CollectionAssert; + +namespace Cassandra.IntegrationTests.Core +{ + /// + /// Common setup for the fixtures that exercise a real cluster containing a zero-token node, i.e. a + /// node started with join_ring: false that therefore does not own any token. + /// + public abstract class ZeroTokenNodeTestBase : SharedClusterTest + { + protected const int NormalNodes = 2; + protected const int ZeroTokenNodeId = ZeroTokenNodeTestBase.NormalNodes + 1; + protected const string TableName = "zero_token_routing"; + + /// + /// Amount of distinct partition keys used to probe the token map. With a handful of token + /// owners this is more than enough to cover every token range at least once. + /// + protected const int RoutingKeySamples = 60; + + private PreparedStatement _insert; + + protected ZeroTokenNodeTestBase() : base(ZeroTokenNodeTestBase.NormalNodes) + { + } + + protected override string[] SetupQueries => new[] + { + $"CREATE TABLE {ZeroTokenNodeTestBase.TableName} (k int PRIMARY KEY, v int)" + }; + + protected string ZeroTokenAddress => TestCluster.ClusterIpPrefix + ZeroTokenNodeTestBase.ZeroTokenNodeId; + + public override void OneTimeSetUp() + { + base.OneTimeSetUp(); + + // Add a node that does not join the token ring. It has to be configured before being + // started, otherwise it would bootstrap as a regular token owner. + TestCluster.BootstrapNode(ZeroTokenNodeTestBase.ZeroTokenNodeId, false); + TestCluster.UpdateConfig(ZeroTokenNodeTestBase.ZeroTokenNodeId, "join_ring: false"); + TestCluster.Start(ZeroTokenNodeTestBase.ZeroTokenNodeId); + TestUtils.WaitForUp(ZeroTokenAddress, SharedClusterTest.DefaultCassandraPort, 120); + + TestHelper.RetryAssert( + () => Assert.AreEqual(ZeroTokenNodeTestBase.NormalNodes + 1, Cluster.AllHosts().Count), + 1000, + 120); + + _insert = Session.Prepare($"INSERT INTO {ZeroTokenNodeTestBase.TableName} (k, v) VALUES (?, ?)"); + } + + protected Host GetZeroTokenHost() + { + return Cluster.AllHosts().Single(h => h.Address.Address.ToString() == ZeroTokenAddress); + } + + protected byte[] GetRoutingKey(int partitionKey) + { + return _insert.Bind(partitionKey, partitionKey).RoutingKey.RawRoutingKey; + } + } + + /// + /// Verifies the driver behaviour against a real cluster that contains a zero-token node. + /// + /// Expected semantics (aligned with the Rust driver): + /// + /// a zero-token node is an ordinary, routable node: it is discovered, kept in the load + /// balancing plans, can act as a coordinator and can be used as a contact point; + /// a zero-token node never owns data, so it must never be returned as a replica by the + /// token map, and consequently token aware routing must never target it. + /// + /// + /// + [TestFixture] + [Category(TestCategory.Short), Category(TestCategory.RealCluster)] + [TestScyllaVersion(2025, 1)] + public class ZeroTokenNodeTests : ZeroTokenNodeTestBase + { + /// + /// Keyspace whose replication factor is higher than the amount of token owners in the cluster. + /// + private const string RfKeyspace = "zero_token_rf"; + + public override void OneTimeSetUp() + { + base.OneTimeSetUp(); + + // Created once the zero-token node is up so that the server does not complain about a + // replication factor higher than the amount of nodes. + Session.Execute( + $"CREATE KEYSPACE {ZeroTokenNodeTests.RfKeyspace} WITH replication = " + + "{'class': 'NetworkTopologyStrategy', 'replication_factor': '3'}"); + } + + [Test] + [Order(1)] + public void Should_DiscoverZeroTokenNode_Without_Tokens() + { + var hosts = Cluster.AllHosts(); + Assert.AreEqual(ZeroTokenNodeTests.NormalNodes + 1, hosts.Count); + + var zeroTokenHost = GetZeroTokenHost(); + Assert.IsTrue(zeroTokenHost.IsUp, "the zero-token node should be seen as UP"); + Assert.IsFalse( + zeroTokenHost.Tokens.Any(), + $"the zero-token node {ZeroTokenAddress} must not own any token"); + + foreach (var host in hosts.Where(h => !h.Equals(zeroTokenHost))) + { + Assert.IsTrue(host.Tokens.Any(), $"the regular node {host.Address} should own tokens"); + } + } + + [Test] + [Order(2)] + public void Should_NeverSelectZeroTokenNode_As_Replica() + { + AssertZeroTokenNodeIsNeverAReplica(); + } + + [Test] + [Order(3)] + public void Should_NeverUseZeroTokenNode_As_Coordinator_When_TokenAware() + { + var localDc = Cluster.AllHosts().First(h => h.Tokens.Any()).Datacenter; + var cluster = GetNewTemporaryCluster( + b => b.WithLoadBalancingPolicy(new TokenAwarePolicy(new DCAwareRoundRobinPolicy(localDc)))); + var session = cluster.Connect(KeyspaceName); + var insert = session.Prepare($"INSERT INTO {ZeroTokenNodeTests.TableName} (k, v) VALUES (?, ?)"); + + for (var i = 0; i < ZeroTokenNodeTests.RoutingKeySamples; i++) + { + var rs = session.Execute(insert.Bind(i, i)); + + Assert.AreNotEqual( + ZeroTokenAddress, + rs.Info.QueriedHost.Address.ToString(), + $"the zero-token node coordinated a token aware query for partition key {i}"); + } + } + + [Test] + [Order(4)] + public void Should_UseZeroTokenNode_As_Coordinator_When_RoutingIsNotTokenAware() + { + var cluster = GetNewTemporaryCluster(b => b.WithLoadBalancingPolicy(new RoundRobinPolicy())); + var session = cluster.Connect(KeyspaceName); + var coordinators = new HashSet(); + + for (var i = 0; i < ZeroTokenNodeTests.RoutingKeySamples; i++) + { + var rs = session.Execute($"SELECT k, v FROM {ZeroTokenNodeTests.TableName} WHERE k = {i}"); + coordinators.Add(rs.Info.QueriedHost.Address.ToString()); + } + + CollectionAssert.Contains( + coordinators, + ZeroTokenAddress, + "a zero-token node owns no data but is still a usable coordinator"); + } + + [Test] + [Order(5)] + public void Should_RetrieveSchemaMetadata_When_ZeroTokenNodeIsPresent() + { + Assert.IsTrue(Cluster.RefreshSchema(KeyspaceName)); + + var keyspace = Cluster.Metadata.GetKeyspace(KeyspaceName); + Assert.IsNotNull(keyspace); + CollectionAssert.Contains(keyspace.GetTablesNames(), ZeroTokenNodeTests.TableName); + + // Note: table-level metadata is deliberately not inspected here. + // Reproduced on Scylla release:2026.1.6 even on clusters with no zero-token nodes: + // SchemaParser V2 can throw when table fields are null for + // bloom_filter_fp_chance, dclocal_read_repair_chance, read_repair_chance. + // TODO: replace with tracking issue link when available. + } + + [Test] + [Order(6)] + public void Should_Connect_When_ZeroTokenNodeIsTheOnlyContactPoint() + { + var cluster = ClusterBuilder() + .AddContactPoint(ZeroTokenAddress) + .WithQueryTimeout(60000) + .WithSocketOptions(new SocketOptions().SetConnectTimeoutMillis(30000).SetReadTimeoutMillis(22000)) + .Build(); + ClusterInstances.Add(cluster); + + var session = cluster.Connect(KeyspaceName); + var insert = session.Prepare($"INSERT INTO {ZeroTokenNodeTests.TableName} (k, v) VALUES (?, ?)"); + + Assert.AreEqual( + ZeroTokenNodeTests.NormalNodes + 1, + cluster.AllHosts().Count, + "the whole topology should be discovered through the zero-token node"); + Assert.IsFalse( + cluster.AllHosts().Single(h => h.Address.Address.ToString() == ZeroTokenAddress).Tokens.Any()); + + for (var i = 0; i < ZeroTokenNodeTests.RoutingKeySamples; i++) + { + session.Execute(insert.Bind(i, i)); + } + } + + [Test] + [Order(7)] + [Category(TestCategory.RealClusterLong)] + public void Should_KeepServingQueries_When_ZeroTokenNodeGoesDownAndUp() + { + // The driver trusts the connection pools rather than the server status change events, + // so the node has to be part of the query plan for its state to be tracked. + var cluster = GetNewTemporaryCluster(b => b.WithLoadBalancingPolicy(new RoundRobinPolicy())); + var session = cluster.Connect(KeyspaceName); + var zeroTokenHost = cluster.AllHosts().Single(h => h.Address.Address.ToString() == ZeroTokenAddress); + var query = new SimpleStatement($"SELECT k, v FROM {ZeroTokenNodeTests.TableName} WHERE k = 1"); + + TestCluster.Stop(ZeroTokenNodeTests.ZeroTokenNodeId); + try + { + // RetryAssert only retries failed assertions, so a query that cannot be served by + // the remaining nodes fails the test right away. + TestHelper.RetryAssert( + () => + { + session.Execute(query); + Assert.IsFalse(zeroTokenHost.IsUp, "the zero-token node should be seen as DOWN"); + }, + 1000, + 120); + } + finally + { + TestCluster.Start(ZeroTokenNodeTests.ZeroTokenNodeId); + TestUtils.WaitForUp(ZeroTokenAddress, SharedClusterTest.DefaultCassandraPort, 120); + } + + TestHelper.RetryAssert( + () => + { + session.Execute(query); + Assert.IsTrue(zeroTokenHost.IsUp, "the zero-token node should be seen as UP again"); + }, + 1000, + 120); + + Assert.IsFalse( + GetZeroTokenHost().Tokens.Any(), + "the node should still be a zero-token node after a restart"); + AssertZeroTokenNodeIsNeverAReplica(); + } + + [Test] + [Order(8)] + public void Should_ReturnOnlyTokenOwners_When_RfExceedsNumberOfTokenOwners() + { + // Characterization test: the shared keyspace is created with RF=1, so this is the only + // place where the multi replica NetworkTopologyStrategy placement loop runs end to end. + // RF is 3 but only two nodes own tokens, so the zero-token node must not be used to pad + // the replica set. + var tokenOwners = Cluster.AllHosts().Where(h => h.Tokens.Any()).ToList(); + Assert.AreEqual(ZeroTokenNodeTests.NormalNodes, tokenOwners.Count); + + // The token map for the keyspace is refreshed asynchronously through a schema event. + TestHelper.RetryAssert( + () => + { + for (var i = 0; i < ZeroTokenNodeTests.RoutingKeySamples; i++) + { + var replicas = Cluster.Metadata + .GetReplicas(ZeroTokenNodeTests.RfKeyspace, GetRoutingKey(i)) + .Select(r => r.Host) + .ToList(); + + CollectionAssert.AreEquivalent(tokenOwners, replicas); + } + }, + 1000, + 30); + } + + private void AssertZeroTokenNodeIsNeverAReplica() + { + var zeroTokenHost = GetZeroTokenHost(); + + for (var i = 0; i < ZeroTokenNodeTests.RoutingKeySamples; i++) + { + var replicas = Cluster.Metadata.GetReplicas(KeyspaceName, GetRoutingKey(i)); + + Assert.IsNotEmpty(replicas, $"no replica was computed for partition key {i}"); + Assert.IsFalse( + replicas.Any(r => r.Host.Equals(zeroTokenHost)), + $"the zero-token node {ZeroTokenAddress} was returned as a replica for partition key {i}"); + } + } + } + + [TestFixture] + [Category(TestCategory.RealCluster), Category(TestCategory.RealClusterLong)] + [TestScyllaVersion(2025, 1)] + public class ZeroTokenNodeReplacementTests : ZeroTokenNodeTestBase + { + [Test] + public void Should_StartRoutingToNode_When_ZeroTokenNodeIsReplacedByRegularNode() + { + Assert.IsFalse(GetZeroTokenHost().Tokens.Any()); + + // ScyllaDB refuses to turn a zero-token node into a token owner in place, it fails to + // start with "Cannot restart with join_ring=true because the node has already joined + // the cluster as a zero-token node". The supported upgrade path is to decommission the + // zero-token node and to add a regular node in its place. + TestCluster.DecommissionNode(ZeroTokenNodeReplacementTests.ZeroTokenNodeId); + TestCluster.Remove(ZeroTokenNodeReplacementTests.ZeroTokenNodeId); + TestHelper.RetryAssert( + () => Assert.AreEqual(ZeroTokenNodeReplacementTests.NormalNodes, Cluster.AllHosts().Count), + 1000, + 120); + + TestCluster.BootstrapNode(ZeroTokenNodeReplacementTests.ZeroTokenNodeId, true); + TestUtils.WaitForUp(ZeroTokenAddress, SharedClusterTest.DefaultCassandraPort, 120); + + // The driver has to pick up the tokens of the replacement node without being restarted. + TestHelper.RetryAssert( + () => + { + var upgraded = Cluster.AllHosts().SingleOrDefault(h => h.Address.Address.ToString() == ZeroTokenAddress); + Assert.IsNotNull(upgraded, "the replacement node should have been discovered"); + Assert.IsTrue(upgraded.IsUp); + Assert.IsTrue(upgraded.Tokens.Any(), "the replacement node should own tokens"); + }, + 1000, + 240); + + // And it has to start using it for token aware routing. + TestHelper.RetryAssert( + () => + { + var isReplica = Enumerable + .Range(0, ZeroTokenNodeReplacementTests.RoutingKeySamples) + .SelectMany(i => Cluster.Metadata.GetReplicas(KeyspaceName, GetRoutingKey(i))) + .Any(r => r.Host.Address.Address.ToString() == ZeroTokenAddress); + Assert.IsTrue(isReplica, "the node that replaced the zero-token node should now own data"); + }, + 1000, + 240); + } + } +} diff --git a/src/Cassandra.Tests/Connections/Control/TopologyRefresherTests.cs b/src/Cassandra.Tests/Connections/Control/TopologyRefresherTests.cs index 8a65876e5..c09276774 100644 --- a/src/Cassandra.Tests/Connections/Control/TopologyRefresherTests.cs +++ b/src/Cassandra.Tests/Connections/Control/TopologyRefresherTests.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Net; using System.Threading.Tasks; @@ -261,6 +262,52 @@ await topologyRefresher.RefreshNodeListAsync( Assert.AreEqual(Version.Parse("2.1.5"), host3.CassandraVersion); } + [Test] + public async Task Should_TolerateZeroTokenPeer_WithoutWarnings() + { + var oldLevel = Diagnostics.CassandraTraceSwitch.Level; + var listener = new TestTraceListener(); + Trace.Listeners.Add(listener); + Diagnostics.CassandraTraceSwitch.Level = TraceLevel.Verbose; + try + { + var zeroTokenPeer = IPAddress.Parse("127.0.0.2"); + var normalPeer = IPAddress.Parse("127.0.0.3"); + var rows = TestHelper.CreateRows(new List> + { + // zero-token node: advertises NULL tokens (e.g. a Scylla coordinator-only node) + new Dictionary{{"rpc_address", zeroTokenPeer}, {"peer", zeroTokenPeer}, { "data_center", "ut-dc" }, { "rack", "ut-rack" }, {"tokens", null}, {"release_version", "3.0.8"}}, + // regular node that owns a token + new Dictionary{{"rpc_address", normalPeer}, {"peer", normalPeer}, { "data_center", "ut-dc" }, { "rack", "ut-rack" }, {"tokens", new [] { "0" }}, {"release_version", "3.0.8"}} + }); + var topologyRefresher = CreateTopologyRefresher(peersRows: rows); + + await topologyRefresher.RefreshNodeListAsync( + new FakeConnectionEndPoint("127.0.0.1", 9042), Mock.Of(), _serializer).ConfigureAwait(false); + + // The zero-token node is a valid, routable host and is part of the cluster metadata. + Assert.AreEqual(3, _metadata.AllHosts().Count); + + var zeroTokenHost = _metadata.GetHost(new IPEndPoint(zeroTokenPeer, ProtocolOptions.DefaultPort)); + Assert.NotNull(zeroTokenHost); + Assert.IsFalse(zeroTokenHost.Tokens.Any(), "zero-token host should have no tokens"); + + var normalHost = _metadata.GetHost(new IPEndPoint(normalPeer, ProtocolOptions.DefaultPort)); + Assert.NotNull(normalHost); + Assert.IsTrue(normalHost.Tokens.Any(), "regular host should keep its tokens"); + + // Processing a zero-token node must not produce any warnings or errors. + Trace.Flush(); + var offending = listener.Queue.Where(m => m.Contains("#ERROR") || m.Contains("#WARNING")).ToList(); + Assert.AreEqual(0, offending.Count, string.Join(Environment.NewLine, offending)); + } + finally + { + Trace.Listeners.Remove(listener); + Diagnostics.CassandraTraceSwitch.Level = oldLevel; + } + } + [Test] public async Task Should_IgnoreNullRpcAddress() { diff --git a/src/Cassandra.Tests/ZeroTokenNodeUnitTests.cs b/src/Cassandra.Tests/ZeroTokenNodeUnitTests.cs new file mode 100644 index 000000000..9c6ac604f --- /dev/null +++ b/src/Cassandra.Tests/ZeroTokenNodeUnitTests.cs @@ -0,0 +1,279 @@ +// +// Copyright (C) DataStax Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Cassandra.Tests.MetadataHelpers.TestHelpers; +using Moq; +using NUnit.Framework; +using Assert = NUnit.Framework.Legacy.ClassicAssert; +using CollectionAssert = NUnit.Framework.Legacy.CollectionAssert; + +namespace Cassandra.Tests +{ + /// + /// Verifies that the driver tolerates zero-token nodes (Scylla coordinator-only nodes that advertise + /// an empty token set). Expected behavior: + /// + /// The node stays a valid, routable host: it is reported with + /// and appears in load balancing query plans, so it can act as a coordinator for non token-aware queries. + /// The node is never selected as a replica for token-aware routing. + /// Processing such a node never produces warnings or errors. + /// + /// + [TestFixture] + public class ZeroTokenNodeUnitTests + { + private static Host NormalHost(string address, string dc, params string[] tokens) + { + return TestHelper.CreateHost(address, dc, "rack1", tokens); + } + + private static Host ZeroTokenHost(string address, string dc) + { + return TestHelper.CreateHost(address, dc, "rack1", new string[0]); + } + + [Test] + public void RoundRobinPolicy_Should_KeepZeroTokenHostRoutable() + { + var normal = NormalHost("0.0.0.1", "dc1", "1"); + var zeroToken = ZeroTokenHost("0.0.0.2", "dc1"); + var hosts = new List { normal, zeroToken }; + var clusterMock = new Mock(); + clusterMock.Setup(c => c.AllHosts()).Returns(hosts); + + var policy = new RoundRobinPolicy(); + policy.Initialize(clusterMock.Object); + + // The zero-token host is still a valid coordinator... + Assert.AreEqual(HostDistance.Local, policy.Distance(zeroToken)); + + // ...and still appears in the query plan alongside the normal host. + var plan = policy.NewQueryPlan(null, new SimpleStatement()).Select(h => h.Host).ToList(); + CollectionAssert.Contains(plan, zeroToken); + CollectionAssert.Contains(plan, normal); + } + + [Test] + public void DCAwareRoundRobinPolicy_Should_KeepLocalZeroTokenHostRoutable() + { + var normal = NormalHost("0.0.0.1", "dc1", "1"); + var zeroToken = ZeroTokenHost("0.0.0.2", "dc1"); + var hosts = new List { normal, zeroToken }; + var clusterMock = new Mock(); + clusterMock.Setup(c => c.AllHosts()).Returns(hosts); + + var policy = new DCAwareRoundRobinPolicy("dc1"); + policy.Initialize(clusterMock.Object); + + Assert.AreEqual(HostDistance.Local, policy.Distance(zeroToken)); + + var plan = policy.NewQueryPlan(null, new SimpleStatement()).Select(h => h.Host).ToList(); + CollectionAssert.Contains(plan, zeroToken); + CollectionAssert.Contains(plan, normal); + } + + [Test] + public void TokenMap_Should_NotSelectZeroTokenHost_AsReplica() + { + var withTokens1 = NormalHost("192.168.0.0", "dc1", "0"); + var zeroToken = ZeroTokenHost("192.168.0.1", "dc1"); + var withTokens2 = NormalHost("192.168.0.2", "dc1", "20"); + var hosts = new List { withTokens1, zeroToken, withTokens2 }; + var keyspaces = new List + { + FakeSchemaParserFactory.CreateSimpleKeyspace("ks1", 2) + }; + + var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces); + + // A token-less host can never enter primaryReplicas; pin the expected token owners. + foreach (var tokenValue in new long[] { -100, 0, 5, 20, 500000 }) + { + var replicas = tokenMap.GetReplicas("ks1", new M3PToken(tokenValue)).Select(r => r.Host).ToList(); + CollectionAssert.AreEquivalent(new[] { withTokens1, withTokens2 }, replicas); + CollectionAssert.DoesNotContain(replicas, zeroToken); + } + } + + [Test] + public void TokenMap_NetworkTopologyStrategy_Should_NotCountZeroTokenRackForRfPlacement() + { + // Two token owners are on rack1 and a zero-token host is on rack2. + // RF=2 in NTS must still return both token owners. + var withTokens1 = TestHelper.CreateHost("192.168.1.0", "dc1", "rack1", new[] { "0" }); + var zeroToken = TestHelper.CreateHost("192.168.1.1", "dc1", "rack2", new string[0]); + var withTokens2 = TestHelper.CreateHost("192.168.1.2", "dc1", "rack1", new[] { "20" }); + var hosts = new List { withTokens1, zeroToken, withTokens2 }; + var keyspaces = new List + { + FakeSchemaParserFactory.CreateNetworkTopologyKeyspace( + "ks_nts", + new Dictionary { { "dc1", "2" } }) + }; + + var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces); + + foreach (var tokenValue in new long[] { -100, 0, 5, 20, 500000 }) + { + var replicas = tokenMap.GetReplicas("ks_nts", new M3PToken(tokenValue)).Select(r => r.Host).ToList(); + CollectionAssert.AreEquivalent(new[] { withTokens1, withTokens2 }, replicas); + CollectionAssert.DoesNotContain(replicas, zeroToken); + } + } + + [Test] + public void TokenMap_Should_IgnoreDatacenter_When_ItOnlyContainsZeroTokenHosts() + { + // dc2 owns no token at all, so it must not take part in replica placement and its + // replication factor must be considered satisfied instead of triggering a full ring scan. + var withTokens1 = NormalHost("192.168.2.0", "dc1", "0"); + var withTokens2 = NormalHost("192.168.2.1", "dc1", "20"); + var zeroTokenOnlyDc = ZeroTokenHost("192.168.2.2", "dc2"); + var hosts = new List { withTokens1, withTokens2, zeroTokenOnlyDc }; + var keyspaces = new List + { + FakeSchemaParserFactory.CreateNetworkTopologyKeyspace( + "ks_nts", + new Dictionary { { "dc1", "2" }, { "dc2", "1" } }) + }; + + var tokenMap = TokenMap.Build("Murmur3Partitioner", hosts, keyspaces); + + foreach (var tokenValue in new long[] { -100, 0, 5, 20, 500000 }) + { + var replicas = tokenMap.GetReplicas("ks_nts", new M3PToken(tokenValue)).Select(r => r.Host).ToList(); + CollectionAssert.AreEquivalent(new[] { withTokens1, withTokens2 }, replicas); + CollectionAssert.DoesNotContain(replicas, zeroTokenOnlyDc); + } + } + + [Test] + public void TokenMap_NetworkTopologyStrategy_Should_StartPlacingReplicasInDc_When_ZeroTokenHostIsReplaced() + { + var dc1Host1 = NormalHost("192.168.3.0", "dc1", "0"); + var dc1Host2 = NormalHost("192.168.3.1", "dc1", "20"); + var dc2ZeroToken = ZeroTokenHost("192.168.3.2", "dc2"); + var keyspaces = new List + { + FakeSchemaParserFactory.CreateNetworkTopologyKeyspace( + "ks_nts", + new Dictionary { { "dc1", "1" }, { "dc2", "1" } }) + }; + + var tokenMapBeforeReplacement = TokenMap.Build( + "Murmur3Partitioner", + new List { dc1Host1, dc1Host2, dc2ZeroToken }, + keyspaces); + + var dc2Replacement = NormalHost("192.168.3.2", "dc2", "10"); + var tokenMapAfterReplacement = TokenMap.Build( + "Murmur3Partitioner", + new List { dc1Host1, dc1Host2, dc2Replacement }, + keyspaces); + + foreach (var tokenValue in new long[] { -100, 0, 5, 10, 20, 500000 }) + { + var replicasBefore = tokenMapBeforeReplacement + .GetReplicas("ks_nts", new M3PToken(tokenValue)) + .Select(r => r.Host) + .ToList(); + + CollectionAssert.DoesNotContain(replicasBefore, dc2ZeroToken); + Assert.IsFalse(replicasBefore.Any(h => h.Datacenter == "dc2")); + Assert.AreEqual(1, replicasBefore.Count(h => h.Datacenter == "dc1")); + Assert.AreEqual(1, replicasBefore.Count); + + var replicasAfter = tokenMapAfterReplacement + .GetReplicas("ks_nts", new M3PToken(tokenValue)) + .Select(r => r.Host) + .ToList(); + + CollectionAssert.Contains(replicasAfter, dc2Replacement); + Assert.AreEqual(1, replicasAfter.Count(h => h.Datacenter == "dc2")); + Assert.AreEqual(1, replicasAfter.Count(h => h.Datacenter == "dc1")); + Assert.AreEqual(2, replicasAfter.Count); + } + } + + [Test] + public void TokenMap_Build_Should_NotThrow_When_HostHasNoTokens() + { + var hosts = new List + { + NormalHost("192.168.0.0", "dc1", "0"), + ZeroTokenHost("192.168.0.1", "dc1"), + }; + var keyspaces = new List + { + FakeSchemaParserFactory.CreateSimpleKeyspace("ks1", 1) + }; + + Assert.DoesNotThrow(() => TokenMap.Build("Murmur3Partitioner", hosts, keyspaces)); + } + + [Test] + public void SetInfo_Should_YieldEmptyTokens_When_TokensAreNullOrEmpty() + { + // Scylla may report the empty token set either as NULL or as an empty collection. + var nullTokenHost = TestHelper.CreateHost("0.0.0.1", "dc1", "rack1", tokens: null); + var emptyTokenHost = TestHelper.CreateHost("0.0.0.2", "dc1", "rack1", tokens: new string[0]); + + Assert.IsNotNull(nullTokenHost.Tokens, "Tokens must never be null"); + Assert.IsNotNull(emptyTokenHost.Tokens, "Tokens must never be null"); + Assert.IsEmpty(nullTokenHost.Tokens); + Assert.IsEmpty(emptyTokenHost.Tokens); + } + + [Test] + public void ZeroTokenNode_Processing_Should_NotLogWarningOrError() + { + var previousLevel = Diagnostics.CassandraTraceSwitch.Level; + var listener = new TestTraceListener(); + Diagnostics.CassandraTraceSwitch.Level = TraceLevel.Verbose; + Trace.Listeners.Add(listener); + try + { + var hosts = new List + { + NormalHost("192.168.0.0", "dc1", "0"), + ZeroTokenHost("192.168.0.1", "dc1"), + NormalHost("192.168.0.2", "dc1", "20"), + }; + var keyspaces = new List + { + FakeSchemaParserFactory.CreateSimpleKeyspace("ks1", 2) + }; + + TokenMap.Build("Murmur3Partitioner", hosts, keyspaces); + + Trace.Flush(); + var offending = listener.Queue + .Where(m => m.Contains("#ERROR") || m.Contains("#WARNING")) + .ToList(); + Assert.AreEqual(0, offending.Count, string.Join(Environment.NewLine, offending)); + } + finally + { + Trace.Listeners.Remove(listener); + Diagnostics.CassandraTraceSwitch.Level = previousLevel; + } + } + } +} diff --git a/src/Cassandra/TokenMap.cs b/src/Cassandra/TokenMap.cs index 569aa2df1..b76244e7c 100644 --- a/src/Cassandra/TokenMap.cs +++ b/src/Cassandra/TokenMap.cs @@ -121,7 +121,13 @@ public static TokenMap Build(string partitioner, ICollection hosts, IColle var datacenters = new Dictionary(); foreach (var host in hosts) { - if (host.Datacenter != null) + // Only hosts that own tokens take part in replica placement, so zero-token hosts must not + // contribute to a datacenter's host count or rack set. This keeps the invariant that every + // host in primaryReplicas with a non null datacenter has an entry in `datacenters`, which + // the replication strategies rely on. A datacenter made up exclusively of zero-token hosts + // is therefore absent from `datacenters` and its replication factor is trivially satisfied. + var hasTokens = host.Tokens.Any(); + if (hasTokens && host.Datacenter != null) { if (!datacenters.TryGetValue(host.Datacenter, out var dc)) {