Skip to content

Commit 36499d2

Browse files
DmitryLukyanovDmitryLukyanov
committed
CSHARP-2486: Add support for Zstandard compression.
1 parent d18e1c1 commit 36499d2

24 files changed

+1418
-10
lines changed

‎THIRD-PARTY-NOTICES‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,3 +116,48 @@ https://github.com/mongodb/mongo-csharp-driver.
116116
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
117117
See the License for the specific language governing permissions and
118118
limitations under the License.
119+
120+
4. The following files: NativeBufferInfo.cs, Zstandard64NativeMethods.cs, ZstandardNativeWrapper.cs, ZstandardStream.cs
121+
122+
Original work:
123+
Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
124+
125+
Redistribution and use in source and binary forms, with or without modification,
126+
are permitted provided that the following conditions are met:
127+
128+
* Redistributions of source code must retain the above copyright notice, this
129+
list of conditions and the following disclaimer.
130+
131+
* Redistributions in binary form must reproduce the above copyright notice,
132+
this list of conditions and the following disclaimer in the documentation
133+
and/or other materials provided with the distribution.
134+
135+
* Neither the name Facebook nor the names of its contributors may be used to
136+
endorse or promote products derived from this software without specific
137+
prior written permission.
138+
139+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
140+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
141+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
142+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
143+
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
144+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
145+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
146+
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
147+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
148+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
149+
150+
Modified work:
151+
Copyright 2020–present MongoDB Inc.
152+
153+
Licensed under the Apache License, Version 2.0 (the "License");
154+
you may not use this file except in compliance with the License.
155+
You may obtain a copy of the License at
156+
157+
http://www.apache.org/licenses/LICENSE-2.0
158+
159+
Unless required by applicable law or agreed to in writing, software
160+
distributed under the License is distributed on an "AS IS" BASIS,
161+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
162+
See the License for the specific language governing permissions and
163+
limitations under the License.

‎evergreen/evergreen.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,6 +1064,10 @@ axes:
10641064
display_name: Snappy
10651065
variables:
10661066
COMPRESSOR: "snappy"
1067+
- id: "zstandard"
1068+
display_name: Zstandard
1069+
variables:
1070+
COMPRESSOR: "zstandard"
10671071

10681072
buildvariants:
10691073

@@ -1102,6 +1106,13 @@ buildvariants:
11021106
tasks:
11031107
- name: "test"
11041108

1109+
- matrix_name: "tests-zstandard-compression"
1110+
matrix_spec: { compressor : "zstandard", auth: "noauth", ssl: "nossl", version: ["3.6", "4.0", "4.2", "4.4", "latest"], topology: "standalone", os: "*" }
1111+
display_name: "${version} ${compressor} ${topology} ${auth} ${ssl} ${os} "
1112+
tags: ["tests-variant"]
1113+
tasks:
1114+
- name: "test"
1115+
11051116
- matrix_name: "ocsp-tests"
11061117
matrix_spec: { version: ["4.4", "latest"], auth: "noauth", ssl: "ssl", topology: "standalone", os: "*" } # matrix_spec: { version: ["latest"], os-ssl-32: ["ubuntu1604-64-go-1-12"] }
11071118
display_name: "OCSP ${version} ${os}" # display_name: "OCSP ${version} ${os-ssl-32}"

‎src/MongoDB.Driver.Core/Core/Compression/CompressorSource.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,8 @@ public static bool IsCompressorSupported(CompressorType compressorType)
4343
switch (compressorType)
4444
{
4545
case CompressorType.Snappy:
46-
return true;
4746
case CompressorType.Zlib:
48-
return true;
47+
case CompressorType.ZStandard:
4948
case CompressorType.Noop: // This is realistically only used for testing
5049
return true;
5150
default:
@@ -86,6 +85,8 @@ private ICompressor CreateCompressor(CompressorConfiguration compressorConfigura
8685

8786
return new ZlibCompressor(zlibCompressionLevel);
8887
}
88+
case CompressorType.ZStandard:
89+
return new ZstandardCompressor();
8990
}
9091

9192
throw new NotSupportedException($"The compressor {compressorConfiguration.Type} is not supported.");

‎src/MongoDB.Driver.Core/Core/Compression/ICompressor.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,11 @@ public enum CompressorType
3535
/// <summary>
3636
/// The content of the message is compressed using zlib.
3737
/// </summary>
38-
Zlib = 2
38+
Zlib = 2,
39+
/// <summary>
40+
/// The content of the message is compressed using zstandard.
41+
/// </summary>
42+
ZStandard = 3
3943
}
4044

4145
/// <summary>

‎src/MongoDB.Driver.Core/Core/Compression/Snappy/Snappy32NativeMethods.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,8 @@ private class Delegates32
105105

106106
private class SnappyLocator : RelativeLibraryLocatorBase
107107
{
108+
public override bool IsX32ModeSupported => true;
109+
108110
public override string GetLibraryRelativePath(SupportedPlatform currentPlatform)
109111
{
110112
switch (currentPlatform)

‎src/MongoDB.Driver.Core/Core/Compression/Snappy/Snappy64NativeMethods.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ private class Delegates64
9999

100100
private class SnappyLocator : RelativeLibraryLocatorBase
101101
{
102+
public override bool IsX32ModeSupported => true;
103+
102104
public override string GetLibraryRelativePath(SupportedPlatform currentPlatform)
103105
{
104106
switch (currentPlatform)

‎src/MongoDB.Driver.Core/Core/Compression/Zstandard/NativeBufferInfo.cs‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/* Original work:
2+
* Copyright(c) 2016-present, Facebook, Inc. All rights reserved.
3+
*
4+
* Redistribution and use in source and binary forms, with or without modification,
5+
* are permitted provided that the following conditions are met:
6+
*
7+
* * Redistributions of source code must retain the above copyright notice, this
8+
* list of conditions and the following disclaimer.
9+
*
10+
* * Redistributions in binary form must reproduce the above copyright notice,
11+
* this list of conditions and the following disclaimer in the documentation
12+
* and/or other materials provided with the distribution.
13+
*
14+
* * Neither the name Facebook nor the names of its contributors may be used to
15+
* endorse or promote products derived from this software without specific
16+
* prior written permission.
17+
*
18+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19+
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20+
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21+
* DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
22+
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23+
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24+
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
25+
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26+
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27+
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28+
*
29+
* Modified work:
30+
* Copyright 2020–present MongoDB Inc.
31+
*
32+
* Licensed under the Apache License, Version 2.0 (the "License");
33+
* you may not use this file except in compliance with the License.
34+
* You may obtain a copy of the License at
35+
*
36+
* http://www.apache.org/licenses/LICENSE-2.0
37+
*
38+
* Unless required by applicable law or agreed to in writing, software
39+
* distributed under the License is distributed on an "AS IS" BASIS,
40+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
41+
* See the License for the specific language governing permissions and
42+
* limitations under the License.
43+
*/
44+
45+
using System;
46+
using System.Runtime.InteropServices;
47+
48+
namespace MongoDB.Driver.Core.Compression.Zstandard
49+
{
50+
[StructLayout(LayoutKind.Sequential)]
51+
internal class NativeBufferInfo
52+
{
53+
public IntPtr DataPointer = IntPtr.Zero;
54+
public ulong Size = 0;
55+
public ulong Position = 0;
56+
}
57+
}

‎src/MongoDB.Driver.Core/Core/Compression/Zstandard/PinnedBufferWalker.cs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/* Copyright 2020–present MongoDB Inc.
2+
*
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
using System;
17+
using System.Runtime.InteropServices;
18+
using MongoDB.Driver.Core.Misc;
19+
20+
namespace MongoDB.Driver.Core.Compression.Zstandard
21+
{
22+
internal class PinnedBufferWalker : IDisposable
23+
{
24+
private readonly byte[] _bytes;
25+
private GCHandle _handle; // not readonly to prevent a temporary copy from being created when calling Free
26+
private IntPtr _intPtr;
27+
private int _offset;
28+
29+
public PinnedBufferWalker(byte[] bytes, int offset)
30+
{
31+
_bytes = Ensure.IsNotNull(bytes, nameof(bytes));
32+
// The array must be pinned by using a GCHandle before it is passed to UnsafeAddrOfPinnedArrayElement.
33+
// For maximum performance, this method does not validate the array passed to it; this can result in unexpected behavior.
34+
_handle = GCHandle.Alloc(_bytes, GCHandleType.Pinned);
35+
_offset = offset;
36+
37+
RefreshIntPtr();
38+
}
39+
40+
public IntPtr IntPtr => _intPtr;
41+
42+
public int Offset
43+
{
44+
get => _offset;
45+
set
46+
{
47+
_offset = value;
48+
RefreshIntPtr();
49+
}
50+
}
51+
52+
// public methods
53+
public void Dispose()
54+
{
55+
try
56+
{
57+
_handle.Free();
58+
}
59+
catch
60+
{
61+
// ignore exceptions
62+
}
63+
}
64+
65+
// private methods
66+
private void RefreshIntPtr()
67+
{
68+
_intPtr = Marshal.UnsafeAddrOfPinnedArrayElement(_bytes, _offset);
69+
}
70+
}
71+
}

0 commit comments

Comments
 (0)