diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java index c334a2d842ed..ef082aa9e0eb 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java @@ -402,7 +402,7 @@ public Statistics getStatistics() { * Also decrement committed bytes against the bytes written. * @param bytes the number of bytes write into the container. */ - private void incrWriteBytes(long bytes) { + public void incrWriteBytes(long bytes) { /* Increase the cached Used Space in VolumeInfo as it maybe not updated, DU or DedicatedDiskSpaceUsage runs @@ -571,7 +571,9 @@ public boolean needsDataChecksum() { public void updateWriteStats(long bytesWritten, boolean overwrite) { getStatistics().updateWrite(bytesWritten, overwrite); - incrWriteBytes(bytesWritten); + if (!overwrite) { + incrWriteBytes(bytesWritten); + } } @Override @@ -672,6 +674,14 @@ public synchronized void updateWrite(long length, boolean overwrite) { writeBytes += length; } + /** + * Increment blockBytes by the given delta. + * This is used for overwrite operations that extend the file. + */ + public synchronized void incrementBlockBytes(long delta) { + blockBytes += delta; + } + public synchronized void decDeletion(long deletedBytes, long processedBytes, long deletedBlockCount, long processedBlockCount) { blockBytes -= deletedBytes; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java index 36ebdc5aa823..33f5bc0bdb4a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java @@ -182,13 +182,14 @@ public void writeChunk(Container container, BlockID blockID, ChunkInfo info, ChunkUtils.writeData(channel, chunkFile.getName(), data, offset, chunkLength, volume); - // When overwriting, update the bytes used if the new length is greater than the old length - // This is to ensure that the bytes used is updated correctly when overwriting a smaller chunk - // with a larger chunk at the end of the block. + // When overwriting, if the file extended beyond its previous length, + // we need to account for the delta in blockBytes, usedSpace and committedBytes. if (overwrite) { long fileLengthAfterWrite = offset + chunkLength; if (fileLengthAfterWrite > fileLengthBeforeWrite) { - containerData.getStatistics().updateWrite(fileLengthAfterWrite - fileLengthBeforeWrite, false); + long delta = fileLengthAfterWrite - fileLengthBeforeWrite; + containerData.getStatistics().incrementBlockBytes(delta); + containerData.incrWriteBytes(delta); } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java index 95475651d014..364ddad2cfd3 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java @@ -229,6 +229,57 @@ public void testWriteChunkForClosedContainer() Assertions.assertEquals(containerData.getBytesUsed(), writeChunkData.remaining() + newWriteChunkData.remaining()); } + /** + * Test that overwrite operations that extend the file correctly update usedSpace by the delta. + */ + @Test + public void testOverwriteFileExtensionUpdatesByDelta() throws Exception { + KeyValueContainer kvContainer = getKeyValueContainer(); + KeyValueContainerData containerData = kvContainer.getContainerData(); + ChunkManager chunkManager = createTestSubject(); + + // Initial write: 4 bytes at offset 0 + byte[] initialData = "test".getBytes(UTF_8); + ChunkInfo initialChunk = new ChunkInfo(String.format("%d.data.%d", getBlockID().getLocalID(), 0), + 0, // offset + initialData.length); + ChunkBuffer initialBuffer = ChunkBuffer.allocate(initialData.length).put(initialData); + initialBuffer.rewind(); + setDataChecksum(initialChunk, initialBuffer); + + long initialUsedSpace = containerData.getVolume().getCurrentUsage().getUsedSpace(); + long initialBlockBytes = containerData.getBytesUsed(); + chunkManager.writeChunk(kvContainer, getBlockID(), initialChunk, initialBuffer, WRITE_STAGE); + long afterFirstWriteUsedSpace = containerData.getVolume().getCurrentUsage().getUsedSpace(); + long afterFirstWriteBlockBytes = containerData.getBytesUsed(); + + assertEquals(initialUsedSpace + initialData.length, afterFirstWriteUsedSpace); + assertEquals(initialBlockBytes + initialData.length, afterFirstWriteBlockBytes); + + // Overwrite that extends file: write 6 bytes at offset 2 (extends file from 4 to 8 bytes) + // File before: [t][e][s][t] + // File after: [t][e][e][x][t][e][n][d] + // File length delta: 8 - 4 = 4 bytes + byte[] overwriteData = "extend".getBytes(UTF_8); + ChunkInfo overwriteChunk = new ChunkInfo(String.format("%d.data.%d", getBlockID().getLocalID(), 0), + 2, // offset - starts at position 2 + overwriteData.length); + ChunkBuffer overwriteBuffer = ChunkBuffer.allocate(overwriteData.length).put(overwriteData); + overwriteBuffer.rewind(); + setDataChecksum(overwriteChunk, overwriteBuffer); + + chunkManager.writeChunk(kvContainer, getBlockID(), overwriteChunk, overwriteBuffer, WRITE_STAGE); + long afterOverwriteUsedSpace = containerData.getVolume().getCurrentUsage().getUsedSpace(); + long afterOverwriteBlockBytes = containerData.getBytesUsed(); + + long expectedDelta = (2 + overwriteData.length) - initialData.length; // 8 - 4 = 4 + long expectedWriteBytes = initialData.length + overwriteData.length; // 4 + 6 = 10 + + assertEquals(afterFirstWriteUsedSpace + expectedDelta, afterOverwriteUsedSpace); + assertEquals(afterFirstWriteBlockBytes + expectedDelta, afterOverwriteBlockBytes); + assertEquals(expectedWriteBytes, containerData.getStatistics().getWriteBytes()); + } + @Test public void testPutBlockForClosedContainer() throws IOException { OzoneConfiguration conf = new OzoneConfiguration();