From 1e58db2205cf5b05b40ad988e5449c4e0da9d605 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 22 Oct 2024 17:25:35 +1000 Subject: [PATCH 01/10] Remove ChunkedMemoryStream --- src/ImageSharp/Formats/Gif/GifDecoderCore.cs | 2 +- .../Sections/GifXmpApplicationExtension.cs | 10 +- src/ImageSharp/Formats/ImageDecoder.cs | 12 +- src/ImageSharp/Formats/ImageEncoder.cs | 7 +- src/ImageSharp/IO/ChunkedMemoryStream.cs | 585 ------------------ src/ImageSharp/Image.FromStream.cs | 5 +- .../Formats/WebP/WebpEncoderTests.cs | 19 + .../IO/ChunkedMemoryStreamTests.cs | 373 ----------- .../Image/NonSeekableStream.cs | 6 +- 9 files changed, 36 insertions(+), 983 deletions(-) delete mode 100644 src/ImageSharp/IO/ChunkedMemoryStream.cs delete mode 100644 tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs diff --git a/src/ImageSharp/Formats/Gif/GifDecoderCore.cs b/src/ImageSharp/Formats/Gif/GifDecoderCore.cs index 68f4e5fa2d..c45450a47b 100644 --- a/src/ImageSharp/Formats/Gif/GifDecoderCore.cs +++ b/src/ImageSharp/Formats/Gif/GifDecoderCore.cs @@ -317,7 +317,7 @@ private void ReadApplicationExtension(BufferedReadStream stream) bool isXmp = this.buffer.Span.StartsWith(GifConstants.XmpApplicationIdentificationBytes); if (isXmp && !this.skipMetadata) { - GifXmpApplicationExtension extension = GifXmpApplicationExtension.Read(stream, this.memoryAllocator); + GifXmpApplicationExtension extension = GifXmpApplicationExtension.Read(stream); if (extension.Data.Length > 0) { this.metadata!.XmpProfile = new XmpProfile(extension.Data); diff --git a/src/ImageSharp/Formats/Gif/Sections/GifXmpApplicationExtension.cs b/src/ImageSharp/Formats/Gif/Sections/GifXmpApplicationExtension.cs index 1c1127c3be..8bd8497eea 100644 --- a/src/ImageSharp/Formats/Gif/Sections/GifXmpApplicationExtension.cs +++ b/src/ImageSharp/Formats/Gif/Sections/GifXmpApplicationExtension.cs @@ -1,7 +1,6 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.Memory; namespace SixLabors.ImageSharp.Formats.Gif; @@ -26,11 +25,10 @@ namespace SixLabors.ImageSharp.Formats.Gif; /// Reads the XMP metadata from the specified stream. /// /// The stream to read from. - /// The memory allocator. /// The XMP metadata - public static GifXmpApplicationExtension Read(Stream stream, MemoryAllocator allocator) + public static GifXmpApplicationExtension Read(Stream stream) { - byte[] xmpBytes = ReadXmpData(stream, allocator); + byte[] xmpBytes = ReadXmpData(stream); // Exclude the "magic trailer", see XMP Specification Part 3, 1.1.2 GIF int xmpLength = xmpBytes.Length - 256; // 257 - unread 0x0 @@ -71,9 +69,9 @@ public int WriteTo(Span buffer) return this.ContentLength; } - private static byte[] ReadXmpData(Stream stream, MemoryAllocator allocator) + private static byte[] ReadXmpData(Stream stream) { - using ChunkedMemoryStream bytes = new(allocator); + using MemoryStream bytes = new(); // XMP data doesn't have a fixed length nor is there an indicator of the length. // So we simply read one byte at a time until we hit the 0x0 value at the end diff --git a/src/ImageSharp/Formats/ImageDecoder.cs b/src/ImageSharp/Formats/ImageDecoder.cs index 549a28d409..03cfa27cfb 100644 --- a/src/ImageSharp/Formats/ImageDecoder.cs +++ b/src/ImageSharp/Formats/ImageDecoder.cs @@ -1,7 +1,6 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; @@ -210,7 +209,7 @@ T PerformActionAndResetPosition(Stream s, long position) } Configuration configuration = options.Configuration; - using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator); + using MemoryStream memoryStream = new(); stream.CopyTo(memoryStream, configuration.StreamProcessingBufferSize); memoryStream.Position = 0; @@ -266,11 +265,6 @@ Task PerformActionAndResetPosition(Stream s, long position, CancellationToken return PerformActionAndResetPosition(ms, ms.Position, cancellationToken); } - if (stream is ChunkedMemoryStream cms) - { - return PerformActionAndResetPosition(cms, cms.Position, cancellationToken); - } - return CopyToMemoryStreamAndActionAsync(options, stream, PerformActionAndResetPosition, cancellationToken); } @@ -282,9 +276,11 @@ private static async Task CopyToMemoryStreamAndActionAsync( { long position = stream.CanSeek ? stream.Position : 0; Configuration configuration = options.Configuration; - await using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator); + + await using MemoryStream memoryStream = new(); await stream.CopyToAsync(memoryStream, configuration.StreamProcessingBufferSize, cancellationToken).ConfigureAwait(false); memoryStream.Position = 0; + return await action(memoryStream, position, cancellationToken).ConfigureAwait(false); } diff --git a/src/ImageSharp/Formats/ImageEncoder.cs b/src/ImageSharp/Formats/ImageEncoder.cs index deb527f698..34d34c3637 100644 --- a/src/ImageSharp/Formats/ImageEncoder.cs +++ b/src/ImageSharp/Formats/ImageEncoder.cs @@ -1,7 +1,6 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp.Formats; @@ -48,8 +47,8 @@ private void EncodeWithSeekableStream(Image image, Stream stream } else { - using ChunkedMemoryStream ms = new(configuration.MemoryAllocator); - this.Encode(image, stream, cancellationToken); + using MemoryStream ms = new(); + this.Encode(image, ms, cancellationToken); ms.Position = 0; ms.CopyTo(stream, configuration.StreamProcessingBufferSize); } @@ -65,7 +64,7 @@ private async Task EncodeWithSeekableStreamAsync(Image image, St } else { - using ChunkedMemoryStream ms = new(configuration.MemoryAllocator); + await using MemoryStream ms = new(); await DoEncodeAsync(ms); ms.Position = 0; await ms.CopyToAsync(stream, configuration.StreamProcessingBufferSize, cancellationToken) diff --git a/src/ImageSharp/IO/ChunkedMemoryStream.cs b/src/ImageSharp/IO/ChunkedMemoryStream.cs deleted file mode 100644 index 2534548141..0000000000 --- a/src/ImageSharp/IO/ChunkedMemoryStream.cs +++ /dev/null @@ -1,585 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using System.Buffers; -using System.Runtime.CompilerServices; -using SixLabors.ImageSharp.Memory; - -namespace SixLabors.ImageSharp.IO; - -/// -/// Provides an in-memory stream composed of non-contiguous chunks that doesn't need to be resized. -/// Chunks are allocated by the assigned via the constructor -/// and is designed to take advantage of buffer pooling when available. -/// -internal sealed class ChunkedMemoryStream : Stream -{ - // The memory allocator. - private readonly MemoryAllocator allocator; - - // Data - private MemoryChunk? memoryChunk; - - // The total number of allocated chunks - private int chunkCount; - - // The length of the largest contiguous buffer that can be handled by the allocator. - private readonly int allocatorCapacity; - - // Has the stream been disposed. - private bool isDisposed; - - // Current chunk to write to - private MemoryChunk? writeChunk; - - // Offset into chunk to write to - private int writeOffset; - - // Current chunk to read from - private MemoryChunk? readChunk; - - // Offset into chunk to read from - private int readOffset; - - /// - /// Initializes a new instance of the class. - /// - /// The memory allocator. - public ChunkedMemoryStream(MemoryAllocator allocator) - { - this.allocatorCapacity = allocator.GetBufferCapacityInBytes(); - this.allocator = allocator; - } - - /// - public override bool CanRead => !this.isDisposed; - - /// - public override bool CanSeek => !this.isDisposed; - - /// - public override bool CanWrite => !this.isDisposed; - - /// - public override long Length - { - get - { - this.EnsureNotDisposed(); - - int length = 0; - MemoryChunk? chunk = this.memoryChunk; - while (chunk != null) - { - MemoryChunk? next = chunk.Next; - if (next != null) - { - length += chunk.Length; - } - else - { - length += this.writeOffset; - } - - chunk = next; - } - - return length; - } - } - - /// - public override long Position - { - get - { - this.EnsureNotDisposed(); - - if (this.readChunk is null) - { - return 0; - } - - int pos = 0; - MemoryChunk? chunk = this.memoryChunk; - while (chunk != this.readChunk && chunk is not null) - { - pos += chunk.Length; - chunk = chunk.Next; - } - - pos += this.readOffset; - - return pos; - } - - set - { - this.EnsureNotDisposed(); - - if (value < 0) - { - ThrowArgumentOutOfRange(nameof(value)); - } - - // Back up current position in case new position is out of range - MemoryChunk? backupReadChunk = this.readChunk; - int backupReadOffset = this.readOffset; - - this.readChunk = null; - this.readOffset = 0; - - int leftUntilAtPos = (int)value; - MemoryChunk? chunk = this.memoryChunk; - while (chunk != null) - { - if ((leftUntilAtPos < chunk.Length) - || ((leftUntilAtPos == chunk.Length) - && (chunk.Next is null))) - { - // The desired position is in this chunk - this.readChunk = chunk; - this.readOffset = leftUntilAtPos; - break; - } - - leftUntilAtPos -= chunk.Length; - chunk = chunk.Next; - } - - if (this.readChunk is null) - { - // Position is out of range - this.readChunk = backupReadChunk; - this.readOffset = backupReadOffset; - } - } - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override long Seek(long offset, SeekOrigin origin) - { - this.EnsureNotDisposed(); - - switch (origin) - { - case SeekOrigin.Begin: - this.Position = offset; - break; - - case SeekOrigin.Current: - this.Position += offset; - break; - - case SeekOrigin.End: - this.Position = this.Length + offset; - break; - default: - ThrowInvalidSeek(); - break; - } - - return this.Position; - } - - /// - public override void SetLength(long value) - => throw new NotSupportedException(); - - /// - protected override void Dispose(bool disposing) - { - if (this.isDisposed) - { - return; - } - - try - { - this.isDisposed = true; - if (disposing) - { - ReleaseMemoryChunks(this.memoryChunk); - } - - this.memoryChunk = null; - this.writeChunk = null; - this.readChunk = null; - this.chunkCount = 0; - } - finally - { - base.Dispose(disposing); - } - } - - /// - public override void Flush() - { - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override int Read(byte[] buffer, int offset, int count) - { - Guard.NotNull(buffer, nameof(buffer)); - Guard.MustBeGreaterThanOrEqualTo(offset, 0, nameof(offset)); - Guard.MustBeGreaterThanOrEqualTo(count, 0, nameof(count)); - - const string bufferMessage = "Offset subtracted from the buffer length is less than count."; - Guard.IsFalse(buffer.Length - offset < count, nameof(buffer), bufferMessage); - - return this.ReadImpl(buffer.AsSpan(offset, count)); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override int Read(Span buffer) => this.ReadImpl(buffer); - - private int ReadImpl(Span buffer) - { - this.EnsureNotDisposed(); - - if (this.readChunk is null) - { - if (this.memoryChunk is null) - { - return 0; - } - - this.readChunk = this.memoryChunk; - this.readOffset = 0; - } - - IMemoryOwner chunkBuffer = this.readChunk.Buffer; - int chunkSize = this.readChunk.Length; - if (this.readChunk.Next is null) - { - chunkSize = this.writeOffset; - } - - int bytesRead = 0; - int offset = 0; - int count = buffer.Length; - while (count > 0) - { - if (this.readOffset == chunkSize) - { - // Exit if no more chunks are currently available - if (this.readChunk.Next is null) - { - break; - } - - this.readChunk = this.readChunk.Next; - this.readOffset = 0; - chunkBuffer = this.readChunk.Buffer; - chunkSize = this.readChunk.Length; - if (this.readChunk.Next is null) - { - chunkSize = this.writeOffset; - } - } - - int readCount = Math.Min(count, chunkSize - this.readOffset); - chunkBuffer.Slice(this.readOffset, readCount).CopyTo(buffer[offset..]); - offset += readCount; - count -= readCount; - this.readOffset += readCount; - bytesRead += readCount; - } - - return bytesRead; - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override int ReadByte() - { - this.EnsureNotDisposed(); - - if (this.readChunk is null) - { - if (this.memoryChunk is null) - { - return 0; - } - - this.readChunk = this.memoryChunk; - this.readOffset = 0; - } - - IMemoryOwner chunkBuffer = this.readChunk.Buffer; - int chunkSize = this.readChunk.Length; - if (this.readChunk.Next is null) - { - chunkSize = this.writeOffset; - } - - if (this.readOffset == chunkSize) - { - // Exit if no more chunks are currently available - if (this.readChunk.Next is null) - { - return -1; - } - - this.readChunk = this.readChunk.Next; - this.readOffset = 0; - chunkBuffer = this.readChunk.Buffer; - } - - return chunkBuffer.GetSpan()[this.readOffset++]; - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override void Write(byte[] buffer, int offset, int count) - { - Guard.NotNull(buffer, nameof(buffer)); - Guard.MustBeGreaterThanOrEqualTo(offset, 0, nameof(offset)); - Guard.MustBeGreaterThanOrEqualTo(count, 0, nameof(count)); - - const string bufferMessage = "Offset subtracted from the buffer length is less than count."; - Guard.IsFalse(buffer.Length - offset < count, nameof(buffer), bufferMessage); - - this.WriteImpl(buffer.AsSpan(offset, count)); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override void Write(ReadOnlySpan buffer) => this.WriteImpl(buffer); - - private void WriteImpl(ReadOnlySpan buffer) - { - this.EnsureNotDisposed(); - - if (this.memoryChunk is null) - { - this.memoryChunk = this.AllocateMemoryChunk(); - this.writeChunk = this.memoryChunk; - this.writeOffset = 0; - } - - Guard.NotNull(this.writeChunk); - - Span chunkBuffer = this.writeChunk.Buffer.GetSpan(); - int chunkSize = this.writeChunk.Length; - int count = buffer.Length; - int offset = 0; - while (count > 0) - { - if (this.writeOffset == chunkSize) - { - // Allocate a new chunk if the current one is full - this.writeChunk.Next = this.AllocateMemoryChunk(); - this.writeChunk = this.writeChunk.Next; - this.writeOffset = 0; - chunkBuffer = this.writeChunk.Buffer.GetSpan(); - chunkSize = this.writeChunk.Length; - } - - int copyCount = Math.Min(count, chunkSize - this.writeOffset); - buffer.Slice(offset, copyCount).CopyTo(chunkBuffer[this.writeOffset..]); - - offset += copyCount; - count -= copyCount; - this.writeOffset += copyCount; - } - } - - /// - public override void WriteByte(byte value) - { - this.EnsureNotDisposed(); - - if (this.memoryChunk is null) - { - this.memoryChunk = this.AllocateMemoryChunk(); - this.writeChunk = this.memoryChunk; - this.writeOffset = 0; - } - - Guard.NotNull(this.writeChunk); - - IMemoryOwner chunkBuffer = this.writeChunk.Buffer; - int chunkSize = this.writeChunk.Length; - - if (this.writeOffset == chunkSize) - { - // Allocate a new chunk if the current one is full - this.writeChunk.Next = this.AllocateMemoryChunk(); - this.writeChunk = this.writeChunk.Next; - this.writeOffset = 0; - chunkBuffer = this.writeChunk.Buffer; - } - - chunkBuffer.GetSpan()[this.writeOffset++] = value; - } - - /// - /// Copy entire buffer into an array. - /// - /// The . - public byte[] ToArray() - { - int length = (int)this.Length; // This will throw if stream is closed - byte[] copy = new byte[this.Length]; - - MemoryChunk? backupReadChunk = this.readChunk; - int backupReadOffset = this.readOffset; - - this.readChunk = this.memoryChunk; - this.readOffset = 0; - this.Read(copy, 0, length); - - this.readChunk = backupReadChunk; - this.readOffset = backupReadOffset; - - return copy; - } - - /// - /// Write remainder of this stream to another stream. - /// - /// The stream to write to. - public void WriteTo(Stream stream) - { - this.EnsureNotDisposed(); - - Guard.NotNull(stream, nameof(stream)); - - if (this.readChunk is null) - { - if (this.memoryChunk is null) - { - return; - } - - this.readChunk = this.memoryChunk; - this.readOffset = 0; - } - - IMemoryOwner chunkBuffer = this.readChunk.Buffer; - int chunkSize = this.readChunk.Length; - if (this.readChunk.Next is null) - { - chunkSize = this.writeOffset; - } - - // Following code mirrors Read() logic (readChunk/readOffset should - // point just past last byte of last chunk when done) - // loop until end of chunks is found - while (true) - { - if (this.readOffset == chunkSize) - { - // Exit if no more chunks are currently available - if (this.readChunk.Next is null) - { - break; - } - - this.readChunk = this.readChunk.Next; - this.readOffset = 0; - chunkBuffer = this.readChunk.Buffer; - chunkSize = this.readChunk.Length; - if (this.readChunk.Next is null) - { - chunkSize = this.writeOffset; - } - } - - int writeCount = chunkSize - this.readOffset; - stream.Write(chunkBuffer.GetSpan(), this.readOffset, writeCount); - this.readOffset = chunkSize; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void EnsureNotDisposed() - { - if (this.isDisposed) - { - ThrowDisposed(); - } - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void ThrowDisposed() => throw new ObjectDisposedException(null, "The stream is closed."); - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void ThrowArgumentOutOfRange(string value) => throw new ArgumentOutOfRangeException(value); - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void ThrowInvalidSeek() => throw new ArgumentException("Invalid seek origin."); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private MemoryChunk AllocateMemoryChunk() - { - // Tweak our buffer sizes to take the minimum of the provided buffer sizes - // or the allocator buffer capacity which provides us with the largest - // available contiguous buffer size. - IMemoryOwner buffer = this.allocator.Allocate(Math.Min(this.allocatorCapacity, GetChunkSize(this.chunkCount++))); - - return new MemoryChunk(buffer) - { - Next = null, - Length = buffer.Length() - }; - } - - private static void ReleaseMemoryChunks(MemoryChunk? chunk) - { - while (chunk != null) - { - chunk.Dispose(); - chunk = chunk.Next; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetChunkSize(int i) - { - // Increment chunks sizes with moderate speed, but without using too many buffers from the same ArrayPool bucket of the default MemoryAllocator. - // https://github.com/SixLabors/ImageSharp/pull/2006#issuecomment-1066244720 -#pragma warning disable IDE1006 // Naming Styles - const int _128K = 1 << 17; - const int _4M = 1 << 22; - return i < 16 ? _128K * (1 << (int)((uint)i / 4)) : _4M; -#pragma warning restore IDE1006 // Naming Styles - } - - private sealed class MemoryChunk : IDisposable - { - private bool isDisposed; - - public MemoryChunk(IMemoryOwner buffer) => this.Buffer = buffer; - - public IMemoryOwner Buffer { get; } - - public MemoryChunk? Next { get; set; } - - public int Length { get; init; } - - private void Dispose(bool disposing) - { - if (!this.isDisposed) - { - if (disposing) - { - this.Buffer.Dispose(); - } - - this.isDisposed = true; - } - } - - public void Dispose() - { - this.Dispose(disposing: true); - GC.SuppressFinalize(this); - } - } -} diff --git a/src/ImageSharp/Image.FromStream.cs b/src/ImageSharp/Image.FromStream.cs index 63f9e64f6c..c73d2880a2 100644 --- a/src/ImageSharp/Image.FromStream.cs +++ b/src/ImageSharp/Image.FromStream.cs @@ -2,7 +2,6 @@ // Licensed under the Six Labors Split License. using SixLabors.ImageSharp.Formats; -using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp; @@ -301,7 +300,7 @@ internal static T WithSeekableStream( return action(stream); } - using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator); + using MemoryStream memoryStream = new(); stream.CopyTo(memoryStream, configuration.StreamProcessingBufferSize); memoryStream.Position = 0; @@ -343,7 +342,7 @@ internal static async Task WithSeekableStreamAsync( return await action(stream, cancellationToken).ConfigureAwait(false); } - using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator); + await using MemoryStream memoryStream = new(); await stream.CopyToAsync(memoryStream, configuration.StreamProcessingBufferSize, cancellationToken).ConfigureAwait(false); memoryStream.Position = 0; diff --git a/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs b/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs index d1d83ffb9a..dd94606084 100644 --- a/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs @@ -529,6 +529,25 @@ public static void RunEncodeLossy_WithPeakImage() [Fact] public void RunEncodeLossy_WithPeakImage_WithoutHardwareIntrinsics_Works() => FeatureTestRunner.RunWithHwIntrinsicsFeature(RunEncodeLossy_WithPeakImage, HwIntrinsics.DisableHWIntrinsic); + [Theory] + [WithFile(TestPatternOpaque, PixelTypes.Rgba32)] + public void CanSave_NonSeekableStream(TestImageProvider provider) + where TPixel : unmanaged, IPixel + { + using Image image = provider.GetImage(); + WebpEncoder encoder = new(); + + using MemoryStream seekable = new(); + image.Save(seekable, encoder); + + using MemoryStream memoryStream = new(); + using NonSeekableStream nonSeekable = new(memoryStream); + + image.Save(nonSeekable, encoder); + + Assert.True(seekable.ToArray().SequenceEqual(memoryStream.ToArray())); + } + private static ImageComparer GetComparer(int quality) { float tolerance = 0.01f; // ~1.0% diff --git a/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs b/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs deleted file mode 100644 index 1803cfddb9..0000000000 --- a/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs +++ /dev/null @@ -1,373 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.IO; -using SixLabors.ImageSharp.Memory; -using SixLabors.ImageSharp.PixelFormats; -using SixLabors.ImageSharp.Tests.TestUtilities.ImageComparison; - -namespace SixLabors.ImageSharp.Tests.IO; - -/// -/// Tests for the class. -/// -public class ChunkedMemoryStreamTests -{ - /// - /// The default length in bytes of each buffer chunk when allocating large buffers. - /// - private const int DefaultLargeChunkSize = 1024 * 1024 * 4; // 4 Mb - - /// - /// The default length in bytes of each buffer chunk when allocating small buffers. - /// - private const int DefaultSmallChunkSize = DefaultLargeChunkSize / 32; // 128 Kb - - private readonly MemoryAllocator allocator; - - public ChunkedMemoryStreamTests() => this.allocator = Configuration.Default.MemoryAllocator; - - [Fact] - public void MemoryStream_GetPositionTest_Negative() - { - using var ms = new ChunkedMemoryStream(this.allocator); - long iCurrentPos = ms.Position; - for (int i = -1; i > -6; i--) - { - Assert.Throws(() => ms.Position = i); - Assert.Equal(ms.Position, iCurrentPos); - } - } - - [Fact] - public void MemoryStream_ReadTest_Negative() - { - var ms2 = new ChunkedMemoryStream(this.allocator); - - Assert.Throws(() => ms2.Read(null, 0, 0)); - Assert.Throws(() => ms2.Read(new byte[] { 1 }, -1, 0)); - Assert.Throws(() => ms2.Read(new byte[] { 1 }, 0, -1)); - Assert.Throws(() => ms2.Read(new byte[] { 1 }, 2, 0)); - Assert.Throws(() => ms2.Read(new byte[] { 1 }, 0, 2)); - - ms2.Dispose(); - - Assert.Throws(() => ms2.Read(new byte[] { 1 }, 0, 1)); - } - - [Theory] - [InlineData(DefaultSmallChunkSize)] - [InlineData((int)(DefaultSmallChunkSize * 1.5))] - [InlineData(DefaultSmallChunkSize * 4)] - [InlineData((int)(DefaultSmallChunkSize * 5.5))] - [InlineData(DefaultSmallChunkSize * 16)] - public void MemoryStream_ReadByteTest(int length) - { - using MemoryStream ms = this.CreateTestStream(length); - using var cms = new ChunkedMemoryStream(this.allocator); - - ms.CopyTo(cms); - cms.Position = 0; - byte[] expected = ms.ToArray(); - - for (int i = 0; i < expected.Length; i++) - { - Assert.Equal(expected[i], cms.ReadByte()); - } - } - - [Theory] - [InlineData(DefaultSmallChunkSize)] - [InlineData((int)(DefaultSmallChunkSize * 1.5))] - [InlineData(DefaultSmallChunkSize * 4)] - [InlineData((int)(DefaultSmallChunkSize * 5.5))] - [InlineData(DefaultSmallChunkSize * 16)] - public void MemoryStream_ReadByteBufferTest(int length) - { - using MemoryStream ms = this.CreateTestStream(length); - using var cms = new ChunkedMemoryStream(this.allocator); - - ms.CopyTo(cms); - cms.Position = 0; - byte[] expected = ms.ToArray(); - byte[] buffer = new byte[2]; - for (int i = 0; i < expected.Length; i += 2) - { - cms.Read(buffer); - Assert.Equal(expected[i], buffer[0]); - Assert.Equal(expected[i + 1], buffer[1]); - } - } - - [Theory] - [InlineData(DefaultSmallChunkSize)] - [InlineData((int)(DefaultSmallChunkSize * 1.5))] - [InlineData(DefaultSmallChunkSize * 4)] - [InlineData((int)(DefaultSmallChunkSize * 5.5))] - [InlineData(DefaultSmallChunkSize * 16)] - public void MemoryStream_ReadByteBufferSpanTest(int length) - { - using MemoryStream ms = this.CreateTestStream(length); - using var cms = new ChunkedMemoryStream(this.allocator); - - ms.CopyTo(cms); - cms.Position = 0; - byte[] expected = ms.ToArray(); - Span buffer = new byte[2]; - for (int i = 0; i < expected.Length; i += 2) - { - cms.Read(buffer); - Assert.Equal(expected[i], buffer[0]); - Assert.Equal(expected[i + 1], buffer[1]); - } - } - - [Fact] - public void MemoryStream_WriteToTests() - { - using (var ms2 = new ChunkedMemoryStream(this.allocator)) - { - byte[] bytArrRet; - byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; - - // [] Write to memoryStream, check the memoryStream - ms2.Write(bytArr, 0, bytArr.Length); - - using var readonlyStream = new ChunkedMemoryStream(this.allocator); - ms2.WriteTo(readonlyStream); - readonlyStream.Flush(); - readonlyStream.Position = 0; - bytArrRet = new byte[(int)readonlyStream.Length]; - readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length); - for (int i = 0; i < bytArr.Length; i++) - { - Assert.Equal(bytArr[i], bytArrRet[i]); - } - } - - // [] Write to memoryStream, check the memoryStream - using (var ms2 = new ChunkedMemoryStream(this.allocator)) - using (var ms3 = new ChunkedMemoryStream(this.allocator)) - { - byte[] bytArrRet; - byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; - - ms2.Write(bytArr, 0, bytArr.Length); - ms2.WriteTo(ms3); - ms3.Position = 0; - bytArrRet = new byte[(int)ms3.Length]; - ms3.Read(bytArrRet, 0, (int)ms3.Length); - for (int i = 0; i < bytArr.Length; i++) - { - Assert.Equal(bytArr[i], bytArrRet[i]); - } - } - } - - [Fact] - public void MemoryStream_WriteToSpanTests() - { - using (var ms2 = new ChunkedMemoryStream(this.allocator)) - { - Span bytArrRet; - Span bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; - - // [] Write to memoryStream, check the memoryStream - ms2.Write(bytArr, 0, bytArr.Length); - - using var readonlyStream = new ChunkedMemoryStream(this.allocator); - ms2.WriteTo(readonlyStream); - readonlyStream.Flush(); - readonlyStream.Position = 0; - bytArrRet = new byte[(int)readonlyStream.Length]; - readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length); - for (int i = 0; i < bytArr.Length; i++) - { - Assert.Equal(bytArr[i], bytArrRet[i]); - } - } - - // [] Write to memoryStream, check the memoryStream - using (var ms2 = new ChunkedMemoryStream(this.allocator)) - using (var ms3 = new ChunkedMemoryStream(this.allocator)) - { - Span bytArrRet; - Span bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; - - ms2.Write(bytArr, 0, bytArr.Length); - ms2.WriteTo(ms3); - ms3.Position = 0; - bytArrRet = new byte[(int)ms3.Length]; - ms3.Read(bytArrRet, 0, (int)ms3.Length); - for (int i = 0; i < bytArr.Length; i++) - { - Assert.Equal(bytArr[i], bytArrRet[i]); - } - } - } - - [Fact] - public void MemoryStream_WriteByteTests() - { - using (var ms2 = new ChunkedMemoryStream(this.allocator)) - { - byte[] bytArrRet; - byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; - - for (int i = 0; i < bytArr.Length; i++) - { - ms2.WriteByte(bytArr[i]); - } - - using var readonlyStream = new ChunkedMemoryStream(this.allocator); - ms2.WriteTo(readonlyStream); - readonlyStream.Flush(); - readonlyStream.Position = 0; - bytArrRet = new byte[(int)readonlyStream.Length]; - readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length); - for (int i = 0; i < bytArr.Length; i++) - { - Assert.Equal(bytArr[i], bytArrRet[i]); - } - } - } - - [Fact] - public void MemoryStream_WriteToTests_Negative() - { - using var ms2 = new ChunkedMemoryStream(this.allocator); - Assert.Throws(() => ms2.WriteTo(null)); - - ms2.Write(new byte[] { 1 }, 0, 1); - var readonlyStream = new MemoryStream(new byte[1028], false); - Assert.Throws(() => ms2.WriteTo(readonlyStream)); - - readonlyStream.Dispose(); - - // [] Pass in a closed stream - Assert.Throws(() => ms2.WriteTo(readonlyStream)); - } - - [Fact] - public void MemoryStream_CopyTo_Invalid() - { - ChunkedMemoryStream memoryStream; - const string bufferSize = nameof(bufferSize); - using (memoryStream = new ChunkedMemoryStream(this.allocator)) - { - const string destination = nameof(destination); - Assert.Throws(destination, () => memoryStream.CopyTo(destination: null)); - - // Validate the destination parameter first. - Assert.Throws(destination, () => memoryStream.CopyTo(destination: null, bufferSize: 0)); - Assert.Throws(destination, () => memoryStream.CopyTo(destination: null, bufferSize: -1)); - - // Then bufferSize. - Assert.Throws(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: 0)); // 0-length buffer doesn't make sense. - Assert.Throws(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: -1)); - } - - // After the Stream is disposed, we should fail on all CopyTos. - Assert.Throws(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: 0)); // Not before bufferSize is validated. - Assert.Throws(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: -1)); - - ChunkedMemoryStream disposedStream = memoryStream; - - // We should throw first for the source being disposed... - Assert.Throws(() => memoryStream.CopyTo(disposedStream, 1)); - - // Then for the destination being disposed. - memoryStream = new ChunkedMemoryStream(this.allocator); - Assert.Throws(() => memoryStream.CopyTo(disposedStream, 1)); - memoryStream.Dispose(); - } - - [Theory] - [MemberData(nameof(CopyToData))] - public void CopyTo(Stream source, byte[] expected) - { - using var destination = new ChunkedMemoryStream(this.allocator); - source.CopyTo(destination); - Assert.InRange(source.Position, source.Length, int.MaxValue); // Copying the data should have read to the end of the stream or stayed past the end. - Assert.Equal(expected, destination.ToArray()); - } - - public static IEnumerable GetAllTestImages() - { - IEnumerable allImageFiles = Directory.EnumerateFiles(TestEnvironment.InputImagesDirectoryFullPath, "*.*", SearchOption.AllDirectories) - .Where(s => !s.EndsWith("txt", StringComparison.OrdinalIgnoreCase)); - - var result = new List(); - foreach (string path in allImageFiles) - { - result.Add(path.Substring(TestEnvironment.InputImagesDirectoryFullPath.Length)); - } - - return result; - } - - public static IEnumerable AllTestImages = GetAllTestImages(); - - [Theory] - [WithFileCollection(nameof(AllTestImages), PixelTypes.Rgba32)] - public void DecoderIntegrationTest(TestImageProvider provider) - where TPixel : unmanaged, IPixel - { - if (!TestEnvironment.Is64BitProcess) - { - return; - } - - Image expected; - try - { - expected = provider.GetImage(); - } - catch - { - // The image is invalid - return; - } - - string fullPath = Path.Combine( - TestEnvironment.InputImagesDirectoryFullPath, - ((TestImageProvider.FileProvider)provider).FilePath); - - using FileStream fs = File.OpenRead(fullPath); - using var nonSeekableStream = new NonSeekableStream(fs); - - var actual = Image.Load(nonSeekableStream); - - ImageComparer.Exact.VerifySimilarity(expected, actual); - } - - public static IEnumerable CopyToData() - { - // Stream is positioned @ beginning of data - byte[] data1 = new byte[] { 1, 2, 3 }; - var stream1 = new MemoryStream(data1); - - yield return new object[] { stream1, data1 }; - - // Stream is positioned in the middle of data - byte[] data2 = new byte[] { 0xff, 0xf3, 0xf0 }; - var stream2 = new MemoryStream(data2) { Position = 1 }; - - yield return new object[] { stream2, new byte[] { 0xf3, 0xf0 } }; - - // Stream is positioned after end of data - byte[] data3 = data2; - var stream3 = new MemoryStream(data3) { Position = data3.Length + 1 }; - - yield return new object[] { stream3, Array.Empty() }; - } - - private MemoryStream CreateTestStream(int length) - { - byte[] buffer = new byte[length]; - var random = new Random(); - random.NextBytes(buffer); - - return new MemoryStream(buffer); - } -} diff --git a/tests/ImageSharp.Tests/Image/NonSeekableStream.cs b/tests/ImageSharp.Tests/Image/NonSeekableStream.cs index 4b1f6e1568..2941490e9a 100644 --- a/tests/ImageSharp.Tests/Image/NonSeekableStream.cs +++ b/tests/ImageSharp.Tests/Image/NonSeekableStream.cs @@ -1,4 +1,4 @@ -// Copyright (c) Six Labors. +// Copyright (c) Six Labors. // Licensed under the Six Labors Split License. namespace SixLabors.ImageSharp.Tests; @@ -14,7 +14,7 @@ public NonSeekableStream(Stream dataStream) public override bool CanSeek => false; - public override bool CanWrite => false; + public override bool CanWrite => this.dataStream.CanWrite; public override bool CanTimeout => this.dataStream.CanTimeout; @@ -91,5 +91,5 @@ public override void SetLength(long value) => throw new NotSupportedException(); public override void Write(byte[] buffer, int offset, int count) - => throw new NotImplementedException(); + => this.dataStream.Write(buffer, offset, count); } From 1a150780fd5dc79194e02568503a12b36cbc42cb Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 22 Oct 2024 19:37:44 +1000 Subject: [PATCH 02/10] Update BufferedStreams.cs --- .../General/IO/BufferedStreams.cs | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/tests/ImageSharp.Benchmarks/General/IO/BufferedStreams.cs b/tests/ImageSharp.Benchmarks/General/IO/BufferedStreams.cs index 2a926d1cd8..a7b22e7ab8 100644 --- a/tests/ImageSharp.Benchmarks/General/IO/BufferedStreams.cs +++ b/tests/ImageSharp.Benchmarks/General/IO/BufferedStreams.cs @@ -19,12 +19,8 @@ public class BufferedStreams private MemoryStream stream4; private MemoryStream stream5; private MemoryStream stream6; - private ChunkedMemoryStream chunkedMemoryStream1; - private ChunkedMemoryStream chunkedMemoryStream2; private BufferedReadStream bufferedStream1; private BufferedReadStream bufferedStream2; - private BufferedReadStream bufferedStream3; - private BufferedReadStream bufferedStream4; private BufferedReadStreamWrapper bufferedStreamWrap1; private BufferedReadStreamWrapper bufferedStreamWrap2; @@ -39,18 +35,8 @@ public void CreateStreams() this.stream6 = new MemoryStream(this.buffer); this.stream6 = new MemoryStream(this.buffer); - this.chunkedMemoryStream1 = new ChunkedMemoryStream(Configuration.Default.MemoryAllocator); - this.chunkedMemoryStream1.Write(this.buffer); - this.chunkedMemoryStream1.Position = 0; - - this.chunkedMemoryStream2 = new ChunkedMemoryStream(Configuration.Default.MemoryAllocator); - this.chunkedMemoryStream2.Write(this.buffer); - this.chunkedMemoryStream2.Position = 0; - this.bufferedStream1 = new BufferedReadStream(Configuration.Default, this.stream3); this.bufferedStream2 = new BufferedReadStream(Configuration.Default, this.stream4); - this.bufferedStream3 = new BufferedReadStream(Configuration.Default, this.chunkedMemoryStream1); - this.bufferedStream4 = new BufferedReadStream(Configuration.Default, this.chunkedMemoryStream2); this.bufferedStreamWrap1 = new BufferedReadStreamWrapper(this.stream5); this.bufferedStreamWrap2 = new BufferedReadStreamWrapper(this.stream6); } @@ -60,12 +46,8 @@ public void DestroyStreams() { this.bufferedStream1?.Dispose(); this.bufferedStream2?.Dispose(); - this.bufferedStream3?.Dispose(); - this.bufferedStream4?.Dispose(); this.bufferedStreamWrap1?.Dispose(); this.bufferedStreamWrap2?.Dispose(); - this.chunkedMemoryStream1?.Dispose(); - this.chunkedMemoryStream2?.Dispose(); this.stream1?.Dispose(); this.stream2?.Dispose(); this.stream3?.Dispose(); @@ -104,21 +86,6 @@ public int BufferedReadStreamRead() return r; } - [Benchmark] - public int BufferedReadStreamChunkedRead() - { - int r = 0; - BufferedReadStream reader = this.bufferedStream3; - byte[] b = this.chunk2; - - for (int i = 0; i < reader.Length / 2; i++) - { - r += reader.Read(b, 0, 2); - } - - return r; - } - [Benchmark] public int BufferedReadStreamWrapRead() { @@ -162,20 +129,6 @@ public int BufferedReadStreamReadByte() return r; } - [Benchmark] - public int BufferedReadStreamChunkedReadByte() - { - int r = 0; - BufferedReadStream reader = this.bufferedStream4; - - for (int i = 0; i < reader.Length; i++) - { - r += reader.ReadByte(); - } - - return r; - } - [Benchmark] public int BufferedReadStreamWrapReadByte() { From c418bb0def340b1a2c25d09e27b0404842f58227 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 23 Oct 2024 09:50:03 +1000 Subject: [PATCH 03/10] Revert "Remove ChunkedMemoryStream" This reverts commit 1e58db2205cf5b05b40ad988e5449c4e0da9d605. --- src/ImageSharp/Formats/Gif/GifDecoderCore.cs | 2 +- .../Sections/GifXmpApplicationExtension.cs | 10 +- src/ImageSharp/Formats/ImageDecoder.cs | 12 +- src/ImageSharp/Formats/ImageEncoder.cs | 7 +- src/ImageSharp/IO/ChunkedMemoryStream.cs | 585 ++++++++++++++++++ src/ImageSharp/Image.FromStream.cs | 5 +- .../Formats/WebP/WebpEncoderTests.cs | 19 - .../IO/ChunkedMemoryStreamTests.cs | 373 +++++++++++ .../Image/NonSeekableStream.cs | 6 +- 9 files changed, 983 insertions(+), 36 deletions(-) create mode 100644 src/ImageSharp/IO/ChunkedMemoryStream.cs create mode 100644 tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs diff --git a/src/ImageSharp/Formats/Gif/GifDecoderCore.cs b/src/ImageSharp/Formats/Gif/GifDecoderCore.cs index c45450a47b..68f4e5fa2d 100644 --- a/src/ImageSharp/Formats/Gif/GifDecoderCore.cs +++ b/src/ImageSharp/Formats/Gif/GifDecoderCore.cs @@ -317,7 +317,7 @@ private void ReadApplicationExtension(BufferedReadStream stream) bool isXmp = this.buffer.Span.StartsWith(GifConstants.XmpApplicationIdentificationBytes); if (isXmp && !this.skipMetadata) { - GifXmpApplicationExtension extension = GifXmpApplicationExtension.Read(stream); + GifXmpApplicationExtension extension = GifXmpApplicationExtension.Read(stream, this.memoryAllocator); if (extension.Data.Length > 0) { this.metadata!.XmpProfile = new XmpProfile(extension.Data); diff --git a/src/ImageSharp/Formats/Gif/Sections/GifXmpApplicationExtension.cs b/src/ImageSharp/Formats/Gif/Sections/GifXmpApplicationExtension.cs index 8bd8497eea..1c1127c3be 100644 --- a/src/ImageSharp/Formats/Gif/Sections/GifXmpApplicationExtension.cs +++ b/src/ImageSharp/Formats/Gif/Sections/GifXmpApplicationExtension.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.Memory; namespace SixLabors.ImageSharp.Formats.Gif; @@ -25,10 +26,11 @@ namespace SixLabors.ImageSharp.Formats.Gif; /// Reads the XMP metadata from the specified stream. /// /// The stream to read from. + /// The memory allocator. /// The XMP metadata - public static GifXmpApplicationExtension Read(Stream stream) + public static GifXmpApplicationExtension Read(Stream stream, MemoryAllocator allocator) { - byte[] xmpBytes = ReadXmpData(stream); + byte[] xmpBytes = ReadXmpData(stream, allocator); // Exclude the "magic trailer", see XMP Specification Part 3, 1.1.2 GIF int xmpLength = xmpBytes.Length - 256; // 257 - unread 0x0 @@ -69,9 +71,9 @@ public int WriteTo(Span buffer) return this.ContentLength; } - private static byte[] ReadXmpData(Stream stream) + private static byte[] ReadXmpData(Stream stream, MemoryAllocator allocator) { - using MemoryStream bytes = new(); + using ChunkedMemoryStream bytes = new(allocator); // XMP data doesn't have a fixed length nor is there an indicator of the length. // So we simply read one byte at a time until we hit the 0x0 value at the end diff --git a/src/ImageSharp/Formats/ImageDecoder.cs b/src/ImageSharp/Formats/ImageDecoder.cs index 03cfa27cfb..549a28d409 100644 --- a/src/ImageSharp/Formats/ImageDecoder.cs +++ b/src/ImageSharp/Formats/ImageDecoder.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; @@ -209,7 +210,7 @@ T PerformActionAndResetPosition(Stream s, long position) } Configuration configuration = options.Configuration; - using MemoryStream memoryStream = new(); + using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator); stream.CopyTo(memoryStream, configuration.StreamProcessingBufferSize); memoryStream.Position = 0; @@ -265,6 +266,11 @@ Task PerformActionAndResetPosition(Stream s, long position, CancellationToken return PerformActionAndResetPosition(ms, ms.Position, cancellationToken); } + if (stream is ChunkedMemoryStream cms) + { + return PerformActionAndResetPosition(cms, cms.Position, cancellationToken); + } + return CopyToMemoryStreamAndActionAsync(options, stream, PerformActionAndResetPosition, cancellationToken); } @@ -276,11 +282,9 @@ private static async Task CopyToMemoryStreamAndActionAsync( { long position = stream.CanSeek ? stream.Position : 0; Configuration configuration = options.Configuration; - - await using MemoryStream memoryStream = new(); + await using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator); await stream.CopyToAsync(memoryStream, configuration.StreamProcessingBufferSize, cancellationToken).ConfigureAwait(false); memoryStream.Position = 0; - return await action(memoryStream, position, cancellationToken).ConfigureAwait(false); } diff --git a/src/ImageSharp/Formats/ImageEncoder.cs b/src/ImageSharp/Formats/ImageEncoder.cs index 34d34c3637..deb527f698 100644 --- a/src/ImageSharp/Formats/ImageEncoder.cs +++ b/src/ImageSharp/Formats/ImageEncoder.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp.Formats; @@ -47,8 +48,8 @@ private void EncodeWithSeekableStream(Image image, Stream stream } else { - using MemoryStream ms = new(); - this.Encode(image, ms, cancellationToken); + using ChunkedMemoryStream ms = new(configuration.MemoryAllocator); + this.Encode(image, stream, cancellationToken); ms.Position = 0; ms.CopyTo(stream, configuration.StreamProcessingBufferSize); } @@ -64,7 +65,7 @@ private async Task EncodeWithSeekableStreamAsync(Image image, St } else { - await using MemoryStream ms = new(); + using ChunkedMemoryStream ms = new(configuration.MemoryAllocator); await DoEncodeAsync(ms); ms.Position = 0; await ms.CopyToAsync(stream, configuration.StreamProcessingBufferSize, cancellationToken) diff --git a/src/ImageSharp/IO/ChunkedMemoryStream.cs b/src/ImageSharp/IO/ChunkedMemoryStream.cs new file mode 100644 index 0000000000..2534548141 --- /dev/null +++ b/src/ImageSharp/IO/ChunkedMemoryStream.cs @@ -0,0 +1,585 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Buffers; +using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.Memory; + +namespace SixLabors.ImageSharp.IO; + +/// +/// Provides an in-memory stream composed of non-contiguous chunks that doesn't need to be resized. +/// Chunks are allocated by the assigned via the constructor +/// and is designed to take advantage of buffer pooling when available. +/// +internal sealed class ChunkedMemoryStream : Stream +{ + // The memory allocator. + private readonly MemoryAllocator allocator; + + // Data + private MemoryChunk? memoryChunk; + + // The total number of allocated chunks + private int chunkCount; + + // The length of the largest contiguous buffer that can be handled by the allocator. + private readonly int allocatorCapacity; + + // Has the stream been disposed. + private bool isDisposed; + + // Current chunk to write to + private MemoryChunk? writeChunk; + + // Offset into chunk to write to + private int writeOffset; + + // Current chunk to read from + private MemoryChunk? readChunk; + + // Offset into chunk to read from + private int readOffset; + + /// + /// Initializes a new instance of the class. + /// + /// The memory allocator. + public ChunkedMemoryStream(MemoryAllocator allocator) + { + this.allocatorCapacity = allocator.GetBufferCapacityInBytes(); + this.allocator = allocator; + } + + /// + public override bool CanRead => !this.isDisposed; + + /// + public override bool CanSeek => !this.isDisposed; + + /// + public override bool CanWrite => !this.isDisposed; + + /// + public override long Length + { + get + { + this.EnsureNotDisposed(); + + int length = 0; + MemoryChunk? chunk = this.memoryChunk; + while (chunk != null) + { + MemoryChunk? next = chunk.Next; + if (next != null) + { + length += chunk.Length; + } + else + { + length += this.writeOffset; + } + + chunk = next; + } + + return length; + } + } + + /// + public override long Position + { + get + { + this.EnsureNotDisposed(); + + if (this.readChunk is null) + { + return 0; + } + + int pos = 0; + MemoryChunk? chunk = this.memoryChunk; + while (chunk != this.readChunk && chunk is not null) + { + pos += chunk.Length; + chunk = chunk.Next; + } + + pos += this.readOffset; + + return pos; + } + + set + { + this.EnsureNotDisposed(); + + if (value < 0) + { + ThrowArgumentOutOfRange(nameof(value)); + } + + // Back up current position in case new position is out of range + MemoryChunk? backupReadChunk = this.readChunk; + int backupReadOffset = this.readOffset; + + this.readChunk = null; + this.readOffset = 0; + + int leftUntilAtPos = (int)value; + MemoryChunk? chunk = this.memoryChunk; + while (chunk != null) + { + if ((leftUntilAtPos < chunk.Length) + || ((leftUntilAtPos == chunk.Length) + && (chunk.Next is null))) + { + // The desired position is in this chunk + this.readChunk = chunk; + this.readOffset = leftUntilAtPos; + break; + } + + leftUntilAtPos -= chunk.Length; + chunk = chunk.Next; + } + + if (this.readChunk is null) + { + // Position is out of range + this.readChunk = backupReadChunk; + this.readOffset = backupReadOffset; + } + } + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override long Seek(long offset, SeekOrigin origin) + { + this.EnsureNotDisposed(); + + switch (origin) + { + case SeekOrigin.Begin: + this.Position = offset; + break; + + case SeekOrigin.Current: + this.Position += offset; + break; + + case SeekOrigin.End: + this.Position = this.Length + offset; + break; + default: + ThrowInvalidSeek(); + break; + } + + return this.Position; + } + + /// + public override void SetLength(long value) + => throw new NotSupportedException(); + + /// + protected override void Dispose(bool disposing) + { + if (this.isDisposed) + { + return; + } + + try + { + this.isDisposed = true; + if (disposing) + { + ReleaseMemoryChunks(this.memoryChunk); + } + + this.memoryChunk = null; + this.writeChunk = null; + this.readChunk = null; + this.chunkCount = 0; + } + finally + { + base.Dispose(disposing); + } + } + + /// + public override void Flush() + { + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override int Read(byte[] buffer, int offset, int count) + { + Guard.NotNull(buffer, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(offset, 0, nameof(offset)); + Guard.MustBeGreaterThanOrEqualTo(count, 0, nameof(count)); + + const string bufferMessage = "Offset subtracted from the buffer length is less than count."; + Guard.IsFalse(buffer.Length - offset < count, nameof(buffer), bufferMessage); + + return this.ReadImpl(buffer.AsSpan(offset, count)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override int Read(Span buffer) => this.ReadImpl(buffer); + + private int ReadImpl(Span buffer) + { + this.EnsureNotDisposed(); + + if (this.readChunk is null) + { + if (this.memoryChunk is null) + { + return 0; + } + + this.readChunk = this.memoryChunk; + this.readOffset = 0; + } + + IMemoryOwner chunkBuffer = this.readChunk.Buffer; + int chunkSize = this.readChunk.Length; + if (this.readChunk.Next is null) + { + chunkSize = this.writeOffset; + } + + int bytesRead = 0; + int offset = 0; + int count = buffer.Length; + while (count > 0) + { + if (this.readOffset == chunkSize) + { + // Exit if no more chunks are currently available + if (this.readChunk.Next is null) + { + break; + } + + this.readChunk = this.readChunk.Next; + this.readOffset = 0; + chunkBuffer = this.readChunk.Buffer; + chunkSize = this.readChunk.Length; + if (this.readChunk.Next is null) + { + chunkSize = this.writeOffset; + } + } + + int readCount = Math.Min(count, chunkSize - this.readOffset); + chunkBuffer.Slice(this.readOffset, readCount).CopyTo(buffer[offset..]); + offset += readCount; + count -= readCount; + this.readOffset += readCount; + bytesRead += readCount; + } + + return bytesRead; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override int ReadByte() + { + this.EnsureNotDisposed(); + + if (this.readChunk is null) + { + if (this.memoryChunk is null) + { + return 0; + } + + this.readChunk = this.memoryChunk; + this.readOffset = 0; + } + + IMemoryOwner chunkBuffer = this.readChunk.Buffer; + int chunkSize = this.readChunk.Length; + if (this.readChunk.Next is null) + { + chunkSize = this.writeOffset; + } + + if (this.readOffset == chunkSize) + { + // Exit if no more chunks are currently available + if (this.readChunk.Next is null) + { + return -1; + } + + this.readChunk = this.readChunk.Next; + this.readOffset = 0; + chunkBuffer = this.readChunk.Buffer; + } + + return chunkBuffer.GetSpan()[this.readOffset++]; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Write(byte[] buffer, int offset, int count) + { + Guard.NotNull(buffer, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(offset, 0, nameof(offset)); + Guard.MustBeGreaterThanOrEqualTo(count, 0, nameof(count)); + + const string bufferMessage = "Offset subtracted from the buffer length is less than count."; + Guard.IsFalse(buffer.Length - offset < count, nameof(buffer), bufferMessage); + + this.WriteImpl(buffer.AsSpan(offset, count)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Write(ReadOnlySpan buffer) => this.WriteImpl(buffer); + + private void WriteImpl(ReadOnlySpan buffer) + { + this.EnsureNotDisposed(); + + if (this.memoryChunk is null) + { + this.memoryChunk = this.AllocateMemoryChunk(); + this.writeChunk = this.memoryChunk; + this.writeOffset = 0; + } + + Guard.NotNull(this.writeChunk); + + Span chunkBuffer = this.writeChunk.Buffer.GetSpan(); + int chunkSize = this.writeChunk.Length; + int count = buffer.Length; + int offset = 0; + while (count > 0) + { + if (this.writeOffset == chunkSize) + { + // Allocate a new chunk if the current one is full + this.writeChunk.Next = this.AllocateMemoryChunk(); + this.writeChunk = this.writeChunk.Next; + this.writeOffset = 0; + chunkBuffer = this.writeChunk.Buffer.GetSpan(); + chunkSize = this.writeChunk.Length; + } + + int copyCount = Math.Min(count, chunkSize - this.writeOffset); + buffer.Slice(offset, copyCount).CopyTo(chunkBuffer[this.writeOffset..]); + + offset += copyCount; + count -= copyCount; + this.writeOffset += copyCount; + } + } + + /// + public override void WriteByte(byte value) + { + this.EnsureNotDisposed(); + + if (this.memoryChunk is null) + { + this.memoryChunk = this.AllocateMemoryChunk(); + this.writeChunk = this.memoryChunk; + this.writeOffset = 0; + } + + Guard.NotNull(this.writeChunk); + + IMemoryOwner chunkBuffer = this.writeChunk.Buffer; + int chunkSize = this.writeChunk.Length; + + if (this.writeOffset == chunkSize) + { + // Allocate a new chunk if the current one is full + this.writeChunk.Next = this.AllocateMemoryChunk(); + this.writeChunk = this.writeChunk.Next; + this.writeOffset = 0; + chunkBuffer = this.writeChunk.Buffer; + } + + chunkBuffer.GetSpan()[this.writeOffset++] = value; + } + + /// + /// Copy entire buffer into an array. + /// + /// The . + public byte[] ToArray() + { + int length = (int)this.Length; // This will throw if stream is closed + byte[] copy = new byte[this.Length]; + + MemoryChunk? backupReadChunk = this.readChunk; + int backupReadOffset = this.readOffset; + + this.readChunk = this.memoryChunk; + this.readOffset = 0; + this.Read(copy, 0, length); + + this.readChunk = backupReadChunk; + this.readOffset = backupReadOffset; + + return copy; + } + + /// + /// Write remainder of this stream to another stream. + /// + /// The stream to write to. + public void WriteTo(Stream stream) + { + this.EnsureNotDisposed(); + + Guard.NotNull(stream, nameof(stream)); + + if (this.readChunk is null) + { + if (this.memoryChunk is null) + { + return; + } + + this.readChunk = this.memoryChunk; + this.readOffset = 0; + } + + IMemoryOwner chunkBuffer = this.readChunk.Buffer; + int chunkSize = this.readChunk.Length; + if (this.readChunk.Next is null) + { + chunkSize = this.writeOffset; + } + + // Following code mirrors Read() logic (readChunk/readOffset should + // point just past last byte of last chunk when done) + // loop until end of chunks is found + while (true) + { + if (this.readOffset == chunkSize) + { + // Exit if no more chunks are currently available + if (this.readChunk.Next is null) + { + break; + } + + this.readChunk = this.readChunk.Next; + this.readOffset = 0; + chunkBuffer = this.readChunk.Buffer; + chunkSize = this.readChunk.Length; + if (this.readChunk.Next is null) + { + chunkSize = this.writeOffset; + } + } + + int writeCount = chunkSize - this.readOffset; + stream.Write(chunkBuffer.GetSpan(), this.readOffset, writeCount); + this.readOffset = chunkSize; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EnsureNotDisposed() + { + if (this.isDisposed) + { + ThrowDisposed(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ThrowDisposed() => throw new ObjectDisposedException(null, "The stream is closed."); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ThrowArgumentOutOfRange(string value) => throw new ArgumentOutOfRangeException(value); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ThrowInvalidSeek() => throw new ArgumentException("Invalid seek origin."); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private MemoryChunk AllocateMemoryChunk() + { + // Tweak our buffer sizes to take the minimum of the provided buffer sizes + // or the allocator buffer capacity which provides us with the largest + // available contiguous buffer size. + IMemoryOwner buffer = this.allocator.Allocate(Math.Min(this.allocatorCapacity, GetChunkSize(this.chunkCount++))); + + return new MemoryChunk(buffer) + { + Next = null, + Length = buffer.Length() + }; + } + + private static void ReleaseMemoryChunks(MemoryChunk? chunk) + { + while (chunk != null) + { + chunk.Dispose(); + chunk = chunk.Next; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetChunkSize(int i) + { + // Increment chunks sizes with moderate speed, but without using too many buffers from the same ArrayPool bucket of the default MemoryAllocator. + // https://github.com/SixLabors/ImageSharp/pull/2006#issuecomment-1066244720 +#pragma warning disable IDE1006 // Naming Styles + const int _128K = 1 << 17; + const int _4M = 1 << 22; + return i < 16 ? _128K * (1 << (int)((uint)i / 4)) : _4M; +#pragma warning restore IDE1006 // Naming Styles + } + + private sealed class MemoryChunk : IDisposable + { + private bool isDisposed; + + public MemoryChunk(IMemoryOwner buffer) => this.Buffer = buffer; + + public IMemoryOwner Buffer { get; } + + public MemoryChunk? Next { get; set; } + + public int Length { get; init; } + + private void Dispose(bool disposing) + { + if (!this.isDisposed) + { + if (disposing) + { + this.Buffer.Dispose(); + } + + this.isDisposed = true; + } + } + + public void Dispose() + { + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/ImageSharp/Image.FromStream.cs b/src/ImageSharp/Image.FromStream.cs index c73d2880a2..63f9e64f6c 100644 --- a/src/ImageSharp/Image.FromStream.cs +++ b/src/ImageSharp/Image.FromStream.cs @@ -2,6 +2,7 @@ // Licensed under the Six Labors Split License. using SixLabors.ImageSharp.Formats; +using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp; @@ -300,7 +301,7 @@ internal static T WithSeekableStream( return action(stream); } - using MemoryStream memoryStream = new(); + using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator); stream.CopyTo(memoryStream, configuration.StreamProcessingBufferSize); memoryStream.Position = 0; @@ -342,7 +343,7 @@ internal static async Task WithSeekableStreamAsync( return await action(stream, cancellationToken).ConfigureAwait(false); } - await using MemoryStream memoryStream = new(); + using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator); await stream.CopyToAsync(memoryStream, configuration.StreamProcessingBufferSize, cancellationToken).ConfigureAwait(false); memoryStream.Position = 0; diff --git a/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs b/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs index 96cdfe8539..031a9ba059 100644 --- a/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs @@ -544,25 +544,6 @@ public static void RunEncodeLossy_WithPeakImage() [Fact] public void RunEncodeLossy_WithPeakImage_WithoutHardwareIntrinsics_Works() => FeatureTestRunner.RunWithHwIntrinsicsFeature(RunEncodeLossy_WithPeakImage, HwIntrinsics.DisableHWIntrinsic); - [Theory] - [WithFile(TestPatternOpaque, PixelTypes.Rgba32)] - public void CanSave_NonSeekableStream(TestImageProvider provider) - where TPixel : unmanaged, IPixel - { - using Image image = provider.GetImage(); - WebpEncoder encoder = new(); - - using MemoryStream seekable = new(); - image.Save(seekable, encoder); - - using MemoryStream memoryStream = new(); - using NonSeekableStream nonSeekable = new(memoryStream); - - image.Save(nonSeekable, encoder); - - Assert.True(seekable.ToArray().SequenceEqual(memoryStream.ToArray())); - } - private static ImageComparer GetComparer(int quality) { float tolerance = 0.01f; // ~1.0% diff --git a/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs b/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs new file mode 100644 index 0000000000..1803cfddb9 --- /dev/null +++ b/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs @@ -0,0 +1,373 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.IO; +using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Tests.TestUtilities.ImageComparison; + +namespace SixLabors.ImageSharp.Tests.IO; + +/// +/// Tests for the class. +/// +public class ChunkedMemoryStreamTests +{ + /// + /// The default length in bytes of each buffer chunk when allocating large buffers. + /// + private const int DefaultLargeChunkSize = 1024 * 1024 * 4; // 4 Mb + + /// + /// The default length in bytes of each buffer chunk when allocating small buffers. + /// + private const int DefaultSmallChunkSize = DefaultLargeChunkSize / 32; // 128 Kb + + private readonly MemoryAllocator allocator; + + public ChunkedMemoryStreamTests() => this.allocator = Configuration.Default.MemoryAllocator; + + [Fact] + public void MemoryStream_GetPositionTest_Negative() + { + using var ms = new ChunkedMemoryStream(this.allocator); + long iCurrentPos = ms.Position; + for (int i = -1; i > -6; i--) + { + Assert.Throws(() => ms.Position = i); + Assert.Equal(ms.Position, iCurrentPos); + } + } + + [Fact] + public void MemoryStream_ReadTest_Negative() + { + var ms2 = new ChunkedMemoryStream(this.allocator); + + Assert.Throws(() => ms2.Read(null, 0, 0)); + Assert.Throws(() => ms2.Read(new byte[] { 1 }, -1, 0)); + Assert.Throws(() => ms2.Read(new byte[] { 1 }, 0, -1)); + Assert.Throws(() => ms2.Read(new byte[] { 1 }, 2, 0)); + Assert.Throws(() => ms2.Read(new byte[] { 1 }, 0, 2)); + + ms2.Dispose(); + + Assert.Throws(() => ms2.Read(new byte[] { 1 }, 0, 1)); + } + + [Theory] + [InlineData(DefaultSmallChunkSize)] + [InlineData((int)(DefaultSmallChunkSize * 1.5))] + [InlineData(DefaultSmallChunkSize * 4)] + [InlineData((int)(DefaultSmallChunkSize * 5.5))] + [InlineData(DefaultSmallChunkSize * 16)] + public void MemoryStream_ReadByteTest(int length) + { + using MemoryStream ms = this.CreateTestStream(length); + using var cms = new ChunkedMemoryStream(this.allocator); + + ms.CopyTo(cms); + cms.Position = 0; + byte[] expected = ms.ToArray(); + + for (int i = 0; i < expected.Length; i++) + { + Assert.Equal(expected[i], cms.ReadByte()); + } + } + + [Theory] + [InlineData(DefaultSmallChunkSize)] + [InlineData((int)(DefaultSmallChunkSize * 1.5))] + [InlineData(DefaultSmallChunkSize * 4)] + [InlineData((int)(DefaultSmallChunkSize * 5.5))] + [InlineData(DefaultSmallChunkSize * 16)] + public void MemoryStream_ReadByteBufferTest(int length) + { + using MemoryStream ms = this.CreateTestStream(length); + using var cms = new ChunkedMemoryStream(this.allocator); + + ms.CopyTo(cms); + cms.Position = 0; + byte[] expected = ms.ToArray(); + byte[] buffer = new byte[2]; + for (int i = 0; i < expected.Length; i += 2) + { + cms.Read(buffer); + Assert.Equal(expected[i], buffer[0]); + Assert.Equal(expected[i + 1], buffer[1]); + } + } + + [Theory] + [InlineData(DefaultSmallChunkSize)] + [InlineData((int)(DefaultSmallChunkSize * 1.5))] + [InlineData(DefaultSmallChunkSize * 4)] + [InlineData((int)(DefaultSmallChunkSize * 5.5))] + [InlineData(DefaultSmallChunkSize * 16)] + public void MemoryStream_ReadByteBufferSpanTest(int length) + { + using MemoryStream ms = this.CreateTestStream(length); + using var cms = new ChunkedMemoryStream(this.allocator); + + ms.CopyTo(cms); + cms.Position = 0; + byte[] expected = ms.ToArray(); + Span buffer = new byte[2]; + for (int i = 0; i < expected.Length; i += 2) + { + cms.Read(buffer); + Assert.Equal(expected[i], buffer[0]); + Assert.Equal(expected[i + 1], buffer[1]); + } + } + + [Fact] + public void MemoryStream_WriteToTests() + { + using (var ms2 = new ChunkedMemoryStream(this.allocator)) + { + byte[] bytArrRet; + byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; + + // [] Write to memoryStream, check the memoryStream + ms2.Write(bytArr, 0, bytArr.Length); + + using var readonlyStream = new ChunkedMemoryStream(this.allocator); + ms2.WriteTo(readonlyStream); + readonlyStream.Flush(); + readonlyStream.Position = 0; + bytArrRet = new byte[(int)readonlyStream.Length]; + readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length); + for (int i = 0; i < bytArr.Length; i++) + { + Assert.Equal(bytArr[i], bytArrRet[i]); + } + } + + // [] Write to memoryStream, check the memoryStream + using (var ms2 = new ChunkedMemoryStream(this.allocator)) + using (var ms3 = new ChunkedMemoryStream(this.allocator)) + { + byte[] bytArrRet; + byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; + + ms2.Write(bytArr, 0, bytArr.Length); + ms2.WriteTo(ms3); + ms3.Position = 0; + bytArrRet = new byte[(int)ms3.Length]; + ms3.Read(bytArrRet, 0, (int)ms3.Length); + for (int i = 0; i < bytArr.Length; i++) + { + Assert.Equal(bytArr[i], bytArrRet[i]); + } + } + } + + [Fact] + public void MemoryStream_WriteToSpanTests() + { + using (var ms2 = new ChunkedMemoryStream(this.allocator)) + { + Span bytArrRet; + Span bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; + + // [] Write to memoryStream, check the memoryStream + ms2.Write(bytArr, 0, bytArr.Length); + + using var readonlyStream = new ChunkedMemoryStream(this.allocator); + ms2.WriteTo(readonlyStream); + readonlyStream.Flush(); + readonlyStream.Position = 0; + bytArrRet = new byte[(int)readonlyStream.Length]; + readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length); + for (int i = 0; i < bytArr.Length; i++) + { + Assert.Equal(bytArr[i], bytArrRet[i]); + } + } + + // [] Write to memoryStream, check the memoryStream + using (var ms2 = new ChunkedMemoryStream(this.allocator)) + using (var ms3 = new ChunkedMemoryStream(this.allocator)) + { + Span bytArrRet; + Span bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; + + ms2.Write(bytArr, 0, bytArr.Length); + ms2.WriteTo(ms3); + ms3.Position = 0; + bytArrRet = new byte[(int)ms3.Length]; + ms3.Read(bytArrRet, 0, (int)ms3.Length); + for (int i = 0; i < bytArr.Length; i++) + { + Assert.Equal(bytArr[i], bytArrRet[i]); + } + } + } + + [Fact] + public void MemoryStream_WriteByteTests() + { + using (var ms2 = new ChunkedMemoryStream(this.allocator)) + { + byte[] bytArrRet; + byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; + + for (int i = 0; i < bytArr.Length; i++) + { + ms2.WriteByte(bytArr[i]); + } + + using var readonlyStream = new ChunkedMemoryStream(this.allocator); + ms2.WriteTo(readonlyStream); + readonlyStream.Flush(); + readonlyStream.Position = 0; + bytArrRet = new byte[(int)readonlyStream.Length]; + readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length); + for (int i = 0; i < bytArr.Length; i++) + { + Assert.Equal(bytArr[i], bytArrRet[i]); + } + } + } + + [Fact] + public void MemoryStream_WriteToTests_Negative() + { + using var ms2 = new ChunkedMemoryStream(this.allocator); + Assert.Throws(() => ms2.WriteTo(null)); + + ms2.Write(new byte[] { 1 }, 0, 1); + var readonlyStream = new MemoryStream(new byte[1028], false); + Assert.Throws(() => ms2.WriteTo(readonlyStream)); + + readonlyStream.Dispose(); + + // [] Pass in a closed stream + Assert.Throws(() => ms2.WriteTo(readonlyStream)); + } + + [Fact] + public void MemoryStream_CopyTo_Invalid() + { + ChunkedMemoryStream memoryStream; + const string bufferSize = nameof(bufferSize); + using (memoryStream = new ChunkedMemoryStream(this.allocator)) + { + const string destination = nameof(destination); + Assert.Throws(destination, () => memoryStream.CopyTo(destination: null)); + + // Validate the destination parameter first. + Assert.Throws(destination, () => memoryStream.CopyTo(destination: null, bufferSize: 0)); + Assert.Throws(destination, () => memoryStream.CopyTo(destination: null, bufferSize: -1)); + + // Then bufferSize. + Assert.Throws(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: 0)); // 0-length buffer doesn't make sense. + Assert.Throws(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: -1)); + } + + // After the Stream is disposed, we should fail on all CopyTos. + Assert.Throws(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: 0)); // Not before bufferSize is validated. + Assert.Throws(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: -1)); + + ChunkedMemoryStream disposedStream = memoryStream; + + // We should throw first for the source being disposed... + Assert.Throws(() => memoryStream.CopyTo(disposedStream, 1)); + + // Then for the destination being disposed. + memoryStream = new ChunkedMemoryStream(this.allocator); + Assert.Throws(() => memoryStream.CopyTo(disposedStream, 1)); + memoryStream.Dispose(); + } + + [Theory] + [MemberData(nameof(CopyToData))] + public void CopyTo(Stream source, byte[] expected) + { + using var destination = new ChunkedMemoryStream(this.allocator); + source.CopyTo(destination); + Assert.InRange(source.Position, source.Length, int.MaxValue); // Copying the data should have read to the end of the stream or stayed past the end. + Assert.Equal(expected, destination.ToArray()); + } + + public static IEnumerable GetAllTestImages() + { + IEnumerable allImageFiles = Directory.EnumerateFiles(TestEnvironment.InputImagesDirectoryFullPath, "*.*", SearchOption.AllDirectories) + .Where(s => !s.EndsWith("txt", StringComparison.OrdinalIgnoreCase)); + + var result = new List(); + foreach (string path in allImageFiles) + { + result.Add(path.Substring(TestEnvironment.InputImagesDirectoryFullPath.Length)); + } + + return result; + } + + public static IEnumerable AllTestImages = GetAllTestImages(); + + [Theory] + [WithFileCollection(nameof(AllTestImages), PixelTypes.Rgba32)] + public void DecoderIntegrationTest(TestImageProvider provider) + where TPixel : unmanaged, IPixel + { + if (!TestEnvironment.Is64BitProcess) + { + return; + } + + Image expected; + try + { + expected = provider.GetImage(); + } + catch + { + // The image is invalid + return; + } + + string fullPath = Path.Combine( + TestEnvironment.InputImagesDirectoryFullPath, + ((TestImageProvider.FileProvider)provider).FilePath); + + using FileStream fs = File.OpenRead(fullPath); + using var nonSeekableStream = new NonSeekableStream(fs); + + var actual = Image.Load(nonSeekableStream); + + ImageComparer.Exact.VerifySimilarity(expected, actual); + } + + public static IEnumerable CopyToData() + { + // Stream is positioned @ beginning of data + byte[] data1 = new byte[] { 1, 2, 3 }; + var stream1 = new MemoryStream(data1); + + yield return new object[] { stream1, data1 }; + + // Stream is positioned in the middle of data + byte[] data2 = new byte[] { 0xff, 0xf3, 0xf0 }; + var stream2 = new MemoryStream(data2) { Position = 1 }; + + yield return new object[] { stream2, new byte[] { 0xf3, 0xf0 } }; + + // Stream is positioned after end of data + byte[] data3 = data2; + var stream3 = new MemoryStream(data3) { Position = data3.Length + 1 }; + + yield return new object[] { stream3, Array.Empty() }; + } + + private MemoryStream CreateTestStream(int length) + { + byte[] buffer = new byte[length]; + var random = new Random(); + random.NextBytes(buffer); + + return new MemoryStream(buffer); + } +} diff --git a/tests/ImageSharp.Tests/Image/NonSeekableStream.cs b/tests/ImageSharp.Tests/Image/NonSeekableStream.cs index 2941490e9a..4b1f6e1568 100644 --- a/tests/ImageSharp.Tests/Image/NonSeekableStream.cs +++ b/tests/ImageSharp.Tests/Image/NonSeekableStream.cs @@ -1,4 +1,4 @@ -// Copyright (c) Six Labors. +// Copyright (c) Six Labors. // Licensed under the Six Labors Split License. namespace SixLabors.ImageSharp.Tests; @@ -14,7 +14,7 @@ public NonSeekableStream(Stream dataStream) public override bool CanSeek => false; - public override bool CanWrite => this.dataStream.CanWrite; + public override bool CanWrite => false; public override bool CanTimeout => this.dataStream.CanTimeout; @@ -91,5 +91,5 @@ public override void SetLength(long value) => throw new NotSupportedException(); public override void Write(byte[] buffer, int offset, int count) - => this.dataStream.Write(buffer, offset, count); + => throw new NotImplementedException(); } From 48645f8b4772a4d3dc07bcc5d96105fe979f94e7 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 23 Oct 2024 12:18:16 +1000 Subject: [PATCH 04/10] Rewrite ChunkedMemoryStream --- src/ImageSharp/Formats/ImageEncoder.cs | 4 +- src/ImageSharp/IO/ChunkedMemoryStream.cs | 628 ++++++++---------- .../Formats/WebP/WebpEncoderTests.cs | 38 ++ .../IO/ChunkedMemoryStreamTests.cs | 93 +-- .../Image/NonSeekableStream.cs | 6 +- 5 files changed, 361 insertions(+), 408 deletions(-) diff --git a/src/ImageSharp/Formats/ImageEncoder.cs b/src/ImageSharp/Formats/ImageEncoder.cs index deb527f698..27a4f11cdd 100644 --- a/src/ImageSharp/Formats/ImageEncoder.cs +++ b/src/ImageSharp/Formats/ImageEncoder.cs @@ -49,7 +49,7 @@ private void EncodeWithSeekableStream(Image image, Stream stream else { using ChunkedMemoryStream ms = new(configuration.MemoryAllocator); - this.Encode(image, stream, cancellationToken); + this.Encode(image, ms, cancellationToken); ms.Position = 0; ms.CopyTo(stream, configuration.StreamProcessingBufferSize); } @@ -65,7 +65,7 @@ private async Task EncodeWithSeekableStreamAsync(Image image, St } else { - using ChunkedMemoryStream ms = new(configuration.MemoryAllocator); + await using ChunkedMemoryStream ms = new(configuration.MemoryAllocator); await DoEncodeAsync(ms); ms.Position = 0; await ms.CopyToAsync(stream, configuration.StreamProcessingBufferSize, cancellationToken) diff --git a/src/ImageSharp/IO/ChunkedMemoryStream.cs b/src/ImageSharp/IO/ChunkedMemoryStream.cs index 2534548141..f178764603 100644 --- a/src/ImageSharp/IO/ChunkedMemoryStream.cs +++ b/src/ImageSharp/IO/ChunkedMemoryStream.cs @@ -2,6 +2,7 @@ // Licensed under the Six Labors Split License. using System.Buffers; +using System.Collections; using System.Runtime.CompilerServices; using SixLabors.ImageSharp.Memory; @@ -12,44 +13,24 @@ namespace SixLabors.ImageSharp.IO; /// Chunks are allocated by the assigned via the constructor /// and is designed to take advantage of buffer pooling when available. /// -internal sealed class ChunkedMemoryStream : Stream +/// Provides an in-memory stream composed of non-contiguous chunks. +public class ChunkedMemoryStream : Stream { - // The memory allocator. - private readonly MemoryAllocator allocator; + private readonly MemoryChunkBuffer memoryChunkBuffer; + private readonly byte[] singleReadBuffer = new byte[1]; - // Data - private MemoryChunk? memoryChunk; - - // The total number of allocated chunks - private int chunkCount; - - // The length of the largest contiguous buffer that can be handled by the allocator. - private readonly int allocatorCapacity; - - // Has the stream been disposed. + private long length; + private long position; + private int currentChunk; + private int currentChunkIndex; private bool isDisposed; - // Current chunk to write to - private MemoryChunk? writeChunk; - - // Offset into chunk to write to - private int writeOffset; - - // Current chunk to read from - private MemoryChunk? readChunk; - - // Offset into chunk to read from - private int readOffset; - /// /// Initializes a new instance of the class. /// /// The memory allocator. public ChunkedMemoryStream(MemoryAllocator allocator) - { - this.allocatorCapacity = allocator.GetBufferCapacityInBytes(); - this.allocator = allocator; - } + => this.memoryChunkBuffer = new(allocator); /// public override bool CanRead => !this.isDisposed; @@ -66,25 +47,7 @@ public override long Length get { this.EnsureNotDisposed(); - - int length = 0; - MemoryChunk? chunk = this.memoryChunk; - while (chunk != null) - { - MemoryChunk? next = chunk.Next; - if (next != null) - { - length += chunk.Length; - } - else - { - length += this.writeOffset; - } - - chunk = next; - } - - return length; + return this.length; } } @@ -94,93 +57,35 @@ public override long Position get { this.EnsureNotDisposed(); - - if (this.readChunk is null) - { - return 0; - } - - int pos = 0; - MemoryChunk? chunk = this.memoryChunk; - while (chunk != this.readChunk && chunk is not null) - { - pos += chunk.Length; - chunk = chunk.Next; - } - - pos += this.readOffset; - - return pos; + return this.position; } set { this.EnsureNotDisposed(); - - if (value < 0) - { - ThrowArgumentOutOfRange(nameof(value)); - } - - // Back up current position in case new position is out of range - MemoryChunk? backupReadChunk = this.readChunk; - int backupReadOffset = this.readOffset; - - this.readChunk = null; - this.readOffset = 0; - - int leftUntilAtPos = (int)value; - MemoryChunk? chunk = this.memoryChunk; - while (chunk != null) - { - if ((leftUntilAtPos < chunk.Length) - || ((leftUntilAtPos == chunk.Length) - && (chunk.Next is null))) - { - // The desired position is in this chunk - this.readChunk = chunk; - this.readOffset = leftUntilAtPos; - break; - } - - leftUntilAtPos -= chunk.Length; - chunk = chunk.Next; - } - - if (this.readChunk is null) - { - // Position is out of range - this.readChunk = backupReadChunk; - this.readOffset = backupReadOffset; - } + this.SetPosition(value); } } /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Flush() + { + } + + /// public override long Seek(long offset, SeekOrigin origin) { this.EnsureNotDisposed(); - switch (origin) + this.Position = origin switch { - case SeekOrigin.Begin: - this.Position = offset; - break; - - case SeekOrigin.Current: - this.Position += offset; - break; - - case SeekOrigin.End: - this.Position = this.Length + offset; - break; - default: - ThrowInvalidSeek(); - break; - } + SeekOrigin.Begin => (int)offset, + SeekOrigin.Current => (int)(this.Position + offset), + SeekOrigin.End => (int)(this.Length + offset), + _ => throw new ArgumentOutOfRangeException(nameof(offset)), + }; - return this.Position; + return this.position; } /// @@ -188,41 +93,23 @@ public override void SetLength(long value) => throw new NotSupportedException(); /// - protected override void Dispose(bool disposing) + public override int ReadByte() { - if (this.isDisposed) - { - return; - } - - try - { - this.isDisposed = true; - if (disposing) - { - ReleaseMemoryChunks(this.memoryChunk); - } - - this.memoryChunk = null; - this.writeChunk = null; - this.readChunk = null; - this.chunkCount = 0; - } - finally + this.EnsureNotDisposed(); + if (this.position >= this.length) { - base.Dispose(disposing); + return -1; } - } - /// - public override void Flush() - { + _ = this.Read(this.singleReadBuffer, 0, 1); + return this.singleReadBuffer[^1]; } /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] public override int Read(byte[] buffer, int offset, int count) { + this.EnsureNotDisposed(); + Guard.NotNull(buffer, nameof(buffer)); Guard.MustBeGreaterThanOrEqualTo(offset, 0, nameof(offset)); Guard.MustBeGreaterThanOrEqualTo(count, 0, nameof(count)); @@ -230,111 +117,63 @@ public override int Read(byte[] buffer, int offset, int count) const string bufferMessage = "Offset subtracted from the buffer length is less than count."; Guard.IsFalse(buffer.Length - offset < count, nameof(buffer), bufferMessage); - return this.ReadImpl(buffer.AsSpan(offset, count)); + return this.Read(buffer.AsSpan(offset, count)); } /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override int Read(Span buffer) => this.ReadImpl(buffer); - - private int ReadImpl(Span buffer) + public override int Read(Span buffer) { this.EnsureNotDisposed(); - if (this.readChunk is null) + int offset = 0; + int count = buffer.Length; + int bytesRead = 0; + long bytesToRead = this.length - this.position; + if (bytesToRead > count) { - if (this.memoryChunk is null) - { - return 0; - } - - this.readChunk = this.memoryChunk; - this.readOffset = 0; + bytesToRead = count; } - IMemoryOwner chunkBuffer = this.readChunk.Buffer; - int chunkSize = this.readChunk.Length; - if (this.readChunk.Next is null) + if (bytesToRead <= 0) { - chunkSize = this.writeOffset; + // Already at the end of the stream, nothing to read + return 0; } - int bytesRead = 0; - int offset = 0; - int count = buffer.Length; - while (count > 0) + while (bytesToRead != 0 && this.currentChunk != this.memoryChunkBuffer.Length) { - if (this.readOffset == chunkSize) + bool moveToNextChunk = false; + MemoryChunk chunk = this.memoryChunkBuffer[this.currentChunk]; + int n = (int)Math.Min(bytesToRead, int.MaxValue); + int remainingBytesInCurrentChunk = chunk.Length - this.currentChunkIndex; + if (n >= remainingBytesInCurrentChunk) { - // Exit if no more chunks are currently available - if (this.readChunk.Next is null) - { - break; - } - - this.readChunk = this.readChunk.Next; - this.readOffset = 0; - chunkBuffer = this.readChunk.Buffer; - chunkSize = this.readChunk.Length; - if (this.readChunk.Next is null) - { - chunkSize = this.writeOffset; - } + n = remainingBytesInCurrentChunk; + moveToNextChunk = true; } - int readCount = Math.Min(count, chunkSize - this.readOffset); - chunkBuffer.Slice(this.readOffset, readCount).CopyTo(buffer[offset..]); - offset += readCount; - count -= readCount; - this.readOffset += readCount; - bytesRead += readCount; - } - - return bytesRead; - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override int ReadByte() - { - this.EnsureNotDisposed(); + // Read n bytes from the current chunk + chunk.Buffer.Memory.Span.Slice(this.currentChunkIndex, n).CopyTo(buffer.Slice(offset, n)); + bytesToRead -= n; + offset += n; + bytesRead += n; - if (this.readChunk is null) - { - if (this.memoryChunk is null) + if (moveToNextChunk) { - return 0; + this.currentChunkIndex = 0; + this.currentChunk++; } - - this.readChunk = this.memoryChunk; - this.readOffset = 0; - } - - IMemoryOwner chunkBuffer = this.readChunk.Buffer; - int chunkSize = this.readChunk.Length; - if (this.readChunk.Next is null) - { - chunkSize = this.writeOffset; - } - - if (this.readOffset == chunkSize) - { - // Exit if no more chunks are currently available - if (this.readChunk.Next is null) + else { - return -1; + this.currentChunkIndex += n; } - - this.readChunk = this.readChunk.Next; - this.readOffset = 0; - chunkBuffer = this.readChunk.Buffer; } - return chunkBuffer.GetSpan()[this.readOffset++]; + this.position += bytesRead; + return bytesRead; } /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] public override void Write(byte[] buffer, int offset, int count) { Guard.NotNull(buffer, nameof(buffer)); @@ -344,157 +183,200 @@ public override void Write(byte[] buffer, int offset, int count) const string bufferMessage = "Offset subtracted from the buffer length is less than count."; Guard.IsFalse(buffer.Length - offset < count, nameof(buffer), bufferMessage); - this.WriteImpl(buffer.AsSpan(offset, count)); + this.Write(buffer.AsSpan(offset, count)); } /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override void Write(ReadOnlySpan buffer) => this.WriteImpl(buffer); - - private void WriteImpl(ReadOnlySpan buffer) + public override void Write(ReadOnlySpan buffer) { this.EnsureNotDisposed(); - if (this.memoryChunk is null) + int offset = 0; + int count = buffer.Length; + int bytesWritten = 0; + long bytesToWrite = this.memoryChunkBuffer.Length - this.position; + + // Ensure we have enough capacity to write the data. + while (bytesToWrite < count) { - this.memoryChunk = this.AllocateMemoryChunk(); - this.writeChunk = this.memoryChunk; - this.writeOffset = 0; + this.memoryChunkBuffer.Expand(); + bytesToWrite = this.memoryChunkBuffer.Length - this.position; } - Guard.NotNull(this.writeChunk); + if (bytesToWrite > count) + { + bytesToWrite = count; + } - Span chunkBuffer = this.writeChunk.Buffer.GetSpan(); - int chunkSize = this.writeChunk.Length; - int count = buffer.Length; - int offset = 0; - while (count > 0) + while (bytesToWrite != 0 && this.currentChunk != this.memoryChunkBuffer.Length) { - if (this.writeOffset == chunkSize) + bool moveToNextChunk = false; + MemoryChunk chunk = this.memoryChunkBuffer[this.currentChunk]; + int n = (int)Math.Min(bytesToWrite, int.MaxValue); + int remainingBytesInCurrentChunk = chunk.Length - this.currentChunkIndex; + if (n >= remainingBytesInCurrentChunk) { - // Allocate a new chunk if the current one is full - this.writeChunk.Next = this.AllocateMemoryChunk(); - this.writeChunk = this.writeChunk.Next; - this.writeOffset = 0; - chunkBuffer = this.writeChunk.Buffer.GetSpan(); - chunkSize = this.writeChunk.Length; + n = remainingBytesInCurrentChunk; + moveToNextChunk = true; } - int copyCount = Math.Min(count, chunkSize - this.writeOffset); - buffer.Slice(offset, copyCount).CopyTo(chunkBuffer[this.writeOffset..]); + // Write n bytes to the current chunk + buffer.Slice(offset, n).CopyTo(chunk.Buffer.Slice(this.currentChunkIndex, n)); + bytesToWrite -= n; + offset += n; + bytesWritten += n; - offset += copyCount; - count -= copyCount; - this.writeOffset += copyCount; + if (moveToNextChunk) + { + this.currentChunkIndex = 0; + this.currentChunk++; + } + else + { + this.currentChunkIndex += n; + } } + + this.position += bytesWritten; + this.length += bytesWritten; } - /// - public override void WriteByte(byte value) + /// + /// Writes the entire contents of this memory stream to another stream. + /// + /// The stream to write this memory stream to. + /// is . + /// The current or target stream is closed. + public void WriteTo(Stream stream) { + Guard.NotNull(stream, nameof(stream)); this.EnsureNotDisposed(); - if (this.memoryChunk is null) + this.Position = 0; + + int bytesRead = 0; + long bytesToRead = this.length - this.position; + if (bytesToRead <= 0) { - this.memoryChunk = this.AllocateMemoryChunk(); - this.writeChunk = this.memoryChunk; - this.writeOffset = 0; + // Already at the end of the stream, nothing to read + return; } - Guard.NotNull(this.writeChunk); + while (bytesToRead != 0 && this.currentChunk != this.memoryChunkBuffer.Length) + { + bool moveToNextChunk = false; + MemoryChunk chunk = this.memoryChunkBuffer[this.currentChunk]; + int n = (int)Math.Min(bytesToRead, int.MaxValue); + int remainingBytesInCurrentChunk = chunk.Length - this.currentChunkIndex; + if (n >= remainingBytesInCurrentChunk) + { + n = remainingBytesInCurrentChunk; + moveToNextChunk = true; + } - IMemoryOwner chunkBuffer = this.writeChunk.Buffer; - int chunkSize = this.writeChunk.Length; + // Read n bytes from the current chunk + stream.Write(chunk.Buffer.Memory.Span.Slice(this.currentChunkIndex, n)); + bytesToRead -= n; + bytesRead += n; - if (this.writeOffset == chunkSize) - { - // Allocate a new chunk if the current one is full - this.writeChunk.Next = this.AllocateMemoryChunk(); - this.writeChunk = this.writeChunk.Next; - this.writeOffset = 0; - chunkBuffer = this.writeChunk.Buffer; + if (moveToNextChunk) + { + this.currentChunkIndex = 0; + this.currentChunk++; + } + else + { + this.currentChunkIndex += n; + } } - chunkBuffer.GetSpan()[this.writeOffset++] = value; + this.position += bytesRead; } /// - /// Copy entire buffer into an array. + /// Writes the stream contents to a byte array, regardless of the property. /// - /// The . + /// A new . public byte[] ToArray() { - int length = (int)this.Length; // This will throw if stream is closed - byte[] copy = new byte[this.Length]; - - MemoryChunk? backupReadChunk = this.readChunk; - int backupReadOffset = this.readOffset; - - this.readChunk = this.memoryChunk; - this.readOffset = 0; - this.Read(copy, 0, length); - - this.readChunk = backupReadChunk; - this.readOffset = backupReadOffset; + this.EnsureNotDisposed(); + long position = this.position; + byte[] copy = new byte[this.length]; + this.Position = 0; + this.Read(copy, 0, copy.Length); + this.Position = position; return copy; } - /// - /// Write remainder of this stream to another stream. - /// - /// The stream to write to. - public void WriteTo(Stream stream) + /// + protected override void Dispose(bool disposing) { - this.EnsureNotDisposed(); - - Guard.NotNull(stream, nameof(stream)); + if (this.isDisposed) + { + return; + } - if (this.readChunk is null) + try { - if (this.memoryChunk is null) + this.isDisposed = true; + if (disposing) { - return; + this.memoryChunkBuffer.Dispose(); } - this.readChunk = this.memoryChunk; - this.readOffset = 0; + this.currentChunk = 0; + this.currentChunkIndex = 0; + this.position = 0; + this.length = 0; + } + finally + { + base.Dispose(disposing); + } + } + + private void SetPosition(long value) + { + long newPosition = value; + if (newPosition < 0) + { + throw new ArgumentOutOfRangeException(nameof(value)); } - IMemoryOwner chunkBuffer = this.readChunk.Buffer; - int chunkSize = this.readChunk.Length; - if (this.readChunk.Next is null) + this.position = newPosition; + + // Find the current chunk & current chunk index + int currentChunkIndex = 0; + long offset = newPosition; + + // If the new position is greater than the length of the stream, set the position to the end of the stream + if (offset > 0 && offset >= this.memoryChunkBuffer.Length) { - chunkSize = this.writeOffset; + this.currentChunk = this.memoryChunkBuffer.ChunkCount - 1; + this.currentChunkIndex = this.memoryChunkBuffer[this.currentChunk].Length - 1; + return; } - // Following code mirrors Read() logic (readChunk/readOffset should - // point just past last byte of last chunk when done) - // loop until end of chunks is found - while (true) + // Loop through the current chunks, as we increment the chunk index, we subtract the length of the chunk + // from the offset. Once the offset is less than the length of the chunk, we have found the correct chunk. + while (offset != 0) { - if (this.readOffset == chunkSize) + int chunkLength = this.memoryChunkBuffer[currentChunkIndex].Length; + if (offset < chunkLength) { - // Exit if no more chunks are currently available - if (this.readChunk.Next is null) - { - break; - } - - this.readChunk = this.readChunk.Next; - this.readOffset = 0; - chunkBuffer = this.readChunk.Buffer; - chunkSize = this.readChunk.Length; - if (this.readChunk.Next is null) - { - chunkSize = this.writeOffset; - } + // Found the correct chunk and the corresponding index + break; } - int writeCount = chunkSize - this.readOffset; - stream.Write(chunkBuffer.GetSpan(), this.readOffset, writeCount); - this.readOffset = chunkSize; + offset -= chunkLength; + currentChunkIndex++; } + + this.currentChunk = currentChunkIndex; + + // Safe to cast here as we know the offset is less than the chunk length. + this.currentChunkIndex = (int)offset; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -507,48 +389,82 @@ private void EnsureNotDisposed() } [MethodImpl(MethodImplOptions.NoInlining)] - private static void ThrowDisposed() => throw new ObjectDisposedException(null, "The stream is closed."); + private static void ThrowDisposed() => throw new ObjectDisposedException(nameof(ChunkedMemoryStream), "The stream is closed."); - [MethodImpl(MethodImplOptions.NoInlining)] - private static void ThrowArgumentOutOfRange(string value) => throw new ArgumentOutOfRangeException(value); + private sealed class MemoryChunkBuffer : IEnumerable, IDisposable + { + private readonly List memoryChunks = new(); + private readonly MemoryAllocator allocator; + private readonly int allocatorCapacity; + private bool isDisposed; - [MethodImpl(MethodImplOptions.NoInlining)] - private static void ThrowInvalidSeek() => throw new ArgumentException("Invalid seek origin."); + public MemoryChunkBuffer(MemoryAllocator allocator) + { + this.allocatorCapacity = allocator.GetBufferCapacityInBytes(); + this.allocator = allocator; + } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private MemoryChunk AllocateMemoryChunk() - { - // Tweak our buffer sizes to take the minimum of the provided buffer sizes - // or the allocator buffer capacity which provides us with the largest - // available contiguous buffer size. - IMemoryOwner buffer = this.allocator.Allocate(Math.Min(this.allocatorCapacity, GetChunkSize(this.chunkCount++))); + public int ChunkCount => this.memoryChunks.Count; + + public long Length { get; private set; } - return new MemoryChunk(buffer) + public MemoryChunk this[int index] => this.memoryChunks[index]; + + public void Expand() { - Next = null, - Length = buffer.Length() - }; - } + IMemoryOwner buffer = + this.allocator.Allocate(Math.Min(this.allocatorCapacity, GetChunkSize(this.ChunkCount))); - private static void ReleaseMemoryChunks(MemoryChunk? chunk) - { - while (chunk != null) + MemoryChunk chunk = new(buffer) + { + Length = buffer.Length() + }; + + this.memoryChunks.Add(chunk); + this.Length += chunk.Length; + } + + public void Dispose() { - chunk.Dispose(); - chunk = chunk.Next; + this.Dispose(true); + GC.SuppressFinalize(this); } - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetChunkSize(int i) - { - // Increment chunks sizes with moderate speed, but without using too many buffers from the same ArrayPool bucket of the default MemoryAllocator. - // https://github.com/SixLabors/ImageSharp/pull/2006#issuecomment-1066244720 -#pragma warning disable IDE1006 // Naming Styles - const int _128K = 1 << 17; - const int _4M = 1 << 22; - return i < 16 ? _128K * (1 << (int)((uint)i / 4)) : _4M; -#pragma warning restore IDE1006 // Naming Styles + public IEnumerator GetEnumerator() + => ((IEnumerable)this.memoryChunks).GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() + => ((IEnumerable)this.memoryChunks).GetEnumerator(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetChunkSize(int i) + { + // Increment chunks sizes with moderate speed, but without using too many buffers from the + // same ArrayPool bucket of the default MemoryAllocator. + // https://github.com/SixLabors/ImageSharp/pull/2006#issuecomment-1066244720 + const int b128K = 1 << 17; + const int b4M = 1 << 22; + return i < 16 ? b128K * (1 << (int)((uint)i / 4)) : b4M; + } + + private void Dispose(bool disposing) + { + if (!this.isDisposed) + { + if (disposing) + { + foreach (MemoryChunk chunk in this.memoryChunks) + { + chunk.Dispose(); + } + + this.memoryChunks.Clear(); + } + + this.Length = 0; + this.isDisposed = true; + } + } } private sealed class MemoryChunk : IDisposable @@ -559,8 +475,6 @@ private sealed class MemoryChunk : IDisposable public IMemoryOwner Buffer { get; } - public MemoryChunk? Next { get; set; } - public int Length { get; init; } private void Dispose(bool disposing) diff --git a/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs b/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs index 031a9ba059..c6751e2a66 100644 --- a/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs @@ -544,6 +544,44 @@ public static void RunEncodeLossy_WithPeakImage() [Fact] public void RunEncodeLossy_WithPeakImage_WithoutHardwareIntrinsics_Works() => FeatureTestRunner.RunWithHwIntrinsicsFeature(RunEncodeLossy_WithPeakImage, HwIntrinsics.DisableHWIntrinsic); + [Theory] + [WithFile(TestPatternOpaque, PixelTypes.Rgba32)] + public void CanSave_NonSeekableStream(TestImageProvider provider) + where TPixel : unmanaged, IPixel + { + using Image image = provider.GetImage(); + WebpEncoder encoder = new(); + + using MemoryStream seekable = new(); + image.Save(seekable, encoder); + + using MemoryStream memoryStream = new(); + using NonSeekableStream nonSeekable = new(memoryStream); + + image.Save(nonSeekable, encoder); + + Assert.True(seekable.ToArray().SequenceEqual(memoryStream.ToArray())); + } + + [Theory] + [WithFile(TestPatternOpaque, PixelTypes.Rgba32)] + public async Task CanSave_NonSeekableStream_Async(TestImageProvider provider) + where TPixel : unmanaged, IPixel + { + using Image image = provider.GetImage(); + WebpEncoder encoder = new(); + + await using MemoryStream seekable = new(); + image.Save(seekable, encoder); + + await using MemoryStream memoryStream = new(); + await using NonSeekableStream nonSeekable = new(memoryStream); + + await image.SaveAsync(nonSeekable, encoder); + + Assert.True(seekable.ToArray().SequenceEqual(memoryStream.ToArray())); + } + private static ImageComparer GetComparer(int quality) { float tolerance = 0.01f; // ~1.0% diff --git a/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs b/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs index 1803cfddb9..8d7ea9a33e 100644 --- a/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs +++ b/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs @@ -30,7 +30,7 @@ public class ChunkedMemoryStreamTests [Fact] public void MemoryStream_GetPositionTest_Negative() { - using var ms = new ChunkedMemoryStream(this.allocator); + using ChunkedMemoryStream ms = new(this.allocator); long iCurrentPos = ms.Position; for (int i = -1; i > -6; i--) { @@ -42,7 +42,7 @@ public void MemoryStream_GetPositionTest_Negative() [Fact] public void MemoryStream_ReadTest_Negative() { - var ms2 = new ChunkedMemoryStream(this.allocator); + ChunkedMemoryStream ms2 = new(this.allocator); Assert.Throws(() => ms2.Read(null, 0, 0)); Assert.Throws(() => ms2.Read(new byte[] { 1 }, -1, 0)); @@ -63,8 +63,8 @@ public void MemoryStream_ReadTest_Negative() [InlineData(DefaultSmallChunkSize * 16)] public void MemoryStream_ReadByteTest(int length) { - using MemoryStream ms = this.CreateTestStream(length); - using var cms = new ChunkedMemoryStream(this.allocator); + using MemoryStream ms = CreateTestStream(length); + using ChunkedMemoryStream cms = new(this.allocator); ms.CopyTo(cms); cms.Position = 0; @@ -84,8 +84,8 @@ public void MemoryStream_ReadByteTest(int length) [InlineData(DefaultSmallChunkSize * 16)] public void MemoryStream_ReadByteBufferTest(int length) { - using MemoryStream ms = this.CreateTestStream(length); - using var cms = new ChunkedMemoryStream(this.allocator); + using MemoryStream ms = CreateTestStream(length); + using ChunkedMemoryStream cms = new(this.allocator); ms.CopyTo(cms); cms.Position = 0; @@ -107,8 +107,8 @@ public void MemoryStream_ReadByteBufferTest(int length) [InlineData(DefaultSmallChunkSize * 16)] public void MemoryStream_ReadByteBufferSpanTest(int length) { - using MemoryStream ms = this.CreateTestStream(length); - using var cms = new ChunkedMemoryStream(this.allocator); + using MemoryStream ms = CreateTestStream(length); + using ChunkedMemoryStream cms = new(this.allocator); ms.CopyTo(cms); cms.Position = 0; @@ -125,7 +125,7 @@ public void MemoryStream_ReadByteBufferSpanTest(int length) [Fact] public void MemoryStream_WriteToTests() { - using (var ms2 = new ChunkedMemoryStream(this.allocator)) + using (ChunkedMemoryStream ms2 = new(this.allocator)) { byte[] bytArrRet; byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; @@ -133,7 +133,7 @@ public void MemoryStream_WriteToTests() // [] Write to memoryStream, check the memoryStream ms2.Write(bytArr, 0, bytArr.Length); - using var readonlyStream = new ChunkedMemoryStream(this.allocator); + using ChunkedMemoryStream readonlyStream = new(this.allocator); ms2.WriteTo(readonlyStream); readonlyStream.Flush(); readonlyStream.Position = 0; @@ -146,8 +146,8 @@ public void MemoryStream_WriteToTests() } // [] Write to memoryStream, check the memoryStream - using (var ms2 = new ChunkedMemoryStream(this.allocator)) - using (var ms3 = new ChunkedMemoryStream(this.allocator)) + using (ChunkedMemoryStream ms2 = new(this.allocator)) + using (ChunkedMemoryStream ms3 = new(this.allocator)) { byte[] bytArrRet; byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; @@ -167,7 +167,7 @@ public void MemoryStream_WriteToTests() [Fact] public void MemoryStream_WriteToSpanTests() { - using (var ms2 = new ChunkedMemoryStream(this.allocator)) + using (ChunkedMemoryStream ms2 = new(this.allocator)) { Span bytArrRet; Span bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; @@ -175,10 +175,12 @@ public void MemoryStream_WriteToSpanTests() // [] Write to memoryStream, check the memoryStream ms2.Write(bytArr, 0, bytArr.Length); - using var readonlyStream = new ChunkedMemoryStream(this.allocator); + using ChunkedMemoryStream readonlyStream = new(this.allocator); ms2.WriteTo(readonlyStream); + readonlyStream.Flush(); readonlyStream.Position = 0; + bytArrRet = new byte[(int)readonlyStream.Length]; readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length); for (int i = 0; i < bytArr.Length; i++) @@ -188,13 +190,14 @@ public void MemoryStream_WriteToSpanTests() } // [] Write to memoryStream, check the memoryStream - using (var ms2 = new ChunkedMemoryStream(this.allocator)) - using (var ms3 = new ChunkedMemoryStream(this.allocator)) + using (ChunkedMemoryStream ms2 = new(this.allocator)) + using (ChunkedMemoryStream ms3 = new(this.allocator)) { Span bytArrRet; Span bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; ms2.Write(bytArr, 0, bytArr.Length); + ms2.WriteTo(ms3); ms3.Position = 0; bytArrRet = new byte[(int)ms3.Length]; @@ -209,37 +212,35 @@ public void MemoryStream_WriteToSpanTests() [Fact] public void MemoryStream_WriteByteTests() { - using (var ms2 = new ChunkedMemoryStream(this.allocator)) - { - byte[] bytArrRet; - byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; + using ChunkedMemoryStream ms2 = new(this.allocator); + byte[] bytArrRet; + byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; - for (int i = 0; i < bytArr.Length; i++) - { - ms2.WriteByte(bytArr[i]); - } + for (int i = 0; i < bytArr.Length; i++) + { + ms2.WriteByte(bytArr[i]); + } - using var readonlyStream = new ChunkedMemoryStream(this.allocator); - ms2.WriteTo(readonlyStream); - readonlyStream.Flush(); - readonlyStream.Position = 0; - bytArrRet = new byte[(int)readonlyStream.Length]; - readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length); - for (int i = 0; i < bytArr.Length; i++) - { - Assert.Equal(bytArr[i], bytArrRet[i]); - } + using ChunkedMemoryStream readonlyStream = new(this.allocator); + ms2.WriteTo(readonlyStream); + readonlyStream.Flush(); + readonlyStream.Position = 0; + bytArrRet = new byte[(int)readonlyStream.Length]; + readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length); + for (int i = 0; i < bytArr.Length; i++) + { + Assert.Equal(bytArr[i], bytArrRet[i]); } } [Fact] public void MemoryStream_WriteToTests_Negative() { - using var ms2 = new ChunkedMemoryStream(this.allocator); + using ChunkedMemoryStream ms2 = new(this.allocator); Assert.Throws(() => ms2.WriteTo(null)); ms2.Write(new byte[] { 1 }, 0, 1); - var readonlyStream = new MemoryStream(new byte[1028], false); + MemoryStream readonlyStream = new(new byte[1028], false); Assert.Throws(() => ms2.WriteTo(readonlyStream)); readonlyStream.Dispose(); @@ -286,7 +287,7 @@ public void MemoryStream_CopyTo_Invalid() [MemberData(nameof(CopyToData))] public void CopyTo(Stream source, byte[] expected) { - using var destination = new ChunkedMemoryStream(this.allocator); + using ChunkedMemoryStream destination = new(this.allocator); source.CopyTo(destination); Assert.InRange(source.Position, source.Length, int.MaxValue); // Copying the data should have read to the end of the stream or stayed past the end. Assert.Equal(expected, destination.ToArray()); @@ -297,10 +298,10 @@ public static IEnumerable GetAllTestImages() IEnumerable allImageFiles = Directory.EnumerateFiles(TestEnvironment.InputImagesDirectoryFullPath, "*.*", SearchOption.AllDirectories) .Where(s => !s.EndsWith("txt", StringComparison.OrdinalIgnoreCase)); - var result = new List(); + List result = new(); foreach (string path in allImageFiles) { - result.Add(path.Substring(TestEnvironment.InputImagesDirectoryFullPath.Length)); + result.Add(path[TestEnvironment.InputImagesDirectoryFullPath.Length..]); } return result; @@ -334,9 +335,9 @@ public void DecoderIntegrationTest(TestImageProvider provider) ((TestImageProvider.FileProvider)provider).FilePath); using FileStream fs = File.OpenRead(fullPath); - using var nonSeekableStream = new NonSeekableStream(fs); + using NonSeekableStream nonSeekableStream = new(fs); - var actual = Image.Load(nonSeekableStream); + Image actual = Image.Load(nonSeekableStream); ImageComparer.Exact.VerifySimilarity(expected, actual); } @@ -345,27 +346,27 @@ public static IEnumerable CopyToData() { // Stream is positioned @ beginning of data byte[] data1 = new byte[] { 1, 2, 3 }; - var stream1 = new MemoryStream(data1); + MemoryStream stream1 = new(data1); yield return new object[] { stream1, data1 }; // Stream is positioned in the middle of data byte[] data2 = new byte[] { 0xff, 0xf3, 0xf0 }; - var stream2 = new MemoryStream(data2) { Position = 1 }; + MemoryStream stream2 = new(data2) { Position = 1 }; yield return new object[] { stream2, new byte[] { 0xf3, 0xf0 } }; // Stream is positioned after end of data byte[] data3 = data2; - var stream3 = new MemoryStream(data3) { Position = data3.Length + 1 }; + MemoryStream stream3 = new(data3) { Position = data3.Length + 1 }; yield return new object[] { stream3, Array.Empty() }; } - private MemoryStream CreateTestStream(int length) + private static MemoryStream CreateTestStream(int length) { byte[] buffer = new byte[length]; - var random = new Random(); + Random random = new(); random.NextBytes(buffer); return new MemoryStream(buffer); diff --git a/tests/ImageSharp.Tests/Image/NonSeekableStream.cs b/tests/ImageSharp.Tests/Image/NonSeekableStream.cs index 4b1f6e1568..2941490e9a 100644 --- a/tests/ImageSharp.Tests/Image/NonSeekableStream.cs +++ b/tests/ImageSharp.Tests/Image/NonSeekableStream.cs @@ -1,4 +1,4 @@ -// Copyright (c) Six Labors. +// Copyright (c) Six Labors. // Licensed under the Six Labors Split License. namespace SixLabors.ImageSharp.Tests; @@ -14,7 +14,7 @@ public NonSeekableStream(Stream dataStream) public override bool CanSeek => false; - public override bool CanWrite => false; + public override bool CanWrite => this.dataStream.CanWrite; public override bool CanTimeout => this.dataStream.CanTimeout; @@ -91,5 +91,5 @@ public override void SetLength(long value) => throw new NotSupportedException(); public override void Write(byte[] buffer, int offset, int count) - => throw new NotImplementedException(); + => this.dataStream.Write(buffer, offset, count); } From f96f8ca285693e70fe095d953bc8f930bb7a81ba Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 23 Oct 2024 12:23:34 +1000 Subject: [PATCH 05/10] Revert "Update BufferedStreams.cs" This reverts commit 1a150780fd5dc79194e02568503a12b36cbc42cb. --- .../General/IO/BufferedStreams.cs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/ImageSharp.Benchmarks/General/IO/BufferedStreams.cs b/tests/ImageSharp.Benchmarks/General/IO/BufferedStreams.cs index a7b22e7ab8..2a926d1cd8 100644 --- a/tests/ImageSharp.Benchmarks/General/IO/BufferedStreams.cs +++ b/tests/ImageSharp.Benchmarks/General/IO/BufferedStreams.cs @@ -19,8 +19,12 @@ public class BufferedStreams private MemoryStream stream4; private MemoryStream stream5; private MemoryStream stream6; + private ChunkedMemoryStream chunkedMemoryStream1; + private ChunkedMemoryStream chunkedMemoryStream2; private BufferedReadStream bufferedStream1; private BufferedReadStream bufferedStream2; + private BufferedReadStream bufferedStream3; + private BufferedReadStream bufferedStream4; private BufferedReadStreamWrapper bufferedStreamWrap1; private BufferedReadStreamWrapper bufferedStreamWrap2; @@ -35,8 +39,18 @@ public void CreateStreams() this.stream6 = new MemoryStream(this.buffer); this.stream6 = new MemoryStream(this.buffer); + this.chunkedMemoryStream1 = new ChunkedMemoryStream(Configuration.Default.MemoryAllocator); + this.chunkedMemoryStream1.Write(this.buffer); + this.chunkedMemoryStream1.Position = 0; + + this.chunkedMemoryStream2 = new ChunkedMemoryStream(Configuration.Default.MemoryAllocator); + this.chunkedMemoryStream2.Write(this.buffer); + this.chunkedMemoryStream2.Position = 0; + this.bufferedStream1 = new BufferedReadStream(Configuration.Default, this.stream3); this.bufferedStream2 = new BufferedReadStream(Configuration.Default, this.stream4); + this.bufferedStream3 = new BufferedReadStream(Configuration.Default, this.chunkedMemoryStream1); + this.bufferedStream4 = new BufferedReadStream(Configuration.Default, this.chunkedMemoryStream2); this.bufferedStreamWrap1 = new BufferedReadStreamWrapper(this.stream5); this.bufferedStreamWrap2 = new BufferedReadStreamWrapper(this.stream6); } @@ -46,8 +60,12 @@ public void DestroyStreams() { this.bufferedStream1?.Dispose(); this.bufferedStream2?.Dispose(); + this.bufferedStream3?.Dispose(); + this.bufferedStream4?.Dispose(); this.bufferedStreamWrap1?.Dispose(); this.bufferedStreamWrap2?.Dispose(); + this.chunkedMemoryStream1?.Dispose(); + this.chunkedMemoryStream2?.Dispose(); this.stream1?.Dispose(); this.stream2?.Dispose(); this.stream3?.Dispose(); @@ -86,6 +104,21 @@ public int BufferedReadStreamRead() return r; } + [Benchmark] + public int BufferedReadStreamChunkedRead() + { + int r = 0; + BufferedReadStream reader = this.bufferedStream3; + byte[] b = this.chunk2; + + for (int i = 0; i < reader.Length / 2; i++) + { + r += reader.Read(b, 0, 2); + } + + return r; + } + [Benchmark] public int BufferedReadStreamWrapRead() { @@ -129,6 +162,20 @@ public int BufferedReadStreamReadByte() return r; } + [Benchmark] + public int BufferedReadStreamChunkedReadByte() + { + int r = 0; + BufferedReadStream reader = this.bufferedStream4; + + for (int i = 0; i < reader.Length; i++) + { + r += reader.ReadByte(); + } + + return r; + } + [Benchmark] public int BufferedReadStreamWrapReadByte() { From 03343c4abee4d33c128f70050505613022e754df Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 23 Oct 2024 14:18:26 +1000 Subject: [PATCH 06/10] Simplify and optimize position checking --- src/ImageSharp/IO/ChunkedMemoryStream.cs | 39 +++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/ImageSharp/IO/ChunkedMemoryStream.cs b/src/ImageSharp/IO/ChunkedMemoryStream.cs index f178764603..06074b25bc 100644 --- a/src/ImageSharp/IO/ChunkedMemoryStream.cs +++ b/src/ImageSharp/IO/ChunkedMemoryStream.cs @@ -127,24 +127,26 @@ public override int Read(Span buffer) int offset = 0; int count = buffer.Length; - int bytesRead = 0; - long bytesToRead = this.length - this.position; - if (bytesToRead > count) + + long remaining = this.length - this.position; + if (remaining > count) { - bytesToRead = count; + remaining = count; } - if (bytesToRead <= 0) + if (remaining <= 0) { // Already at the end of the stream, nothing to read return 0; } + int bytesToRead = (int)remaining; + int bytesRead = 0; while (bytesToRead != 0 && this.currentChunk != this.memoryChunkBuffer.Length) { bool moveToNextChunk = false; MemoryChunk chunk = this.memoryChunkBuffer[this.currentChunk]; - int n = (int)Math.Min(bytesToRead, int.MaxValue); + int n = bytesToRead; int remainingBytesInCurrentChunk = chunk.Length - this.currentChunkIndex; if (n >= remainingBytesInCurrentChunk) { @@ -193,26 +195,28 @@ public override void Write(ReadOnlySpan buffer) int offset = 0; int count = buffer.Length; - int bytesWritten = 0; - long bytesToWrite = this.memoryChunkBuffer.Length - this.position; + + long remaining = this.memoryChunkBuffer.Length - this.position; // Ensure we have enough capacity to write the data. - while (bytesToWrite < count) + while (remaining < count) { this.memoryChunkBuffer.Expand(); - bytesToWrite = this.memoryChunkBuffer.Length - this.position; + remaining = this.memoryChunkBuffer.Length - this.position; } - if (bytesToWrite > count) + if (remaining > count) { - bytesToWrite = count; + remaining = count; } + int bytesToWrite = (int)remaining; + int bytesWritten = 0; while (bytesToWrite != 0 && this.currentChunk != this.memoryChunkBuffer.Length) { bool moveToNextChunk = false; MemoryChunk chunk = this.memoryChunkBuffer[this.currentChunk]; - int n = (int)Math.Min(bytesToWrite, int.MaxValue); + int n = bytesToWrite; int remainingBytesInCurrentChunk = chunk.Length - this.currentChunkIndex; if (n >= remainingBytesInCurrentChunk) { @@ -254,19 +258,20 @@ public void WriteTo(Stream stream) this.Position = 0; - int bytesRead = 0; - long bytesToRead = this.length - this.position; - if (bytesToRead <= 0) + long remaining = this.length - this.position; + if (remaining <= 0) { // Already at the end of the stream, nothing to read return; } + int bytesToRead = (int)remaining; + int bytesRead = 0; while (bytesToRead != 0 && this.currentChunk != this.memoryChunkBuffer.Length) { bool moveToNextChunk = false; MemoryChunk chunk = this.memoryChunkBuffer[this.currentChunk]; - int n = (int)Math.Min(bytesToRead, int.MaxValue); + int n = bytesToRead; int remainingBytesInCurrentChunk = chunk.Length - this.currentChunkIndex; if (n >= remainingBytesInCurrentChunk) { From b74d2e425774333e4be9b1e776f81b10fb003eee Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 30 Oct 2024 09:34:07 +1000 Subject: [PATCH 07/10] Add WriteByte and optimize ReadByte --- src/ImageSharp/IO/ChunkedMemoryStream.cs | 28 +++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/ImageSharp/IO/ChunkedMemoryStream.cs b/src/ImageSharp/IO/ChunkedMemoryStream.cs index 06074b25bc..59c42ec387 100644 --- a/src/ImageSharp/IO/ChunkedMemoryStream.cs +++ b/src/ImageSharp/IO/ChunkedMemoryStream.cs @@ -4,6 +4,7 @@ using System.Buffers; using System.Collections; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using SixLabors.ImageSharp.Memory; namespace SixLabors.ImageSharp.IO; @@ -13,11 +14,10 @@ namespace SixLabors.ImageSharp.IO; /// Chunks are allocated by the assigned via the constructor /// and is designed to take advantage of buffer pooling when available. /// -/// Provides an in-memory stream composed of non-contiguous chunks. public class ChunkedMemoryStream : Stream { private readonly MemoryChunkBuffer memoryChunkBuffer; - private readonly byte[] singleReadBuffer = new byte[1]; + private readonly byte[] singleByteBuffer = new byte[1]; private long length; private long position; @@ -101,8 +101,8 @@ public override int ReadByte() return -1; } - _ = this.Read(this.singleReadBuffer, 0, 1); - return this.singleReadBuffer[^1]; + _ = this.Read(this.singleByteBuffer, 0, 1); + return MemoryMarshal.GetReference(this.singleByteBuffer); } /// @@ -129,17 +129,17 @@ public override int Read(Span buffer) int count = buffer.Length; long remaining = this.length - this.position; - if (remaining > count) - { - remaining = count; - } - if (remaining <= 0) { // Already at the end of the stream, nothing to read return 0; } + if (remaining > count) + { + remaining = count; + } + int bytesToRead = (int)remaining; int bytesRead = 0; while (bytesToRead != 0 && this.currentChunk != this.memoryChunkBuffer.Length) @@ -175,6 +175,14 @@ public override int Read(Span buffer) return bytesRead; } + /// + public override void WriteByte(byte value) + { + this.EnsureNotDisposed(); + MemoryMarshal.Write(this.singleByteBuffer, ref value); + this.Write(this.singleByteBuffer, 0, 1); + } + /// public override void Write(byte[] buffer, int offset, int count) { @@ -309,7 +317,7 @@ public byte[] ToArray() byte[] copy = new byte[this.length]; this.Position = 0; - this.Read(copy, 0, copy.Length); + _ = this.Read(copy, 0, copy.Length); this.Position = position; return copy; } From 630166211c7c3d1b6b56f250bf198cc967d3e42b Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Mon, 4 Nov 2024 10:12:11 +1000 Subject: [PATCH 08/10] Feedback updates and massively expand write tests --- src/ImageSharp/IO/ChunkedMemoryStream.cs | 148 ++++++------------ .../IO/ChunkedMemoryStreamTests.cs | 88 ++++++++--- 2 files changed, 119 insertions(+), 117 deletions(-) diff --git a/src/ImageSharp/IO/ChunkedMemoryStream.cs b/src/ImageSharp/IO/ChunkedMemoryStream.cs index 59c42ec387..53de2c3cb5 100644 --- a/src/ImageSharp/IO/ChunkedMemoryStream.cs +++ b/src/ImageSharp/IO/ChunkedMemoryStream.cs @@ -2,7 +2,6 @@ // Licensed under the Six Labors Split License. using System.Buffers; -using System.Collections; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using SixLabors.ImageSharp.Memory; @@ -14,15 +13,13 @@ namespace SixLabors.ImageSharp.IO; /// Chunks are allocated by the assigned via the constructor /// and is designed to take advantage of buffer pooling when available. /// -public class ChunkedMemoryStream : Stream +internal sealed class ChunkedMemoryStream : Stream { private readonly MemoryChunkBuffer memoryChunkBuffer; - private readonly byte[] singleByteBuffer = new byte[1]; - private long length; private long position; - private int currentChunk; - private int currentChunkIndex; + private int bufferIndex; + private int chunkIndex; private bool isDisposed; /// @@ -95,21 +92,13 @@ public override void SetLength(long value) /// public override int ReadByte() { - this.EnsureNotDisposed(); - if (this.position >= this.length) - { - return -1; - } - - _ = this.Read(this.singleByteBuffer, 0, 1); - return MemoryMarshal.GetReference(this.singleByteBuffer); + Unsafe.SkipInit(out byte b); + return this.Read(MemoryMarshal.CreateSpan(ref b, 1)) == 1 ? b : -1; } /// public override int Read(byte[] buffer, int offset, int count) { - this.EnsureNotDisposed(); - Guard.NotNull(buffer, nameof(buffer)); Guard.MustBeGreaterThanOrEqualTo(offset, 0, nameof(offset)); Guard.MustBeGreaterThanOrEqualTo(count, 0, nameof(count)); @@ -135,19 +124,14 @@ public override int Read(Span buffer) return 0; } - if (remaining > count) - { - remaining = count; - } - - int bytesToRead = (int)remaining; + int bytesToRead = count; int bytesRead = 0; - while (bytesToRead != 0 && this.currentChunk != this.memoryChunkBuffer.Length) + while (bytesToRead > 0 && this.bufferIndex != this.memoryChunkBuffer.Length) { bool moveToNextChunk = false; - MemoryChunk chunk = this.memoryChunkBuffer[this.currentChunk]; + MemoryChunk chunk = this.memoryChunkBuffer[this.bufferIndex]; int n = bytesToRead; - int remainingBytesInCurrentChunk = chunk.Length - this.currentChunkIndex; + int remainingBytesInCurrentChunk = chunk.Length - this.chunkIndex; if (n >= remainingBytesInCurrentChunk) { n = remainingBytesInCurrentChunk; @@ -155,19 +139,19 @@ public override int Read(Span buffer) } // Read n bytes from the current chunk - chunk.Buffer.Memory.Span.Slice(this.currentChunkIndex, n).CopyTo(buffer.Slice(offset, n)); + chunk.Buffer.Memory.Span.Slice(this.chunkIndex, n).CopyTo(buffer.Slice(offset, n)); bytesToRead -= n; offset += n; bytesRead += n; if (moveToNextChunk) { - this.currentChunkIndex = 0; - this.currentChunk++; + this.chunkIndex = 0; + this.bufferIndex++; } else { - this.currentChunkIndex += n; + this.chunkIndex += n; } } @@ -177,11 +161,7 @@ public override int Read(Span buffer) /// public override void WriteByte(byte value) - { - this.EnsureNotDisposed(); - MemoryMarshal.Write(this.singleByteBuffer, ref value); - this.Write(this.singleByteBuffer, 0, 1); - } + => this.Write(MemoryMarshal.CreateSpan(ref value, 1)); /// public override void Write(byte[] buffer, int offset, int count) @@ -213,19 +193,14 @@ public override void Write(ReadOnlySpan buffer) remaining = this.memoryChunkBuffer.Length - this.position; } - if (remaining > count) - { - remaining = count; - } - - int bytesToWrite = (int)remaining; + int bytesToWrite = count; int bytesWritten = 0; - while (bytesToWrite != 0 && this.currentChunk != this.memoryChunkBuffer.Length) + while (bytesToWrite > 0 && this.bufferIndex != this.memoryChunkBuffer.Length) { bool moveToNextChunk = false; - MemoryChunk chunk = this.memoryChunkBuffer[this.currentChunk]; + MemoryChunk chunk = this.memoryChunkBuffer[this.bufferIndex]; int n = bytesToWrite; - int remainingBytesInCurrentChunk = chunk.Length - this.currentChunkIndex; + int remainingBytesInCurrentChunk = chunk.Length - this.chunkIndex; if (n >= remainingBytesInCurrentChunk) { n = remainingBytesInCurrentChunk; @@ -233,19 +208,19 @@ public override void Write(ReadOnlySpan buffer) } // Write n bytes to the current chunk - buffer.Slice(offset, n).CopyTo(chunk.Buffer.Slice(this.currentChunkIndex, n)); + buffer.Slice(offset, n).CopyTo(chunk.Buffer.Slice(this.chunkIndex, n)); bytesToWrite -= n; offset += n; bytesWritten += n; if (moveToNextChunk) { - this.currentChunkIndex = 0; - this.currentChunk++; + this.chunkIndex = 0; + this.bufferIndex++; } else { - this.currentChunkIndex += n; + this.chunkIndex += n; } } @@ -275,12 +250,12 @@ public void WriteTo(Stream stream) int bytesToRead = (int)remaining; int bytesRead = 0; - while (bytesToRead != 0 && this.currentChunk != this.memoryChunkBuffer.Length) + while (bytesToRead > 0 && this.bufferIndex != this.memoryChunkBuffer.Length) { bool moveToNextChunk = false; - MemoryChunk chunk = this.memoryChunkBuffer[this.currentChunk]; + MemoryChunk chunk = this.memoryChunkBuffer[this.bufferIndex]; int n = bytesToRead; - int remainingBytesInCurrentChunk = chunk.Length - this.currentChunkIndex; + int remainingBytesInCurrentChunk = chunk.Length - this.chunkIndex; if (n >= remainingBytesInCurrentChunk) { n = remainingBytesInCurrentChunk; @@ -288,18 +263,18 @@ public void WriteTo(Stream stream) } // Read n bytes from the current chunk - stream.Write(chunk.Buffer.Memory.Span.Slice(this.currentChunkIndex, n)); + stream.Write(chunk.Buffer.Memory.Span.Slice(this.chunkIndex, n)); bytesToRead -= n; bytesRead += n; if (moveToNextChunk) { - this.currentChunkIndex = 0; - this.currentChunk++; + this.chunkIndex = 0; + this.bufferIndex++; } else { - this.currentChunkIndex += n; + this.chunkIndex += n; } } @@ -338,8 +313,8 @@ protected override void Dispose(bool disposing) this.memoryChunkBuffer.Dispose(); } - this.currentChunk = 0; - this.currentChunkIndex = 0; + this.bufferIndex = 0; + this.chunkIndex = 0; this.position = 0; this.length = 0; } @@ -366,8 +341,8 @@ private void SetPosition(long value) // If the new position is greater than the length of the stream, set the position to the end of the stream if (offset > 0 && offset >= this.memoryChunkBuffer.Length) { - this.currentChunk = this.memoryChunkBuffer.ChunkCount - 1; - this.currentChunkIndex = this.memoryChunkBuffer[this.currentChunk].Length - 1; + this.bufferIndex = this.memoryChunkBuffer.ChunkCount - 1; + this.chunkIndex = this.memoryChunkBuffer[this.bufferIndex].Length - 1; return; } @@ -386,10 +361,10 @@ private void SetPosition(long value) currentChunkIndex++; } - this.currentChunk = currentChunkIndex; + this.bufferIndex = currentChunkIndex; // Safe to cast here as we know the offset is less than the chunk length. - this.currentChunkIndex = (int)offset; + this.chunkIndex = (int)offset; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -404,7 +379,7 @@ private void EnsureNotDisposed() [MethodImpl(MethodImplOptions.NoInlining)] private static void ThrowDisposed() => throw new ObjectDisposedException(nameof(ChunkedMemoryStream), "The stream is closed."); - private sealed class MemoryChunkBuffer : IEnumerable, IDisposable + private sealed class MemoryChunkBuffer : IDisposable { private readonly List memoryChunks = new(); private readonly MemoryAllocator allocator; @@ -439,15 +414,19 @@ public void Expand() public void Dispose() { - this.Dispose(true); - GC.SuppressFinalize(this); - } + if (!this.isDisposed) + { + foreach (MemoryChunk chunk in this.memoryChunks) + { + chunk.Dispose(); + } - public IEnumerator GetEnumerator() - => ((IEnumerable)this.memoryChunks).GetEnumerator(); + this.memoryChunks.Clear(); - IEnumerator IEnumerable.GetEnumerator() - => ((IEnumerable)this.memoryChunks).GetEnumerator(); + this.Length = 0; + this.isDisposed = true; + } + } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int GetChunkSize(int i) @@ -459,25 +438,6 @@ private static int GetChunkSize(int i) const int b4M = 1 << 22; return i < 16 ? b128K * (1 << (int)((uint)i / 4)) : b4M; } - - private void Dispose(bool disposing) - { - if (!this.isDisposed) - { - if (disposing) - { - foreach (MemoryChunk chunk in this.memoryChunks) - { - chunk.Dispose(); - } - - this.memoryChunks.Clear(); - } - - this.Length = 0; - this.isDisposed = true; - } - } } private sealed class MemoryChunk : IDisposable @@ -490,23 +450,13 @@ private sealed class MemoryChunk : IDisposable public int Length { get; init; } - private void Dispose(bool disposing) + public void Dispose() { if (!this.isDisposed) { - if (disposing) - { - this.Buffer.Dispose(); - } - + this.Buffer.Dispose(); this.isDisposed = true; } } - - public void Dispose() - { - this.Dispose(disposing: true); - GC.SuppressFinalize(this); - } } } diff --git a/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs b/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs index 8d7ea9a33e..b1bb7a9f5a 100644 --- a/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs +++ b/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs @@ -13,6 +13,8 @@ namespace SixLabors.ImageSharp.Tests.IO; /// public class ChunkedMemoryStreamTests { + private readonly Random bufferFiller = new(); + /// /// The default length in bytes of each buffer chunk when allocating large buffers. /// @@ -63,7 +65,7 @@ public void MemoryStream_ReadTest_Negative() [InlineData(DefaultSmallChunkSize * 16)] public void MemoryStream_ReadByteTest(int length) { - using MemoryStream ms = CreateTestStream(length); + using MemoryStream ms = this.CreateTestStream(length); using ChunkedMemoryStream cms = new(this.allocator); ms.CopyTo(cms); @@ -84,7 +86,7 @@ public void MemoryStream_ReadByteTest(int length) [InlineData(DefaultSmallChunkSize * 16)] public void MemoryStream_ReadByteBufferTest(int length) { - using MemoryStream ms = CreateTestStream(length); + using MemoryStream ms = this.CreateTestStream(length); using ChunkedMemoryStream cms = new(this.allocator); ms.CopyTo(cms); @@ -105,9 +107,10 @@ public void MemoryStream_ReadByteBufferTest(int length) [InlineData(DefaultSmallChunkSize * 4)] [InlineData((int)(DefaultSmallChunkSize * 5.5))] [InlineData(DefaultSmallChunkSize * 16)] + [InlineData(DefaultSmallChunkSize * 32)] public void MemoryStream_ReadByteBufferSpanTest(int length) { - using MemoryStream ms = CreateTestStream(length); + using MemoryStream ms = this.CreateTestStream(length); using ChunkedMemoryStream cms = new(this.allocator); ms.CopyTo(cms); @@ -122,13 +125,19 @@ public void MemoryStream_ReadByteBufferSpanTest(int length) } } - [Fact] - public void MemoryStream_WriteToTests() + [Theory] + [InlineData(DefaultSmallChunkSize)] + [InlineData((int)(DefaultSmallChunkSize * 1.5))] + [InlineData(DefaultSmallChunkSize * 4)] + [InlineData((int)(DefaultSmallChunkSize * 5.5))] + [InlineData(DefaultSmallChunkSize * 16)] + [InlineData(DefaultSmallChunkSize * 32)] + public void MemoryStream_WriteToTests(int length) { using (ChunkedMemoryStream ms2 = new(this.allocator)) { byte[] bytArrRet; - byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; + byte[] bytArr = this.CreateTestBuffer(length); // [] Write to memoryStream, check the memoryStream ms2.Write(bytArr, 0, bytArr.Length); @@ -150,7 +159,7 @@ public void MemoryStream_WriteToTests() using (ChunkedMemoryStream ms3 = new(this.allocator)) { byte[] bytArrRet; - byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; + byte[] bytArr = this.CreateTestBuffer(length); ms2.Write(bytArr, 0, bytArr.Length); ms2.WriteTo(ms3); @@ -164,13 +173,19 @@ public void MemoryStream_WriteToTests() } } - [Fact] - public void MemoryStream_WriteToSpanTests() + [Theory] + [InlineData(DefaultSmallChunkSize)] + [InlineData((int)(DefaultSmallChunkSize * 1.5))] + [InlineData(DefaultSmallChunkSize * 4)] + [InlineData((int)(DefaultSmallChunkSize * 5.5))] + [InlineData(DefaultSmallChunkSize * 16)] + [InlineData(DefaultSmallChunkSize * 32)] + public void MemoryStream_WriteToSpanTests(int length) { using (ChunkedMemoryStream ms2 = new(this.allocator)) { Span bytArrRet; - Span bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; + Span bytArr = this.CreateTestBuffer(length); // [] Write to memoryStream, check the memoryStream ms2.Write(bytArr, 0, bytArr.Length); @@ -194,7 +209,7 @@ public void MemoryStream_WriteToSpanTests() using (ChunkedMemoryStream ms3 = new(this.allocator)) { Span bytArrRet; - Span bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 }; + Span bytArr = this.CreateTestBuffer(length); ms2.Write(bytArr, 0, bytArr.Length); @@ -307,7 +322,7 @@ public static IEnumerable GetAllTestImages() return result; } - public static IEnumerable AllTestImages = GetAllTestImages(); + public static IEnumerable AllTestImages { get; } = GetAllTestImages(); [Theory] [WithFileCollection(nameof(AllTestImages), PixelTypes.Rgba32)] @@ -337,9 +352,45 @@ public void DecoderIntegrationTest(TestImageProvider provider) using FileStream fs = File.OpenRead(fullPath); using NonSeekableStream nonSeekableStream = new(fs); - Image actual = Image.Load(nonSeekableStream); + using Image actual = Image.Load(nonSeekableStream); + + ImageComparer.Exact.VerifySimilarity(expected, actual); + expected.Dispose(); + } + + [Theory] + [WithFileCollection(nameof(AllTestImages), PixelTypes.Rgba32)] + public void EncoderIntegrationTest(TestImageProvider provider) + where TPixel : unmanaged, IPixel + { + if (!TestEnvironment.Is64BitProcess) + { + return; + } + + Image expected; + try + { + expected = provider.GetImage(); + } + catch + { + // The image is invalid + return; + } + + string fullPath = Path.Combine( + TestEnvironment.InputImagesDirectoryFullPath, + ((TestImageProvider.FileProvider)provider).FilePath); + + using MemoryStream ms = new(); + using NonSeekableStream nonSeekableStream = new(ms); + expected.SaveAsWebp(nonSeekableStream); + + using Image actual = Image.Load(nonSeekableStream); ImageComparer.Exact.VerifySimilarity(expected, actual); + expected.Dispose(); } public static IEnumerable CopyToData() @@ -363,12 +414,13 @@ public static IEnumerable CopyToData() yield return new object[] { stream3, Array.Empty() }; } - private static MemoryStream CreateTestStream(int length) + private byte[] CreateTestBuffer(int length) { byte[] buffer = new byte[length]; - Random random = new(); - random.NextBytes(buffer); - - return new MemoryStream(buffer); + this.bufferFiller.NextBytes(buffer); + return buffer; } + + private MemoryStream CreateTestStream(int length) + => new(this.CreateTestBuffer(length)); } From c45702df5b2c0464098bb3f7def7bf396bd537fd Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Mon, 4 Nov 2024 13:14:16 +1000 Subject: [PATCH 09/10] Fix read bug. --- src/ImageSharp/IO/ChunkedMemoryStream.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/ImageSharp/IO/ChunkedMemoryStream.cs b/src/ImageSharp/IO/ChunkedMemoryStream.cs index 53de2c3cb5..760d1d3345 100644 --- a/src/ImageSharp/IO/ChunkedMemoryStream.cs +++ b/src/ImageSharp/IO/ChunkedMemoryStream.cs @@ -124,7 +124,13 @@ public override int Read(Span buffer) return 0; } - int bytesToRead = count; + if (remaining > count) + { + remaining = count; + } + + // 'remaining' can be less than the provided buffer length. + int bytesToRead = (int)remaining; int bytesRead = 0; while (bytesToRead > 0 && this.bufferIndex != this.memoryChunkBuffer.Length) { @@ -422,7 +428,6 @@ public void Dispose() } this.memoryChunks.Clear(); - this.Length = 0; this.isDisposed = true; } From c4fd666018266f1db59efc584dd4d0e195ed1736 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 13 Nov 2024 09:54:19 +1000 Subject: [PATCH 10/10] Update tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs Co-authored-by: Anton Firszov --- tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs b/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs index b1bb7a9f5a..390170cfef 100644 --- a/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs +++ b/tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs @@ -13,7 +13,7 @@ namespace SixLabors.ImageSharp.Tests.IO; /// public class ChunkedMemoryStreamTests { - private readonly Random bufferFiller = new(); + private readonly Random bufferFiller = new(123); /// /// The default length in bytes of each buffer chunk when allocating large buffers.