csharp/src/ProtocolBuffers.Test/CodedInputStreamTest.cs

#region Copyright notice and license

// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// http://github.com/jskeet/dotnet-protobufs/
// Original C++/Java/Python code:
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#endregion

using System;
using System.Collections.Generic;
using System.IO;
using Google.ProtocolBuffers.Descriptors;
using Google.ProtocolBuffers.TestProtos;
using Xunit;

namespace Google.ProtocolBuffers
{
    public class CodedInputStreamTest
    {
        /// <summary>
        /// Helper to construct a byte array from a bunch of bytes.  The inputs are
        /// actually ints so that I can use hex notation and not get stupid errors
        /// about precision.
        /// </summary>
        private static byte[] Bytes(params int[] bytesAsInts)
        {
            byte[] bytes = new byte[bytesAsInts.Length];
            for (int i = 0; i < bytesAsInts.Length; i++)
            {
                bytes[i] = (byte) bytesAsInts[i];
            }
            return bytes;
        }

        /// <summary>
        /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and
        /// </summary>
        private static void AssertReadVarint(byte[] data, ulong value)
        {
            CodedInputStream input = CodedInputStream.CreateInstance(data);
            Assert.Equal((uint) value, input.ReadRawVarint32());

            input = CodedInputStream.CreateInstance(data);
            Assert.Equal(value, input.ReadRawVarint64());
            Assert.True(input.IsAtEnd);

            // Try different block sizes.
            for (int bufferSize = 1; bufferSize <= 16; bufferSize *= 2)
            {
                input = CodedInputStream.CreateInstance(new SmallBlockInputStream(data, bufferSize));
                Assert.Equal((uint) value, input.ReadRawVarint32());

                input = CodedInputStream.CreateInstance(new SmallBlockInputStream(data, bufferSize));
                Assert.Equal(value, input.ReadRawVarint64());
                Assert.True(input.IsAtEnd);
            }

            // Try reading directly from a MemoryStream. We want to verify that it
            // doesn't read past the end of the input, so write an extra byte - this
            // lets us test the position at the end.
            MemoryStream memoryStream = new MemoryStream();
            memoryStream.Write(data, 0, data.Length);
            memoryStream.WriteByte(0);
            memoryStream.Position = 0;
            Assert.Equal((uint) value, CodedInputStream.ReadRawVarint32(memoryStream));
            Assert.Equal(data.Length, memoryStream.Position);
        }

        /// <summary>
        /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and
        /// expects them to fail with an InvalidProtocolBufferException whose
        /// description matches the given one.
        /// </summary>
        private static void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data)
        {
            CodedInputStream input = CodedInputStream.CreateInstance(data);
            var exception = Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawVarint32());
            Assert.Equal(expected.Message, exception.Message);

            input = CodedInputStream.CreateInstance(data);
            exception = Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawVarint64());
            Assert.Equal(expected.Message, exception.Message);

            // Make sure we get the same error when reading directly from a Stream.
            exception = Assert.Throws<InvalidProtocolBufferException>(() => CodedInputStream.ReadRawVarint32(new MemoryStream(data)));
            Assert.Equal(expected.Message, exception.Message);
        }

        [Fact]
        public void ReadVarint()
        {
            AssertReadVarint(Bytes(0x00), 0);
            AssertReadVarint(Bytes(0x01), 1);
            AssertReadVarint(Bytes(0x7f), 127);
            // 14882
            AssertReadVarint(Bytes(0xa2, 0x74), (0x22 << 0) | (0x74 << 7));
            // 2961488830
            AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b),
                             (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) |
                             (0x0bL << 28));

            // 64-bit
            // 7256456126
            AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b),
                             (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) |
                             (0x1bL << 28));
            // 41256202580718336
            AssertReadVarint(Bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49),
                             (0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) |
                             (0x43L << 28) | (0x49L << 35) | (0x24L << 42) | (0x49L << 49));
            // 11964378330978735131
            AssertReadVarint(Bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01),
                             (0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) |
                             (0x3bUL << 28) | (0x56UL << 35) | (0x00UL << 42) |
                             (0x05UL << 49) | (0x26UL << 56) | (0x01UL << 63));

            // Failures
            AssertReadVarintFailure(
                InvalidProtocolBufferException.MalformedVarint(),
                Bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
                      0x00));
            AssertReadVarintFailure(
                InvalidProtocolBufferException.TruncatedMessage(),
                Bytes(0x80));
        }

        /// <summary>
        /// Parses the given bytes using ReadRawLittleEndian32() and checks
        /// that the result matches the given value.
        /// </summary>
        private static void AssertReadLittleEndian32(byte[] data, uint value)
        {
            CodedInputStream input = CodedInputStream.CreateInstance(data);
            Assert.Equal(value, input.ReadRawLittleEndian32());
            Assert.True(input.IsAtEnd);

            // Try different block sizes.
            for (int blockSize = 1; blockSize <= 16; blockSize *= 2)
            {
                input = CodedInputStream.CreateInstance(
                    new SmallBlockInputStream(data, blockSize));
                Assert.Equal(value, input.ReadRawLittleEndian32());
                Assert.True(input.IsAtEnd);
            }
        }

        /// <summary>
        /// Parses the given bytes using ReadRawLittleEndian64() and checks
        /// that the result matches the given value.
        /// </summary>
        private static void AssertReadLittleEndian64(byte[] data, ulong value)
        {
            CodedInputStream input = CodedInputStream.CreateInstance(data);
            Assert.Equal(value, input.ReadRawLittleEndian64());
            Assert.True(input.IsAtEnd);

            // Try different block sizes.
            for (int blockSize = 1; blockSize <= 16; blockSize *= 2)
            {
                input = CodedInputStream.CreateInstance(
                    new SmallBlockInputStream(data, blockSize));
                Assert.Equal(value, input.ReadRawLittleEndian64());
                Assert.True(input.IsAtEnd);
            }
        }

        [Fact]
        public void ReadLittleEndian()
        {
            AssertReadLittleEndian32(Bytes(0x78, 0x56, 0x34, 0x12), 0x12345678);
            AssertReadLittleEndian32(Bytes(0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef0);

            AssertReadLittleEndian64(Bytes(0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12),
                                     0x123456789abcdef0L);
            AssertReadLittleEndian64(
                Bytes(0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef012345678UL);
        }

        [Fact]
        public void DecodeZigZag32()
        {
            Assert.Equal(0, CodedInputStream.DecodeZigZag32(0));
            Assert.Equal(-1, CodedInputStream.DecodeZigZag32(1));
            Assert.Equal(1, CodedInputStream.DecodeZigZag32(2));
            Assert.Equal(-2, CodedInputStream.DecodeZigZag32(3));
            Assert.Equal(0x3FFFFFFF, CodedInputStream.DecodeZigZag32(0x7FFFFFFE));
            Assert.Equal(unchecked((int) 0xC0000000), CodedInputStream.DecodeZigZag32(0x7FFFFFFF));
            Assert.Equal(0x7FFFFFFF, CodedInputStream.DecodeZigZag32(0xFFFFFFFE));
            Assert.Equal(unchecked((int) 0x80000000), CodedInputStream.DecodeZigZag32(0xFFFFFFFF));
        }

        [Fact]
        public void DecodeZigZag64()
        {
            Assert.Equal(0, CodedInputStream.DecodeZigZag64(0));
            Assert.Equal(-1, CodedInputStream.DecodeZigZag64(1));
            Assert.Equal(1, CodedInputStream.DecodeZigZag64(2));
            Assert.Equal(-2, CodedInputStream.DecodeZigZag64(3));
            Assert.Equal(0x000000003FFFFFFFL, CodedInputStream.DecodeZigZag64(0x000000007FFFFFFEL));
            Assert.Equal(unchecked((long) 0xFFFFFFFFC0000000L), CodedInputStream.DecodeZigZag64(0x000000007FFFFFFFL));
            Assert.Equal(0x000000007FFFFFFFL, CodedInputStream.DecodeZigZag64(0x00000000FFFFFFFEL));
            Assert.Equal(unchecked((long) 0xFFFFFFFF80000000L), CodedInputStream.DecodeZigZag64(0x00000000FFFFFFFFL));
            Assert.Equal(0x7FFFFFFFFFFFFFFFL, CodedInputStream.DecodeZigZag64(0xFFFFFFFFFFFFFFFEL));
            Assert.Equal(unchecked((long) 0x8000000000000000L), CodedInputStream.DecodeZigZag64(0xFFFFFFFFFFFFFFFFL));
        }

        [Fact]
        public void ReadWholeMessage()
        {
            TestAllTypes message = TestUtil.GetAllSet();

            byte[] rawBytes = message.ToByteArray();
            Assert.Equal(rawBytes.Length, message.SerializedSize);
            TestAllTypes message2 = TestAllTypes.ParseFrom(rawBytes);
            TestUtil.AssertAllFieldsSet(message2);

            // Try different block sizes.
            for (int blockSize = 1; blockSize < 256; blockSize *= 2)
            {
                message2 = TestAllTypes.ParseFrom(new SmallBlockInputStream(rawBytes, blockSize));
                TestUtil.AssertAllFieldsSet(message2);
            }
        }

        [Fact]
        public void SkipWholeMessage()
        {
            TestAllTypes message = TestUtil.GetAllSet();
            byte[] rawBytes = message.ToByteArray();

            // Create two parallel inputs.  Parse one as unknown fields while using
            // skipField() to skip each field on the other.  Expect the same tags.
            CodedInputStream input1 = CodedInputStream.CreateInstance(rawBytes);
            CodedInputStream input2 = CodedInputStream.CreateInstance(rawBytes);
            UnknownFieldSet.Builder unknownFields = UnknownFieldSet.CreateBuilder();

            uint tag;
            string name;
            while (input1.ReadTag(out tag, out name))
            {
                uint tag2;
                Assert.True(input2.ReadTag(out tag2, out name));
                Assert.Equal(tag, tag2);

                unknownFields.MergeFieldFrom(tag, input1);
                input2.SkipField();
            }
        }

        /// <summary>
        /// Test that a bug in SkipRawBytes has been fixed: if the skip
        /// skips exactly up to a limit, this should bnot break things
        /// </summary>
        [Fact]
        public void SkipRawBytesBug()
        {
            byte[] rawBytes = new byte[] {1, 2};
            CodedInputStream input = CodedInputStream.CreateInstance(rawBytes);

            int limit = input.PushLimit(1);
            input.SkipRawBytes(1);
            input.PopLimit(limit);
            Assert.Equal(2, input.ReadRawByte());
        }

        public void ReadHugeBlob()
        {
            // Allocate and initialize a 1MB blob.
            byte[] blob = new byte[1 << 20];
            for (int i = 0; i < blob.Length; i++)
            {
                blob[i] = (byte) i;
            }

            // Make a message containing it.
            TestAllTypes.Builder builder = TestAllTypes.CreateBuilder();
            TestUtil.SetAllFields(builder);
            builder.SetOptionalBytes(ByteString.CopyFrom(blob));
            TestAllTypes message = builder.Build();

            // Serialize and parse it.  Make sure to parse from an InputStream, not
            // directly from a ByteString, so that CodedInputStream uses buffered
            // reading.
            TestAllTypes message2 = TestAllTypes.ParseFrom(message.ToByteString().CreateCodedInput());

            Assert.Equal(message.OptionalBytes, message2.OptionalBytes);

            // Make sure all the other fields were parsed correctly.
            TestAllTypes message3 = TestAllTypes.CreateBuilder(message2)
                .SetOptionalBytes(TestUtil.GetAllSet().OptionalBytes)
                .Build();
            TestUtil.AssertAllFieldsSet(message3);
        }

        [Fact]
        public void ReadMaliciouslyLargeBlob()
        {
            MemoryStream ms = new MemoryStream();
            CodedOutputStream output = CodedOutputStream.CreateInstance(ms);

            uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited);
            output.WriteRawVarint32(tag);
            output.WriteRawVarint32(0x7FFFFFFF);
            output.WriteRawBytes(new byte[32]); // Pad with a few random bytes.
            output.Flush();
            ms.Position = 0;

            CodedInputStream input = CodedInputStream.CreateInstance(ms);
            uint testtag;
            string ignore;
            Assert.True(input.ReadTag(out testtag, out ignore));
            Assert.Equal(tag, testtag);

            ByteString bytes = null;
            // TODO(jonskeet): Should this be ArgumentNullException instead?
            Assert.Throws<InvalidProtocolBufferException>(() => input.ReadBytes(ref bytes));
        }

        private static TestRecursiveMessage MakeRecursiveMessage(int depth)
        {
            if (depth == 0)
            {
                return TestRecursiveMessage.CreateBuilder().SetI(5).Build();
            }
            else
            {
                return TestRecursiveMessage.CreateBuilder()
                    .SetA(MakeRecursiveMessage(depth - 1)).Build();
            }
        }

        private static void AssertMessageDepth(TestRecursiveMessage message, int depth)
        {
            if (depth == 0)
            {
                Assert.False(message.HasA);
                Assert.Equal(5, message.I);
            }
            else
            {
                Assert.True(message.HasA);
                AssertMessageDepth(message.A, depth - 1);
            }
        }

        [Fact]
        public void MaliciousRecursion()
        {
            ByteString data64 = MakeRecursiveMessage(64).ToByteString();
            ByteString data65 = MakeRecursiveMessage(65).ToByteString();

            AssertMessageDepth(TestRecursiveMessage.ParseFrom(data64), 64);

            Assert.Throws<InvalidProtocolBufferException>(() => TestRecursiveMessage.ParseFrom(data65));

            CodedInputStream input = data64.CreateCodedInput();
            input.SetRecursionLimit(8);
            Assert.Throws<InvalidProtocolBufferException>(() => TestRecursiveMessage.ParseFrom(input));
        }

        [Fact]
        public void SizeLimit()
        {
            // Have to use a Stream rather than ByteString.CreateCodedInput as SizeLimit doesn't
            // apply to the latter case.
            MemoryStream ms = new MemoryStream(TestUtil.GetAllSet().ToByteString().ToByteArray());
            CodedInputStream input = CodedInputStream.CreateInstance(ms);
            input.SetSizeLimit(16);

            Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.ParseFrom(input));
        }

        [Fact]
        public void ResetSizeCounter()
        {
            CodedInputStream input = CodedInputStream.CreateInstance(
                new SmallBlockInputStream(new byte[256], 8));
            input.SetSizeLimit(16);
            input.ReadRawBytes(16);

            Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawByte());

            input.ResetSizeCounter();
            input.ReadRawByte(); // No exception thrown.

            Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawBytes(16));
        }

        /// <summary>
        /// Tests that if we read an string that contains invalid UTF-8, no exception
        /// is thrown.  Instead, the invalid bytes are replaced with the Unicode
        /// "replacement character" U+FFFD.
        /// </summary>
        [Fact]
        public void ReadInvalidUtf8()
        {
            MemoryStream ms = new MemoryStream();
            CodedOutputStream output = CodedOutputStream.CreateInstance(ms);

            uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited);
            output.WriteRawVarint32(tag);
            output.WriteRawVarint32(1);
            output.WriteRawBytes(new byte[] {0x80});
            output.Flush();
            ms.Position = 0;

            CodedInputStream input = CodedInputStream.CreateInstance(ms);

            uint testtag;
            string ignored;

            Assert.True(input.ReadTag(out testtag, out ignored));
            Assert.Equal(tag, testtag);
            string text = null;
            input.ReadString(ref text);
            Assert.Equal('\ufffd', text[0]);
        }

        /// <summary>
        /// A stream which limits the number of bytes it reads at a time.
        /// We use this to make sure that CodedInputStream doesn't screw up when
        /// reading in small blocks.
        /// </summary>
        private sealed class SmallBlockInputStream : MemoryStream
        {
            private readonly int blockSize;

            public SmallBlockInputStream(byte[] data, int blockSize)
                : base(data)
            {
                this.blockSize = blockSize;
            }

            public override int Read(byte[] buffer, int offset, int count)
            {
                return base.Read(buffer, offset, Math.Min(count, blockSize));
            }
        }

        enum TestNegEnum { None = 0, Value = -2 }

        [Fact]
        public void TestNegativeEnum()
        {
            byte[] bytes = new byte[10] { 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01 };
            CodedInputStream input = CodedInputStream.CreateInstance(bytes);
            object unk;
            TestNegEnum val = TestNegEnum.None;

            Assert.True(input.ReadEnum(ref val, out unk));
            Assert.True(input.IsAtEnd);
            Assert.Equal(TestNegEnum.Value, val);
        }

        [Fact]
        public void TestNegativeEnumPackedArray()
        {
            int arraySize = 1 + (10 * 5);
            int msgSize = 1 + 1 + arraySize;
            byte[] bytes = new byte[msgSize];
            CodedOutputStream output = CodedOutputStream.CreateInstance(bytes);
            output.WritePackedInt32Array(8, "", arraySize, new int[] { 0, -1, -2, -3, -4, -5 });

            Assert.Equal(0, output.SpaceLeft);

            CodedInputStream input = CodedInputStream.CreateInstance(bytes);
            uint tag;
            string name;
            Assert.True(input.ReadTag(out tag, out name));

            List<TestNegEnum> values = new List<TestNegEnum>();
            ICollection<object> unk;
            input.ReadEnumArray(tag, name, values, out unk);

            Assert.Equal(2, values.Count);
            Assert.Equal(TestNegEnum.None, values[0]);
            Assert.Equal(TestNegEnum.Value, values[1]);

            Assert.NotNull(unk);
            Assert.Equal(4, unk.Count);
        }

        [Fact]
        public void TestNegativeEnumArray()
        {
            int arraySize = 1 + 1 + (11 * 5);
            int msgSize = arraySize;
            byte[] bytes = new byte[msgSize];
            CodedOutputStream output = CodedOutputStream.CreateInstance(bytes);
            output.WriteInt32Array(8, "", new int[] { 0, -1, -2, -3, -4, -5 });

            Assert.Equal(0, output.SpaceLeft);

            CodedInputStream input = CodedInputStream.CreateInstance(bytes);
            uint tag;
            string name;
            Assert.True(input.ReadTag(out tag, out name));

            List<TestNegEnum> values = new List<TestNegEnum>();
            ICollection<object> unk;
            input.ReadEnumArray(tag, name, values, out unk);

            Assert.Equal(2, values.Count);
            Assert.Equal(TestNegEnum.None, values[0]);
            Assert.Equal(TestNegEnum.Value, values[1]);

            Assert.NotNull(unk);
            Assert.Equal(4, unk.Count);
        }

        //Issue 71:	CodedInputStream.ReadBytes go to slow path unnecessarily
        [Fact]
        public void TestSlowPathAvoidance()
        {
            using (var ms = new MemoryStream())
            {
                CodedOutputStream output = CodedOutputStream.CreateInstance(ms);
                output.WriteField(FieldType.Bytes, 1, "bytes", ByteString.CopyFrom(new byte[100]));
                output.WriteField(FieldType.Bytes, 2, "bytes", ByteString.CopyFrom(new byte[100]));
                output.Flush();

                ms.Position = 0;
                CodedInputStream input = CodedInputStream.CreateInstance(ms, new byte[ms.Length / 2]);

                uint tag;
                string ignore;
                ByteString value;

                Assert.True(input.ReadTag(out tag, out ignore));
                Assert.Equal(1, WireFormat.GetTagFieldNumber(tag));
                value = ByteString.Empty;
                Assert.True(input.ReadBytes(ref value) && value.Length == 100);

                Assert.True(input.ReadTag(out tag, out ignore));
                Assert.Equal(2, WireFormat.GetTagFieldNumber(tag));
                value = ByteString.Empty;
                Assert.True(input.ReadBytes(ref value) && value.Length == 100);
            }
        }
    }
}