Java/C#/Python prefixed size support (#4445) * initial changes to support size prefixed buffers in Java * add slice equivalent to CSharp ByteBuffer * resolve TODO for slicing in CSharp code generation * add newly generated Java and CSharp test sources * fix typo in comment * add FinishSizePrefixed methods to CSharp FlatBufferBuilder as well * add option to allow writing the prefix as well * generate size-prefixed monster binary as well * extend JavaTest to test the size prefixed binary as well * use constants for size prefix length * fuse common code for getRootAs and getSizePrefixedRootAs * pulled file identifier out of if * add FinishSizePrefixed, GetSizePrefixedRootAs support for Python * Revert "extend JavaTest to test the size prefixed binary as well" This reverts commit 68be4420dda47e8d0600bb19691f03be71503a68. * Revert "generate size-prefixed monster binary as well" This reverts commit 2939516fdf78df4f061c627221e232b312301417. * fix ByteBuffer.cs Slice() method; add proper CSharp and Java tests * fix unused parameter * increment version number * pulled out generated methods into separate utility class * pulled out generated methods into separate utility class for Python * fix indentation * remove unnecessary comment * fix newline and copyright * add ByteBufferUtil to csproj compilation * hide ByteBuffer's internal data; track offset into parent's array * test unsafe versions as well; compile and run in debug mode * clarify help text for size prefix * move ByteBuffer slicing behavior to subclass * fix protection levels * add size prefix support for text generation * add ByteBufferSlice to csproj compilation * revert size prefix handling for nested buffers * use duplicate instead of slice for removing size prefix * remove slice subclass and use duplicate for removing size prefix * remove slice specific tests * remove superfluous command line option
diff --git a/tests/FlatBuffers.Test/ByteBufferTests.cs b/tests/FlatBuffers.Test/ByteBufferTests.cs index 3324f12..58bd71e 100644 --- a/tests/FlatBuffers.Test/ByteBufferTests.cs +++ b/tests/FlatBuffers.Test/ByteBufferTests.cs
@@ -44,8 +44,7 @@ [FlatBuffersTestMethod] public void ByteBuffer_PutByteCannotPutAtOffsetPastLength() { - var buffer = new byte[1]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(1); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutByte(1, 99)); } #endif @@ -66,8 +65,7 @@ [FlatBuffersTestMethod] public void ByteBuffer_PutShortCannotPutAtOffsetPastLength() { - var buffer = new byte[2]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(2); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutShort(2, 99)); } #endif @@ -76,16 +74,14 @@ [FlatBuffersTestMethod] public void ByteBuffer_PutShortChecksLength() { - var buffer = new byte[1]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(1); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutShort(0, 99)); } [FlatBuffersTestMethod] public void ByteBuffer_PutShortChecksLengthAndOffset() { - var buffer = new byte[2]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(2); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutShort(1, 99)); } #endif @@ -108,24 +104,21 @@ [FlatBuffersTestMethod] public void ByteBuffer_PutIntCannotPutAtOffsetPastLength() { - var buffer = new byte[4]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(4); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutInt(2, 0x0A0B0C0D)); } [FlatBuffersTestMethod] public void ByteBuffer_PutIntChecksLength() { - var buffer = new byte[1]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(1); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutInt(0, 0x0A0B0C0D)); } [FlatBuffersTestMethod] public void ByteBuffer_PutIntChecksLengthAndOffset() { - var buffer = new byte[4]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(4); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutInt(2, 0x0A0B0C0D)); } #endif @@ -152,24 +145,21 @@ [FlatBuffersTestMethod] public void ByteBuffer_PutLongCannotPutAtOffsetPastLength() { - var buffer = new byte[8]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(8); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutLong(2, 0x010203040A0B0C0D)); } [FlatBuffersTestMethod] public void ByteBuffer_PutLongChecksLength() { - var buffer = new byte[1]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(1); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutLong(0, 0x010203040A0B0C0D)); } [FlatBuffersTestMethod] public void ByteBuffer_PutLongChecksLengthAndOffset() { - var buffer = new byte[8]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(8); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutLong(2, 0x010203040A0B0C0D)); } #endif @@ -187,8 +177,7 @@ [FlatBuffersTestMethod] public void ByteBuffer_GetByteChecksOffset() { - var buffer = new byte[1]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(1); Assert.Throws<ArgumentOutOfRangeException>(()=>uut.Get(1)); } #endif @@ -207,16 +196,14 @@ [FlatBuffersTestMethod] public void ByteBuffer_GetShortChecksOffset() { - var buffer = new byte[2]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(2); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetShort(2)); } [FlatBuffersTestMethod] public void ByteBuffer_GetShortChecksLength() { - var buffer = new byte[2]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(2); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetShort(1)); } #endif @@ -237,16 +224,14 @@ [FlatBuffersTestMethod] public void ByteBuffer_GetIntChecksOffset() { - var buffer = new byte[4]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(4); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetInt(4)); } [FlatBuffersTestMethod] public void ByteBuffer_GetIntChecksLength() { - var buffer = new byte[2]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(2); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetInt(0)); } #endif @@ -271,16 +256,14 @@ [FlatBuffersTestMethod] public void ByteBuffer_GetLongChecksOffset() { - var buffer = new byte[8]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(8); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetLong(8)); } [FlatBuffersTestMethod] public void ByteBuffer_GetLongChecksLength() { - var buffer = new byte[7]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(7); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetLong(0)); } #endif @@ -317,5 +300,49 @@ var rereverse = ByteBuffer.ReverseBytes(reverse); Assert.AreEqual(original, rereverse); } + + [FlatBuffersTestMethod] + public void ByteBuffer_ToFullArray_MatchesBuffer() + { + var buffer = new byte[4]; + buffer[0] = 0x0D; + buffer[1] = 0x0C; + buffer[2] = 0x0B; + buffer[3] = 0x0A; + var uut = new ByteBuffer(buffer); + Assert.ArrayEqual(buffer, uut.ToFullArray()); + } + + [FlatBuffersTestMethod] + public void ByteBuffer_ToSizedArray_MatchesBuffer() + { + var buffer = new byte[4]; + buffer[0] = 0x0D; + buffer[1] = 0x0C; + buffer[2] = 0x0B; + buffer[3] = 0x0A; + var uut = new ByteBuffer(buffer); + Assert.ArrayEqual(buffer, uut.ToFullArray()); + } + + [FlatBuffersTestMethod] + public void ByteBuffer_Duplicate_MatchesBuffer() + { + var buffer = new byte[4]; + buffer[0] = 0x0D; + buffer[1] = 0x0C; + buffer[2] = 0x0B; + buffer[3] = 0x0A; + var uut = new ByteBuffer(buffer); + Assert.AreEqual(0x0A0B0C0D, uut.GetInt(0)); + + // Advance by two bytes + uut.Position = 2; uut = uut.Duplicate(); + Assert.AreEqual(0x0A0B, uut.GetShort(2)); + + // Advance by one more byte + uut.Position = 1; uut = uut.Duplicate(); + Assert.AreEqual(0x0A, uut.Get(3)); + } } }
diff --git a/tests/FlatBuffers.Test/FlatBuffers.Test.csproj b/tests/FlatBuffers.Test/FlatBuffers.Test.csproj index 38d62d6..7c67274 100644 --- a/tests/FlatBuffers.Test/FlatBuffers.Test.csproj +++ b/tests/FlatBuffers.Test/FlatBuffers.Test.csproj
@@ -41,6 +41,9 @@ <Compile Include="..\..\net\FlatBuffers\ByteBuffer.cs"> <Link>FlatBuffers\ByteBuffer.cs</Link> </Compile> + <Compile Include="..\..\net\FlatBuffers\ByteBufferUtil.cs"> + <Link>FlatBuffers\ByteBufferUtil.cs</Link> + </Compile> <Compile Include="..\..\net\FlatBuffers\IFlatbufferObject.cs"> <Link>FlatBuffers\IFlatbufferObject.cs</Link> </Compile>
diff --git a/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs b/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs index ccd5b05..82d4fdf 100644 --- a/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs +++ b/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs
@@ -33,6 +33,12 @@ [FlatBuffersTestMethod] public void CanCreateNewFlatBufferFromScratch() { + CanCreateNewFlatBufferFromScratch(true); + CanCreateNewFlatBufferFromScratch(false); + } + + private void CanCreateNewFlatBufferFromScratch(bool sizePrefix) + { // Second, let's create a FlatBuffer from scratch in C#, and test it also. // We use an initial size of 1 to exercise the reallocation algorithm, // normally a size larger than the typical FlatBuffer you generate would be @@ -95,22 +101,40 @@ Monster.AddTestarrayoftables(fbb, sortMons); var mon = Monster.EndMonster(fbb); - Monster.FinishMonsterBuffer(fbb, mon); + if (sizePrefix) + { + Monster.FinishSizePrefixedMonsterBuffer(fbb, mon); + } + else + { + Monster.FinishMonsterBuffer(fbb, mon); + } // Dump to output directory so we can inspect later, if needed - using (var ms = new MemoryStream(fbb.DataBuffer.Data, fbb.DataBuffer.Position, fbb.Offset)) + using (var ms = fbb.DataBuffer.ToMemoryStream(fbb.DataBuffer.Position, fbb.Offset)) { var data = ms.ToArray(); - File.WriteAllBytes(@"Resources/monsterdata_cstest.mon",data); + string filename = @"Resources/monsterdata_cstest" + (sizePrefix ? "_sp" : "") + ".mon"; + File.WriteAllBytes(filename, data); + } + + // Remove the size prefix if necessary for further testing + ByteBuffer dataBuffer = fbb.DataBuffer; + if (sizePrefix) + { + Assert.AreEqual(ByteBufferUtil.GetSizePrefix(dataBuffer) + FlatBufferConstants.SizePrefixLength, + dataBuffer.Length - dataBuffer.Position); + dataBuffer = ByteBufferUtil.RemoveSizePrefix(dataBuffer); } // Now assert the buffer - TestBuffer(fbb.DataBuffer); + TestBuffer(dataBuffer); //Attempt to mutate Monster fields and check whether the buffer has been mutated properly // revert to original values after testing - Monster monster = Monster.GetRootAsMonster(fbb.DataBuffer); + Monster monster = Monster.GetRootAsMonster(dataBuffer); + // mana is optional and does not exist in the buffer so the mutation should fail // the mana field should retain its default value @@ -161,12 +185,12 @@ pos.MutateX(1.0f); Assert.AreEqual(pos.X, 1.0f); - TestBuffer(fbb.DataBuffer); + TestBuffer(dataBuffer); } private void TestBuffer(ByteBuffer bb) { - var monster = Monster.GetRootAsMonster(bb); + Monster monster = Monster.GetRootAsMonster(bb); Assert.AreEqual(80, monster.Hp); Assert.AreEqual(150, monster.Mana);
diff --git a/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs b/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs index d933246..2d411a2 100644 --- a/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs +++ b/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs
@@ -33,21 +33,21 @@ public void TestNumbers() { var builder = new FlatBufferBuilder(1); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddBool(true); - Assert.ArrayEqual(new byte[] { 1 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 1 }, builder.DataBuffer.ToFullArray()); builder.AddSbyte(-127); - Assert.ArrayEqual(new byte[] { 129, 1 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 129, 1 }, builder.DataBuffer.ToFullArray()); builder.AddByte(255); - Assert.ArrayEqual(new byte[] { 0, 255, 129, 1 }, builder.DataBuffer.Data); // First pad + Assert.ArrayEqual(new byte[] { 0, 255, 129, 1 }, builder.DataBuffer.ToFullArray()); // First pad builder.AddShort(-32222); - Assert.ArrayEqual(new byte[] { 0, 0, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.Data); // Second pad + Assert.ArrayEqual(new byte[] { 0, 0, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.ToFullArray()); // Second pad builder.AddUshort(0xFEEE); - Assert.ArrayEqual(new byte[] { 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.Data); // no pad + Assert.ArrayEqual(new byte[] { 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.ToFullArray()); // no pad builder.AddInt(-53687092); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.Data); // third pad + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.ToFullArray()); // third pad builder.AddUint(0x98765432); - Assert.ArrayEqual(new byte[] { 0x32, 0x54, 0x76, 0x98, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.Data); // no pad + Assert.ArrayEqual(new byte[] { 0x32, 0x54, 0x76, 0x98, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.ToFullArray()); // no pad } [FlatBuffersTestMethod] @@ -55,11 +55,11 @@ { var builder = new FlatBufferBuilder(1); builder.AddUlong(0x1122334455667788); - Assert.ArrayEqual(new byte[] { 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11 }, builder.DataBuffer.ToFullArray()); builder = new FlatBufferBuilder(1); builder.AddLong(0x1122334455667788); - Assert.ArrayEqual(new byte[] { 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11 }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -67,11 +67,11 @@ { var builder = new FlatBufferBuilder(1); builder.StartVector(sizeof(byte), 1, 1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.AddByte(1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.EndVector(); - Assert.ArrayEqual(new byte[] { 1, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 1, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -79,13 +79,13 @@ { var builder = new FlatBufferBuilder(1); builder.StartVector(sizeof(byte), 2, 1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.AddByte(1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 1, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 1, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.AddByte(2); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 2, 1, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 2, 1, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.EndVector(); - Assert.ArrayEqual(new byte[] { 2, 0, 0, 0, 2, 1, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 2, 0, 0, 0, 2, 1, 0, 0 }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -93,11 +93,11 @@ { var builder = new FlatBufferBuilder(1); builder.StartVector(sizeof(ushort), 1, 1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.AddUshort(1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.EndVector(); - Assert.ArrayEqual(new byte[] { 1, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 1, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -105,13 +105,13 @@ { var builder = new FlatBufferBuilder(1); builder.StartVector(sizeof(ushort), 2, 1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.AddUshort(0xABCD); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0xCD, 0xAB }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0xCD, 0xAB }, builder.DataBuffer.ToFullArray()); builder.AddUshort(0xDCBA); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB }, builder.DataBuffer.ToFullArray()); builder.EndVector(); - Assert.ArrayEqual(new byte[] { 2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -119,7 +119,7 @@ { var builder = new FlatBufferBuilder(1); builder.CreateString("foo"); - Assert.ArrayEqual(new byte[] { 3, 0, 0, 0, (byte)'f', (byte)'o', (byte)'o', 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 3, 0, 0, 0, (byte)'f', (byte)'o', (byte)'o', 0 }, builder.DataBuffer.ToFullArray()); builder.CreateString("moop"); Assert.ArrayEqual(new byte[] @@ -132,7 +132,7 @@ 0, 0, 0, 0, // zero terminator with 3 byte pad 3, 0, 0, 0, (byte)'f', (byte)'o', (byte)'o', 0 - }, builder.DataBuffer.Data); + }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -144,7 +144,7 @@ { 3, 0, 0, 0, 0x01, 0x02, 0x03, 0 - }, builder.DataBuffer.Data); // No padding + }, builder.DataBuffer.ToFullArray()); // No padding builder.CreateString("\x04\x05\x06\x07"); Assert.ArrayEqual(new byte[] { @@ -156,7 +156,7 @@ 0, 0, 0, 0, // zero terminator with 3 byte pad 3, 0, 0, 0, 0x01, 0x02, 0x03, 0 - }, builder.DataBuffer.Data); // No padding + }, builder.DataBuffer.ToFullArray()); // No padding } [FlatBuffersTestMethod] @@ -164,14 +164,14 @@ { var builder = new FlatBufferBuilder(1); builder.StartObject(0); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.EndObject(); Assert.ArrayEqual(new byte[] { 4, 0, 4, 0, 4, 0, 0, 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -179,7 +179,7 @@ { var builder = new FlatBufferBuilder(1); builder.StartObject(1); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddBool(0, true, false); builder.EndObject(); Assert.ArrayEqual(new byte[] @@ -192,7 +192,7 @@ 0, 0, 0, // padding 1, // value 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -200,7 +200,7 @@ { var builder = new FlatBufferBuilder(1); builder.StartObject(1); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddBool(0, false, false); builder.EndObject(); Assert.ArrayEqual(new byte[] @@ -211,7 +211,7 @@ // entry 0 is not stored (trimmed end of vtable) 4, 0, 0, 0, // int32 offset for start of vtable }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -219,7 +219,7 @@ { var builder = new FlatBufferBuilder(1); builder.StartObject(1); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddShort(0, 0x789A, 0); builder.EndObject(); Assert.ArrayEqual(new byte[] @@ -232,7 +232,7 @@ 0, 0, // padding 0x9A, 0x78, //value 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -240,7 +240,7 @@ { var builder = new FlatBufferBuilder(1); builder.StartObject(2); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddShort(0, 0x3456, 0); builder.AddShort(1, 0x789A, 0); builder.EndObject(); @@ -254,7 +254,7 @@ 0x9A, 0x78, // value 1 0x56, 0x34, // value 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -262,7 +262,7 @@ { var builder = new FlatBufferBuilder(1); builder.StartObject(2); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddShort(0, 0x3456, 0); builder.AddBool(1, true, false); builder.EndObject(); @@ -276,7 +276,7 @@ 0, 1, // padding + value 1 0x56, 0x34, // value 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -303,7 +303,7 @@ 4, 0, 0, 0, 0, 0, 0, 0, }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -330,7 +330,7 @@ 0, 0, 55, 0, // value 0 0, 0, 0, 0, // length of vector (not in sctruc) }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } @@ -361,7 +361,7 @@ 0x78, 0x56, // vector value 0 0x34, 0x12, // vector value 1 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -391,7 +391,7 @@ 0x00, 0x00, 0x34, 0x12, // struct value 1 0x00, 0x00, 0x00, 55, // struct value 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -425,7 +425,7 @@ 44, // vector 0, 1 33, // vector 0, 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -438,7 +438,7 @@ var off = builder.EndObject(); builder.Finish(off); - Assert.ArrayEqual(new byte[] + byte[] padded = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, @@ -452,8 +452,13 @@ 66, 0, // value 1 0, 33, // value 0 - }, - builder.DataBuffer.Data); + }; + Assert.ArrayEqual(padded, builder.DataBuffer.ToFullArray()); + + // no padding in sized array + byte[] unpadded = new byte[padded.Length - 12]; + Buffer.BlockCopy(padded, 12, unpadded, 0, unpadded.Length); + Assert.ArrayEqual(unpadded, builder.DataBuffer.ToSizedArray()); } [FlatBuffersTestMethod] @@ -504,7 +509,7 @@ 44, // value 1, 0 33, }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -519,7 +524,7 @@ var off = builder.EndObject(); builder.Finish(off); - Assert.ArrayEqual(new byte[] + byte[] padded = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, @@ -546,8 +551,61 @@ 1, 1, 1, 1, // values 1, 1, 1, 1, - }, - builder.DataBuffer.Data); + }; + Assert.ArrayEqual(padded, builder.DataBuffer.ToFullArray()); + + // no padding in sized array + byte[] unpadded = new byte[padded.Length - 28]; + Buffer.BlockCopy(padded, 28, unpadded, 0, unpadded.Length); + Assert.ArrayEqual(unpadded, builder.DataBuffer.ToSizedArray()); + } + + [FlatBuffersTestMethod] + public void TestBunchOfBoolsSizePrefixed() + { + var builder = new FlatBufferBuilder(1); + builder.StartObject(8); + for (var i = 0; i < 8; i++) + { + builder.AddBool(i, true, false); + } + var off = builder.EndObject(); + builder.FinishSizePrefixed(off); + + byte[] padded = new byte[] + { + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, // padding to 64 bytes + + 36, 0, 0, 0, // size prefix + 24, 0, 0, 0, // root of table, pointing to vtable offset (obj0) + 20, 0, // vtable bytes + 12, 0, // object length + 11, 0, // start of value 0 + 10, 0, // start of value 1 + 9, 0, // start of value 2 + 8, 0, // start of value 3 + 7, 0, // start of value 4 + 6, 0, // start of value 5 + 5, 0, // start of value 6 + 4, 0, // start of value 7 + + 20, 0, 0, 0, // int32 offset for start of vtable + + 1, 1, 1, 1, // values + 1, 1, 1, 1, + + }; + Assert.ArrayEqual(padded, builder.DataBuffer.ToFullArray()); + + // no padding in sized array + byte[] unpadded = new byte[padded.Length - 24]; + Buffer.BlockCopy(padded, 24, unpadded, 0, unpadded.Length); + Assert.ArrayEqual(unpadded, builder.DataBuffer.ToSizedArray()); } [FlatBuffersTestMethod] @@ -569,7 +627,7 @@ 0, 0, 128, 63, // value }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } private void CheckObjects(int fieldCount, int objectCount)
diff --git a/tests/FlatBuffers.Test/NetTest.sh b/tests/FlatBuffers.Test/NetTest.sh index ea16e47..6201549 100644 --- a/tests/FlatBuffers.Test/NetTest.sh +++ b/tests/FlatBuffers.Test/NetTest.sh
@@ -2,8 +2,22 @@ # Testing C# on Linux using Mono. -mcs -out:fbnettest.exe ../../net/FlatBuffers/*.cs ../MyGame/Example/*.cs FlatBuffersTestClassAttribute.cs FlatBuffersTestMethodAttribute.cs Assert.cs FlatBuffersExampleTests.cs Program.cs ByteBufferTests.cs FlatBufferBuilderTests.cs FlatBuffersFuzzTests.cs FuzzTestData.cs Lcg.cs TestTable.cs -./fbnettest.exe +mcs -debug -out:./fbnettest.exe \ + ../../net/FlatBuffers/*.cs ../MyGame/Example/*.cs ../MyGame/*.cs \ + FlatBuffersTestClassAttribute.cs FlatBuffersTestMethodAttribute.cs Assert.cs FlatBuffersExampleTests.cs Program.cs ByteBufferTests.cs FlatBufferBuilderTests.cs FlatBuffersFuzzTests.cs FuzzTestData.cs Lcg.cs TestTable.cs +mono --debug ./fbnettest.exe rm fbnettest.exe rm Resources/monsterdata_cstest.mon +rm Resources/monsterdata_cstest_sp.mon + +# Repeat with unsafe versions + +mcs -debug -out:./fbnettest.exe \ + -unsafe -d:UNSAFE_BYTEBUFFER \ + ../../net/FlatBuffers/*.cs ../MyGame/Example/*.cs ../MyGame/*.cs \ + FlatBuffersTestClassAttribute.cs FlatBuffersTestMethodAttribute.cs Assert.cs FlatBuffersExampleTests.cs Program.cs ByteBufferTests.cs FlatBufferBuilderTests.cs FlatBuffersFuzzTests.cs FuzzTestData.cs Lcg.cs TestTable.cs +mono --debug ./fbnettest.exe +rm fbnettest.exe +rm Resources/monsterdata_cstest.mon +rm Resources/monsterdata_cstest_sp.mon
diff --git a/tests/JavaTest.java b/tests/JavaTest.java index 9a5b676..a9dedf9 100644 --- a/tests/JavaTest.java +++ b/tests/JavaTest.java
@@ -21,6 +21,8 @@ import MyGame.Example.*; import NamespaceA.*; import NamespaceA.NamespaceB.*; +import com.google.flatbuffers.ByteBufferUtil; +import static com.google.flatbuffers.Constants.*; import com.google.flatbuffers.FlatBufferBuilder; class JavaTest { @@ -53,7 +55,8 @@ // better for performance. FlatBufferBuilder fbb = new FlatBufferBuilder(1); - TestBuilderBasics(fbb); + TestBuilderBasics(fbb, true); + TestBuilderBasics(fbb, false); TestExtendedBuffer(fbb.dataBuffer().asReadOnlyBuffer()); @@ -244,14 +247,14 @@ FlatBufferBuilder fbb = new FlatBufferBuilder(1, new MappedByteBufferFactory()); - TestBuilderBasics(fbb); + TestBuilderBasics(fbb, false); } static void TestSizedInputStream() { // Test on default FlatBufferBuilder that uses HeapByteBuffer FlatBufferBuilder fbb = new FlatBufferBuilder(1); - TestBuilderBasics(fbb); + TestBuilderBasics(fbb, false); InputStream in = fbb.sizedInputStream(); byte[] array = fbb.sizedByteArray(); @@ -271,7 +274,7 @@ TestEq(count, array.length); } - static void TestBuilderBasics(FlatBufferBuilder fbb) { + static void TestBuilderBasics(FlatBufferBuilder fbb, boolean sizePrefix) { int[] names = {fbb.createString("Frodo"), fbb.createString("Barney"), fbb.createString("Wilma")}; int[] off = new int[3]; Monster.startMonster(fbb); @@ -321,7 +324,11 @@ Monster.addTestarrayoftables(fbb, sortMons); int mon = Monster.endMonster(fbb); - Monster.finishMonsterBuffer(fbb, mon); + if (sizePrefix) { + Monster.finishSizePrefixedMonsterBuffer(fbb, mon); + } else { + Monster.finishMonsterBuffer(fbb, mon); + } // Write the result to a file for debugging purposes: // Note that the binaries are not necessarily identical, since the JSON @@ -329,7 +336,8 @@ // Java code. They are functionally equivalent though. try { - FileChannel fc = new FileOutputStream("monsterdata_java_wire.mon").getChannel(); + String filename = "monsterdata_java_wire" + (sizePrefix ? "_sp" : "") + ".mon"; + FileChannel fc = new FileOutputStream(filename).getChannel(); fc.write(fbb.dataBuffer().duplicate()); fc.close(); } catch(java.io.IOException e) { @@ -338,18 +346,24 @@ } // Test it: - TestExtendedBuffer(fbb.dataBuffer()); + ByteBuffer dataBuffer = fbb.dataBuffer(); + if (sizePrefix) { + TestEq(ByteBufferUtil.getSizePrefix(dataBuffer) + SIZE_PREFIX_LENGTH, + dataBuffer.remaining()); + dataBuffer = ByteBufferUtil.removeSizePrefix(dataBuffer); + } + TestExtendedBuffer(dataBuffer); // Make sure it also works with read only ByteBuffers. This is slower, // since creating strings incurs an additional copy // (see Table.__string). - TestExtendedBuffer(fbb.dataBuffer().asReadOnlyBuffer()); + TestExtendedBuffer(dataBuffer.asReadOnlyBuffer()); TestEnums(); //Attempt to mutate Monster fields and check whether the buffer has been mutated properly // revert to original values after testing - Monster monster = Monster.getRootAsMonster(fbb.dataBuffer()); + Monster monster = Monster.getRootAsMonster(dataBuffer); // mana is optional and does not exist in the buffer so the mutation should fail // the mana field should retain its default value
diff --git a/tests/MyGame/Example/Monster.cs b/tests/MyGame/Example/Monster.cs index c554c3c..b7844fb 100644 --- a/tests/MyGame/Example/Monster.cs +++ b/tests/MyGame/Example/Monster.cs
@@ -203,6 +203,7 @@ return new Offset<Monster>(o); } public static void FinishMonsterBuffer(FlatBufferBuilder builder, Offset<Monster> offset) { builder.Finish(offset.Value, "MONS"); } + public static void FinishSizePrefixedMonsterBuffer(FlatBufferBuilder builder, Offset<Monster> offset) { builder.FinishSizePrefixed(offset.Value, "MONS"); } public static VectorOffset CreateSortedVectorOfMonster(FlatBufferBuilder builder, Offset<Monster>[] offsets) { Array.Sort(offsets, (Offset<Monster> o1, Offset<Monster> o2) => Table.CompareStrings(Table.__offset(10, o1.Value, builder.DataBuffer), Table.__offset(10, o2.Value, builder.DataBuffer), builder.DataBuffer));
diff --git a/tests/MyGame/Example/Monster.java b/tests/MyGame/Example/Monster.java index 6048dc7..8a5bb0e 100644 --- a/tests/MyGame/Example/Monster.java +++ b/tests/MyGame/Example/Monster.java
@@ -228,6 +228,7 @@ return o; } public static void finishMonsterBuffer(FlatBufferBuilder builder, int offset) { builder.finish(offset, "MONS"); } + public static void finishSizePrefixedMonsterBuffer(FlatBufferBuilder builder, int offset) { builder.finishSizePrefixed(offset, "MONS"); } @Override protected int keysCompare(Integer o1, Integer o2, ByteBuffer _bb) { return compareStrings(__offset(10, o1, _bb), __offset(10, o2, _bb), _bb); }
diff --git a/tests/monster_test.bfbs b/tests/monster_test.bfbs index 0093def..7605728 100644 --- a/tests/monster_test.bfbs +++ b/tests/monster_test.bfbs Binary files differ
diff --git a/tests/namespace_test/namespace_test2_generated.ts b/tests/namespace_test/namespace_test2_generated.ts index 61cf631..aa623a8 100644 --- a/tests/namespace_test/namespace_test2_generated.ts +++ b/tests/namespace_test/namespace_test2_generated.ts
@@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -import * as NS39599748 from "./namespace_test1_generated"; +import * as NS4989953370203581498 from "./namespace_test1_generated"; /** * @constructor */ @@ -39,24 +39,24 @@ * @param {NamespaceA.NamespaceB.TableInNestedNS=} obj * @returns {NamespaceA.NamespaceB.TableInNestedNS|null} */ -fooTable(obj?:NS39599748.NamespaceA.NamespaceB.TableInNestedNS):NS39599748.NamespaceA.NamespaceB.TableInNestedNS|null { +fooTable(obj?:NS4989953370203581498.NamespaceA.NamespaceB.TableInNestedNS):NS4989953370203581498.NamespaceA.NamespaceB.TableInNestedNS|null { var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? (obj || new NS39599748.NamespaceA.NamespaceB.TableInNestedNS).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; + return offset ? (obj || new NS4989953370203581498.NamespaceA.NamespaceB.TableInNestedNS).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; }; /** * @returns {NamespaceA.NamespaceB.EnumInNestedNS} */ -fooEnum():NS39599748.NamespaceA.NamespaceB.EnumInNestedNS { +fooEnum():NS4989953370203581498.NamespaceA.NamespaceB.EnumInNestedNS { var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? /** @type {NamespaceA.NamespaceB.EnumInNestedNS} */ (this.bb!.readInt8(this.bb_pos + offset)) : NS39599748.NamespaceA.NamespaceB.EnumInNestedNS.A; + return offset ? /** @type {NamespaceA.NamespaceB.EnumInNestedNS} */ (this.bb!.readInt8(this.bb_pos + offset)) : NS4989953370203581498.NamespaceA.NamespaceB.EnumInNestedNS.A; }; /** * @param {NamespaceA.NamespaceB.EnumInNestedNS} value * @returns {boolean} */ -mutate_foo_enum(value:NS39599748.NamespaceA.NamespaceB.EnumInNestedNS):boolean { +mutate_foo_enum(value:NS4989953370203581498.NamespaceA.NamespaceB.EnumInNestedNS):boolean { var offset = this.bb!.__offset(this.bb_pos, 6); if (offset === 0) { @@ -71,9 +71,9 @@ * @param {NamespaceA.NamespaceB.StructInNestedNS=} obj * @returns {NamespaceA.NamespaceB.StructInNestedNS|null} */ -fooStruct(obj?:NS39599748.NamespaceA.NamespaceB.StructInNestedNS):NS39599748.NamespaceA.NamespaceB.StructInNestedNS|null { +fooStruct(obj?:NS4989953370203581498.NamespaceA.NamespaceB.StructInNestedNS):NS4989953370203581498.NamespaceA.NamespaceB.StructInNestedNS|null { var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? (obj || new NS39599748.NamespaceA.NamespaceB.StructInNestedNS).__init(this.bb_pos + offset, this.bb!) : null; + return offset ? (obj || new NS4989953370203581498.NamespaceA.NamespaceB.StructInNestedNS).__init(this.bb_pos + offset, this.bb!) : null; }; /** @@ -95,8 +95,8 @@ * @param {flatbuffers.Builder} builder * @param {NamespaceA.NamespaceB.EnumInNestedNS} fooEnum */ -static addFooEnum(builder:flatbuffers.Builder, fooEnum:NS39599748.NamespaceA.NamespaceB.EnumInNestedNS) { - builder.addFieldInt8(1, fooEnum, NS39599748.NamespaceA.NamespaceB.EnumInNestedNS.A); +static addFooEnum(builder:flatbuffers.Builder, fooEnum:NS4989953370203581498.NamespaceA.NamespaceB.EnumInNestedNS) { + builder.addFieldInt8(1, fooEnum, NS4989953370203581498.NamespaceA.NamespaceB.EnumInNestedNS.A); }; /**
diff --git a/tests/py_test.py b/tests/py_test.py index aeef729..1b76b61 100644 --- a/tests/py_test.py +++ b/tests/py_test.py
@@ -25,6 +25,7 @@ from flatbuffers import compat +from flatbuffers import util from flatbuffers.compat import range_func as compat_range from flatbuffers.compat import NumpyRequiredForThisFeature @@ -56,9 +57,11 @@ class TestWireFormat(unittest.TestCase): def test_wire_format(self): # Verify that using the generated Python code builds a buffer without - # returning errors, and is interpreted correctly: - gen_buf, gen_off = make_monster_from_generated_code() - CheckReadBuffer(gen_buf, gen_off) + # returning errors, and is interpreted correctly, for size prefixed + # representation and regular: + for sizePrefix in [True, False]: + gen_buf, gen_off = make_monster_from_generated_code(sizePrefix = sizePrefix) + CheckReadBuffer(gen_buf, gen_off, sizePrefix = sizePrefix) # Verify that the canonical flatbuffer file is readable by the # generated Python code. Note that context managers are not part of @@ -74,7 +77,7 @@ f.close() -def CheckReadBuffer(buf, offset): +def CheckReadBuffer(buf, offset, sizePrefix = False): ''' CheckReadBuffer checks that the given buffer is evaluated correctly as the example Monster. ''' @@ -83,6 +86,11 @@ if not stmt: raise AssertionError('CheckReadBuffer case failed') + if sizePrefix: + size = util.GetSizePrefix(buf, offset) + # taken from the size of monsterdata_python_wire.mon, minus 4 + asserter(size == 348) + buf, offset = util.RemoveSizePrefix(buf, offset) monster = MyGame.Example.Monster.Monster.GetRootAsMonster(buf, offset) asserter(monster.Hp() == 80) @@ -810,7 +818,7 @@ ]) -def make_monster_from_generated_code(): +def make_monster_from_generated_code(sizePrefix = False): ''' Use generated code to build the example Monster. ''' b = flatbuffers.Builder(0) @@ -871,7 +879,10 @@ MyGame.Example.Monster.MonsterAddVectorOfDoubles(b, VectorOfDoubles) mon = MyGame.Example.Monster.MonsterEnd(b) - b.Finish(mon) + if sizePrefix: + b.FinishSizePrefixed(mon) + else: + b.Finish(mon) return b.Bytes, b.Head()