Toolbox: Add data hashes to exported xml.gz

BililiveRecorder.Core/RecyclableMemoryStreamProvider.cs

@@ -28,6 +28,6 @@ namespace BililiveRecorder.Core
             //};
         }
 
-        public Stream CreateMemoryStream(string tag) => this.manager.GetStream(tag);
+        public MemoryStream CreateMemoryStream(string tag) => this.manager.GetStream(tag);
     }
 }

BililiveRecorder.Flv/BililiveRecorder.Flv.csproj

@@ -8,6 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="FastHashes" Version="2.4.0" />
     <PackageReference Include="JsonSubTypes" Version="1.8.0" />
     <PackageReference Include="Microsoft.Bcl.HashCode" Version="1.1.1" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="5.0.0" />

BililiveRecorder.Flv/DefaultMemoryStreamProvider.cs

@@ -4,6 +4,6 @@ namespace BililiveRecorder.Flv
 {
     public class DefaultMemoryStreamProvider : IMemoryStreamProvider
     {
-        public Stream CreateMemoryStream(string tag) => new MemoryStream();
+        public MemoryStream CreateMemoryStream(string tag) => new MemoryStream();
     }
 }

BililiveRecorder.Flv/H264Nalu.cs

@@ -84,6 +84,12 @@ namespace BililiveRecorder.Flv
         /// </summary>
         [XmlAttribute]
         public H264NaluType Type { get; set; }
+
+        /// <summary>
+        /// nal_unit data hash
+        /// </summary>
+        [XmlAttribute]
+        public string? NaluHash { get; set; }
     }
 
     /// <summary>

BililiveRecorder.Flv/IMemoryStreamProvider.cs

@@ -4,6 +4,6 @@ namespace BililiveRecorder.Flv
 {
     public interface IMemoryStreamProvider
     {
-        Stream CreateMemoryStream(string tag);
+        MemoryStream CreateMemoryStream(string tag);
     }
 }

BililiveRecorder.Flv/Tag.cs

@@ -3,8 +3,10 @@ using System.Collections.Generic;
 using System.ComponentModel;
 using System.Diagnostics;
 using System.IO;
+using System.Runtime.CompilerServices;
 using System.Xml.Serialization;
 using BililiveRecorder.Flv.Amf;
+using FastHashes;
 
 namespace BililiveRecorder.Flv
 {
@@ -26,8 +28,11 @@ namespace BililiveRecorder.Flv
         [XmlAttribute]
         public int Timestamp { get; set; }
 
+        [XmlAttribute]
+        public string? DataHash { get; set; }
+
         [XmlIgnore]
-        public Stream? BinaryData { get; set; }
+        public MemoryStream? BinaryData { get; set; }
 
         [XmlElement]
         public ScriptTagBody? ScriptData { get; set; }
@@ -43,7 +48,7 @@ namespace BililiveRecorder.Flv
                 : BinaryConvertUtilities.StreamToHexString(this.BinaryData);
             set
             {
-                Stream? new_stream = null;
+                MemoryStream? new_stream = null;
                 if (value != null)
                     new_stream = BinaryConvertUtilities.HexStringToMemoryStream(value);
 
@@ -52,10 +57,13 @@ namespace BililiveRecorder.Flv
             }
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public bool ShouldSerializeBinaryDataForSerializationUseOnly() => 0 != (this.Flag & TagFlag.Header);
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public bool ShouldSerializeScriptData() => this.Type == TagType.Script;
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public bool ShouldSerializeNalus() => this.Type == TagType.Video && (0 == (this.Flag & TagFlag.Header));
 
         public override string ToString() => this.DebuggerDisplay;
@@ -64,7 +72,7 @@ namespace BililiveRecorder.Flv
         public Tag Clone() => this.Clone(null);
         public Tag Clone(IMemoryStreamProvider? provider = null)
         {
-            Stream? binaryData = null;
+            MemoryStream? binaryData = null;
             if (this.BinaryData != null)
             {
                 binaryData = provider?.CreateMemoryStream(nameof(Tag) + ":" + nameof(Clone)) ?? new MemoryStream();
@@ -88,12 +96,37 @@ namespace BililiveRecorder.Flv
                 Size = this.Size,
                 Index = this.Index,
                 Timestamp = this.Timestamp,
+                DataHash = this.DataHash,
                 BinaryData = binaryData,
                 ScriptData = scriptData,
                 Nalus = this.Nalus is null ? null : new List<H264Nalu>(this.Nalus),
             };
         }
 
+        private static readonly FarmHash64 farmHash64 = new FarmHash64();
+
+        public string? UpdateDataHash()
+        {
+            if (this.BinaryData is null)
+            {
+                this.DataHash = null;
+            }
+            else
+            {
+                var buffer = this.BinaryData.GetBuffer();
+                this.DataHash = BinaryConvertUtilities.ByteArrayToHexString(farmHash64.ComputeHash(buffer, (int)this.BinaryData.Length));
+
+                if (this.Nalus?.Count > 0)
+                {
+                    foreach (var nalu in this.Nalus)
+                    {
+                        nalu.NaluHash = BinaryConvertUtilities.ByteArrayToHexString(farmHash64.ComputeHash(buffer, nalu.StartPosition, (int)nalu.FullSize));
+                    }
+                }
+            }
+            return this.DataHash;
+        }
+
         private string DebuggerDisplay => string.Format("{0}, {1}{2}{3}, TS={4}, Size={5}",
             this.Type switch
             {
@@ -158,7 +191,7 @@ namespace BililiveRecorder.Flv
                 return new string(result);
             }
 
-            internal static Stream HexStringToMemoryStream(string hex)
+            internal static MemoryStream HexStringToMemoryStream(string hex)
             {
                 var stream = new MemoryStream(hex.Length / 2);
                 for (var i = 0; i < hex.Length; i += 2)

BililiveRecorder.ToolBox/RecyclableMemoryStreamProvider.cs

@@ -28,6 +28,6 @@ namespace BililiveRecorder.ToolBox
             //};
         }
 
-        public Stream CreateMemoryStream(string tag) => this.manager.GetStream(tag);
+        public MemoryStream CreateMemoryStream(string tag) => this.manager.GetStream(tag);
     }
 }

BililiveRecorder.ToolBox/Tool/Export/ExportHandler.cs

@@ -66,11 +66,17 @@ namespace BililiveRecorder.ToolBox.Tool.Export
                     var count = 0;
                     var tags = new List<Tag>();
                     var memoryStreamProvider = new RecyclableMemoryStreamProvider();
-                    using var reader = new FlvTagPipeReader(PipeReader.Create(inputStream), memoryStreamProvider, skipData: true, logger: logger);
+                    using var reader = new FlvTagPipeReader(PipeReader.Create(inputStream), memoryStreamProvider, skipData: false, logger: logger);
                     while (!cancellationToken.IsCancellationRequested)
                     {
                         var tag = await reader.ReadTagAsync(cancellationToken).ConfigureAwait(false);
-                        if (tag is null) break;
+                        if (tag is null)
+                            break;
+
+                        tag.UpdateDataHash();
+                        if (!tag.ShouldSerializeBinaryDataForSerializationUseOnly())
+                            tag.BinaryData?.Dispose();
+
                         tags.Add(tag);
 
                         if (count++ % 300 == 0 && progress is not null)

test/BililiveRecorder.Flv.Tests/FlvTests/TagTests.cs

@@ -31,6 +31,7 @@ namespace BililiveRecorder.Flv.Tests.FlvTests
             Assert.Equal(source.Index, result.Index);
             Assert.Equal(source.Size, result.Size);
             Assert.Equal(source.Timestamp, result.Timestamp);
+            Assert.Equal(source.DataHash, result.DataHash);
 
             Assert.NotSame(source.BinaryData, result.BinaryData);
             Assert.Equal(source.BinaryDataForSerializationUseOnly, result.BinaryDataForSerializationUseOnly);

test/BililiveRecorder.Flv.Tests/TestRecyclableMemoryStreamProvider.cs

@@ -12,6 +12,6 @@ namespace BililiveRecorder.Flv.Tests
                 MaximumFreeLargePoolBytes = 64 * 1024 * 32,
             };
 
-        public Stream CreateMemoryStream(string tag) => manager.GetStream(tag);
+        public MemoryStream CreateMemoryStream(string tag) => manager.GetStream(tag);
     }
 }