update to 0.6.1

changes.txt

@@ -1,11 +1,16 @@
+  ES_R37 (0.6.2)
+  - added feature to inject libraries to main executable
+
+  ES_R36 (0.6.1)
+  - added fast lzma2 compression for portable mode
+  - fixed minor issue with stream deduplication feature
+
   ES_R35 (0.6.0)
   - added wav stream detector
   - added flac codec
   - added jpg stream detector
   - added packjpg, brunsli, jojpeg codec
-  - added fast lzma2 compression for portable mode
   - added feature that allows input to be a directory
-  - added feature to inject libraries to main executable
   - added feature to extract detected streams
   - updated database feature
   - updated deduplication feature
@@ -248,4 +253,4 @@
   - added grittibanzli (also handles deflate stream but slow af)
 
   2008_R1 (0.2.1)
-  - initial release
+  - initial release

common/Utils.pas

@@ -490,6 +490,8 @@ function GetCmdStr(CommandLine: String; Index: Integer;
   KeepQuotes: Boolean = False): string;
 function GetCmdCount(CommandLine: String): Integer;
 
+procedure UpdateFileResource(Source, Dest, ResName: string);
+
 implementation
 
 function GetBits(Data: Int64; Index: TInt64_BitIndex;
@@ -3615,4 +3617,41 @@ begin
     Inc(Result);
 end;
 
+procedure UpdateFileResource(Source, Dest, ResName: string);
+var
+  Stream: TFileStream;
+  hDestRes: THandle;
+  lpData: Pointer;
+  cbData: DWORD;
+begin
+  Stream := TFileStream.Create(Source, fmOpenRead or fmShareDenyNone);
+  try
+    Stream.Seek(0, soFromBeginning);
+    cbData := Stream.Size;
+    if cbData > 0 then
+    begin
+      GetMem(lpData, cbData);
+      try
+        Stream.Read(lpData^, cbData);
+        hDestRes := BeginUpdateResource(PChar(Dest), False);
+        if hDestRes <> 0 then
+          if UpdateResource(hDestRes, RT_RCDATA, PWideChar(ResName), 0, lpData,
+            cbData) then
+          begin
+            if not EndUpdateResource(hDestRes, False) then
+              RaiseLastOSError
+          end
+          else
+            RaiseLastOSError
+        else
+          RaiseLastOSError;
+      finally
+        FreeMem(lpData);
+      end;
+    end;
+  finally
+    Stream.Free;
+  end;
+end;
+
 end.

imports/FLZMA2DLL.pas

@@ -0,0 +1,312 @@
+unit FLZMA2DLL;
+
+interface
+
+uses
+  MemoryModule,
+  WinAPI.Windows,
+  System.SysUtils, System.Classes, System.Types;
+
+type
+  PFL2_inBuffer = ^FL2_inBuffer;
+
+  FL2_inBuffer = record
+    src: Pointer;
+    size: size_t;
+    pos: size_t;
+  end;
+
+  PFL2_outBuffer = ^FL2_outBuffer;
+
+  FL2_outBuffer = record
+    dst: Pointer;
+    size: size_t;
+    pos: size_t;
+  end;
+
+var
+  FL2_compress: function(dst: Pointer; dstCapacity: size_t; const src: Pointer;
+    srcSize: size_t; compressionLevel: Integer): size_t cdecl;
+  FL2_compressMt: function(dst: Pointer; dstCapacity: size_t;
+    const src: Pointer; srcSize: size_t; compressionLevel: Integer;
+    nbThreads: Cardinal): size_t cdecl;
+  FL2_decompress: function(dst: Pointer; dstCapacity: size_t;
+    const src: Pointer; srcSize: size_t): size_t cdecl;
+  FL2_decompressMt: function(dst: Pointer; dstCapacity: size_t;
+    const src: Pointer; srcSize: size_t; nbThreads: Cardinal): size_t cdecl;
+
+  FL2_createCCtx: function: Pointer cdecl;
+  FL2_createCCtxMt: function(nbThreads: Cardinal): Pointer cdecl;
+  FL2_freeCCtx: procedure(cctx: Pointer)cdecl;
+  FL2_compressCCtx: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
+    src: Pointer; srcSize: size_t; compressionLevel: Integer): size_t cdecl;
+  FL2_createDCtx: function: Pointer cdecl;
+  FL2_createDCtxMt: function(nbThreads: Cardinal): Pointer cdecl;
+  FL2_freeDCtx: function(dctx: Pointer): size_t cdecl;
+  FL2_decompressDCtx: function(dctx: Pointer; dst: Pointer; dstCapacity: size_t;
+    src: Pointer; srcSize: size_t): size_t cdecl;
+
+  FL2_createCStream: function: Pointer cdecl;
+  FL2_createCStreamMt: function(nbThreads: Cardinal; dualBuffer: Integer)
+    : Pointer cdecl;
+  FL2_freeCStream: procedure(fcs: Pointer)cdecl;
+  FL2_initCStream: function(fcs: Pointer; compressionLevel: Integer)
+    : size_t cdecl;
+  FL2_compressStream: function(fcs: Pointer; output: PFL2_outBuffer;
+    input: PFL2_inBuffer): size_t cdecl;
+
+  FL2_createDStream: function: Pointer cdecl;
+  FL2_createDStreamMt: function(nbThreads: Cardinal): Pointer cdecl;
+  FL2_freeDStream: procedure(fds: Pointer)cdecl;
+  FL2_initDStream: function(fds: Pointer): size_t cdecl;
+  FL2_decompressStream: function(fds: Pointer; output: PFL2_outBuffer;
+    input: PFL2_inBuffer): size_t cdecl;
+
+  FL2_endStream: function(fcs: Pointer; output: PFL2_outBuffer): size_t cdecl;
+  FL2_isError: function(code: size_t): Cardinal cdecl;
+  DLLLoaded: boolean = False;
+
+type
+  TLZMACRec = record
+    Threads: Integer;
+    Level: Integer;
+    procedure Parse(S: String);
+  end;
+
+  TLZMADRec = record
+    Threads: Integer;
+    procedure Parse(S: String);
+  end;
+
+  TLZMACompressStream = class(TStream)
+  private const
+    FBufferSize = 65536;
+  private
+    FCtx: Pointer;
+    FProp: TLZMACRec;
+    FOutput: TStream;
+    FBuffer: array [0 .. FBufferSize - 1] of Byte;
+  public
+    constructor Create(AOutput: TStream; AConfig: String = 't50p');
+    destructor Destroy; override;
+    function Write(const Buffer; Count: LongInt): LongInt; override;
+  end;
+
+  TLZMADecompressStream = class(TStream)
+  private const
+    FBufferSize = 65536;
+  private
+    FCtx: Pointer;
+    FProp: TLZMADRec;
+    FInp: FL2_inBuffer;
+    FInput: TStream;
+    FBuffer: array [0 .. FBufferSize - 1] of Byte;
+  public
+    constructor Create(AInput: TStream; AConfig: String = '');
+    destructor Destroy; override;
+    function Read(var Buffer; Count: Integer): Integer; override;
+  end;
+
+implementation
+
+uses
+  Utils;
+
+var
+  DLLStream: TResourceStream;
+  DLLHandle: TMemoryModule;
+
+procedure TLZMACRec.Parse(S: string);
+var
+  List: TStringDynArray;
+  I, J: Integer;
+begin
+  Threads := 1;
+  Level := 5;
+  List := DecodeStr(S, ':');
+  for I := Low(List) to High(List) do
+  begin
+    if List[I].StartsWith('t', True) then
+      Threads := ConvertToThreads(List[I].Substring(1));
+    if List[I].StartsWith('l', True) then
+      Level := List[I].Substring(1).ToInteger;
+  end;
+end;
+
+procedure TLZMADRec.Parse(S: string);
+var
+  List: TStringDynArray;
+  I: Integer;
+begin
+  Threads := 1;
+  List := DecodeStr(S, ':');
+  for I := Low(List) to High(List) do
+  begin
+    if List[I].StartsWith('t', True) then
+      Threads := ConvertToThreads(List[I].Substring(1));
+  end;
+end;
+
+constructor TLZMACompressStream.Create(AOutput: TStream; AConfig: String);
+begin
+  inherited Create;
+  FProp.Parse(AConfig);
+  FOutput := AOutput;
+  if FProp.Threads > 1 then
+    FCtx := FL2_createCStreamMt(FProp.Threads, 0)
+  else
+    FCtx := FL2_createCStream;
+  FL2_initCStream(FCtx, FProp.Level);
+end;
+
+destructor TLZMACompressStream.Destroy;
+var
+  Oup: FL2_outBuffer;
+  Res: size_t;
+begin
+  Oup.dst := @FBuffer[0];
+  Oup.size := FBufferSize;
+  Oup.pos := 0;
+  repeat
+    Res := FL2_endStream(FCtx, @Oup);
+    FOutput.WriteBuffer(FBuffer[0], Oup.pos);
+    Oup.pos := 0;
+  until Res = 0;
+  FL2_freeCCtx(FCtx);
+  inherited Destroy;
+end;
+
+function TLZMACompressStream.Write(const Buffer; Count: LongInt): LongInt;
+var
+  Inp: FL2_inBuffer;
+  Oup: FL2_outBuffer;
+begin
+  Result := 0;
+  Inp.src := PByte(@Buffer);
+  Inp.size := Count;
+  Inp.pos := 0;
+  Oup.dst := @FBuffer[0];
+  Oup.size := FBufferSize;
+  Oup.pos := 0;
+  while Inp.pos < Inp.size do
+  begin
+    if not boolean(FL2_isError(FL2_compressStream(FCtx, @Oup, @Inp))) then
+    begin
+      FOutput.WriteBuffer(FBuffer[0], Oup.pos);
+      Oup.pos := 0;
+    end;
+  end;
+  Result := Inp.pos;
+end;
+
+constructor TLZMADecompressStream.Create(AInput: TStream; AConfig: String);
+begin
+  inherited Create;
+  FProp.Parse(AConfig);
+  FInput := AInput;
+  if FProp.Threads > 1 then
+    FCtx := FL2_createDStreamMt(FProp.Threads)
+  else
+    FCtx := FL2_createDStream;
+  FL2_initDStream(FCtx);
+  FillChar(FInp, SizeOf(FL2_inBuffer), 0);
+end;
+
+destructor TLZMADecompressStream.Destroy;
+begin
+  FL2_freeDCtx(FCtx);
+  inherited Destroy;
+end;
+
+function TLZMADecompressStream.Read(var Buffer; Count: Integer): Integer;
+var
+  Oup: FL2_outBuffer;
+begin
+  Result := 0;
+  if FInp.pos = FInp.size then
+  begin
+    FInp.src := @FBuffer[0];
+    FInp.size := FInput.Read(FBuffer[0], FBufferSize);
+    FInp.pos := 0;
+    if FInp.size = 0 then
+      exit;
+  end;
+  Oup.dst := PByte(@Buffer);
+  Oup.size := Count;
+  Oup.pos := 0;
+  while Oup.pos < Oup.size do
+  begin
+    if not boolean(FL2_isError(FL2_decompressStream(FCtx, @Oup, @FInp))) then
+    begin
+      if FInp.pos = FInp.size then
+      begin
+        FInp.src := @FBuffer[0];
+        FInp.size := FInput.Read(FBuffer[0], FBufferSize);
+        FInp.pos := 0;
+        if FInp.size = 0 then
+          break;
+      end;
+    end
+    else
+      break;
+  end;
+  Result := Oup.pos;
+end;
+
+procedure Init;
+begin
+  DLLStream := TResourceStream.Create(HInstance, 'fast_lzma2', RT_RCDATA);
+  DLLHandle := MemoryLoadLibary(DLLStream.Memory);
+  if Assigned(DLLHandle) then
+  begin
+    @FL2_compress := MemoryGetProcAddress(DLLHandle, 'FL2_compress');
+    @FL2_compressMt := MemoryGetProcAddress(DLLHandle, 'FL2_compressMt');
+    @FL2_decompress := MemoryGetProcAddress(DLLHandle, 'FL2_decompress');
+    @FL2_decompressMt := MemoryGetProcAddress(DLLHandle, 'FL2_decompressMt');
+    @FL2_createCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_createCCtx');
+    @FL2_createCCtxMt := MemoryGetProcAddress(DLLHandle, 'FL2_createCCtxMt');
+    @FL2_freeCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_freeCCtx');
+    @FL2_compressCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_compressCCtx');
+    @FL2_createDCtx := MemoryGetProcAddress(DLLHandle, 'FL2_createDCtx');
+    @FL2_createDCtxMt := MemoryGetProcAddress(DLLHandle, 'FL2_createDCtxMt');
+    @FL2_freeDCtx := MemoryGetProcAddress(DLLHandle, 'FL2_freeDCtx');
+    @FL2_decompressDCtx := MemoryGetProcAddress(DLLHandle,
+      'FL2_decompressDCtx');
+    @FL2_createCStream := MemoryGetProcAddress(DLLHandle, 'FL2_createCStream');
+    @FL2_createCStreamMt := MemoryGetProcAddress(DLLHandle,
+      'FL2_createCStreamMt');
+    @FL2_freeCStream := MemoryGetProcAddress(DLLHandle, 'FL2_freeCStream');
+    @FL2_initCStream := MemoryGetProcAddress(DLLHandle, 'FL2_initCStream');
+    @FL2_compressStream := MemoryGetProcAddress(DLLHandle,
+      'FL2_compressStream');
+    @FL2_createDStream := MemoryGetProcAddress(DLLHandle, 'FL2_createDStream');
+    @FL2_createDStreamMt := MemoryGetProcAddress(DLLHandle,
+      'FL2_createDStreamMt');
+    @FL2_freeDStream := MemoryGetProcAddress(DLLHandle, 'FL2_freeDStream');
+    @FL2_initDStream := MemoryGetProcAddress(DLLHandle, 'FL2_initDStream');
+    @FL2_decompressStream := MemoryGetProcAddress(DLLHandle,
+      'FL2_decompressStream');
+    @FL2_endStream := MemoryGetProcAddress(DLLHandle, 'FL2_endStream');
+    @FL2_isError := MemoryGetProcAddress(DLLHandle, 'FL2_isError');
+    DLLLoaded := Assigned(FL2_compress) and Assigned(FL2_decompress);
+  end
+  else
+    DLLLoaded := False;
+end;
+
+procedure Deinit;
+begin
+  if not DLLLoaded then
+    exit;
+  MemoryFreeLibrary(DLLHandle);
+end;
+
+initialization
+
+Init;
+
+finalization
+
+Deinit;
+
+end.

imports/LZMADLL.pas

@@ -1,28 +0,0 @@
-unit LZMADLL;
-
-interface
-
-uses
-  WinAPI.Windows,
-  System.SysUtils;
-
-type
-  PFL2_inBuffer = ^FL2_inBuffer;
-
-  FL2_inBuffer = record
-    src: Pointer;
-    size: size_t;
-    pos: size_t;
-  end;
-
-  PFL2_outBuffer = ^FL2_outBuffer;
-
-  FL2_outBuffer = record
-    dst: Pointer;
-    size: size_t;
-    pos: size_t;
-  end;
-
-implementation
-
-end.

imports/XDeltaDLL.pas

@@ -46,11 +46,7 @@ var
 
 procedure Init;
 begin
-{$IFDEF WIN32}
+  DLLStream := TResourceStream.Create(HInstance, 'xdelta3_dll', RT_RCDATA);
-  DLLStream := TResourceStream.Create(HInstance, 'XDELTA86_DLL', RT_RCDATA);
-{$ELSE}
-  DLLStream := TResourceStream.Create(HInstance, 'XDELTA64_DLL', RT_RCDATA);
-{$ENDIF}
   DLLHandle := MemoryLoadLibary(DLLStream.Memory);
   if Assigned(DLLHandle) then
   begin

imports/ZSTDDLL.pas

@@ -9,10 +9,11 @@ uses
 type
   ZSTD_strategy = (ZSTD_fast = 1, ZSTD_dfast = 2, ZSTD_greedy = 3,
     ZSTD_lazy = 4, ZSTD_lazy2 = 5, ZSTD_btlazy2 = 6, ZSTD_btopt = 7,
-    ZSTD_btultra = 8, ZSTD_btultra2 = 9);
+    ZSTD_btultra = 8, ZSTD_btultra2 = 9, ZSTD_strategy_Force32 = $40000000);
 
   ZSTD_ResetDirective = (ZSTD_reset_session_only = 1, ZSTD_reset_parameters = 2,
-    ZSTD_reset_session_and_parameters = 3);
+    ZSTD_reset_session_and_parameters = 3,
+    ZSTD_ResetDirective_Force32 = $40000000);
 
   PZSTD_inBuffer = ^ZSTD_inBuffer;
 
@@ -47,7 +48,7 @@ type
     ZSTD_c_experimentalParam9 = 1006, ZSTD_c_experimentalParam10 = 1007,
     ZSTD_c_experimentalParam11 = 1008, ZSTD_c_experimentalParam12 = 1009,
     ZSTD_c_experimentalParam13 = 1010, ZSTD_c_experimentalParam14 = 1011,
-    ZSTD_c_experimentalParam15 = 1012);
+    ZSTD_c_experimentalParam15 = 1012, ZSTD_cParameter_Force32 = $40000000);
 
   ZSTD_compressionParameters = record
     windowLog: Cardinal;
@@ -73,6 +74,8 @@ type
 var
   ZSTD_compress: function(dst: Pointer; dstCapacity: size_t; const src: Pointer;
     srcSize: size_t; compressionLevel: Integer): size_t cdecl;
+  ZSTD_compress2: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
+    const src: Pointer; srcSize: size_t): size_t cdecl;
   ZSTD_decompress: function(dst: Pointer; dstCapacity: size_t;
     const src: Pointer; srcSize: size_t): SSIZE_T cdecl;
   ZSTD_findFrameCompressedSize: function(const src: Pointer; srcSize: size_t)
@@ -81,6 +84,10 @@ var
     : int64 cdecl;
   ZSTD_createCCtx: function: Pointer cdecl;
   ZSTD_freeCCtx: function(cctx: Pointer): size_t cdecl;
+  ZSTD_CCtx_reset: function(cctx: Pointer; reset: ZSTD_ResetDirective)
+    : size_t cdecl;
+  ZSTD_CCtx_setParameter: function(cctx: Pointer; param: ZSTD_cParameter;
+    value: Integer): size_t cdecl;
   ZSTD_compressCCtx: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
     src: Pointer; srcSize: size_t; compressionLevel: Integer): size_t cdecl;
   ZSTD_createDCtx: function: Pointer cdecl;
@@ -99,6 +106,13 @@ var
   ZSTD_decompress_usingDDict: function(dctx: Pointer; dst: Pointer;
     dstCapacity: size_t; const src: Pointer; srcSize: size_t;
     const ddict: Pointer): size_t cdecl;
+  ZSTD_initCStream: function(zcs: Pointer; compressionLevel: Integer)
+    : size_t cdecl;
+  ZSTD_compressStream: function(zcs: Pointer; output: PZSTD_outBuffer;
+    input: PZSTD_inBuffer): size_t cdecl;
+  ZSTD_flushStream: function(zcs: Pointer; output: PZSTD_outBuffer)
+    : size_t cdecl;
+  ZSTD_endStream: function(zcs: Pointer; output: PZSTD_outBuffer): size_t cdecl;
 
   DLLLoaded: Boolean = False;
 
@@ -138,6 +152,7 @@ begin
   if DLLHandle >= 32 then
   begin
     @ZSTD_compress := GetProcAddress(DLLHandle, 'ZSTD_compress');
+    @ZSTD_compress2 := GetProcAddress(DLLHandle, 'ZSTD_compress2');
     @ZSTD_decompress := GetProcAddress(DLLHandle, 'ZSTD_decompress');
     @ZSTD_findFrameCompressedSize := GetProcAddress(DLLHandle,
       'ZSTD_findFrameCompressedSize');
@@ -145,6 +160,9 @@ begin
       'ZSTD_findDecompressedSize');
     @ZSTD_createCCtx := GetProcAddress(DLLHandle, 'ZSTD_createCCtx');
     @ZSTD_freeCCtx := GetProcAddress(DLLHandle, 'ZSTD_freeCCtx');
+    @ZSTD_CCtx_reset := GetProcAddress(DLLHandle, 'ZSTD_CCtx_reset');
+    @ZSTD_CCtx_setParameter := GetProcAddress(DLLHandle,
+      'ZSTD_CCtx_setParameter');
     @ZSTD_createDCtx := GetProcAddress(DLLHandle, 'ZSTD_createDCtx');
     @ZSTD_freeDCtx := GetProcAddress(DLLHandle, 'ZSTD_freeDCtx');
     @ZSTD_createCDict := GetProcAddress(DLLHandle, 'ZSTD_createCDict');
@@ -157,6 +175,10 @@ begin
       'ZSTD_compress_usingCDict');
     @ZSTD_decompress_usingDDict := GetProcAddress(DLLHandle,
       'ZSTD_decompress_usingDDict');
+    @ZSTD_initCStream := GetProcAddress(DLLHandle, 'ZSTD_initCStream');
+    @ZSTD_compressStream := GetProcAddress(DLLHandle, 'ZSTD_compressStream');
+    @ZSTD_flushStream := GetProcAddress(DLLHandle, 'ZSTD_flushStream');
+    @ZSTD_endStream := GetProcAddress(DLLHandle, 'ZSTD_endStream');
     DLLLoaded := Assigned(ZSTD_compress) and Assigned(ZSTD_decompress);
   end
   else

precompressor/PrecompLZ4.pas

@@ -266,21 +266,32 @@ begin
   SOList[Instance][X].Index := 0;
   while SOList[Instance][X].Get(I) >= 0 do
   begin
-    if StreamInfo^.Status = TStreamStatus.Predicted then
+    if StreamInfo^.Status >= TStreamStatus.Predicted then
+    begin
       if GetBits(StreamInfo^.Option, 5, 7) <> I then
         continue;
+      if (StreamInfo^.Status = TStreamStatus.Database) and
+        (GetBits(StreamInfo^.Option, 1, 31) = 0) then
+      begin
+        Res1 := StreamInfo^.OldSize;
+        Result := True;
+      end;
+    end;
     Params := '';
     case X of
       LZ4_CODEC:
         begin
           Params := 'a' + GetBits(StreamInfo^.Option, 15, 7).ToString;
-          Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y,
+          if not Result then
-            GetBits(StreamInfo^.Option, 15, 7));
+            Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y,
+              GetBits(StreamInfo^.Option, 15, 7));
         end;
       LZ4HC_CODEC:
         begin
           Params := 'l' + I.ToString;
-          Res1 := LZ4_compress_HC(NewInput, Buffer, StreamInfo^.NewSize, Y, I);
+          if not Result then
+            Res1 := LZ4_compress_HC(NewInput, Buffer,
+              StreamInfo^.NewSize, Y, I);
         end;
       LZ4F_CODEC:
         begin
@@ -293,18 +304,20 @@ begin
           Params := 'l' + I.ToString + ':' + 'b' +
             (GetBits(StreamInfo^.Option, 12, 2) + 4).ToString + ':' + 'd' +
             GetBits(StreamInfo^.Option, 14, 1).ToString;
-          Res1 := LZ4F_compressFrame(Buffer, Y, NewInput,
+          if not Result then
-            StreamInfo^.NewSize, @LZ4FT);
+            Res1 := LZ4F_compressFrame(Buffer, Y, NewInput,
+              StreamInfo^.NewSize, @LZ4FT);
         end;
     end;
-    Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+    if not Result then
-      StreamInfo^.OldSize);
+      Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+        StreamInfo^.OldSize);
     Funcs^.LogProcess(LZ4Codecs[GetBits(StreamInfo^.Option, 0, 5)],
       PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
     if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
       break;
   end;
-  if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
+  if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
     (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
   begin
     Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

precompressor/PrecompLZO.pas

@@ -298,9 +298,17 @@ begin
   SOList[Instance][X].Index := 0;
   while SOList[Instance][X].Get(I) >= 0 do
   begin
-    if StreamInfo^.Status = TStreamStatus.Predicted then
+    if StreamInfo^.Status >= TStreamStatus.Predicted then
+    begin
       if GetBits(StreamInfo^.Option, 5, 7) <> I then
         continue;
+      if (StreamInfo^.Status = TStreamStatus.Database) and
+        (GetBits(StreamInfo^.Option, 1, 31) = 0) then
+      begin
+        Res1 := StreamInfo^.OldSize;
+        Result := True;
+      end;
+    end;
     Params := '';
     Res1 := StreamInfo^.NewSize;
     case X of
@@ -310,23 +318,25 @@ begin
             begin
               Params := 'l' + I.ToString + ':' + 'v' +
                 GetBits(StreamInfo^.Option, 12, 5).ToString;
-              if not lzo1x_999_compress_level(NewInput, StreamInfo^.NewSize,
+              if not Result then
-                Buffer, @Res1, @WrkMem[Instance, 0], nil, 0, nil, I) = 0 then
+                if not lzo1x_999_compress_level(NewInput, StreamInfo^.NewSize,
-                Res1 := 0;
+                  Buffer, @Res1, @WrkMem[Instance, 0], nil, 0, nil, I) = 0 then
+                  Res1 := 0;
             end;
           { if not lzo1x_1_compress(NewInput, StreamInfo^.NewSize, Buffer,
             @Res1, @WrkMem[Instance, 0]) = 0 then
             Res1 := 0; }
         end;
     end;
-    Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+    if not Result then
-      StreamInfo^.OldSize);
+      Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+        StreamInfo^.OldSize);
     Funcs^.LogProcess(LZOCodecs[GetBits(StreamInfo^.Option, 0, 5)],
       PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
     if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
       break;
   end;
-  if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
+  if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
     (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
   begin
     Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

precompressor/PrecompMain.pas

@@ -5,7 +5,7 @@ unit PrecompMain;
 interface
 
 uses
-  Threading, Utils, SynCommons, ParseClass, ParseExpr,
+  Threading, Utils, SynCommons, ParseClass, ParseExpr, FLZMA2DLL,
   PrecompUtils, PrecompCrypto, PrecompZLib, PrecompLZ4, PrecompLZO, PrecompZSTD,
   PrecompOodle, PrecompMedia, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE,
   WinAPI.Windows, WinAPI.ShlObj,
@@ -25,6 +25,8 @@ type
     Depth: Integer;
     LowMem: Boolean;
     DBaseFile, ExtractDir: String;
+    DoCompress: Boolean;
+    CompressCfg: String;
   end;
 
   PDecodeOptions = ^TDecodeOptions;
@@ -34,6 +36,7 @@ type
     ChunkCount, Threads: Integer;
     Depth: Integer;
     DedupSysMem, DedupGPUMem: Int64;
+    CompressCfg: String;
   end;
 
 procedure PrintHelp;
@@ -122,7 +125,7 @@ begin
   WriteLn(ErrOutput, '');
   WriteLn(ErrOutput, 'Parameters:');
   WriteLn(ErrOutput,
-    '  -m#  - codecs to use for precompression (separate by "+" if more than one)');
+    '  -m#  - codecs to use for precompression (separate with "+" if more than one)');
   WriteLn(ErrOutput, '  -c#  - scanning range of precompressor [16mb]');
   WriteLn(ErrOutput, '  -t#  - number of working threads [50p]');
   WriteLn(ErrOutput, '  -lm  - low memory mode');
@@ -131,13 +134,16 @@ begin
   WriteLn(ErrOutput, 'Advanced parameters:');
   WriteLn(ErrOutput,
     '  --dbase=#   - use database (#=filename to save db, optional)');
-  WriteLn(ErrOutput,
+  WriteLn(ErrOutput, '  --dedup   - use stream deduplication');
-    '  --dedup=#   - use stream deduplication (#=filename to save db, optional)');
   WriteLn(ErrOutput,
     '  --mem=#     - deduplication ram usage limit (#=size) [75p]');
   WriteLn(ErrOutput,
     '  --diff=#    - set xdelta threshold to accept streams [5p]');
   WriteLn(ErrOutput, '  --extract=# - extract streams to directory path');
+  WriteLn(ErrOutput,
+    '  --compress=# - compress data using fast lzma2 (separate params with ":"');
+  WriteLn(ErrOutput, '               l# - compression level [5]');
+  WriteLn(ErrOutput, '               t# - number of threads [50p]');
   WriteLn(ErrOutput, '');
 end;
 
@@ -191,6 +197,12 @@ begin
     Options.ExtractDir := ArgParse.AsString('--extract=');
     if Options.ExtractDir <> '' then
       EXTRACT := DirectoryExists(Options.ExtractDir);
+    Options.DoCompress := ArgParse.AsBoolean('--compress');
+    S := ArgParse.AsString('--compress=');
+    S := ReplaceText(S, SPrecompSep3, SPrecompSep2);
+    Options.CompressCfg := S;
+    if Options.CompressCfg <> '' then
+      Options.DoCompress := True;
   finally
     ArgParse.Free;
     ExpParse.Free;
@@ -228,6 +240,9 @@ begin
     if B then
       Options.DedupSysMem := -Options.DedupSysMem;
     VERBOSE := ArgParse.AsBoolean('--verbose');
+    S := ArgParse.AsString('--compress=', 0, 't50p');
+    S := ReplaceText(S, SPrecompSep3, SPrecompSep2);
+    Options.CompressCfg := S;
   finally
     ArgParse.Free;
     ExpParse.Free;
@@ -1905,8 +1920,8 @@ begin
   for I := Low(ThreadSync) to High(ThreadSync) do
     ThreadSync[I] := TCriticalSection.Create;
   DupSysMem := Options^.DedupSysMem;
-  NStream.Add(TypeInfo(TMemoryStream) { , CalcSysMem } );
+  NStream.Add(TypeInfo(TMemoryStream), CalcSysMem);
-  // NStream.Add(TypeInfo(TPrecompVMStream));
+  NStream.Add(TypeInfo(TPrecompVMStream));
   Input.ReadBuffer(Options^.Depth, Options^.Depth.Size);
   Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size);
   SetLength(Bytes, LongRec(I).Bytes[0]);
@@ -1975,19 +1990,8 @@ begin
         MemOutput2[I] := TMemoryStream.Create;
       end;
     end;
-  Input.ReadBuffer(StoreDD, StoreDD.Size);
-  UI32 := 0;
-  if StoreDD then
-  begin
-    Input.ReadBuffer(UI32, UI32.Size);
-    SetLength(DDList2, UI32);
-    DDCount2 := UI32;
-    for I := Low(DDList2) to High(DDList2) do
-      Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
-    DDIndex1 := -1;
-    DDIndex2 := 0;
-  end;
   DataMgr := TDataManager.Create(NStream);
+  Input.ReadBuffer(StoreDD, StoreDD.Size);
 end;
 
 procedure DecFree;
@@ -2037,7 +2041,20 @@ var
   I, J: Integer;
 begin
   if Depth = 0 then
+  begin
+    UI32 := 0;
+    if StoreDD then
+    begin
+      Input.ReadBuffer(UI32, UI32.Size);
+      SetLength(DDList2, UI32);
+      DDCount2 := UI32;
+      for I := Low(DDList2) to High(DDList2) do
+        Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
+      DDIndex1 := -1;
+      DDIndex2 := 0;
+    end;
     LogInt64 := 0;
+  end;
   with ComVars2[Depth] do
   begin
     DecInput[Index] := Input;
@@ -2252,6 +2269,9 @@ begin
 end;
 
 procedure Encode(Input, Output: TStream; Options: TEncodeOptions);
+var
+  Compressed: Boolean;
+  LOutput: TStream;
 begin
   InternalSync.Enter;
   FillChar(EncInfo, SizeOf(EncInfo), 0);
@@ -2263,8 +2283,16 @@ begin
     ConTask.Start;
   try
     EncInit(Input, Output, @Options);
-    EncData(Input, Output, 0, 0);
+    Compressed := Options.DoCompress;
+    Output.WriteBuffer(Compressed, Compressed.Size);
+    if Options.DoCompress then
+      LOutput := TLZMACompressStream.Create(Output, Options.CompressCfg)
+    else
+      LOutput := Output;
+    EncData(Input, LOutput, 0, 0);
   finally
+    if Options.DoCompress then
+      LOutput.Free;
     try
       EncFree;
     finally
@@ -2279,6 +2307,9 @@ begin
 end;
 
 procedure Decode(Input, Output: TStream; Options: TDecodeOptions);
+var
+  Compressed: Boolean;
+  LInput: TStream;
 begin
   InternalSync.Enter;
   FillChar(EncInfo, SizeOf(EncInfo), 0);
@@ -2291,8 +2322,15 @@ begin
   NStream := TArrayStream.Create;
   try
     DecInit(Input, Output, @Options);
-    DecChunk(Input, Output, 0, 0);
+    Input.ReadBuffer(Compressed, Compressed.Size);
+    if Compressed then
+      LInput := TLZMADecompressStream.Create(Input, Options.CompressCfg)
+    else
+      LInput := Input;
+    DecChunk(LInput, Output, 0, 0);
   finally
+    if Compressed then
+      LInput.Free;
     try
       NStream.Free;
       DecFree;

precompressor/PrecompMedia.pas

@@ -140,9 +140,10 @@ begin
     end
     else if subchunk_hdr.subchunk_id = data_SIGN then
     begin
-      Result := True;
+      chunk_size^ := Min(subchunk_hdr.subchunk_size,
-      chunk_size^ := subchunk_hdr.subchunk_size;
+        (riffhdr.chunk_size + 8 - Pos));
       header_size^ := Pos;
+      Result := chunk_size^ + header_size^ <= InSize;
       exit;
     end
     else
@@ -750,7 +751,6 @@ begin
           StreamInfo^.NewSize := Res;
           Result := True;
         end;
-        ShowMessage('');
       end;
     BRUNSLI_CODEC:
       begin

precompressor/PrecompOodle.pas

@@ -711,25 +711,35 @@ begin
   SOList[Instance][X].Index := 0;
   while SOList[Instance][X].Get(I) >= 0 do
   begin
-    if StreamInfo^.Status = TStreamStatus.Predicted then
+    if StreamInfo^.Status >= TStreamStatus.Predicted then
+    begin
       if GetBits(StreamInfo^.Option, 5, 7) <> I then
         continue;
+      if (StreamInfo^.Status = TStreamStatus.Database) and
+        (GetBits(StreamInfo^.Option, 1, 31) = 0) then
+      begin
+        Res1 := StreamInfo^.OldSize;
+        Result := True;
+      end;
+    end;
     Move(OodleLZ_CompressOptions_GetDefault(Y, I)^, COptions,
       SizeOf(TOodleLZ_CompressOptions));
     COptions.sendQuantumCRCs := GetBits(StreamInfo^.Option, 12, 1) = 1;
     COptions.spaceSpeedTradeoffBytes := GetBits(StreamInfo^.Option, 13, 11);
     Params := 'l' + I.ToString + ':' + 'c' + GetBits(StreamInfo^.Option, 12, 1)
       .ToString + ':' + 't' + GetBits(StreamInfo^.Option, 13, 11).ToString;
-    Res1 := OodleLZ_Compress(Y, NewInput, StreamInfo^.NewSize, Buffer, I,
+    if not Result then
-      @COptions);
+      Res1 := OodleLZ_Compress(Y, NewInput, StreamInfo^.NewSize, Buffer, I,
-    Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+        @COptions);
-      StreamInfo^.OldSize);
+    if not Result then
+      Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+        StreamInfo^.OldSize);
     Funcs^.LogProcess(OodleCodecs[GetBits(StreamInfo^.Option, 0, 5)],
       PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
     if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
       break;
   end;
-  if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
+  if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
     (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
   begin
     Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

precompressor/PrecompUtils.pas

@@ -16,6 +16,7 @@ resourcestring
   SPrecompSep2 = ':';
   SPrecompSep3 = ',';
   SPrecompSep4 = '/';
+  SPrecompSep5 = '/';
 
 const
   SuccessStatus = 4;

precompressor/PrecompZSTD.pas

@@ -250,9 +250,9 @@ var
   X: Integer;
   Res1: Integer;
   Res2: NativeUInt;
-  // Inp: ZSTD_inBuffer;
+  Inp: ZSTD_inBuffer;
-  // Oup: ZSTD_outBuffer;
+  Oup: ZSTD_outBuffer;
-  // Progress: NativeInt;
+  Progress: NativeInt;
 begin
   Result := False;
   X := GetBits(StreamInfo^.Option, 0, 5);
@@ -262,29 +262,43 @@ begin
   SOList[Instance][X].Index := 0;
   while SOList[Instance][X].Get(I) >= 0 do
   begin
-    if StreamInfo^.Status = TStreamStatus.Predicted then
+    if StreamInfo^.Status >= TStreamStatus.Predicted then
+    begin
       if GetBits(StreamInfo^.Option, 5, 7) <> I then
         continue;
+      if (StreamInfo^.Status = TStreamStatus.Database) and
+        (GetBits(StreamInfo^.Option, 1, 31) = 0) then
+      begin
+        Res1 := StreamInfo^.OldSize;
+        Result := True;
+      end;
+    end;
     Params := '';
     case X of
       ZSTD_CODEC:
         begin
           Params := 'l' + I.ToString;
-          Res1 := ZSTD_compressCCtx(cctx[Instance], Buffer, StreamInfo^.NewSize,
+          { ZSTD_CCtx_reset(cctx[Instance], ZSTD_reset_session_and_parameters);
-            NewInput, StreamInfo^.NewSize, I);
+            ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_strategy, 5);
+            ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_compressionLevel, I); }
+          if not Result then
+            Res1 := ZSTD_compressCCtx(cctx[Instance], Buffer,
+              StreamInfo^.NewSize, NewInput, StreamInfo^.NewSize, I);
         end;
       { Res1 := ZSTD_compress_usingCDict(cctx[Instance], Buffer,
         StreamInfo^.NewSize, NewInput, StreamInfo^.NewSize, cdict); }
       { begin
+        Params := 'l' + I.ToString;
         Progress := 0;
         Oup.dst := Buffer;
         Oup.Size := StreamInfo^.NewSize;
         Oup.Pos := 0;
         ZSTD_initCStream(cctx[Instance], I);
+        ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_strategy, 9);
         while Progress < StreamInfo^.NewSize do
         begin
         Inp.src := PByte(NewInput) + Progress;
-        Inp.Size := Min(StreamInfo^.NewSize - Progress, 32768);
+        Inp.Size := Min(StreamInfo^.NewSize - Progress, 64 * 1024);
         Inp.Pos := 0;
         if ZSTD_compressStream(cctx[Instance], @Oup, @Inp) > 0 then
         begin
@@ -298,8 +312,9 @@ begin
         Res1 := Oup.Pos;
         end; }
     end;
-    Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+    if not Result then
-      StreamInfo^.OldSize);
+      Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+        StreamInfo^.OldSize);
     Funcs^.LogProcess(ZSTDCodecs[GetBits(StreamInfo^.Option, 0, 5)],
       PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
     if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
@@ -307,7 +322,7 @@ begin
   end;
   if Res1 < 0 then
     exit;
-  if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
+  if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
     (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
   begin
     Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

xtool.dpr

@@ -54,6 +54,7 @@ uses
   ParseExpr in 'contrib\ParseExpression\ParseExpr.pas',
   BrunsliDLL in 'imports\BrunsliDLL.pas',
   FLACDLL in 'imports\FLACDLL.pas',
+  FLZMA2DLL in 'imports\FLZMA2DLL.pas',
   JoJpegDLL in 'imports\JoJpegDLL.pas',
   LZ4DLL in 'imports\LZ4DLL.pas',
   LZODLL in 'imports\LZODLL.pas',
@@ -87,8 +88,7 @@ uses
   IOPatch in 'io\IOPatch.pas',
   IOExecute in 'io\IOExecute.pas',
   IODecode in 'io\IODecode.pas',
-  IOUtils in 'io\IOUtils.pas',
+  IOUtils in 'io\IOUtils.pas';
-  LZMADLL in 'imports\LZMADLL.pas';
 
 {$SETPEFLAGS IMAGE_FILE_LARGE_ADDRESS_AWARE or IMAGE_FILE_RELOCS_STRIPPED}
 

xtool.dproj

@@ -113,6 +113,7 @@
         <DCCReference Include="contrib\ParseExpression\ParseExpr.pas"/>
         <DCCReference Include="imports\BrunsliDLL.pas"/>
         <DCCReference Include="imports\FLACDLL.pas"/>
+        <DCCReference Include="imports\FLZMA2DLL.pas"/>
         <DCCReference Include="imports\JoJpegDLL.pas"/>
         <DCCReference Include="imports\LZ4DLL.pas"/>
         <DCCReference Include="imports\LZODLL.pas"/>
@@ -147,18 +148,17 @@
         <DCCReference Include="io\IOExecute.pas"/>
         <DCCReference Include="io\IODecode.pas"/>
         <DCCReference Include="io\IOUtils.pas"/>
-        <DCCReference Include="imports\LZMADLL.pas"/>
-        <RcItem Include="resources\Win32\xdelta3_dll.dll">
-            <ContainerId>ResourceItem</ContainerId>
-            <ResourceType>RCDATA</ResourceType>
-            <ResourceId>XDELTA86_DLL</ResourceId>
-        </RcItem>
         <RcItem Include="resources\Win64\xdelta3_dll.dll">
             <ContainerId>ResourceItem</ContainerId>
             <ResourceType>RCDATA</ResourceType>
-            <ResourceId>XDELTA64_DLL</ResourceId>
+            <ResourceId>xdelta3_dll</ResourceId>
         </RcItem>
         <None Include="changes.txt"/>
+        <RcItem Include="resources\Win64\fast-lzma2.dll">
+            <ContainerId>ResourceItem</ContainerId>
+            <ResourceType>RCDATA</ResourceType>
+            <ResourceId>fast_lzma2</ResourceId>
+        </RcItem>
         <BuildConfiguration Include="Base">
             <Key>Base</Key>
         </BuildConfiguration>
@@ -208,7 +208,7 @@
                         <Overwrite>true</Overwrite>
                     </Platform>
                 </DeployFile>
-                <DeployFile LocalName="resources\Win32\xdelta3_dll.dll" Configuration="Release" Class="ProjectFile">
+                <DeployFile LocalName="resources\Win64\fast-lzma2.dll" Configuration="Release" Class="ProjectFile">
                     <Platform Name="Win64">
                         <RemoteDir>.\</RemoteDir>
                         <Overwrite>true</Overwrite>

xtoolResource.rc

@@ -1,2 +1,2 @@
-XDELTA86_DLL RCDATA "resources\\Win32\\xdelta3_dll.dll"
+xdelta3_dll RCDATA "resources\\Win64\\xdelta3_dll.dll"
-XDELTA64_DLL RCDATA "resources\\Win64\\xdelta3_dll.dll"
+fast_lzma2 RCDATA "resources\\Win64\\fast-lzma2.dll"