update to 0.6.0

major update

changes.txt

@@ -0,0 +1,251 @@
+  ES_R35 (0.6.0)
+  - added wav stream detector
+  - added flac codec
+  - added jpg stream detector
+  - added packjpg, brunsli, jojpeg codec
+  - added fast lzma2 compression for portable mode
+  - added feature that allows input to be a directory
+  - added feature to inject libraries to main executable
+  - added feature to extract detected streams
+  - updated database feature
+  - updated deduplication feature
+  - IO function decode updated
+
+  ES_R34 (0.5.3)
+  - added png stream preprocessor
+  - removed grittibanzli codec (since nobody uses it)
+
+  ES_R33 (0.5.2)
+  - added IO functions (archive, execute)
+  - fixed issue in patch io function
+  - removed compression on patch diff files
+
+  ES_R32 (0.5.1)
+  - added IO functions (find, extract, patch)
+  - generate database feature and IO functions now can search for streams larger than chunk size
+
+  ES_R31 (0.5.0)
+  - added IO functions (erase, replace)
+  - fixed external executable support bugs
+
+  ES_R30 (0.4.8)
+  - fixed issue with storing incorrect recompression information when stream patching is performed
+
+  ES_R29 (0.4.7)
+  - updated oodle scanner
+  - updated external executable support
+  - updated configuration based plugin support to add depth information
+  - updated verbose mode
+
+  ES_R28 (0.4.6)
+  - generate database feature fixed
+  - fixed external executable support issues
+  - fixed lz4f level setting bug
+
+  ES_R28 (0.4.5)
+  - removed leviathan codec restriction
+
+  ES_R27 (0.4.4)
+  - fixed issue of lz4 codec loading incorrect library
+  - fixed issue with handling endianess via configuration based plugins
+  - updated framework of library based plugins
+
+  ES_R26 (0.4.3)
+  - added verbose mode
+  - added feature that allows you to enforce a different library to be loaded
+  - fixed issues related to imperfect stream patching
+  - fixed issues with old libraries with missing functions that cause xtool to crash on startup
+  - updated oodle codec
+  - updated reflate codec
+  - updated zstd codec
+
+  ES_R25 (0.4.2)
+  - removed debugging code from encryption and executable codec
+  - fixed issue with depth when using search codec
+  - fixed external executable support issues
+
+  ES_R24 (0.4.1)
+  - fixed issue of status not reporting when encoding
+  - added depth method support for search support
+  - fixed zlib encoding issues for different window bits
+  - fixed zlib memory leak issue
+  - updated all internal codecs to support information relayed by external codecs
+  - updated lz4f codec and removed temporarily removed support for universal scanning
+  - added option to change recompression level to be used by reflate
+  - updated external executable support
+  - generate database feature currently bugged, wait for next update
+  - search database structure changed, older database files will no longer work with newer releases
+
+  ES_R23 (0.4.0)
+  - project made open source
+  - added external executable support
+  - added generate database feature
+  - fixed search support bug
+
+  ES_R22 (0.3.22)
+  - updated search support (speed improvements)
+  - updated command line parser
+  - added partial universal scanner for lzo1x streams
+  - added universal scanner for lz4f streams
+  - fixed issue with configuration files failing to execute without conditions
+
+  ES_R21 (0.3.21)
+  - updated search support
+
+  ES_R20 (0.3.20)
+  - fixed library support bug
+  - x86 build discontinued (has bugs from nowhere)
+
+  ES_R19 (0.3.19)
+  - updated lzo codec
+
+  ES_R18 (0.3.18)
+  - fixed depth bug
+  - fixed library plugin bugs
+
+  ES_R17 (0.3.17)
+  - fixed multi-threading bug
+
+  ES_R16 (0.3.16)
+  - minor bug fixes
+
+  ES_R15 (0.3.15)
+  - converted library support to unicode (don't know why I used ansi in the first place)
+  - added library support functions
+  - added rc4 encryption support
+
+  ES_R14 (0.3.14)
+  - fixed library support bug
+  - updated library structure
+
+  ES_R13 (0.3.13)
+  - updated lz4 codec
+  - updated library structure
+  - updated depth info functions
+  - updated depth feature
+
+  ES_R12 (0.3.12)
+  - added depth info functions
+  - added support for oodle 2.9.0+ functions
+  - fixed data patching bug
+  - updated oodle codec
+  - updated command line parser
+
+  ES_R11 (0.3.11)
+  - fixed x86 build bugs
+  - fixed config multi-threading bug
+  - fixed resource management bug
+  - fixed deduplication bug
+
+  ES_R10 (0.3.10)
+  - minor bug fixes
+  - added diff tolerance parameter (--diff=)
+  - fixed plugin database bug
+  - fixed lz4 codec bug
+  - updated oodle codec
+  - updated library structure
+  - added resource management
+  - added direct use encryption codecs
+  - added embedded deduplication feature (--dedup) [makes temps during encoding]
+
+  ES_R9 (0.3.9)
+  - fixed future stream bug
+
+  ES_R8 (0.3.8)
+  - fixed command line parser bug
+  - updated library support
+
+  ES_R7 (0.3.7)
+  - updated library structure
+
+  ES_R6 (0.3.6)
+  - updated oodle codec (fixed more lzna bugs)
+
+  ES_R5 (0.3.5)
+  - updated oodle codec (fixed lzna bug)
+  - added custom method configuration
+
+  ES_R4 (0.3.4)
+  - fixed bug depthing
+
+  ES_R3 (0.3.3)
+  - updated lz4 codec
+  - updated library support
+
+  ES_R2 (0.3.2)
+  - improved depthing
+  - updated library support
+  - fixed zstd codec issues
+  - removed fast memory
+
+  ES_R1 (0.3.1)
+  - updated library support
+  - updated command line parser
+  - included x86 build
+  - fixed depthing issues
+
+  2012_R2 (0.2.14)
+  - added library support
+  - added compress, decompress, encrypt, decrypt, hash, delta functions (used by library)
+  - added lzo codec placeholders
+  - fixed oodle bug
+  - fixed lz4 bug
+  - removed libdunia codec
+
+  2012_R1 (0.2.13)
+  - added oo2ext* dll support
+  - updated search support
+
+  2011_R1 (0.2.12)
+  - added temporary libdunia codec
+
+  2010_R5 (0.2.11)
+  - fixed search/config support bug
+
+  2010_R4 (0.2.10)
+  - updated search/config support
+
+  2010_R3 (0.2.9)
+  - added database search
+  - updated zlib scanner
+  - fixed reflate bug
+  - fixed 2GB memory limit
+
+  2010_R2 (0.2.8)
+  - fixed zstd codec
+
+  2010_R1 (0.2.7)
+  - added zstd codec
+  - added lz4, lz4hc, lzna, mermaid, selkie, hydra, leviathan codec placeholders
+  - added configuration support
+  - added xdelta support to handle crc mismatch streams
+
+  2009_R3 (0.2.6)
+  - documentation added
+
+  2009_R2 (0.2.5)
+  - added kraken codec
+  - fixed depthing issues
+
+  2009_R1 (0.2.4)
+  - added reflate forced verification
+  - updated deflate scanner
+  - fixed depthing issues
+  - fixed low memory mode issues
+  - fixed hanging issues when encoding
+
+  2008_R3 (0.2.3)
+  - fixed deduplication memory calculation error
+  - added virtual memory support for deduplication
+  - added --mem=# parameter to control deduplication memory usage
+
+  2008_R2 (0.2.2)
+  - fixed command line parser
+  - updated deflate scanner
+  - added stream deduplication
+  - added stream database
+  - added decompression memory limiter
+  - added grittibanzli (also handles deflate stream but slow af)
+
+  2008_R1 (0.2.1)
+  - initial release

common/Utils.pas

@@ -44,6 +44,16 @@ procedure SetBits(var Data: UInt64; Value: Int64; Index: TInt64_BitIndex;
   Count: TInt64_BitCount); overload;
 
 type
+  PDynArrayRec = ^TDynArrayRec;
+
+  TDynArrayRec = packed record
+{$IFDEF CPUX64}
+    _Padding: LongInt;
+{$ENDIF}
+    RefCnt: LongInt;
+    Length: NativeInt;
+  end;
+
   TListEx<T> = class(TList<T>)
   private
     FIndex: Integer;
@@ -85,6 +95,13 @@ type
     property Method: TSOMethod read FSOMethod write FSOMethod;
   end;
 
+  TNullStream = class(TStream)
+  public
+    constructor Create;
+    destructor Destroy; override;
+    function Write(const Buffer; Count: LongInt): LongInt; override;
+  end;
+
   TArrayStream = class(TStream)
   private type
     _Stream = ^IStream;
@@ -97,7 +114,7 @@ type
     FMaxStreamSize = $FFFFFFFFFF;
   protected
     function GetSize: Int64; override;
-    procedure SetSize(NewSize: Longint); override;
+    procedure SetSize(NewSize: LongInt); override;
     procedure SetSize(const NewSize: Int64); override;
   private
     FStreams: TArray<IStream>;
@@ -110,8 +127,8 @@ type
   public
     constructor Create;
     destructor Destroy; override;
-    function Read(var Buffer; Count: Longint): Longint; override;
+    function Read(var Buffer; Count: LongInt): LongInt; override;
-    function Write(const Buffer; Count: Longint): Longint; override;
+    function Write(const Buffer; Count: LongInt): LongInt; override;
     function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override;
     procedure Clear;
     function Add(AStreamType: Pointer; MaxSize: Int64 = FMaxStreamSize)
@@ -122,7 +139,7 @@ type
   TPointersStream = class(TStream)
   protected
     function GetSize: Int64; override;
-    procedure SetSize(NewSize: Longint); override;
+    procedure SetSize(NewSize: LongInt); override;
     procedure SetSize(const NewSize: Int64); override;
   private
     FPointers: TArray<Pointer>;
@@ -135,8 +152,8 @@ type
   public
     constructor Create;
     destructor Destroy; override;
-    function Read(var Buffer; Count: Longint): Longint; override;
+    function Read(var Buffer; Count: LongInt): LongInt; override;
-    function Write(const Buffer; Count: Longint): Longint; override;
+    function Write(const Buffer; Count: LongInt): LongInt; override;
     function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override;
     function Add(Ptr: Pointer; Size: NativeInt): Integer;
     procedure Delete(Index: Integer);
@@ -157,13 +174,48 @@ type
       AMaxSize: NativeInt = 0); overload;
     destructor Destroy; override;
     procedure SetSize(const NewSize: Int64); override;
-    procedure SetSize(NewSize: Longint); override;
+    procedure SetSize(NewSize: LongInt); override;
-    function Read(var Buffer; Count: Longint): Longint; override;
+    function Read(var Buffer; Count: LongInt): LongInt; override;
-    function Write(const Buffer; Count: Longint): Longint; override;
+    function Write(const Buffer; Count: LongInt): LongInt; override;
     function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override;
     procedure Update(const AMemory: Pointer = nil; AMaxSize: NativeInt = 0);
   end;
 
+  TDirInputStream = class(TStream)
+  protected type
+    TState = (iNone, iLength, iFilename, iSize, iData);
+  private
+    FState: TState;
+    FPath: String;
+    FBaseDir: String;
+    FList: TArray<String>;
+    FIndex, FCount: Integer;
+    FLength: Word;
+    FBytes: TBytes;
+    FStream: TFileStream;
+    FPosition, FSize: Int64;
+  public
+    constructor Create(const APath: String);
+    destructor Destroy; override;
+    function Read(var Buffer; Count: LongInt): LongInt; override;
+  end;
+
+  TDirOutputStream = class(TStream)
+  protected type
+    TState = (oNone, oLength, oFilename, oSize, oData);
+  private
+    FState: TState;
+    FPath: String;
+    FLength: Word;
+    FBytes: TBytes;
+    FStream: TFileStream;
+    FPosition, FSize: Int64;
+  public
+    constructor Create(const APath: String);
+    destructor Destroy; override;
+    function Write(const Buffer; Count: LongInt): LongInt; override;
+  end;
+
   TSharedMemoryStream = class(TMemoryStreamEx)
   private const
     FIncSize = 64 * 1024 * 1024;
@@ -179,7 +231,7 @@ type
     constructor Create(const AMapName: String; ASize: NativeInt); overload;
     destructor Destroy; override;
     procedure SetSize(const NewSize: Int64); override;
-    function Write(const Buffer; Count: Longint): Longint; override;
+    function Write(const Buffer; Count: LongInt): LongInt; override;
   end;
 
   TDownloadStream = class(TStream)
@@ -195,7 +247,7 @@ type
   public
     constructor Create(Url: string);
     destructor Destroy; override;
-    function Read(var Buffer; Count: Longint): Longint; override;
+    function Read(var Buffer; Count: LongInt): LongInt; override;
     function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override;
   end;
 
@@ -297,23 +349,23 @@ type
     PBlockInfo = ^TBlockInfo;
 
     TBlockInfo = record
+      ID: Integer;
       Position, CurrSize, FullSize: Int64;
       Count: Integer;
     end;
-
-    TBlockInfoDynArray = TArray<TBlockInfo>;
   private
-    FSync: TSynLocker;
+    FSearchList: TArray<TBlockInfo>;
-    FDictionary: TSynDictionary;
     FStream: TStream;
-    FStreamPos: Int64;
+    FStreamPos, FStreamSize: Int64;
   public
-    constructor Create(AStream: TStream; ACapacity: Integer = 0);
+    constructor Create(AStream: TStream);
     destructor Destroy; override;
     procedure Add(ID: Integer; Size: Int64; Count: Integer = Integer.MaxValue);
-    procedure Write(ID: Integer; const Buffer; Size: Integer);
+    procedure Write(ID: Integer; Buffer: Pointer; Size: Integer);
     procedure CopyData(ID: Integer; Stream: TStream); overload;
-    function CopyData(ID: Integer): Pointer; overload;
+    function CopyData(ID: Integer; Data: Pointer): Integer; overload;
+    procedure Update(ID: Integer; Count: Integer);
+    procedure Reset(ID: Integer);
   end;
 
   TArgParser = class(TObject)
@@ -679,6 +731,21 @@ begin
   Result := FList.Count;
 end;
 
+constructor TNullStream.Create;
+begin
+  inherited Create;
+end;
+
+destructor TNullStream.Destroy;
+begin
+  inherited Destroy;
+end;
+
+function TNullStream.Write(const Buffer; Count: LongInt): LongInt;
+begin
+  Result := Count;
+end;
+
 constructor TArrayStream.Create;
 begin
   inherited Create;
@@ -781,7 +848,7 @@ begin
   Result := FSize;
 end;
 
-procedure TArrayStream.SetSize(NewSize: Longint);
+procedure TArrayStream.SetSize(NewSize: LongInt);
 begin
   SetSize(Int64(NewSize));
 end;
@@ -794,7 +861,7 @@ begin
     Seek(0, soEnd);
 end;
 
-function TArrayStream.Read(var Buffer; Count: Longint): Longint;
+function TArrayStream.Read(var Buffer; Count: LongInt): LongInt;
 var
   LCount: Int64;
 begin
@@ -808,7 +875,7 @@ begin
   Inc(FPosition, Result);
 end;
 
-function TArrayStream.Write(const Buffer; Count: Longint): Longint;
+function TArrayStream.Write(const Buffer; Count: LongInt): LongInt;
 var
   LCount: Int64;
 begin
@@ -888,7 +955,7 @@ begin
   Result := FSize;
 end;
 
-procedure TPointersStream.SetSize(NewSize: Longint);
+procedure TPointersStream.SetSize(NewSize: LongInt);
 begin
   SetSize(Int64(NewSize));
 end;
@@ -917,12 +984,12 @@ begin
     FSize := FMaxSize;
 end;
 
-function TPointersStream.Read(var Buffer; Count: Longint): Longint;
+function TPointersStream.Read(var Buffer; Count: LongInt): LongInt;
 begin
   // 2121212
 end;
 
-function TPointersStream.Write(const Buffer; Count: Longint): Longint;
+function TPointersStream.Write(const Buffer; Count: LongInt): LongInt;
 begin
 
 end;
@@ -986,7 +1053,7 @@ begin
   inherited Destroy;
 end;
 
-procedure TMemoryStreamEx.SetSize(NewSize: Longint);
+procedure TMemoryStreamEx.SetSize(NewSize: LongInt);
 begin
   SetSize(Int64(NewSize));
 end;
@@ -1002,7 +1069,7 @@ begin
     Seek(0, soEnd);
 end;
 
-function TMemoryStreamEx.Read(var Buffer; Count: Longint): Longint;
+function TMemoryStreamEx.Read(var Buffer; Count: LongInt): LongInt;
 begin
   Result := 0;
   if (FPosition >= 0) and (Count >= 0) then
@@ -1019,9 +1086,9 @@ begin
   end;
 end;
 
-function TMemoryStreamEx.Write(const Buffer; Count: Longint): Longint;
+function TMemoryStreamEx.Write(const Buffer; Count: LongInt): LongInt;
 var
-  FCount: Longint;
+  FCount: LongInt;
 begin
   Result := 0;
   FCount := Count;
@@ -1077,6 +1144,210 @@ begin
   SetSize(LSize);
 end;
 
+constructor TDirInputStream.Create(const APath: String);
+begin
+  inherited Create;
+  FState := TState.iNone;
+  FPath := TPath.GetFullPath(APath);
+  if FileExists(FPath) then
+    FBaseDir := ExtractFilePath(TPath.GetFullPath(FPath))
+  else if DirectoryExists(FPath) then
+    FBaseDir := IncludeTrailingBackSlash(TPath.GetFullPath(FPath))
+  else
+    FBaseDir := ExtractFilePath(TPath.GetFullPath(FPath));
+  FList := GetFileList([FPath], True);
+  FCount := Length(FList);
+  if FCount = 0 then
+    raise EFOpenError.CreateRes(@SEmptyPath);
+  FIndex := -1;
+  FStream := nil;
+end;
+
+destructor TDirInputStream.Destroy;
+begin
+  if Assigned(FStream) then
+    FStream.Free;
+  FStream := nil;
+  inherited Destroy;
+end;
+
+function TDirInputStream.Read(var Buffer; Count: LongInt): LongInt;
+var
+  LCount: Integer;
+begin
+  Result := 0;
+  if Count <= 0 then
+    exit;
+  if FState = TState.iNone then
+  begin
+    if Succ(FIndex) >= FCount then
+      exit;
+    Inc(FIndex);
+    FBytes := BytesOf(ReplaceText(FList[FIndex], FBaseDir, ''));
+    FLength := Length(FBytes);
+    FPosition := 0;
+    FSize := FileSize(FList[FIndex]);
+    FState := TState.iLength;
+  end;
+  if FState = TState.iLength then
+    if FPosition < FLength.Size then
+    begin
+      LCount := Min(FLength.Size - FPosition, Count);
+      Move(WordRec(FLength).Bytes[FPosition], Buffer, LCount);
+      Inc(FPosition, LCount);
+      if FPosition = FLength.Size then
+      begin
+        FState := TState.iFilename;
+        FPosition := 0;
+      end;
+      exit(LCount);
+    end;
+  if FState = TState.iFilename then
+    if FPosition < FLength then
+    begin
+      LCount := Min(FLength - FPosition, Count);
+      Move(FBytes[FPosition], Buffer, LCount);
+      Inc(FPosition, LCount);
+      if FPosition = FLength then
+      begin
+        FState := TState.iSize;
+        FPosition := 0;
+      end;
+      exit(LCount);
+    end;
+  if FState = TState.iSize then
+    if FPosition < FSize.Size then
+    begin
+      LCount := Min(FSize.Size - FPosition, Count);
+      Move(Int64Rec(FSize).Bytes[FPosition], Buffer, LCount);
+      Inc(FPosition, LCount);
+      if FPosition = FSize.Size then
+      begin
+        if FSize = 0 then
+          FState := TState.iNone
+        else
+        begin
+          FState := TState.iData;
+          FPosition := 0;
+          FStream := TFileStream.Create(FList[FIndex], fmShareDenyNone);
+        end;
+      end;
+      exit(LCount);
+    end;
+  if FState = TState.iData then
+    if FPosition < FSize then
+    begin
+      LCount := Min(FSize - FPosition, Count);
+      LCount := FStream.Read(Buffer, LCount);
+      Inc(FPosition, LCount);
+      if FPosition = FSize then
+      begin
+        FState := TState.iNone;
+        FStream.Free;
+        FStream := nil;
+      end;
+      exit(LCount);
+    end;
+end;
+
+constructor TDirOutputStream.Create(const APath: String);
+begin
+  inherited Create;
+  FState := TState.oNone;
+  FPath := IncludeTrailingBackSlash(TPath.GetFullPath(APath));
+  FStream := nil;
+end;
+
+destructor TDirOutputStream.Destroy;
+begin
+  if Assigned(FStream) then
+    FStream.Free;
+  FStream := nil;
+  inherited Destroy;
+end;
+
+function TDirOutputStream.Write(const Buffer; Count: LongInt): LongInt;
+var
+  LCount: Integer;
+  LStr: String;
+begin
+  Result := 0;
+  if Count <= 0 then
+    exit;
+  if FState = TState.oNone then
+  begin
+    FPosition := 0;
+    FState := TState.oLength;
+  end;
+  if FState = TState.oLength then
+    if FPosition < FLength.Size then
+    begin
+      LCount := Min(FLength.Size - FPosition, Count);
+      Move(Buffer, WordRec(FLength).Bytes[FPosition], LCount);
+      Inc(FPosition, LCount);
+      if FPosition = FLength.Size then
+      begin
+        SetLength(FBytes, FLength);
+        FState := TState.oFilename;
+        FPosition := 0;
+      end;
+      exit(LCount);
+    end;
+  if FState = TState.oFilename then
+    if FPosition < FLength then
+    begin
+      LCount := Min(FLength - FPosition, Count);
+      Move(Buffer, FBytes[FPosition], LCount);
+      Inc(FPosition, LCount);
+      if FPosition = FLength then
+      begin
+        FState := TState.oSize;
+        FPosition := 0;
+      end;
+      exit(LCount);
+    end;
+  if FState = TState.oSize then
+    if FPosition < FSize.Size then
+    begin
+      LCount := Min(FSize.Size - FPosition, Count);
+      Move(Buffer, Int64Rec(FSize).Bytes[FPosition], LCount);
+      Inc(FPosition, LCount);
+      if FPosition = FSize.Size then
+      begin
+        LStr := FPath + StringOf(FBytes);
+        if not DirectoryExists(ExtractFilePath(LStr)) then
+          ForceDirectories(ExtractFilePath(LStr));
+        FStream := TFileStream.Create(LStr, fmCreate);
+        if FSize = 0 then
+        begin
+          FState := TState.oNone;
+          FStream.Free;
+          FStream := nil;
+        end
+        else
+        begin
+          FState := TState.oData;
+          FPosition := 0;
+        end;
+      end;
+      exit(LCount);
+    end;
+  if FState = TState.oData then
+    if FPosition < FSize then
+    begin
+      LCount := Min(FSize - FPosition, Count);
+      LCount := FStream.Write(Buffer, LCount);
+      Inc(FPosition, LCount);
+      if FPosition = FSize then
+      begin
+        FState := TState.oNone;
+        FStream.Free;
+        FStream := nil;
+      end;
+      exit(LCount);
+    end;
+end;
+
 constructor TSharedMemoryStream.Create(const AMapName: String;
   AFileName: string);
 
@@ -1191,7 +1462,7 @@ begin
   inherited SetSize(NewSize);
 end;
 
-function TSharedMemoryStream.Write(const Buffer; Count: Longint): Longint;
+function TSharedMemoryStream.Write(const Buffer; Count: LongInt): LongInt;
 begin
   if FPosition + Count > FMaxSize then
     IncMemory(FPosition + Count);
@@ -1227,7 +1498,7 @@ begin
   Abort := True;
 end;
 
-function TDownloadStream.Read(var Buffer; Count: Longint): Longint;
+function TDownloadStream.Read(var Buffer; Count: LongInt): LongInt;
 begin
   if (FPosition >= 0) and (Count >= 0) then
   begin
@@ -1779,107 +2050,132 @@ begin
   FSizes[Index] := 0;
 end;
 
-constructor TDataManager.Create(AStream: TStream; ACapacity: Integer);
+constructor TDataManager.Create(AStream: TStream);
 begin
   inherited Create;
-  FSync.Init;
-  FDictionary := TSynDictionary.Create(TypeInfo(TIntegerDynArray),
-    TypeInfo(TBlockInfoDynArray));
-  FDictionary.Capacity := ACapacity;
   FStream := AStream;
   FStreamPos := FStream.Position;
+  FStreamSize := 0;
 end;
 
 destructor TDataManager.Destroy;
 begin
-  FDictionary.Free;
-  FSync.Done;
   inherited Destroy;
 end;
 
 procedure TDataManager.Add(ID: Integer; Size: Int64; Count: Integer);
 var
-  LBlockInfo: TBlockInfo;
-  _BlockInfo: PBlockInfo;
   I: Integer;
+  LBlockInfo: TBlockInfo;
 begin
   if Count <= 0 then
     exit;
-  FSync.Lock;
+  for I := Low(FSearchList) to High(FSearchList) do
-  try
+  begin
-    LBlockInfo.Position := 0;
+    if (FSearchList[I].Count <= 0) and (Size <= FSearchList[I].FullSize) then
-    for I := 0 to FDictionary.Count - 1 do
     begin
-      _BlockInfo := PBlockInfo(FDictionary.Values.ElemPtr(I));
+      FSearchList[I].ID := ID;
-      LBlockInfo.Position := Max(LBlockInfo.Position, _BlockInfo^.Position +
+      FSearchList[I].CurrSize := 0;
-        _BlockInfo^.FullSize);
+      FSearchList[I].Count := Count;
+      exit;
     end;
-    LBlockInfo.CurrSize := 0;
-    LBlockInfo.FullSize := Size;
-    LBlockInfo.Count := Count;
-    FStream.Size := Max(FStream.Size, FStreamPos + LBlockInfo.Position +
-      LBlockInfo.FullSize);
-    FDictionary.Add(ID, LBlockInfo);
-  finally
-    FSync.UnLock;
   end;
-  // once reads reaches 0, add list of all available spaces
+  LBlockInfo.ID := ID;
+  LBlockInfo.Position := FStreamPos + FStreamSize;
+  LBlockInfo.CurrSize := 0;
+  LBlockInfo.FullSize := Size;
+  LBlockInfo.Count := Count;
+  Insert(LBlockInfo, FSearchList, Length(FSearchList));
+  Inc(FStreamSize, Size);
 end;
 
-procedure TDataManager.Write(ID: Integer; const Buffer; Size: Integer);
+procedure TDataManager.Write(ID: Integer; Buffer: Pointer; Size: Integer);
 var
-  _BlockInfo: PBlockInfo;
+  I: Integer;
 begin
   if Size <= 0 then
     exit;
-  FSync.Lock;
+  for I := Low(FSearchList) to High(FSearchList) do
-  try
+  begin
-    _BlockInfo := FDictionary.FindValue(ID);
+    if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then
-    if (_BlockInfo^.Position + _BlockInfo^.CurrSize + Size) >
+    begin
-      (_BlockInfo^.Position + _BlockInfo^.FullSize) then
+      if FSearchList[I].CurrSize + Size > FSearchList[I].FullSize then
-      raise EWriteError.CreateRes(@SWriteError);
+        raise EWriteError.CreateRes(@SWriteError);
-    FStream.Position := FStreamPos + _BlockInfo^.Position +
+      FStream.Position := FSearchList[I].Position + FSearchList[I].CurrSize;
-      _BlockInfo^.CurrSize;
+      FStream.WriteBuffer(Buffer^, Size);
-    FStream.WriteBuffer(Buffer, Size);
+      Inc(FSearchList[I].CurrSize, Size);
-    Inc(_BlockInfo^.CurrSize, Size);
+      exit;
-  finally
+    end;
-    FSync.UnLock;
   end;
+  raise Exception.CreateRes(@SGenericItemNotFound);
 end;
 
 procedure TDataManager.CopyData(ID: Integer; Stream: TStream);
 var
-  _BlockInfo: PBlockInfo;
+  I: Integer;
 begin
-  FSync.Lock;
+  for I := Low(FSearchList) to High(FSearchList) do
-  try
+  begin
-    _BlockInfo := FDictionary.FindValue(ID);
+    if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then
-    if _BlockInfo^.CurrSize <> _BlockInfo^.FullSize then
+    begin
-      raise EReadError.CreateRes(@SReadError);
+      FStream.Position := FSearchList[I].Position;
-    FStream.Position := FStreamPos + _BlockInfo^.Position;
+      CopyStreamEx(FStream, Stream, FSearchList[I].CurrSize);
-    CopyStreamEx(FStream, Stream, _BlockInfo^.FullSize);
+      Dec(FSearchList[I].Count);
-    Dec(_BlockInfo^.Count);
+      exit;
-  finally
+    end;
-    FSync.UnLock;
   end;
+  raise Exception.CreateRes(@SGenericItemNotFound);
 end;
 
-function TDataManager.CopyData(ID: Integer): Pointer;
+function TDataManager.CopyData(ID: Integer; Data: Pointer): Integer;
 var
-  _BlockInfo: PBlockInfo;
+  I: Integer;
 begin
-  FSync.Lock;
+  Result := 0;
-  try
+  for I := Low(FSearchList) to High(FSearchList) do
-    _BlockInfo := FDictionary.FindValue(ID);
+  begin
-    if _BlockInfo^.CurrSize <> _BlockInfo^.FullSize then
+    if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then
-      raise EReadError.CreateRes(@SReadError);
+    begin
-    FStream.Position := FStreamPos + _BlockInfo^.Position +
+      FStream.Position := FSearchList[I].Position;
-      _BlockInfo^.CurrSize;
+      FStream.ReadBuffer(Data^, FSearchList[I].CurrSize);
-    FStream.ReadBuffer(Result^, _BlockInfo^.FullSize);
+      Result := FSearchList[I].CurrSize;
-    Dec(_BlockInfo^.Count);
+      Dec(FSearchList[I].Count);
-  finally
+      if FSearchList[I].Count = 0 then
-    FSync.UnLock;
+        FSearchList[I].ID := -1;
+      exit;
+    end;
   end;
+  raise Exception.CreateRes(@SGenericItemNotFound);
+end;
+
+procedure TDataManager.Update(ID: Integer; Count: Integer);
+var
+  I: Integer;
+begin
+  for I := Low(FSearchList) to High(FSearchList) do
+  begin
+    if (ID = FSearchList[I].ID) then
+    begin
+      FSearchList[I].Count := Count;
+      exit;
+    end;
+  end;
+  raise Exception.CreateRes(@SGenericItemNotFound);
+end;
+
+procedure TDataManager.Reset(ID: Integer);
+var
+  I: Integer;
+begin
+  for I := Low(FSearchList) to High(FSearchList) do
+  begin
+    if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then
+    begin
+      FSearchList[I].CurrSize := 0;
+      exit;
+    end;
+  end;
+  raise Exception.CreateRes(@SGenericItemNotFound);
 end;
 
 constructor TArgParser.Create(Arguments: TStringDynArray);

imports/BrunsliDLL.pas

@@ -0,0 +1,87 @@
+unit BrunsliDLL;
+
+interface
+
+uses
+  WinAPI.Windows,
+  System.SysUtils, System.Classes;
+
+const
+  BRUNSLI_OK = 0;
+  BRUNSLI_NON_REPRESENTABLE = 1;
+  BRUNSLI_MEMORY_ERROR = 2;
+  BRUNSLI_INVALID_PARAM = 3;
+  BRUNSLI_COMPRESSION_ERROR = 4;
+  BRUNSLI_INVALID_BRN = 5;
+  BRUNSLI_DECOMPRESSION_ERROR = 6;
+  BRUNSLI_NOT_ENOUGH_DATA = 7;
+
+type
+  TBrunsliWriter = function(ctx: Pointer; data: Pointer; Size: NativeUInt)
+    : Integer cdecl;
+
+var
+  brunsli_alloc_JPEGData: function: Pointer cdecl;
+  brunsli_free_JPEGData: procedure(P: Pointer)cdecl;
+  brunsli_GetMaximumEncodedSize: function(P: Pointer): Integer cdecl;
+  brunsli_ReadJpeg: function(P: Pointer; data: Pointer; len: Integer)
+    : Integer cdecl;
+  brunsli_EncodeJpeg: function(P: Pointer; data: Pointer; len: Integer)
+    : Integer cdecl;
+  brunsli_DecodeJpeg: function(P: Pointer; data: Pointer; len: Integer)
+    : Integer cdecl;
+  brunsli_alloc_JPEGOutput: function(P: TBrunsliWriter; data: Pointer)
+    : Pointer cdecl;
+  brunsli_free_JPEGOutput: procedure(P: Pointer)cdecl;
+  brunsli_WriteJpeg: function(P: Pointer; oup: Pointer): Integer cdecl;
+  DLLLoaded: Boolean = False;
+
+implementation
+
+var
+  DLLHandle: THandle;
+  S: String;
+
+procedure Init;
+begin
+  S := 'brunsli.dll';
+  DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + S));
+  if DLLHandle >= 32 then
+  begin
+    @brunsli_alloc_JPEGData := GetProcAddress(DLLHandle,
+      'brunsli_alloc_JPEGData');
+    @brunsli_free_JPEGData := GetProcAddress(DLLHandle,
+      'brunsli_free_JPEGData');
+    @brunsli_GetMaximumEncodedSize := GetProcAddress(DLLHandle,
+      'brunsli_GetMaximumEncodedSize');
+    @brunsli_ReadJpeg := GetProcAddress(DLLHandle, 'brunsli_ReadJpeg');
+    @brunsli_EncodeJpeg := GetProcAddress(DLLHandle, 'brunsli_EncodeJpeg');
+    @brunsli_DecodeJpeg := GetProcAddress(DLLHandle, 'brunsli_DecodeJpeg');
+    @brunsli_alloc_JPEGOutput := GetProcAddress(DLLHandle,
+      'brunsli_alloc_JPEGOutput');
+    @brunsli_free_JPEGOutput := GetProcAddress(DLLHandle,
+      'brunsli_free_JPEGOutput');
+    @brunsli_WriteJpeg := GetProcAddress(DLLHandle, 'brunsli_WriteJpeg');
+    DLLLoaded := Assigned(brunsli_alloc_JPEGData) and
+      Assigned(brunsli_alloc_JPEGOutput);
+  end
+  else
+    DLLLoaded := False;
+end;
+
+procedure Deinit;
+begin
+  if not DLLLoaded then
+    exit;
+  FreeLibrary(DLLHandle);
+end;
+
+initialization
+
+Init;
+
+finalization
+
+Deinit;
+
+end.

imports/FLACDLL.pas

@@ -0,0 +1,196 @@
+unit FLACDLL;
+
+interface
+
+uses
+  WinAPI.Windows,
+  System.SysUtils, System.Classes;
+
+const
+  FLAC__MAX_CHANNELS = 8;
+  FLAC__MAX_FIXED_ORDER = 4;
+  FLAC__MAX_LPC_ORDER = 32;
+
+type
+  TFLAC__FrameHeader = record
+    blocksize, sample_rate, channels: Cardinal;
+    channel_assignment: Integer;
+    bits_per_sample: Cardinal;
+    case number_type: Integer of
+      0:
+        (frame_number: Cardinal; crc1: Byte);
+      1:
+        (sample_number: UInt64; crc2: Byte);
+  end;
+
+  PFLAC__EntropyCodingMethod_PartitionedRiceContents = ^
+    TFLAC__EntropyCodingMethod_PartitionedRiceContents;
+
+  TFLAC__EntropyCodingMethod_PartitionedRiceContents = record
+    parameters, raw_bits: PCardinal;
+    capacity_by_order: Cardinal;
+  end;
+
+  TFLAC__EntropyCodingMethod_PartitionedRice = record
+    order: Cardinal;
+    contents: PFLAC__EntropyCodingMethod_PartitionedRiceContents;
+  end;
+
+  TFLAC__EntropyCodingMethod = record
+    case ftype: Integer of
+      0:
+        (partitioned_rice: TFLAC__EntropyCodingMethod_PartitionedRice);
+  end;
+
+  TFLAC__Subframe_Fixed = record
+    entropy_coding_method: TFLAC__EntropyCodingMethod;
+    order: Cardinal;
+    warmup: array [0 .. FLAC__MAX_CHANNELS - 1] of Integer;
+    residual: PInteger;
+  end;
+
+  TFLAC__Subframe_LPC = record
+    entropy_coding_method: TFLAC__EntropyCodingMethod;
+    order, qlp_coeff_precision: Cardinal;
+    quantization_level: Integer;
+    qlp_coeff, warmup: array [0 .. FLAC__MAX_LPC_ORDER - 1] of Integer;
+    residual: PInteger;
+  end;
+
+  TFLAC__Subframe = record
+    case ftype: Integer of
+      0:
+        (constant: Integer; wb1: Cardinal);
+      1:
+        (fixed: TFLAC__Subframe_Fixed; wb2: Cardinal);
+      2:
+        (lpc: TFLAC__Subframe_LPC; wb3: Cardinal);
+      3:
+        (verbatim: PInteger; wb4: Cardinal);
+  end;
+
+  PFLAC__Frame = ^TFLAC__Frame;
+
+  TFLAC__Frame = record
+    header: TFLAC__FrameHeader;
+    subframes: array [0 .. FLAC__MAX_CHANNELS - 1] of TFLAC__Subframe;
+    footer: Word;
+  end;
+
+var
+  FLAC__stream_encoder_new: function: Pointer cdecl;
+  FLAC__stream_encoder_set_verify: function(encoder: Pointer; value: Boolean)
+    : Boolean cdecl;
+  FLAC__stream_encoder_set_compression_level: function(encoder: Pointer;
+    value: Cardinal): Boolean cdecl;
+  FLAC__stream_encoder_set_channels: function(encoder: Pointer; value: Cardinal)
+    : Boolean cdecl;
+  FLAC__stream_encoder_set_bits_per_sample: function(encoder: Pointer;
+    value: Cardinal): Boolean cdecl;
+  FLAC__stream_encoder_set_sample_rate: function(encoder: Pointer;
+    value: Cardinal): Boolean cdecl;
+  FLAC__stream_encoder_set_total_samples_estimate: function(encoder: Pointer;
+    value: UInt64): Boolean cdecl;
+  FLAC__stream_encoder_init_stream: function(encoder: Pointer;
+    write_callback, seek_callback, tell_callback, metadata_callback: Pointer;
+    client_data: Pointer): Integer cdecl;
+  FLAC__stream_encoder_init_file: function(encoder: Pointer;
+    filename: PAnsiChar; progress_callback: Pointer; client_data: Pointer)
+    : Integer cdecl;
+  FLAC__stream_encoder_process_interleaved: function(encoder: Pointer;
+    const buffer; samples: Cardinal): Boolean cdecl;
+  FLAC__stream_encoder_finish: function(encoder: Pointer): Boolean cdecl;
+  FLAC__stream_encoder_delete: procedure(encoder: Pointer)cdecl;
+  FLAC__stream_decoder_new: function: Pointer cdecl;
+  FLAC__stream_decoder_init_stream: function(decoder: Pointer;
+    read_callback, seek_callback, tell_callback, length_callback, eof_callback,
+    write_callback, metadata_callback, error_callback: Pointer;
+    client_data: Pointer): Integer cdecl;
+  FLAC__stream_decoder_init_file: function(decoder: Pointer;
+    filename: PAnsiChar; write_callback, metadata_callback, error_callback
+    : Pointer; client_data: Pointer): Integer cdecl;
+  FLAC__stream_decoder_get_channels: function(decoder: Pointer): Cardinal cdecl;
+  FLAC__stream_decoder_get_bits_per_sample: function(decoder: Pointer)
+    : Cardinal cdecl;
+  FLAC__stream_decoder_process_until_end_of_stream: function(decoder: Pointer)
+    : Boolean cdecl;
+  FLAC__stream_decoder_finish: function(encoder: Pointer): Boolean cdecl;
+  FLAC__stream_decoder_delete: procedure(encoder: Pointer)cdecl;
+  DLLLoaded: Boolean = False;
+
+implementation
+
+var
+  DLLHandle: THandle;
+
+procedure Init;
+begin
+  DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) +
+    'libFLAC_dynamic.dll'));
+  if DLLHandle >= 32 then
+  begin
+    @FLAC__stream_encoder_new := GetProcAddress(DLLHandle,
+      'FLAC__stream_encoder_new');
+    @FLAC__stream_encoder_set_verify := GetProcAddress(DLLHandle,
+      'FLAC__stream_encoder_set_verify');
+    @FLAC__stream_encoder_set_channels := GetProcAddress(DLLHandle,
+      'FLAC__stream_encoder_set_channels');
+    @FLAC__stream_encoder_set_compression_level :=
+      GetProcAddress(DLLHandle, 'FLAC__stream_encoder_set_compression_level');
+    @FLAC__stream_encoder_set_bits_per_sample :=
+      GetProcAddress(DLLHandle, 'FLAC__stream_encoder_set_bits_per_sample');
+    @FLAC__stream_encoder_set_sample_rate :=
+      GetProcAddress(DLLHandle, 'FLAC__stream_encoder_set_sample_rate');
+    @FLAC__stream_encoder_set_total_samples_estimate :=
+      GetProcAddress(DLLHandle,
+      'FLAC__stream_encoder_set_total_samples_estimate');
+    @FLAC__stream_encoder_init_stream := GetProcAddress(DLLHandle,
+      'FLAC__stream_encoder_init_stream');
+    @FLAC__stream_encoder_init_file := GetProcAddress(DLLHandle,
+      'FLAC__stream_encoder_init_file');
+    @FLAC__stream_encoder_process_interleaved :=
+      GetProcAddress(DLLHandle, 'FLAC__stream_encoder_process_interleaved');
+    @FLAC__stream_encoder_finish := GetProcAddress(DLLHandle,
+      'FLAC__stream_encoder_finish');
+    @FLAC__stream_encoder_delete := GetProcAddress(DLLHandle,
+      'FLAC__stream_encoder_delete');
+    @FLAC__stream_decoder_new := GetProcAddress(DLLHandle,
+      'FLAC__stream_decoder_new');
+    @FLAC__stream_decoder_init_stream := GetProcAddress(DLLHandle,
+      'FLAC__stream_decoder_init_stream');
+    @FLAC__stream_decoder_init_file := GetProcAddress(DLLHandle,
+      'FLAC__stream_decoder_init_file');
+    @FLAC__stream_decoder_get_channels := GetProcAddress(DLLHandle,
+      'FLAC__stream_decoder_get_channels');
+    @FLAC__stream_decoder_get_bits_per_sample :=
+      GetProcAddress(DLLHandle, 'FLAC__stream_decoder_get_bits_per_sample');
+    @FLAC__stream_decoder_process_until_end_of_stream :=
+      GetProcAddress(DLLHandle,
+      'FLAC__stream_decoder_process_until_end_of_stream');
+    @FLAC__stream_decoder_finish := GetProcAddress(DLLHandle,
+      'FLAC__stream_decoder_finish');
+    @FLAC__stream_decoder_delete := GetProcAddress(DLLHandle,
+      'FLAC__stream_decoder_delete');
+    DLLLoaded := Assigned(FLAC__stream_encoder_new) and
+      Assigned(FLAC__stream_decoder_new);
+  end
+  else
+    DLLLoaded := False;
+end;
+
+procedure Deinit;
+begin
+  if not DLLLoaded then
+    exit;
+  FreeLibrary(DLLHandle);
+end;
+
+initialization
+
+Init;
+
+finalization
+
+Deinit;
+
+end.

imports/JoJpegDLL.pas

@@ -0,0 +1,69 @@
+unit JoJpegDLL;
+
+interface
+
+uses
+  WinAPI.Windows,
+  System.SysUtils, System.Classes;
+
+const
+  jojpeg_Size = 51320000;
+
+  jojpeg_Compress = 0;
+  jojpeg_Decompress = 1;
+
+  jojpeg_enc_Input = 1;
+  jojpeg_enc_Output1 = 2;
+  jojpeg_enc_Output2 = 3;
+
+  jojpeg_dec_Input1 = 1;
+  jojpeg_dec_Input2 = 3;
+  jojpeg_dec_Output = 2;
+
+var
+  jojpeg_Init: function(p: Pointer; f_DEC: integer): integer stdcall;
+  jojpeg_Quit: procedure(p: Pointer; f_DEC: integer)stdcall;
+  jojpeg_Loop: function(p: Pointer; f_DEC: integer): integer stdcall;
+  jojpeg_Getvalue: function(p: Pointer; f_DEC, typ: integer): Int64;
+  jojpeg_Addbuf: procedure(p: Pointer; f_DEC: integer; buf: Pointer;
+    bufsize, state: integer)stdcall;
+  DLLLoaded: boolean = False;
+
+implementation
+
+var
+  DLLHandle: THandle;
+
+procedure Init;
+begin
+  DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) +
+    'jojpeg_dll.dll'));
+  if DLLHandle >= 32 then
+  begin
+    @jojpeg_Init := GetProcAddress(DLLHandle, 'jojpeg_Init');
+    @jojpeg_Quit := GetProcAddress(DLLHandle, 'jojpeg_Quit');
+    @jojpeg_Loop := GetProcAddress(DLLHandle, 'jojpeg_Loop');
+    @jojpeg_Getvalue := GetProcAddress(DLLHandle, 'jojpeg_Getvalue');
+    @jojpeg_Addbuf := GetProcAddress(DLLHandle, 'jojpeg_Addbuf');
+    DLLLoaded := Assigned(jojpeg_Init);
+  end
+  else
+    DLLLoaded := False;
+end;
+
+procedure Deinit;
+begin
+  if not DLLLoaded then
+    exit;
+  FreeLibrary(DLLHandle);
+end;
+
+initialization
+
+Init;
+
+finalization
+
+Deinit;
+
+end.

imports/LZ4DLL.pas

@@ -69,6 +69,8 @@ var
   LZ4F_compressFrame: function(dstBuffer: Pointer; dstCapacity: size_t;
     srcBuffer: Pointer; srcSize: size_t; preferencesPtr: PLZ4F_preferences_t)
     : size_t cdecl;
+  LZ4_compressHC2: function(const src: Pointer; dst: Pointer; srcSize: Integer;
+    compressionLevel: Integer): Integer cdecl;
   LZ4F_compressFrameBound: function(srcSize: size_t;
     preferencesPtr: PLZ4F_preferences_t): size_t cdecl;
   LZ4F_createDecompressionContext: function(out dctxPtr: LZ4F_dctx;
@@ -104,6 +106,7 @@ begin
     @LZ4_compress_default := GetProcAddress(DLLHandle, 'LZ4_compress_default');
     @LZ4_compress_fast := GetProcAddress(DLLHandle, 'LZ4_compress_fast');
     @LZ4_compress_HC := GetProcAddress(DLLHandle, 'LZ4_compress_HC');
+    @LZ4_compressHC2 := GetProcAddress(DLLHandle, 'LZ4_compressHC2');
     @LZ4F_compressFrame := GetProcAddress(DLLHandle, 'LZ4F_compressFrame');
     @LZ4F_compressFrameBound := GetProcAddress(DLLHandle,
       'LZ4F_compressFrameBound');

imports/LZMADLL.pas

@@ -0,0 +1,28 @@
+unit LZMADLL;
+
+interface
+
+uses
+  WinAPI.Windows,
+  System.SysUtils;
+
+type
+  PFL2_inBuffer = ^FL2_inBuffer;
+
+  FL2_inBuffer = record
+    src: Pointer;
+    size: size_t;
+    pos: size_t;
+  end;
+
+  PFL2_outBuffer = ^FL2_outBuffer;
+
+  FL2_outBuffer = record
+    dst: Pointer;
+    size: size_t;
+    pos: size_t;
+  end;
+
+implementation
+
+end.

imports/PackJPGDLL.pas

@@ -0,0 +1,67 @@
+unit PackJPGDLL;
+
+interface
+
+uses
+  WinAPI.Windows,
+  System.SysUtils, System.Classes;
+
+const
+  pjglib_file = 0;
+  pjglib_memory = 1;
+  pjglib_handle = 2;
+
+var
+  pjglib_convert_stream2stream: function(msg: PAnsiChar): Boolean cdecl;
+  pjglib_convert_file2file: function(ain, aout, msg: PAnsiChar): Boolean cdecl;
+  pjglib_convert_stream2mem: function(out_file: PPAnsiChar; out_size: PCardinal;
+    msg: PAnsiChar): Boolean cdecl;
+  pjglib_init_streams: procedure(in_src: Pointer; in_type: Integer;
+    in_size: Integer; out_dest: Pointer; out_type: Integer)cdecl;
+  pjglib_version_info: function: PAnsiChar cdecl;
+  pjglib_short_name: function: PAnsiChar cdecl;
+  DLLLoaded: Boolean = False;
+
+implementation
+
+var
+  DLLHandle: THandle;
+
+procedure Init;
+begin
+  DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) +
+    'packjpg_dll.dll'));
+  if DLLHandle >= 32 then
+  begin
+    @pjglib_convert_stream2stream := GetProcAddress(DLLHandle,
+      'pjglib_convert_stream2stream');
+    @pjglib_convert_file2file := GetProcAddress(DLLHandle,
+      'pjglib_convert_file2file');
+    @pjglib_convert_stream2mem := GetProcAddress(DLLHandle,
+      'pjglib_convert_stream2mem');
+    @pjglib_init_streams := GetProcAddress(DLLHandle, 'pjglib_init_streams');
+    @pjglib_version_info := GetProcAddress(DLLHandle, 'pjglib_version_info');
+    @pjglib_short_name := GetProcAddress(DLLHandle, 'pjglib_short_name');
+    DLLLoaded := Assigned(pjglib_init_streams) and
+      Assigned(pjglib_convert_stream2stream);
+  end
+  else
+    DLLLoaded := False;
+end;
+
+procedure Deinit;
+begin
+  if not DLLLoaded then
+    exit;
+  FreeLibrary(DLLHandle);
+end;
+
+initialization
+
+Init;
+
+finalization
+
+Deinit;
+
+end.

imports/TTADLL.pas

@@ -0,0 +1,53 @@
+unit TTADLL;
+
+interface
+
+uses
+  WinAPI.Windows,
+  System.SysUtils, System.Classes;
+
+var
+  tta_encode: function(const src: Pointer; srcSize: PInteger; dst: Pointer;
+    dstCapacity: PInteger): Boolean cdecl;
+  tta_decode: function(const src: Pointer; srcSize: Integer; dst: Pointer;
+    dstCapacity: PInteger): Boolean cdecl;
+  tta_getsize: function(const src: Pointer; srcSize: Integer;
+    headerSize: PInteger): Integer cdecl;
+  DLLLoaded: Boolean = False;
+
+implementation
+
+var
+  DLLHandle: THandle;
+
+procedure Init;
+begin
+  DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) +
+    'libtta_dll.dll'));
+  if DLLHandle >= 32 then
+  begin
+    @tta_encode := GetProcAddress(DLLHandle, 'encode');
+    @tta_decode := GetProcAddress(DLLHandle, 'decode');
+    @tta_getsize := GetProcAddress(DLLHandle, 'getsize');
+    DLLLoaded := Assigned(tta_encode) and Assigned(tta_decode);
+  end
+  else
+    DLLLoaded := False;
+end;
+
+procedure Deinit;
+begin
+  if not DLLLoaded then
+    exit;
+  FreeLibrary(DLLHandle);
+end;
+
+initialization
+
+Init;
+
+finalization
+
+Deinit;
+
+end.

io/IODecode.pas

@@ -128,7 +128,7 @@ begin
             '_' + Random($7FFFFFFF).ToHexString + XTOOL_MAPSUF2)), LFilename);
           try
             Move(SS2.Memory^, (PByte(SS1.Memory) + LEntry.Position)^,
-              LEntry.Size);
+              Min(SS2.Size, LEntry.Size));
           finally
             SS2.Free;
           end;

precompressor/PrecompCrypto.pas

@@ -91,7 +91,7 @@ begin
     DI2.OldSize := SI.NewSize;
     DI2.NewSize := SI.NewSize;
     Funcs^.LogScan1(CryptoCodecs[GetBits(SI.Option, 0, 5)], SI.Position,
-      SI.OldSize, SI.NewSize);
+      SI.OldSize, -1);
     Add(Instance, @SI, DI1.Codec, @DI2);
   end;
 end;
@@ -108,10 +108,10 @@ begin
     exit;
   if (Res > 0) and (StreamInfo^.OldSize > 0) then
   begin
-    StreamInfo^.NewSize := StreamInfo^.OldSize;
     Output(Instance, Input, StreamInfo^.OldSize);
+    StreamInfo^.NewSize := StreamInfo^.OldSize;
     Funcs^.LogScan2(CryptoCodecs[GetBits(StreamInfo^.Option, 0, 5)],
-      StreamInfo^.OldSize, StreamInfo^.NewSize);
+      StreamInfo^.OldSize, -1);
     Result := True;
   end;
 end;
@@ -143,7 +143,7 @@ begin
     end;
     Result := True;
     Funcs^.LogProcess(CryptoCodecs[GetBits(StreamInfo^.Option, 0, 5)], nil,
-      StreamInfo^.OldSize, StreamInfo^.NewSize, StreamInfo^.OldSize, Result);
+      StreamInfo^.OldSize, -1, -1, Result);
   end;
 end;
 
@@ -175,7 +175,7 @@ begin
     Output(Instance, Input, StreamInfo.OldSize);
     Result := True;
     Funcs^.LogRestore(CryptoCodecs[GetBits(StreamInfo.Option, 0, 5)], nil,
-      StreamInfo.OldSize, StreamInfo.NewSize, StreamInfo.OldSize, Result);
+      StreamInfo.OldSize, -1, -1, Result);
   end;
 end;
 

precompressor/PrecompEXE.pas

@@ -589,7 +589,7 @@ begin
       StreamInfo^.OldSize);
     Funcs^.LogProcess(PChar(Codec.Names[X]), nil, StreamInfo^.OldSize,
       StreamInfo^.NewSize, Res1, Result);
-    if Result = False then
+    if (Result = False) and (DIFF_TOLERANCE > 0) then
     begin
       Buffer := Funcs^.Allocator(Instance,
         Res1 + Max(StreamInfo^.OldSize, Res1));

precompressor/PrecompLZ4.pas

@@ -24,12 +24,14 @@ const
   L_MAXSIZE = 16 * 1024 * 1024;
   L_BLOCKSIZE = 0;
   L_BLOCKDEPENDENCY = 0;
+  L_ACCELERATION = 1;
 
 var
   SOList: array of array [0 .. CODEC_COUNT - 1] of TSOList;
   CodecAvailable, CodecEnabled: TArray<Boolean>;
   LBlockSize: Integer = L_BLOCKSIZE;
   LBlockDependency: Integer = L_BLOCKDEPENDENCY;
+  LAcceleration: Integer = L_ACCELERATION;
 
 function LZ4Init(Command: PChar; Count: Integer; Funcs: PPrecompFuncs): Boolean;
 var
@@ -57,6 +59,8 @@ begin
     if (CompareText(S, LZ4Codecs[LZ4_CODEC]) = 0) and LZ4DLL.DLLLoaded then
     begin
       CodecEnabled[LZ4_CODEC] := True;
+      if Funcs^.GetParam(Command, X, 'a') <> '' then
+        LAcceleration := StrToInt(Funcs^.GetParam(Command, X, 'a'));
     end
     else if (CompareText(S, LZ4Codecs[LZ4HC_CODEC]) = 0) and LZ4DLL.DLLLoaded
     then
@@ -86,7 +90,7 @@ begin
     if SOList[X, LZ4_CODEC].Count = 0 then
       SOList[X, LZ4_CODEC].Update([1]);
   SetLength(Options, 0);
-  for I := 3 to 12 do
+  for I := 2 to 12 do
     Insert(I, Options, Length(Options));
   for X := Low(SOList) to High(SOList) do
     if SOList[X, LZ4HC_CODEC].Count = 0 then
@@ -118,6 +122,7 @@ begin
   Option^ := 0;
   SetBits(Option^, LBlockSize, 12, 2);
   SetBits(Option^, LBlockDependency, 14, 1);
+  SetBits(Option^, LAcceleration, 15, 7);
   I := 0;
   while Funcs^.GetCodec(Command, I, False) <> '' do
   begin
@@ -125,6 +130,8 @@ begin
     if (CompareText(S, LZ4Codecs[LZ4_CODEC]) = 0) and LZ4DLL.DLLLoaded then
     begin
       SetBits(Option^, LZ4_CODEC, 0, 5);
+      if Funcs^.GetParam(Command, I, 'a') <> '' then
+        SetBits(Option^, StrToInt(Funcs^.GetParam(Command, I, 'a')), 15, 7);
       Result := True;
     end
     else if (CompareText(S, LZ4Codecs[LZ4HC_CODEC]) = 0) and LZ4DLL.DLLLoaded
@@ -188,6 +195,7 @@ begin
       SetBits(SI.Option, X, 0, 5);
       SetBits(SI.Option, LBlockSize, 12, 2);
       SetBits(SI.Option, LBlockDependency, 14, 1);
+      SetBits(SI.Option, LAcceleration, 15, 7);
       if System.Pos(SPrecompSep2, DI1.Codec) > 0 then
         SI.Status := TStreamStatus.Predicted
       else
@@ -230,8 +238,8 @@ begin
   end;
   if Res > StreamInfo^.OldSize then
   begin
-    StreamInfo^.NewSize := Res;
     Output(Instance, Buffer, Res);
+    StreamInfo^.NewSize := Res;
     Funcs^.LogScan2(LZ4Codecs[GetBits(StreamInfo^.Option, 0, 5)],
       StreamInfo^.OldSize, StreamInfo^.NewSize);
     Result := True;
@@ -261,12 +269,13 @@ begin
     if StreamInfo^.Status = TStreamStatus.Predicted then
       if GetBits(StreamInfo^.Option, 5, 7) <> I then
         continue;
+    Params := '';
     case X of
       LZ4_CODEC:
         begin
-          Params := '';
+          Params := 'a' + GetBits(StreamInfo^.Option, 15, 7).ToString;
-          Res1 := LZ4_compress_default(NewInput, Buffer,
+          Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y,
-            StreamInfo^.NewSize, Y);
+            GetBits(StreamInfo^.Option, 15, 7));
         end;
       LZ4HC_CODEC:
         begin
@@ -296,7 +305,7 @@ begin
       break;
   end;
   if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
-    (SOList[Instance][X].Count = 1)) then
+    (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
   begin
     Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));
     Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1,
@@ -332,14 +341,16 @@ begin
   X := GetBits(StreamInfo.Option, 0, 5);
   if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then
     exit;
+  Params := '';
   Buffer := Funcs^.Allocator(Instance,
     LZ4F_compressFrameBound(StreamInfo.NewSize, nil));
   case X of
     LZ4_CODEC:
       begin
-        Params := '';
+        Params := 'a' + GetBits(StreamInfo.Option, 15, 7).ToString;
-        Res1 := LZ4_compress_default(Input, Buffer, StreamInfo.NewSize,
+        Res1 := LZ4_compress_fast(Input, Buffer, StreamInfo.NewSize,
-          LZ4F_compressFrameBound(StreamInfo.NewSize, nil));
+          LZ4F_compressFrameBound(StreamInfo.NewSize, nil),
+          GetBits(StreamInfo.Option, 15, 7));
       end;
     LZ4HC_CODEC:
       begin

precompressor/PrecompLZO.pas

@@ -272,8 +272,8 @@ begin
   end;
   if Res > StreamInfo^.OldSize then
   begin
-    StreamInfo^.NewSize := Res;
     Output(Instance, Buffer, Res);
+    StreamInfo^.NewSize := Res;
     Funcs^.LogScan2(LZOCodecs[GetBits(StreamInfo^.Option, 0, 5)],
       StreamInfo^.OldSize, StreamInfo^.NewSize);
     Result := True;
@@ -301,6 +301,7 @@ begin
     if StreamInfo^.Status = TStreamStatus.Predicted then
       if GetBits(StreamInfo^.Option, 5, 7) <> I then
         continue;
+    Params := '';
     Res1 := StreamInfo^.NewSize;
     case X of
       LZO1X_CODEC:
@@ -326,7 +327,7 @@ begin
       break;
   end;
   if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
-    (SOList[Instance][X].Count = 1)) then
+    (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
   begin
     Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));
     Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1,
@@ -361,6 +362,7 @@ begin
   X := GetBits(StreamInfo.Option, 0, 5);
   if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then
     exit;
+  Params := '';
   Buffer := Funcs^.Allocator(Instance, StreamInfo.NewSize);
   Res1 := StreamInfo.NewSize;
   case X of

precompressor/PrecompMain.pas

@@ -7,7 +7,7 @@ interface
 uses
   Threading, Utils, SynCommons, ParseClass, ParseExpr,
   PrecompUtils, PrecompCrypto, PrecompZLib, PrecompLZ4, PrecompLZO, PrecompZSTD,
-  PrecompOodle, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE,
+  PrecompOodle, PrecompMedia, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE,
   WinAPI.Windows, WinAPI.ShlObj,
   System.SysUtils, System.Classes, System.SyncObjs, System.Math, System.Types,
   System.StrUtils, System.RTLConsts, System.TimeSpan, System.Diagnostics,
@@ -24,8 +24,7 @@ type
     ChunkSize, Threads: Integer;
     Depth: Integer;
     LowMem: Boolean;
-    DBaseFile: String;
+    DBaseFile, ExtractDir: String;
-    DedupFile: String;
   end;
 
   PDecodeOptions = ^TDecodeOptions;
@@ -34,7 +33,6 @@ type
     Method: String;
     ChunkCount, Threads: Integer;
     Depth: Integer;
-    DedupFile: String;
     DedupSysMem, DedupGPUMem: Int64;
   end;
 
@@ -53,10 +51,10 @@ function PrecompReadFuture(Index: Integer; Position: NativeInt; Buffer: Pointer;
 procedure PrecompLogScan1(Codec: PChar; Position: Int64;
   InSize, OutSize: Integer)cdecl;
 procedure PrecompLogScan2(Codec: PChar; InSize, OutSize: Integer)cdecl;
-procedure PrecompLogProcess(Codec, Method: PChar;
+procedure PrecompLogProcess(Codec, Method: PChar; Size1, Size2, Size3: Integer;
-  OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl;
+  Status: Boolean)cdecl;
-procedure PrecompLogRestore(Codec, Method: PChar;
+procedure PrecompLogRestore(Codec, Method: PChar; Size1, Size2, Size3: Integer;
-  OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl;
+  Status: Boolean)cdecl;
 procedure PrecompLogPatch1(OldSize, NewSize, PatchSize: Integer;
   Status: Boolean)cdecl;
 procedure PrecompLogPatch2(OldSize, NewSize, PatchSize: Integer;
@@ -101,10 +99,11 @@ var
   IntArray: array [0 .. 1] of Int64;
   Codecs: array of TPrecompressor;
   DBFile: String = '';
+  ExtDir: String = '';
   UseDB: Boolean = False;
-  DupFile: String = '';
   StoreDD: Boolean = False;
-  DupGUID: TGUID;
+  VERBOSE: Boolean = False;
+  EXTRACT: Boolean = False;
   DupSysMem: Int64 = 0;
   EncInfo: TEncInfo;
   ConTask: TTask;
@@ -131,11 +130,14 @@ begin
   WriteLn(ErrOutput, '');
   WriteLn(ErrOutput, 'Advanced parameters:');
   WriteLn(ErrOutput,
-    '  --dbase=#  - use database (#=filename to save db, optional)');
+    '  --dbase=#   - use database (#=filename to save db, optional)');
   WriteLn(ErrOutput,
-    '  --dedup=#  - use stream deduplication (#=filename to save db, optional)');
+    '  --dedup=#   - use stream deduplication (#=filename to save db, optional)');
   WriteLn(ErrOutput,
-    '  --mem=#    - deduplication ram usage limit (#=size) [75p]');
+    '  --mem=#     - deduplication ram usage limit (#=size) [75p]');
+  WriteLn(ErrOutput,
+    '  --diff=#    - set xdelta threshold to accept streams [5p]');
+  WriteLn(ErrOutput, '  --extract=# - extract streams to directory path');
   WriteLn(ErrOutput, '');
 end;
 
@@ -182,16 +184,18 @@ begin
     if Options.DBaseFile <> '' then
       UseDB := True;
     StoreDD := ArgParse.AsBoolean('--dedup');
-    Options.DedupFile := ArgParse.AsString('--dedup=');
     S := ArgParse.AsString('--diff=', 0, '5p');
     S := ReplaceText(S, 'p', '%');
     DIFF_TOLERANCE := Max(0.00, ExpParse.Evaluate(S));
     VERBOSE := ArgParse.AsBoolean('--verbose');
+    Options.ExtractDir := ArgParse.AsString('--extract=');
+    if Options.ExtractDir <> '' then
+      EXTRACT := DirectoryExists(Options.ExtractDir);
   finally
     ArgParse.Free;
     ExpParse.Free;
   end;
-  if VERBOSE then
+  if VERBOSE or EXTRACT then
     Options.Threads := 1;
 end;
 
@@ -210,7 +214,6 @@ begin
     S := ReplaceText(S, 'p', '%');
     S := ReplaceText(S, '%', '%*' + CPUCount.ToString);
     Options.Threads := Max(1, Round(ExpParse.Evaluate(S)));
-    Options.DedupFile := ArgParse.AsString('--dedup=');
     S := ArgParse.AsString('--mem=', 0, '75p');
     S := ReplaceText(S, 'KB', '* 1024^1');
     S := ReplaceText(S, 'MB', '* 1024^2');
@@ -267,14 +270,13 @@ type
     StrIdx: TArray<Integer>;
   end;
 
-  TDupRec = record
-    Dict: TSynDictionary;
-    Index: Integer;
-  end;
-
 var
-  Database: TSynDictionary;
+  DBInfo: TArray<TArray<TDatabase>>;
-  Duplicates1: array [0 .. 1] of TDupRec;
+  DBCount: TArray<Integer>;
+  DDInfo: TArray<TArray<TDuplicate1>>;
+  DDCount1: TArray<Integer>;
+  DDList1: TArray<Int64>;
+  DDIndex: Integer;
   ComVars1: TArray<TCommonVarsEnc>;
   Tasks: TArray<TTask>;
   CurCodec: TArray<Byte>;
@@ -285,6 +287,7 @@ var
   Scanned1, Scanned2, Processed: TArray<Boolean>;
   LogInt: Integer;
   LogInt64: Int64;
+  LogPtr: Pointer;
 
 procedure CodecInit(Count: Integer; Method: String);
 var
@@ -303,6 +306,7 @@ begin
   Insert(PrecompLZO.Codec, Codecs, Length(Codecs));
   Insert(PrecompZSTD.Codec, Codecs, Length(Codecs));
   Insert(PrecompOodle.Codec, Codecs, Length(Codecs));
+  Insert(PrecompMedia.Codec, Codecs, Length(Codecs));
   for X := Low(Codecs) to High(Codecs) do
     for Y := Low(Codecs[X].Names) to High(Codecs[X].Names) do
       Insert(Codecs[X].Names[Y], List, Length(List));
@@ -394,64 +398,101 @@ end;
 
 procedure PrecompLogScan1(Codec: PChar; Position: Int64;
   InSize, OutSize: Integer);
+var
+  S: String;
 begin
   if not VERBOSE then
     exit;
   with ComVars1[CurDepth[0]] do
   begin
+    if OutSize < 0 then
+      S := '(%d)'
+    else
+      S := '(%d >> %d)';
     if (OutSize > 0) and (Position < DataStore.Size(0)) and
       (MemOutput1[0].Position - CurPos1[0] = OutSize) then
-      WriteLn(ErrOutput, Format('[%d] Actual %s stream found at %s (%d >> %d)',
+      WriteLn(ErrOutput, Format('[%d] Actual %s stream found at %s ' + S,
         [CurDepth[0], Codec, (DataStore.Position(0) + Position).ToHexString,
         InSize, OutSize]))
     else
-      WriteLn(ErrOutput,
+      WriteLn(ErrOutput, Format('[%d] Possible %s stream located at %s ' + S,
-        Format('[%d] Possible %s stream located at %s (%d >> %d)',
         [CurDepth[0], Codec, (DataStore.Position(0) + Position).ToHexString,
         InSize, OutSize]));
   end;
 end;
 
 procedure PrecompLogScan2(Codec: PChar; InSize, OutSize: Integer);
+var
+  S: String;
 begin
   if not VERBOSE then
     exit;
-  WriteLn(ErrOutput, Format('[%d] Confirmed %s stream at %s (%d >> %d)',
+  if OutSize < 0 then
+    S := '(%d)'
+  else
+    S := '(%d >> %d)';
+  WriteLn(ErrOutput, Format('[%d] Confirmed %s stream at %s ' + S,
     [CurDepth[0], Codec, LogInt64.ToHexString, InSize, OutSize]));
 end;
 
-procedure PrecompLogProcess(Codec, Method: PChar;
+procedure PrecompLogProcess(Codec, Method: PChar; Size1, Size2, Size3: Integer;
-  OriginalSize, InSize, OutSize: Integer; Status: Boolean);
+  Status: Boolean);
 var
-  S: String;
+  S1, S2: String;
 begin
-  if not VERBOSE then
+  if VERBOSE then
-    exit;
+  begin
-  if Status then
+    if Size2 < 0 then
-    S := '[%d] Processed %s stream at %s (%d >> %d >> %d)' +
+      S1 := '(%d)'
-      IfThen(String(Method) <> '', ' using %s', '') + ' successfully'
+    else if Size3 < 0 then
-  else
+      S1 := '(%d >> %d)'
-    S := '[%d] Processing %s stream at %s (%d >> %d >> %d)' +
+    else
-      IfThen(String(Method) <> '', ' using %s', '') + ' has failed';
+      S1 := '(%d >> %d >> %d)';
-  WriteLn(ErrOutput, Format(S, [CurDepth[0], Codec, LogInt64.ToHexString,
+    if Status then
-    OriginalSize, InSize, OutSize, Method]));
+      S2 := '[%d] Processed %s stream at %s ' + S1 +
+        IfThen(String(Method) <> '', ' using ' + String(Method), '') +
+        ' successfully'
+    else
+      S2 := '[%d] Processing %s stream at %s ' + S1 +
+        IfThen(String(Method) <> '', ' using ' + String(Method), '') +
+        ' has failed';
+    WriteLn(ErrOutput, Format(S2, [CurDepth[0], Codec, LogInt64.ToHexString,
+      Size1, Size2, Size3]));
+  end;
+  if EXTRACT and (CurDepth[0] = 0) then
+  begin
+    S1 := '%s_%s.raw';
+    with TFileStream.Create(ExtDir + Format(S1, [LogInt64.ToHexString, Codec]),
+      fmCreate) do
+      try
+        WriteBuffer(LogPtr^, Size1);
+      finally
+        Free;
+      end;
+  end;
 end;
 
-procedure PrecompLogRestore(Codec, Method: PChar;
+procedure PrecompLogRestore(Codec, Method: PChar; Size1, Size2, Size3: Integer;
-  OriginalSize, InSize, OutSize: Integer; Status: Boolean);
+  Status: Boolean);
 var
-  S: String;
+  S1, S2: String;
 begin
   if not VERBOSE then
     exit;
-  if Status then
+  if Size2 < 0 then
-    S := '[%d] Restored %s stream at %s (%d >> %d >> %d)' +
+    S1 := '(%d)'
-      IfThen(String(Method) <> '', ' using %s', '') + ' successfully'
+  else if Size3 < 0 then
+    S1 := '(%d >> %d)'
   else
-    S := '[%d] Restoring %s stream at %s (%d >> %d >> %d)' +
+    S1 := '(%d >> %d >> %d)';
-      IfThen(String(Method) <> '', ' using %s', '') + ' has failed';
+  if Status then
-  WriteLn(ErrOutput, Format(S, [CurDepth[0], Codec, LogInt64.ToHexString,
+    S2 := '[%d] Restored %s stream at %s ' + S1 + IfThen(String(Method) <> '',
-    OriginalSize, InSize, OutSize, Method]));
+      ' using ' + String(Method), '') + ' successfully'
+  else
+    S2 := '[%d] Restoring %s stream at %s ' + S1 + IfThen(String(Method) <> '',
+      ' using ' + String(Method), '') + ' has failed';
+  WriteLn(ErrOutput, Format(S2, [CurDepth[0], Codec, LogInt64.ToHexString,
+    Size1, Size2, Size3]));
 end;
 
 procedure PrecompLogPatch1(OldSize, NewSize, PatchSize: Integer;
@@ -553,7 +594,9 @@ begin
       begin
         MemOutput1[Instance].Position := CurPos1[Instance];
         exit;
-      end;
+      end
+      else if LCodec = CurCodec[Instance] then
+        LOption := Info^.Option;
     end
     else
     begin
@@ -616,53 +659,130 @@ begin
     CurTransfer[Instance] := String(Codec);
 end;
 
-function CheckDB(Dictionary: TSynDictionary; const StreamInfo: TEncodeSI;
+function CheckDB(StreamInfo: TEncodeSI; Database: PDatabase): Boolean;
-  var Database: TDatabase): Boolean;
 var
-  DBKey: Int64;
+  A: Word;
+  I: Integer;
+  LCount: Integer;
+  DB: PDatabase;
 begin
   Result := False;
-  Int64Rec(DBKey).Lo := StreamInfo.Checksum;
+  A := LongRec(StreamInfo.Checksum).Lo;
-  Int64Rec(DBKey).Hi := StreamInfo.OldSize;
+  AtomicExchange(LCount, DBCount[A]);
-  Result := Dictionary.FindAndCopy(DBKey, Database);
+  for I := 0 to LCount - 1 do
-end;
-
-procedure AddDB(Dictionary: TSynDictionary; const StreamInfo: TEncodeSI;
-  const Database: TDatabase);
-var
-  DBKey: Int64;
-begin
-  Int64Rec(DBKey).Lo := StreamInfo.Checksum;
-  Int64Rec(DBKey).Hi := StreamInfo.OldSize;
-  Dictionary.AddOrUpdate(DBKey, Database);
-end;
-
-function CheckDup(var DupRec: TDupRec; const StreamInfo: TEncodeSI;
-  var StreamKey, DupCount: Integer): Boolean;
-var
-  DupKey: Int64;
-  DupInfo: PDuplicate;
-  DupAdded: Boolean;
-begin
-  Result := False;
-  Inc(DupRec.Index);
-  Int64Rec(DupKey).Lo := StreamInfo.Checksum;
-  Int64Rec(DupKey).Hi := StreamInfo.OldSize;
-  DupInfo := DupRec.Dict.FindValueOrAdd(DupKey, DupAdded);
-  if not DupAdded then
   begin
+    DB := @DBInfo[A, I];
+    if (DB^.Size = StreamInfo.OldSize) and (DB^.Checksum = StreamInfo.Checksum)
+    then
+    begin
+      if Assigned(Database) then
+        Move(DB^, Database^, SizeOf(TDatabase));
+      Result := True;
+      break;
+    end;
+  end;
+end;
+
+procedure AddDB(StreamInfo: TEncodeSI);
+var
+  A: Word;
+  I: Integer;
+  DB: TDatabase;
+begin
+  A := LongRec(StreamInfo.Checksum).Lo;
+  if not CheckDB(StreamInfo, nil) then
+  begin
+    GlobalSync.Acquire;
+    try
+      DB.Size := StreamInfo.OldSize;
+      DB.Codec := StreamInfo.Codec;
+      DB.Option := StreamInfo.Option;
+      DB.Checksum := StreamInfo.Checksum;
+      DB.Status := StreamInfo.Status;
+      Insert(DB, DBInfo[A], Length(DBInfo[A]));
+      Inc(DBCount[A]);
+    finally
+      GlobalSync.Release;
+    end;
+  end;
+end;
+
+function CheckDD(StreamInfo: TEncodeSI; Database: PDuplicate1;
+  Index: PInteger): Boolean;
+var
+  A: Word;
+  I: Integer;
+  LCount: Integer;
+  DD: PDuplicate1;
+begin
+  Result := False;
+  A := LongRec(StreamInfo.Checksum).Lo;
+  LCount := DDCount1[A];
+  for I := 0 to LCount - 1 do
+  begin
+    DD := @DDInfo[A, I];
+    if (DD^.Size = StreamInfo.OldSize) and (DD^.Checksum = StreamInfo.Checksum)
+    then
+    begin
+      if Assigned(Database) then
+        Move(DD^, Database^, SizeOf(TDuplicate1));
+      if Assigned(Index) then
+        Index^ := I;
+      Result := True;
+      break;
+    end;
+  end;
+end;
+
+function FindDD(StreamInfo: TEncodeSI; Index, Count: PInteger): Boolean;
+var
+  A: Word;
+  I: Integer;
+  DD: PDuplicate1;
+begin
+  Result := False;
+  if CheckDD(StreamInfo, nil, @I) then
+  begin
+    A := LongRec(StreamInfo.Checksum).Lo;
+    DD := @DDInfo[A, I];
+    if Assigned(Index) then
+      Index^ := DD^.Index;
+    if Assigned(Count) then
+      Count^ := DD^.Count;
+    Result := True;
+  end;
+end;
+
+function FindOrAddDD(StreamInfo: TEncodeSI; Index, Count: PInteger): Boolean;
+var
+  A: Word;
+  I: Integer;
+  DD: TDuplicate1;
+  I64: Int64;
+begin
+  Result := False;
+  Inc(DDIndex);
+  A := LongRec(StreamInfo.Checksum).Lo;
+  if not CheckDD(StreamInfo, nil, @I) then
+  begin
+    DD.Size := StreamInfo.OldSize;
+    DD.Checksum := StreamInfo.Checksum;
+    DD.Index := DDIndex;
+    DD.Count := 0;
+    I := Length(DDInfo[A]);
+    Insert(DD, DDInfo[A], I);
+    Int64Rec(I64).Words[0] := A;
+    Int64Rec(I64).Hi := DDCount1[A];
+    Insert(I64, DDList1, Length(DDList1));
+    Inc(DDCount1[A]);
     Result := True;
-    Inc(DupInfo^.Count);
-    StreamKey := DupInfo^.Index;
-    DupCount := DupInfo^.Count;
   end
   else
-  begin
+    Inc(DDInfo[A, I].Count);
-    DupInfo^.Count := 0;
+  if Assigned(Index) then
-    DupInfo^.Index := DupRec.Index;
+    Index^ := DDInfo[A, I].Index;
-    StreamKey := -1;
+  if Assigned(Count) then
-    DupCount := 0;
+    Count^ := DDInfo[A, I].Count;
-  end;
 end;
 
 procedure Scan1(Index, Depth: Integer);
@@ -729,6 +849,8 @@ begin
           X := DataStore.ActualSize(Index) -
             NativeInt(SI2.Position - DataStore.Position(Index));
           LogInt64 := SI2.Position;
+          LogPtr := PByte(DataStore.Slot(Index).Memory) +
+            NativeInt(SI2.Position - DataStore.Position(Index));
           if (SI1.OldSize <= X) and Codecs[SI2.Codec].Scan2(Index, Depth,
             PByte(DataStore.Slot(Index).Memory) +
             NativeInt(SI2.Position - DataStore.Position(Index)), X, @SI1, @J,
@@ -830,10 +952,11 @@ begin
     SI1.Resource := SI2.Resource;
     SI1.Option := SI2.Option;
     SI1.Status := SI2.Status;
-    LogInt64 := DataStore.Position(0) + SI2.ActualPosition;
+    LogInt64 := DataStore.Position(ThreadIndex) + SI2.ActualPosition;
+    LogPtr := PByte(DataStore.Slot(ThreadIndex).Memory) + SI2.ActualPosition;
     if UseDB and (SI2.Codec > 2) then
     begin
-      DBBool := CheckDB(Database, SI2, DBTyp);
+      DBBool := CheckDB(SI2, @DBTyp);
       if DBBool and (SI2.Codec = DBTyp.Codec) then
       begin
         if DBTyp.Status = TStreamStatus.Invalid then
@@ -841,7 +964,7 @@ begin
         else
         begin
           SI1.Option := DBTyp.Option;
-          SI1.Status := TStreamStatus.Predicted;
+          SI1.Status := TStreamStatus.Database;
         end;
       end;
     end;
@@ -882,13 +1005,14 @@ begin
     if UseDB then
       if not DBBool then
       begin
-        DBTyp.Codec := SI2.Codec;
-        DBTyp.Option := SI1.Option;
         if Result then
-          DBTyp.Status := TStreamStatus.Predicted
+        begin
+          SI2.Option := SI1.Option;
+          SI2.Status := TStreamStatus.Predicted
+        end
         else
-          DBTyp.Status := TStreamStatus.Invalid;
+          SI2.Status := TStreamStatus.Invalid;
-        AddDB(Database, SI2, DBTyp);
+        AddDB(SI2);
       end;
     if Result then
     begin
@@ -1033,10 +1157,9 @@ procedure EncInit(Input, Output: TStream; Options: PEncodeOptions);
 var
   UI32: UInt32;
   I, J, K: Integer;
+  W: Word;
   Bytes: TBytes;
   NI: NativeInt;
-  DBKey: Int64;
-  DBTyp: TDatabase;
   S: String;
   DupMethod: Boolean;
 begin
@@ -1046,15 +1169,21 @@ begin
     ThreadSync[I] := TCriticalSection.Create;
   I := XTOOL_PRECOMP;
   Output.WriteBuffer(I, I.Size);
-  CreateGUID(DupGUID);
+  if UseDB then
-  Output.WriteBuffer(DupGUID, SizeOf(TGUID));
-  Database := TSynDictionary.Create(TypeInfo(TInt64DynArray),
-    TypeInfo(TDatabaseDynArray));
-  for I := Low(Duplicates1) to High(Duplicates1) do
   begin
-    Duplicates1[I].Dict := TSynDictionary.Create(TypeInfo(TInt64DynArray),
+    SetLength(DBInfo, $10000);
-      TypeInfo(TDuplicateDynArray));
+    SetLength(DBCount, $10000);
-    Duplicates1[I].Index := -1;
+    for I := Low(DBInfo) to High(DBInfo) do
+      DBCount[I] := 0;
+  end;
+  if StoreDD then
+  begin
+    SetLength(DDInfo, $10000);
+    SetLength(DDCount1, $10000);
+    SetLength(DDList1, 0);
+    for I := Low(DDInfo) to High(DDInfo) do
+      DDCount1[I] := 0;
+    DDIndex := -1;
   end;
   SetLength(Tasks, Options^.Threads);
   SetLength(CurCodec, Options^.Threads);
@@ -1126,7 +1255,6 @@ begin
     end;
   CodecInit(Options^.Threads, Options^.Method);
   DBFile := Options^.DBaseFile;
-  DupFile := Options^.DedupFile;
   if FileExists(ExtractFilePath(Utils.GetModuleName) + DBFile) then
   begin
     with TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + DBFile,
@@ -1140,16 +1268,19 @@ begin
     end;
     with WorkStream[0] do
     begin
-      J := PInteger(Memory)^;
+      Position := 0;
-      for I := 0 to J - 1 do
+      while Position < Size do
       begin
-        NI := Integer.Size + (I * (SizeOf(Int64) + SizeOf(TDatabase)));
+        ReadBuffer(W, W.Size);
-        DBKey := PInt64(PByte(Memory) + NI)^;
+        ReadBuffer(J, J.Size);
-        DBTyp := PDatabase(PByte(Memory) + NI + SizeOf(Int64))^;
+        DBCount[W] := J;
-        Database.Add(DBKey, DBTyp);
+        SetLength(DBInfo[W], J);
+        for K := 0 to J - 1 do
+          ReadBuffer(DBInfo[W, K], SizeOf(TDatabase));
       end;
     end;
   end;
+  ExtDir := IncludeTrailingBackSlash(Options^.ExtractDir);
   Output.WriteBuffer(Options^.Depth, Options^.Depth.Size);
   S := '';
   I := 0;
@@ -1237,9 +1368,6 @@ begin
       Tasks[I].Free;
     WorkStream[I].Free;
   end;
-  Database.Free;
-  for I := Low(Duplicates1) to High(Duplicates1) do
-    Duplicates1[I].Dict.Free;
   FreeResources;
   GlobalSync.Free;
   for I := Low(ThreadSync) to High(ThreadSync) do
@@ -1265,19 +1393,19 @@ var
   StreamCount: Integer;
   BlockSize: Int64;
   UI32: UInt32;
-  I, J, X: Integer;
+  I, J, K, X: Integer;
+  W: Word;
+  I64: Int64;
   LastStream, LastPos: Int64;
   LastIndex: Integer;
   CurrSize: Cardinal;
   DupBool: Boolean;
-  DupKey, DupCount: Integer;
+  DupIdx1, DupIdx2, DupCount: Integer;
-  DBKey: Int64;
+  DupTyp: TDuplicate2;
-  DBTyp: TDatabase;
-  DupTyp: TDuplicate;
 begin
   if (Depth = 0) then
   begin
-    if (DupFile = '') and StoreDD then
+    if StoreDD then
       TempOutput := TPrecompVMStream.Create
     else
       TempOutput := Output;
@@ -1285,6 +1413,7 @@ begin
   else
     TempOutput := Output;
   Result := False;
+  DupIdx1 := 0;
   with ComVars1[Depth] do
   begin
     LastStream := 0;
@@ -1381,16 +1510,15 @@ begin
             begin
               Inc(StreamCount);
               DupBool := False;
-              if (Depth = 0) and ((DupFile <> '') or StoreDD) then
+              if (Depth = 0) and StoreDD then
-                DupBool := CheckDup(Duplicates1[0], StreamInfo, DupKey,
+                DupBool := not FindOrAddDD(StreamInfo, @DupIdx2, @DupCount);
-                  DupCount);
               if DupBool then
               begin
-                if DupCount = 2 then
+                if DupCount = 1 then
                   Inc(EncInfo.DecMem2, StreamInfo.OldSize);
                 FillChar(StreamHeader, SizeOf(TStreamHeader), 0);
                 StreamHeader.Kind := DUPLICATED_STREAM;
-                StreamHeader.Option := DupKey;
+                StreamHeader.Option := DupIdx2;
               end
               else
               begin
@@ -1427,19 +1555,19 @@ begin
             Result := True
           else if Depth > 0 then
             exit;
+          I64 := MemStream[I].Position;
           MemStream[I].Position := 0;
           MemStream[I].WriteBuffer(StreamCount, StreamCount.Size);
           MemStream[I].WriteBuffer(BlockSize, BlockSize.Size);
-          TempOutput.WriteBuffer(MemStream[I].Memory^, MemStream[I].Position +
+          TempOutput.WriteBuffer(MemStream[I].Memory^, I64);
-            StreamCount * SizeOf(TStreamHeader));
           InfoStore1[I].Index := LastIndex;
           J := InfoStore1[I].Get(StreamInfo);
           while J >= 0 do
           begin
             DupBool := False;
-            if (Depth = 0) and ((DupFile <> '') or StoreDD) then
+            if (Depth = 0) and StoreDD then
-              DupBool := CheckDup(Duplicates1[1], StreamInfo, DupKey, DupCount);
+              DupBool := FindDD(StreamInfo, @DupIdx2, @DupCount);
-            if not DupBool then
+            if (DupBool = False) or (DupIdx1 = DupIdx2) then
             begin
               if StreamInfo.ExtSize < 0 then
               begin
@@ -1465,6 +1593,7 @@ begin
                   StreamInfo.ExtSize.Size);
               end;
             end;
+            Inc(DupIdx1);
             if Succ(J - LastIndex) = StreamCount then
               break;
             J := InfoStore1[I].Get(StreamInfo);
@@ -1519,14 +1648,16 @@ begin
       with WorkStream[0] do
       begin
         Position := 0;
-        J := Database.Count;
+        for W := 0 to $10000 - 1 do
-        WriteBuffer(J, J.Size);
-        for I := 0 to J - 1 do
         begin
-          DBKey := PInt64(Database.Keys.ElemPtr(I))^;
+          J := DBCount[I];
-          WriteBuffer(DBKey, SizeOf(Int64));
+          if J > 0 then
-          DBTyp := PDatabase(Database.Values.ElemPtr(I))^;
+          begin
-          WriteBuffer(DBTyp, SizeOf(TDatabase));
+            WriteBuffer(W, W.Size);
+            WriteBuffer(J, J.Size);
+            for K := 0 to J - 1 do
+              WriteBuffer(DBInfo[W, K], SizeOf(TDatabase));
+          end;
         end;
       end;
       with TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + DBFile,
@@ -1536,40 +1667,27 @@ begin
         Free;
       end;
     end;
-    if (DupFile <> '') or StoreDD then
+    if StoreDD then
     begin
-      for I := Duplicates1[0].Dict.Count - 1 downto 0 do
-      begin
-        if PDuplicate(Duplicates1[0].Dict.Values.ElemPtr(I))^.Count < 1 then
-          Duplicates1[0].Dict.DeleteAt(I);
-      end;
       with WorkStream[0] do
       begin
         Position := 0;
-        WriteBuffer(DupGUID, SizeOf(TGUID));
+        UI32 := 0;
-        Duplicates1[0].Dict.Values.Sort(DuplicateSortCompare);
+        for I := Low(DDList1) to High(DDList1) do
-        J := Duplicates1[0].Dict.Count;
-        WriteBuffer(J, J.Size);
-        for I := 0 to J - 1 do
         begin
-          DupTyp := PDuplicate(Duplicates1[0].Dict.Values.ElemPtr(I))^;
+          J := Int64Rec(DDList1[I]).Words[0];
-          WriteBuffer(DupTyp, SizeOf(TDuplicate));
+          X := Int64Rec(DDList1[I]).Hi;
+          if DDInfo[J, X].Count > 0 then
+          begin
+            DupTyp.Index := DDInfo[J, X].Index;
+            DupTyp.Count := DDInfo[J, X].Count;
+            WriteBuffer(DupTyp, SizeOf(TDuplicate2));
+            Inc(UI32);
+          end;
         end;
       end;
-      if DupFile <> '' then
+      Output.WriteBuffer(UI32, UI32.Size);
-      begin
+      Output.WriteBuffer(WorkStream[0].Memory^, WorkStream[0].Position);
-        with TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + DupFile,
-          FSMode(FileExists(ExtractFilePath(Utils.GetModuleName) + DupFile))) do
-        begin
-          Position := Size;
-          WriteBuffer(WorkStream[0].Memory^, WorkStream[0].Position);
-        end;
-      end
-      else
-        Output.WriteBuffer(WorkStream[0].Memory^, WorkStream[0].Position);
-    end;
-    if (DupFile = '') and StoreDD then
-    begin
       Output.CopyFrom(TempOutput, 0);
       TempOutput.Free;
     end;
@@ -1633,10 +1751,9 @@ var
   NStream: TArrayStream;
   DataMgr: TDataManager;
   ComVars2: TArray<TCommonVarsDec>;
-  Duplicates2: TSynDictionary;
+  DDList2: TArray<TDuplicate2>;
-  DupIdx1: Integer;
+  DDCount2: Integer;
-  DupIdx2: TArray<Integer>;
+  DDIndex1, DDIndex2: Integer;
-  DupBool: TArray<Boolean>;
   BlockPos: Int64;
 
 procedure PrecompOutput2(Instance: Integer; const Buffer: Pointer;
@@ -1644,8 +1761,9 @@ procedure PrecompOutput2(Instance: Integer; const Buffer: Pointer;
 begin
   with ComVars2[CurDepth[Instance]] do
     DecOutput[Instance].WriteBuffer(Buffer^, Size);
-  if (CurDepth[Instance] = 0) and (DupBool[Instance]) then
+  if StoreDD and (CurDepth[Instance] = 0) then
-    DataMgr.Write(DupIdx2[Instance], Buffer^, Size);
+    if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index)) then
+      DataMgr.Write(DDIndex1, Buffer, Size);
 end;
 
 procedure PrecompOutput3(Instance: Integer; const Buffer: Pointer;
@@ -1653,8 +1771,6 @@ procedure PrecompOutput3(Instance: Integer; const Buffer: Pointer;
 begin
   with ComVars2[CurDepth[Instance]] do
     MemOutput1[Instance].WriteBuffer(Buffer^, Size);
-  if (CurDepth[Instance] = 0) and (DupBool[Instance]) then
-    DataMgr.Write(DupIdx2[Instance], Buffer^, Size);
 end;
 
 procedure Restore(MT: Boolean; Index, Depth: Integer);
@@ -1675,18 +1791,11 @@ begin
     while X < StreamCount[Index]^ do
     begin
       SH := PStreamHeader(MemStream1[Index].Memory) + X;
-      if (Depth = 0) then
-      begin
-        DupIdx2[Index] := DupIdx1 + X;
-        DupBool[Index] := Duplicates2.FindAndCopy(DupIdx2[Index], Y);
-        if DupBool[Index] then
-          DataMgr.Add(DupIdx2[Index], SH^.OldSize, Y);
-      end;
       if MT then
       begin
         LOutput := @PrecompOutput3;
         Pos := StreamInfo[Index]^.Pos[X];
-        X64 := Pos + SH^.NewSize;
+        X64 := Pos + Max(SH^.OldSize, SH^.NewSize);
         while (BlockPos < X64) do
         begin
           if IsErrored(Tasks) or (BlockPos < 0) then
@@ -1697,6 +1806,13 @@ begin
       end
       else
       begin
+        if StoreDD and (Depth = 0) then
+        begin
+          Inc(DDIndex1);
+          if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index))
+          then
+            DataMgr.Add(DDIndex1, SH^.OldSize, DDList2[DDIndex2].Count);
+        end;
         LOutput := @PrecompOutput2;
         DecInput[Index].ReadBuffer(UI32, UI32.Size);
         if UI32 > 0 then
@@ -1740,9 +1856,12 @@ begin
       end;
       CurCodec[Index] := SH^.Codec;
       CurDepth[Index] := Depth;
+      Y := GetBits(SI.Option, 0, 5);
+      if not InRange(Y, 0, Pred(Length(Codecs[SH^.Codec].Names))) then
+        Y := 0;
       if (Codecs[SH^.Codec].Restore(Index, Depth, Ptr1, Ptr2, SI, LOutput,
         @PrecompFunctions) = False) then
-        raise Exception.CreateFmt(SPrecompError3, [Codecs[SH^.Codec].Names[0]]);
+        raise Exception.CreateFmt(SPrecompError3, [Codecs[SH^.Codec].Names[Y]]);
       NStream.Update(0, CalcSysMem);
       if MT then
       begin
@@ -1751,7 +1870,13 @@ begin
         StreamInfo[Index]^.Completed[X] := True;
       end
       else
+      begin
+        if StoreDD and (Depth = 0) then
+          if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index))
+          then
+            Inc(DDIndex2);
         Inc(Pos, SH^.NewSize);
+      end;
       X := AtomicIncrement(StreamIdx[Index]^);
     end;
   end;
@@ -1772,24 +1897,16 @@ var
   I, J: Integer;
   Bytes: TBytes;
   UI32: UInt32;
-  DupTyp: TDuplicate;
+  DupTyp: TDuplicate1;
-  LStream: TStream;
+  LResData: TResData;
-  LGUID: TGUID;
-  LResData: PResData;
 begin
   GlobalSync := TCriticalSection.Create;
   SetLength(ThreadSync, Options^.Threads);
   for I := Low(ThreadSync) to High(ThreadSync) do
     ThreadSync[I] := TCriticalSection.Create;
   DupSysMem := Options^.DedupSysMem;
-  NStream.Add(TypeInfo(TMemoryStream), CalcSysMem);
+  NStream.Add(TypeInfo(TMemoryStream) { , CalcSysMem } );
-  NStream.Add(TypeInfo(TPrecompVMStream));
+  // NStream.Add(TypeInfo(TPrecompVMStream));
-  Duplicates2 := TSynDictionary.Create(TypeInfo(TIntegerDynArray),
-    TypeInfo(TIntegerDynArray));
-  DupIdx1 := 0;
-  SetLength(DupIdx2, Options^.Threads);
-  SetLength(DupBool, Options^.Threads);
-  Input.ReadBuffer(DupGUID, SizeOf(TGUID));
   Input.ReadBuffer(Options^.Depth, Options^.Depth.Size);
   Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size);
   SetLength(Bytes, LongRec(I).Bytes[0]);
@@ -1798,15 +1915,14 @@ begin
   Input.ReadBuffer(I, I.Size);
   for J := 0 to I - 1 do
   begin
-    New(LResData);
     Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size);
     SetLength(Bytes, LongRec(I).Bytes[0]);
     Input.ReadBuffer(Bytes[0], LongRec(I).Bytes[0]);
-    LResData^.Name := StringOf(Bytes);
+    LResData.Name := StringOf(Bytes);
-    Input.ReadBuffer(LResData^.Size, LResData^.Size.Size);
+    Input.ReadBuffer(LResData.Size, LResData.Size.Size);
-    GetMem(LResData^.Data, LResData^.Size);
+    GetMem(LResData.Data, LResData.Size);
-    Input.ReadBuffer(LResData^.Data^, LResData^.Size);
+    Input.ReadBuffer(LResData.Data^, LResData.Size);
-    Insert(LResData^, Resources, Length(Resources));
+    Insert(LResData, Resources, Length(Resources));
   end;
   SetLength(Tasks, Options^.Threads);
   SetLength(CurCodec, Options^.Threads);
@@ -1860,45 +1976,18 @@ begin
       end;
     end;
   Input.ReadBuffer(StoreDD, StoreDD.Size);
-  if StoreDD or FileExists(ExtractFilePath(Utils.GetModuleName) +
+  UI32 := 0;
-    Options^.DedupFile) then
+  if StoreDD then
   begin
-    if StoreDD then
+    Input.ReadBuffer(UI32, UI32.Size);
-      LStream := Input
+    SetLength(DDList2, UI32);
-    else
+    DDCount2 := UI32;
-    begin
+    for I := Low(DDList2) to High(DDList2) do
-      LStream := TFileStream.Create(ExtractFilePath(Utils.GetModuleName) +
+      Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
-        Options^.DedupFile, fmShareDenyNone);
+    DDIndex1 := -1;
-      LStream.Position := 0;
+    DDIndex2 := 0;
-    end;
-    while True do
-    begin
-      LStream.ReadBuffer(LGUID, SizeOf(TGUID));
-      LStream.ReadBuffer(J, J.Size);
-      I := J * SizeOf(TDuplicate);
-      if CompareMem(@DupGUID, @LGUID, SizeOf(TGUID)) then
-      begin
-        if WorkStream[0].Size < I then
-          WorkStream[0].Size := I;
-        LStream.ReadBuffer(WorkStream[0].Memory^, I);
-        for I := 0 to J - 1 do
-        begin
-          DupTyp := (PDuplicate(WorkStream[0].Memory) + I)^;
-          Duplicates2.Add(DupTyp.Index, DupTyp.Count);
-        end;
-        break;
-      end
-      else if StoreDD then
-        raise EReadError.CreateRes(@SInvalidProperty)
-      else
-        LStream.Seek(I, TSeekOrigin.soCurrent);
-      if StoreDD or (LStream.Position >= LStream.Size) then
-        break;
-    end;
-    if not StoreDD then
-      LStream.Free;
   end;
-  DataMgr := TDataManager.Create(NStream, Duplicates2.Count);
+  DataMgr := TDataManager.Create(NStream);
 end;
 
 procedure DecFree;
@@ -1933,7 +2022,6 @@ begin
     WorkStream[I].Free;
   end;
   DataMgr.Free;
-  Duplicates2.Free;
   FreeResources;
   GlobalSync.Free;
   for I := Low(ThreadSync) to High(ThreadSync) do
@@ -1981,8 +2069,17 @@ begin
             Inc(CurrPos, Max(StreamHeader^.OldSize, StreamHeader^.NewSize));
           end;
         end;
-        if MemInput[Index].Size < BlockSize then
+        if (Depth = 0) and (Length(Tasks) > 1) and (StreamCount[Index]^ > 1)
-          MemInput[Index].Size := BlockSize;
+        then
+        begin
+          if MemInput[Index].Size < CurrPos then
+            MemInput[Index].Size := CurrPos;
+        end
+        else
+        begin
+          if MemInput[Index].Size < BlockSize then
+            MemInput[Index].Size := BlockSize;
+        end;
         MemInput[Index].Position := 0;
         StreamIdx[Index]^ := -1;
         if (Depth = 0) and (Length(Tasks) > 1) and (StreamCount[Index]^ > 1)
@@ -1996,9 +2093,6 @@ begin
           for J := 0 to StreamCount[Index]^ - 1 do
           begin
             StreamHeader := PStreamHeader(MemStream1[Index].Memory) + J;
-            MemInput[Index].Size := Max(MemInput[Index].Size,
-              StreamInfo[Index]^.Pos[J] + Max(StreamHeader^.OldSize,
-              StreamHeader^.NewSize));
             MemInput[Index].Position := StreamInfo[Index]^.Pos[J];
             if CopyStream(DecInput[Index], MemInput[Index],
               StreamHeader^.NewSize) <> StreamHeader^.NewSize then
@@ -2026,12 +2120,26 @@ begin
             if IsErrored(Tasks) then
               for I := Low(Tasks) to High(Tasks) do
                 Tasks[I].RaiseLastError;
+            if StoreDD and (Depth = 0) then
+            begin
+              Inc(DDIndex1);
+              if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index))
+              then
+              begin
+                DataMgr.Add(DDIndex1, StreamHeader^.OldSize,
+                  DDList2[DDIndex2].Count);
+                DataMgr.Write(DDIndex1,
+                  (PByte(MemInput[Index].Memory) + StreamInfo[Index]^.Pos[J]),
+                  StreamHeader^.OldSize);
+                Inc(DDIndex2);
+              end;
+            end;
             if StreamHeader^.Kind and DUPLICATED_STREAM = DUPLICATED_STREAM then
               DataMgr.CopyData(StreamHeader^.Option, DecOutput[Index])
             else
               DecOutput[Index].WriteBuffer
                 ((PByte(MemInput[Index].Memory) + StreamInfo[Index]^.Pos[J])^,
-                (PStreamHeader(MemStream1[Index].Memory) + J)^.OldSize);
+                StreamHeader^.OldSize);
           end;
           WaitForAll(Tasks);
         end
@@ -2041,8 +2149,6 @@ begin
       DecInput[Index].ReadBuffer(UI32, UI32.Size);
       if UI32 > 0 then
         CopyStreamEx(DecInput[Index], DecOutput[Index], UI32);
-      if Depth = 0 then
-        Inc(DupIdx1, StreamCount[Index]^);
       DecInput[Index].ReadBuffer(StreamCount[Index]^, StreamCount[Index]^.Size);
     end;
   end;

precompressor/PrecompOodle.pas

@@ -681,8 +681,8 @@ begin
     if CustomLZ_Decompress0(Input, Buffer, StreamInfo^.OldSize,
       OodleSI.DSize, Res) then
     begin
-      StreamInfo^.NewSize := Res;
       Output(Instance, Buffer, Res);
+      StreamInfo^.NewSize := Res;
       Funcs^.LogScan2(OodleCodecs[GetBits(StreamInfo^.Option, 0, 5)],
         StreamInfo^.OldSize, StreamInfo^.NewSize);
       Result := True;
@@ -730,7 +730,7 @@ begin
       break;
   end;
   if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
-    (SOList[Instance][X].Count = 1)) then
+    (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
   begin
     Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));
     Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1,

precompressor/PrecompUtils.pas

@@ -18,7 +18,7 @@ resourcestring
   SPrecompSep4 = '/';
 
 const
-  SuccessStatus = 3;
+  SuccessStatus = 4;
 
   DEFAULT_STREAM = 0;
   EXTENDED_STREAM = 1;
@@ -32,7 +32,7 @@ type
 
   TPrecompStr = array [0 .. 255] of Char;
 
-  TStreamStatus = (None, Invalid, Predicted);
+  TStreamStatus = (None, Invalid, Predicted, Database);
 
   PDepthInfo = ^TDepthInfo;
 
@@ -182,10 +182,10 @@ type
     LogScan1: procedure(Codec: PChar; Position: Int64;
       InSize, OutSize: Integer)cdecl;
     LogScan2: procedure(Codec: PChar; InSize, OutSize: Integer)cdecl; // 35
-    LogProcess: procedure(Codec, Method: PChar;
+    LogProcess: procedure(Codec, Method: PChar; Size1, Size2, Size3: Integer;
-      OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl;
+      Status: Boolean)cdecl;
-    LogRestore: procedure(Codec, Method: PChar;
+    LogRestore: procedure(Codec, Method: PChar; Size1, Size2, Size3: Integer;
-      OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl;
+      Status: Boolean)cdecl;
     LogPatch1: procedure(OldSize, NewSize, PatchSize: Integer;
       Status: Boolean)cdecl;
     LogPatch2: procedure(OldSize, NewSize, PatchSize: Integer;
@@ -243,22 +243,29 @@ type
 
   PDatabase = ^TDatabase;
 
-  TDatabase = record
+  TDatabase = packed record
+    Size: Integer;
     Codec: Byte;
-    Status: TStreamStatus;
     Option: Integer;
+    Checksum: Cardinal;
+    Status: TStreamStatus;
   end;
 
-  TDatabaseDynArray = TArray<TDatabase>;
+  PDuplicate1 = ^TDuplicate1;
 
-  PDuplicate = ^TDuplicate;
+  TDuplicate1 = packed record
-
+    Size: Integer;
-  TDuplicate = record
+    Checksum: Cardinal;
     Index: Integer;
     Count: Integer;
   end;
 
-  TDuplicateDynArray = TArray<TDuplicate>;
+  PDuplicate2 = ^TDuplicate2;
+
+  TDuplicate2 = packed record
+    Index: Integer;
+    Count: Integer;
+  end;
 
   TPrecompVMStream = class(TStream)
   private const
@@ -288,8 +295,6 @@ type
     Size: Integer;
   end;
 
-function DuplicateSortCompare(const Left, Right): Integer;
-
 procedure AddMethod(Method: String);
 procedure ClearMethods;
 
@@ -358,7 +363,6 @@ function PrecompAcceptPatch(OldSize, NewSize, PatchSize: Integer)
 var
   PrecompFunctions: _PrecompFuncs;
   DIFF_TOLERANCE: Single = 0.05;
-  VERBOSE: Boolean = False;
   EncodeSICmp: TEncodeSIComparer;
   FutureSICmp: TFutureSIComparer;
   StockMethods, ExternalMethods: TStringList;
@@ -380,11 +384,6 @@ begin
   Result := Integer(CompareValue(Left.Position, Right.Position));
 end;
 
-function DuplicateSortCompare(const Left, Right): Integer;
-begin
-  Result := TDuplicate(Left).Index - TDuplicate(Right).Index;
-end;
-
 procedure AddMethod(Method: String);
 begin
   if (StockMethods.IndexOf(Method) < 0) and (ExternalMethods.IndexOf(Method) < 0)
@@ -856,7 +855,7 @@ var
 begin
   Result := 0;
   if xd3_encode(OldBuff, OldSize, NewBuff, NewSize, PatchBuff, @Res, PatchSize,
-    0) = 0 then
+    Integer(XD3_NOCOMPRESS)) = 0 then
     Result := Res;
   // MakeDiff(OldBuff, NewBuff, PatchBuff, OldSize, NewSize, Result);
 end;
@@ -869,7 +868,7 @@ var
 begin
   Result := 0;
   if xd3_decode(PatchBuff, PatchSize, OldBuff, OldSize, NewBuff, @Res, NewSize,
-    0) = 0 then
+    Integer(XD3_NOCOMPRESS)) = 0 then
     Result := Res;
   // MakePatch(OldBuff, PatchBuff, NewBuff, OldSize, PatchSize, Result);
 end;

precompressor/PrecompZLib.pas

@@ -255,7 +255,8 @@ begin
   CodecAvailable[ZLIB_CODEC] := ZLibDLL.DLLLoaded;
   CodecAvailable[REFLATE_CODEC] := ReflateDLL.DLLLoaded;
   CodecAvailable[PREFLATE_CODEC] := PreflateDLL.DLLLoaded;
-  CodecAvailable[PNG_CODEC] := True;
+  CodecAvailable[PNG_CODEC] := ZLibDLL.DLLLoaded or ReflateDLL.DLLLoaded or
+    PreflateDLL.DLLLoaded;
   X := 0;
   while Funcs^.GetCodec(Command, X, False) <> '' do
   begin
@@ -433,12 +434,13 @@ var
 begin
   DI1 := Funcs^.GetDepthInfo(Instance);
   DS := Funcs^.GetCodec(DI1.Codec, 0, False);
+  X := -1;
   if DS <> '' then
   begin
     X := IndexTextW(@DS[0], ZlibCodecs);
     if (X < 0) or (DI1.OldSize <> SizeEx) then
       exit;
-    if CodecAvailable[X] then
+    if not CodecAvailable[X] then
       exit;
   end
   else if BoolArray(CodecEnabled, False) then
@@ -567,7 +569,7 @@ begin
             if (I = ZLIB_CODEC) and (WinBits = 0) then
               SetBits(SI.Option, 1, 12, 3);
             SetBits(SI.Option, I, 0, 5);
-            if CodecEnabled[I] then
+            if CodecEnabled[I] or (I = X) then
             begin
               DS := Funcs^.GetDepthCodec(DI1.Codec);
               Move(DS[0], DI2.Codec, SizeOf(DI2.Codec));
@@ -632,18 +634,14 @@ function ZlibProcess(Instance, Depth: Integer; OldInput, NewInput: Pointer;
 
   function IsValidLevel(CLevel, ZLevel: Integer): Boolean;
   begin
+    Result := False;
     case CLevel of
       1, 6:
-        if CLevel = ZLevel then
+        Result := CLevel = ZLevel;
-          Result := True;
       2 .. 5:
-        if ZLevel = 5 then
+        Result := ZLevel = 5;
-          Result := True;
       7 .. 9:
-        if ZLevel = 9 then
+        Result := ZLevel = 9;
-          Result := True;
-    else
-      Result := False;
     end;
   end;
 
@@ -664,6 +662,7 @@ begin
   if not X in [PNG_CODEC] then
     if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then
       exit;
+  Params := '';
   case X of
     ZLIB_CODEC:
       begin
@@ -673,7 +672,7 @@ begin
         begin
           L := I div 10;
           M := I mod 10;
-          if StreamInfo^.Status = TStreamStatus.Predicted then
+          if StreamInfo^.Status >= TStreamStatus.Predicted then
           begin
             if InRange(GetBits(StreamInfo^.Option, 5, 7), 1, 9) then
             begin
@@ -684,36 +683,40 @@ begin
             begin
               if GetBits(StreamInfo^.Option, 5, 7) <> I then
                 continue;
-              { I := GetBits(StreamInfo^.Option, 5, 7);
+              if StreamInfo^.Status = TStreamStatus.Database then
-                SOList[Instance][ZLIB_CODEC].Add(I);
                 Result := True;
-                break; }
             end;
           end;
           Params := 'l' + I.ToString + ':' + 'w' +
             (GetBits(StreamInfo^.Option, 12, 3) + 8).ToString;
           ZStream := @ZStream1[Instance, L, M,
             GetBits(StreamInfo^.Option, 12, 3)];
-          ZStream^.next_in := NewInput;
+          if not Result then
-          ZStream^.avail_in := StreamInfo^.NewSize;
+          begin
-          deflateReset(ZStream^);
+            ZStream^.next_in := NewInput;
-          repeat
+            ZStream^.avail_in := StreamInfo^.NewSize;
-            ZStream^.next_out := Buffer;
+            deflateReset(ZStream^);
-            ZStream^.avail_out := Z_BLKSIZE;
+            repeat
-            Res1 := deflate(ZStream^, Z_FINISH);
+              ZStream^.next_out := Buffer;
-            if Res1 < 0 then
+              ZStream^.avail_out := Z_BLKSIZE;
-              raise EZCompressionError.Create(string(_z_errmsg[2 - Res1]))
+              Res1 := deflate(ZStream^, Z_FINISH);
-                at ReturnAddress;
+              if Res1 < 0 then
-            Res2 := Z_BLKSIZE - ZStream^.avail_out;
+                raise EZCompressionError.Create(string(_z_errmsg[2 - Res1]))
-            Verified := CompareMem(PByte(OldInput) + ZStream^.total_out - Res2,
+                  at ReturnAddress;
-              Buffer, Res2);
+              Res2 := Z_BLKSIZE - ZStream^.avail_out;
-            if not Verified then
+              Verified := CompareMem(PByte(OldInput) + ZStream^.total_out -
-              break;
+                Res2, Buffer, Res2);
-          until (ZStream^.avail_in = 0) and (ZStream^.avail_out > 0);
+              if not Verified then
+                break;
+            until (ZStream^.avail_in = 0) and (ZStream^.avail_out > 0);
+          end
+          else
+            ZStream.total_out := StreamInfo^.OldSize;
           Funcs^.LogProcess(ZlibCodecs[GetBits(StreamInfo^.Option, 0, 5)],
             PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize,
-            ZStream^.total_out, Verified and (Res1 = Z_STREAM_END));
+            ZStream^.total_out, (Result = True) or
-          if Verified and (Res1 = Z_STREAM_END) then
+            (Verified and (Res1 = Z_STREAM_END)));
+          if (Result = True) or (Verified and (Res1 = Z_STREAM_END)) then
           begin
             SetBits(StreamInfo^.Option, I, 5, 7);
             SOList[Instance][ZLIB_CODEC].Add(I);
@@ -739,7 +742,7 @@ begin
         Buffer := Funcs^.Allocator(Instance, R_WORKMEM * 2);
         J := 0;
         HR := RefInst1[Instance];
-        if StreamInfo^.Status = TStreamStatus.Predicted then
+        if StreamInfo^.Status >= TStreamStatus.Predicted then
           L := GetBits(StreamInfo^.Option, 5, 7)
         else
           L := RLevel;
@@ -804,7 +807,6 @@ begin
     PNG_CODEC:
       begin
         Buffer := Funcs^.Allocator(Instance, StreamInfo^.OldSize);
-        Params := '';
         if DecodePNG(NewInput, Buffer, StreamInfo^.OldSize) then
           Result := CompareMem(OldInput, Buffer, StreamInfo^.OldSize);
         Funcs^.LogProcess(ZlibCodecs[GetBits(StreamInfo^.Option, 0, 5)],
@@ -831,6 +833,7 @@ begin
   if not X in [PNG_CODEC] then
     if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then
       exit;
+  Params := '';
   case X of
     ZLIB_CODEC:
       begin
@@ -915,7 +918,6 @@ begin
     PNG_CODEC:
       begin
         Buffer := Funcs^.Allocator(Instance, StreamInfo.OldSize);
-        Params := '';
         if DecodePNG(Input, Buffer, StreamInfo.OldSize) then
         begin
           Output(Instance, Buffer, StreamInfo.OldSize);

precompressor/PrecompZSTD.pas

@@ -233,8 +233,8 @@ begin
   end;
   if Res > StreamInfo^.OldSize then
   begin
-    StreamInfo^.NewSize := Res;
     Output(Instance, Buffer, Res);
+    StreamInfo^.NewSize := Res;
     Funcs^.LogScan2(ZSTDCodecs[GetBits(StreamInfo^.Option, 0, 5)],
       StreamInfo^.OldSize, StreamInfo^.NewSize);
     Result := True;
@@ -265,6 +265,7 @@ begin
     if StreamInfo^.Status = TStreamStatus.Predicted then
       if GetBits(StreamInfo^.Option, 5, 7) <> I then
         continue;
+    Params := '';
     case X of
       ZSTD_CODEC:
         begin
@@ -307,7 +308,7 @@ begin
   if Res1 < 0 then
     exit;
   if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
-    (SOList[Instance][X].Count = 1)) then
+    (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
   begin
     Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));
     Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1,

xtool.dpr

@@ -52,9 +52,13 @@ uses
   oObjects in 'contrib\ParseExpression\oObjects.pas',
   ParseClass in 'contrib\ParseExpression\ParseClass.pas',
   ParseExpr in 'contrib\ParseExpression\ParseExpr.pas',
+  BrunsliDLL in 'imports\BrunsliDLL.pas',
+  FLACDLL in 'imports\FLACDLL.pas',
+  JoJpegDLL in 'imports\JoJpegDLL.pas',
   LZ4DLL in 'imports\LZ4DLL.pas',
   LZODLL in 'imports\LZODLL.pas',
   OodleDLL in 'imports\OodleDLL.pas',
+  PackJPGDLL in 'imports\PackJPGDLL.pas',
   PreflateDLL in 'imports\PreflateDLL.pas',
   ReflateDLL in 'imports\ReflateDLL.pas',
   XDeltaDLL in 'imports\XDeltaDLL.pas',
@@ -68,6 +72,7 @@ uses
   PrecompLZ4 in 'precompressor\PrecompLZ4.pas',
   PrecompLZO in 'precompressor\PrecompLZO.pas',
   PrecompZSTD in 'precompressor\PrecompZSTD.pas',
+  PrecompMedia in 'precompressor\PrecompMedia.pas',
   PrecompOodle in 'precompressor\PrecompOodle.pas',
   PrecompINI in 'precompressor\PrecompINI.pas',
   PrecompSearch in 'precompressor\PrecompSearch.pas',
@@ -82,7 +87,8 @@ uses
   IOPatch in 'io\IOPatch.pas',
   IOExecute in 'io\IOExecute.pas',
   IODecode in 'io\IODecode.pas',
-  IOUtils in 'io\IOUtils.pas';
+  IOUtils in 'io\IOUtils.pas',
+  LZMADLL in 'imports\LZMADLL.pas';
 
 {$SETPEFLAGS IMAGE_FILE_LARGE_ADDRESS_AWARE or IMAGE_FILE_RELOCS_STRIPPED}
 
@@ -141,265 +147,24 @@ begin
     Result := THandleStream.Create(GetStdHandle(STD_INPUT_HANDLE))
   else if Pos('://', Input) > 0 then
     Result := TDownloadStream.Create(Input)
+  else if FileExists(Input) then
+    Result := TFileStream.Create(Input, fmShareDenyNone)
   else
-    Result := TFileStream.Create(Input, fmShareDenyNone);
+    Result := TDirInputStream.Create(Input);
 end;
 
-function GetOutStream(Output: string): TStream;
+function GetOutStream(Output: string; MultiInput: Boolean = False): TStream;
 begin
-  if (Output = '-') or (Output = '') then
+  if (Output = '') then
+    Result := TNullStream.Create
+  else if DirectoryExists(Output) then
+    Result := TDirOutputStream.Create(Output)
+  else if (Output = '-') then
     Result := THandleStream.Create(GetStdHandle(STD_OUTPUT_HANDLE))
   else
     Result := TFileStream.Create(Output, fmCreate);
 end;
 
-{ changelog
-  ES_R34 (0.5.3)
-  - added png stream preprocessor
-  - removed grittibanzli codec (since nobody uses it)
-
-  ES_R33 (0.5.2)
-  - added IO functions (archive, execute)
-  - fixed issue in patch io function
-  - removed compression on patch diff files
-
-  ES_R32 (0.5.1)
-  - added IO functions (find, extract, patch)
-  - generate database feature and IO functions now can search for streams larger than chunk size
-
-  ES_R31 (0.5.0)
-  - added IO functions (erase, replace)
-  - fixed external executable support bugs
-
-  ES_R30 (0.4.8)
-  - fixed issue with storing incorrect recompression information when stream patching is performed
-
-  ES_R29 (0.4.7)
-  - updated oodle scanner
-  - updated external executable support
-  - updated configuration based plugin support to add depth information
-  - updated verbose mode
-
-  ES_R28 (0.4.6)
-  - generate database feature fixed
-  - fixed external executable support issues
-  - fixed lz4f level setting bug
-
-  ES_R28 (0.4.5)
-  - removed leviathan codec restriction
-
-  ES_R27 (0.4.4)
-  - fixed issue of lz4 codec loading incorrect library
-  - fixed issue with handling endianess via configuration based plugins
-  - updated framework of library based plugins
-
-  ES_R26 (0.4.3)
-  - added verbose mode
-  - added feature that allows you to enforce a different library to be loaded
-  - fixed issues related to imperfect stream patching
-  - fixed issues with old libraries with missing functions that cause xtool to crash on startup
-  - updated oodle codec
-  - updated reflate codec
-  - updated zstd codec
-
-  ES_R25 (0.4.2)
-  - removed debugging code from encryption and executable codec
-  - fixed issue with depth when using search codec
-  - fixed external executable support issues
-
-  ES_R24 (0.4.1)
-  - fixed issue of status not reporting when encoding
-  - added depth method support for search support
-  - fixed zlib encoding issues for different window bits
-  - fixed zlib memory leak issue
-  - updated all internal codecs to support information relayed by external codecs
-  - updated lz4f codec and removed temporarily removed support for universal scanning
-  - added option to change recompression level to be used by reflate
-  - updated external executable support
-  - generate database feature currently bugged, wait for next update
-  - search database structure changed, older database files will no longer work with newer releases
-
-  ES_R23 (0.4.0)
-  - project made open source
-  - added external executable support
-  - added generate database feature
-  - fixed search support bug
-
-  ES_R22 (0.3.22)
-  - updated search support (speed improvements)
-  - updated command line parser
-  - added partial universal scanner for lzo1x streams
-  - added universal scanner for lz4f streams
-  - fixed issue with configuration files failing to execute without conditions
-
-  ES_R21 (0.3.21)
-  - updated search support
-
-  ES_R20 (0.3.20)
-  - fixed library support bug
-  - x86 build discontinued (has bugs from nowhere)
-
-  ES_R19 (0.3.19)
-  - updated lzo codec
-
-  ES_R18 (0.3.18)
-  - fixed depth bug
-  - fixed library plugin bugs
-
-  ES_R17 (0.3.17)
-  - fixed multi-threading bug
-
-  ES_R16 (0.3.16)
-  - minor bug fixes
-
-  ES_R15 (0.3.15)
-  - converted library support to unicode (don't know why I used ansi in the first place)
-  - added library support functions
-  - added rc4 encryption support
-
-  ES_R14 (0.3.14)
-  - fixed library support bug
-  - updated library structure
-
-  ES_R13 (0.3.13)
-  - updated lz4 codec
-  - updated library structure
-  - updated depth info functions
-  - updated depth feature
-
-  ES_R12 (0.3.12)
-  - added depth info functions
-  - added support for oodle 2.9.0+ functions
-  - fixed data patching bug
-  - updated oodle codec
-  - updated command line parser
-
-  ES_R11 (0.3.11)
-  - fixed x86 build bugs
-  - fixed config multi-threading bug
-  - fixed resource management bug
-  - fixed deduplication bug
-
-  ES_R10 (0.3.10)
-  - minor bug fixes
-  - added diff tolerance parameter (--diff=)
-  - fixed plugin database bug
-  - fixed lz4 codec bug
-  - updated oodle codec
-  - updated library structure
-  - added resource management
-  - added direct use encryption codecs
-  - added embedded deduplication feature (--dedup) [makes temps during encoding]
-
-  ES_R9 (0.3.9)
-  - fixed future stream bug
-
-  ES_R8 (0.3.8)
-  - fixed command line parser bug
-  - updated library support
-
-  ES_R7 (0.3.7)
-  - updated library structure
-
-  ES_R6 (0.3.6)
-  - updated oodle codec (fixed more lzna bugs)
-
-  ES_R5 (0.3.5)
-  - updated oodle codec (fixed lzna bug)
-  - added custom method configuration
-
-  ES_R4 (0.3.4)
-  - fixed bug depthing
-
-  ES_R3 (0.3.3)
-  - updated lz4 codec
-  - updated library support
-
-  ES_R2 (0.3.2)
-  - improved depthing
-  - updated library support
-  - fixed zstd codec issues
-  - removed fast memory
-
-  ES_R1 (0.3.1)
-  - updated library support
-  - updated command line parser
-  - included x86 build
-  - fixed depthing issues
-
-  2012_R2 (0.2.14)
-  - added library support
-  - added compress, decompress, encrypt, decrypt, hash, delta functions (used by library)
-  - added lzo codec placeholders
-  - fixed oodle bug
-  - fixed lz4 bug
-  - removed libdunia codec
-
-  2012_R1 (0.2.13)
-  - added oo2ext* dll support
-  - updated search support
-
-  2011_R1 (0.2.12)
-  - added temporary libdunia codec
-
-  2010_R5 (0.2.11)
-  - fixed search/config support bug
-
-  2010_R4 (0.2.10)
-  - updated search/config support
-
-  2010_R3 (0.2.9)
-  - added database search
-  - updated zlib scanner
-  - fixed reflate bug
-  - fixed 2GB memory limit
-
-  2010_R2 (0.2.8)
-  - fixed zstd codec
-
-  2010_R1 (0.2.7)
-  - added zstd codec
-  - added lz4, lz4hc, lzna, mermaid, selkie, hydra, leviathan codec placeholders
-  - added configuration support
-  - added xdelta support to handle crc mismatch streams
-
-  2009_R3 (0.2.6)
-  - documentation added
-
-  2009_R2 (0.2.5)
-  - added kraken codec
-  - fixed depthing issues
-
-  2009_R1 (0.2.4)
-  - added reflate forced verification
-  - updated deflate scanner
-  - fixed depthing issues
-  - fixed low memory mode issues
-  - fixed hanging issues when encoding
-
-  2008_R3 (0.2.3)
-  - fixed deduplication memory calculation error
-  - added virtual memory support for deduplication
-  - added --mem=# parameter to control deduplication memory usage
-
-  2008_R2 (0.2.2)
-  - fixed command line parser
-  - updated deflate scanner
-  - added stream deduplication
-  - added stream database
-  - added decompression memory limiter
-  - added grittibanzli (also handles deflate stream but slow af)
-
-  2008_R1 (0.2.1)
-  - initial release
-
-  changelog }
-
-procedure EncodePNG(Input: PByte; Output: PByte);
-begin
-
-end;
-
 const
   BufferSize = 1048576;
 
@@ -460,12 +225,10 @@ begin
         PrecompMain.PrintHelp
       else
       begin
-        while Length(ParamArg[1]) < 2 do
+        Input := TBufferedStream.Create(GetInStream(ParamArgSafe(1, 0)), True,
-          Insert('-', ParamArg[1], Length(ParamArg[1]));
-        Input := TBufferedStream.Create(GetInStream(ParamArg[1, 0]), True,
-          BufferSize);
-        Output := TBufferedStream.Create(GetOutStream(ParamArg[1, 1]), False,
           BufferSize);
+        Output := TBufferedStream.Create(GetOutStream(ParamArgSafe(1, 1)),
+          False, BufferSize);
         try
           PrecompMain.Parse(ParamArg[0], PrecompEnc);
           PrecompMain.Encode(Input, Output, PrecompEnc);
@@ -550,8 +313,6 @@ begin
         IOArchive.PrintHelp
       else
       begin
-        while Length(ParamArg[1]) < 2 do
-          Insert('-', ParamArg[1], Length(ParamArg[1]));
         SetLength(StrArray, 0);
         for I := 0 to High(ParamArg[1]) - 1 do
           Insert(ParamArg[1, I], StrArray, Length(StrArray));
@@ -569,8 +330,6 @@ begin
         IOExecute.PrintHelp
       else
       begin
-        while Length(ParamArg[1]) < 2 do
-          Insert('-', ParamArg[1], Length(ParamArg[1]));
         SetLength(StrArray, 0);
         for I := 2 to High(ParamArg[1]) do
           Insert(ParamArg[1, I], StrArray, Length(StrArray));

xtool.dproj

@@ -111,9 +111,13 @@
         <DCCReference Include="contrib\ParseExpression\oObjects.pas"/>
         <DCCReference Include="contrib\ParseExpression\ParseClass.pas"/>
         <DCCReference Include="contrib\ParseExpression\ParseExpr.pas"/>
+        <DCCReference Include="imports\BrunsliDLL.pas"/>
+        <DCCReference Include="imports\FLACDLL.pas"/>
+        <DCCReference Include="imports\JoJpegDLL.pas"/>
         <DCCReference Include="imports\LZ4DLL.pas"/>
         <DCCReference Include="imports\LZODLL.pas"/>
         <DCCReference Include="imports\OodleDLL.pas"/>
+        <DCCReference Include="imports\PackJPGDLL.pas"/>
         <DCCReference Include="imports\PreflateDLL.pas"/>
         <DCCReference Include="imports\ReflateDLL.pas"/>
         <DCCReference Include="imports\XDeltaDLL.pas"/>
@@ -127,6 +131,7 @@
         <DCCReference Include="precompressor\PrecompLZ4.pas"/>
         <DCCReference Include="precompressor\PrecompLZO.pas"/>
         <DCCReference Include="precompressor\PrecompZSTD.pas"/>
+        <DCCReference Include="precompressor\PrecompMedia.pas"/>
         <DCCReference Include="precompressor\PrecompOodle.pas"/>
         <DCCReference Include="precompressor\PrecompINI.pas"/>
         <DCCReference Include="precompressor\PrecompSearch.pas"/>
@@ -142,6 +147,7 @@
         <DCCReference Include="io\IOExecute.pas"/>
         <DCCReference Include="io\IODecode.pas"/>
         <DCCReference Include="io\IOUtils.pas"/>
+        <DCCReference Include="imports\LZMADLL.pas"/>
         <RcItem Include="resources\Win32\xdelta3_dll.dll">
             <ContainerId>ResourceItem</ContainerId>
             <ResourceType>RCDATA</ResourceType>
@@ -152,6 +158,7 @@
             <ResourceType>RCDATA</ResourceType>
             <ResourceId>XDELTA64_DLL</ResourceId>
         </RcItem>
+        <None Include="changes.txt"/>
         <BuildConfiguration Include="Base">
             <Key>Base</Key>
         </BuildConfiguration>
@@ -189,6 +196,12 @@
                         <Overwrite>true</Overwrite>
                     </Platform>
                 </DeployFile>
+                <DeployFile LocalName="changes.txt" Configuration="Release" Class="ProjectFile">
+                    <Platform Name="Win64">
+                        <RemoteDir>.\</RemoteDir>
+                        <Overwrite>true</Overwrite>
+                    </Platform>
+                </DeployFile>
                 <DeployFile LocalName="resources\Win32\xdelta3_dll.dll" Configuration="Release" Class="ProjectFile">
                     <Platform Name="Win32">
                         <RemoteDir>.\</RemoteDir>