diff --git a/changes.txt b/changes.txt
index c04878d..a340325 100644
--- a/changes.txt
+++ b/changes.txt
@@ -1,11 +1,16 @@
+ ES_R37 (0.6.2)
+ - added feature to inject libraries to main executable
+
+ ES_R36 (0.6.1)
+ - added fast lzma2 compression for portable mode
+ - fixed minor issue with stream deduplication feature
+
ES_R35 (0.6.0)
- added wav stream detector
- added flac codec
- added jpg stream detector
- added packjpg, brunsli, jojpeg codec
- - added fast lzma2 compression for portable mode
- added feature that allows input to be a directory
- - added feature to inject libraries to main executable
- added feature to extract detected streams
- updated database feature
- updated deduplication feature
@@ -248,4 +253,4 @@
- added grittibanzli (also handles deflate stream but slow af)
2008_R1 (0.2.1)
- - initial release
\ No newline at end of file
+ - initial release
diff --git a/common/Utils.pas b/common/Utils.pas
index 94c7042..09cfbdd 100644
--- a/common/Utils.pas
+++ b/common/Utils.pas
@@ -490,6 +490,8 @@ function GetCmdStr(CommandLine: String; Index: Integer;
KeepQuotes: Boolean = False): string;
function GetCmdCount(CommandLine: String): Integer;
+procedure UpdateFileResource(Source, Dest, ResName: string);
+
implementation
function GetBits(Data: Int64; Index: TInt64_BitIndex;
@@ -3615,4 +3617,41 @@ begin
Inc(Result);
end;
+procedure UpdateFileResource(Source, Dest, ResName: string);
+var
+ Stream: TFileStream;
+ hDestRes: THandle;
+ lpData: Pointer;
+ cbData: DWORD;
+begin
+ Stream := TFileStream.Create(Source, fmOpenRead or fmShareDenyNone);
+ try
+ Stream.Seek(0, soFromBeginning);
+ cbData := Stream.Size;
+ if cbData > 0 then
+ begin
+ GetMem(lpData, cbData);
+ try
+ Stream.Read(lpData^, cbData);
+ hDestRes := BeginUpdateResource(PChar(Dest), False);
+ if hDestRes <> 0 then
+ if UpdateResource(hDestRes, RT_RCDATA, PWideChar(ResName), 0, lpData,
+ cbData) then
+ begin
+ if not EndUpdateResource(hDestRes, False) then
+ RaiseLastOSError
+ end
+ else
+ RaiseLastOSError
+ else
+ RaiseLastOSError;
+ finally
+ FreeMem(lpData);
+ end;
+ end;
+ finally
+ Stream.Free;
+ end;
+end;
+
end.
diff --git a/imports/FLZMA2DLL.pas b/imports/FLZMA2DLL.pas
new file mode 100644
index 0000000..23ba52b
--- /dev/null
+++ b/imports/FLZMA2DLL.pas
@@ -0,0 +1,312 @@
+unit FLZMA2DLL;
+
+interface
+
+uses
+ MemoryModule,
+ WinAPI.Windows,
+ System.SysUtils, System.Classes, System.Types;
+
+type
+ PFL2_inBuffer = ^FL2_inBuffer;
+
+ FL2_inBuffer = record
+ src: Pointer;
+ size: size_t;
+ pos: size_t;
+ end;
+
+ PFL2_outBuffer = ^FL2_outBuffer;
+
+ FL2_outBuffer = record
+ dst: Pointer;
+ size: size_t;
+ pos: size_t;
+ end;
+
+var
+ FL2_compress: function(dst: Pointer; dstCapacity: size_t; const src: Pointer;
+ srcSize: size_t; compressionLevel: Integer): size_t cdecl;
+ FL2_compressMt: function(dst: Pointer; dstCapacity: size_t;
+ const src: Pointer; srcSize: size_t; compressionLevel: Integer;
+ nbThreads: Cardinal): size_t cdecl;
+ FL2_decompress: function(dst: Pointer; dstCapacity: size_t;
+ const src: Pointer; srcSize: size_t): size_t cdecl;
+ FL2_decompressMt: function(dst: Pointer; dstCapacity: size_t;
+ const src: Pointer; srcSize: size_t; nbThreads: Cardinal): size_t cdecl;
+
+ FL2_createCCtx: function: Pointer cdecl;
+ FL2_createCCtxMt: function(nbThreads: Cardinal): Pointer cdecl;
+ FL2_freeCCtx: procedure(cctx: Pointer)cdecl;
+ FL2_compressCCtx: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
+ src: Pointer; srcSize: size_t; compressionLevel: Integer): size_t cdecl;
+ FL2_createDCtx: function: Pointer cdecl;
+ FL2_createDCtxMt: function(nbThreads: Cardinal): Pointer cdecl;
+ FL2_freeDCtx: function(dctx: Pointer): size_t cdecl;
+ FL2_decompressDCtx: function(dctx: Pointer; dst: Pointer; dstCapacity: size_t;
+ src: Pointer; srcSize: size_t): size_t cdecl;
+
+ FL2_createCStream: function: Pointer cdecl;
+ FL2_createCStreamMt: function(nbThreads: Cardinal; dualBuffer: Integer)
+ : Pointer cdecl;
+ FL2_freeCStream: procedure(fcs: Pointer)cdecl;
+ FL2_initCStream: function(fcs: Pointer; compressionLevel: Integer)
+ : size_t cdecl;
+ FL2_compressStream: function(fcs: Pointer; output: PFL2_outBuffer;
+ input: PFL2_inBuffer): size_t cdecl;
+
+ FL2_createDStream: function: Pointer cdecl;
+ FL2_createDStreamMt: function(nbThreads: Cardinal): Pointer cdecl;
+ FL2_freeDStream: procedure(fds: Pointer)cdecl;
+ FL2_initDStream: function(fds: Pointer): size_t cdecl;
+ FL2_decompressStream: function(fds: Pointer; output: PFL2_outBuffer;
+ input: PFL2_inBuffer): size_t cdecl;
+
+ FL2_endStream: function(fcs: Pointer; output: PFL2_outBuffer): size_t cdecl;
+ FL2_isError: function(code: size_t): Cardinal cdecl;
+ DLLLoaded: boolean = False;
+
+type
+ TLZMACRec = record
+ Threads: Integer;
+ Level: Integer;
+ procedure Parse(S: String);
+ end;
+
+ TLZMADRec = record
+ Threads: Integer;
+ procedure Parse(S: String);
+ end;
+
+ TLZMACompressStream = class(TStream)
+ private const
+ FBufferSize = 65536;
+ private
+ FCtx: Pointer;
+ FProp: TLZMACRec;
+ FOutput: TStream;
+ FBuffer: array [0 .. FBufferSize - 1] of Byte;
+ public
+ constructor Create(AOutput: TStream; AConfig: String = 't50p');
+ destructor Destroy; override;
+ function Write(const Buffer; Count: LongInt): LongInt; override;
+ end;
+
+ TLZMADecompressStream = class(TStream)
+ private const
+ FBufferSize = 65536;
+ private
+ FCtx: Pointer;
+ FProp: TLZMADRec;
+ FInp: FL2_inBuffer;
+ FInput: TStream;
+ FBuffer: array [0 .. FBufferSize - 1] of Byte;
+ public
+ constructor Create(AInput: TStream; AConfig: String = '');
+ destructor Destroy; override;
+ function Read(var Buffer; Count: Integer): Integer; override;
+ end;
+
+implementation
+
+uses
+ Utils;
+
+var
+ DLLStream: TResourceStream;
+ DLLHandle: TMemoryModule;
+
+procedure TLZMACRec.Parse(S: string);
+var
+ List: TStringDynArray;
+ I, J: Integer;
+begin
+ Threads := 1;
+ Level := 5;
+ List := DecodeStr(S, ':');
+ for I := Low(List) to High(List) do
+ begin
+ if List[I].StartsWith('t', True) then
+ Threads := ConvertToThreads(List[I].Substring(1));
+ if List[I].StartsWith('l', True) then
+ Level := List[I].Substring(1).ToInteger;
+ end;
+end;
+
+procedure TLZMADRec.Parse(S: string);
+var
+ List: TStringDynArray;
+ I: Integer;
+begin
+ Threads := 1;
+ List := DecodeStr(S, ':');
+ for I := Low(List) to High(List) do
+ begin
+ if List[I].StartsWith('t', True) then
+ Threads := ConvertToThreads(List[I].Substring(1));
+ end;
+end;
+
+constructor TLZMACompressStream.Create(AOutput: TStream; AConfig: String);
+begin
+ inherited Create;
+ FProp.Parse(AConfig);
+ FOutput := AOutput;
+ if FProp.Threads > 1 then
+ FCtx := FL2_createCStreamMt(FProp.Threads, 0)
+ else
+ FCtx := FL2_createCStream;
+ FL2_initCStream(FCtx, FProp.Level);
+end;
+
+destructor TLZMACompressStream.Destroy;
+var
+ Oup: FL2_outBuffer;
+ Res: size_t;
+begin
+ Oup.dst := @FBuffer[0];
+ Oup.size := FBufferSize;
+ Oup.pos := 0;
+ repeat
+ Res := FL2_endStream(FCtx, @Oup);
+ FOutput.WriteBuffer(FBuffer[0], Oup.pos);
+ Oup.pos := 0;
+ until Res = 0;
+ FL2_freeCCtx(FCtx);
+ inherited Destroy;
+end;
+
+function TLZMACompressStream.Write(const Buffer; Count: LongInt): LongInt;
+var
+ Inp: FL2_inBuffer;
+ Oup: FL2_outBuffer;
+begin
+ Result := 0;
+ Inp.src := PByte(@Buffer);
+ Inp.size := Count;
+ Inp.pos := 0;
+ Oup.dst := @FBuffer[0];
+ Oup.size := FBufferSize;
+ Oup.pos := 0;
+ while Inp.pos < Inp.size do
+ begin
+ if not boolean(FL2_isError(FL2_compressStream(FCtx, @Oup, @Inp))) then
+ begin
+ FOutput.WriteBuffer(FBuffer[0], Oup.pos);
+ Oup.pos := 0;
+ end;
+ end;
+ Result := Inp.pos;
+end;
+
+constructor TLZMADecompressStream.Create(AInput: TStream; AConfig: String);
+begin
+ inherited Create;
+ FProp.Parse(AConfig);
+ FInput := AInput;
+ if FProp.Threads > 1 then
+ FCtx := FL2_createDStreamMt(FProp.Threads)
+ else
+ FCtx := FL2_createDStream;
+ FL2_initDStream(FCtx);
+ FillChar(FInp, SizeOf(FL2_inBuffer), 0);
+end;
+
+destructor TLZMADecompressStream.Destroy;
+begin
+ FL2_freeDCtx(FCtx);
+ inherited Destroy;
+end;
+
+function TLZMADecompressStream.Read(var Buffer; Count: Integer): Integer;
+var
+ Oup: FL2_outBuffer;
+begin
+ Result := 0;
+ if FInp.pos = FInp.size then
+ begin
+ FInp.src := @FBuffer[0];
+ FInp.size := FInput.Read(FBuffer[0], FBufferSize);
+ FInp.pos := 0;
+ if FInp.size = 0 then
+ exit;
+ end;
+ Oup.dst := PByte(@Buffer);
+ Oup.size := Count;
+ Oup.pos := 0;
+ while Oup.pos < Oup.size do
+ begin
+ if not boolean(FL2_isError(FL2_decompressStream(FCtx, @Oup, @FInp))) then
+ begin
+ if FInp.pos = FInp.size then
+ begin
+ FInp.src := @FBuffer[0];
+ FInp.size := FInput.Read(FBuffer[0], FBufferSize);
+ FInp.pos := 0;
+ if FInp.size = 0 then
+ break;
+ end;
+ end
+ else
+ break;
+ end;
+ Result := Oup.pos;
+end;
+
+procedure Init;
+begin
+ DLLStream := TResourceStream.Create(HInstance, 'fast_lzma2', RT_RCDATA);
+ DLLHandle := MemoryLoadLibary(DLLStream.Memory);
+ if Assigned(DLLHandle) then
+ begin
+ @FL2_compress := MemoryGetProcAddress(DLLHandle, 'FL2_compress');
+ @FL2_compressMt := MemoryGetProcAddress(DLLHandle, 'FL2_compressMt');
+ @FL2_decompress := MemoryGetProcAddress(DLLHandle, 'FL2_decompress');
+ @FL2_decompressMt := MemoryGetProcAddress(DLLHandle, 'FL2_decompressMt');
+ @FL2_createCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_createCCtx');
+ @FL2_createCCtxMt := MemoryGetProcAddress(DLLHandle, 'FL2_createCCtxMt');
+ @FL2_freeCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_freeCCtx');
+ @FL2_compressCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_compressCCtx');
+ @FL2_createDCtx := MemoryGetProcAddress(DLLHandle, 'FL2_createDCtx');
+ @FL2_createDCtxMt := MemoryGetProcAddress(DLLHandle, 'FL2_createDCtxMt');
+ @FL2_freeDCtx := MemoryGetProcAddress(DLLHandle, 'FL2_freeDCtx');
+ @FL2_decompressDCtx := MemoryGetProcAddress(DLLHandle,
+ 'FL2_decompressDCtx');
+ @FL2_createCStream := MemoryGetProcAddress(DLLHandle, 'FL2_createCStream');
+ @FL2_createCStreamMt := MemoryGetProcAddress(DLLHandle,
+ 'FL2_createCStreamMt');
+ @FL2_freeCStream := MemoryGetProcAddress(DLLHandle, 'FL2_freeCStream');
+ @FL2_initCStream := MemoryGetProcAddress(DLLHandle, 'FL2_initCStream');
+ @FL2_compressStream := MemoryGetProcAddress(DLLHandle,
+ 'FL2_compressStream');
+ @FL2_createDStream := MemoryGetProcAddress(DLLHandle, 'FL2_createDStream');
+ @FL2_createDStreamMt := MemoryGetProcAddress(DLLHandle,
+ 'FL2_createDStreamMt');
+ @FL2_freeDStream := MemoryGetProcAddress(DLLHandle, 'FL2_freeDStream');
+ @FL2_initDStream := MemoryGetProcAddress(DLLHandle, 'FL2_initDStream');
+ @FL2_decompressStream := MemoryGetProcAddress(DLLHandle,
+ 'FL2_decompressStream');
+ @FL2_endStream := MemoryGetProcAddress(DLLHandle, 'FL2_endStream');
+ @FL2_isError := MemoryGetProcAddress(DLLHandle, 'FL2_isError');
+ DLLLoaded := Assigned(FL2_compress) and Assigned(FL2_decompress);
+ end
+ else
+ DLLLoaded := False;
+end;
+
+procedure Deinit;
+begin
+ if not DLLLoaded then
+ exit;
+ MemoryFreeLibrary(DLLHandle);
+end;
+
+initialization
+
+Init;
+
+finalization
+
+Deinit;
+
+end.
diff --git a/imports/LZMADLL.pas b/imports/LZMADLL.pas
deleted file mode 100644
index 58ac065..0000000
--- a/imports/LZMADLL.pas
+++ /dev/null
@@ -1,28 +0,0 @@
-unit LZMADLL;
-
-interface
-
-uses
- WinAPI.Windows,
- System.SysUtils;
-
-type
- PFL2_inBuffer = ^FL2_inBuffer;
-
- FL2_inBuffer = record
- src: Pointer;
- size: size_t;
- pos: size_t;
- end;
-
- PFL2_outBuffer = ^FL2_outBuffer;
-
- FL2_outBuffer = record
- dst: Pointer;
- size: size_t;
- pos: size_t;
- end;
-
-implementation
-
-end.
diff --git a/imports/XDeltaDLL.pas b/imports/XDeltaDLL.pas
index c8c6f6a..c7ff24e 100644
--- a/imports/XDeltaDLL.pas
+++ b/imports/XDeltaDLL.pas
@@ -46,11 +46,7 @@ var
procedure Init;
begin
-{$IFDEF WIN32}
- DLLStream := TResourceStream.Create(HInstance, 'XDELTA86_DLL', RT_RCDATA);
-{$ELSE}
- DLLStream := TResourceStream.Create(HInstance, 'XDELTA64_DLL', RT_RCDATA);
-{$ENDIF}
+ DLLStream := TResourceStream.Create(HInstance, 'xdelta3_dll', RT_RCDATA);
DLLHandle := MemoryLoadLibary(DLLStream.Memory);
if Assigned(DLLHandle) then
begin
diff --git a/imports/ZSTDDLL.pas b/imports/ZSTDDLL.pas
index d1498c2..480e454 100644
--- a/imports/ZSTDDLL.pas
+++ b/imports/ZSTDDLL.pas
@@ -9,10 +9,11 @@ uses
type
ZSTD_strategy = (ZSTD_fast = 1, ZSTD_dfast = 2, ZSTD_greedy = 3,
ZSTD_lazy = 4, ZSTD_lazy2 = 5, ZSTD_btlazy2 = 6, ZSTD_btopt = 7,
- ZSTD_btultra = 8, ZSTD_btultra2 = 9);
+ ZSTD_btultra = 8, ZSTD_btultra2 = 9, ZSTD_strategy_Force32 = $40000000);
ZSTD_ResetDirective = (ZSTD_reset_session_only = 1, ZSTD_reset_parameters = 2,
- ZSTD_reset_session_and_parameters = 3);
+ ZSTD_reset_session_and_parameters = 3,
+ ZSTD_ResetDirective_Force32 = $40000000);
PZSTD_inBuffer = ^ZSTD_inBuffer;
@@ -47,7 +48,7 @@ type
ZSTD_c_experimentalParam9 = 1006, ZSTD_c_experimentalParam10 = 1007,
ZSTD_c_experimentalParam11 = 1008, ZSTD_c_experimentalParam12 = 1009,
ZSTD_c_experimentalParam13 = 1010, ZSTD_c_experimentalParam14 = 1011,
- ZSTD_c_experimentalParam15 = 1012);
+ ZSTD_c_experimentalParam15 = 1012, ZSTD_cParameter_Force32 = $40000000);
ZSTD_compressionParameters = record
windowLog: Cardinal;
@@ -73,6 +74,8 @@ type
var
ZSTD_compress: function(dst: Pointer; dstCapacity: size_t; const src: Pointer;
srcSize: size_t; compressionLevel: Integer): size_t cdecl;
+ ZSTD_compress2: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
+ const src: Pointer; srcSize: size_t): size_t cdecl;
ZSTD_decompress: function(dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t): SSIZE_T cdecl;
ZSTD_findFrameCompressedSize: function(const src: Pointer; srcSize: size_t)
@@ -81,6 +84,10 @@ var
: int64 cdecl;
ZSTD_createCCtx: function: Pointer cdecl;
ZSTD_freeCCtx: function(cctx: Pointer): size_t cdecl;
+ ZSTD_CCtx_reset: function(cctx: Pointer; reset: ZSTD_ResetDirective)
+ : size_t cdecl;
+ ZSTD_CCtx_setParameter: function(cctx: Pointer; param: ZSTD_cParameter;
+ value: Integer): size_t cdecl;
ZSTD_compressCCtx: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
src: Pointer; srcSize: size_t; compressionLevel: Integer): size_t cdecl;
ZSTD_createDCtx: function: Pointer cdecl;
@@ -99,6 +106,13 @@ var
ZSTD_decompress_usingDDict: function(dctx: Pointer; dst: Pointer;
dstCapacity: size_t; const src: Pointer; srcSize: size_t;
const ddict: Pointer): size_t cdecl;
+ ZSTD_initCStream: function(zcs: Pointer; compressionLevel: Integer)
+ : size_t cdecl;
+ ZSTD_compressStream: function(zcs: Pointer; output: PZSTD_outBuffer;
+ input: PZSTD_inBuffer): size_t cdecl;
+ ZSTD_flushStream: function(zcs: Pointer; output: PZSTD_outBuffer)
+ : size_t cdecl;
+ ZSTD_endStream: function(zcs: Pointer; output: PZSTD_outBuffer): size_t cdecl;
DLLLoaded: Boolean = False;
@@ -138,6 +152,7 @@ begin
if DLLHandle >= 32 then
begin
@ZSTD_compress := GetProcAddress(DLLHandle, 'ZSTD_compress');
+ @ZSTD_compress2 := GetProcAddress(DLLHandle, 'ZSTD_compress2');
@ZSTD_decompress := GetProcAddress(DLLHandle, 'ZSTD_decompress');
@ZSTD_findFrameCompressedSize := GetProcAddress(DLLHandle,
'ZSTD_findFrameCompressedSize');
@@ -145,6 +160,9 @@ begin
'ZSTD_findDecompressedSize');
@ZSTD_createCCtx := GetProcAddress(DLLHandle, 'ZSTD_createCCtx');
@ZSTD_freeCCtx := GetProcAddress(DLLHandle, 'ZSTD_freeCCtx');
+ @ZSTD_CCtx_reset := GetProcAddress(DLLHandle, 'ZSTD_CCtx_reset');
+ @ZSTD_CCtx_setParameter := GetProcAddress(DLLHandle,
+ 'ZSTD_CCtx_setParameter');
@ZSTD_createDCtx := GetProcAddress(DLLHandle, 'ZSTD_createDCtx');
@ZSTD_freeDCtx := GetProcAddress(DLLHandle, 'ZSTD_freeDCtx');
@ZSTD_createCDict := GetProcAddress(DLLHandle, 'ZSTD_createCDict');
@@ -157,6 +175,10 @@ begin
'ZSTD_compress_usingCDict');
@ZSTD_decompress_usingDDict := GetProcAddress(DLLHandle,
'ZSTD_decompress_usingDDict');
+ @ZSTD_initCStream := GetProcAddress(DLLHandle, 'ZSTD_initCStream');
+ @ZSTD_compressStream := GetProcAddress(DLLHandle, 'ZSTD_compressStream');
+ @ZSTD_flushStream := GetProcAddress(DLLHandle, 'ZSTD_flushStream');
+ @ZSTD_endStream := GetProcAddress(DLLHandle, 'ZSTD_endStream');
DLLLoaded := Assigned(ZSTD_compress) and Assigned(ZSTD_decompress);
end
else
diff --git a/precompressor/PrecompLZ4.pas b/precompressor/PrecompLZ4.pas
index ff9005a..a407415 100644
--- a/precompressor/PrecompLZ4.pas
+++ b/precompressor/PrecompLZ4.pas
@@ -266,21 +266,32 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
- if StreamInfo^.Status = TStreamStatus.Predicted then
+ if StreamInfo^.Status >= TStreamStatus.Predicted then
+ begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
+ if (StreamInfo^.Status = TStreamStatus.Database) and
+ (GetBits(StreamInfo^.Option, 1, 31) = 0) then
+ begin
+ Res1 := StreamInfo^.OldSize;
+ Result := True;
+ end;
+ end;
Params := '';
case X of
LZ4_CODEC:
begin
Params := 'a' + GetBits(StreamInfo^.Option, 15, 7).ToString;
- Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y,
- GetBits(StreamInfo^.Option, 15, 7));
+ if not Result then
+ Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y,
+ GetBits(StreamInfo^.Option, 15, 7));
end;
LZ4HC_CODEC:
begin
Params := 'l' + I.ToString;
- Res1 := LZ4_compress_HC(NewInput, Buffer, StreamInfo^.NewSize, Y, I);
+ if not Result then
+ Res1 := LZ4_compress_HC(NewInput, Buffer,
+ StreamInfo^.NewSize, Y, I);
end;
LZ4F_CODEC:
begin
@@ -293,18 +304,20 @@ begin
Params := 'l' + I.ToString + ':' + 'b' +
(GetBits(StreamInfo^.Option, 12, 2) + 4).ToString + ':' + 'd' +
GetBits(StreamInfo^.Option, 14, 1).ToString;
- Res1 := LZ4F_compressFrame(Buffer, Y, NewInput,
- StreamInfo^.NewSize, @LZ4FT);
+ if not Result then
+ Res1 := LZ4F_compressFrame(Buffer, Y, NewInput,
+ StreamInfo^.NewSize, @LZ4FT);
end;
end;
- Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
- StreamInfo^.OldSize);
+ if not Result then
+ Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+ StreamInfo^.OldSize);
Funcs^.LogProcess(LZ4Codecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
- if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
+ if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));
diff --git a/precompressor/PrecompLZO.pas b/precompressor/PrecompLZO.pas
index f090292..f5d8e80 100644
--- a/precompressor/PrecompLZO.pas
+++ b/precompressor/PrecompLZO.pas
@@ -298,9 +298,17 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
- if StreamInfo^.Status = TStreamStatus.Predicted then
+ if StreamInfo^.Status >= TStreamStatus.Predicted then
+ begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
+ if (StreamInfo^.Status = TStreamStatus.Database) and
+ (GetBits(StreamInfo^.Option, 1, 31) = 0) then
+ begin
+ Res1 := StreamInfo^.OldSize;
+ Result := True;
+ end;
+ end;
Params := '';
Res1 := StreamInfo^.NewSize;
case X of
@@ -310,23 +318,25 @@ begin
begin
Params := 'l' + I.ToString + ':' + 'v' +
GetBits(StreamInfo^.Option, 12, 5).ToString;
- if not lzo1x_999_compress_level(NewInput, StreamInfo^.NewSize,
- Buffer, @Res1, @WrkMem[Instance, 0], nil, 0, nil, I) = 0 then
- Res1 := 0;
+ if not Result then
+ if not lzo1x_999_compress_level(NewInput, StreamInfo^.NewSize,
+ Buffer, @Res1, @WrkMem[Instance, 0], nil, 0, nil, I) = 0 then
+ Res1 := 0;
end;
{ if not lzo1x_1_compress(NewInput, StreamInfo^.NewSize, Buffer,
@Res1, @WrkMem[Instance, 0]) = 0 then
Res1 := 0; }
end;
end;
- Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
- StreamInfo^.OldSize);
+ if not Result then
+ Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+ StreamInfo^.OldSize);
Funcs^.LogProcess(LZOCodecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
- if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
+ if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));
diff --git a/precompressor/PrecompMain.pas b/precompressor/PrecompMain.pas
index aa53630..c02530c 100644
--- a/precompressor/PrecompMain.pas
+++ b/precompressor/PrecompMain.pas
@@ -5,7 +5,7 @@ unit PrecompMain;
interface
uses
- Threading, Utils, SynCommons, ParseClass, ParseExpr,
+ Threading, Utils, SynCommons, ParseClass, ParseExpr, FLZMA2DLL,
PrecompUtils, PrecompCrypto, PrecompZLib, PrecompLZ4, PrecompLZO, PrecompZSTD,
PrecompOodle, PrecompMedia, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE,
WinAPI.Windows, WinAPI.ShlObj,
@@ -25,6 +25,8 @@ type
Depth: Integer;
LowMem: Boolean;
DBaseFile, ExtractDir: String;
+ DoCompress: Boolean;
+ CompressCfg: String;
end;
PDecodeOptions = ^TDecodeOptions;
@@ -34,6 +36,7 @@ type
ChunkCount, Threads: Integer;
Depth: Integer;
DedupSysMem, DedupGPUMem: Int64;
+ CompressCfg: String;
end;
procedure PrintHelp;
@@ -122,7 +125,7 @@ begin
WriteLn(ErrOutput, '');
WriteLn(ErrOutput, 'Parameters:');
WriteLn(ErrOutput,
- ' -m# - codecs to use for precompression (separate by "+" if more than one)');
+ ' -m# - codecs to use for precompression (separate with "+" if more than one)');
WriteLn(ErrOutput, ' -c# - scanning range of precompressor [16mb]');
WriteLn(ErrOutput, ' -t# - number of working threads [50p]');
WriteLn(ErrOutput, ' -lm - low memory mode');
@@ -131,13 +134,16 @@ begin
WriteLn(ErrOutput, 'Advanced parameters:');
WriteLn(ErrOutput,
' --dbase=# - use database (#=filename to save db, optional)');
- WriteLn(ErrOutput,
- ' --dedup=# - use stream deduplication (#=filename to save db, optional)');
+ WriteLn(ErrOutput, ' --dedup - use stream deduplication');
WriteLn(ErrOutput,
' --mem=# - deduplication ram usage limit (#=size) [75p]');
WriteLn(ErrOutput,
' --diff=# - set xdelta threshold to accept streams [5p]');
WriteLn(ErrOutput, ' --extract=# - extract streams to directory path');
+ WriteLn(ErrOutput,
+ ' --compress=# - compress data using fast lzma2 (separate params with ":"');
+ WriteLn(ErrOutput, ' l# - compression level [5]');
+ WriteLn(ErrOutput, ' t# - number of threads [50p]');
WriteLn(ErrOutput, '');
end;
@@ -191,6 +197,12 @@ begin
Options.ExtractDir := ArgParse.AsString('--extract=');
if Options.ExtractDir <> '' then
EXTRACT := DirectoryExists(Options.ExtractDir);
+ Options.DoCompress := ArgParse.AsBoolean('--compress');
+ S := ArgParse.AsString('--compress=');
+ S := ReplaceText(S, SPrecompSep3, SPrecompSep2);
+ Options.CompressCfg := S;
+ if Options.CompressCfg <> '' then
+ Options.DoCompress := True;
finally
ArgParse.Free;
ExpParse.Free;
@@ -228,6 +240,9 @@ begin
if B then
Options.DedupSysMem := -Options.DedupSysMem;
VERBOSE := ArgParse.AsBoolean('--verbose');
+ S := ArgParse.AsString('--compress=', 0, 't50p');
+ S := ReplaceText(S, SPrecompSep3, SPrecompSep2);
+ Options.CompressCfg := S;
finally
ArgParse.Free;
ExpParse.Free;
@@ -1905,8 +1920,8 @@ begin
for I := Low(ThreadSync) to High(ThreadSync) do
ThreadSync[I] := TCriticalSection.Create;
DupSysMem := Options^.DedupSysMem;
- NStream.Add(TypeInfo(TMemoryStream) { , CalcSysMem } );
- // NStream.Add(TypeInfo(TPrecompVMStream));
+ NStream.Add(TypeInfo(TMemoryStream), CalcSysMem);
+ NStream.Add(TypeInfo(TPrecompVMStream));
Input.ReadBuffer(Options^.Depth, Options^.Depth.Size);
Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size);
SetLength(Bytes, LongRec(I).Bytes[0]);
@@ -1975,19 +1990,8 @@ begin
MemOutput2[I] := TMemoryStream.Create;
end;
end;
- Input.ReadBuffer(StoreDD, StoreDD.Size);
- UI32 := 0;
- if StoreDD then
- begin
- Input.ReadBuffer(UI32, UI32.Size);
- SetLength(DDList2, UI32);
- DDCount2 := UI32;
- for I := Low(DDList2) to High(DDList2) do
- Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
- DDIndex1 := -1;
- DDIndex2 := 0;
- end;
DataMgr := TDataManager.Create(NStream);
+ Input.ReadBuffer(StoreDD, StoreDD.Size);
end;
procedure DecFree;
@@ -2037,7 +2041,20 @@ var
I, J: Integer;
begin
if Depth = 0 then
+ begin
+ UI32 := 0;
+ if StoreDD then
+ begin
+ Input.ReadBuffer(UI32, UI32.Size);
+ SetLength(DDList2, UI32);
+ DDCount2 := UI32;
+ for I := Low(DDList2) to High(DDList2) do
+ Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
+ DDIndex1 := -1;
+ DDIndex2 := 0;
+ end;
LogInt64 := 0;
+ end;
with ComVars2[Depth] do
begin
DecInput[Index] := Input;
@@ -2252,6 +2269,9 @@ begin
end;
procedure Encode(Input, Output: TStream; Options: TEncodeOptions);
+var
+ Compressed: Boolean;
+ LOutput: TStream;
begin
InternalSync.Enter;
FillChar(EncInfo, SizeOf(EncInfo), 0);
@@ -2263,8 +2283,16 @@ begin
ConTask.Start;
try
EncInit(Input, Output, @Options);
- EncData(Input, Output, 0, 0);
+ Compressed := Options.DoCompress;
+ Output.WriteBuffer(Compressed, Compressed.Size);
+ if Options.DoCompress then
+ LOutput := TLZMACompressStream.Create(Output, Options.CompressCfg)
+ else
+ LOutput := Output;
+ EncData(Input, LOutput, 0, 0);
finally
+ if Options.DoCompress then
+ LOutput.Free;
try
EncFree;
finally
@@ -2279,6 +2307,9 @@ begin
end;
procedure Decode(Input, Output: TStream; Options: TDecodeOptions);
+var
+ Compressed: Boolean;
+ LInput: TStream;
begin
InternalSync.Enter;
FillChar(EncInfo, SizeOf(EncInfo), 0);
@@ -2291,8 +2322,15 @@ begin
NStream := TArrayStream.Create;
try
DecInit(Input, Output, @Options);
- DecChunk(Input, Output, 0, 0);
+ Input.ReadBuffer(Compressed, Compressed.Size);
+ if Compressed then
+ LInput := TLZMADecompressStream.Create(Input, Options.CompressCfg)
+ else
+ LInput := Input;
+ DecChunk(LInput, Output, 0, 0);
finally
+ if Compressed then
+ LInput.Free;
try
NStream.Free;
DecFree;
diff --git a/precompressor/PrecompMedia.pas b/precompressor/PrecompMedia.pas
index 5c7c616..81a71f9 100644
--- a/precompressor/PrecompMedia.pas
+++ b/precompressor/PrecompMedia.pas
@@ -140,9 +140,10 @@ begin
end
else if subchunk_hdr.subchunk_id = data_SIGN then
begin
- Result := True;
- chunk_size^ := subchunk_hdr.subchunk_size;
+ chunk_size^ := Min(subchunk_hdr.subchunk_size,
+ (riffhdr.chunk_size + 8 - Pos));
header_size^ := Pos;
+ Result := chunk_size^ + header_size^ <= InSize;
exit;
end
else
@@ -750,7 +751,6 @@ begin
StreamInfo^.NewSize := Res;
Result := True;
end;
- ShowMessage('');
end;
BRUNSLI_CODEC:
begin
diff --git a/precompressor/PrecompOodle.pas b/precompressor/PrecompOodle.pas
index 7950c03..eae05c0 100644
--- a/precompressor/PrecompOodle.pas
+++ b/precompressor/PrecompOodle.pas
@@ -711,25 +711,35 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
- if StreamInfo^.Status = TStreamStatus.Predicted then
+ if StreamInfo^.Status >= TStreamStatus.Predicted then
+ begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
+ if (StreamInfo^.Status = TStreamStatus.Database) and
+ (GetBits(StreamInfo^.Option, 1, 31) = 0) then
+ begin
+ Res1 := StreamInfo^.OldSize;
+ Result := True;
+ end;
+ end;
Move(OodleLZ_CompressOptions_GetDefault(Y, I)^, COptions,
SizeOf(TOodleLZ_CompressOptions));
COptions.sendQuantumCRCs := GetBits(StreamInfo^.Option, 12, 1) = 1;
COptions.spaceSpeedTradeoffBytes := GetBits(StreamInfo^.Option, 13, 11);
Params := 'l' + I.ToString + ':' + 'c' + GetBits(StreamInfo^.Option, 12, 1)
.ToString + ':' + 't' + GetBits(StreamInfo^.Option, 13, 11).ToString;
- Res1 := OodleLZ_Compress(Y, NewInput, StreamInfo^.NewSize, Buffer, I,
- @COptions);
- Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
- StreamInfo^.OldSize);
+ if not Result then
+ Res1 := OodleLZ_Compress(Y, NewInput, StreamInfo^.NewSize, Buffer, I,
+ @COptions);
+ if not Result then
+ Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+ StreamInfo^.OldSize);
Funcs^.LogProcess(OodleCodecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
- if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
+ if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));
diff --git a/precompressor/PrecompUtils.pas b/precompressor/PrecompUtils.pas
index 377e612..82460bd 100644
--- a/precompressor/PrecompUtils.pas
+++ b/precompressor/PrecompUtils.pas
@@ -16,6 +16,7 @@ resourcestring
SPrecompSep2 = ':';
SPrecompSep3 = ',';
SPrecompSep4 = '/';
+ SPrecompSep5 = '/';
const
SuccessStatus = 4;
diff --git a/precompressor/PrecompZSTD.pas b/precompressor/PrecompZSTD.pas
index c4bd9d1..bb60918 100644
--- a/precompressor/PrecompZSTD.pas
+++ b/precompressor/PrecompZSTD.pas
@@ -250,9 +250,9 @@ var
X: Integer;
Res1: Integer;
Res2: NativeUInt;
- // Inp: ZSTD_inBuffer;
- // Oup: ZSTD_outBuffer;
- // Progress: NativeInt;
+ Inp: ZSTD_inBuffer;
+ Oup: ZSTD_outBuffer;
+ Progress: NativeInt;
begin
Result := False;
X := GetBits(StreamInfo^.Option, 0, 5);
@@ -262,29 +262,43 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
- if StreamInfo^.Status = TStreamStatus.Predicted then
+ if StreamInfo^.Status >= TStreamStatus.Predicted then
+ begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
+ if (StreamInfo^.Status = TStreamStatus.Database) and
+ (GetBits(StreamInfo^.Option, 1, 31) = 0) then
+ begin
+ Res1 := StreamInfo^.OldSize;
+ Result := True;
+ end;
+ end;
Params := '';
case X of
ZSTD_CODEC:
begin
Params := 'l' + I.ToString;
- Res1 := ZSTD_compressCCtx(cctx[Instance], Buffer, StreamInfo^.NewSize,
- NewInput, StreamInfo^.NewSize, I);
+ { ZSTD_CCtx_reset(cctx[Instance], ZSTD_reset_session_and_parameters);
+ ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_strategy, 5);
+ ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_compressionLevel, I); }
+ if not Result then
+ Res1 := ZSTD_compressCCtx(cctx[Instance], Buffer,
+ StreamInfo^.NewSize, NewInput, StreamInfo^.NewSize, I);
end;
{ Res1 := ZSTD_compress_usingCDict(cctx[Instance], Buffer,
StreamInfo^.NewSize, NewInput, StreamInfo^.NewSize, cdict); }
{ begin
+ Params := 'l' + I.ToString;
Progress := 0;
Oup.dst := Buffer;
Oup.Size := StreamInfo^.NewSize;
Oup.Pos := 0;
ZSTD_initCStream(cctx[Instance], I);
+ ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_strategy, 9);
while Progress < StreamInfo^.NewSize do
begin
Inp.src := PByte(NewInput) + Progress;
- Inp.Size := Min(StreamInfo^.NewSize - Progress, 32768);
+ Inp.Size := Min(StreamInfo^.NewSize - Progress, 64 * 1024);
Inp.Pos := 0;
if ZSTD_compressStream(cctx[Instance], @Oup, @Inp) > 0 then
begin
@@ -298,8 +312,9 @@ begin
Res1 := Oup.Pos;
end; }
end;
- Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
- StreamInfo^.OldSize);
+ if not Result then
+ Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
+ StreamInfo^.OldSize);
Funcs^.LogProcess(ZSTDCodecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
@@ -307,7 +322,7 @@ begin
end;
if Res1 < 0 then
exit;
- if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
+ if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));
diff --git a/xtool.dpr b/xtool.dpr
index 9dc958e..516684e 100644
--- a/xtool.dpr
+++ b/xtool.dpr
@@ -54,6 +54,7 @@ uses
ParseExpr in 'contrib\ParseExpression\ParseExpr.pas',
BrunsliDLL in 'imports\BrunsliDLL.pas',
FLACDLL in 'imports\FLACDLL.pas',
+ FLZMA2DLL in 'imports\FLZMA2DLL.pas',
JoJpegDLL in 'imports\JoJpegDLL.pas',
LZ4DLL in 'imports\LZ4DLL.pas',
LZODLL in 'imports\LZODLL.pas',
@@ -87,8 +88,7 @@ uses
IOPatch in 'io\IOPatch.pas',
IOExecute in 'io\IOExecute.pas',
IODecode in 'io\IODecode.pas',
- IOUtils in 'io\IOUtils.pas',
- LZMADLL in 'imports\LZMADLL.pas';
+ IOUtils in 'io\IOUtils.pas';
{$SETPEFLAGS IMAGE_FILE_LARGE_ADDRESS_AWARE or IMAGE_FILE_RELOCS_STRIPPED}
diff --git a/xtool.dproj b/xtool.dproj
index 194f8e7..b28a609 100644
--- a/xtool.dproj
+++ b/xtool.dproj
@@ -113,6 +113,7 @@
+
@@ -147,18 +148,17 @@
-
-
- ResourceItem
- RCDATA
- XDELTA86_DLL
-
ResourceItem
RCDATA
- XDELTA64_DLL
+ xdelta3_dll
+
+ ResourceItem
+ RCDATA
+ fast_lzma2
+
Base
@@ -208,7 +208,7 @@
true
-
+
.\
true
diff --git a/xtool.dres b/xtool.dres
index 2a18a3b..41f324e 100644
Binary files a/xtool.dres and b/xtool.dres differ
diff --git a/xtoolResource.rc b/xtoolResource.rc
index dac36ac..88aceb4 100644
--- a/xtoolResource.rc
+++ b/xtoolResource.rc
@@ -1,2 +1,2 @@
-XDELTA86_DLL RCDATA "resources\\Win32\\xdelta3_dll.dll"
-XDELTA64_DLL RCDATA "resources\\Win64\\xdelta3_dll.dll"
+xdelta3_dll RCDATA "resources\\Win64\\xdelta3_dll.dll"
+fast_lzma2 RCDATA "resources\\Win64\\fast-lzma2.dll"