Vichingo455-RePacks
/
xtool
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

update to 0.6.1

This commit is contained in:
Razor12911 2022-07-12 05:19:10 +02:00
parent 762160b0c9
commit fb6bcfa239
17 changed files with 539 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
changes.txt
View File

@ -1,11 +1,16 @@
ES_R37 (0.6.2)
- added feature to inject libraries to main executable
ES_R36 (0.6.1)
- added fast lzma2 compression for portable mode
- fixed minor issue with stream deduplication feature
ES_R35 (0.6.0)
- added wav stream detector
- added flac codec
- added jpg stream detector
- added packjpg, brunsli, jojpeg codec
- added fast lzma2 compression for portable mode
- added feature that allows input to be a directory
- added feature to inject libraries to main executable
- added feature to extract detected streams
- updated database feature
- updated deduplication feature
@ -248,4 +253,4 @@
- added grittibanzli (also handles deflate stream but slow af)
2008_R1 (0.2.1)
- initial release
- initial release

39
common/Utils.pas
View File

@ -490,6 +490,8 @@ function GetCmdStr(CommandLine: String; Index: Integer;
KeepQuotes: Boolean = False): string;
function GetCmdCount(CommandLine: String): Integer;
procedure UpdateFileResource(Source, Dest, ResName: string);
implementation
function GetBits(Data: Int64; Index: TInt64_BitIndex;
@ -3615,4 +3617,41 @@ begin
Inc(Result);
end;
procedure UpdateFileResource(Source, Dest, ResName: string);
var
Stream: TFileStream;
hDestRes: THandle;
lpData: Pointer;
cbData: DWORD;
begin
Stream := TFileStream.Create(Source, fmOpenRead or fmShareDenyNone);
try
Stream.Seek(0, soFromBeginning);
cbData := Stream.Size;
if cbData > 0 then
begin
GetMem(lpData, cbData);
try
Stream.Read(lpData^, cbData);
hDestRes := BeginUpdateResource(PChar(Dest), False);
if hDestRes <> 0 then
if UpdateResource(hDestRes, RT_RCDATA, PWideChar(ResName), 0, lpData,
cbData) then
begin
if not EndUpdateResource(hDestRes, False) then
RaiseLastOSError
end
else
RaiseLastOSError
else
RaiseLastOSError;
finally
FreeMem(lpData);
end;
end;
finally
Stream.Free;
end;
end;
end.

312
imports/FLZMA2DLL.pas Normal file
View File

@ -0,0 +1,312 @@
unit FLZMA2DLL;
interface
uses
MemoryModule,
WinAPI.Windows,
System.SysUtils, System.Classes, System.Types;
type
PFL2_inBuffer = ^FL2_inBuffer;
FL2_inBuffer = record
src: Pointer;
size: size_t;
pos: size_t;
end;
PFL2_outBuffer = ^FL2_outBuffer;
FL2_outBuffer = record
dst: Pointer;
size: size_t;
pos: size_t;
end;
var
FL2_compress: function(dst: Pointer; dstCapacity: size_t; const src: Pointer;
srcSize: size_t; compressionLevel: Integer): size_t cdecl;
FL2_compressMt: function(dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t; compressionLevel: Integer;
nbThreads: Cardinal): size_t cdecl;
FL2_decompress: function(dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t): size_t cdecl;
FL2_decompressMt: function(dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t; nbThreads: Cardinal): size_t cdecl;
FL2_createCCtx: function: Pointer cdecl;
FL2_createCCtxMt: function(nbThreads: Cardinal): Pointer cdecl;
FL2_freeCCtx: procedure(cctx: Pointer)cdecl;
FL2_compressCCtx: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
src: Pointer; srcSize: size_t; compressionLevel: Integer): size_t cdecl;
FL2_createDCtx: function: Pointer cdecl;
FL2_createDCtxMt: function(nbThreads: Cardinal): Pointer cdecl;
FL2_freeDCtx: function(dctx: Pointer): size_t cdecl;
FL2_decompressDCtx: function(dctx: Pointer; dst: Pointer; dstCapacity: size_t;
src: Pointer; srcSize: size_t): size_t cdecl;
FL2_createCStream: function: Pointer cdecl;
FL2_createCStreamMt: function(nbThreads: Cardinal; dualBuffer: Integer)
: Pointer cdecl;
FL2_freeCStream: procedure(fcs: Pointer)cdecl;
FL2_initCStream: function(fcs: Pointer; compressionLevel: Integer)
: size_t cdecl;
FL2_compressStream: function(fcs: Pointer; output: PFL2_outBuffer;
input: PFL2_inBuffer): size_t cdecl;
FL2_createDStream: function: Pointer cdecl;
FL2_createDStreamMt: function(nbThreads: Cardinal): Pointer cdecl;
FL2_freeDStream: procedure(fds: Pointer)cdecl;
FL2_initDStream: function(fds: Pointer): size_t cdecl;
FL2_decompressStream: function(fds: Pointer; output: PFL2_outBuffer;
input: PFL2_inBuffer): size_t cdecl;
FL2_endStream: function(fcs: Pointer; output: PFL2_outBuffer): size_t cdecl;
FL2_isError: function(code: size_t): Cardinal cdecl;
DLLLoaded: boolean = False;
type
TLZMACRec = record
Threads: Integer;
Level: Integer;
procedure Parse(S: String);
end;
TLZMADRec = record
Threads: Integer;
procedure Parse(S: String);
end;
TLZMACompressStream = class(TStream)
private const
FBufferSize = 65536;
private
FCtx: Pointer;
FProp: TLZMACRec;
FOutput: TStream;
FBuffer: array [0 .. FBufferSize - 1] of Byte;
public
constructor Create(AOutput: TStream; AConfig: String = 't50p');
destructor Destroy; override;
function Write(const Buffer; Count: LongInt): LongInt; override;
end;
TLZMADecompressStream = class(TStream)
private const
FBufferSize = 65536;
private
FCtx: Pointer;
FProp: TLZMADRec;
FInp: FL2_inBuffer;
FInput: TStream;
FBuffer: array [0 .. FBufferSize - 1] of Byte;
public
constructor Create(AInput: TStream; AConfig: String = '');
destructor Destroy; override;
function Read(var Buffer; Count: Integer): Integer; override;
end;
implementation
uses
Utils;
var
DLLStream: TResourceStream;
DLLHandle: TMemoryModule;
procedure TLZMACRec.Parse(S: string);
var
List: TStringDynArray;
I, J: Integer;
begin
Threads := 1;
Level := 5;
List := DecodeStr(S, ':');
for I := Low(List) to High(List) do
begin
if List[I].StartsWith('t', True) then
Threads := ConvertToThreads(List[I].Substring(1));
if List[I].StartsWith('l', True) then
Level := List[I].Substring(1).ToInteger;
end;
end;
procedure TLZMADRec.Parse(S: string);
var
List: TStringDynArray;
I: Integer;
begin
Threads := 1;
List := DecodeStr(S, ':');
for I := Low(List) to High(List) do
begin
if List[I].StartsWith('t', True) then
Threads := ConvertToThreads(List[I].Substring(1));
end;
end;
constructor TLZMACompressStream.Create(AOutput: TStream; AConfig: String);
begin
inherited Create;
FProp.Parse(AConfig);
FOutput := AOutput;
if FProp.Threads > 1 then
FCtx := FL2_createCStreamMt(FProp.Threads, 0)
else
FCtx := FL2_createCStream;
FL2_initCStream(FCtx, FProp.Level);
end;
destructor TLZMACompressStream.Destroy;
var
Oup: FL2_outBuffer;
Res: size_t;
begin
Oup.dst := @FBuffer[0];
Oup.size := FBufferSize;
Oup.pos := 0;
repeat
Res := FL2_endStream(FCtx, @Oup);
FOutput.WriteBuffer(FBuffer[0], Oup.pos);
Oup.pos := 0;
until Res = 0;
FL2_freeCCtx(FCtx);
inherited Destroy;
end;
function TLZMACompressStream.Write(const Buffer; Count: LongInt): LongInt;
var
Inp: FL2_inBuffer;
Oup: FL2_outBuffer;
begin
Result := 0;
Inp.src := PByte(@Buffer);
Inp.size := Count;
Inp.pos := 0;
Oup.dst := @FBuffer[0];
Oup.size := FBufferSize;
Oup.pos := 0;
while Inp.pos < Inp.size do
begin
if not boolean(FL2_isError(FL2_compressStream(FCtx, @Oup, @Inp))) then
begin
FOutput.WriteBuffer(FBuffer[0], Oup.pos);
Oup.pos := 0;
end;
end;
Result := Inp.pos;
end;
constructor TLZMADecompressStream.Create(AInput: TStream; AConfig: String);
begin
inherited Create;
FProp.Parse(AConfig);
FInput := AInput;
if FProp.Threads > 1 then
FCtx := FL2_createDStreamMt(FProp.Threads)
else
FCtx := FL2_createDStream;
FL2_initDStream(FCtx);
FillChar(FInp, SizeOf(FL2_inBuffer), 0);
end;
destructor TLZMADecompressStream.Destroy;
begin
FL2_freeDCtx(FCtx);
inherited Destroy;
end;
function TLZMADecompressStream.Read(var Buffer; Count: Integer): Integer;
var
Oup: FL2_outBuffer;
begin
Result := 0;
if FInp.pos = FInp.size then
begin
FInp.src := @FBuffer[0];
FInp.size := FInput.Read(FBuffer[0], FBufferSize);
FInp.pos := 0;
if FInp.size = 0 then
exit;
end;
Oup.dst := PByte(@Buffer);
Oup.size := Count;
Oup.pos := 0;
while Oup.pos < Oup.size do
begin
if not boolean(FL2_isError(FL2_decompressStream(FCtx, @Oup, @FInp))) then
begin
if FInp.pos = FInp.size then
begin
FInp.src := @FBuffer[0];
FInp.size := FInput.Read(FBuffer[0], FBufferSize);
FInp.pos := 0;
if FInp.size = 0 then
break;
end;
end
else
break;
end;
Result := Oup.pos;
end;
procedure Init;
begin
DLLStream := TResourceStream.Create(HInstance, 'fast_lzma2', RT_RCDATA);
DLLHandle := MemoryLoadLibary(DLLStream.Memory);
if Assigned(DLLHandle) then
begin
@FL2_compress := MemoryGetProcAddress(DLLHandle, 'FL2_compress');
@FL2_compressMt := MemoryGetProcAddress(DLLHandle, 'FL2_compressMt');
@FL2_decompress := MemoryGetProcAddress(DLLHandle, 'FL2_decompress');
@FL2_decompressMt := MemoryGetProcAddress(DLLHandle, 'FL2_decompressMt');
@FL2_createCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_createCCtx');
@FL2_createCCtxMt := MemoryGetProcAddress(DLLHandle, 'FL2_createCCtxMt');
@FL2_freeCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_freeCCtx');
@FL2_compressCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_compressCCtx');
@FL2_createDCtx := MemoryGetProcAddress(DLLHandle, 'FL2_createDCtx');
@FL2_createDCtxMt := MemoryGetProcAddress(DLLHandle, 'FL2_createDCtxMt');
@FL2_freeDCtx := MemoryGetProcAddress(DLLHandle, 'FL2_freeDCtx');
@FL2_decompressDCtx := MemoryGetProcAddress(DLLHandle,
'FL2_decompressDCtx');
@FL2_createCStream := MemoryGetProcAddress(DLLHandle, 'FL2_createCStream');
@FL2_createCStreamMt := MemoryGetProcAddress(DLLHandle,
'FL2_createCStreamMt');
@FL2_freeCStream := MemoryGetProcAddress(DLLHandle, 'FL2_freeCStream');
@FL2_initCStream := MemoryGetProcAddress(DLLHandle, 'FL2_initCStream');
@FL2_compressStream := MemoryGetProcAddress(DLLHandle,
'FL2_compressStream');
@FL2_createDStream := MemoryGetProcAddress(DLLHandle, 'FL2_createDStream');
@FL2_createDStreamMt := MemoryGetProcAddress(DLLHandle,
'FL2_createDStreamMt');
@FL2_freeDStream := MemoryGetProcAddress(DLLHandle, 'FL2_freeDStream');
@FL2_initDStream := MemoryGetProcAddress(DLLHandle, 'FL2_initDStream');
@FL2_decompressStream := MemoryGetProcAddress(DLLHandle,
'FL2_decompressStream');
@FL2_endStream := MemoryGetProcAddress(DLLHandle, 'FL2_endStream');
@FL2_isError := MemoryGetProcAddress(DLLHandle, 'FL2_isError');
DLLLoaded := Assigned(FL2_compress) and Assigned(FL2_decompress);
end
else
DLLLoaded := False;
end;
procedure Deinit;
begin
if not DLLLoaded then
exit;
MemoryFreeLibrary(DLLHandle);
end;
initialization
Init;
finalization
Deinit;
end.

28
imports/LZMADLL.pas
View File

@ -1,28 +0,0 @@
unit LZMADLL;
interface
uses
WinAPI.Windows,
System.SysUtils;
type
PFL2_inBuffer = ^FL2_inBuffer;
FL2_inBuffer = record
src: Pointer;
size: size_t;
pos: size_t;
end;
PFL2_outBuffer = ^FL2_outBuffer;
FL2_outBuffer = record
dst: Pointer;
size: size_t;
pos: size_t;
end;
implementation
end.

6
imports/XDeltaDLL.pas
View File

@ -46,11 +46,7 @@ var
procedure Init;
begin
{$IFDEF WIN32}
DLLStream := TResourceStream.Create(HInstance, 'XDELTA86_DLL', RT_RCDATA);
{$ELSE}
DLLStream := TResourceStream.Create(HInstance, 'XDELTA64_DLL', RT_RCDATA);
{$ENDIF}
DLLStream := TResourceStream.Create(HInstance, 'xdelta3_dll', RT_RCDATA);
DLLHandle := MemoryLoadLibary(DLLStream.Memory);
if Assigned(DLLHandle) then
begin

28
imports/ZSTDDLL.pas
View File

@ -9,10 +9,11 @@ uses
type
ZSTD_strategy = (ZSTD_fast = 1, ZSTD_dfast = 2, ZSTD_greedy = 3,
ZSTD_lazy = 4, ZSTD_lazy2 = 5, ZSTD_btlazy2 = 6, ZSTD_btopt = 7,
ZSTD_btultra = 8, ZSTD_btultra2 = 9);
ZSTD_btultra = 8, ZSTD_btultra2 = 9, ZSTD_strategy_Force32 = $40000000);
ZSTD_ResetDirective = (ZSTD_reset_session_only = 1, ZSTD_reset_parameters = 2,
ZSTD_reset_session_and_parameters = 3);
ZSTD_reset_session_and_parameters = 3,
ZSTD_ResetDirective_Force32 = $40000000);
PZSTD_inBuffer = ^ZSTD_inBuffer;
@ -47,7 +48,7 @@ type
ZSTD_c_experimentalParam9 = 1006, ZSTD_c_experimentalParam10 = 1007,
ZSTD_c_experimentalParam11 = 1008, ZSTD_c_experimentalParam12 = 1009,
ZSTD_c_experimentalParam13 = 1010, ZSTD_c_experimentalParam14 = 1011,
ZSTD_c_experimentalParam15 = 1012);
ZSTD_c_experimentalParam15 = 1012, ZSTD_cParameter_Force32 = $40000000);
ZSTD_compressionParameters = record
windowLog: Cardinal;
@ -73,6 +74,8 @@ type
var
ZSTD_compress: function(dst: Pointer; dstCapacity: size_t; const src: Pointer;
srcSize: size_t; compressionLevel: Integer): size_t cdecl;
ZSTD_compress2: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t): size_t cdecl;
ZSTD_decompress: function(dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t): SSIZE_T cdecl;
ZSTD_findFrameCompressedSize: function(const src: Pointer; srcSize: size_t)
@ -81,6 +84,10 @@ var
: int64 cdecl;
ZSTD_createCCtx: function: Pointer cdecl;
ZSTD_freeCCtx: function(cctx: Pointer): size_t cdecl;
ZSTD_CCtx_reset: function(cctx: Pointer; reset: ZSTD_ResetDirective)
: size_t cdecl;
ZSTD_CCtx_setParameter: function(cctx: Pointer; param: ZSTD_cParameter;
value: Integer): size_t cdecl;
ZSTD_compressCCtx: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
src: Pointer; srcSize: size_t; compressionLevel: Integer): size_t cdecl;
ZSTD_createDCtx: function: Pointer cdecl;
@ -99,6 +106,13 @@ var
ZSTD_decompress_usingDDict: function(dctx: Pointer; dst: Pointer;
dstCapacity: size_t; const src: Pointer; srcSize: size_t;
const ddict: Pointer): size_t cdecl;
ZSTD_initCStream: function(zcs: Pointer; compressionLevel: Integer)
: size_t cdecl;
ZSTD_compressStream: function(zcs: Pointer; output: PZSTD_outBuffer;
input: PZSTD_inBuffer): size_t cdecl;
ZSTD_flushStream: function(zcs: Pointer; output: PZSTD_outBuffer)
: size_t cdecl;
ZSTD_endStream: function(zcs: Pointer; output: PZSTD_outBuffer): size_t cdecl;
DLLLoaded: Boolean = False;
@ -138,6 +152,7 @@ begin
if DLLHandle >= 32 then
begin
@ZSTD_compress := GetProcAddress(DLLHandle, 'ZSTD_compress');
@ZSTD_compress2 := GetProcAddress(DLLHandle, 'ZSTD_compress2');
@ZSTD_decompress := GetProcAddress(DLLHandle, 'ZSTD_decompress');
@ZSTD_findFrameCompressedSize := GetProcAddress(DLLHandle,
'ZSTD_findFrameCompressedSize');
@ -145,6 +160,9 @@ begin
'ZSTD_findDecompressedSize');
@ZSTD_createCCtx := GetProcAddress(DLLHandle, 'ZSTD_createCCtx');
@ZSTD_freeCCtx := GetProcAddress(DLLHandle, 'ZSTD_freeCCtx');
@ZSTD_CCtx_reset := GetProcAddress(DLLHandle, 'ZSTD_CCtx_reset');
@ZSTD_CCtx_setParameter := GetProcAddress(DLLHandle,
'ZSTD_CCtx_setParameter');
@ZSTD_createDCtx := GetProcAddress(DLLHandle, 'ZSTD_createDCtx');
@ZSTD_freeDCtx := GetProcAddress(DLLHandle, 'ZSTD_freeDCtx');
@ZSTD_createCDict := GetProcAddress(DLLHandle, 'ZSTD_createCDict');
@ -157,6 +175,10 @@ begin
'ZSTD_compress_usingCDict');
@ZSTD_decompress_usingDDict := GetProcAddress(DLLHandle,
'ZSTD_decompress_usingDDict');
@ZSTD_initCStream := GetProcAddress(DLLHandle, 'ZSTD_initCStream');
@ZSTD_compressStream := GetProcAddress(DLLHandle, 'ZSTD_compressStream');
@ZSTD_flushStream := GetProcAddress(DLLHandle, 'ZSTD_flushStream');
@ZSTD_endStream := GetProcAddress(DLLHandle, 'ZSTD_endStream');
DLLLoaded := Assigned(ZSTD_compress) and Assigned(ZSTD_decompress);
end
else

31
precompressor/PrecompLZ4.pas
View File

@ -266,21 +266,32 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Params := '';
case X of
LZ4_CODEC:
begin
Params := 'a' + GetBits(StreamInfo^.Option, 15, 7).ToString;
Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y,
GetBits(StreamInfo^.Option, 15, 7));
if not Result then
Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y,
GetBits(StreamInfo^.Option, 15, 7));
end;
LZ4HC_CODEC:
begin
Params := 'l' + I.ToString;
Res1 := LZ4_compress_HC(NewInput, Buffer, StreamInfo^.NewSize, Y, I);
if not Result then
Res1 := LZ4_compress_HC(NewInput, Buffer,
StreamInfo^.NewSize, Y, I);
end;
LZ4F_CODEC:
begin
@ -293,18 +304,20 @@ begin
Params := 'l' + I.ToString + ':' + 'b' +
(GetBits(StreamInfo^.Option, 12, 2) + 4).ToString + ':' + 'd' +
GetBits(StreamInfo^.Option, 14, 1).ToString;
Res1 := LZ4F_compressFrame(Buffer, Y, NewInput,
StreamInfo^.NewSize, @LZ4FT);
if not Result then
Res1 := LZ4F_compressFrame(Buffer, Y, NewInput,
StreamInfo^.NewSize, @LZ4FT);
end;
end;
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(LZ4Codecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

24
precompressor/PrecompLZO.pas
View File

@ -298,9 +298,17 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Params := '';
Res1 := StreamInfo^.NewSize;
case X of
@ -310,23 +318,25 @@ begin
begin
Params := 'l' + I.ToString + ':' + 'v' +
GetBits(StreamInfo^.Option, 12, 5).ToString;
if not lzo1x_999_compress_level(NewInput, StreamInfo^.NewSize,
Buffer, @Res1, @WrkMem[Instance, 0], nil, 0, nil, I) = 0 then
Res1 := 0;
if not Result then
if not lzo1x_999_compress_level(NewInput, StreamInfo^.NewSize,
Buffer, @Res1, @WrkMem[Instance, 0], nil, 0, nil, I) = 0 then
Res1 := 0;
end;
{ if not lzo1x_1_compress(NewInput, StreamInfo^.NewSize, Buffer,
@Res1, @WrkMem[Instance, 0]) = 0 then
Res1 := 0; }
end;
end;
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(LZOCodecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

78
precompressor/PrecompMain.pas
View File

@ -5,7 +5,7 @@ unit PrecompMain;
interface
uses
Threading, Utils, SynCommons, ParseClass, ParseExpr,
Threading, Utils, SynCommons, ParseClass, ParseExpr, FLZMA2DLL,
PrecompUtils, PrecompCrypto, PrecompZLib, PrecompLZ4, PrecompLZO, PrecompZSTD,
PrecompOodle, PrecompMedia, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE,
WinAPI.Windows, WinAPI.ShlObj,
@ -25,6 +25,8 @@ type
Depth: Integer;
LowMem: Boolean;
DBaseFile, ExtractDir: String;
DoCompress: Boolean;
CompressCfg: String;
end;
PDecodeOptions = ^TDecodeOptions;
@ -34,6 +36,7 @@ type
ChunkCount, Threads: Integer;
Depth: Integer;
DedupSysMem, DedupGPUMem: Int64;
CompressCfg: String;
end;
procedure PrintHelp;
@ -122,7 +125,7 @@ begin
WriteLn(ErrOutput, '');
WriteLn(ErrOutput, 'Parameters:');
WriteLn(ErrOutput,
' -m# - codecs to use for precompression (separate by "+" if more than one)');
' -m# - codecs to use for precompression (separate with "+" if more than one)');
WriteLn(ErrOutput, ' -c# - scanning range of precompressor [16mb]');
WriteLn(ErrOutput, ' -t# - number of working threads [50p]');
WriteLn(ErrOutput, ' -lm - low memory mode');
@ -131,13 +134,16 @@ begin
WriteLn(ErrOutput, 'Advanced parameters:');
WriteLn(ErrOutput,
' --dbase=# - use database (#=filename to save db, optional)');
WriteLn(ErrOutput,
' --dedup=# - use stream deduplication (#=filename to save db, optional)');
WriteLn(ErrOutput, ' --dedup - use stream deduplication');
WriteLn(ErrOutput,
' --mem=# - deduplication ram usage limit (#=size) [75p]');
WriteLn(ErrOutput,
' --diff=# - set xdelta threshold to accept streams [5p]');
WriteLn(ErrOutput, ' --extract=# - extract streams to directory path');
WriteLn(ErrOutput,
' --compress=# - compress data using fast lzma2 (separate params with ":"');
WriteLn(ErrOutput, ' l# - compression level [5]');
WriteLn(ErrOutput, ' t# - number of threads [50p]');
WriteLn(ErrOutput, '');
end;
@ -191,6 +197,12 @@ begin
Options.ExtractDir := ArgParse.AsString('--extract=');
if Options.ExtractDir <> '' then
EXTRACT := DirectoryExists(Options.ExtractDir);
Options.DoCompress := ArgParse.AsBoolean('--compress');
S := ArgParse.AsString('--compress=');
S := ReplaceText(S, SPrecompSep3, SPrecompSep2);
Options.CompressCfg := S;
if Options.CompressCfg <> '' then
Options.DoCompress := True;
finally
ArgParse.Free;
ExpParse.Free;
@ -228,6 +240,9 @@ begin
if B then
Options.DedupSysMem := -Options.DedupSysMem;
VERBOSE := ArgParse.AsBoolean('--verbose');
S := ArgParse.AsString('--compress=', 0, 't50p');
S := ReplaceText(S, SPrecompSep3, SPrecompSep2);
Options.CompressCfg := S;
finally
ArgParse.Free;
ExpParse.Free;
@ -1905,8 +1920,8 @@ begin
for I := Low(ThreadSync) to High(ThreadSync) do
ThreadSync[I] := TCriticalSection.Create;
DupSysMem := Options^.DedupSysMem;
NStream.Add(TypeInfo(TMemoryStream) { , CalcSysMem } );
// NStream.Add(TypeInfo(TPrecompVMStream));
NStream.Add(TypeInfo(TMemoryStream), CalcSysMem);
NStream.Add(TypeInfo(TPrecompVMStream));
Input.ReadBuffer(Options^.Depth, Options^.Depth.Size);
Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size);
SetLength(Bytes, LongRec(I).Bytes[0]);
@ -1975,19 +1990,8 @@ begin
MemOutput2[I] := TMemoryStream.Create;
end;
end;
Input.ReadBuffer(StoreDD, StoreDD.Size);
UI32 := 0;
if StoreDD then
begin
Input.ReadBuffer(UI32, UI32.Size);
SetLength(DDList2, UI32);
DDCount2 := UI32;
for I := Low(DDList2) to High(DDList2) do
Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
DDIndex1 := -1;
DDIndex2 := 0;
end;
DataMgr := TDataManager.Create(NStream);
Input.ReadBuffer(StoreDD, StoreDD.Size);
end;
procedure DecFree;
@ -2037,7 +2041,20 @@ var
I, J: Integer;
begin
if Depth = 0 then
begin
UI32 := 0;
if StoreDD then
begin
Input.ReadBuffer(UI32, UI32.Size);
SetLength(DDList2, UI32);
DDCount2 := UI32;
for I := Low(DDList2) to High(DDList2) do
Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
DDIndex1 := -1;
DDIndex2 := 0;
end;
LogInt64 := 0;
end;
with ComVars2[Depth] do
begin
DecInput[Index] := Input;
@ -2252,6 +2269,9 @@ begin
end;
procedure Encode(Input, Output: TStream; Options: TEncodeOptions);
var
Compressed: Boolean;
LOutput: TStream;
begin
InternalSync.Enter;
FillChar(EncInfo, SizeOf(EncInfo), 0);
@ -2263,8 +2283,16 @@ begin
ConTask.Start;
try
EncInit(Input, Output, @Options);
EncData(Input, Output, 0, 0);
Compressed := Options.DoCompress;
Output.WriteBuffer(Compressed, Compressed.Size);
if Options.DoCompress then
LOutput := TLZMACompressStream.Create(Output, Options.CompressCfg)
else
LOutput := Output;
EncData(Input, LOutput, 0, 0);
finally
if Options.DoCompress then
LOutput.Free;
try
EncFree;
finally
@ -2279,6 +2307,9 @@ begin
end;
procedure Decode(Input, Output: TStream; Options: TDecodeOptions);
var
Compressed: Boolean;
LInput: TStream;
begin
InternalSync.Enter;
FillChar(EncInfo, SizeOf(EncInfo), 0);
@ -2291,8 +2322,15 @@ begin
NStream := TArrayStream.Create;
try
DecInit(Input, Output, @Options);
DecChunk(Input, Output, 0, 0);
Input.ReadBuffer(Compressed, Compressed.Size);
if Compressed then
LInput := TLZMADecompressStream.Create(Input, Options.CompressCfg)
else
LInput := Input;
DecChunk(LInput, Output, 0, 0);
finally
if Compressed then
LInput.Free;
try
NStream.Free;
DecFree;

6
precompressor/PrecompMedia.pas
View File

@ -140,9 +140,10 @@ begin
end
else if subchunk_hdr.subchunk_id = data_SIGN then
begin
Result := True;
chunk_size^ := subchunk_hdr.subchunk_size;
chunk_size^ := Min(subchunk_hdr.subchunk_size,
(riffhdr.chunk_size + 8 - Pos));
header_size^ := Pos;
Result := chunk_size^ + header_size^ <= InSize;
exit;
end
else
@ -750,7 +751,6 @@ begin
StreamInfo^.NewSize := Res;
Result := True;
end;
ShowMessage('');
end;
BRUNSLI_CODEC:
begin

22
precompressor/PrecompOodle.pas
View File

@ -711,25 +711,35 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Move(OodleLZ_CompressOptions_GetDefault(Y, I)^, COptions,
SizeOf(TOodleLZ_CompressOptions));
COptions.sendQuantumCRCs := GetBits(StreamInfo^.Option, 12, 1) = 1;
COptions.spaceSpeedTradeoffBytes := GetBits(StreamInfo^.Option, 13, 11);
Params := 'l' + I.ToString + ':' + 'c' + GetBits(StreamInfo^.Option, 12, 1)
.ToString + ':' + 't' + GetBits(StreamInfo^.Option, 13, 11).ToString;
Res1 := OodleLZ_Compress(Y, NewInput, StreamInfo^.NewSize, Buffer, I,
@COptions);
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
if not Result then
Res1 := OodleLZ_Compress(Y, NewInput, StreamInfo^.NewSize, Buffer, I,
@COptions);
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(OodleCodecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

1
precompressor/PrecompUtils.pas
View File

@ -16,6 +16,7 @@ resourcestring
SPrecompSep2 = ':';
SPrecompSep3 = ',';
SPrecompSep4 = '/';
SPrecompSep5 = '/';
const
SuccessStatus = 4;

35
precompressor/PrecompZSTD.pas
View File

@ -250,9 +250,9 @@ var
X: Integer;
Res1: Integer;
Res2: NativeUInt;
// Inp: ZSTD_inBuffer;
// Oup: ZSTD_outBuffer;
// Progress: NativeInt;
Inp: ZSTD_inBuffer;
Oup: ZSTD_outBuffer;
Progress: NativeInt;
begin
Result := False;
X := GetBits(StreamInfo^.Option, 0, 5);
@ -262,29 +262,43 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Params := '';
case X of
ZSTD_CODEC:
begin
Params := 'l' + I.ToString;
Res1 := ZSTD_compressCCtx(cctx[Instance], Buffer, StreamInfo^.NewSize,
NewInput, StreamInfo^.NewSize, I);
{ ZSTD_CCtx_reset(cctx[Instance], ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_strategy, 5);
ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_compressionLevel, I); }
if not Result then
Res1 := ZSTD_compressCCtx(cctx[Instance], Buffer,
StreamInfo^.NewSize, NewInput, StreamInfo^.NewSize, I);
end;
{ Res1 := ZSTD_compress_usingCDict(cctx[Instance], Buffer,
StreamInfo^.NewSize, NewInput, StreamInfo^.NewSize, cdict); }
{ begin
Params := 'l' + I.ToString;
Progress := 0;
Oup.dst := Buffer;
Oup.Size := StreamInfo^.NewSize;
Oup.Pos := 0;
ZSTD_initCStream(cctx[Instance], I);
ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_strategy, 9);
while Progress < StreamInfo^.NewSize do
begin
Inp.src := PByte(NewInput) + Progress;
Inp.Size := Min(StreamInfo^.NewSize - Progress, 32768);
Inp.Size := Min(StreamInfo^.NewSize - Progress, 64 * 1024);
Inp.Pos := 0;
if ZSTD_compressStream(cctx[Instance], @Oup, @Inp) > 0 then
begin
@ -298,8 +312,9 @@ begin
Res1 := Oup.Pos;
end; }
end;
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(ZSTDCodecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
@ -307,7 +322,7 @@ begin
end;
if Res1 < 0 then
exit;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

4
xtool.dpr
View File

@ -54,6 +54,7 @@ uses
ParseExpr in 'contrib\ParseExpression\ParseExpr.pas',
BrunsliDLL in 'imports\BrunsliDLL.pas',
FLACDLL in 'imports\FLACDLL.pas',
FLZMA2DLL in 'imports\FLZMA2DLL.pas',
JoJpegDLL in 'imports\JoJpegDLL.pas',
LZ4DLL in 'imports\LZ4DLL.pas',
LZODLL in 'imports\LZODLL.pas',
@ -87,8 +88,7 @@ uses
IOPatch in 'io\IOPatch.pas',
IOExecute in 'io\IOExecute.pas',
IODecode in 'io\IODecode.pas',
IOUtils in 'io\IOUtils.pas',
LZMADLL in 'imports\LZMADLL.pas';
IOUtils in 'io\IOUtils.pas';
{$SETPEFLAGS IMAGE_FILE_LARGE_ADDRESS_AWARE or IMAGE_FILE_RELOCS_STRIPPED}

16
xtool.dproj
View File

@ -113,6 +113,7 @@
<DCCReference Include="contrib\ParseExpression\ParseExpr.pas"/>
<DCCReference Include="imports\BrunsliDLL.pas"/>
<DCCReference Include="imports\FLACDLL.pas"/>
<DCCReference Include="imports\FLZMA2DLL.pas"/>
<DCCReference Include="imports\JoJpegDLL.pas"/>
<DCCReference Include="imports\LZ4DLL.pas"/>
<DCCReference Include="imports\LZODLL.pas"/>
@ -147,18 +148,17 @@
<DCCReference Include="io\IOExecute.pas"/>
<DCCReference Include="io\IODecode.pas"/>
<DCCReference Include="io\IOUtils.pas"/>
<DCCReference Include="imports\LZMADLL.pas"/>
<RcItem Include="resources\Win32\xdelta3_dll.dll">
<ContainerId>ResourceItem</ContainerId>
<ResourceType>RCDATA</ResourceType>
<ResourceId>XDELTA86_DLL</ResourceId>
</RcItem>
<RcItem Include="resources\Win64\xdelta3_dll.dll">
<ContainerId>ResourceItem</ContainerId>
<ResourceType>RCDATA</ResourceType>
<ResourceId>XDELTA64_DLL</ResourceId>
<ResourceId>xdelta3_dll</ResourceId>
</RcItem>
<None Include="changes.txt"/>
<RcItem Include="resources\Win64\fast-lzma2.dll">
<ContainerId>ResourceItem</ContainerId>
<ResourceType>RCDATA</ResourceType>
<ResourceId>fast_lzma2</ResourceId>
</RcItem>
<BuildConfiguration Include="Base">
<Key>Base</Key>
</BuildConfiguration>
@ -208,7 +208,7 @@
<Overwrite>true</Overwrite>
</Platform>
</DeployFile>
<DeployFile LocalName="resources\Win32\xdelta3_dll.dll" Configuration="Release" Class="ProjectFile">
<DeployFile LocalName="resources\Win64\fast-lzma2.dll" Configuration="Release" Class="ProjectFile">
<Platform Name="Win64">
<RemoteDir>.\</RemoteDir>
<Overwrite>true</Overwrite>

BIN
xtool.dres
View File

Binary file not shown.

4
xtoolResource.rc
View File

@ -1,2 +1,2 @@
XDELTA86_DLL RCDATA "resources\\Win32\\xdelta3_dll.dll"
XDELTA64_DLL RCDATA "resources\\Win64\\xdelta3_dll.dll"
xdelta3_dll RCDATA "resources\\Win64\\xdelta3_dll.dll"
fast_lzma2 RCDATA "resources\\Win64\\fast-lzma2.dll"