Vichingo455-RePacks
/
xtool
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

update to 0.6.1

This commit is contained in:
Razor12911 2022-07-12 05:19:10 +02:00
parent 762160b0c9
commit fb6bcfa239
17 changed files with 539 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
changes.txt
View File

@ -1,11 +1,16 @@
ES_R37 (0.6.2)
- added feature to inject libraries to main executable
ES_R36 (0.6.1)
- added fast lzma2 compression for portable mode
- fixed minor issue with stream deduplication feature
ES_R35 (0.6.0)
- added wav stream detector
- added flac codec
- added jpg stream detector
- added packjpg, brunsli, jojpeg codec
- added fast lzma2 compression for portable mode
- added feature that allows input to be a directory
- added feature to inject libraries to main executable
- added feature to extract detected streams
- updated database feature
- updated deduplication feature

39
common/Utils.pas
View File

@ -490,6 +490,8 @@ function GetCmdStr(CommandLine: String; Index: Integer;
KeepQuotes: Boolean = False): string;
function GetCmdCount(CommandLine: String): Integer;
procedure UpdateFileResource(Source, Dest, ResName: string);
implementation
function GetBits(Data: Int64; Index: TInt64_BitIndex;
@ -3615,4 +3617,41 @@ begin
Inc(Result);
end;
procedure UpdateFileResource(Source, Dest, ResName: string);
var
Stream: TFileStream;
hDestRes: THandle;
lpData: Pointer;
cbData: DWORD;
begin
Stream := TFileStream.Create(Source, fmOpenRead or fmShareDenyNone);
try
Stream.Seek(0, soFromBeginning);
cbData := Stream.Size;
if cbData > 0 then
begin
GetMem(lpData, cbData);
try
Stream.Read(lpData^, cbData);
hDestRes := BeginUpdateResource(PChar(Dest), False);
if hDestRes <> 0 then
if UpdateResource(hDestRes, RT_RCDATA, PWideChar(ResName), 0, lpData,
cbData) then
begin
if not EndUpdateResource(hDestRes, False) then
RaiseLastOSError
end
else
RaiseLastOSError
else
RaiseLastOSError;
finally
FreeMem(lpData);
end;
end;
finally
Stream.Free;
end;
end;
end.

312
imports/FLZMA2DLL.pas Normal file
View File

@ -0,0 +1,312 @@
unit FLZMA2DLL;
interface
uses
MemoryModule,
WinAPI.Windows,
System.SysUtils, System.Classes, System.Types;
type
PFL2_inBuffer = ^FL2_inBuffer;
FL2_inBuffer = record
src: Pointer;
size: size_t;
pos: size_t;
end;
PFL2_outBuffer = ^FL2_outBuffer;
FL2_outBuffer = record
dst: Pointer;
size: size_t;
pos: size_t;
end;
var
FL2_compress: function(dst: Pointer; dstCapacity: size_t; const src: Pointer;
srcSize: size_t; compressionLevel: Integer): size_t cdecl;
FL2_compressMt: function(dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t; compressionLevel: Integer;
nbThreads: Cardinal): size_t cdecl;
FL2_decompress: function(dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t): size_t cdecl;
FL2_decompressMt: function(dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t; nbThreads: Cardinal): size_t cdecl;
FL2_createCCtx: function: Pointer cdecl;
FL2_createCCtxMt: function(nbThreads: Cardinal): Pointer cdecl;
FL2_freeCCtx: procedure(cctx: Pointer)cdecl;
FL2_compressCCtx: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
src: Pointer; srcSize: size_t; compressionLevel: Integer): size_t cdecl;
FL2_createDCtx: function: Pointer cdecl;
FL2_createDCtxMt: function(nbThreads: Cardinal): Pointer cdecl;
FL2_freeDCtx: function(dctx: Pointer): size_t cdecl;
FL2_decompressDCtx: function(dctx: Pointer; dst: Pointer; dstCapacity: size_t;
src: Pointer; srcSize: size_t): size_t cdecl;
FL2_createCStream: function: Pointer cdecl;
FL2_createCStreamMt: function(nbThreads: Cardinal; dualBuffer: Integer)
: Pointer cdecl;
FL2_freeCStream: procedure(fcs: Pointer)cdecl;
FL2_initCStream: function(fcs: Pointer; compressionLevel: Integer)
: size_t cdecl;
FL2_compressStream: function(fcs: Pointer; output: PFL2_outBuffer;
input: PFL2_inBuffer): size_t cdecl;
FL2_createDStream: function: Pointer cdecl;
FL2_createDStreamMt: function(nbThreads: Cardinal): Pointer cdecl;
FL2_freeDStream: procedure(fds: Pointer)cdecl;
FL2_initDStream: function(fds: Pointer): size_t cdecl;
FL2_decompressStream: function(fds: Pointer; output: PFL2_outBuffer;
input: PFL2_inBuffer): size_t cdecl;
FL2_endStream: function(fcs: Pointer; output: PFL2_outBuffer): size_t cdecl;
FL2_isError: function(code: size_t): Cardinal cdecl;
DLLLoaded: boolean = False;
type
TLZMACRec = record
Threads: Integer;
Level: Integer;
procedure Parse(S: String);
end;
TLZMADRec = record
Threads: Integer;
procedure Parse(S: String);
end;
TLZMACompressStream = class(TStream)
private const
FBufferSize = 65536;
private
FCtx: Pointer;
FProp: TLZMACRec;
FOutput: TStream;
FBuffer: array [0 .. FBufferSize - 1] of Byte;
public
constructor Create(AOutput: TStream; AConfig: String = 't50p');
destructor Destroy; override;
function Write(const Buffer; Count: LongInt): LongInt; override;
end;
TLZMADecompressStream = class(TStream)
private const
FBufferSize = 65536;
private
FCtx: Pointer;
FProp: TLZMADRec;
FInp: FL2_inBuffer;
FInput: TStream;
FBuffer: array [0 .. FBufferSize - 1] of Byte;
public
constructor Create(AInput: TStream; AConfig: String = '');
destructor Destroy; override;
function Read(var Buffer; Count: Integer): Integer; override;
end;
implementation
uses
Utils;
var
DLLStream: TResourceStream;
DLLHandle: TMemoryModule;
procedure TLZMACRec.Parse(S: string);
var
List: TStringDynArray;
I, J: Integer;
begin
Threads := 1;
Level := 5;
List := DecodeStr(S, ':');
for I := Low(List) to High(List) do
begin
if List[I].StartsWith('t', True) then
Threads := ConvertToThreads(List[I].Substring(1));
if List[I].StartsWith('l', True) then
Level := List[I].Substring(1).ToInteger;
end;
end;
procedure TLZMADRec.Parse(S: string);
var
List: TStringDynArray;
I: Integer;
begin
Threads := 1;
List := DecodeStr(S, ':');
for I := Low(List) to High(List) do
begin
if List[I].StartsWith('t', True) then
Threads := ConvertToThreads(List[I].Substring(1));
end;
end;
constructor TLZMACompressStream.Create(AOutput: TStream; AConfig: String);
begin
inherited Create;
FProp.Parse(AConfig);
FOutput := AOutput;
if FProp.Threads > 1 then
FCtx := FL2_createCStreamMt(FProp.Threads, 0)
else
FCtx := FL2_createCStream;
FL2_initCStream(FCtx, FProp.Level);
end;
destructor TLZMACompressStream.Destroy;
var
Oup: FL2_outBuffer;
Res: size_t;
begin
Oup.dst := @FBuffer[0];
Oup.size := FBufferSize;
Oup.pos := 0;
repeat
Res := FL2_endStream(FCtx, @Oup);
FOutput.WriteBuffer(FBuffer[0], Oup.pos);
Oup.pos := 0;
until Res = 0;
FL2_freeCCtx(FCtx);
inherited Destroy;
end;
function TLZMACompressStream.Write(const Buffer; Count: LongInt): LongInt;
var
Inp: FL2_inBuffer;
Oup: FL2_outBuffer;
begin
Result := 0;
Inp.src := PByte(@Buffer);
Inp.size := Count;
Inp.pos := 0;
Oup.dst := @FBuffer[0];
Oup.size := FBufferSize;
Oup.pos := 0;
while Inp.pos < Inp.size do
begin
if not boolean(FL2_isError(FL2_compressStream(FCtx, @Oup, @Inp))) then
begin
FOutput.WriteBuffer(FBuffer[0], Oup.pos);
Oup.pos := 0;
end;
end;
Result := Inp.pos;
end;
constructor TLZMADecompressStream.Create(AInput: TStream; AConfig: String);
begin
inherited Create;
FProp.Parse(AConfig);
FInput := AInput;
if FProp.Threads > 1 then
FCtx := FL2_createDStreamMt(FProp.Threads)
else
FCtx := FL2_createDStream;
FL2_initDStream(FCtx);
FillChar(FInp, SizeOf(FL2_inBuffer), 0);
end;
destructor TLZMADecompressStream.Destroy;
begin
FL2_freeDCtx(FCtx);
inherited Destroy;
end;
function TLZMADecompressStream.Read(var Buffer; Count: Integer): Integer;
var
Oup: FL2_outBuffer;
begin
Result := 0;
if FInp.pos = FInp.size then
begin
FInp.src := @FBuffer[0];
FInp.size := FInput.Read(FBuffer[0], FBufferSize);
FInp.pos := 0;
if FInp.size = 0 then
exit;
end;
Oup.dst := PByte(@Buffer);
Oup.size := Count;
Oup.pos := 0;
while Oup.pos < Oup.size do
begin
if not boolean(FL2_isError(FL2_decompressStream(FCtx, @Oup, @FInp))) then
begin
if FInp.pos = FInp.size then
begin
FInp.src := @FBuffer[0];
FInp.size := FInput.Read(FBuffer[0], FBufferSize);
FInp.pos := 0;
if FInp.size = 0 then
break;
end;
end
else
break;
end;
Result := Oup.pos;
end;
procedure Init;
begin
DLLStream := TResourceStream.Create(HInstance, 'fast_lzma2', RT_RCDATA);
DLLHandle := MemoryLoadLibary(DLLStream.Memory);
if Assigned(DLLHandle) then
begin
@FL2_compress := MemoryGetProcAddress(DLLHandle, 'FL2_compress');
@FL2_compressMt := MemoryGetProcAddress(DLLHandle, 'FL2_compressMt');
@FL2_decompress := MemoryGetProcAddress(DLLHandle, 'FL2_decompress');
@FL2_decompressMt := MemoryGetProcAddress(DLLHandle, 'FL2_decompressMt');
@FL2_createCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_createCCtx');
@FL2_createCCtxMt := MemoryGetProcAddress(DLLHandle, 'FL2_createCCtxMt');
@FL2_freeCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_freeCCtx');
@FL2_compressCCtx := MemoryGetProcAddress(DLLHandle, 'FL2_compressCCtx');
@FL2_createDCtx := MemoryGetProcAddress(DLLHandle, 'FL2_createDCtx');
@FL2_createDCtxMt := MemoryGetProcAddress(DLLHandle, 'FL2_createDCtxMt');
@FL2_freeDCtx := MemoryGetProcAddress(DLLHandle, 'FL2_freeDCtx');
@FL2_decompressDCtx := MemoryGetProcAddress(DLLHandle,
'FL2_decompressDCtx');
@FL2_createCStream := MemoryGetProcAddress(DLLHandle, 'FL2_createCStream');
@FL2_createCStreamMt := MemoryGetProcAddress(DLLHandle,
'FL2_createCStreamMt');
@FL2_freeCStream := MemoryGetProcAddress(DLLHandle, 'FL2_freeCStream');
@FL2_initCStream := MemoryGetProcAddress(DLLHandle, 'FL2_initCStream');
@FL2_compressStream := MemoryGetProcAddress(DLLHandle,
'FL2_compressStream');
@FL2_createDStream := MemoryGetProcAddress(DLLHandle, 'FL2_createDStream');
@FL2_createDStreamMt := MemoryGetProcAddress(DLLHandle,
'FL2_createDStreamMt');
@FL2_freeDStream := MemoryGetProcAddress(DLLHandle, 'FL2_freeDStream');
@FL2_initDStream := MemoryGetProcAddress(DLLHandle, 'FL2_initDStream');
@FL2_decompressStream := MemoryGetProcAddress(DLLHandle,
'FL2_decompressStream');
@FL2_endStream := MemoryGetProcAddress(DLLHandle, 'FL2_endStream');
@FL2_isError := MemoryGetProcAddress(DLLHandle, 'FL2_isError');
DLLLoaded := Assigned(FL2_compress) and Assigned(FL2_decompress);
end
else
DLLLoaded := False;
end;
procedure Deinit;
begin
if not DLLLoaded then
exit;
MemoryFreeLibrary(DLLHandle);
end;
initialization
Init;
finalization
Deinit;
end.

28
imports/LZMADLL.pas
View File

@ -1,28 +0,0 @@
unit LZMADLL;
interface
uses
WinAPI.Windows,
System.SysUtils;
type
PFL2_inBuffer = ^FL2_inBuffer;
FL2_inBuffer = record
src: Pointer;
size: size_t;
pos: size_t;
end;
PFL2_outBuffer = ^FL2_outBuffer;
FL2_outBuffer = record
dst: Pointer;
size: size_t;
pos: size_t;
end;
implementation
end.

6
imports/XDeltaDLL.pas
View File

@ -46,11 +46,7 @@ var
procedure Init;
begin
{$IFDEF WIN32}
DLLStream := TResourceStream.Create(HInstance, 'XDELTA86_DLL', RT_RCDATA);
{$ELSE}
DLLStream := TResourceStream.Create(HInstance, 'XDELTA64_DLL', RT_RCDATA);
{$ENDIF}
DLLStream := TResourceStream.Create(HInstance, 'xdelta3_dll', RT_RCDATA);
DLLHandle := MemoryLoadLibary(DLLStream.Memory);
if Assigned(DLLHandle) then
begin

28
imports/ZSTDDLL.pas
View File

@ -9,10 +9,11 @@ uses
type
ZSTD_strategy = (ZSTD_fast = 1, ZSTD_dfast = 2, ZSTD_greedy = 3,
ZSTD_lazy = 4, ZSTD_lazy2 = 5, ZSTD_btlazy2 = 6, ZSTD_btopt = 7,
ZSTD_btultra = 8, ZSTD_btultra2 = 9);
ZSTD_btultra = 8, ZSTD_btultra2 = 9, ZSTD_strategy_Force32 = $40000000);
ZSTD_ResetDirective = (ZSTD_reset_session_only = 1, ZSTD_reset_parameters = 2,
ZSTD_reset_session_and_parameters = 3);
ZSTD_reset_session_and_parameters = 3,
ZSTD_ResetDirective_Force32 = $40000000);
PZSTD_inBuffer = ^ZSTD_inBuffer;
@ -47,7 +48,7 @@ type
ZSTD_c_experimentalParam9 = 1006, ZSTD_c_experimentalParam10 = 1007,
ZSTD_c_experimentalParam11 = 1008, ZSTD_c_experimentalParam12 = 1009,
ZSTD_c_experimentalParam13 = 1010, ZSTD_c_experimentalParam14 = 1011,
ZSTD_c_experimentalParam15 = 1012);
ZSTD_c_experimentalParam15 = 1012, ZSTD_cParameter_Force32 = $40000000);
ZSTD_compressionParameters = record
windowLog: Cardinal;
@ -73,6 +74,8 @@ type
var
ZSTD_compress: function(dst: Pointer; dstCapacity: size_t; const src: Pointer;
srcSize: size_t; compressionLevel: Integer): size_t cdecl;
ZSTD_compress2: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t): size_t cdecl;
ZSTD_decompress: function(dst: Pointer; dstCapacity: size_t;
const src: Pointer; srcSize: size_t): SSIZE_T cdecl;
ZSTD_findFrameCompressedSize: function(const src: Pointer; srcSize: size_t)
@ -81,6 +84,10 @@ var
: int64 cdecl;
ZSTD_createCCtx: function: Pointer cdecl;
ZSTD_freeCCtx: function(cctx: Pointer): size_t cdecl;
ZSTD_CCtx_reset: function(cctx: Pointer; reset: ZSTD_ResetDirective)
: size_t cdecl;
ZSTD_CCtx_setParameter: function(cctx: Pointer; param: ZSTD_cParameter;
value: Integer): size_t cdecl;
ZSTD_compressCCtx: function(cctx: Pointer; dst: Pointer; dstCapacity: size_t;
src: Pointer; srcSize: size_t; compressionLevel: Integer): size_t cdecl;
ZSTD_createDCtx: function: Pointer cdecl;
@ -99,6 +106,13 @@ var
ZSTD_decompress_usingDDict: function(dctx: Pointer; dst: Pointer;
dstCapacity: size_t; const src: Pointer; srcSize: size_t;
const ddict: Pointer): size_t cdecl;
ZSTD_initCStream: function(zcs: Pointer; compressionLevel: Integer)
: size_t cdecl;
ZSTD_compressStream: function(zcs: Pointer; output: PZSTD_outBuffer;
input: PZSTD_inBuffer): size_t cdecl;
ZSTD_flushStream: function(zcs: Pointer; output: PZSTD_outBuffer)
: size_t cdecl;
ZSTD_endStream: function(zcs: Pointer; output: PZSTD_outBuffer): size_t cdecl;
DLLLoaded: Boolean = False;
@ -138,6 +152,7 @@ begin
if DLLHandle >= 32 then
begin
@ZSTD_compress := GetProcAddress(DLLHandle, 'ZSTD_compress');
@ZSTD_compress2 := GetProcAddress(DLLHandle, 'ZSTD_compress2');
@ZSTD_decompress := GetProcAddress(DLLHandle, 'ZSTD_decompress');
@ZSTD_findFrameCompressedSize := GetProcAddress(DLLHandle,
'ZSTD_findFrameCompressedSize');
@ -145,6 +160,9 @@ begin
'ZSTD_findDecompressedSize');
@ZSTD_createCCtx := GetProcAddress(DLLHandle, 'ZSTD_createCCtx');
@ZSTD_freeCCtx := GetProcAddress(DLLHandle, 'ZSTD_freeCCtx');
@ZSTD_CCtx_reset := GetProcAddress(DLLHandle, 'ZSTD_CCtx_reset');
@ZSTD_CCtx_setParameter := GetProcAddress(DLLHandle,
'ZSTD_CCtx_setParameter');
@ZSTD_createDCtx := GetProcAddress(DLLHandle, 'ZSTD_createDCtx');
@ZSTD_freeDCtx := GetProcAddress(DLLHandle, 'ZSTD_freeDCtx');
@ZSTD_createCDict := GetProcAddress(DLLHandle, 'ZSTD_createCDict');
@ -157,6 +175,10 @@ begin
'ZSTD_compress_usingCDict');
@ZSTD_decompress_usingDDict := GetProcAddress(DLLHandle,
'ZSTD_decompress_usingDDict');
@ZSTD_initCStream := GetProcAddress(DLLHandle, 'ZSTD_initCStream');
@ZSTD_compressStream := GetProcAddress(DLLHandle, 'ZSTD_compressStream');
@ZSTD_flushStream := GetProcAddress(DLLHandle, 'ZSTD_flushStream');
@ZSTD_endStream := GetProcAddress(DLLHandle, 'ZSTD_endStream');
DLLLoaded := Assigned(ZSTD_compress) and Assigned(ZSTD_decompress);
end
else

19
precompressor/PrecompLZ4.pas
View File

@ -266,21 +266,32 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Params := '';
case X of
LZ4_CODEC:
begin
Params := 'a' + GetBits(StreamInfo^.Option, 15, 7).ToString;
if not Result then
Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y,
GetBits(StreamInfo^.Option, 15, 7));
end;
LZ4HC_CODEC:
begin
Params := 'l' + I.ToString;
Res1 := LZ4_compress_HC(NewInput, Buffer, StreamInfo^.NewSize, Y, I);
if not Result then
Res1 := LZ4_compress_HC(NewInput, Buffer,
StreamInfo^.NewSize, Y, I);
end;
LZ4F_CODEC:
begin
@ -293,10 +304,12 @@ begin
Params := 'l' + I.ToString + ':' + 'b' +
(GetBits(StreamInfo^.Option, 12, 2) + 4).ToString + ':' + 'd' +
GetBits(StreamInfo^.Option, 14, 1).ToString;
if not Result then
Res1 := LZ4F_compressFrame(Buffer, Y, NewInput,
StreamInfo^.NewSize, @LZ4FT);
end;
end;
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(LZ4Codecs[GetBits(StreamInfo^.Option, 0, 5)],
@ -304,7 +317,7 @@ begin
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

14
precompressor/PrecompLZO.pas
View File

@ -298,9 +298,17 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Params := '';
Res1 := StreamInfo^.NewSize;
case X of
@ -310,6 +318,7 @@ begin
begin
Params := 'l' + I.ToString + ':' + 'v' +
GetBits(StreamInfo^.Option, 12, 5).ToString;
if not Result then
if not lzo1x_999_compress_level(NewInput, StreamInfo^.NewSize,
Buffer, @Res1, @WrkMem[Instance, 0], nil, 0, nil, I) = 0 then
Res1 := 0;
@ -319,6 +328,7 @@ begin
Res1 := 0; }
end;
end;
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(LZOCodecs[GetBits(StreamInfo^.Option, 0, 5)],
@ -326,7 +336,7 @@ begin
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

78
precompressor/PrecompMain.pas
View File

@ -5,7 +5,7 @@ unit PrecompMain;
interface
uses
Threading, Utils, SynCommons, ParseClass, ParseExpr,
Threading, Utils, SynCommons, ParseClass, ParseExpr, FLZMA2DLL,
PrecompUtils, PrecompCrypto, PrecompZLib, PrecompLZ4, PrecompLZO, PrecompZSTD,
PrecompOodle, PrecompMedia, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE,
WinAPI.Windows, WinAPI.ShlObj,
@ -25,6 +25,8 @@ type
Depth: Integer;
LowMem: Boolean;
DBaseFile, ExtractDir: String;
DoCompress: Boolean;
CompressCfg: String;
end;
PDecodeOptions = ^TDecodeOptions;
@ -34,6 +36,7 @@ type
ChunkCount, Threads: Integer;
Depth: Integer;
DedupSysMem, DedupGPUMem: Int64;
CompressCfg: String;
end;
procedure PrintHelp;
@ -122,7 +125,7 @@ begin
WriteLn(ErrOutput, '');
WriteLn(ErrOutput, 'Parameters:');
WriteLn(ErrOutput,
' -m# - codecs to use for precompression (separate by "+" if more than one)');
' -m# - codecs to use for precompression (separate with "+" if more than one)');
WriteLn(ErrOutput, ' -c# - scanning range of precompressor [16mb]');
WriteLn(ErrOutput, ' -t# - number of working threads [50p]');
WriteLn(ErrOutput, ' -lm - low memory mode');
@ -131,13 +134,16 @@ begin
WriteLn(ErrOutput, 'Advanced parameters:');
WriteLn(ErrOutput,
' --dbase=# - use database (#=filename to save db, optional)');
WriteLn(ErrOutput,
' --dedup=# - use stream deduplication (#=filename to save db, optional)');
WriteLn(ErrOutput, ' --dedup - use stream deduplication');
WriteLn(ErrOutput,
' --mem=# - deduplication ram usage limit (#=size) [75p]');
WriteLn(ErrOutput,
' --diff=# - set xdelta threshold to accept streams [5p]');
WriteLn(ErrOutput, ' --extract=# - extract streams to directory path');
WriteLn(ErrOutput,
' --compress=# - compress data using fast lzma2 (separate params with ":"');
WriteLn(ErrOutput, ' l# - compression level [5]');
WriteLn(ErrOutput, ' t# - number of threads [50p]');
WriteLn(ErrOutput, '');
end;
@ -191,6 +197,12 @@ begin
Options.ExtractDir := ArgParse.AsString('--extract=');
if Options.ExtractDir <> '' then
EXTRACT := DirectoryExists(Options.ExtractDir);
Options.DoCompress := ArgParse.AsBoolean('--compress');
S := ArgParse.AsString('--compress=');
S := ReplaceText(S, SPrecompSep3, SPrecompSep2);
Options.CompressCfg := S;
if Options.CompressCfg <> '' then
Options.DoCompress := True;
finally
ArgParse.Free;
ExpParse.Free;
@ -228,6 +240,9 @@ begin
if B then
Options.DedupSysMem := -Options.DedupSysMem;
VERBOSE := ArgParse.AsBoolean('--verbose');
S := ArgParse.AsString('--compress=', 0, 't50p');
S := ReplaceText(S, SPrecompSep3, SPrecompSep2);
Options.CompressCfg := S;
finally
ArgParse.Free;
ExpParse.Free;
@ -1905,8 +1920,8 @@ begin
for I := Low(ThreadSync) to High(ThreadSync) do
ThreadSync[I] := TCriticalSection.Create;
DupSysMem := Options^.DedupSysMem;
NStream.Add(TypeInfo(TMemoryStream) { , CalcSysMem } );
// NStream.Add(TypeInfo(TPrecompVMStream));
NStream.Add(TypeInfo(TMemoryStream), CalcSysMem);
NStream.Add(TypeInfo(TPrecompVMStream));
Input.ReadBuffer(Options^.Depth, Options^.Depth.Size);
Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size);
SetLength(Bytes, LongRec(I).Bytes[0]);
@ -1975,19 +1990,8 @@ begin
MemOutput2[I] := TMemoryStream.Create;
end;
end;
Input.ReadBuffer(StoreDD, StoreDD.Size);
UI32 := 0;
if StoreDD then
begin
Input.ReadBuffer(UI32, UI32.Size);
SetLength(DDList2, UI32);
DDCount2 := UI32;
for I := Low(DDList2) to High(DDList2) do
Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
DDIndex1 := -1;
DDIndex2 := 0;
end;
DataMgr := TDataManager.Create(NStream);
Input.ReadBuffer(StoreDD, StoreDD.Size);
end;
procedure DecFree;
@ -2037,7 +2041,20 @@ var
I, J: Integer;
begin
if Depth = 0 then
begin
UI32 := 0;
if StoreDD then
begin
Input.ReadBuffer(UI32, UI32.Size);
SetLength(DDList2, UI32);
DDCount2 := UI32;
for I := Low(DDList2) to High(DDList2) do
Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
DDIndex1 := -1;
DDIndex2 := 0;
end;
LogInt64 := 0;
end;
with ComVars2[Depth] do
begin
DecInput[Index] := Input;
@ -2252,6 +2269,9 @@ begin
end;
procedure Encode(Input, Output: TStream; Options: TEncodeOptions);
var
Compressed: Boolean;
LOutput: TStream;
begin
InternalSync.Enter;
FillChar(EncInfo, SizeOf(EncInfo), 0);
@ -2263,8 +2283,16 @@ begin
ConTask.Start;
try
EncInit(Input, Output, @Options);
EncData(Input, Output, 0, 0);
Compressed := Options.DoCompress;
Output.WriteBuffer(Compressed, Compressed.Size);
if Options.DoCompress then
LOutput := TLZMACompressStream.Create(Output, Options.CompressCfg)
else
LOutput := Output;
EncData(Input, LOutput, 0, 0);
finally
if Options.DoCompress then
LOutput.Free;
try
EncFree;
finally
@ -2279,6 +2307,9 @@ begin
end;
procedure Decode(Input, Output: TStream; Options: TDecodeOptions);
var
Compressed: Boolean;
LInput: TStream;
begin
InternalSync.Enter;
FillChar(EncInfo, SizeOf(EncInfo), 0);
@ -2291,8 +2322,15 @@ begin
NStream := TArrayStream.Create;
try
DecInit(Input, Output, @Options);
DecChunk(Input, Output, 0, 0);
Input.ReadBuffer(Compressed, Compressed.Size);
if Compressed then
LInput := TLZMADecompressStream.Create(Input, Options.CompressCfg)
else
LInput := Input;
DecChunk(LInput, Output, 0, 0);
finally
if Compressed then
LInput.Free;
try
NStream.Free;
DecFree;

6
precompressor/PrecompMedia.pas
View File

@ -140,9 +140,10 @@ begin
end
else if subchunk_hdr.subchunk_id = data_SIGN then
begin
Result := True;
chunk_size^ := subchunk_hdr.subchunk_size;
chunk_size^ := Min(subchunk_hdr.subchunk_size,
(riffhdr.chunk_size + 8 - Pos));
header_size^ := Pos;
Result := chunk_size^ + header_size^ <= InSize;
exit;
end
else
@ -750,7 +751,6 @@ begin
StreamInfo^.NewSize := Res;
Result := True;
end;
ShowMessage('');
end;
BRUNSLI_CODEC:
begin

14
precompressor/PrecompOodle.pas
View File

@ -711,17 +711,27 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Move(OodleLZ_CompressOptions_GetDefault(Y, I)^, COptions,
SizeOf(TOodleLZ_CompressOptions));
COptions.sendQuantumCRCs := GetBits(StreamInfo^.Option, 12, 1) = 1;
COptions.spaceSpeedTradeoffBytes := GetBits(StreamInfo^.Option, 13, 11);
Params := 'l' + I.ToString + ':' + 'c' + GetBits(StreamInfo^.Option, 12, 1)
.ToString + ':' + 't' + GetBits(StreamInfo^.Option, 13, 11).ToString;
if not Result then
Res1 := OodleLZ_Compress(Y, NewInput, StreamInfo^.NewSize, Buffer, I,
@COptions);
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(OodleCodecs[GetBits(StreamInfo^.Option, 0, 5)],
@ -729,7 +739,7 @@ begin
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

1
precompressor/PrecompUtils.pas
View File

@ -16,6 +16,7 @@ resourcestring
SPrecompSep2 = ':';
SPrecompSep3 = ',';
SPrecompSep4 = '/';
SPrecompSep5 = '/';
const
SuccessStatus = 4;

31
precompressor/PrecompZSTD.pas
View File

@ -250,9 +250,9 @@ var
X: Integer;
Res1: Integer;
Res2: NativeUInt;
// Inp: ZSTD_inBuffer;
// Oup: ZSTD_outBuffer;
// Progress: NativeInt;
Inp: ZSTD_inBuffer;
Oup: ZSTD_outBuffer;
Progress: NativeInt;
begin
Result := False;
X := GetBits(StreamInfo^.Option, 0, 5);
@ -262,29 +262,43 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Params := '';
case X of
ZSTD_CODEC:
begin
Params := 'l' + I.ToString;
Res1 := ZSTD_compressCCtx(cctx[Instance], Buffer, StreamInfo^.NewSize,
NewInput, StreamInfo^.NewSize, I);
{ ZSTD_CCtx_reset(cctx[Instance], ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_strategy, 5);
ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_compressionLevel, I); }
if not Result then
Res1 := ZSTD_compressCCtx(cctx[Instance], Buffer,
StreamInfo^.NewSize, NewInput, StreamInfo^.NewSize, I);
end;
{ Res1 := ZSTD_compress_usingCDict(cctx[Instance], Buffer,
StreamInfo^.NewSize, NewInput, StreamInfo^.NewSize, cdict); }
{ begin
Params := 'l' + I.ToString;
Progress := 0;
Oup.dst := Buffer;
Oup.Size := StreamInfo^.NewSize;
Oup.Pos := 0;
ZSTD_initCStream(cctx[Instance], I);
ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_strategy, 9);
while Progress < StreamInfo^.NewSize do
begin
Inp.src := PByte(NewInput) + Progress;
Inp.Size := Min(StreamInfo^.NewSize - Progress, 32768);
Inp.Size := Min(StreamInfo^.NewSize - Progress, 64 * 1024);
Inp.Pos := 0;
if ZSTD_compressStream(cctx[Instance], @Oup, @Inp) > 0 then
begin
@ -298,6 +312,7 @@ begin
Res1 := Oup.Pos;
end; }
end;
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(ZSTDCodecs[GetBits(StreamInfo^.Option, 0, 5)],
@ -307,7 +322,7 @@ begin
end;
if Res1 < 0 then
exit;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

4
xtool.dpr
View File

@ -54,6 +54,7 @@ uses
ParseExpr in 'contrib\ParseExpression\ParseExpr.pas',
BrunsliDLL in 'imports\BrunsliDLL.pas',
FLACDLL in 'imports\FLACDLL.pas',
FLZMA2DLL in 'imports\FLZMA2DLL.pas',
JoJpegDLL in 'imports\JoJpegDLL.pas',
LZ4DLL in 'imports\LZ4DLL.pas',
LZODLL in 'imports\LZODLL.pas',
@ -87,8 +88,7 @@ uses
IOPatch in 'io\IOPatch.pas',
IOExecute in 'io\IOExecute.pas',
IODecode in 'io\IODecode.pas',
IOUtils in 'io\IOUtils.pas',
LZMADLL in 'imports\LZMADLL.pas';
IOUtils in 'io\IOUtils.pas';
{$SETPEFLAGS IMAGE_FILE_LARGE_ADDRESS_AWARE or IMAGE_FILE_RELOCS_STRIPPED}

16
xtool.dproj
View File

@ -113,6 +113,7 @@
<DCCReference Include="contrib\ParseExpression\ParseExpr.pas"/>
<DCCReference Include="imports\BrunsliDLL.pas"/>
<DCCReference Include="imports\FLACDLL.pas"/>
<DCCReference Include="imports\FLZMA2DLL.pas"/>
<DCCReference Include="imports\JoJpegDLL.pas"/>
<DCCReference Include="imports\LZ4DLL.pas"/>
<DCCReference Include="imports\LZODLL.pas"/>
@ -147,18 +148,17 @@
<DCCReference Include="io\IOExecute.pas"/>
<DCCReference Include="io\IODecode.pas"/>
<DCCReference Include="io\IOUtils.pas"/>
<DCCReference Include="imports\LZMADLL.pas"/>
<RcItem Include="resources\Win32\xdelta3_dll.dll">
<ContainerId>ResourceItem</ContainerId>
<ResourceType>RCDATA</ResourceType>
<ResourceId>XDELTA86_DLL</ResourceId>
</RcItem>
<RcItem Include="resources\Win64\xdelta3_dll.dll">
<ContainerId>ResourceItem</ContainerId>
<ResourceType>RCDATA</ResourceType>
<ResourceId>XDELTA64_DLL</ResourceId>
<ResourceId>xdelta3_dll</ResourceId>
</RcItem>
<None Include="changes.txt"/>
<RcItem Include="resources\Win64\fast-lzma2.dll">
<ContainerId>ResourceItem</ContainerId>
<ResourceType>RCDATA</ResourceType>
<ResourceId>fast_lzma2</ResourceId>
</RcItem>
<BuildConfiguration Include="Base">
<Key>Base</Key>
</BuildConfiguration>
@ -208,7 +208,7 @@
<Overwrite>true</Overwrite>
</Platform>
</DeployFile>
<DeployFile LocalName="resources\Win32\xdelta3_dll.dll" Configuration="Release" Class="ProjectFile">
<DeployFile LocalName="resources\Win64\fast-lzma2.dll" Configuration="Release" Class="ProjectFile">
<Platform Name="Win64">
<RemoteDir>.\</RemoteDir>
<Overwrite>true</Overwrite>

BIN
xtool.dres
View File

Binary file not shown.

4
xtoolResource.rc
View File

@ -1,2 +1,2 @@
XDELTA86_DLL RCDATA "resources\\Win32\\xdelta3_dll.dll"
XDELTA64_DLL RCDATA "resources\\Win64\\xdelta3_dll.dll"
xdelta3_dll RCDATA "resources\\Win64\\xdelta3_dll.dll"
fast_lzma2 RCDATA "resources\\Win64\\fast-lzma2.dll"