Vichingo455-RePacks/xtool
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

update to 0.6.1

Browse Source
This commit is contained in:
Razor12911
2022-07-12 05:19:10 +02:00
parent 762160b0c9
commit fb6bcfa239
17 changed files with 539 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
precompressor/PrecompLZ4.pas
View File

@@ -266,21 +266,32 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Params := '';
case X of
LZ4_CODEC:
begin
Params := 'a' + GetBits(StreamInfo^.Option, 15, 7).ToString;
Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y,
GetBits(StreamInfo^.Option, 15, 7));
if not Result then
Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y,
GetBits(StreamInfo^.Option, 15, 7));
end;
LZ4HC_CODEC:
begin
Params := 'l' + I.ToString;
Res1 := LZ4_compress_HC(NewInput, Buffer, StreamInfo^.NewSize, Y, I);
if not Result then
Res1 := LZ4_compress_HC(NewInput, Buffer,
StreamInfo^.NewSize, Y, I);
end;
LZ4F_CODEC:
begin
@@ -293,18 +304,20 @@ begin
Params := 'l' + I.ToString + ':' + 'b' +
(GetBits(StreamInfo^.Option, 12, 2) + 4).ToString + ':' + 'd' +
GetBits(StreamInfo^.Option, 14, 1).ToString;
Res1 := LZ4F_compressFrame(Buffer, Y, NewInput,
StreamInfo^.NewSize, @LZ4FT);
if not Result then
Res1 := LZ4F_compressFrame(Buffer, Y, NewInput,
StreamInfo^.NewSize, @LZ4FT);
end;
end;
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(LZ4Codecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

24
precompressor/PrecompLZO.pas
View File

@@ -298,9 +298,17 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Params := '';
Res1 := StreamInfo^.NewSize;
case X of
@@ -310,23 +318,25 @@ begin
begin
Params := 'l' + I.ToString + ':' + 'v' +
GetBits(StreamInfo^.Option, 12, 5).ToString;
if not lzo1x_999_compress_level(NewInput, StreamInfo^.NewSize,
Buffer, @Res1, @WrkMem[Instance, 0], nil, 0, nil, I) = 0 then
Res1 := 0;
if not Result then
if not lzo1x_999_compress_level(NewInput, StreamInfo^.NewSize,
Buffer, @Res1, @WrkMem[Instance, 0], nil, 0, nil, I) = 0 then
Res1 := 0;
end;
{ if not lzo1x_1_compress(NewInput, StreamInfo^.NewSize, Buffer,
@Res1, @WrkMem[Instance, 0]) = 0 then
Res1 := 0; }
end;
end;
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(LZOCodecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

78
precompressor/PrecompMain.pas
View File

@@ -5,7 +5,7 @@ unit PrecompMain;
interface
uses
Threading, Utils, SynCommons, ParseClass, ParseExpr,
Threading, Utils, SynCommons, ParseClass, ParseExpr, FLZMA2DLL,
PrecompUtils, PrecompCrypto, PrecompZLib, PrecompLZ4, PrecompLZO, PrecompZSTD,
PrecompOodle, PrecompMedia, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE,
WinAPI.Windows, WinAPI.ShlObj,
@@ -25,6 +25,8 @@ type
Depth: Integer;
LowMem: Boolean;
DBaseFile, ExtractDir: String;
DoCompress: Boolean;
CompressCfg: String;
end;
PDecodeOptions = ^TDecodeOptions;
@@ -34,6 +36,7 @@ type
ChunkCount, Threads: Integer;
Depth: Integer;
DedupSysMem, DedupGPUMem: Int64;
CompressCfg: String;
end;
procedure PrintHelp;
@@ -122,7 +125,7 @@ begin
WriteLn(ErrOutput, '');
WriteLn(ErrOutput, 'Parameters:');
WriteLn(ErrOutput,
' -m# - codecs to use for precompression (separate by "+" if more than one)');
' -m# - codecs to use for precompression (separate with "+" if more than one)');
WriteLn(ErrOutput, ' -c# - scanning range of precompressor [16mb]');
WriteLn(ErrOutput, ' -t# - number of working threads [50p]');
WriteLn(ErrOutput, ' -lm - low memory mode');
@@ -131,13 +134,16 @@ begin
WriteLn(ErrOutput, 'Advanced parameters:');
WriteLn(ErrOutput,
' --dbase=# - use database (#=filename to save db, optional)');
WriteLn(ErrOutput,
' --dedup=# - use stream deduplication (#=filename to save db, optional)');
WriteLn(ErrOutput, ' --dedup - use stream deduplication');
WriteLn(ErrOutput,
' --mem=# - deduplication ram usage limit (#=size) [75p]');
WriteLn(ErrOutput,
' --diff=# - set xdelta threshold to accept streams [5p]');
WriteLn(ErrOutput, ' --extract=# - extract streams to directory path');
WriteLn(ErrOutput,
' --compress=# - compress data using fast lzma2 (separate params with ":"');
WriteLn(ErrOutput, ' l# - compression level [5]');
WriteLn(ErrOutput, ' t# - number of threads [50p]');
WriteLn(ErrOutput, '');
end;
@@ -191,6 +197,12 @@ begin
Options.ExtractDir := ArgParse.AsString('--extract=');
if Options.ExtractDir <> '' then
EXTRACT := DirectoryExists(Options.ExtractDir);
Options.DoCompress := ArgParse.AsBoolean('--compress');
S := ArgParse.AsString('--compress=');
S := ReplaceText(S, SPrecompSep3, SPrecompSep2);
Options.CompressCfg := S;
if Options.CompressCfg <> '' then
Options.DoCompress := True;
finally
ArgParse.Free;
ExpParse.Free;
@@ -228,6 +240,9 @@ begin
if B then
Options.DedupSysMem := -Options.DedupSysMem;
VERBOSE := ArgParse.AsBoolean('--verbose');
S := ArgParse.AsString('--compress=', 0, 't50p');
S := ReplaceText(S, SPrecompSep3, SPrecompSep2);
Options.CompressCfg := S;
finally
ArgParse.Free;
ExpParse.Free;
@@ -1905,8 +1920,8 @@ begin
for I := Low(ThreadSync) to High(ThreadSync) do
ThreadSync[I] := TCriticalSection.Create;
DupSysMem := Options^.DedupSysMem;
NStream.Add(TypeInfo(TMemoryStream) { , CalcSysMem } );
// NStream.Add(TypeInfo(TPrecompVMStream));
NStream.Add(TypeInfo(TMemoryStream), CalcSysMem);
NStream.Add(TypeInfo(TPrecompVMStream));
Input.ReadBuffer(Options^.Depth, Options^.Depth.Size);
Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size);
SetLength(Bytes, LongRec(I).Bytes[0]);
@@ -1975,19 +1990,8 @@ begin
MemOutput2[I] := TMemoryStream.Create;
end;
end;
Input.ReadBuffer(StoreDD, StoreDD.Size);
UI32 := 0;
if StoreDD then
begin
Input.ReadBuffer(UI32, UI32.Size);
SetLength(DDList2, UI32);
DDCount2 := UI32;
for I := Low(DDList2) to High(DDList2) do
Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
DDIndex1 := -1;
DDIndex2 := 0;
end;
DataMgr := TDataManager.Create(NStream);
Input.ReadBuffer(StoreDD, StoreDD.Size);
end;
procedure DecFree;
@@ -2037,7 +2041,20 @@ var
I, J: Integer;
begin
if Depth = 0 then
begin
UI32 := 0;
if StoreDD then
begin
Input.ReadBuffer(UI32, UI32.Size);
SetLength(DDList2, UI32);
DDCount2 := UI32;
for I := Low(DDList2) to High(DDList2) do
Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2));
DDIndex1 := -1;
DDIndex2 := 0;
end;
LogInt64 := 0;
end;
with ComVars2[Depth] do
begin
DecInput[Index] := Input;
@@ -2252,6 +2269,9 @@ begin
end;
procedure Encode(Input, Output: TStream; Options: TEncodeOptions);
var
Compressed: Boolean;
LOutput: TStream;
begin
InternalSync.Enter;
FillChar(EncInfo, SizeOf(EncInfo), 0);
@@ -2263,8 +2283,16 @@ begin
ConTask.Start;
try
EncInit(Input, Output, @Options);
EncData(Input, Output, 0, 0);
Compressed := Options.DoCompress;
Output.WriteBuffer(Compressed, Compressed.Size);
if Options.DoCompress then
LOutput := TLZMACompressStream.Create(Output, Options.CompressCfg)
else
LOutput := Output;
EncData(Input, LOutput, 0, 0);
finally
if Options.DoCompress then
LOutput.Free;
try
EncFree;
finally
@@ -2279,6 +2307,9 @@ begin
end;
procedure Decode(Input, Output: TStream; Options: TDecodeOptions);
var
Compressed: Boolean;
LInput: TStream;
begin
InternalSync.Enter;
FillChar(EncInfo, SizeOf(EncInfo), 0);
@@ -2291,8 +2322,15 @@ begin
NStream := TArrayStream.Create;
try
DecInit(Input, Output, @Options);
DecChunk(Input, Output, 0, 0);
Input.ReadBuffer(Compressed, Compressed.Size);
if Compressed then
LInput := TLZMADecompressStream.Create(Input, Options.CompressCfg)
else
LInput := Input;
DecChunk(LInput, Output, 0, 0);
finally
if Compressed then
LInput.Free;
try
NStream.Free;
DecFree;

6
precompressor/PrecompMedia.pas
View File

@@ -140,9 +140,10 @@ begin
end
else if subchunk_hdr.subchunk_id = data_SIGN then
begin
Result := True;
chunk_size^ := subchunk_hdr.subchunk_size;
chunk_size^ := Min(subchunk_hdr.subchunk_size,
(riffhdr.chunk_size + 8 - Pos));
header_size^ := Pos;
Result := chunk_size^ + header_size^ <= InSize;
exit;
end
else
@@ -750,7 +751,6 @@ begin
StreamInfo^.NewSize := Res;
Result := True;
end;
ShowMessage('');
end;
BRUNSLI_CODEC:
begin

22
precompressor/PrecompOodle.pas
View File

@@ -711,25 +711,35 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Move(OodleLZ_CompressOptions_GetDefault(Y, I)^, COptions,
SizeOf(TOodleLZ_CompressOptions));
COptions.sendQuantumCRCs := GetBits(StreamInfo^.Option, 12, 1) = 1;
COptions.spaceSpeedTradeoffBytes := GetBits(StreamInfo^.Option, 13, 11);
Params := 'l' + I.ToString + ':' + 'c' + GetBits(StreamInfo^.Option, 12, 1)
.ToString + ':' + 't' + GetBits(StreamInfo^.Option, 13, 11).ToString;
Res1 := OodleLZ_Compress(Y, NewInput, StreamInfo^.NewSize, Buffer, I,
@COptions);
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
if not Result then
Res1 := OodleLZ_Compress(Y, NewInput, StreamInfo^.NewSize, Buffer, I,
@COptions);
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(OodleCodecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
break;
end;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));

1
precompressor/PrecompUtils.pas
View File

@@ -16,6 +16,7 @@ resourcestring
SPrecompSep2 = ':';
SPrecompSep3 = ',';
SPrecompSep4 = '/';
SPrecompSep5 = '/';
const
SuccessStatus = 4;

35
precompressor/PrecompZSTD.pas
View File

@@ -250,9 +250,9 @@ var
X: Integer;
Res1: Integer;
Res2: NativeUInt;
// Inp: ZSTD_inBuffer;
// Oup: ZSTD_outBuffer;
// Progress: NativeInt;
Inp: ZSTD_inBuffer;
Oup: ZSTD_outBuffer;
Progress: NativeInt;
begin
Result := False;
X := GetBits(StreamInfo^.Option, 0, 5);
@@ -262,29 +262,43 @@ begin
SOList[Instance][X].Index := 0;
while SOList[Instance][X].Get(I) >= 0 do
begin
if StreamInfo^.Status = TStreamStatus.Predicted then
if StreamInfo^.Status >= TStreamStatus.Predicted then
begin
if GetBits(StreamInfo^.Option, 5, 7) <> I then
continue;
if (StreamInfo^.Status = TStreamStatus.Database) and
(GetBits(StreamInfo^.Option, 1, 31) = 0) then
begin
Res1 := StreamInfo^.OldSize;
Result := True;
end;
end;
Params := '';
case X of
ZSTD_CODEC:
begin
Params := 'l' + I.ToString;
Res1 := ZSTD_compressCCtx(cctx[Instance], Buffer, StreamInfo^.NewSize,
NewInput, StreamInfo^.NewSize, I);
{ ZSTD_CCtx_reset(cctx[Instance], ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_strategy, 5);
ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_compressionLevel, I); }
if not Result then
Res1 := ZSTD_compressCCtx(cctx[Instance], Buffer,
StreamInfo^.NewSize, NewInput, StreamInfo^.NewSize, I);
end;
{ Res1 := ZSTD_compress_usingCDict(cctx[Instance], Buffer,
StreamInfo^.NewSize, NewInput, StreamInfo^.NewSize, cdict); }
{ begin
Params := 'l' + I.ToString;
Progress := 0;
Oup.dst := Buffer;
Oup.Size := StreamInfo^.NewSize;
Oup.Pos := 0;
ZSTD_initCStream(cctx[Instance], I);
ZSTD_CCtx_setParameter(cctx[Instance], ZSTD_c_strategy, 9);
while Progress < StreamInfo^.NewSize do
begin
Inp.src := PByte(NewInput) + Progress;
Inp.Size := Min(StreamInfo^.NewSize - Progress, 32768);
Inp.Size := Min(StreamInfo^.NewSize - Progress, 64 * 1024);
Inp.Pos := 0;
if ZSTD_compressStream(cctx[Instance], @Oup, @Inp) > 0 then
begin
@@ -298,8 +312,9 @@ begin
Res1 := Oup.Pos;
end; }
end;
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
if not Result then
Result := (Res1 = StreamInfo^.OldSize) and CompareMem(OldInput, Buffer,
StreamInfo^.OldSize);
Funcs^.LogProcess(ZSTDCodecs[GetBits(StreamInfo^.Option, 0, 5)],
PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result);
if Result or (StreamInfo^.Status = TStreamStatus.Predicted) then
@@ -307,7 +322,7 @@ begin
end;
if Res1 < 0 then
exit;
if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or
if (Result = False) and ((StreamInfo^.Status >= TStreamStatus.Predicted) or
(SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then
begin
Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1));