diff --git a/changes.txt b/changes.txt new file mode 100644 index 0000000..c04878d --- /dev/null +++ b/changes.txt @@ -0,0 +1,251 @@ + ES_R35 (0.6.0) + - added wav stream detector + - added flac codec + - added jpg stream detector + - added packjpg, brunsli, jojpeg codec + - added fast lzma2 compression for portable mode + - added feature that allows input to be a directory + - added feature to inject libraries to main executable + - added feature to extract detected streams + - updated database feature + - updated deduplication feature + - IO function decode updated + + ES_R34 (0.5.3) + - added png stream preprocessor + - removed grittibanzli codec (since nobody uses it) + + ES_R33 (0.5.2) + - added IO functions (archive, execute) + - fixed issue in patch io function + - removed compression on patch diff files + + ES_R32 (0.5.1) + - added IO functions (find, extract, patch) + - generate database feature and IO functions now can search for streams larger than chunk size + + ES_R31 (0.5.0) + - added IO functions (erase, replace) + - fixed external executable support bugs + + ES_R30 (0.4.8) + - fixed issue with storing incorrect recompression information when stream patching is performed + + ES_R29 (0.4.7) + - updated oodle scanner + - updated external executable support + - updated configuration based plugin support to add depth information + - updated verbose mode + + ES_R28 (0.4.6) + - generate database feature fixed + - fixed external executable support issues + - fixed lz4f level setting bug + + ES_R28 (0.4.5) + - removed leviathan codec restriction + + ES_R27 (0.4.4) + - fixed issue of lz4 codec loading incorrect library + - fixed issue with handling endianess via configuration based plugins + - updated framework of library based plugins + + ES_R26 (0.4.3) + - added verbose mode + - added feature that allows you to enforce a different library to be loaded + - fixed issues related to imperfect stream patching + - fixed issues with old libraries with missing functions that cause xtool to crash on startup + - updated oodle codec + - updated reflate codec + - updated zstd codec + + ES_R25 (0.4.2) + - removed debugging code from encryption and executable codec + - fixed issue with depth when using search codec + - fixed external executable support issues + + ES_R24 (0.4.1) + - fixed issue of status not reporting when encoding + - added depth method support for search support + - fixed zlib encoding issues for different window bits + - fixed zlib memory leak issue + - updated all internal codecs to support information relayed by external codecs + - updated lz4f codec and removed temporarily removed support for universal scanning + - added option to change recompression level to be used by reflate + - updated external executable support + - generate database feature currently bugged, wait for next update + - search database structure changed, older database files will no longer work with newer releases + + ES_R23 (0.4.0) + - project made open source + - added external executable support + - added generate database feature + - fixed search support bug + + ES_R22 (0.3.22) + - updated search support (speed improvements) + - updated command line parser + - added partial universal scanner for lzo1x streams + - added universal scanner for lz4f streams + - fixed issue with configuration files failing to execute without conditions + + ES_R21 (0.3.21) + - updated search support + + ES_R20 (0.3.20) + - fixed library support bug + - x86 build discontinued (has bugs from nowhere) + + ES_R19 (0.3.19) + - updated lzo codec + + ES_R18 (0.3.18) + - fixed depth bug + - fixed library plugin bugs + + ES_R17 (0.3.17) + - fixed multi-threading bug + + ES_R16 (0.3.16) + - minor bug fixes + + ES_R15 (0.3.15) + - converted library support to unicode (don't know why I used ansi in the first place) + - added library support functions + - added rc4 encryption support + + ES_R14 (0.3.14) + - fixed library support bug + - updated library structure + + ES_R13 (0.3.13) + - updated lz4 codec + - updated library structure + - updated depth info functions + - updated depth feature + + ES_R12 (0.3.12) + - added depth info functions + - added support for oodle 2.9.0+ functions + - fixed data patching bug + - updated oodle codec + - updated command line parser + + ES_R11 (0.3.11) + - fixed x86 build bugs + - fixed config multi-threading bug + - fixed resource management bug + - fixed deduplication bug + + ES_R10 (0.3.10) + - minor bug fixes + - added diff tolerance parameter (--diff=) + - fixed plugin database bug + - fixed lz4 codec bug + - updated oodle codec + - updated library structure + - added resource management + - added direct use encryption codecs + - added embedded deduplication feature (--dedup) [makes temps during encoding] + + ES_R9 (0.3.9) + - fixed future stream bug + + ES_R8 (0.3.8) + - fixed command line parser bug + - updated library support + + ES_R7 (0.3.7) + - updated library structure + + ES_R6 (0.3.6) + - updated oodle codec (fixed more lzna bugs) + + ES_R5 (0.3.5) + - updated oodle codec (fixed lzna bug) + - added custom method configuration + + ES_R4 (0.3.4) + - fixed bug depthing + + ES_R3 (0.3.3) + - updated lz4 codec + - updated library support + + ES_R2 (0.3.2) + - improved depthing + - updated library support + - fixed zstd codec issues + - removed fast memory + + ES_R1 (0.3.1) + - updated library support + - updated command line parser + - included x86 build + - fixed depthing issues + + 2012_R2 (0.2.14) + - added library support + - added compress, decompress, encrypt, decrypt, hash, delta functions (used by library) + - added lzo codec placeholders + - fixed oodle bug + - fixed lz4 bug + - removed libdunia codec + + 2012_R1 (0.2.13) + - added oo2ext* dll support + - updated search support + + 2011_R1 (0.2.12) + - added temporary libdunia codec + + 2010_R5 (0.2.11) + - fixed search/config support bug + + 2010_R4 (0.2.10) + - updated search/config support + + 2010_R3 (0.2.9) + - added database search + - updated zlib scanner + - fixed reflate bug + - fixed 2GB memory limit + + 2010_R2 (0.2.8) + - fixed zstd codec + + 2010_R1 (0.2.7) + - added zstd codec + - added lz4, lz4hc, lzna, mermaid, selkie, hydra, leviathan codec placeholders + - added configuration support + - added xdelta support to handle crc mismatch streams + + 2009_R3 (0.2.6) + - documentation added + + 2009_R2 (0.2.5) + - added kraken codec + - fixed depthing issues + + 2009_R1 (0.2.4) + - added reflate forced verification + - updated deflate scanner + - fixed depthing issues + - fixed low memory mode issues + - fixed hanging issues when encoding + + 2008_R3 (0.2.3) + - fixed deduplication memory calculation error + - added virtual memory support for deduplication + - added --mem=# parameter to control deduplication memory usage + + 2008_R2 (0.2.2) + - fixed command line parser + - updated deflate scanner + - added stream deduplication + - added stream database + - added decompression memory limiter + - added grittibanzli (also handles deflate stream but slow af) + + 2008_R1 (0.2.1) + - initial release \ No newline at end of file diff --git a/common/Utils.pas b/common/Utils.pas index f48dabd..94c7042 100644 --- a/common/Utils.pas +++ b/common/Utils.pas @@ -44,6 +44,16 @@ procedure SetBits(var Data: UInt64; Value: Int64; Index: TInt64_BitIndex; Count: TInt64_BitCount); overload; type + PDynArrayRec = ^TDynArrayRec; + + TDynArrayRec = packed record +{$IFDEF CPUX64} + _Padding: LongInt; +{$ENDIF} + RefCnt: LongInt; + Length: NativeInt; + end; + TListEx = class(TList) private FIndex: Integer; @@ -85,6 +95,13 @@ type property Method: TSOMethod read FSOMethod write FSOMethod; end; + TNullStream = class(TStream) + public + constructor Create; + destructor Destroy; override; + function Write(const Buffer; Count: LongInt): LongInt; override; + end; + TArrayStream = class(TStream) private type _Stream = ^IStream; @@ -97,7 +114,7 @@ type FMaxStreamSize = $FFFFFFFFFF; protected function GetSize: Int64; override; - procedure SetSize(NewSize: Longint); override; + procedure SetSize(NewSize: LongInt); override; procedure SetSize(const NewSize: Int64); override; private FStreams: TArray; @@ -110,8 +127,8 @@ type public constructor Create; destructor Destroy; override; - function Read(var Buffer; Count: Longint): Longint; override; - function Write(const Buffer; Count: Longint): Longint; override; + function Read(var Buffer; Count: LongInt): LongInt; override; + function Write(const Buffer; Count: LongInt): LongInt; override; function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override; procedure Clear; function Add(AStreamType: Pointer; MaxSize: Int64 = FMaxStreamSize) @@ -122,7 +139,7 @@ type TPointersStream = class(TStream) protected function GetSize: Int64; override; - procedure SetSize(NewSize: Longint); override; + procedure SetSize(NewSize: LongInt); override; procedure SetSize(const NewSize: Int64); override; private FPointers: TArray; @@ -135,8 +152,8 @@ type public constructor Create; destructor Destroy; override; - function Read(var Buffer; Count: Longint): Longint; override; - function Write(const Buffer; Count: Longint): Longint; override; + function Read(var Buffer; Count: LongInt): LongInt; override; + function Write(const Buffer; Count: LongInt): LongInt; override; function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override; function Add(Ptr: Pointer; Size: NativeInt): Integer; procedure Delete(Index: Integer); @@ -157,13 +174,48 @@ type AMaxSize: NativeInt = 0); overload; destructor Destroy; override; procedure SetSize(const NewSize: Int64); override; - procedure SetSize(NewSize: Longint); override; - function Read(var Buffer; Count: Longint): Longint; override; - function Write(const Buffer; Count: Longint): Longint; override; + procedure SetSize(NewSize: LongInt); override; + function Read(var Buffer; Count: LongInt): LongInt; override; + function Write(const Buffer; Count: LongInt): LongInt; override; function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override; procedure Update(const AMemory: Pointer = nil; AMaxSize: NativeInt = 0); end; + TDirInputStream = class(TStream) + protected type + TState = (iNone, iLength, iFilename, iSize, iData); + private + FState: TState; + FPath: String; + FBaseDir: String; + FList: TArray; + FIndex, FCount: Integer; + FLength: Word; + FBytes: TBytes; + FStream: TFileStream; + FPosition, FSize: Int64; + public + constructor Create(const APath: String); + destructor Destroy; override; + function Read(var Buffer; Count: LongInt): LongInt; override; + end; + + TDirOutputStream = class(TStream) + protected type + TState = (oNone, oLength, oFilename, oSize, oData); + private + FState: TState; + FPath: String; + FLength: Word; + FBytes: TBytes; + FStream: TFileStream; + FPosition, FSize: Int64; + public + constructor Create(const APath: String); + destructor Destroy; override; + function Write(const Buffer; Count: LongInt): LongInt; override; + end; + TSharedMemoryStream = class(TMemoryStreamEx) private const FIncSize = 64 * 1024 * 1024; @@ -179,7 +231,7 @@ type constructor Create(const AMapName: String; ASize: NativeInt); overload; destructor Destroy; override; procedure SetSize(const NewSize: Int64); override; - function Write(const Buffer; Count: Longint): Longint; override; + function Write(const Buffer; Count: LongInt): LongInt; override; end; TDownloadStream = class(TStream) @@ -195,7 +247,7 @@ type public constructor Create(Url: string); destructor Destroy; override; - function Read(var Buffer; Count: Longint): Longint; override; + function Read(var Buffer; Count: LongInt): LongInt; override; function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override; end; @@ -297,23 +349,23 @@ type PBlockInfo = ^TBlockInfo; TBlockInfo = record + ID: Integer; Position, CurrSize, FullSize: Int64; Count: Integer; end; - - TBlockInfoDynArray = TArray; private - FSync: TSynLocker; - FDictionary: TSynDictionary; + FSearchList: TArray; FStream: TStream; - FStreamPos: Int64; + FStreamPos, FStreamSize: Int64; public - constructor Create(AStream: TStream; ACapacity: Integer = 0); + constructor Create(AStream: TStream); destructor Destroy; override; procedure Add(ID: Integer; Size: Int64; Count: Integer = Integer.MaxValue); - procedure Write(ID: Integer; const Buffer; Size: Integer); + procedure Write(ID: Integer; Buffer: Pointer; Size: Integer); procedure CopyData(ID: Integer; Stream: TStream); overload; - function CopyData(ID: Integer): Pointer; overload; + function CopyData(ID: Integer; Data: Pointer): Integer; overload; + procedure Update(ID: Integer; Count: Integer); + procedure Reset(ID: Integer); end; TArgParser = class(TObject) @@ -679,6 +731,21 @@ begin Result := FList.Count; end; +constructor TNullStream.Create; +begin + inherited Create; +end; + +destructor TNullStream.Destroy; +begin + inherited Destroy; +end; + +function TNullStream.Write(const Buffer; Count: LongInt): LongInt; +begin + Result := Count; +end; + constructor TArrayStream.Create; begin inherited Create; @@ -781,7 +848,7 @@ begin Result := FSize; end; -procedure TArrayStream.SetSize(NewSize: Longint); +procedure TArrayStream.SetSize(NewSize: LongInt); begin SetSize(Int64(NewSize)); end; @@ -794,7 +861,7 @@ begin Seek(0, soEnd); end; -function TArrayStream.Read(var Buffer; Count: Longint): Longint; +function TArrayStream.Read(var Buffer; Count: LongInt): LongInt; var LCount: Int64; begin @@ -808,7 +875,7 @@ begin Inc(FPosition, Result); end; -function TArrayStream.Write(const Buffer; Count: Longint): Longint; +function TArrayStream.Write(const Buffer; Count: LongInt): LongInt; var LCount: Int64; begin @@ -888,7 +955,7 @@ begin Result := FSize; end; -procedure TPointersStream.SetSize(NewSize: Longint); +procedure TPointersStream.SetSize(NewSize: LongInt); begin SetSize(Int64(NewSize)); end; @@ -917,12 +984,12 @@ begin FSize := FMaxSize; end; -function TPointersStream.Read(var Buffer; Count: Longint): Longint; +function TPointersStream.Read(var Buffer; Count: LongInt): LongInt; begin // 2121212 end; -function TPointersStream.Write(const Buffer; Count: Longint): Longint; +function TPointersStream.Write(const Buffer; Count: LongInt): LongInt; begin end; @@ -986,7 +1053,7 @@ begin inherited Destroy; end; -procedure TMemoryStreamEx.SetSize(NewSize: Longint); +procedure TMemoryStreamEx.SetSize(NewSize: LongInt); begin SetSize(Int64(NewSize)); end; @@ -1002,7 +1069,7 @@ begin Seek(0, soEnd); end; -function TMemoryStreamEx.Read(var Buffer; Count: Longint): Longint; +function TMemoryStreamEx.Read(var Buffer; Count: LongInt): LongInt; begin Result := 0; if (FPosition >= 0) and (Count >= 0) then @@ -1019,9 +1086,9 @@ begin end; end; -function TMemoryStreamEx.Write(const Buffer; Count: Longint): Longint; +function TMemoryStreamEx.Write(const Buffer; Count: LongInt): LongInt; var - FCount: Longint; + FCount: LongInt; begin Result := 0; FCount := Count; @@ -1077,6 +1144,210 @@ begin SetSize(LSize); end; +constructor TDirInputStream.Create(const APath: String); +begin + inherited Create; + FState := TState.iNone; + FPath := TPath.GetFullPath(APath); + if FileExists(FPath) then + FBaseDir := ExtractFilePath(TPath.GetFullPath(FPath)) + else if DirectoryExists(FPath) then + FBaseDir := IncludeTrailingBackSlash(TPath.GetFullPath(FPath)) + else + FBaseDir := ExtractFilePath(TPath.GetFullPath(FPath)); + FList := GetFileList([FPath], True); + FCount := Length(FList); + if FCount = 0 then + raise EFOpenError.CreateRes(@SEmptyPath); + FIndex := -1; + FStream := nil; +end; + +destructor TDirInputStream.Destroy; +begin + if Assigned(FStream) then + FStream.Free; + FStream := nil; + inherited Destroy; +end; + +function TDirInputStream.Read(var Buffer; Count: LongInt): LongInt; +var + LCount: Integer; +begin + Result := 0; + if Count <= 0 then + exit; + if FState = TState.iNone then + begin + if Succ(FIndex) >= FCount then + exit; + Inc(FIndex); + FBytes := BytesOf(ReplaceText(FList[FIndex], FBaseDir, '')); + FLength := Length(FBytes); + FPosition := 0; + FSize := FileSize(FList[FIndex]); + FState := TState.iLength; + end; + if FState = TState.iLength then + if FPosition < FLength.Size then + begin + LCount := Min(FLength.Size - FPosition, Count); + Move(WordRec(FLength).Bytes[FPosition], Buffer, LCount); + Inc(FPosition, LCount); + if FPosition = FLength.Size then + begin + FState := TState.iFilename; + FPosition := 0; + end; + exit(LCount); + end; + if FState = TState.iFilename then + if FPosition < FLength then + begin + LCount := Min(FLength - FPosition, Count); + Move(FBytes[FPosition], Buffer, LCount); + Inc(FPosition, LCount); + if FPosition = FLength then + begin + FState := TState.iSize; + FPosition := 0; + end; + exit(LCount); + end; + if FState = TState.iSize then + if FPosition < FSize.Size then + begin + LCount := Min(FSize.Size - FPosition, Count); + Move(Int64Rec(FSize).Bytes[FPosition], Buffer, LCount); + Inc(FPosition, LCount); + if FPosition = FSize.Size then + begin + if FSize = 0 then + FState := TState.iNone + else + begin + FState := TState.iData; + FPosition := 0; + FStream := TFileStream.Create(FList[FIndex], fmShareDenyNone); + end; + end; + exit(LCount); + end; + if FState = TState.iData then + if FPosition < FSize then + begin + LCount := Min(FSize - FPosition, Count); + LCount := FStream.Read(Buffer, LCount); + Inc(FPosition, LCount); + if FPosition = FSize then + begin + FState := TState.iNone; + FStream.Free; + FStream := nil; + end; + exit(LCount); + end; +end; + +constructor TDirOutputStream.Create(const APath: String); +begin + inherited Create; + FState := TState.oNone; + FPath := IncludeTrailingBackSlash(TPath.GetFullPath(APath)); + FStream := nil; +end; + +destructor TDirOutputStream.Destroy; +begin + if Assigned(FStream) then + FStream.Free; + FStream := nil; + inherited Destroy; +end; + +function TDirOutputStream.Write(const Buffer; Count: LongInt): LongInt; +var + LCount: Integer; + LStr: String; +begin + Result := 0; + if Count <= 0 then + exit; + if FState = TState.oNone then + begin + FPosition := 0; + FState := TState.oLength; + end; + if FState = TState.oLength then + if FPosition < FLength.Size then + begin + LCount := Min(FLength.Size - FPosition, Count); + Move(Buffer, WordRec(FLength).Bytes[FPosition], LCount); + Inc(FPosition, LCount); + if FPosition = FLength.Size then + begin + SetLength(FBytes, FLength); + FState := TState.oFilename; + FPosition := 0; + end; + exit(LCount); + end; + if FState = TState.oFilename then + if FPosition < FLength then + begin + LCount := Min(FLength - FPosition, Count); + Move(Buffer, FBytes[FPosition], LCount); + Inc(FPosition, LCount); + if FPosition = FLength then + begin + FState := TState.oSize; + FPosition := 0; + end; + exit(LCount); + end; + if FState = TState.oSize then + if FPosition < FSize.Size then + begin + LCount := Min(FSize.Size - FPosition, Count); + Move(Buffer, Int64Rec(FSize).Bytes[FPosition], LCount); + Inc(FPosition, LCount); + if FPosition = FSize.Size then + begin + LStr := FPath + StringOf(FBytes); + if not DirectoryExists(ExtractFilePath(LStr)) then + ForceDirectories(ExtractFilePath(LStr)); + FStream := TFileStream.Create(LStr, fmCreate); + if FSize = 0 then + begin + FState := TState.oNone; + FStream.Free; + FStream := nil; + end + else + begin + FState := TState.oData; + FPosition := 0; + end; + end; + exit(LCount); + end; + if FState = TState.oData then + if FPosition < FSize then + begin + LCount := Min(FSize - FPosition, Count); + LCount := FStream.Write(Buffer, LCount); + Inc(FPosition, LCount); + if FPosition = FSize then + begin + FState := TState.oNone; + FStream.Free; + FStream := nil; + end; + exit(LCount); + end; +end; + constructor TSharedMemoryStream.Create(const AMapName: String; AFileName: string); @@ -1191,7 +1462,7 @@ begin inherited SetSize(NewSize); end; -function TSharedMemoryStream.Write(const Buffer; Count: Longint): Longint; +function TSharedMemoryStream.Write(const Buffer; Count: LongInt): LongInt; begin if FPosition + Count > FMaxSize then IncMemory(FPosition + Count); @@ -1227,7 +1498,7 @@ begin Abort := True; end; -function TDownloadStream.Read(var Buffer; Count: Longint): Longint; +function TDownloadStream.Read(var Buffer; Count: LongInt): LongInt; begin if (FPosition >= 0) and (Count >= 0) then begin @@ -1779,107 +2050,132 @@ begin FSizes[Index] := 0; end; -constructor TDataManager.Create(AStream: TStream; ACapacity: Integer); +constructor TDataManager.Create(AStream: TStream); begin inherited Create; - FSync.Init; - FDictionary := TSynDictionary.Create(TypeInfo(TIntegerDynArray), - TypeInfo(TBlockInfoDynArray)); - FDictionary.Capacity := ACapacity; FStream := AStream; FStreamPos := FStream.Position; + FStreamSize := 0; end; destructor TDataManager.Destroy; begin - FDictionary.Free; - FSync.Done; inherited Destroy; end; procedure TDataManager.Add(ID: Integer; Size: Int64; Count: Integer); var - LBlockInfo: TBlockInfo; - _BlockInfo: PBlockInfo; I: Integer; + LBlockInfo: TBlockInfo; begin if Count <= 0 then exit; - FSync.Lock; - try - LBlockInfo.Position := 0; - for I := 0 to FDictionary.Count - 1 do + for I := Low(FSearchList) to High(FSearchList) do + begin + if (FSearchList[I].Count <= 0) and (Size <= FSearchList[I].FullSize) then begin - _BlockInfo := PBlockInfo(FDictionary.Values.ElemPtr(I)); - LBlockInfo.Position := Max(LBlockInfo.Position, _BlockInfo^.Position + - _BlockInfo^.FullSize); + FSearchList[I].ID := ID; + FSearchList[I].CurrSize := 0; + FSearchList[I].Count := Count; + exit; end; - LBlockInfo.CurrSize := 0; - LBlockInfo.FullSize := Size; - LBlockInfo.Count := Count; - FStream.Size := Max(FStream.Size, FStreamPos + LBlockInfo.Position + - LBlockInfo.FullSize); - FDictionary.Add(ID, LBlockInfo); - finally - FSync.UnLock; end; - // once reads reaches 0, add list of all available spaces + LBlockInfo.ID := ID; + LBlockInfo.Position := FStreamPos + FStreamSize; + LBlockInfo.CurrSize := 0; + LBlockInfo.FullSize := Size; + LBlockInfo.Count := Count; + Insert(LBlockInfo, FSearchList, Length(FSearchList)); + Inc(FStreamSize, Size); end; -procedure TDataManager.Write(ID: Integer; const Buffer; Size: Integer); +procedure TDataManager.Write(ID: Integer; Buffer: Pointer; Size: Integer); var - _BlockInfo: PBlockInfo; + I: Integer; begin if Size <= 0 then exit; - FSync.Lock; - try - _BlockInfo := FDictionary.FindValue(ID); - if (_BlockInfo^.Position + _BlockInfo^.CurrSize + Size) > - (_BlockInfo^.Position + _BlockInfo^.FullSize) then - raise EWriteError.CreateRes(@SWriteError); - FStream.Position := FStreamPos + _BlockInfo^.Position + - _BlockInfo^.CurrSize; - FStream.WriteBuffer(Buffer, Size); - Inc(_BlockInfo^.CurrSize, Size); - finally - FSync.UnLock; + for I := Low(FSearchList) to High(FSearchList) do + begin + if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then + begin + if FSearchList[I].CurrSize + Size > FSearchList[I].FullSize then + raise EWriteError.CreateRes(@SWriteError); + FStream.Position := FSearchList[I].Position + FSearchList[I].CurrSize; + FStream.WriteBuffer(Buffer^, Size); + Inc(FSearchList[I].CurrSize, Size); + exit; + end; end; + raise Exception.CreateRes(@SGenericItemNotFound); end; procedure TDataManager.CopyData(ID: Integer; Stream: TStream); var - _BlockInfo: PBlockInfo; + I: Integer; begin - FSync.Lock; - try - _BlockInfo := FDictionary.FindValue(ID); - if _BlockInfo^.CurrSize <> _BlockInfo^.FullSize then - raise EReadError.CreateRes(@SReadError); - FStream.Position := FStreamPos + _BlockInfo^.Position; - CopyStreamEx(FStream, Stream, _BlockInfo^.FullSize); - Dec(_BlockInfo^.Count); - finally - FSync.UnLock; + for I := Low(FSearchList) to High(FSearchList) do + begin + if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then + begin + FStream.Position := FSearchList[I].Position; + CopyStreamEx(FStream, Stream, FSearchList[I].CurrSize); + Dec(FSearchList[I].Count); + exit; + end; end; + raise Exception.CreateRes(@SGenericItemNotFound); end; -function TDataManager.CopyData(ID: Integer): Pointer; +function TDataManager.CopyData(ID: Integer; Data: Pointer): Integer; var - _BlockInfo: PBlockInfo; + I: Integer; begin - FSync.Lock; - try - _BlockInfo := FDictionary.FindValue(ID); - if _BlockInfo^.CurrSize <> _BlockInfo^.FullSize then - raise EReadError.CreateRes(@SReadError); - FStream.Position := FStreamPos + _BlockInfo^.Position + - _BlockInfo^.CurrSize; - FStream.ReadBuffer(Result^, _BlockInfo^.FullSize); - Dec(_BlockInfo^.Count); - finally - FSync.UnLock; + Result := 0; + for I := Low(FSearchList) to High(FSearchList) do + begin + if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then + begin + FStream.Position := FSearchList[I].Position; + FStream.ReadBuffer(Data^, FSearchList[I].CurrSize); + Result := FSearchList[I].CurrSize; + Dec(FSearchList[I].Count); + if FSearchList[I].Count = 0 then + FSearchList[I].ID := -1; + exit; + end; end; + raise Exception.CreateRes(@SGenericItemNotFound); +end; + +procedure TDataManager.Update(ID: Integer; Count: Integer); +var + I: Integer; +begin + for I := Low(FSearchList) to High(FSearchList) do + begin + if (ID = FSearchList[I].ID) then + begin + FSearchList[I].Count := Count; + exit; + end; + end; + raise Exception.CreateRes(@SGenericItemNotFound); +end; + +procedure TDataManager.Reset(ID: Integer); +var + I: Integer; +begin + for I := Low(FSearchList) to High(FSearchList) do + begin + if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then + begin + FSearchList[I].CurrSize := 0; + exit; + end; + end; + raise Exception.CreateRes(@SGenericItemNotFound); end; constructor TArgParser.Create(Arguments: TStringDynArray); diff --git a/imports/BrunsliDLL.pas b/imports/BrunsliDLL.pas new file mode 100644 index 0000000..13bb944 --- /dev/null +++ b/imports/BrunsliDLL.pas @@ -0,0 +1,87 @@ +unit BrunsliDLL; + +interface + +uses + WinAPI.Windows, + System.SysUtils, System.Classes; + +const + BRUNSLI_OK = 0; + BRUNSLI_NON_REPRESENTABLE = 1; + BRUNSLI_MEMORY_ERROR = 2; + BRUNSLI_INVALID_PARAM = 3; + BRUNSLI_COMPRESSION_ERROR = 4; + BRUNSLI_INVALID_BRN = 5; + BRUNSLI_DECOMPRESSION_ERROR = 6; + BRUNSLI_NOT_ENOUGH_DATA = 7; + +type + TBrunsliWriter = function(ctx: Pointer; data: Pointer; Size: NativeUInt) + : Integer cdecl; + +var + brunsli_alloc_JPEGData: function: Pointer cdecl; + brunsli_free_JPEGData: procedure(P: Pointer)cdecl; + brunsli_GetMaximumEncodedSize: function(P: Pointer): Integer cdecl; + brunsli_ReadJpeg: function(P: Pointer; data: Pointer; len: Integer) + : Integer cdecl; + brunsli_EncodeJpeg: function(P: Pointer; data: Pointer; len: Integer) + : Integer cdecl; + brunsli_DecodeJpeg: function(P: Pointer; data: Pointer; len: Integer) + : Integer cdecl; + brunsli_alloc_JPEGOutput: function(P: TBrunsliWriter; data: Pointer) + : Pointer cdecl; + brunsli_free_JPEGOutput: procedure(P: Pointer)cdecl; + brunsli_WriteJpeg: function(P: Pointer; oup: Pointer): Integer cdecl; + DLLLoaded: Boolean = False; + +implementation + +var + DLLHandle: THandle; + S: String; + +procedure Init; +begin + S := 'brunsli.dll'; + DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + S)); + if DLLHandle >= 32 then + begin + @brunsli_alloc_JPEGData := GetProcAddress(DLLHandle, + 'brunsli_alloc_JPEGData'); + @brunsli_free_JPEGData := GetProcAddress(DLLHandle, + 'brunsli_free_JPEGData'); + @brunsli_GetMaximumEncodedSize := GetProcAddress(DLLHandle, + 'brunsli_GetMaximumEncodedSize'); + @brunsli_ReadJpeg := GetProcAddress(DLLHandle, 'brunsli_ReadJpeg'); + @brunsli_EncodeJpeg := GetProcAddress(DLLHandle, 'brunsli_EncodeJpeg'); + @brunsli_DecodeJpeg := GetProcAddress(DLLHandle, 'brunsli_DecodeJpeg'); + @brunsli_alloc_JPEGOutput := GetProcAddress(DLLHandle, + 'brunsli_alloc_JPEGOutput'); + @brunsli_free_JPEGOutput := GetProcAddress(DLLHandle, + 'brunsli_free_JPEGOutput'); + @brunsli_WriteJpeg := GetProcAddress(DLLHandle, 'brunsli_WriteJpeg'); + DLLLoaded := Assigned(brunsli_alloc_JPEGData) and + Assigned(brunsli_alloc_JPEGOutput); + end + else + DLLLoaded := False; +end; + +procedure Deinit; +begin + if not DLLLoaded then + exit; + FreeLibrary(DLLHandle); +end; + +initialization + +Init; + +finalization + +Deinit; + +end. diff --git a/imports/FLACDLL.pas b/imports/FLACDLL.pas new file mode 100644 index 0000000..4c14445 --- /dev/null +++ b/imports/FLACDLL.pas @@ -0,0 +1,196 @@ +unit FLACDLL; + +interface + +uses + WinAPI.Windows, + System.SysUtils, System.Classes; + +const + FLAC__MAX_CHANNELS = 8; + FLAC__MAX_FIXED_ORDER = 4; + FLAC__MAX_LPC_ORDER = 32; + +type + TFLAC__FrameHeader = record + blocksize, sample_rate, channels: Cardinal; + channel_assignment: Integer; + bits_per_sample: Cardinal; + case number_type: Integer of + 0: + (frame_number: Cardinal; crc1: Byte); + 1: + (sample_number: UInt64; crc2: Byte); + end; + + PFLAC__EntropyCodingMethod_PartitionedRiceContents = ^ + TFLAC__EntropyCodingMethod_PartitionedRiceContents; + + TFLAC__EntropyCodingMethod_PartitionedRiceContents = record + parameters, raw_bits: PCardinal; + capacity_by_order: Cardinal; + end; + + TFLAC__EntropyCodingMethod_PartitionedRice = record + order: Cardinal; + contents: PFLAC__EntropyCodingMethod_PartitionedRiceContents; + end; + + TFLAC__EntropyCodingMethod = record + case ftype: Integer of + 0: + (partitioned_rice: TFLAC__EntropyCodingMethod_PartitionedRice); + end; + + TFLAC__Subframe_Fixed = record + entropy_coding_method: TFLAC__EntropyCodingMethod; + order: Cardinal; + warmup: array [0 .. FLAC__MAX_CHANNELS - 1] of Integer; + residual: PInteger; + end; + + TFLAC__Subframe_LPC = record + entropy_coding_method: TFLAC__EntropyCodingMethod; + order, qlp_coeff_precision: Cardinal; + quantization_level: Integer; + qlp_coeff, warmup: array [0 .. FLAC__MAX_LPC_ORDER - 1] of Integer; + residual: PInteger; + end; + + TFLAC__Subframe = record + case ftype: Integer of + 0: + (constant: Integer; wb1: Cardinal); + 1: + (fixed: TFLAC__Subframe_Fixed; wb2: Cardinal); + 2: + (lpc: TFLAC__Subframe_LPC; wb3: Cardinal); + 3: + (verbatim: PInteger; wb4: Cardinal); + end; + + PFLAC__Frame = ^TFLAC__Frame; + + TFLAC__Frame = record + header: TFLAC__FrameHeader; + subframes: array [0 .. FLAC__MAX_CHANNELS - 1] of TFLAC__Subframe; + footer: Word; + end; + +var + FLAC__stream_encoder_new: function: Pointer cdecl; + FLAC__stream_encoder_set_verify: function(encoder: Pointer; value: Boolean) + : Boolean cdecl; + FLAC__stream_encoder_set_compression_level: function(encoder: Pointer; + value: Cardinal): Boolean cdecl; + FLAC__stream_encoder_set_channels: function(encoder: Pointer; value: Cardinal) + : Boolean cdecl; + FLAC__stream_encoder_set_bits_per_sample: function(encoder: Pointer; + value: Cardinal): Boolean cdecl; + FLAC__stream_encoder_set_sample_rate: function(encoder: Pointer; + value: Cardinal): Boolean cdecl; + FLAC__stream_encoder_set_total_samples_estimate: function(encoder: Pointer; + value: UInt64): Boolean cdecl; + FLAC__stream_encoder_init_stream: function(encoder: Pointer; + write_callback, seek_callback, tell_callback, metadata_callback: Pointer; + client_data: Pointer): Integer cdecl; + FLAC__stream_encoder_init_file: function(encoder: Pointer; + filename: PAnsiChar; progress_callback: Pointer; client_data: Pointer) + : Integer cdecl; + FLAC__stream_encoder_process_interleaved: function(encoder: Pointer; + const buffer; samples: Cardinal): Boolean cdecl; + FLAC__stream_encoder_finish: function(encoder: Pointer): Boolean cdecl; + FLAC__stream_encoder_delete: procedure(encoder: Pointer)cdecl; + FLAC__stream_decoder_new: function: Pointer cdecl; + FLAC__stream_decoder_init_stream: function(decoder: Pointer; + read_callback, seek_callback, tell_callback, length_callback, eof_callback, + write_callback, metadata_callback, error_callback: Pointer; + client_data: Pointer): Integer cdecl; + FLAC__stream_decoder_init_file: function(decoder: Pointer; + filename: PAnsiChar; write_callback, metadata_callback, error_callback + : Pointer; client_data: Pointer): Integer cdecl; + FLAC__stream_decoder_get_channels: function(decoder: Pointer): Cardinal cdecl; + FLAC__stream_decoder_get_bits_per_sample: function(decoder: Pointer) + : Cardinal cdecl; + FLAC__stream_decoder_process_until_end_of_stream: function(decoder: Pointer) + : Boolean cdecl; + FLAC__stream_decoder_finish: function(encoder: Pointer): Boolean cdecl; + FLAC__stream_decoder_delete: procedure(encoder: Pointer)cdecl; + DLLLoaded: Boolean = False; + +implementation + +var + DLLHandle: THandle; + +procedure Init; +begin + DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + + 'libFLAC_dynamic.dll')); + if DLLHandle >= 32 then + begin + @FLAC__stream_encoder_new := GetProcAddress(DLLHandle, + 'FLAC__stream_encoder_new'); + @FLAC__stream_encoder_set_verify := GetProcAddress(DLLHandle, + 'FLAC__stream_encoder_set_verify'); + @FLAC__stream_encoder_set_channels := GetProcAddress(DLLHandle, + 'FLAC__stream_encoder_set_channels'); + @FLAC__stream_encoder_set_compression_level := + GetProcAddress(DLLHandle, 'FLAC__stream_encoder_set_compression_level'); + @FLAC__stream_encoder_set_bits_per_sample := + GetProcAddress(DLLHandle, 'FLAC__stream_encoder_set_bits_per_sample'); + @FLAC__stream_encoder_set_sample_rate := + GetProcAddress(DLLHandle, 'FLAC__stream_encoder_set_sample_rate'); + @FLAC__stream_encoder_set_total_samples_estimate := + GetProcAddress(DLLHandle, + 'FLAC__stream_encoder_set_total_samples_estimate'); + @FLAC__stream_encoder_init_stream := GetProcAddress(DLLHandle, + 'FLAC__stream_encoder_init_stream'); + @FLAC__stream_encoder_init_file := GetProcAddress(DLLHandle, + 'FLAC__stream_encoder_init_file'); + @FLAC__stream_encoder_process_interleaved := + GetProcAddress(DLLHandle, 'FLAC__stream_encoder_process_interleaved'); + @FLAC__stream_encoder_finish := GetProcAddress(DLLHandle, + 'FLAC__stream_encoder_finish'); + @FLAC__stream_encoder_delete := GetProcAddress(DLLHandle, + 'FLAC__stream_encoder_delete'); + @FLAC__stream_decoder_new := GetProcAddress(DLLHandle, + 'FLAC__stream_decoder_new'); + @FLAC__stream_decoder_init_stream := GetProcAddress(DLLHandle, + 'FLAC__stream_decoder_init_stream'); + @FLAC__stream_decoder_init_file := GetProcAddress(DLLHandle, + 'FLAC__stream_decoder_init_file'); + @FLAC__stream_decoder_get_channels := GetProcAddress(DLLHandle, + 'FLAC__stream_decoder_get_channels'); + @FLAC__stream_decoder_get_bits_per_sample := + GetProcAddress(DLLHandle, 'FLAC__stream_decoder_get_bits_per_sample'); + @FLAC__stream_decoder_process_until_end_of_stream := + GetProcAddress(DLLHandle, + 'FLAC__stream_decoder_process_until_end_of_stream'); + @FLAC__stream_decoder_finish := GetProcAddress(DLLHandle, + 'FLAC__stream_decoder_finish'); + @FLAC__stream_decoder_delete := GetProcAddress(DLLHandle, + 'FLAC__stream_decoder_delete'); + DLLLoaded := Assigned(FLAC__stream_encoder_new) and + Assigned(FLAC__stream_decoder_new); + end + else + DLLLoaded := False; +end; + +procedure Deinit; +begin + if not DLLLoaded then + exit; + FreeLibrary(DLLHandle); +end; + +initialization + +Init; + +finalization + +Deinit; + +end. diff --git a/imports/JoJpegDLL.pas b/imports/JoJpegDLL.pas new file mode 100644 index 0000000..e2c7741 --- /dev/null +++ b/imports/JoJpegDLL.pas @@ -0,0 +1,69 @@ +unit JoJpegDLL; + +interface + +uses + WinAPI.Windows, + System.SysUtils, System.Classes; + +const + jojpeg_Size = 51320000; + + jojpeg_Compress = 0; + jojpeg_Decompress = 1; + + jojpeg_enc_Input = 1; + jojpeg_enc_Output1 = 2; + jojpeg_enc_Output2 = 3; + + jojpeg_dec_Input1 = 1; + jojpeg_dec_Input2 = 3; + jojpeg_dec_Output = 2; + +var + jojpeg_Init: function(p: Pointer; f_DEC: integer): integer stdcall; + jojpeg_Quit: procedure(p: Pointer; f_DEC: integer)stdcall; + jojpeg_Loop: function(p: Pointer; f_DEC: integer): integer stdcall; + jojpeg_Getvalue: function(p: Pointer; f_DEC, typ: integer): Int64; + jojpeg_Addbuf: procedure(p: Pointer; f_DEC: integer; buf: Pointer; + bufsize, state: integer)stdcall; + DLLLoaded: boolean = False; + +implementation + +var + DLLHandle: THandle; + +procedure Init; +begin + DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + + 'jojpeg_dll.dll')); + if DLLHandle >= 32 then + begin + @jojpeg_Init := GetProcAddress(DLLHandle, 'jojpeg_Init'); + @jojpeg_Quit := GetProcAddress(DLLHandle, 'jojpeg_Quit'); + @jojpeg_Loop := GetProcAddress(DLLHandle, 'jojpeg_Loop'); + @jojpeg_Getvalue := GetProcAddress(DLLHandle, 'jojpeg_Getvalue'); + @jojpeg_Addbuf := GetProcAddress(DLLHandle, 'jojpeg_Addbuf'); + DLLLoaded := Assigned(jojpeg_Init); + end + else + DLLLoaded := False; +end; + +procedure Deinit; +begin + if not DLLLoaded then + exit; + FreeLibrary(DLLHandle); +end; + +initialization + +Init; + +finalization + +Deinit; + +end. diff --git a/imports/LZ4DLL.pas b/imports/LZ4DLL.pas index 58cf0f0..1eccf75 100644 --- a/imports/LZ4DLL.pas +++ b/imports/LZ4DLL.pas @@ -69,6 +69,8 @@ var LZ4F_compressFrame: function(dstBuffer: Pointer; dstCapacity: size_t; srcBuffer: Pointer; srcSize: size_t; preferencesPtr: PLZ4F_preferences_t) : size_t cdecl; + LZ4_compressHC2: function(const src: Pointer; dst: Pointer; srcSize: Integer; + compressionLevel: Integer): Integer cdecl; LZ4F_compressFrameBound: function(srcSize: size_t; preferencesPtr: PLZ4F_preferences_t): size_t cdecl; LZ4F_createDecompressionContext: function(out dctxPtr: LZ4F_dctx; @@ -104,6 +106,7 @@ begin @LZ4_compress_default := GetProcAddress(DLLHandle, 'LZ4_compress_default'); @LZ4_compress_fast := GetProcAddress(DLLHandle, 'LZ4_compress_fast'); @LZ4_compress_HC := GetProcAddress(DLLHandle, 'LZ4_compress_HC'); + @LZ4_compressHC2 := GetProcAddress(DLLHandle, 'LZ4_compressHC2'); @LZ4F_compressFrame := GetProcAddress(DLLHandle, 'LZ4F_compressFrame'); @LZ4F_compressFrameBound := GetProcAddress(DLLHandle, 'LZ4F_compressFrameBound'); diff --git a/imports/LZMADLL.pas b/imports/LZMADLL.pas new file mode 100644 index 0000000..58ac065 --- /dev/null +++ b/imports/LZMADLL.pas @@ -0,0 +1,28 @@ +unit LZMADLL; + +interface + +uses + WinAPI.Windows, + System.SysUtils; + +type + PFL2_inBuffer = ^FL2_inBuffer; + + FL2_inBuffer = record + src: Pointer; + size: size_t; + pos: size_t; + end; + + PFL2_outBuffer = ^FL2_outBuffer; + + FL2_outBuffer = record + dst: Pointer; + size: size_t; + pos: size_t; + end; + +implementation + +end. diff --git a/imports/PackJPGDLL.pas b/imports/PackJPGDLL.pas new file mode 100644 index 0000000..b1dbe99 --- /dev/null +++ b/imports/PackJPGDLL.pas @@ -0,0 +1,67 @@ +unit PackJPGDLL; + +interface + +uses + WinAPI.Windows, + System.SysUtils, System.Classes; + +const + pjglib_file = 0; + pjglib_memory = 1; + pjglib_handle = 2; + +var + pjglib_convert_stream2stream: function(msg: PAnsiChar): Boolean cdecl; + pjglib_convert_file2file: function(ain, aout, msg: PAnsiChar): Boolean cdecl; + pjglib_convert_stream2mem: function(out_file: PPAnsiChar; out_size: PCardinal; + msg: PAnsiChar): Boolean cdecl; + pjglib_init_streams: procedure(in_src: Pointer; in_type: Integer; + in_size: Integer; out_dest: Pointer; out_type: Integer)cdecl; + pjglib_version_info: function: PAnsiChar cdecl; + pjglib_short_name: function: PAnsiChar cdecl; + DLLLoaded: Boolean = False; + +implementation + +var + DLLHandle: THandle; + +procedure Init; +begin + DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + + 'packjpg_dll.dll')); + if DLLHandle >= 32 then + begin + @pjglib_convert_stream2stream := GetProcAddress(DLLHandle, + 'pjglib_convert_stream2stream'); + @pjglib_convert_file2file := GetProcAddress(DLLHandle, + 'pjglib_convert_file2file'); + @pjglib_convert_stream2mem := GetProcAddress(DLLHandle, + 'pjglib_convert_stream2mem'); + @pjglib_init_streams := GetProcAddress(DLLHandle, 'pjglib_init_streams'); + @pjglib_version_info := GetProcAddress(DLLHandle, 'pjglib_version_info'); + @pjglib_short_name := GetProcAddress(DLLHandle, 'pjglib_short_name'); + DLLLoaded := Assigned(pjglib_init_streams) and + Assigned(pjglib_convert_stream2stream); + end + else + DLLLoaded := False; +end; + +procedure Deinit; +begin + if not DLLLoaded then + exit; + FreeLibrary(DLLHandle); +end; + +initialization + +Init; + +finalization + +Deinit; + +end. diff --git a/imports/TTADLL.pas b/imports/TTADLL.pas new file mode 100644 index 0000000..12ba17e --- /dev/null +++ b/imports/TTADLL.pas @@ -0,0 +1,53 @@ +unit TTADLL; + +interface + +uses + WinAPI.Windows, + System.SysUtils, System.Classes; + +var + tta_encode: function(const src: Pointer; srcSize: PInteger; dst: Pointer; + dstCapacity: PInteger): Boolean cdecl; + tta_decode: function(const src: Pointer; srcSize: Integer; dst: Pointer; + dstCapacity: PInteger): Boolean cdecl; + tta_getsize: function(const src: Pointer; srcSize: Integer; + headerSize: PInteger): Integer cdecl; + DLLLoaded: Boolean = False; + +implementation + +var + DLLHandle: THandle; + +procedure Init; +begin + DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + + 'libtta_dll.dll')); + if DLLHandle >= 32 then + begin + @tta_encode := GetProcAddress(DLLHandle, 'encode'); + @tta_decode := GetProcAddress(DLLHandle, 'decode'); + @tta_getsize := GetProcAddress(DLLHandle, 'getsize'); + DLLLoaded := Assigned(tta_encode) and Assigned(tta_decode); + end + else + DLLLoaded := False; +end; + +procedure Deinit; +begin + if not DLLLoaded then + exit; + FreeLibrary(DLLHandle); +end; + +initialization + +Init; + +finalization + +Deinit; + +end. diff --git a/io/IODecode.pas b/io/IODecode.pas index 4ee9769..a82389b 100644 --- a/io/IODecode.pas +++ b/io/IODecode.pas @@ -128,7 +128,7 @@ begin '_' + Random($7FFFFFFF).ToHexString + XTOOL_MAPSUF2)), LFilename); try Move(SS2.Memory^, (PByte(SS1.Memory) + LEntry.Position)^, - LEntry.Size); + Min(SS2.Size, LEntry.Size)); finally SS2.Free; end; diff --git a/precompressor/PrecompCrypto.pas b/precompressor/PrecompCrypto.pas index a16fb9b..26a198c 100644 --- a/precompressor/PrecompCrypto.pas +++ b/precompressor/PrecompCrypto.pas @@ -91,7 +91,7 @@ begin DI2.OldSize := SI.NewSize; DI2.NewSize := SI.NewSize; Funcs^.LogScan1(CryptoCodecs[GetBits(SI.Option, 0, 5)], SI.Position, - SI.OldSize, SI.NewSize); + SI.OldSize, -1); Add(Instance, @SI, DI1.Codec, @DI2); end; end; @@ -108,10 +108,10 @@ begin exit; if (Res > 0) and (StreamInfo^.OldSize > 0) then begin - StreamInfo^.NewSize := StreamInfo^.OldSize; Output(Instance, Input, StreamInfo^.OldSize); + StreamInfo^.NewSize := StreamInfo^.OldSize; Funcs^.LogScan2(CryptoCodecs[GetBits(StreamInfo^.Option, 0, 5)], - StreamInfo^.OldSize, StreamInfo^.NewSize); + StreamInfo^.OldSize, -1); Result := True; end; end; @@ -143,7 +143,7 @@ begin end; Result := True; Funcs^.LogProcess(CryptoCodecs[GetBits(StreamInfo^.Option, 0, 5)], nil, - StreamInfo^.OldSize, StreamInfo^.NewSize, StreamInfo^.OldSize, Result); + StreamInfo^.OldSize, -1, -1, Result); end; end; @@ -175,7 +175,7 @@ begin Output(Instance, Input, StreamInfo.OldSize); Result := True; Funcs^.LogRestore(CryptoCodecs[GetBits(StreamInfo.Option, 0, 5)], nil, - StreamInfo.OldSize, StreamInfo.NewSize, StreamInfo.OldSize, Result); + StreamInfo.OldSize, -1, -1, Result); end; end; diff --git a/precompressor/PrecompEXE.pas b/precompressor/PrecompEXE.pas index 3c9cd8a..9dfca77 100644 --- a/precompressor/PrecompEXE.pas +++ b/precompressor/PrecompEXE.pas @@ -589,7 +589,7 @@ begin StreamInfo^.OldSize); Funcs^.LogProcess(PChar(Codec.Names[X]), nil, StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result); - if Result = False then + if (Result = False) and (DIFF_TOLERANCE > 0) then begin Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1)); diff --git a/precompressor/PrecompLZ4.pas b/precompressor/PrecompLZ4.pas index f51b8b0..ff9005a 100644 --- a/precompressor/PrecompLZ4.pas +++ b/precompressor/PrecompLZ4.pas @@ -24,12 +24,14 @@ const L_MAXSIZE = 16 * 1024 * 1024; L_BLOCKSIZE = 0; L_BLOCKDEPENDENCY = 0; + L_ACCELERATION = 1; var SOList: array of array [0 .. CODEC_COUNT - 1] of TSOList; CodecAvailable, CodecEnabled: TArray; LBlockSize: Integer = L_BLOCKSIZE; LBlockDependency: Integer = L_BLOCKDEPENDENCY; + LAcceleration: Integer = L_ACCELERATION; function LZ4Init(Command: PChar; Count: Integer; Funcs: PPrecompFuncs): Boolean; var @@ -57,6 +59,8 @@ begin if (CompareText(S, LZ4Codecs[LZ4_CODEC]) = 0) and LZ4DLL.DLLLoaded then begin CodecEnabled[LZ4_CODEC] := True; + if Funcs^.GetParam(Command, X, 'a') <> '' then + LAcceleration := StrToInt(Funcs^.GetParam(Command, X, 'a')); end else if (CompareText(S, LZ4Codecs[LZ4HC_CODEC]) = 0) and LZ4DLL.DLLLoaded then @@ -86,7 +90,7 @@ begin if SOList[X, LZ4_CODEC].Count = 0 then SOList[X, LZ4_CODEC].Update([1]); SetLength(Options, 0); - for I := 3 to 12 do + for I := 2 to 12 do Insert(I, Options, Length(Options)); for X := Low(SOList) to High(SOList) do if SOList[X, LZ4HC_CODEC].Count = 0 then @@ -118,6 +122,7 @@ begin Option^ := 0; SetBits(Option^, LBlockSize, 12, 2); SetBits(Option^, LBlockDependency, 14, 1); + SetBits(Option^, LAcceleration, 15, 7); I := 0; while Funcs^.GetCodec(Command, I, False) <> '' do begin @@ -125,6 +130,8 @@ begin if (CompareText(S, LZ4Codecs[LZ4_CODEC]) = 0) and LZ4DLL.DLLLoaded then begin SetBits(Option^, LZ4_CODEC, 0, 5); + if Funcs^.GetParam(Command, I, 'a') <> '' then + SetBits(Option^, StrToInt(Funcs^.GetParam(Command, I, 'a')), 15, 7); Result := True; end else if (CompareText(S, LZ4Codecs[LZ4HC_CODEC]) = 0) and LZ4DLL.DLLLoaded @@ -188,6 +195,7 @@ begin SetBits(SI.Option, X, 0, 5); SetBits(SI.Option, LBlockSize, 12, 2); SetBits(SI.Option, LBlockDependency, 14, 1); + SetBits(SI.Option, LAcceleration, 15, 7); if System.Pos(SPrecompSep2, DI1.Codec) > 0 then SI.Status := TStreamStatus.Predicted else @@ -230,8 +238,8 @@ begin end; if Res > StreamInfo^.OldSize then begin - StreamInfo^.NewSize := Res; Output(Instance, Buffer, Res); + StreamInfo^.NewSize := Res; Funcs^.LogScan2(LZ4Codecs[GetBits(StreamInfo^.Option, 0, 5)], StreamInfo^.OldSize, StreamInfo^.NewSize); Result := True; @@ -261,12 +269,13 @@ begin if StreamInfo^.Status = TStreamStatus.Predicted then if GetBits(StreamInfo^.Option, 5, 7) <> I then continue; + Params := ''; case X of LZ4_CODEC: begin - Params := ''; - Res1 := LZ4_compress_default(NewInput, Buffer, - StreamInfo^.NewSize, Y); + Params := 'a' + GetBits(StreamInfo^.Option, 15, 7).ToString; + Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y, + GetBits(StreamInfo^.Option, 15, 7)); end; LZ4HC_CODEC: begin @@ -296,7 +305,7 @@ begin break; end; if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or - (SOList[Instance][X].Count = 1)) then + (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then begin Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1)); Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1, @@ -332,14 +341,16 @@ begin X := GetBits(StreamInfo.Option, 0, 5); if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then exit; + Params := ''; Buffer := Funcs^.Allocator(Instance, LZ4F_compressFrameBound(StreamInfo.NewSize, nil)); case X of LZ4_CODEC: begin - Params := ''; - Res1 := LZ4_compress_default(Input, Buffer, StreamInfo.NewSize, - LZ4F_compressFrameBound(StreamInfo.NewSize, nil)); + Params := 'a' + GetBits(StreamInfo.Option, 15, 7).ToString; + Res1 := LZ4_compress_fast(Input, Buffer, StreamInfo.NewSize, + LZ4F_compressFrameBound(StreamInfo.NewSize, nil), + GetBits(StreamInfo.Option, 15, 7)); end; LZ4HC_CODEC: begin diff --git a/precompressor/PrecompLZO.pas b/precompressor/PrecompLZO.pas index 472fbcf..f090292 100644 --- a/precompressor/PrecompLZO.pas +++ b/precompressor/PrecompLZO.pas @@ -272,8 +272,8 @@ begin end; if Res > StreamInfo^.OldSize then begin - StreamInfo^.NewSize := Res; Output(Instance, Buffer, Res); + StreamInfo^.NewSize := Res; Funcs^.LogScan2(LZOCodecs[GetBits(StreamInfo^.Option, 0, 5)], StreamInfo^.OldSize, StreamInfo^.NewSize); Result := True; @@ -301,6 +301,7 @@ begin if StreamInfo^.Status = TStreamStatus.Predicted then if GetBits(StreamInfo^.Option, 5, 7) <> I then continue; + Params := ''; Res1 := StreamInfo^.NewSize; case X of LZO1X_CODEC: @@ -326,7 +327,7 @@ begin break; end; if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or - (SOList[Instance][X].Count = 1)) then + (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then begin Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1)); Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1, @@ -361,6 +362,7 @@ begin X := GetBits(StreamInfo.Option, 0, 5); if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then exit; + Params := ''; Buffer := Funcs^.Allocator(Instance, StreamInfo.NewSize); Res1 := StreamInfo.NewSize; case X of diff --git a/precompressor/PrecompMain.pas b/precompressor/PrecompMain.pas index b7a1cba..aa53630 100644 --- a/precompressor/PrecompMain.pas +++ b/precompressor/PrecompMain.pas @@ -7,7 +7,7 @@ interface uses Threading, Utils, SynCommons, ParseClass, ParseExpr, PrecompUtils, PrecompCrypto, PrecompZLib, PrecompLZ4, PrecompLZO, PrecompZSTD, - PrecompOodle, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE, + PrecompOodle, PrecompMedia, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE, WinAPI.Windows, WinAPI.ShlObj, System.SysUtils, System.Classes, System.SyncObjs, System.Math, System.Types, System.StrUtils, System.RTLConsts, System.TimeSpan, System.Diagnostics, @@ -24,8 +24,7 @@ type ChunkSize, Threads: Integer; Depth: Integer; LowMem: Boolean; - DBaseFile: String; - DedupFile: String; + DBaseFile, ExtractDir: String; end; PDecodeOptions = ^TDecodeOptions; @@ -34,7 +33,6 @@ type Method: String; ChunkCount, Threads: Integer; Depth: Integer; - DedupFile: String; DedupSysMem, DedupGPUMem: Int64; end; @@ -53,10 +51,10 @@ function PrecompReadFuture(Index: Integer; Position: NativeInt; Buffer: Pointer; procedure PrecompLogScan1(Codec: PChar; Position: Int64; InSize, OutSize: Integer)cdecl; procedure PrecompLogScan2(Codec: PChar; InSize, OutSize: Integer)cdecl; -procedure PrecompLogProcess(Codec, Method: PChar; - OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl; -procedure PrecompLogRestore(Codec, Method: PChar; - OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl; +procedure PrecompLogProcess(Codec, Method: PChar; Size1, Size2, Size3: Integer; + Status: Boolean)cdecl; +procedure PrecompLogRestore(Codec, Method: PChar; Size1, Size2, Size3: Integer; + Status: Boolean)cdecl; procedure PrecompLogPatch1(OldSize, NewSize, PatchSize: Integer; Status: Boolean)cdecl; procedure PrecompLogPatch2(OldSize, NewSize, PatchSize: Integer; @@ -101,10 +99,11 @@ var IntArray: array [0 .. 1] of Int64; Codecs: array of TPrecompressor; DBFile: String = ''; + ExtDir: String = ''; UseDB: Boolean = False; - DupFile: String = ''; StoreDD: Boolean = False; - DupGUID: TGUID; + VERBOSE: Boolean = False; + EXTRACT: Boolean = False; DupSysMem: Int64 = 0; EncInfo: TEncInfo; ConTask: TTask; @@ -131,11 +130,14 @@ begin WriteLn(ErrOutput, ''); WriteLn(ErrOutput, 'Advanced parameters:'); WriteLn(ErrOutput, - ' --dbase=# - use database (#=filename to save db, optional)'); + ' --dbase=# - use database (#=filename to save db, optional)'); WriteLn(ErrOutput, - ' --dedup=# - use stream deduplication (#=filename to save db, optional)'); + ' --dedup=# - use stream deduplication (#=filename to save db, optional)'); WriteLn(ErrOutput, - ' --mem=# - deduplication ram usage limit (#=size) [75p]'); + ' --mem=# - deduplication ram usage limit (#=size) [75p]'); + WriteLn(ErrOutput, + ' --diff=# - set xdelta threshold to accept streams [5p]'); + WriteLn(ErrOutput, ' --extract=# - extract streams to directory path'); WriteLn(ErrOutput, ''); end; @@ -182,16 +184,18 @@ begin if Options.DBaseFile <> '' then UseDB := True; StoreDD := ArgParse.AsBoolean('--dedup'); - Options.DedupFile := ArgParse.AsString('--dedup='); S := ArgParse.AsString('--diff=', 0, '5p'); S := ReplaceText(S, 'p', '%'); DIFF_TOLERANCE := Max(0.00, ExpParse.Evaluate(S)); VERBOSE := ArgParse.AsBoolean('--verbose'); + Options.ExtractDir := ArgParse.AsString('--extract='); + if Options.ExtractDir <> '' then + EXTRACT := DirectoryExists(Options.ExtractDir); finally ArgParse.Free; ExpParse.Free; end; - if VERBOSE then + if VERBOSE or EXTRACT then Options.Threads := 1; end; @@ -210,7 +214,6 @@ begin S := ReplaceText(S, 'p', '%'); S := ReplaceText(S, '%', '%*' + CPUCount.ToString); Options.Threads := Max(1, Round(ExpParse.Evaluate(S))); - Options.DedupFile := ArgParse.AsString('--dedup='); S := ArgParse.AsString('--mem=', 0, '75p'); S := ReplaceText(S, 'KB', '* 1024^1'); S := ReplaceText(S, 'MB', '* 1024^2'); @@ -267,14 +270,13 @@ type StrIdx: TArray; end; - TDupRec = record - Dict: TSynDictionary; - Index: Integer; - end; - var - Database: TSynDictionary; - Duplicates1: array [0 .. 1] of TDupRec; + DBInfo: TArray>; + DBCount: TArray; + DDInfo: TArray>; + DDCount1: TArray; + DDList1: TArray; + DDIndex: Integer; ComVars1: TArray; Tasks: TArray; CurCodec: TArray; @@ -285,6 +287,7 @@ var Scanned1, Scanned2, Processed: TArray; LogInt: Integer; LogInt64: Int64; + LogPtr: Pointer; procedure CodecInit(Count: Integer; Method: String); var @@ -303,6 +306,7 @@ begin Insert(PrecompLZO.Codec, Codecs, Length(Codecs)); Insert(PrecompZSTD.Codec, Codecs, Length(Codecs)); Insert(PrecompOodle.Codec, Codecs, Length(Codecs)); + Insert(PrecompMedia.Codec, Codecs, Length(Codecs)); for X := Low(Codecs) to High(Codecs) do for Y := Low(Codecs[X].Names) to High(Codecs[X].Names) do Insert(Codecs[X].Names[Y], List, Length(List)); @@ -394,64 +398,101 @@ end; procedure PrecompLogScan1(Codec: PChar; Position: Int64; InSize, OutSize: Integer); +var + S: String; begin if not VERBOSE then exit; with ComVars1[CurDepth[0]] do begin + if OutSize < 0 then + S := '(%d)' + else + S := '(%d >> %d)'; if (OutSize > 0) and (Position < DataStore.Size(0)) and (MemOutput1[0].Position - CurPos1[0] = OutSize) then - WriteLn(ErrOutput, Format('[%d] Actual %s stream found at %s (%d >> %d)', + WriteLn(ErrOutput, Format('[%d] Actual %s stream found at %s ' + S, [CurDepth[0], Codec, (DataStore.Position(0) + Position).ToHexString, InSize, OutSize])) else - WriteLn(ErrOutput, - Format('[%d] Possible %s stream located at %s (%d >> %d)', + WriteLn(ErrOutput, Format('[%d] Possible %s stream located at %s ' + S, [CurDepth[0], Codec, (DataStore.Position(0) + Position).ToHexString, InSize, OutSize])); end; end; procedure PrecompLogScan2(Codec: PChar; InSize, OutSize: Integer); +var + S: String; begin if not VERBOSE then exit; - WriteLn(ErrOutput, Format('[%d] Confirmed %s stream at %s (%d >> %d)', + if OutSize < 0 then + S := '(%d)' + else + S := '(%d >> %d)'; + WriteLn(ErrOutput, Format('[%d] Confirmed %s stream at %s ' + S, [CurDepth[0], Codec, LogInt64.ToHexString, InSize, OutSize])); end; -procedure PrecompLogProcess(Codec, Method: PChar; - OriginalSize, InSize, OutSize: Integer; Status: Boolean); +procedure PrecompLogProcess(Codec, Method: PChar; Size1, Size2, Size3: Integer; + Status: Boolean); var - S: String; + S1, S2: String; begin - if not VERBOSE then - exit; - if Status then - S := '[%d] Processed %s stream at %s (%d >> %d >> %d)' + - IfThen(String(Method) <> '', ' using %s', '') + ' successfully' - else - S := '[%d] Processing %s stream at %s (%d >> %d >> %d)' + - IfThen(String(Method) <> '', ' using %s', '') + ' has failed'; - WriteLn(ErrOutput, Format(S, [CurDepth[0], Codec, LogInt64.ToHexString, - OriginalSize, InSize, OutSize, Method])); + if VERBOSE then + begin + if Size2 < 0 then + S1 := '(%d)' + else if Size3 < 0 then + S1 := '(%d >> %d)' + else + S1 := '(%d >> %d >> %d)'; + if Status then + S2 := '[%d] Processed %s stream at %s ' + S1 + + IfThen(String(Method) <> '', ' using ' + String(Method), '') + + ' successfully' + else + S2 := '[%d] Processing %s stream at %s ' + S1 + + IfThen(String(Method) <> '', ' using ' + String(Method), '') + + ' has failed'; + WriteLn(ErrOutput, Format(S2, [CurDepth[0], Codec, LogInt64.ToHexString, + Size1, Size2, Size3])); + end; + if EXTRACT and (CurDepth[0] = 0) then + begin + S1 := '%s_%s.raw'; + with TFileStream.Create(ExtDir + Format(S1, [LogInt64.ToHexString, Codec]), + fmCreate) do + try + WriteBuffer(LogPtr^, Size1); + finally + Free; + end; + end; end; -procedure PrecompLogRestore(Codec, Method: PChar; - OriginalSize, InSize, OutSize: Integer; Status: Boolean); +procedure PrecompLogRestore(Codec, Method: PChar; Size1, Size2, Size3: Integer; + Status: Boolean); var - S: String; + S1, S2: String; begin if not VERBOSE then exit; - if Status then - S := '[%d] Restored %s stream at %s (%d >> %d >> %d)' + - IfThen(String(Method) <> '', ' using %s', '') + ' successfully' + if Size2 < 0 then + S1 := '(%d)' + else if Size3 < 0 then + S1 := '(%d >> %d)' else - S := '[%d] Restoring %s stream at %s (%d >> %d >> %d)' + - IfThen(String(Method) <> '', ' using %s', '') + ' has failed'; - WriteLn(ErrOutput, Format(S, [CurDepth[0], Codec, LogInt64.ToHexString, - OriginalSize, InSize, OutSize, Method])); + S1 := '(%d >> %d >> %d)'; + if Status then + S2 := '[%d] Restored %s stream at %s ' + S1 + IfThen(String(Method) <> '', + ' using ' + String(Method), '') + ' successfully' + else + S2 := '[%d] Restoring %s stream at %s ' + S1 + IfThen(String(Method) <> '', + ' using ' + String(Method), '') + ' has failed'; + WriteLn(ErrOutput, Format(S2, [CurDepth[0], Codec, LogInt64.ToHexString, + Size1, Size2, Size3])); end; procedure PrecompLogPatch1(OldSize, NewSize, PatchSize: Integer; @@ -553,7 +594,9 @@ begin begin MemOutput1[Instance].Position := CurPos1[Instance]; exit; - end; + end + else if LCodec = CurCodec[Instance] then + LOption := Info^.Option; end else begin @@ -616,53 +659,130 @@ begin CurTransfer[Instance] := String(Codec); end; -function CheckDB(Dictionary: TSynDictionary; const StreamInfo: TEncodeSI; - var Database: TDatabase): Boolean; +function CheckDB(StreamInfo: TEncodeSI; Database: PDatabase): Boolean; var - DBKey: Int64; + A: Word; + I: Integer; + LCount: Integer; + DB: PDatabase; begin Result := False; - Int64Rec(DBKey).Lo := StreamInfo.Checksum; - Int64Rec(DBKey).Hi := StreamInfo.OldSize; - Result := Dictionary.FindAndCopy(DBKey, Database); -end; - -procedure AddDB(Dictionary: TSynDictionary; const StreamInfo: TEncodeSI; - const Database: TDatabase); -var - DBKey: Int64; -begin - Int64Rec(DBKey).Lo := StreamInfo.Checksum; - Int64Rec(DBKey).Hi := StreamInfo.OldSize; - Dictionary.AddOrUpdate(DBKey, Database); -end; - -function CheckDup(var DupRec: TDupRec; const StreamInfo: TEncodeSI; - var StreamKey, DupCount: Integer): Boolean; -var - DupKey: Int64; - DupInfo: PDuplicate; - DupAdded: Boolean; -begin - Result := False; - Inc(DupRec.Index); - Int64Rec(DupKey).Lo := StreamInfo.Checksum; - Int64Rec(DupKey).Hi := StreamInfo.OldSize; - DupInfo := DupRec.Dict.FindValueOrAdd(DupKey, DupAdded); - if not DupAdded then + A := LongRec(StreamInfo.Checksum).Lo; + AtomicExchange(LCount, DBCount[A]); + for I := 0 to LCount - 1 do begin + DB := @DBInfo[A, I]; + if (DB^.Size = StreamInfo.OldSize) and (DB^.Checksum = StreamInfo.Checksum) + then + begin + if Assigned(Database) then + Move(DB^, Database^, SizeOf(TDatabase)); + Result := True; + break; + end; + end; +end; + +procedure AddDB(StreamInfo: TEncodeSI); +var + A: Word; + I: Integer; + DB: TDatabase; +begin + A := LongRec(StreamInfo.Checksum).Lo; + if not CheckDB(StreamInfo, nil) then + begin + GlobalSync.Acquire; + try + DB.Size := StreamInfo.OldSize; + DB.Codec := StreamInfo.Codec; + DB.Option := StreamInfo.Option; + DB.Checksum := StreamInfo.Checksum; + DB.Status := StreamInfo.Status; + Insert(DB, DBInfo[A], Length(DBInfo[A])); + Inc(DBCount[A]); + finally + GlobalSync.Release; + end; + end; +end; + +function CheckDD(StreamInfo: TEncodeSI; Database: PDuplicate1; + Index: PInteger): Boolean; +var + A: Word; + I: Integer; + LCount: Integer; + DD: PDuplicate1; +begin + Result := False; + A := LongRec(StreamInfo.Checksum).Lo; + LCount := DDCount1[A]; + for I := 0 to LCount - 1 do + begin + DD := @DDInfo[A, I]; + if (DD^.Size = StreamInfo.OldSize) and (DD^.Checksum = StreamInfo.Checksum) + then + begin + if Assigned(Database) then + Move(DD^, Database^, SizeOf(TDuplicate1)); + if Assigned(Index) then + Index^ := I; + Result := True; + break; + end; + end; +end; + +function FindDD(StreamInfo: TEncodeSI; Index, Count: PInteger): Boolean; +var + A: Word; + I: Integer; + DD: PDuplicate1; +begin + Result := False; + if CheckDD(StreamInfo, nil, @I) then + begin + A := LongRec(StreamInfo.Checksum).Lo; + DD := @DDInfo[A, I]; + if Assigned(Index) then + Index^ := DD^.Index; + if Assigned(Count) then + Count^ := DD^.Count; + Result := True; + end; +end; + +function FindOrAddDD(StreamInfo: TEncodeSI; Index, Count: PInteger): Boolean; +var + A: Word; + I: Integer; + DD: TDuplicate1; + I64: Int64; +begin + Result := False; + Inc(DDIndex); + A := LongRec(StreamInfo.Checksum).Lo; + if not CheckDD(StreamInfo, nil, @I) then + begin + DD.Size := StreamInfo.OldSize; + DD.Checksum := StreamInfo.Checksum; + DD.Index := DDIndex; + DD.Count := 0; + I := Length(DDInfo[A]); + Insert(DD, DDInfo[A], I); + Int64Rec(I64).Words[0] := A; + Int64Rec(I64).Hi := DDCount1[A]; + Insert(I64, DDList1, Length(DDList1)); + Inc(DDCount1[A]); Result := True; - Inc(DupInfo^.Count); - StreamKey := DupInfo^.Index; - DupCount := DupInfo^.Count; end else - begin - DupInfo^.Count := 0; - DupInfo^.Index := DupRec.Index; - StreamKey := -1; - DupCount := 0; - end; + Inc(DDInfo[A, I].Count); + if Assigned(Index) then + Index^ := DDInfo[A, I].Index; + if Assigned(Count) then + Count^ := DDInfo[A, I].Count; end; procedure Scan1(Index, Depth: Integer); @@ -729,6 +849,8 @@ begin X := DataStore.ActualSize(Index) - NativeInt(SI2.Position - DataStore.Position(Index)); LogInt64 := SI2.Position; + LogPtr := PByte(DataStore.Slot(Index).Memory) + + NativeInt(SI2.Position - DataStore.Position(Index)); if (SI1.OldSize <= X) and Codecs[SI2.Codec].Scan2(Index, Depth, PByte(DataStore.Slot(Index).Memory) + NativeInt(SI2.Position - DataStore.Position(Index)), X, @SI1, @J, @@ -830,10 +952,11 @@ begin SI1.Resource := SI2.Resource; SI1.Option := SI2.Option; SI1.Status := SI2.Status; - LogInt64 := DataStore.Position(0) + SI2.ActualPosition; + LogInt64 := DataStore.Position(ThreadIndex) + SI2.ActualPosition; + LogPtr := PByte(DataStore.Slot(ThreadIndex).Memory) + SI2.ActualPosition; if UseDB and (SI2.Codec > 2) then begin - DBBool := CheckDB(Database, SI2, DBTyp); + DBBool := CheckDB(SI2, @DBTyp); if DBBool and (SI2.Codec = DBTyp.Codec) then begin if DBTyp.Status = TStreamStatus.Invalid then @@ -841,7 +964,7 @@ begin else begin SI1.Option := DBTyp.Option; - SI1.Status := TStreamStatus.Predicted; + SI1.Status := TStreamStatus.Database; end; end; end; @@ -882,13 +1005,14 @@ begin if UseDB then if not DBBool then begin - DBTyp.Codec := SI2.Codec; - DBTyp.Option := SI1.Option; if Result then - DBTyp.Status := TStreamStatus.Predicted + begin + SI2.Option := SI1.Option; + SI2.Status := TStreamStatus.Predicted + end else - DBTyp.Status := TStreamStatus.Invalid; - AddDB(Database, SI2, DBTyp); + SI2.Status := TStreamStatus.Invalid; + AddDB(SI2); end; if Result then begin @@ -1033,10 +1157,9 @@ procedure EncInit(Input, Output: TStream; Options: PEncodeOptions); var UI32: UInt32; I, J, K: Integer; + W: Word; Bytes: TBytes; NI: NativeInt; - DBKey: Int64; - DBTyp: TDatabase; S: String; DupMethod: Boolean; begin @@ -1046,15 +1169,21 @@ begin ThreadSync[I] := TCriticalSection.Create; I := XTOOL_PRECOMP; Output.WriteBuffer(I, I.Size); - CreateGUID(DupGUID); - Output.WriteBuffer(DupGUID, SizeOf(TGUID)); - Database := TSynDictionary.Create(TypeInfo(TInt64DynArray), - TypeInfo(TDatabaseDynArray)); - for I := Low(Duplicates1) to High(Duplicates1) do + if UseDB then begin - Duplicates1[I].Dict := TSynDictionary.Create(TypeInfo(TInt64DynArray), - TypeInfo(TDuplicateDynArray)); - Duplicates1[I].Index := -1; + SetLength(DBInfo, $10000); + SetLength(DBCount, $10000); + for I := Low(DBInfo) to High(DBInfo) do + DBCount[I] := 0; + end; + if StoreDD then + begin + SetLength(DDInfo, $10000); + SetLength(DDCount1, $10000); + SetLength(DDList1, 0); + for I := Low(DDInfo) to High(DDInfo) do + DDCount1[I] := 0; + DDIndex := -1; end; SetLength(Tasks, Options^.Threads); SetLength(CurCodec, Options^.Threads); @@ -1126,7 +1255,6 @@ begin end; CodecInit(Options^.Threads, Options^.Method); DBFile := Options^.DBaseFile; - DupFile := Options^.DedupFile; if FileExists(ExtractFilePath(Utils.GetModuleName) + DBFile) then begin with TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + DBFile, @@ -1140,16 +1268,19 @@ begin end; with WorkStream[0] do begin - J := PInteger(Memory)^; - for I := 0 to J - 1 do + Position := 0; + while Position < Size do begin - NI := Integer.Size + (I * (SizeOf(Int64) + SizeOf(TDatabase))); - DBKey := PInt64(PByte(Memory) + NI)^; - DBTyp := PDatabase(PByte(Memory) + NI + SizeOf(Int64))^; - Database.Add(DBKey, DBTyp); + ReadBuffer(W, W.Size); + ReadBuffer(J, J.Size); + DBCount[W] := J; + SetLength(DBInfo[W], J); + for K := 0 to J - 1 do + ReadBuffer(DBInfo[W, K], SizeOf(TDatabase)); end; end; end; + ExtDir := IncludeTrailingBackSlash(Options^.ExtractDir); Output.WriteBuffer(Options^.Depth, Options^.Depth.Size); S := ''; I := 0; @@ -1237,9 +1368,6 @@ begin Tasks[I].Free; WorkStream[I].Free; end; - Database.Free; - for I := Low(Duplicates1) to High(Duplicates1) do - Duplicates1[I].Dict.Free; FreeResources; GlobalSync.Free; for I := Low(ThreadSync) to High(ThreadSync) do @@ -1265,19 +1393,19 @@ var StreamCount: Integer; BlockSize: Int64; UI32: UInt32; - I, J, X: Integer; + I, J, K, X: Integer; + W: Word; + I64: Int64; LastStream, LastPos: Int64; LastIndex: Integer; CurrSize: Cardinal; DupBool: Boolean; - DupKey, DupCount: Integer; - DBKey: Int64; - DBTyp: TDatabase; - DupTyp: TDuplicate; + DupIdx1, DupIdx2, DupCount: Integer; + DupTyp: TDuplicate2; begin if (Depth = 0) then begin - if (DupFile = '') and StoreDD then + if StoreDD then TempOutput := TPrecompVMStream.Create else TempOutput := Output; @@ -1285,6 +1413,7 @@ begin else TempOutput := Output; Result := False; + DupIdx1 := 0; with ComVars1[Depth] do begin LastStream := 0; @@ -1381,16 +1510,15 @@ begin begin Inc(StreamCount); DupBool := False; - if (Depth = 0) and ((DupFile <> '') or StoreDD) then - DupBool := CheckDup(Duplicates1[0], StreamInfo, DupKey, - DupCount); + if (Depth = 0) and StoreDD then + DupBool := not FindOrAddDD(StreamInfo, @DupIdx2, @DupCount); if DupBool then begin - if DupCount = 2 then + if DupCount = 1 then Inc(EncInfo.DecMem2, StreamInfo.OldSize); FillChar(StreamHeader, SizeOf(TStreamHeader), 0); StreamHeader.Kind := DUPLICATED_STREAM; - StreamHeader.Option := DupKey; + StreamHeader.Option := DupIdx2; end else begin @@ -1427,19 +1555,19 @@ begin Result := True else if Depth > 0 then exit; + I64 := MemStream[I].Position; MemStream[I].Position := 0; MemStream[I].WriteBuffer(StreamCount, StreamCount.Size); MemStream[I].WriteBuffer(BlockSize, BlockSize.Size); - TempOutput.WriteBuffer(MemStream[I].Memory^, MemStream[I].Position + - StreamCount * SizeOf(TStreamHeader)); + TempOutput.WriteBuffer(MemStream[I].Memory^, I64); InfoStore1[I].Index := LastIndex; J := InfoStore1[I].Get(StreamInfo); while J >= 0 do begin DupBool := False; - if (Depth = 0) and ((DupFile <> '') or StoreDD) then - DupBool := CheckDup(Duplicates1[1], StreamInfo, DupKey, DupCount); - if not DupBool then + if (Depth = 0) and StoreDD then + DupBool := FindDD(StreamInfo, @DupIdx2, @DupCount); + if (DupBool = False) or (DupIdx1 = DupIdx2) then begin if StreamInfo.ExtSize < 0 then begin @@ -1465,6 +1593,7 @@ begin StreamInfo.ExtSize.Size); end; end; + Inc(DupIdx1); if Succ(J - LastIndex) = StreamCount then break; J := InfoStore1[I].Get(StreamInfo); @@ -1519,14 +1648,16 @@ begin with WorkStream[0] do begin Position := 0; - J := Database.Count; - WriteBuffer(J, J.Size); - for I := 0 to J - 1 do + for W := 0 to $10000 - 1 do begin - DBKey := PInt64(Database.Keys.ElemPtr(I))^; - WriteBuffer(DBKey, SizeOf(Int64)); - DBTyp := PDatabase(Database.Values.ElemPtr(I))^; - WriteBuffer(DBTyp, SizeOf(TDatabase)); + J := DBCount[I]; + if J > 0 then + begin + WriteBuffer(W, W.Size); + WriteBuffer(J, J.Size); + for K := 0 to J - 1 do + WriteBuffer(DBInfo[W, K], SizeOf(TDatabase)); + end; end; end; with TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + DBFile, @@ -1536,40 +1667,27 @@ begin Free; end; end; - if (DupFile <> '') or StoreDD then + if StoreDD then begin - for I := Duplicates1[0].Dict.Count - 1 downto 0 do - begin - if PDuplicate(Duplicates1[0].Dict.Values.ElemPtr(I))^.Count < 1 then - Duplicates1[0].Dict.DeleteAt(I); - end; with WorkStream[0] do begin Position := 0; - WriteBuffer(DupGUID, SizeOf(TGUID)); - Duplicates1[0].Dict.Values.Sort(DuplicateSortCompare); - J := Duplicates1[0].Dict.Count; - WriteBuffer(J, J.Size); - for I := 0 to J - 1 do + UI32 := 0; + for I := Low(DDList1) to High(DDList1) do begin - DupTyp := PDuplicate(Duplicates1[0].Dict.Values.ElemPtr(I))^; - WriteBuffer(DupTyp, SizeOf(TDuplicate)); + J := Int64Rec(DDList1[I]).Words[0]; + X := Int64Rec(DDList1[I]).Hi; + if DDInfo[J, X].Count > 0 then + begin + DupTyp.Index := DDInfo[J, X].Index; + DupTyp.Count := DDInfo[J, X].Count; + WriteBuffer(DupTyp, SizeOf(TDuplicate2)); + Inc(UI32); + end; end; end; - if DupFile <> '' then - begin - with TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + DupFile, - FSMode(FileExists(ExtractFilePath(Utils.GetModuleName) + DupFile))) do - begin - Position := Size; - WriteBuffer(WorkStream[0].Memory^, WorkStream[0].Position); - end; - end - else - Output.WriteBuffer(WorkStream[0].Memory^, WorkStream[0].Position); - end; - if (DupFile = '') and StoreDD then - begin + Output.WriteBuffer(UI32, UI32.Size); + Output.WriteBuffer(WorkStream[0].Memory^, WorkStream[0].Position); Output.CopyFrom(TempOutput, 0); TempOutput.Free; end; @@ -1633,10 +1751,9 @@ var NStream: TArrayStream; DataMgr: TDataManager; ComVars2: TArray; - Duplicates2: TSynDictionary; - DupIdx1: Integer; - DupIdx2: TArray; - DupBool: TArray; + DDList2: TArray; + DDCount2: Integer; + DDIndex1, DDIndex2: Integer; BlockPos: Int64; procedure PrecompOutput2(Instance: Integer; const Buffer: Pointer; @@ -1644,8 +1761,9 @@ procedure PrecompOutput2(Instance: Integer; const Buffer: Pointer; begin with ComVars2[CurDepth[Instance]] do DecOutput[Instance].WriteBuffer(Buffer^, Size); - if (CurDepth[Instance] = 0) and (DupBool[Instance]) then - DataMgr.Write(DupIdx2[Instance], Buffer^, Size); + if StoreDD and (CurDepth[Instance] = 0) then + if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index)) then + DataMgr.Write(DDIndex1, Buffer, Size); end; procedure PrecompOutput3(Instance: Integer; const Buffer: Pointer; @@ -1653,8 +1771,6 @@ procedure PrecompOutput3(Instance: Integer; const Buffer: Pointer; begin with ComVars2[CurDepth[Instance]] do MemOutput1[Instance].WriteBuffer(Buffer^, Size); - if (CurDepth[Instance] = 0) and (DupBool[Instance]) then - DataMgr.Write(DupIdx2[Instance], Buffer^, Size); end; procedure Restore(MT: Boolean; Index, Depth: Integer); @@ -1675,18 +1791,11 @@ begin while X < StreamCount[Index]^ do begin SH := PStreamHeader(MemStream1[Index].Memory) + X; - if (Depth = 0) then - begin - DupIdx2[Index] := DupIdx1 + X; - DupBool[Index] := Duplicates2.FindAndCopy(DupIdx2[Index], Y); - if DupBool[Index] then - DataMgr.Add(DupIdx2[Index], SH^.OldSize, Y); - end; if MT then begin LOutput := @PrecompOutput3; Pos := StreamInfo[Index]^.Pos[X]; - X64 := Pos + SH^.NewSize; + X64 := Pos + Max(SH^.OldSize, SH^.NewSize); while (BlockPos < X64) do begin if IsErrored(Tasks) or (BlockPos < 0) then @@ -1697,6 +1806,13 @@ begin end else begin + if StoreDD and (Depth = 0) then + begin + Inc(DDIndex1); + if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index)) + then + DataMgr.Add(DDIndex1, SH^.OldSize, DDList2[DDIndex2].Count); + end; LOutput := @PrecompOutput2; DecInput[Index].ReadBuffer(UI32, UI32.Size); if UI32 > 0 then @@ -1740,9 +1856,12 @@ begin end; CurCodec[Index] := SH^.Codec; CurDepth[Index] := Depth; + Y := GetBits(SI.Option, 0, 5); + if not InRange(Y, 0, Pred(Length(Codecs[SH^.Codec].Names))) then + Y := 0; if (Codecs[SH^.Codec].Restore(Index, Depth, Ptr1, Ptr2, SI, LOutput, @PrecompFunctions) = False) then - raise Exception.CreateFmt(SPrecompError3, [Codecs[SH^.Codec].Names[0]]); + raise Exception.CreateFmt(SPrecompError3, [Codecs[SH^.Codec].Names[Y]]); NStream.Update(0, CalcSysMem); if MT then begin @@ -1751,7 +1870,13 @@ begin StreamInfo[Index]^.Completed[X] := True; end else + begin + if StoreDD and (Depth = 0) then + if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index)) + then + Inc(DDIndex2); Inc(Pos, SH^.NewSize); + end; X := AtomicIncrement(StreamIdx[Index]^); end; end; @@ -1772,24 +1897,16 @@ var I, J: Integer; Bytes: TBytes; UI32: UInt32; - DupTyp: TDuplicate; - LStream: TStream; - LGUID: TGUID; - LResData: PResData; + DupTyp: TDuplicate1; + LResData: TResData; begin GlobalSync := TCriticalSection.Create; SetLength(ThreadSync, Options^.Threads); for I := Low(ThreadSync) to High(ThreadSync) do ThreadSync[I] := TCriticalSection.Create; DupSysMem := Options^.DedupSysMem; - NStream.Add(TypeInfo(TMemoryStream), CalcSysMem); - NStream.Add(TypeInfo(TPrecompVMStream)); - Duplicates2 := TSynDictionary.Create(TypeInfo(TIntegerDynArray), - TypeInfo(TIntegerDynArray)); - DupIdx1 := 0; - SetLength(DupIdx2, Options^.Threads); - SetLength(DupBool, Options^.Threads); - Input.ReadBuffer(DupGUID, SizeOf(TGUID)); + NStream.Add(TypeInfo(TMemoryStream) { , CalcSysMem } ); + // NStream.Add(TypeInfo(TPrecompVMStream)); Input.ReadBuffer(Options^.Depth, Options^.Depth.Size); Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size); SetLength(Bytes, LongRec(I).Bytes[0]); @@ -1798,15 +1915,14 @@ begin Input.ReadBuffer(I, I.Size); for J := 0 to I - 1 do begin - New(LResData); Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size); SetLength(Bytes, LongRec(I).Bytes[0]); Input.ReadBuffer(Bytes[0], LongRec(I).Bytes[0]); - LResData^.Name := StringOf(Bytes); - Input.ReadBuffer(LResData^.Size, LResData^.Size.Size); - GetMem(LResData^.Data, LResData^.Size); - Input.ReadBuffer(LResData^.Data^, LResData^.Size); - Insert(LResData^, Resources, Length(Resources)); + LResData.Name := StringOf(Bytes); + Input.ReadBuffer(LResData.Size, LResData.Size.Size); + GetMem(LResData.Data, LResData.Size); + Input.ReadBuffer(LResData.Data^, LResData.Size); + Insert(LResData, Resources, Length(Resources)); end; SetLength(Tasks, Options^.Threads); SetLength(CurCodec, Options^.Threads); @@ -1860,45 +1976,18 @@ begin end; end; Input.ReadBuffer(StoreDD, StoreDD.Size); - if StoreDD or FileExists(ExtractFilePath(Utils.GetModuleName) + - Options^.DedupFile) then + UI32 := 0; + if StoreDD then begin - if StoreDD then - LStream := Input - else - begin - LStream := TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + - Options^.DedupFile, fmShareDenyNone); - LStream.Position := 0; - end; - while True do - begin - LStream.ReadBuffer(LGUID, SizeOf(TGUID)); - LStream.ReadBuffer(J, J.Size); - I := J * SizeOf(TDuplicate); - if CompareMem(@DupGUID, @LGUID, SizeOf(TGUID)) then - begin - if WorkStream[0].Size < I then - WorkStream[0].Size := I; - LStream.ReadBuffer(WorkStream[0].Memory^, I); - for I := 0 to J - 1 do - begin - DupTyp := (PDuplicate(WorkStream[0].Memory) + I)^; - Duplicates2.Add(DupTyp.Index, DupTyp.Count); - end; - break; - end - else if StoreDD then - raise EReadError.CreateRes(@SInvalidProperty) - else - LStream.Seek(I, TSeekOrigin.soCurrent); - if StoreDD or (LStream.Position >= LStream.Size) then - break; - end; - if not StoreDD then - LStream.Free; + Input.ReadBuffer(UI32, UI32.Size); + SetLength(DDList2, UI32); + DDCount2 := UI32; + for I := Low(DDList2) to High(DDList2) do + Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2)); + DDIndex1 := -1; + DDIndex2 := 0; end; - DataMgr := TDataManager.Create(NStream, Duplicates2.Count); + DataMgr := TDataManager.Create(NStream); end; procedure DecFree; @@ -1933,7 +2022,6 @@ begin WorkStream[I].Free; end; DataMgr.Free; - Duplicates2.Free; FreeResources; GlobalSync.Free; for I := Low(ThreadSync) to High(ThreadSync) do @@ -1981,8 +2069,17 @@ begin Inc(CurrPos, Max(StreamHeader^.OldSize, StreamHeader^.NewSize)); end; end; - if MemInput[Index].Size < BlockSize then - MemInput[Index].Size := BlockSize; + if (Depth = 0) and (Length(Tasks) > 1) and (StreamCount[Index]^ > 1) + then + begin + if MemInput[Index].Size < CurrPos then + MemInput[Index].Size := CurrPos; + end + else + begin + if MemInput[Index].Size < BlockSize then + MemInput[Index].Size := BlockSize; + end; MemInput[Index].Position := 0; StreamIdx[Index]^ := -1; if (Depth = 0) and (Length(Tasks) > 1) and (StreamCount[Index]^ > 1) @@ -1996,9 +2093,6 @@ begin for J := 0 to StreamCount[Index]^ - 1 do begin StreamHeader := PStreamHeader(MemStream1[Index].Memory) + J; - MemInput[Index].Size := Max(MemInput[Index].Size, - StreamInfo[Index]^.Pos[J] + Max(StreamHeader^.OldSize, - StreamHeader^.NewSize)); MemInput[Index].Position := StreamInfo[Index]^.Pos[J]; if CopyStream(DecInput[Index], MemInput[Index], StreamHeader^.NewSize) <> StreamHeader^.NewSize then @@ -2026,12 +2120,26 @@ begin if IsErrored(Tasks) then for I := Low(Tasks) to High(Tasks) do Tasks[I].RaiseLastError; + if StoreDD and (Depth = 0) then + begin + Inc(DDIndex1); + if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index)) + then + begin + DataMgr.Add(DDIndex1, StreamHeader^.OldSize, + DDList2[DDIndex2].Count); + DataMgr.Write(DDIndex1, + (PByte(MemInput[Index].Memory) + StreamInfo[Index]^.Pos[J]), + StreamHeader^.OldSize); + Inc(DDIndex2); + end; + end; if StreamHeader^.Kind and DUPLICATED_STREAM = DUPLICATED_STREAM then DataMgr.CopyData(StreamHeader^.Option, DecOutput[Index]) else DecOutput[Index].WriteBuffer ((PByte(MemInput[Index].Memory) + StreamInfo[Index]^.Pos[J])^, - (PStreamHeader(MemStream1[Index].Memory) + J)^.OldSize); + StreamHeader^.OldSize); end; WaitForAll(Tasks); end @@ -2041,8 +2149,6 @@ begin DecInput[Index].ReadBuffer(UI32, UI32.Size); if UI32 > 0 then CopyStreamEx(DecInput[Index], DecOutput[Index], UI32); - if Depth = 0 then - Inc(DupIdx1, StreamCount[Index]^); DecInput[Index].ReadBuffer(StreamCount[Index]^, StreamCount[Index]^.Size); end; end; diff --git a/precompressor/PrecompMedia.pas b/precompressor/PrecompMedia.pas new file mode 100644 index 0000000..5c7c616 --- /dev/null +++ b/precompressor/PrecompMedia.pas @@ -0,0 +1,1005 @@ +unit PrecompMedia; + +{$POINTERMATH ON} + +interface + +uses + BrunsliDLL, FLACDLL, PackJPGDLL, JoJpegDLL, XDeltaDLL, + Utils, + PrecompUtils, + System.SysUtils, System.Classes, System.Math; + +var + Codec: TPrecompressor; + +implementation + +const + MediaCodecs: array of PChar = ['flac', 'packjpg', 'brunsli', 'jojpeg']; + CODEC_COUNT = 4; + FLAC_CODEC = 0; + PACKJPG_CODEC = 1; + BRUNSLI_CODEC = 2; + JOJPEG_CODEC = 3; + +const + FLAC_LEVEL = 5; + J_WORKMEM = 262144; + +type + PFlacEncCD = ^TFlacEncCD; + + TFlacEncCD = record + Output: TMemoryStreamEx; + Error: Boolean; + end; + + PFlacDecCD = ^TFlacDecCD; + + TFlacDecCD = record + Input, Output: TMemoryStreamEx; + end; + +var + cctx, dctx: array of Pointer; + ccd: array of TFlacEncCD; + dcd: array of TFlacDecCD; + JJInst: TArray; + FlacLevel: Integer = FLAC_LEVEL; + CodecAvailable, CodecEnabled: TArray; + RestoreOut: TArray<_PrecompOutput>; + RestoreSize: TArray; + +const + RIFF_SIGN = $46464952; + WAVE_SIGN = $45564157; + fmt_SIGN = $20746D66; + data_SIGN = $61746164; + + WAVE_FORMAT_PCM = 1; + WAVE_FORMAT_EXTENSIBLE = $FFFE; + +type + PRIFF_hdr = ^TRIFF_hdr; + + TRIFF_hdr = packed record + chunk_id, chunk_size, form_type: Cardinal; + end; + + PWAVE_hdr = ^TWAVE_hdr; + + TWAVE_hdr = packed record + audio_format, num_channels: Word; + sample_rate, byte_rate: Cardinal; + block_align, bits_per_sample: Word; + end; + + PWAVE_subchunk_hdr = ^TWAVE_subchunk_hdr; + + TWAVE_subchunk_hdr = packed record + subchunk_id, subchunk_size: Cardinal; + end; + + PWAVE_subformat = ^TWAVE_subformat; + + TWAVE_subformat = packed record + f1: Cardinal; + f2, f3: Word; + f4: array [0 .. 7] of Byte; + end; + + PWAVE_ext_hdr = ^TWAVE_ext_hdr; + + TWAVE_ext_hdr = packed record + cb_size, valid_bits: Word; + ch_mask: Cardinal; + est: TWAVE_subformat; + end; + +function GetWAVEInfo(InBuff: PByte; InSize: NativeInt; wave_hdr: PWAVE_hdr; + chunk_size, header_size: PCardinal): Boolean; +var + Pos, LastPos: NativeInt; + riffhdr: TRIFF_hdr; + subchunk_hdr: TWAVE_subchunk_hdr; + wave_hdr_ex: TWAVE_ext_hdr; +begin + Result := False; + Pos := 0; + if InSize - Pos < SizeOf(TRIFF_hdr) then + exit; + Move((InBuff + Pos)^, riffhdr, SizeOf(TRIFF_hdr)); + Inc(Pos, SizeOf(TRIFF_hdr)); + if (riffhdr.chunk_id <> RIFF_SIGN) or (riffhdr.form_type <> WAVE_SIGN) then + exit; + while True do + begin + if InSize - Pos < SizeOf(TWAVE_subchunk_hdr) then + exit; + Move((InBuff + Pos)^, subchunk_hdr, SizeOf(TWAVE_subchunk_hdr)); + Inc(Pos, SizeOf(TWAVE_subchunk_hdr)); + if subchunk_hdr.subchunk_id = fmt_SIGN then + begin + LastPos := Pos; + if InSize - Pos < SizeOf(TWAVE_hdr) then + exit; + Move((InBuff + Pos)^, wave_hdr^, SizeOf(TWAVE_hdr)); + Inc(Pos, SizeOf(TWAVE_hdr)); + if wave_hdr^.audio_format = WAVE_FORMAT_EXTENSIBLE then + begin + if InSize - Pos < SizeOf(TWAVE_ext_hdr) then + exit; + Move((InBuff + Pos)^, wave_hdr_ex, SizeOf(TWAVE_ext_hdr)); + Inc(Pos, SizeOf(TWAVE_ext_hdr)); + wave_hdr^.audio_format := wave_hdr_ex.est.f1; + end + else if wave_hdr^.audio_format <> WAVE_FORMAT_PCM then + exit; + Pos := LastPos + subchunk_hdr.subchunk_size; + end + else if subchunk_hdr.subchunk_id = data_SIGN then + begin + Result := True; + chunk_size^ := subchunk_hdr.subchunk_size; + header_size^ := Pos; + exit; + end + else + Inc(Pos, subchunk_hdr.subchunk_size); + end; +end; + +function FlacEncoderWriteCB(encoder: Pointer; const Buffer; bytes: NativeUInt; + samples, current_frame: Cardinal; client_data: PFlacEncCD): Integer cdecl; +begin + try + client_data^.Output.WriteBuffer(Buffer, bytes); + except + client_data^.Error := True; + end; + Result := 0; +end; + +function FlacDecoderReadCB(decoder: Pointer; var Buffer; bytes: PNativeUInt; + client_data: PFlacDecCD): Integer cdecl; +begin + bytes^ := client_data^.Input.Read(Buffer, bytes^); + if bytes^ > 0 then + Result := 0 + else + Result := 1; +end; + +function FlacDecoderWriteCB(decoder: Pointer; frame: PFLAC__Frame; + Buffer: Pointer; client_data: PFlacDecCD): Integer cdecl; +var + I, J, X: Integer; +begin + for I := 0 to frame^.header.blocksize - 1 do + for J := 0 to frame^.header.channels - 1 do + begin + X := PInteger(PNativeInt(Buffer)[J])[I]; + client_data^.Output.WriteBuffer(X, frame^.header.bits_per_sample div 8); + end; + Result := 0; +end; + +procedure FlacDecoderErrorCB(decoder: Pointer; Status: Integer; + client_data: Pointer)cdecl; +begin + +end; + +function FlacEncode(ctx: Pointer; cd: PFlacEncCD; InBuff: PByte; + InSize: Integer; OutBuff: PByte; OutSize: Integer; + Level: Integer = 5): Integer; +const + READSIZE = 1000; +var + I, J, X: Integer; + Ptr: PByte; + wave_hdr: TWAVE_hdr; + byte_per_sample: Word; + data_size, hdr_size, smp_size: Cardinal; + ok: Boolean; + pcm: TArray; +begin + Result := 0; + ok := False; + if GetWAVEInfo(InBuff, InSize, @wave_hdr, @data_size, @hdr_size) then + begin + ok := True; + byte_per_sample := (wave_hdr.bits_per_sample + 7) div 8; + smp_size := wave_hdr.num_channels * byte_per_sample; + ok := ok and FLAC__stream_encoder_set_verify(ctx, True); + ok := ok and FLAC__stream_encoder_set_compression_level(ctx, Level); + ok := ok and FLAC__stream_encoder_set_channels(ctx, wave_hdr.num_channels); + ok := ok and FLAC__stream_encoder_set_bits_per_sample(ctx, + wave_hdr.bits_per_sample); + ok := ok and FLAC__stream_encoder_set_sample_rate(ctx, + wave_hdr.sample_rate); + end; + if ok then + try + cd^.Error := False; + cd^.Output.Update(OutBuff, OutSize); + cd^.Output.Size := 0; + if FLAC__stream_encoder_init_stream(ctx, @FlacEncoderWriteCB, nil, nil, + nil, cd) = 0 then + begin + Ptr := InBuff + hdr_size; + SetLength(pcm, READSIZE * wave_hdr.num_channels); + while True do + begin + X := 0; + for I := 0 to READSIZE - 1 do + begin + if NativeInt(Ptr + smp_size - InBuff) > hdr_size + data_size then + break; + Inc(X); + for J := 0 to wave_hdr.num_channels - 1 do + begin + if NativeInt(Ptr - InBuff) > hdr_size + data_size then + break; + pcm[I * wave_hdr.num_channels + J] := + GetBits(PInteger(Ptr)^, 0, Pred(wave_hdr.bits_per_sample)) - + IfThen(GetBits(PInteger(Ptr)^, Pred(wave_hdr.bits_per_sample), + 1) = 0, 0, 1 shl Pred(wave_hdr.bits_per_sample)); + Inc(Ptr, byte_per_sample); + end; + end; + ok := ok and FLAC__stream_encoder_process_interleaved(ctx, pcm[0], X); + if (NativeInt(Ptr + smp_size - InBuff) > hdr_size + data_size) or + (ok = False) then + break; + end; + end; + finally + FLAC__stream_encoder_finish(ctx); + if ok and (cd^.Error = False) then + Result := cd^.Output.Size; + end; +end; + +function FlacDecode(ctx: Pointer; cd: PFlacDecCD; InBuff: PByte; + InSize: Integer; OutBuff: PByte; OutSize: Integer): Integer; +var + ok: Boolean; +begin + Result := 0; + ok := False; + cd^.Input.Update(InBuff, InSize); + cd^.Input.Position := 0; + cd^.Input.Size := InSize; + cd^.Output.Update(OutBuff, OutSize); + cd^.Output.Size := 0; + try + if FLAC__stream_decoder_init_stream(ctx, @FlacDecoderReadCB, nil, nil, nil, + nil, @FlacDecoderWriteCB, nil, @FlacDecoderErrorCB, cd) = 0 then + begin + ok := True; + ok := ok and FLAC__stream_decoder_process_until_end_of_stream(ctx); + end; + finally + FLAC__stream_decoder_finish(ctx); + if ok then + Result := cd^.Output.Size; + end; +end; + +const + JPG_HEADER = $D8FF; + JPG_FOOTER = $D9FF; + +function GetJPEGInfo(InBuff: PByte; InSize: NativeInt; ImageSize: PCardinal; + IsProgressive: PBoolean): Boolean; +// https://github.com/schnaader/precomp-cpp/blob/master/precomp.cpp +var + Pos: NativeInt; + hasQuantTable: Boolean; + done, found: Boolean; + length: Integer; + isMarker: Boolean; + bytesRead: Cardinal; +begin + Result := False; + Pos := 0; + done := False; + found := False; + if (PWord(InBuff)^ = JPG_HEADER) and ((InBuff + 2)^ = $FF) then + begin + Pos := 3; + if (InBuff + Pos)^ in [$C0, $C2, $C4, $DB .. $FE] then + begin + hasQuantTable := (InBuff + Pos)^ = $DB; + IsProgressive^ := (InBuff + Pos)^ = $C2; + Pos := 2; + repeat + if (Pos + 5 > InSize) or ((InBuff + Pos)[0] <> $FF) then + break; + length := Integer((InBuff + Pos)[2]) * 256 + Integer((InBuff + Pos)[3]); + case (InBuff + Pos)[1] of + $DB: + begin + if (length <= 262) and (((length - 2) mod 65) = 0) and + ((InBuff + Pos)[4] <= 3) then + begin + hasQuantTable := True; + Inc(Pos, length + 2); + end + else + done := True; + end; + $C4: + begin + done := (((InBuff + Pos)[4] and $F) > 3) or + (((InBuff + Pos)[4] shr 4) > 1); + Inc(Pos, length + 2); + end; + $DA: + begin + found := hasQuantTable; + Inc(Pos, length + 2); + end; + $D9: + done := True; + $C2: + begin + IsProgressive^ := True; + Inc(Pos, length + 2); + end; + $C0: + begin + done := (InBuff + Pos)[4] <> $08; + Inc(Pos, length + 2); + end; + else + Inc(Pos, length + 2); + end; + until done; + if found then + while Pos < InSize - 1 do + begin + if PWord(InBuff + Pos)^ = JPG_FOOTER then + begin + ImageSize^ := Pos + 2; + Result := True; + break; + end; + Inc(Pos); + end; + end; + end; +end; + +function BrunsliWriter(cd: Pointer; data: Pointer; Size: NativeUInt) + : Integer cdecl; +begin + RestoreOut[PInteger(cd)^](PInteger(cd)^, data, Size); + Inc(RestoreSize[PInteger(cd)^], Size); + Result := Size; +end; + +function MediaInit(Command: PChar; Count: Integer; + Funcs: PPrecompFuncs): Boolean; +var + I: Integer; + Options: TArray; + S: String; + X, Y: Integer; +begin + Result := True; + SetLength(RestoreOut, Count); + SetLength(RestoreSize, Count); + for X := Low(CodecAvailable) to High(CodecAvailable) do + begin + CodecAvailable[X] := False; + CodecEnabled[X] := False; + end; + CodecAvailable[FLAC_CODEC] := FLACDLL.DLLLoaded; + CodecAvailable[PACKJPG_CODEC] := PackJPGDLL.DLLLoaded; + CodecAvailable[BRUNSLI_CODEC] := BrunsliDLL.DLLLoaded; + CodecAvailable[JOJPEG_CODEC] := JoJpegDLL.DLLLoaded; + X := 0; + while Funcs^.GetCodec(Command, X, False) <> '' do + begin + S := Funcs^.GetCodec(Command, X, False); + if (CompareText(S, MediaCodecs[FLAC_CODEC]) = 0) and FLACDLL.DLLLoaded then + begin + CodecEnabled[FLAC_CODEC] := True; + if Funcs^.GetParam(Command, X, 'l') <> '' then + FlacLevel := StrToInt(Funcs^.GetParam(Command, X, 'l')); + end + else if (CompareText(S, MediaCodecs[PACKJPG_CODEC]) = 0) and PackJPGDLL.DLLLoaded + then + CodecEnabled[PACKJPG_CODEC] := True + else if (CompareText(S, MediaCodecs[BRUNSLI_CODEC]) = 0) and BrunsliDLL.DLLLoaded + then + CodecEnabled[BRUNSLI_CODEC] := True + else if (CompareText(S, MediaCodecs[JOJPEG_CODEC]) = 0) and JoJpegDLL.DLLLoaded + then + CodecEnabled[JOJPEG_CODEC] := True;; + Inc(X); + end; + if CodecAvailable[FLAC_CODEC] then + begin + SetLength(cctx, Count); + SetLength(dctx, Count); + SetLength(ccd, Count); + SetLength(dcd, Count); + for X := Low(cctx) to High(cctx) do + begin + cctx[X] := FLAC__stream_encoder_new; + dctx[X] := FLAC__stream_decoder_new; + ccd[X].Output := TMemoryStreamEx.Create(False); + dcd[X].Input := TMemoryStreamEx.Create(False); + dcd[X].Output := TMemoryStreamEx.Create(False); + end; + end; + if CodecAvailable[JOJPEG_CODEC] then + begin + SetLength(JJInst, Count); + for X := Low(JJInst) to High(JJInst) do + JJInst[X] := GetMemory(jojpeg_Size); + end; +end; + +procedure MediaFree(Funcs: PPrecompFuncs); +var + X: Integer; +begin + if CodecAvailable[FLAC_CODEC] then + begin + for X := Low(cctx) to High(cctx) do + begin + FLAC__stream_encoder_delete(cctx[X]); + FLAC__stream_decoder_delete(dctx[X]); + ccd[X].Output.Free; + dcd[X].Input.Free; + dcd[X].Output.Free; + end; + end; + if CodecAvailable[JOJPEG_CODEC] then + for X := Low(JJInst) to High(JJInst) do + FreeMemory(JJInst[X]); +end; + +function MediaParse(Command: PChar; Option: PInteger; + Funcs: PPrecompFuncs): Boolean; +var + S: String; + I: Integer; +begin + Result := False; + Option^ := 0; + I := 0; + while Funcs^.GetCodec(Command, I, False) <> '' do + begin + S := Funcs^.GetCodec(Command, I, False); + if (CompareText(S, MediaCodecs[FLAC_CODEC]) = 0) and FLACDLL.DLLLoaded then + begin + SetBits(Option^, FLAC_CODEC, 0, 5); + Result := True; + end + else if (CompareText(S, MediaCodecs[PACKJPG_CODEC]) = 0) and PackJPGDLL.DLLLoaded + then + begin + SetBits(Option^, PACKJPG_CODEC, 0, 5); + Result := True; + end + else if (CompareText(S, MediaCodecs[BRUNSLI_CODEC]) = 0) and BrunsliDLL.DLLLoaded + then + begin + SetBits(Option^, BRUNSLI_CODEC, 0, 5); + Result := True; + end + else if (CompareText(S, MediaCodecs[JOJPEG_CODEC]) = 0) and JoJpegDLL.DLLLoaded + then + begin + SetBits(Option^, JOJPEG_CODEC, 0, 5); + Result := True; + end; + Inc(I); + end; +end; + +procedure MediaScan1(Instance, Depth: Integer; Input: PByte; + Size, SizeEx: NativeInt; Output: _PrecompOutput; Add: _PrecompAdd; + Funcs: PPrecompFuncs); +var + Buffer: PByte; + Pos: NativeInt; + X, Y, Z: Integer; + SI: _StrInfo1; + DI1, DI2: TDepthInfo; + DS: TPrecompStr; + wave_hdr: TWAVE_hdr; + data_size, hdr_size: Cardinal; + progressive: Boolean; +begin + DI1 := Funcs^.GetDepthInfo(Instance); + DS := Funcs^.GetCodec(DI1.Codec, 0, False); + if DS <> '' then + begin + X := IndexTextW(@DS[0], MediaCodecs); + if (X < 0) or (DI1.OldSize <> SizeEx) then + exit; + if not CodecAvailable[X] then + exit; + Y := DI1.OldSize; + case X of + FLAC_CODEC: + begin + if GetWAVEInfo(Input, Y, @wave_hdr, @data_size, @hdr_size) then + Y := data_size + hdr_size + else + Y := 0; + end; + PACKJPG_CODEC, BRUNSLI_CODEC, JOJPEG_CODEC: + begin + if GetJPEGInfo(Input, Y, @data_size, @progressive) then + Y := data_size + else + Y := 0; + end; + else + Y := 0; + end; + if Y > 0 then + begin + Z := data_size; + if X = FLAC_CODEC then + begin + Buffer := Funcs^.Allocator(Instance, hdr_size); + Move(Input^, Buffer^, hdr_size); + Inc(PInteger(Buffer)^); + Output(Instance, @hdr_size, hdr_size.Size); + Output(Instance, Buffer, hdr_size); + Output(Instance, Input + hdr_size, Z); + Inc(Z, hdr_size.Size); + Inc(Z, hdr_size); + end + else + Output(Instance, Input, Z); + SI.Position := 0; + SI.OldSize := Y; + SI.NewSize := Z; + SI.Option := 0; + SetBits(SI.Option, X, 0, 5); + if System.Pos(SPrecompSep2, DI1.Codec) > 0 then + SI.Status := TStreamStatus.Predicted + else + SI.Status := TStreamStatus.None; + DS := Funcs^.GetDepthCodec(DI1.Codec); + Move(DS[0], DI2.Codec, SizeOf(DI2.Codec)); + DI2.OldSize := SI.NewSize; + DI2.NewSize := SI.NewSize; + Funcs^.LogScan1(MediaCodecs[GetBits(SI.Option, 0, 5)], SI.Position, + SI.OldSize, -1); + Add(Instance, @SI, DI1.Codec, @DI2); + end; + exit; + end; + if BoolArray(CodecEnabled, False) then + exit; + Pos := 0; + while Pos < Size do + begin + if CodecEnabled[FLAC_CODEC] and (PCardinal(Input + Pos)^ = RIFF_SIGN) then + begin + Y := SizeEx - Pos; + if GetWAVEInfo(Input + Pos, Y, @wave_hdr, @data_size, @hdr_size) then + Y := data_size + hdr_size + else + Y := 0; + if Y > 0 then + begin + Z := data_size; + Buffer := Funcs^.Allocator(Instance, hdr_size); + Move((Input + Pos)^, Buffer^, hdr_size); + Inc(PInteger(Buffer)^); + Output(Instance, @hdr_size, hdr_size.Size); + Output(Instance, Buffer, hdr_size); + Output(Instance, Input + Pos + hdr_size, Z); + Inc(Z, hdr_size.Size); + Inc(Z, hdr_size); + SI.Position := Pos; + SI.OldSize := Y; + SI.NewSize := Z; + SI.Option := 0; + SetBits(SI.Option, FLAC_CODEC, 0, 5); + SI.Status := TStreamStatus.None; + Funcs^.LogScan1(MediaCodecs[GetBits(SI.Option, 0, 5)], SI.Position, + SI.OldSize, -1); + Add(Instance, @SI, nil, nil); + Inc(Pos, SI.OldSize); + continue; + end; + end; + if (CodecEnabled[PACKJPG_CODEC] or CodecEnabled[BRUNSLI_CODEC] or + CodecEnabled[JOJPEG_CODEC]) and (PWord(Input + Pos)^ = JPG_HEADER) then + begin + Y := SizeEx - Pos; + if GetJPEGInfo(Input + Pos, Y, @data_size, @progressive) then + Y := data_size + else + Y := 0; + if Y > 0 then + begin + Z := data_size; + Output(Instance, Input + Pos, Z); + SI.Position := Pos; + SI.OldSize := Y; + SI.NewSize := Z; + SI.Option := 0; + if CodecEnabled[BRUNSLI_CODEC] then + SetBits(SI.Option, BRUNSLI_CODEC, 0, 5) + else if CodecEnabled[JOJPEG_CODEC] then + SetBits(SI.Option, JOJPEG_CODEC, 0, 5) + else if CodecEnabled[PACKJPG_CODEC] then + SetBits(SI.Option, PACKJPG_CODEC, 0, 5); + SI.Status := TStreamStatus.None; + Funcs^.LogScan1(MediaCodecs[GetBits(SI.Option, 0, 5)], SI.Position, + SI.OldSize, -1); + Add(Instance, @SI, nil, nil); + Inc(Pos, SI.OldSize); + continue; + end; + end; + Inc(Pos); + end; +end; + +function MediaScan2(Instance, Depth: Integer; Input: Pointer; Size: NativeInt; + StreamInfo: PStrInfo2; Offset: PInteger; Output: _PrecompOutput; + Funcs: PPrecompFuncs): Boolean; +var + Buffer: PByte; + X, Y, Z: Integer; + wave_hdr: TWAVE_hdr; + data_size, hdr_size: Cardinal; + progressive: Boolean; +begin + Result := False; + X := GetBits(StreamInfo^.Option, 0, 5); + if StreamInfo^.OldSize <= 0 then + exit; + Y := StreamInfo^.OldSize; + case X of + FLAC_CODEC: + begin + if GetWAVEInfo(Input, Y, @wave_hdr, @data_size, @hdr_size) then + Y := data_size + hdr_size + else + Y := 0; + end; + PACKJPG_CODEC, BRUNSLI_CODEC, JOJPEG_CODEC: + begin + if GetJPEGInfo(Input, Y, @data_size, @progressive) then + Y := data_size + else + Y := 0; + end; + else + Y := 0; + end; + if Y > 0 then + begin + Z := data_size; + if X = FLAC_CODEC then + begin + Buffer := Funcs^.Allocator(Instance, hdr_size); + Move(Input^, Buffer^, hdr_size); + Inc(PInteger(Buffer)^); + Output(Instance, @hdr_size, hdr_size.Size); + Output(Instance, Buffer, hdr_size); + Output(Instance, PByte(Input) + hdr_size, Z); + Inc(Z, hdr_size.Size); + Inc(Z, hdr_size); + end + else + Output(Instance, Input, Z); + StreamInfo^.NewSize := Z; + Funcs^.LogScan2(MediaCodecs[GetBits(StreamInfo^.Option, 0, 5)], + StreamInfo^.OldSize, -1); + Result := True; + end; +end; + +function MediaProcess(Instance, Depth: Integer; OldInput, NewInput: Pointer; + StreamInfo: PStrInfo2; Output: _PrecompOutput; Funcs: PPrecompFuncs): Boolean; +var + Buffer, Ptr: PByte; + Params: String; + ctx: Pointer; + I, J: Integer; + X, Y: Integer; + Res: Cardinal; + Res1, Res2: Integer; +begin + Result := False; + X := GetBits(StreamInfo^.Option, 0, 5); + if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then + exit; + Params := ''; + case X of + FLAC_CODEC: + begin + Params := 'l' + FlacLevel.ToString; + Y := Integer.Size + PInteger(NewInput)^; + Res := StreamInfo.NewSize - Y; + Res := FlacEncode(cctx[Instance], @ccd[Instance], OldInput, + StreamInfo^.OldSize, PByte(NewInput) + Y, Res, FlacLevel); + if (Res > 0) and (Res + Y < StreamInfo^.NewSize) then + begin + StreamInfo^.NewSize := Res + Y; + Result := True; + end; + end; + PACKJPG_CODEC: + begin + Buffer := nil; + Res := StreamInfo.OldSize; + pjglib_init_streams(OldInput, pjglib_memory, StreamInfo^.OldSize, + Buffer, pjglib_memory); + if pjglib_convert_stream2mem(@Buffer, @Res, nil) and + (Res < StreamInfo^.NewSize) then + begin + Move(Buffer^, NewInput^, Res); + StreamInfo^.NewSize := Res; + Result := True; + end; + ShowMessage(''); + end; + BRUNSLI_CODEC: + begin + ctx := brunsli_alloc_JPEGData; + try + if brunsli_ReadJpeg(ctx, OldInput, StreamInfo^.OldSize) = Integer(True) + then + begin + Res := brunsli_GetMaximumEncodedSize(ctx); + Buffer := Funcs^.Allocator(Instance, Res); + Res := brunsli_EncodeJpeg(ctx, Buffer, Res); + if Res < StreamInfo^.NewSize then + begin + Move(Buffer^, NewInput^, Res); + StreamInfo^.NewSize := Res; + Result := True; + end; + end; + finally + brunsli_free_JPEGData(ctx); + end; + end; + JOJPEG_CODEC: + begin + ctx := JJInst[Instance]; + Buffer := Funcs^.Allocator(Instance, J_WORKMEM * 2); + I := 0; + J := 0; + Y := 0; + jojpeg_Init(ctx, jojpeg_Compress); + try + jojpeg_Addbuf(ctx, jojpeg_Compress, Buffer, J_WORKMEM, + jojpeg_enc_Output1); + jojpeg_Addbuf(ctx, jojpeg_Compress, Buffer + J_WORKMEM, J_WORKMEM, + jojpeg_enc_Output2); + while True do + begin + Res1 := jojpeg_Loop(ctx, jojpeg_Compress); + if (Res1 = jojpeg_enc_Input) then + begin + Res2 := Min(StreamInfo^.OldSize - I, J_WORKMEM); + jojpeg_Addbuf(ctx, jojpeg_Compress, PByte(OldInput) + I, Res2, + jojpeg_enc_Input); + Inc(I, Res2); + end; + if (Res1 in [0, jojpeg_enc_Output1]) then + begin + Res2 := jojpeg_Getvalue(ctx, jojpeg_Compress, jojpeg_enc_Output1); + Move(Buffer^, (PByte(NewInput) + J)^, Res2); + Inc(J, Res2); + jojpeg_Addbuf(ctx, jojpeg_Compress, Buffer, J_WORKMEM, + jojpeg_enc_Output1); + end; + if (Res1 = jojpeg_enc_Output2) or (Res1 = 0) then + begin + Res2 := jojpeg_Getvalue(ctx, jojpeg_Compress, jojpeg_enc_Output2); + Output(Instance, Buffer + J_WORKMEM, Res2); + Inc(Y, Res2); + jojpeg_Addbuf(ctx, jojpeg_Compress, Buffer + J_WORKMEM, J_WORKMEM, + jojpeg_enc_Output2); + if Res1 = 0 then + break; + end; + end; + if I = StreamInfo^.OldSize then + begin + StreamInfo^.NewSize := J; + Result := True; + end; + finally + jojpeg_Quit(ctx, jojpeg_Compress); + end; + end; + end; + Funcs^.LogProcess(MediaCodecs[GetBits(StreamInfo^.Option, 0, 5)], + PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, -1, Result); +end; + +function MediaRestore(Instance, Depth: Integer; Input, InputExt: Pointer; + StreamInfo: _StrInfo3; Output: _PrecompOutput; Funcs: PPrecompFuncs): Boolean; +var + Buffer: PByte; + Params: String; + ctx, writer: Pointer; + I, J: Integer; + X, Y: Integer; + Res: Cardinal; + Res1, Res2: Integer; +begin + Result := False; + X := GetBits(StreamInfo.Option, 0, 5); + if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then + exit; + Params := ''; + case X of + FLAC_CODEC: + begin + Buffer := Funcs^.Allocator(Instance, StreamInfo.OldSize); + Y := Integer.Size + PInteger(Input)^; + Res := StreamInfo.OldSize; + Res := FlacDecode(dctx[Instance], @dcd[Instance], PByte(Input) + Y, + StreamInfo.NewSize - Y, Buffer + PInteger(Input)^, Res); + if Res > 0 then + begin + Move((PByte(Input) + Integer.Size)^, Buffer^, PInteger(Input)^); + Inc(Res, PInteger(Input)^); + Dec(PInteger(Buffer)^); + Result := True; + end; + if Result then + Output(Instance, Buffer, Res); + end; + PACKJPG_CODEC: + begin + Buffer := nil; + pjglib_init_streams(Input, pjglib_memory, StreamInfo.NewSize, Buffer, + pjglib_memory); + Res := StreamInfo.OldSize; + Result := pjglib_convert_stream2mem(@Buffer, @Res, nil); + if Result then + Output(Instance, Buffer, Res); + end; + BRUNSLI_CODEC: + begin + RestoreOut[Instance] := Output; + RestoreSize[Instance] := 0; + Res := RestoreSize[Instance]; + ctx := brunsli_alloc_JPEGData; + try + if brunsli_DecodeJpeg(ctx, Input, StreamInfo.NewSize) = BRUNSLI_OK + then + begin + writer := brunsli_alloc_JPEGOutput(BrunsliWriter, @Instance); + try + Result := Boolean(brunsli_WriteJpeg(ctx, writer)); + finally + brunsli_free_JPEGOutput(writer); + end; + end; + finally + brunsli_free_JPEGData(ctx); + end; + Res := RestoreSize[Instance]; + end; + JOJPEG_CODEC: + begin + ctx := JJInst[Instance]; + Buffer := Funcs^.Allocator(Instance, J_WORKMEM); + I := 0; + J := 0; + Y := 0; + jojpeg_Init(ctx, jojpeg_Decompress); + try + jojpeg_Addbuf(ctx, jojpeg_Decompress, Buffer, J_WORKMEM, + jojpeg_dec_Output); + while True do + begin + Res1 := jojpeg_Loop(ctx, jojpeg_Decompress); + if (Res1 = jojpeg_dec_Input1) then + begin + Res2 := Min(StreamInfo.NewSize - I, J_WORKMEM); + jojpeg_Addbuf(ctx, jojpeg_Decompress, PByte(Input) + I, Res2, + jojpeg_dec_Input1); + Inc(I, Res2); + end; + if (Res1 = jojpeg_dec_Input2) then + begin + Res2 := Min(StreamInfo.ExtSize - J, J_WORKMEM); + jojpeg_Addbuf(ctx, jojpeg_Decompress, PByte(InputExt) + J, Res2, + jojpeg_dec_Input2); + Inc(J, Res2); + end; + if (Res1 in [0, jojpeg_dec_Output]) then + begin + Res2 := jojpeg_Getvalue(ctx, jojpeg_Decompress, + jojpeg_dec_Output); + Output(Instance, Buffer, Res2); + Inc(Y, Res2); + jojpeg_Addbuf(ctx, jojpeg_Decompress, Buffer, J_WORKMEM, + jojpeg_dec_Output); + if Res1 = 0 then + break; + end; + end; + { while True do + begin + Res1 := jojpeg_Loop(ctx, jojpeg_Decompress); + if Res1 = 0 then + break; + if (Res1 = jojpeg_dec_Input1) then + begin + Res2 := Min(StreamInfo.NewSize - I, J_WORKMEM); + jojpeg_Addbuf(ctx, jojpeg_Decompress, PByte(Input) + I, Res2, + jojpeg_dec_Input1); + Inc(I, Res2); + end; + if (Res1 = jojpeg_dec_Input2) then + begin + Res2 := Min(StreamInfo.ExtSize - J, J_WORKMEM); + jojpeg_Addbuf(ctx, jojpeg_Decompress, PByte(InputExt) + J, Res2, + jojpeg_dec_Input2); + Inc(J, Res2); + end; + if (Res1 = jojpeg_dec_Output) then + begin + Output(Instance, Buffer, J_WORKMEM); + Inc(Y, J_WORKMEM); + jojpeg_Addbuf(ctx, jojpeg_Decompress, Buffer, J_WORKMEM, + jojpeg_dec_Output); + end; + end; + if (Y <> StreamInfo.OldSize) and (StreamInfo.OldSize - Y < J_WORKMEM) + then + begin + Res2 := StreamInfo.OldSize - Y; + Output(Instance, Buffer, Res2); + Inc(Y, Res2); + end; } + Res := Y; + Result := Y = StreamInfo.OldSize; + finally + jojpeg_Quit(ctx, jojpeg_Decompress); + end; + end; + end; + Funcs^.LogRestore(MediaCodecs[GetBits(StreamInfo.Option, 0, 5)], + PChar(Params), StreamInfo.NewSize, Res, -1, Result); +end; + +var + I: Integer; + +initialization + +Codec.Names := []; +for I := Low(MediaCodecs) to High(MediaCodecs) do +begin + Codec.Names := Codec.Names + [MediaCodecs[I]]; + StockMethods.Add(MediaCodecs[I]); +end; +Codec.Initialised := False; +Codec.Init := @MediaInit; +Codec.Free := @MediaFree; +Codec.Parse := @MediaParse; +Codec.Scan1 := @MediaScan1; +Codec.Scan2 := @MediaScan2; +Codec.Process := @MediaProcess; +Codec.Restore := @MediaRestore; +SetLength(CodecAvailable, length(Codec.Names)); +SetLength(CodecEnabled, length(Codec.Names)); + +end. diff --git a/precompressor/PrecompOodle.pas b/precompressor/PrecompOodle.pas index 53710e3..7950c03 100644 --- a/precompressor/PrecompOodle.pas +++ b/precompressor/PrecompOodle.pas @@ -681,8 +681,8 @@ begin if CustomLZ_Decompress0(Input, Buffer, StreamInfo^.OldSize, OodleSI.DSize, Res) then begin - StreamInfo^.NewSize := Res; Output(Instance, Buffer, Res); + StreamInfo^.NewSize := Res; Funcs^.LogScan2(OodleCodecs[GetBits(StreamInfo^.Option, 0, 5)], StreamInfo^.OldSize, StreamInfo^.NewSize); Result := True; @@ -730,7 +730,7 @@ begin break; end; if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or - (SOList[Instance][X].Count = 1)) then + (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then begin Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1)); Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1, diff --git a/precompressor/PrecompUtils.pas b/precompressor/PrecompUtils.pas index 72a4e6e..377e612 100644 --- a/precompressor/PrecompUtils.pas +++ b/precompressor/PrecompUtils.pas @@ -18,7 +18,7 @@ resourcestring SPrecompSep4 = '/'; const - SuccessStatus = 3; + SuccessStatus = 4; DEFAULT_STREAM = 0; EXTENDED_STREAM = 1; @@ -32,7 +32,7 @@ type TPrecompStr = array [0 .. 255] of Char; - TStreamStatus = (None, Invalid, Predicted); + TStreamStatus = (None, Invalid, Predicted, Database); PDepthInfo = ^TDepthInfo; @@ -182,10 +182,10 @@ type LogScan1: procedure(Codec: PChar; Position: Int64; InSize, OutSize: Integer)cdecl; LogScan2: procedure(Codec: PChar; InSize, OutSize: Integer)cdecl; // 35 - LogProcess: procedure(Codec, Method: PChar; - OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl; - LogRestore: procedure(Codec, Method: PChar; - OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl; + LogProcess: procedure(Codec, Method: PChar; Size1, Size2, Size3: Integer; + Status: Boolean)cdecl; + LogRestore: procedure(Codec, Method: PChar; Size1, Size2, Size3: Integer; + Status: Boolean)cdecl; LogPatch1: procedure(OldSize, NewSize, PatchSize: Integer; Status: Boolean)cdecl; LogPatch2: procedure(OldSize, NewSize, PatchSize: Integer; @@ -243,22 +243,29 @@ type PDatabase = ^TDatabase; - TDatabase = record + TDatabase = packed record + Size: Integer; Codec: Byte; - Status: TStreamStatus; Option: Integer; + Checksum: Cardinal; + Status: TStreamStatus; end; - TDatabaseDynArray = TArray; + PDuplicate1 = ^TDuplicate1; - PDuplicate = ^TDuplicate; - - TDuplicate = record + TDuplicate1 = packed record + Size: Integer; + Checksum: Cardinal; Index: Integer; Count: Integer; end; - TDuplicateDynArray = TArray; + PDuplicate2 = ^TDuplicate2; + + TDuplicate2 = packed record + Index: Integer; + Count: Integer; + end; TPrecompVMStream = class(TStream) private const @@ -288,8 +295,6 @@ type Size: Integer; end; -function DuplicateSortCompare(const Left, Right): Integer; - procedure AddMethod(Method: String); procedure ClearMethods; @@ -358,7 +363,6 @@ function PrecompAcceptPatch(OldSize, NewSize, PatchSize: Integer) var PrecompFunctions: _PrecompFuncs; DIFF_TOLERANCE: Single = 0.05; - VERBOSE: Boolean = False; EncodeSICmp: TEncodeSIComparer; FutureSICmp: TFutureSIComparer; StockMethods, ExternalMethods: TStringList; @@ -380,11 +384,6 @@ begin Result := Integer(CompareValue(Left.Position, Right.Position)); end; -function DuplicateSortCompare(const Left, Right): Integer; -begin - Result := TDuplicate(Left).Index - TDuplicate(Right).Index; -end; - procedure AddMethod(Method: String); begin if (StockMethods.IndexOf(Method) < 0) and (ExternalMethods.IndexOf(Method) < 0) @@ -856,7 +855,7 @@ var begin Result := 0; if xd3_encode(OldBuff, OldSize, NewBuff, NewSize, PatchBuff, @Res, PatchSize, - 0) = 0 then + Integer(XD3_NOCOMPRESS)) = 0 then Result := Res; // MakeDiff(OldBuff, NewBuff, PatchBuff, OldSize, NewSize, Result); end; @@ -869,7 +868,7 @@ var begin Result := 0; if xd3_decode(PatchBuff, PatchSize, OldBuff, OldSize, NewBuff, @Res, NewSize, - 0) = 0 then + Integer(XD3_NOCOMPRESS)) = 0 then Result := Res; // MakePatch(OldBuff, PatchBuff, NewBuff, OldSize, PatchSize, Result); end; diff --git a/precompressor/PrecompZLib.pas b/precompressor/PrecompZLib.pas index 82b44ce..5ae50ee 100644 --- a/precompressor/PrecompZLib.pas +++ b/precompressor/PrecompZLib.pas @@ -255,7 +255,8 @@ begin CodecAvailable[ZLIB_CODEC] := ZLibDLL.DLLLoaded; CodecAvailable[REFLATE_CODEC] := ReflateDLL.DLLLoaded; CodecAvailable[PREFLATE_CODEC] := PreflateDLL.DLLLoaded; - CodecAvailable[PNG_CODEC] := True; + CodecAvailable[PNG_CODEC] := ZLibDLL.DLLLoaded or ReflateDLL.DLLLoaded or + PreflateDLL.DLLLoaded; X := 0; while Funcs^.GetCodec(Command, X, False) <> '' do begin @@ -433,12 +434,13 @@ var begin DI1 := Funcs^.GetDepthInfo(Instance); DS := Funcs^.GetCodec(DI1.Codec, 0, False); + X := -1; if DS <> '' then begin X := IndexTextW(@DS[0], ZlibCodecs); if (X < 0) or (DI1.OldSize <> SizeEx) then exit; - if CodecAvailable[X] then + if not CodecAvailable[X] then exit; end else if BoolArray(CodecEnabled, False) then @@ -567,7 +569,7 @@ begin if (I = ZLIB_CODEC) and (WinBits = 0) then SetBits(SI.Option, 1, 12, 3); SetBits(SI.Option, I, 0, 5); - if CodecEnabled[I] then + if CodecEnabled[I] or (I = X) then begin DS := Funcs^.GetDepthCodec(DI1.Codec); Move(DS[0], DI2.Codec, SizeOf(DI2.Codec)); @@ -632,18 +634,14 @@ function ZlibProcess(Instance, Depth: Integer; OldInput, NewInput: Pointer; function IsValidLevel(CLevel, ZLevel: Integer): Boolean; begin + Result := False; case CLevel of 1, 6: - if CLevel = ZLevel then - Result := True; + Result := CLevel = ZLevel; 2 .. 5: - if ZLevel = 5 then - Result := True; + Result := ZLevel = 5; 7 .. 9: - if ZLevel = 9 then - Result := True; - else - Result := False; + Result := ZLevel = 9; end; end; @@ -664,6 +662,7 @@ begin if not X in [PNG_CODEC] then if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then exit; + Params := ''; case X of ZLIB_CODEC: begin @@ -673,7 +672,7 @@ begin begin L := I div 10; M := I mod 10; - if StreamInfo^.Status = TStreamStatus.Predicted then + if StreamInfo^.Status >= TStreamStatus.Predicted then begin if InRange(GetBits(StreamInfo^.Option, 5, 7), 1, 9) then begin @@ -684,36 +683,40 @@ begin begin if GetBits(StreamInfo^.Option, 5, 7) <> I then continue; - { I := GetBits(StreamInfo^.Option, 5, 7); - SOList[Instance][ZLIB_CODEC].Add(I); + if StreamInfo^.Status = TStreamStatus.Database then Result := True; - break; } end; end; Params := 'l' + I.ToString + ':' + 'w' + (GetBits(StreamInfo^.Option, 12, 3) + 8).ToString; ZStream := @ZStream1[Instance, L, M, GetBits(StreamInfo^.Option, 12, 3)]; - ZStream^.next_in := NewInput; - ZStream^.avail_in := StreamInfo^.NewSize; - deflateReset(ZStream^); - repeat - ZStream^.next_out := Buffer; - ZStream^.avail_out := Z_BLKSIZE; - Res1 := deflate(ZStream^, Z_FINISH); - if Res1 < 0 then - raise EZCompressionError.Create(string(_z_errmsg[2 - Res1])) - at ReturnAddress; - Res2 := Z_BLKSIZE - ZStream^.avail_out; - Verified := CompareMem(PByte(OldInput) + ZStream^.total_out - Res2, - Buffer, Res2); - if not Verified then - break; - until (ZStream^.avail_in = 0) and (ZStream^.avail_out > 0); + if not Result then + begin + ZStream^.next_in := NewInput; + ZStream^.avail_in := StreamInfo^.NewSize; + deflateReset(ZStream^); + repeat + ZStream^.next_out := Buffer; + ZStream^.avail_out := Z_BLKSIZE; + Res1 := deflate(ZStream^, Z_FINISH); + if Res1 < 0 then + raise EZCompressionError.Create(string(_z_errmsg[2 - Res1])) + at ReturnAddress; + Res2 := Z_BLKSIZE - ZStream^.avail_out; + Verified := CompareMem(PByte(OldInput) + ZStream^.total_out - + Res2, Buffer, Res2); + if not Verified then + break; + until (ZStream^.avail_in = 0) and (ZStream^.avail_out > 0); + end + else + ZStream.total_out := StreamInfo^.OldSize; Funcs^.LogProcess(ZlibCodecs[GetBits(StreamInfo^.Option, 0, 5)], PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, - ZStream^.total_out, Verified and (Res1 = Z_STREAM_END)); - if Verified and (Res1 = Z_STREAM_END) then + ZStream^.total_out, (Result = True) or + (Verified and (Res1 = Z_STREAM_END))); + if (Result = True) or (Verified and (Res1 = Z_STREAM_END)) then begin SetBits(StreamInfo^.Option, I, 5, 7); SOList[Instance][ZLIB_CODEC].Add(I); @@ -739,7 +742,7 @@ begin Buffer := Funcs^.Allocator(Instance, R_WORKMEM * 2); J := 0; HR := RefInst1[Instance]; - if StreamInfo^.Status = TStreamStatus.Predicted then + if StreamInfo^.Status >= TStreamStatus.Predicted then L := GetBits(StreamInfo^.Option, 5, 7) else L := RLevel; @@ -804,7 +807,6 @@ begin PNG_CODEC: begin Buffer := Funcs^.Allocator(Instance, StreamInfo^.OldSize); - Params := ''; if DecodePNG(NewInput, Buffer, StreamInfo^.OldSize) then Result := CompareMem(OldInput, Buffer, StreamInfo^.OldSize); Funcs^.LogProcess(ZlibCodecs[GetBits(StreamInfo^.Option, 0, 5)], @@ -831,6 +833,7 @@ begin if not X in [PNG_CODEC] then if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then exit; + Params := ''; case X of ZLIB_CODEC: begin @@ -915,7 +918,6 @@ begin PNG_CODEC: begin Buffer := Funcs^.Allocator(Instance, StreamInfo.OldSize); - Params := ''; if DecodePNG(Input, Buffer, StreamInfo.OldSize) then begin Output(Instance, Buffer, StreamInfo.OldSize); diff --git a/precompressor/PrecompZSTD.pas b/precompressor/PrecompZSTD.pas index 8615493..c4bd9d1 100644 --- a/precompressor/PrecompZSTD.pas +++ b/precompressor/PrecompZSTD.pas @@ -233,8 +233,8 @@ begin end; if Res > StreamInfo^.OldSize then begin - StreamInfo^.NewSize := Res; Output(Instance, Buffer, Res); + StreamInfo^.NewSize := Res; Funcs^.LogScan2(ZSTDCodecs[GetBits(StreamInfo^.Option, 0, 5)], StreamInfo^.OldSize, StreamInfo^.NewSize); Result := True; @@ -265,6 +265,7 @@ begin if StreamInfo^.Status = TStreamStatus.Predicted then if GetBits(StreamInfo^.Option, 5, 7) <> I then continue; + Params := ''; case X of ZSTD_CODEC: begin @@ -307,7 +308,7 @@ begin if Res1 < 0 then exit; if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or - (SOList[Instance][X].Count = 1)) then + (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then begin Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1)); Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1, diff --git a/resources/Win32/fast-lzma2.dll b/resources/Win32/fast-lzma2.dll new file mode 100644 index 0000000..c166eb4 Binary files /dev/null and b/resources/Win32/fast-lzma2.dll differ diff --git a/resources/Win64/fast-lzma2.dll b/resources/Win64/fast-lzma2.dll new file mode 100644 index 0000000..397329f Binary files /dev/null and b/resources/Win64/fast-lzma2.dll differ diff --git a/xtool.dpr b/xtool.dpr index bbe9006..9dc958e 100644 --- a/xtool.dpr +++ b/xtool.dpr @@ -52,9 +52,13 @@ uses oObjects in 'contrib\ParseExpression\oObjects.pas', ParseClass in 'contrib\ParseExpression\ParseClass.pas', ParseExpr in 'contrib\ParseExpression\ParseExpr.pas', + BrunsliDLL in 'imports\BrunsliDLL.pas', + FLACDLL in 'imports\FLACDLL.pas', + JoJpegDLL in 'imports\JoJpegDLL.pas', LZ4DLL in 'imports\LZ4DLL.pas', LZODLL in 'imports\LZODLL.pas', OodleDLL in 'imports\OodleDLL.pas', + PackJPGDLL in 'imports\PackJPGDLL.pas', PreflateDLL in 'imports\PreflateDLL.pas', ReflateDLL in 'imports\ReflateDLL.pas', XDeltaDLL in 'imports\XDeltaDLL.pas', @@ -68,6 +72,7 @@ uses PrecompLZ4 in 'precompressor\PrecompLZ4.pas', PrecompLZO in 'precompressor\PrecompLZO.pas', PrecompZSTD in 'precompressor\PrecompZSTD.pas', + PrecompMedia in 'precompressor\PrecompMedia.pas', PrecompOodle in 'precompressor\PrecompOodle.pas', PrecompINI in 'precompressor\PrecompINI.pas', PrecompSearch in 'precompressor\PrecompSearch.pas', @@ -82,7 +87,8 @@ uses IOPatch in 'io\IOPatch.pas', IOExecute in 'io\IOExecute.pas', IODecode in 'io\IODecode.pas', - IOUtils in 'io\IOUtils.pas'; + IOUtils in 'io\IOUtils.pas', + LZMADLL in 'imports\LZMADLL.pas'; {$SETPEFLAGS IMAGE_FILE_LARGE_ADDRESS_AWARE or IMAGE_FILE_RELOCS_STRIPPED} @@ -141,265 +147,24 @@ begin Result := THandleStream.Create(GetStdHandle(STD_INPUT_HANDLE)) else if Pos('://', Input) > 0 then Result := TDownloadStream.Create(Input) + else if FileExists(Input) then + Result := TFileStream.Create(Input, fmShareDenyNone) else - Result := TFileStream.Create(Input, fmShareDenyNone); + Result := TDirInputStream.Create(Input); end; -function GetOutStream(Output: string): TStream; +function GetOutStream(Output: string; MultiInput: Boolean = False): TStream; begin - if (Output = '-') or (Output = '') then + if (Output = '') then + Result := TNullStream.Create + else if DirectoryExists(Output) then + Result := TDirOutputStream.Create(Output) + else if (Output = '-') then Result := THandleStream.Create(GetStdHandle(STD_OUTPUT_HANDLE)) else Result := TFileStream.Create(Output, fmCreate); end; -{ changelog - ES_R34 (0.5.3) - - added png stream preprocessor - - removed grittibanzli codec (since nobody uses it) - - ES_R33 (0.5.2) - - added IO functions (archive, execute) - - fixed issue in patch io function - - removed compression on patch diff files - - ES_R32 (0.5.1) - - added IO functions (find, extract, patch) - - generate database feature and IO functions now can search for streams larger than chunk size - - ES_R31 (0.5.0) - - added IO functions (erase, replace) - - fixed external executable support bugs - - ES_R30 (0.4.8) - - fixed issue with storing incorrect recompression information when stream patching is performed - - ES_R29 (0.4.7) - - updated oodle scanner - - updated external executable support - - updated configuration based plugin support to add depth information - - updated verbose mode - - ES_R28 (0.4.6) - - generate database feature fixed - - fixed external executable support issues - - fixed lz4f level setting bug - - ES_R28 (0.4.5) - - removed leviathan codec restriction - - ES_R27 (0.4.4) - - fixed issue of lz4 codec loading incorrect library - - fixed issue with handling endianess via configuration based plugins - - updated framework of library based plugins - - ES_R26 (0.4.3) - - added verbose mode - - added feature that allows you to enforce a different library to be loaded - - fixed issues related to imperfect stream patching - - fixed issues with old libraries with missing functions that cause xtool to crash on startup - - updated oodle codec - - updated reflate codec - - updated zstd codec - - ES_R25 (0.4.2) - - removed debugging code from encryption and executable codec - - fixed issue with depth when using search codec - - fixed external executable support issues - - ES_R24 (0.4.1) - - fixed issue of status not reporting when encoding - - added depth method support for search support - - fixed zlib encoding issues for different window bits - - fixed zlib memory leak issue - - updated all internal codecs to support information relayed by external codecs - - updated lz4f codec and removed temporarily removed support for universal scanning - - added option to change recompression level to be used by reflate - - updated external executable support - - generate database feature currently bugged, wait for next update - - search database structure changed, older database files will no longer work with newer releases - - ES_R23 (0.4.0) - - project made open source - - added external executable support - - added generate database feature - - fixed search support bug - - ES_R22 (0.3.22) - - updated search support (speed improvements) - - updated command line parser - - added partial universal scanner for lzo1x streams - - added universal scanner for lz4f streams - - fixed issue with configuration files failing to execute without conditions - - ES_R21 (0.3.21) - - updated search support - - ES_R20 (0.3.20) - - fixed library support bug - - x86 build discontinued (has bugs from nowhere) - - ES_R19 (0.3.19) - - updated lzo codec - - ES_R18 (0.3.18) - - fixed depth bug - - fixed library plugin bugs - - ES_R17 (0.3.17) - - fixed multi-threading bug - - ES_R16 (0.3.16) - - minor bug fixes - - ES_R15 (0.3.15) - - converted library support to unicode (don't know why I used ansi in the first place) - - added library support functions - - added rc4 encryption support - - ES_R14 (0.3.14) - - fixed library support bug - - updated library structure - - ES_R13 (0.3.13) - - updated lz4 codec - - updated library structure - - updated depth info functions - - updated depth feature - - ES_R12 (0.3.12) - - added depth info functions - - added support for oodle 2.9.0+ functions - - fixed data patching bug - - updated oodle codec - - updated command line parser - - ES_R11 (0.3.11) - - fixed x86 build bugs - - fixed config multi-threading bug - - fixed resource management bug - - fixed deduplication bug - - ES_R10 (0.3.10) - - minor bug fixes - - added diff tolerance parameter (--diff=) - - fixed plugin database bug - - fixed lz4 codec bug - - updated oodle codec - - updated library structure - - added resource management - - added direct use encryption codecs - - added embedded deduplication feature (--dedup) [makes temps during encoding] - - ES_R9 (0.3.9) - - fixed future stream bug - - ES_R8 (0.3.8) - - fixed command line parser bug - - updated library support - - ES_R7 (0.3.7) - - updated library structure - - ES_R6 (0.3.6) - - updated oodle codec (fixed more lzna bugs) - - ES_R5 (0.3.5) - - updated oodle codec (fixed lzna bug) - - added custom method configuration - - ES_R4 (0.3.4) - - fixed bug depthing - - ES_R3 (0.3.3) - - updated lz4 codec - - updated library support - - ES_R2 (0.3.2) - - improved depthing - - updated library support - - fixed zstd codec issues - - removed fast memory - - ES_R1 (0.3.1) - - updated library support - - updated command line parser - - included x86 build - - fixed depthing issues - - 2012_R2 (0.2.14) - - added library support - - added compress, decompress, encrypt, decrypt, hash, delta functions (used by library) - - added lzo codec placeholders - - fixed oodle bug - - fixed lz4 bug - - removed libdunia codec - - 2012_R1 (0.2.13) - - added oo2ext* dll support - - updated search support - - 2011_R1 (0.2.12) - - added temporary libdunia codec - - 2010_R5 (0.2.11) - - fixed search/config support bug - - 2010_R4 (0.2.10) - - updated search/config support - - 2010_R3 (0.2.9) - - added database search - - updated zlib scanner - - fixed reflate bug - - fixed 2GB memory limit - - 2010_R2 (0.2.8) - - fixed zstd codec - - 2010_R1 (0.2.7) - - added zstd codec - - added lz4, lz4hc, lzna, mermaid, selkie, hydra, leviathan codec placeholders - - added configuration support - - added xdelta support to handle crc mismatch streams - - 2009_R3 (0.2.6) - - documentation added - - 2009_R2 (0.2.5) - - added kraken codec - - fixed depthing issues - - 2009_R1 (0.2.4) - - added reflate forced verification - - updated deflate scanner - - fixed depthing issues - - fixed low memory mode issues - - fixed hanging issues when encoding - - 2008_R3 (0.2.3) - - fixed deduplication memory calculation error - - added virtual memory support for deduplication - - added --mem=# parameter to control deduplication memory usage - - 2008_R2 (0.2.2) - - fixed command line parser - - updated deflate scanner - - added stream deduplication - - added stream database - - added decompression memory limiter - - added grittibanzli (also handles deflate stream but slow af) - - 2008_R1 (0.2.1) - - initial release - - changelog } - -procedure EncodePNG(Input: PByte; Output: PByte); -begin - -end; - const BufferSize = 1048576; @@ -460,12 +225,10 @@ begin PrecompMain.PrintHelp else begin - while Length(ParamArg[1]) < 2 do - Insert('-', ParamArg[1], Length(ParamArg[1])); - Input := TBufferedStream.Create(GetInStream(ParamArg[1, 0]), True, - BufferSize); - Output := TBufferedStream.Create(GetOutStream(ParamArg[1, 1]), False, + Input := TBufferedStream.Create(GetInStream(ParamArgSafe(1, 0)), True, BufferSize); + Output := TBufferedStream.Create(GetOutStream(ParamArgSafe(1, 1)), + False, BufferSize); try PrecompMain.Parse(ParamArg[0], PrecompEnc); PrecompMain.Encode(Input, Output, PrecompEnc); @@ -550,8 +313,6 @@ begin IOArchive.PrintHelp else begin - while Length(ParamArg[1]) < 2 do - Insert('-', ParamArg[1], Length(ParamArg[1])); SetLength(StrArray, 0); for I := 0 to High(ParamArg[1]) - 1 do Insert(ParamArg[1, I], StrArray, Length(StrArray)); @@ -569,8 +330,6 @@ begin IOExecute.PrintHelp else begin - while Length(ParamArg[1]) < 2 do - Insert('-', ParamArg[1], Length(ParamArg[1])); SetLength(StrArray, 0); for I := 2 to High(ParamArg[1]) do Insert(ParamArg[1, I], StrArray, Length(StrArray)); diff --git a/xtool.dproj b/xtool.dproj index 9936163..194f8e7 100644 --- a/xtool.dproj +++ b/xtool.dproj @@ -111,9 +111,13 @@ + + + + @@ -127,6 +131,7 @@ + @@ -142,6 +147,7 @@ + ResourceItem RCDATA @@ -152,6 +158,7 @@ RCDATA XDELTA64_DLL + Base @@ -189,6 +196,12 @@ true + + + .\ + true + + .\