diff --git a/changes.txt b/changes.txt deleted file mode 100644 index c04878d..0000000 --- a/changes.txt +++ /dev/null @@ -1,251 +0,0 @@ - ES_R35 (0.6.0) - - added wav stream detector - - added flac codec - - added jpg stream detector - - added packjpg, brunsli, jojpeg codec - - added fast lzma2 compression for portable mode - - added feature that allows input to be a directory - - added feature to inject libraries to main executable - - added feature to extract detected streams - - updated database feature - - updated deduplication feature - - IO function decode updated - - ES_R34 (0.5.3) - - added png stream preprocessor - - removed grittibanzli codec (since nobody uses it) - - ES_R33 (0.5.2) - - added IO functions (archive, execute) - - fixed issue in patch io function - - removed compression on patch diff files - - ES_R32 (0.5.1) - - added IO functions (find, extract, patch) - - generate database feature and IO functions now can search for streams larger than chunk size - - ES_R31 (0.5.0) - - added IO functions (erase, replace) - - fixed external executable support bugs - - ES_R30 (0.4.8) - - fixed issue with storing incorrect recompression information when stream patching is performed - - ES_R29 (0.4.7) - - updated oodle scanner - - updated external executable support - - updated configuration based plugin support to add depth information - - updated verbose mode - - ES_R28 (0.4.6) - - generate database feature fixed - - fixed external executable support issues - - fixed lz4f level setting bug - - ES_R28 (0.4.5) - - removed leviathan codec restriction - - ES_R27 (0.4.4) - - fixed issue of lz4 codec loading incorrect library - - fixed issue with handling endianess via configuration based plugins - - updated framework of library based plugins - - ES_R26 (0.4.3) - - added verbose mode - - added feature that allows you to enforce a different library to be loaded - - fixed issues related to imperfect stream patching - - fixed issues with old libraries with missing functions that cause xtool to crash on startup - - updated oodle codec - - updated reflate codec - - updated zstd codec - - ES_R25 (0.4.2) - - removed debugging code from encryption and executable codec - - fixed issue with depth when using search codec - - fixed external executable support issues - - ES_R24 (0.4.1) - - fixed issue of status not reporting when encoding - - added depth method support for search support - - fixed zlib encoding issues for different window bits - - fixed zlib memory leak issue - - updated all internal codecs to support information relayed by external codecs - - updated lz4f codec and removed temporarily removed support for universal scanning - - added option to change recompression level to be used by reflate - - updated external executable support - - generate database feature currently bugged, wait for next update - - search database structure changed, older database files will no longer work with newer releases - - ES_R23 (0.4.0) - - project made open source - - added external executable support - - added generate database feature - - fixed search support bug - - ES_R22 (0.3.22) - - updated search support (speed improvements) - - updated command line parser - - added partial universal scanner for lzo1x streams - - added universal scanner for lz4f streams - - fixed issue with configuration files failing to execute without conditions - - ES_R21 (0.3.21) - - updated search support - - ES_R20 (0.3.20) - - fixed library support bug - - x86 build discontinued (has bugs from nowhere) - - ES_R19 (0.3.19) - - updated lzo codec - - ES_R18 (0.3.18) - - fixed depth bug - - fixed library plugin bugs - - ES_R17 (0.3.17) - - fixed multi-threading bug - - ES_R16 (0.3.16) - - minor bug fixes - - ES_R15 (0.3.15) - - converted library support to unicode (don't know why I used ansi in the first place) - - added library support functions - - added rc4 encryption support - - ES_R14 (0.3.14) - - fixed library support bug - - updated library structure - - ES_R13 (0.3.13) - - updated lz4 codec - - updated library structure - - updated depth info functions - - updated depth feature - - ES_R12 (0.3.12) - - added depth info functions - - added support for oodle 2.9.0+ functions - - fixed data patching bug - - updated oodle codec - - updated command line parser - - ES_R11 (0.3.11) - - fixed x86 build bugs - - fixed config multi-threading bug - - fixed resource management bug - - fixed deduplication bug - - ES_R10 (0.3.10) - - minor bug fixes - - added diff tolerance parameter (--diff=) - - fixed plugin database bug - - fixed lz4 codec bug - - updated oodle codec - - updated library structure - - added resource management - - added direct use encryption codecs - - added embedded deduplication feature (--dedup) [makes temps during encoding] - - ES_R9 (0.3.9) - - fixed future stream bug - - ES_R8 (0.3.8) - - fixed command line parser bug - - updated library support - - ES_R7 (0.3.7) - - updated library structure - - ES_R6 (0.3.6) - - updated oodle codec (fixed more lzna bugs) - - ES_R5 (0.3.5) - - updated oodle codec (fixed lzna bug) - - added custom method configuration - - ES_R4 (0.3.4) - - fixed bug depthing - - ES_R3 (0.3.3) - - updated lz4 codec - - updated library support - - ES_R2 (0.3.2) - - improved depthing - - updated library support - - fixed zstd codec issues - - removed fast memory - - ES_R1 (0.3.1) - - updated library support - - updated command line parser - - included x86 build - - fixed depthing issues - - 2012_R2 (0.2.14) - - added library support - - added compress, decompress, encrypt, decrypt, hash, delta functions (used by library) - - added lzo codec placeholders - - fixed oodle bug - - fixed lz4 bug - - removed libdunia codec - - 2012_R1 (0.2.13) - - added oo2ext* dll support - - updated search support - - 2011_R1 (0.2.12) - - added temporary libdunia codec - - 2010_R5 (0.2.11) - - fixed search/config support bug - - 2010_R4 (0.2.10) - - updated search/config support - - 2010_R3 (0.2.9) - - added database search - - updated zlib scanner - - fixed reflate bug - - fixed 2GB memory limit - - 2010_R2 (0.2.8) - - fixed zstd codec - - 2010_R1 (0.2.7) - - added zstd codec - - added lz4, lz4hc, lzna, mermaid, selkie, hydra, leviathan codec placeholders - - added configuration support - - added xdelta support to handle crc mismatch streams - - 2009_R3 (0.2.6) - - documentation added - - 2009_R2 (0.2.5) - - added kraken codec - - fixed depthing issues - - 2009_R1 (0.2.4) - - added reflate forced verification - - updated deflate scanner - - fixed depthing issues - - fixed low memory mode issues - - fixed hanging issues when encoding - - 2008_R3 (0.2.3) - - fixed deduplication memory calculation error - - added virtual memory support for deduplication - - added --mem=# parameter to control deduplication memory usage - - 2008_R2 (0.2.2) - - fixed command line parser - - updated deflate scanner - - added stream deduplication - - added stream database - - added decompression memory limiter - - added grittibanzli (also handles deflate stream but slow af) - - 2008_R1 (0.2.1) - - initial release \ No newline at end of file diff --git a/common/Utils.pas b/common/Utils.pas index 94c7042..f48dabd 100644 --- a/common/Utils.pas +++ b/common/Utils.pas @@ -44,16 +44,6 @@ procedure SetBits(var Data: UInt64; Value: Int64; Index: TInt64_BitIndex; Count: TInt64_BitCount); overload; type - PDynArrayRec = ^TDynArrayRec; - - TDynArrayRec = packed record -{$IFDEF CPUX64} - _Padding: LongInt; -{$ENDIF} - RefCnt: LongInt; - Length: NativeInt; - end; - TListEx = class(TList) private FIndex: Integer; @@ -95,13 +85,6 @@ type property Method: TSOMethod read FSOMethod write FSOMethod; end; - TNullStream = class(TStream) - public - constructor Create; - destructor Destroy; override; - function Write(const Buffer; Count: LongInt): LongInt; override; - end; - TArrayStream = class(TStream) private type _Stream = ^IStream; @@ -114,7 +97,7 @@ type FMaxStreamSize = $FFFFFFFFFF; protected function GetSize: Int64; override; - procedure SetSize(NewSize: LongInt); override; + procedure SetSize(NewSize: Longint); override; procedure SetSize(const NewSize: Int64); override; private FStreams: TArray; @@ -127,8 +110,8 @@ type public constructor Create; destructor Destroy; override; - function Read(var Buffer; Count: LongInt): LongInt; override; - function Write(const Buffer; Count: LongInt): LongInt; override; + function Read(var Buffer; Count: Longint): Longint; override; + function Write(const Buffer; Count: Longint): Longint; override; function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override; procedure Clear; function Add(AStreamType: Pointer; MaxSize: Int64 = FMaxStreamSize) @@ -139,7 +122,7 @@ type TPointersStream = class(TStream) protected function GetSize: Int64; override; - procedure SetSize(NewSize: LongInt); override; + procedure SetSize(NewSize: Longint); override; procedure SetSize(const NewSize: Int64); override; private FPointers: TArray; @@ -152,8 +135,8 @@ type public constructor Create; destructor Destroy; override; - function Read(var Buffer; Count: LongInt): LongInt; override; - function Write(const Buffer; Count: LongInt): LongInt; override; + function Read(var Buffer; Count: Longint): Longint; override; + function Write(const Buffer; Count: Longint): Longint; override; function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override; function Add(Ptr: Pointer; Size: NativeInt): Integer; procedure Delete(Index: Integer); @@ -174,48 +157,13 @@ type AMaxSize: NativeInt = 0); overload; destructor Destroy; override; procedure SetSize(const NewSize: Int64); override; - procedure SetSize(NewSize: LongInt); override; - function Read(var Buffer; Count: LongInt): LongInt; override; - function Write(const Buffer; Count: LongInt): LongInt; override; + procedure SetSize(NewSize: Longint); override; + function Read(var Buffer; Count: Longint): Longint; override; + function Write(const Buffer; Count: Longint): Longint; override; function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override; procedure Update(const AMemory: Pointer = nil; AMaxSize: NativeInt = 0); end; - TDirInputStream = class(TStream) - protected type - TState = (iNone, iLength, iFilename, iSize, iData); - private - FState: TState; - FPath: String; - FBaseDir: String; - FList: TArray; - FIndex, FCount: Integer; - FLength: Word; - FBytes: TBytes; - FStream: TFileStream; - FPosition, FSize: Int64; - public - constructor Create(const APath: String); - destructor Destroy; override; - function Read(var Buffer; Count: LongInt): LongInt; override; - end; - - TDirOutputStream = class(TStream) - protected type - TState = (oNone, oLength, oFilename, oSize, oData); - private - FState: TState; - FPath: String; - FLength: Word; - FBytes: TBytes; - FStream: TFileStream; - FPosition, FSize: Int64; - public - constructor Create(const APath: String); - destructor Destroy; override; - function Write(const Buffer; Count: LongInt): LongInt; override; - end; - TSharedMemoryStream = class(TMemoryStreamEx) private const FIncSize = 64 * 1024 * 1024; @@ -231,7 +179,7 @@ type constructor Create(const AMapName: String; ASize: NativeInt); overload; destructor Destroy; override; procedure SetSize(const NewSize: Int64); override; - function Write(const Buffer; Count: LongInt): LongInt; override; + function Write(const Buffer; Count: Longint): Longint; override; end; TDownloadStream = class(TStream) @@ -247,7 +195,7 @@ type public constructor Create(Url: string); destructor Destroy; override; - function Read(var Buffer; Count: LongInt): LongInt; override; + function Read(var Buffer; Count: Longint): Longint; override; function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override; end; @@ -349,23 +297,23 @@ type PBlockInfo = ^TBlockInfo; TBlockInfo = record - ID: Integer; Position, CurrSize, FullSize: Int64; Count: Integer; end; + + TBlockInfoDynArray = TArray; private - FSearchList: TArray; + FSync: TSynLocker; + FDictionary: TSynDictionary; FStream: TStream; - FStreamPos, FStreamSize: Int64; + FStreamPos: Int64; public - constructor Create(AStream: TStream); + constructor Create(AStream: TStream; ACapacity: Integer = 0); destructor Destroy; override; procedure Add(ID: Integer; Size: Int64; Count: Integer = Integer.MaxValue); - procedure Write(ID: Integer; Buffer: Pointer; Size: Integer); + procedure Write(ID: Integer; const Buffer; Size: Integer); procedure CopyData(ID: Integer; Stream: TStream); overload; - function CopyData(ID: Integer; Data: Pointer): Integer; overload; - procedure Update(ID: Integer; Count: Integer); - procedure Reset(ID: Integer); + function CopyData(ID: Integer): Pointer; overload; end; TArgParser = class(TObject) @@ -731,21 +679,6 @@ begin Result := FList.Count; end; -constructor TNullStream.Create; -begin - inherited Create; -end; - -destructor TNullStream.Destroy; -begin - inherited Destroy; -end; - -function TNullStream.Write(const Buffer; Count: LongInt): LongInt; -begin - Result := Count; -end; - constructor TArrayStream.Create; begin inherited Create; @@ -848,7 +781,7 @@ begin Result := FSize; end; -procedure TArrayStream.SetSize(NewSize: LongInt); +procedure TArrayStream.SetSize(NewSize: Longint); begin SetSize(Int64(NewSize)); end; @@ -861,7 +794,7 @@ begin Seek(0, soEnd); end; -function TArrayStream.Read(var Buffer; Count: LongInt): LongInt; +function TArrayStream.Read(var Buffer; Count: Longint): Longint; var LCount: Int64; begin @@ -875,7 +808,7 @@ begin Inc(FPosition, Result); end; -function TArrayStream.Write(const Buffer; Count: LongInt): LongInt; +function TArrayStream.Write(const Buffer; Count: Longint): Longint; var LCount: Int64; begin @@ -955,7 +888,7 @@ begin Result := FSize; end; -procedure TPointersStream.SetSize(NewSize: LongInt); +procedure TPointersStream.SetSize(NewSize: Longint); begin SetSize(Int64(NewSize)); end; @@ -984,12 +917,12 @@ begin FSize := FMaxSize; end; -function TPointersStream.Read(var Buffer; Count: LongInt): LongInt; +function TPointersStream.Read(var Buffer; Count: Longint): Longint; begin // 2121212 end; -function TPointersStream.Write(const Buffer; Count: LongInt): LongInt; +function TPointersStream.Write(const Buffer; Count: Longint): Longint; begin end; @@ -1053,7 +986,7 @@ begin inherited Destroy; end; -procedure TMemoryStreamEx.SetSize(NewSize: LongInt); +procedure TMemoryStreamEx.SetSize(NewSize: Longint); begin SetSize(Int64(NewSize)); end; @@ -1069,7 +1002,7 @@ begin Seek(0, soEnd); end; -function TMemoryStreamEx.Read(var Buffer; Count: LongInt): LongInt; +function TMemoryStreamEx.Read(var Buffer; Count: Longint): Longint; begin Result := 0; if (FPosition >= 0) and (Count >= 0) then @@ -1086,9 +1019,9 @@ begin end; end; -function TMemoryStreamEx.Write(const Buffer; Count: LongInt): LongInt; +function TMemoryStreamEx.Write(const Buffer; Count: Longint): Longint; var - FCount: LongInt; + FCount: Longint; begin Result := 0; FCount := Count; @@ -1144,210 +1077,6 @@ begin SetSize(LSize); end; -constructor TDirInputStream.Create(const APath: String); -begin - inherited Create; - FState := TState.iNone; - FPath := TPath.GetFullPath(APath); - if FileExists(FPath) then - FBaseDir := ExtractFilePath(TPath.GetFullPath(FPath)) - else if DirectoryExists(FPath) then - FBaseDir := IncludeTrailingBackSlash(TPath.GetFullPath(FPath)) - else - FBaseDir := ExtractFilePath(TPath.GetFullPath(FPath)); - FList := GetFileList([FPath], True); - FCount := Length(FList); - if FCount = 0 then - raise EFOpenError.CreateRes(@SEmptyPath); - FIndex := -1; - FStream := nil; -end; - -destructor TDirInputStream.Destroy; -begin - if Assigned(FStream) then - FStream.Free; - FStream := nil; - inherited Destroy; -end; - -function TDirInputStream.Read(var Buffer; Count: LongInt): LongInt; -var - LCount: Integer; -begin - Result := 0; - if Count <= 0 then - exit; - if FState = TState.iNone then - begin - if Succ(FIndex) >= FCount then - exit; - Inc(FIndex); - FBytes := BytesOf(ReplaceText(FList[FIndex], FBaseDir, '')); - FLength := Length(FBytes); - FPosition := 0; - FSize := FileSize(FList[FIndex]); - FState := TState.iLength; - end; - if FState = TState.iLength then - if FPosition < FLength.Size then - begin - LCount := Min(FLength.Size - FPosition, Count); - Move(WordRec(FLength).Bytes[FPosition], Buffer, LCount); - Inc(FPosition, LCount); - if FPosition = FLength.Size then - begin - FState := TState.iFilename; - FPosition := 0; - end; - exit(LCount); - end; - if FState = TState.iFilename then - if FPosition < FLength then - begin - LCount := Min(FLength - FPosition, Count); - Move(FBytes[FPosition], Buffer, LCount); - Inc(FPosition, LCount); - if FPosition = FLength then - begin - FState := TState.iSize; - FPosition := 0; - end; - exit(LCount); - end; - if FState = TState.iSize then - if FPosition < FSize.Size then - begin - LCount := Min(FSize.Size - FPosition, Count); - Move(Int64Rec(FSize).Bytes[FPosition], Buffer, LCount); - Inc(FPosition, LCount); - if FPosition = FSize.Size then - begin - if FSize = 0 then - FState := TState.iNone - else - begin - FState := TState.iData; - FPosition := 0; - FStream := TFileStream.Create(FList[FIndex], fmShareDenyNone); - end; - end; - exit(LCount); - end; - if FState = TState.iData then - if FPosition < FSize then - begin - LCount := Min(FSize - FPosition, Count); - LCount := FStream.Read(Buffer, LCount); - Inc(FPosition, LCount); - if FPosition = FSize then - begin - FState := TState.iNone; - FStream.Free; - FStream := nil; - end; - exit(LCount); - end; -end; - -constructor TDirOutputStream.Create(const APath: String); -begin - inherited Create; - FState := TState.oNone; - FPath := IncludeTrailingBackSlash(TPath.GetFullPath(APath)); - FStream := nil; -end; - -destructor TDirOutputStream.Destroy; -begin - if Assigned(FStream) then - FStream.Free; - FStream := nil; - inherited Destroy; -end; - -function TDirOutputStream.Write(const Buffer; Count: LongInt): LongInt; -var - LCount: Integer; - LStr: String; -begin - Result := 0; - if Count <= 0 then - exit; - if FState = TState.oNone then - begin - FPosition := 0; - FState := TState.oLength; - end; - if FState = TState.oLength then - if FPosition < FLength.Size then - begin - LCount := Min(FLength.Size - FPosition, Count); - Move(Buffer, WordRec(FLength).Bytes[FPosition], LCount); - Inc(FPosition, LCount); - if FPosition = FLength.Size then - begin - SetLength(FBytes, FLength); - FState := TState.oFilename; - FPosition := 0; - end; - exit(LCount); - end; - if FState = TState.oFilename then - if FPosition < FLength then - begin - LCount := Min(FLength - FPosition, Count); - Move(Buffer, FBytes[FPosition], LCount); - Inc(FPosition, LCount); - if FPosition = FLength then - begin - FState := TState.oSize; - FPosition := 0; - end; - exit(LCount); - end; - if FState = TState.oSize then - if FPosition < FSize.Size then - begin - LCount := Min(FSize.Size - FPosition, Count); - Move(Buffer, Int64Rec(FSize).Bytes[FPosition], LCount); - Inc(FPosition, LCount); - if FPosition = FSize.Size then - begin - LStr := FPath + StringOf(FBytes); - if not DirectoryExists(ExtractFilePath(LStr)) then - ForceDirectories(ExtractFilePath(LStr)); - FStream := TFileStream.Create(LStr, fmCreate); - if FSize = 0 then - begin - FState := TState.oNone; - FStream.Free; - FStream := nil; - end - else - begin - FState := TState.oData; - FPosition := 0; - end; - end; - exit(LCount); - end; - if FState = TState.oData then - if FPosition < FSize then - begin - LCount := Min(FSize - FPosition, Count); - LCount := FStream.Write(Buffer, LCount); - Inc(FPosition, LCount); - if FPosition = FSize then - begin - FState := TState.oNone; - FStream.Free; - FStream := nil; - end; - exit(LCount); - end; -end; - constructor TSharedMemoryStream.Create(const AMapName: String; AFileName: string); @@ -1462,7 +1191,7 @@ begin inherited SetSize(NewSize); end; -function TSharedMemoryStream.Write(const Buffer; Count: LongInt): LongInt; +function TSharedMemoryStream.Write(const Buffer; Count: Longint): Longint; begin if FPosition + Count > FMaxSize then IncMemory(FPosition + Count); @@ -1498,7 +1227,7 @@ begin Abort := True; end; -function TDownloadStream.Read(var Buffer; Count: LongInt): LongInt; +function TDownloadStream.Read(var Buffer; Count: Longint): Longint; begin if (FPosition >= 0) and (Count >= 0) then begin @@ -2050,132 +1779,107 @@ begin FSizes[Index] := 0; end; -constructor TDataManager.Create(AStream: TStream); +constructor TDataManager.Create(AStream: TStream; ACapacity: Integer); begin inherited Create; + FSync.Init; + FDictionary := TSynDictionary.Create(TypeInfo(TIntegerDynArray), + TypeInfo(TBlockInfoDynArray)); + FDictionary.Capacity := ACapacity; FStream := AStream; FStreamPos := FStream.Position; - FStreamSize := 0; end; destructor TDataManager.Destroy; begin + FDictionary.Free; + FSync.Done; inherited Destroy; end; procedure TDataManager.Add(ID: Integer; Size: Int64; Count: Integer); var - I: Integer; LBlockInfo: TBlockInfo; + _BlockInfo: PBlockInfo; + I: Integer; begin if Count <= 0 then exit; - for I := Low(FSearchList) to High(FSearchList) do - begin - if (FSearchList[I].Count <= 0) and (Size <= FSearchList[I].FullSize) then + FSync.Lock; + try + LBlockInfo.Position := 0; + for I := 0 to FDictionary.Count - 1 do begin - FSearchList[I].ID := ID; - FSearchList[I].CurrSize := 0; - FSearchList[I].Count := Count; - exit; + _BlockInfo := PBlockInfo(FDictionary.Values.ElemPtr(I)); + LBlockInfo.Position := Max(LBlockInfo.Position, _BlockInfo^.Position + + _BlockInfo^.FullSize); end; + LBlockInfo.CurrSize := 0; + LBlockInfo.FullSize := Size; + LBlockInfo.Count := Count; + FStream.Size := Max(FStream.Size, FStreamPos + LBlockInfo.Position + + LBlockInfo.FullSize); + FDictionary.Add(ID, LBlockInfo); + finally + FSync.UnLock; end; - LBlockInfo.ID := ID; - LBlockInfo.Position := FStreamPos + FStreamSize; - LBlockInfo.CurrSize := 0; - LBlockInfo.FullSize := Size; - LBlockInfo.Count := Count; - Insert(LBlockInfo, FSearchList, Length(FSearchList)); - Inc(FStreamSize, Size); + // once reads reaches 0, add list of all available spaces end; -procedure TDataManager.Write(ID: Integer; Buffer: Pointer; Size: Integer); +procedure TDataManager.Write(ID: Integer; const Buffer; Size: Integer); var - I: Integer; + _BlockInfo: PBlockInfo; begin if Size <= 0 then exit; - for I := Low(FSearchList) to High(FSearchList) do - begin - if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then - begin - if FSearchList[I].CurrSize + Size > FSearchList[I].FullSize then - raise EWriteError.CreateRes(@SWriteError); - FStream.Position := FSearchList[I].Position + FSearchList[I].CurrSize; - FStream.WriteBuffer(Buffer^, Size); - Inc(FSearchList[I].CurrSize, Size); - exit; - end; + FSync.Lock; + try + _BlockInfo := FDictionary.FindValue(ID); + if (_BlockInfo^.Position + _BlockInfo^.CurrSize + Size) > + (_BlockInfo^.Position + _BlockInfo^.FullSize) then + raise EWriteError.CreateRes(@SWriteError); + FStream.Position := FStreamPos + _BlockInfo^.Position + + _BlockInfo^.CurrSize; + FStream.WriteBuffer(Buffer, Size); + Inc(_BlockInfo^.CurrSize, Size); + finally + FSync.UnLock; end; - raise Exception.CreateRes(@SGenericItemNotFound); end; procedure TDataManager.CopyData(ID: Integer; Stream: TStream); var - I: Integer; + _BlockInfo: PBlockInfo; begin - for I := Low(FSearchList) to High(FSearchList) do - begin - if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then - begin - FStream.Position := FSearchList[I].Position; - CopyStreamEx(FStream, Stream, FSearchList[I].CurrSize); - Dec(FSearchList[I].Count); - exit; - end; + FSync.Lock; + try + _BlockInfo := FDictionary.FindValue(ID); + if _BlockInfo^.CurrSize <> _BlockInfo^.FullSize then + raise EReadError.CreateRes(@SReadError); + FStream.Position := FStreamPos + _BlockInfo^.Position; + CopyStreamEx(FStream, Stream, _BlockInfo^.FullSize); + Dec(_BlockInfo^.Count); + finally + FSync.UnLock; end; - raise Exception.CreateRes(@SGenericItemNotFound); end; -function TDataManager.CopyData(ID: Integer; Data: Pointer): Integer; +function TDataManager.CopyData(ID: Integer): Pointer; var - I: Integer; + _BlockInfo: PBlockInfo; begin - Result := 0; - for I := Low(FSearchList) to High(FSearchList) do - begin - if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then - begin - FStream.Position := FSearchList[I].Position; - FStream.ReadBuffer(Data^, FSearchList[I].CurrSize); - Result := FSearchList[I].CurrSize; - Dec(FSearchList[I].Count); - if FSearchList[I].Count = 0 then - FSearchList[I].ID := -1; - exit; - end; + FSync.Lock; + try + _BlockInfo := FDictionary.FindValue(ID); + if _BlockInfo^.CurrSize <> _BlockInfo^.FullSize then + raise EReadError.CreateRes(@SReadError); + FStream.Position := FStreamPos + _BlockInfo^.Position + + _BlockInfo^.CurrSize; + FStream.ReadBuffer(Result^, _BlockInfo^.FullSize); + Dec(_BlockInfo^.Count); + finally + FSync.UnLock; end; - raise Exception.CreateRes(@SGenericItemNotFound); -end; - -procedure TDataManager.Update(ID: Integer; Count: Integer); -var - I: Integer; -begin - for I := Low(FSearchList) to High(FSearchList) do - begin - if (ID = FSearchList[I].ID) then - begin - FSearchList[I].Count := Count; - exit; - end; - end; - raise Exception.CreateRes(@SGenericItemNotFound); -end; - -procedure TDataManager.Reset(ID: Integer); -var - I: Integer; -begin - for I := Low(FSearchList) to High(FSearchList) do - begin - if (ID = FSearchList[I].ID) and (FSearchList[I].Count > 0) then - begin - FSearchList[I].CurrSize := 0; - exit; - end; - end; - raise Exception.CreateRes(@SGenericItemNotFound); end; constructor TArgParser.Create(Arguments: TStringDynArray); diff --git a/imports/BrunsliDLL.pas b/imports/BrunsliDLL.pas deleted file mode 100644 index 13bb944..0000000 --- a/imports/BrunsliDLL.pas +++ /dev/null @@ -1,87 +0,0 @@ -unit BrunsliDLL; - -interface - -uses - WinAPI.Windows, - System.SysUtils, System.Classes; - -const - BRUNSLI_OK = 0; - BRUNSLI_NON_REPRESENTABLE = 1; - BRUNSLI_MEMORY_ERROR = 2; - BRUNSLI_INVALID_PARAM = 3; - BRUNSLI_COMPRESSION_ERROR = 4; - BRUNSLI_INVALID_BRN = 5; - BRUNSLI_DECOMPRESSION_ERROR = 6; - BRUNSLI_NOT_ENOUGH_DATA = 7; - -type - TBrunsliWriter = function(ctx: Pointer; data: Pointer; Size: NativeUInt) - : Integer cdecl; - -var - brunsli_alloc_JPEGData: function: Pointer cdecl; - brunsli_free_JPEGData: procedure(P: Pointer)cdecl; - brunsli_GetMaximumEncodedSize: function(P: Pointer): Integer cdecl; - brunsli_ReadJpeg: function(P: Pointer; data: Pointer; len: Integer) - : Integer cdecl; - brunsli_EncodeJpeg: function(P: Pointer; data: Pointer; len: Integer) - : Integer cdecl; - brunsli_DecodeJpeg: function(P: Pointer; data: Pointer; len: Integer) - : Integer cdecl; - brunsli_alloc_JPEGOutput: function(P: TBrunsliWriter; data: Pointer) - : Pointer cdecl; - brunsli_free_JPEGOutput: procedure(P: Pointer)cdecl; - brunsli_WriteJpeg: function(P: Pointer; oup: Pointer): Integer cdecl; - DLLLoaded: Boolean = False; - -implementation - -var - DLLHandle: THandle; - S: String; - -procedure Init; -begin - S := 'brunsli.dll'; - DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + S)); - if DLLHandle >= 32 then - begin - @brunsli_alloc_JPEGData := GetProcAddress(DLLHandle, - 'brunsli_alloc_JPEGData'); - @brunsli_free_JPEGData := GetProcAddress(DLLHandle, - 'brunsli_free_JPEGData'); - @brunsli_GetMaximumEncodedSize := GetProcAddress(DLLHandle, - 'brunsli_GetMaximumEncodedSize'); - @brunsli_ReadJpeg := GetProcAddress(DLLHandle, 'brunsli_ReadJpeg'); - @brunsli_EncodeJpeg := GetProcAddress(DLLHandle, 'brunsli_EncodeJpeg'); - @brunsli_DecodeJpeg := GetProcAddress(DLLHandle, 'brunsli_DecodeJpeg'); - @brunsli_alloc_JPEGOutput := GetProcAddress(DLLHandle, - 'brunsli_alloc_JPEGOutput'); - @brunsli_free_JPEGOutput := GetProcAddress(DLLHandle, - 'brunsli_free_JPEGOutput'); - @brunsli_WriteJpeg := GetProcAddress(DLLHandle, 'brunsli_WriteJpeg'); - DLLLoaded := Assigned(brunsli_alloc_JPEGData) and - Assigned(brunsli_alloc_JPEGOutput); - end - else - DLLLoaded := False; -end; - -procedure Deinit; -begin - if not DLLLoaded then - exit; - FreeLibrary(DLLHandle); -end; - -initialization - -Init; - -finalization - -Deinit; - -end. diff --git a/imports/FLACDLL.pas b/imports/FLACDLL.pas deleted file mode 100644 index 4c14445..0000000 --- a/imports/FLACDLL.pas +++ /dev/null @@ -1,196 +0,0 @@ -unit FLACDLL; - -interface - -uses - WinAPI.Windows, - System.SysUtils, System.Classes; - -const - FLAC__MAX_CHANNELS = 8; - FLAC__MAX_FIXED_ORDER = 4; - FLAC__MAX_LPC_ORDER = 32; - -type - TFLAC__FrameHeader = record - blocksize, sample_rate, channels: Cardinal; - channel_assignment: Integer; - bits_per_sample: Cardinal; - case number_type: Integer of - 0: - (frame_number: Cardinal; crc1: Byte); - 1: - (sample_number: UInt64; crc2: Byte); - end; - - PFLAC__EntropyCodingMethod_PartitionedRiceContents = ^ - TFLAC__EntropyCodingMethod_PartitionedRiceContents; - - TFLAC__EntropyCodingMethod_PartitionedRiceContents = record - parameters, raw_bits: PCardinal; - capacity_by_order: Cardinal; - end; - - TFLAC__EntropyCodingMethod_PartitionedRice = record - order: Cardinal; - contents: PFLAC__EntropyCodingMethod_PartitionedRiceContents; - end; - - TFLAC__EntropyCodingMethod = record - case ftype: Integer of - 0: - (partitioned_rice: TFLAC__EntropyCodingMethod_PartitionedRice); - end; - - TFLAC__Subframe_Fixed = record - entropy_coding_method: TFLAC__EntropyCodingMethod; - order: Cardinal; - warmup: array [0 .. FLAC__MAX_CHANNELS - 1] of Integer; - residual: PInteger; - end; - - TFLAC__Subframe_LPC = record - entropy_coding_method: TFLAC__EntropyCodingMethod; - order, qlp_coeff_precision: Cardinal; - quantization_level: Integer; - qlp_coeff, warmup: array [0 .. FLAC__MAX_LPC_ORDER - 1] of Integer; - residual: PInteger; - end; - - TFLAC__Subframe = record - case ftype: Integer of - 0: - (constant: Integer; wb1: Cardinal); - 1: - (fixed: TFLAC__Subframe_Fixed; wb2: Cardinal); - 2: - (lpc: TFLAC__Subframe_LPC; wb3: Cardinal); - 3: - (verbatim: PInteger; wb4: Cardinal); - end; - - PFLAC__Frame = ^TFLAC__Frame; - - TFLAC__Frame = record - header: TFLAC__FrameHeader; - subframes: array [0 .. FLAC__MAX_CHANNELS - 1] of TFLAC__Subframe; - footer: Word; - end; - -var - FLAC__stream_encoder_new: function: Pointer cdecl; - FLAC__stream_encoder_set_verify: function(encoder: Pointer; value: Boolean) - : Boolean cdecl; - FLAC__stream_encoder_set_compression_level: function(encoder: Pointer; - value: Cardinal): Boolean cdecl; - FLAC__stream_encoder_set_channels: function(encoder: Pointer; value: Cardinal) - : Boolean cdecl; - FLAC__stream_encoder_set_bits_per_sample: function(encoder: Pointer; - value: Cardinal): Boolean cdecl; - FLAC__stream_encoder_set_sample_rate: function(encoder: Pointer; - value: Cardinal): Boolean cdecl; - FLAC__stream_encoder_set_total_samples_estimate: function(encoder: Pointer; - value: UInt64): Boolean cdecl; - FLAC__stream_encoder_init_stream: function(encoder: Pointer; - write_callback, seek_callback, tell_callback, metadata_callback: Pointer; - client_data: Pointer): Integer cdecl; - FLAC__stream_encoder_init_file: function(encoder: Pointer; - filename: PAnsiChar; progress_callback: Pointer; client_data: Pointer) - : Integer cdecl; - FLAC__stream_encoder_process_interleaved: function(encoder: Pointer; - const buffer; samples: Cardinal): Boolean cdecl; - FLAC__stream_encoder_finish: function(encoder: Pointer): Boolean cdecl; - FLAC__stream_encoder_delete: procedure(encoder: Pointer)cdecl; - FLAC__stream_decoder_new: function: Pointer cdecl; - FLAC__stream_decoder_init_stream: function(decoder: Pointer; - read_callback, seek_callback, tell_callback, length_callback, eof_callback, - write_callback, metadata_callback, error_callback: Pointer; - client_data: Pointer): Integer cdecl; - FLAC__stream_decoder_init_file: function(decoder: Pointer; - filename: PAnsiChar; write_callback, metadata_callback, error_callback - : Pointer; client_data: Pointer): Integer cdecl; - FLAC__stream_decoder_get_channels: function(decoder: Pointer): Cardinal cdecl; - FLAC__stream_decoder_get_bits_per_sample: function(decoder: Pointer) - : Cardinal cdecl; - FLAC__stream_decoder_process_until_end_of_stream: function(decoder: Pointer) - : Boolean cdecl; - FLAC__stream_decoder_finish: function(encoder: Pointer): Boolean cdecl; - FLAC__stream_decoder_delete: procedure(encoder: Pointer)cdecl; - DLLLoaded: Boolean = False; - -implementation - -var - DLLHandle: THandle; - -procedure Init; -begin - DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + - 'libFLAC_dynamic.dll')); - if DLLHandle >= 32 then - begin - @FLAC__stream_encoder_new := GetProcAddress(DLLHandle, - 'FLAC__stream_encoder_new'); - @FLAC__stream_encoder_set_verify := GetProcAddress(DLLHandle, - 'FLAC__stream_encoder_set_verify'); - @FLAC__stream_encoder_set_channels := GetProcAddress(DLLHandle, - 'FLAC__stream_encoder_set_channels'); - @FLAC__stream_encoder_set_compression_level := - GetProcAddress(DLLHandle, 'FLAC__stream_encoder_set_compression_level'); - @FLAC__stream_encoder_set_bits_per_sample := - GetProcAddress(DLLHandle, 'FLAC__stream_encoder_set_bits_per_sample'); - @FLAC__stream_encoder_set_sample_rate := - GetProcAddress(DLLHandle, 'FLAC__stream_encoder_set_sample_rate'); - @FLAC__stream_encoder_set_total_samples_estimate := - GetProcAddress(DLLHandle, - 'FLAC__stream_encoder_set_total_samples_estimate'); - @FLAC__stream_encoder_init_stream := GetProcAddress(DLLHandle, - 'FLAC__stream_encoder_init_stream'); - @FLAC__stream_encoder_init_file := GetProcAddress(DLLHandle, - 'FLAC__stream_encoder_init_file'); - @FLAC__stream_encoder_process_interleaved := - GetProcAddress(DLLHandle, 'FLAC__stream_encoder_process_interleaved'); - @FLAC__stream_encoder_finish := GetProcAddress(DLLHandle, - 'FLAC__stream_encoder_finish'); - @FLAC__stream_encoder_delete := GetProcAddress(DLLHandle, - 'FLAC__stream_encoder_delete'); - @FLAC__stream_decoder_new := GetProcAddress(DLLHandle, - 'FLAC__stream_decoder_new'); - @FLAC__stream_decoder_init_stream := GetProcAddress(DLLHandle, - 'FLAC__stream_decoder_init_stream'); - @FLAC__stream_decoder_init_file := GetProcAddress(DLLHandle, - 'FLAC__stream_decoder_init_file'); - @FLAC__stream_decoder_get_channels := GetProcAddress(DLLHandle, - 'FLAC__stream_decoder_get_channels'); - @FLAC__stream_decoder_get_bits_per_sample := - GetProcAddress(DLLHandle, 'FLAC__stream_decoder_get_bits_per_sample'); - @FLAC__stream_decoder_process_until_end_of_stream := - GetProcAddress(DLLHandle, - 'FLAC__stream_decoder_process_until_end_of_stream'); - @FLAC__stream_decoder_finish := GetProcAddress(DLLHandle, - 'FLAC__stream_decoder_finish'); - @FLAC__stream_decoder_delete := GetProcAddress(DLLHandle, - 'FLAC__stream_decoder_delete'); - DLLLoaded := Assigned(FLAC__stream_encoder_new) and - Assigned(FLAC__stream_decoder_new); - end - else - DLLLoaded := False; -end; - -procedure Deinit; -begin - if not DLLLoaded then - exit; - FreeLibrary(DLLHandle); -end; - -initialization - -Init; - -finalization - -Deinit; - -end. diff --git a/imports/JoJpegDLL.pas b/imports/JoJpegDLL.pas deleted file mode 100644 index e2c7741..0000000 --- a/imports/JoJpegDLL.pas +++ /dev/null @@ -1,69 +0,0 @@ -unit JoJpegDLL; - -interface - -uses - WinAPI.Windows, - System.SysUtils, System.Classes; - -const - jojpeg_Size = 51320000; - - jojpeg_Compress = 0; - jojpeg_Decompress = 1; - - jojpeg_enc_Input = 1; - jojpeg_enc_Output1 = 2; - jojpeg_enc_Output2 = 3; - - jojpeg_dec_Input1 = 1; - jojpeg_dec_Input2 = 3; - jojpeg_dec_Output = 2; - -var - jojpeg_Init: function(p: Pointer; f_DEC: integer): integer stdcall; - jojpeg_Quit: procedure(p: Pointer; f_DEC: integer)stdcall; - jojpeg_Loop: function(p: Pointer; f_DEC: integer): integer stdcall; - jojpeg_Getvalue: function(p: Pointer; f_DEC, typ: integer): Int64; - jojpeg_Addbuf: procedure(p: Pointer; f_DEC: integer; buf: Pointer; - bufsize, state: integer)stdcall; - DLLLoaded: boolean = False; - -implementation - -var - DLLHandle: THandle; - -procedure Init; -begin - DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + - 'jojpeg_dll.dll')); - if DLLHandle >= 32 then - begin - @jojpeg_Init := GetProcAddress(DLLHandle, 'jojpeg_Init'); - @jojpeg_Quit := GetProcAddress(DLLHandle, 'jojpeg_Quit'); - @jojpeg_Loop := GetProcAddress(DLLHandle, 'jojpeg_Loop'); - @jojpeg_Getvalue := GetProcAddress(DLLHandle, 'jojpeg_Getvalue'); - @jojpeg_Addbuf := GetProcAddress(DLLHandle, 'jojpeg_Addbuf'); - DLLLoaded := Assigned(jojpeg_Init); - end - else - DLLLoaded := False; -end; - -procedure Deinit; -begin - if not DLLLoaded then - exit; - FreeLibrary(DLLHandle); -end; - -initialization - -Init; - -finalization - -Deinit; - -end. diff --git a/imports/LZ4DLL.pas b/imports/LZ4DLL.pas index 1eccf75..58cf0f0 100644 --- a/imports/LZ4DLL.pas +++ b/imports/LZ4DLL.pas @@ -69,8 +69,6 @@ var LZ4F_compressFrame: function(dstBuffer: Pointer; dstCapacity: size_t; srcBuffer: Pointer; srcSize: size_t; preferencesPtr: PLZ4F_preferences_t) : size_t cdecl; - LZ4_compressHC2: function(const src: Pointer; dst: Pointer; srcSize: Integer; - compressionLevel: Integer): Integer cdecl; LZ4F_compressFrameBound: function(srcSize: size_t; preferencesPtr: PLZ4F_preferences_t): size_t cdecl; LZ4F_createDecompressionContext: function(out dctxPtr: LZ4F_dctx; @@ -106,7 +104,6 @@ begin @LZ4_compress_default := GetProcAddress(DLLHandle, 'LZ4_compress_default'); @LZ4_compress_fast := GetProcAddress(DLLHandle, 'LZ4_compress_fast'); @LZ4_compress_HC := GetProcAddress(DLLHandle, 'LZ4_compress_HC'); - @LZ4_compressHC2 := GetProcAddress(DLLHandle, 'LZ4_compressHC2'); @LZ4F_compressFrame := GetProcAddress(DLLHandle, 'LZ4F_compressFrame'); @LZ4F_compressFrameBound := GetProcAddress(DLLHandle, 'LZ4F_compressFrameBound'); diff --git a/imports/LZMADLL.pas b/imports/LZMADLL.pas deleted file mode 100644 index 58ac065..0000000 --- a/imports/LZMADLL.pas +++ /dev/null @@ -1,28 +0,0 @@ -unit LZMADLL; - -interface - -uses - WinAPI.Windows, - System.SysUtils; - -type - PFL2_inBuffer = ^FL2_inBuffer; - - FL2_inBuffer = record - src: Pointer; - size: size_t; - pos: size_t; - end; - - PFL2_outBuffer = ^FL2_outBuffer; - - FL2_outBuffer = record - dst: Pointer; - size: size_t; - pos: size_t; - end; - -implementation - -end. diff --git a/imports/PackJPGDLL.pas b/imports/PackJPGDLL.pas deleted file mode 100644 index b1dbe99..0000000 --- a/imports/PackJPGDLL.pas +++ /dev/null @@ -1,67 +0,0 @@ -unit PackJPGDLL; - -interface - -uses - WinAPI.Windows, - System.SysUtils, System.Classes; - -const - pjglib_file = 0; - pjglib_memory = 1; - pjglib_handle = 2; - -var - pjglib_convert_stream2stream: function(msg: PAnsiChar): Boolean cdecl; - pjglib_convert_file2file: function(ain, aout, msg: PAnsiChar): Boolean cdecl; - pjglib_convert_stream2mem: function(out_file: PPAnsiChar; out_size: PCardinal; - msg: PAnsiChar): Boolean cdecl; - pjglib_init_streams: procedure(in_src: Pointer; in_type: Integer; - in_size: Integer; out_dest: Pointer; out_type: Integer)cdecl; - pjglib_version_info: function: PAnsiChar cdecl; - pjglib_short_name: function: PAnsiChar cdecl; - DLLLoaded: Boolean = False; - -implementation - -var - DLLHandle: THandle; - -procedure Init; -begin - DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + - 'packjpg_dll.dll')); - if DLLHandle >= 32 then - begin - @pjglib_convert_stream2stream := GetProcAddress(DLLHandle, - 'pjglib_convert_stream2stream'); - @pjglib_convert_file2file := GetProcAddress(DLLHandle, - 'pjglib_convert_file2file'); - @pjglib_convert_stream2mem := GetProcAddress(DLLHandle, - 'pjglib_convert_stream2mem'); - @pjglib_init_streams := GetProcAddress(DLLHandle, 'pjglib_init_streams'); - @pjglib_version_info := GetProcAddress(DLLHandle, 'pjglib_version_info'); - @pjglib_short_name := GetProcAddress(DLLHandle, 'pjglib_short_name'); - DLLLoaded := Assigned(pjglib_init_streams) and - Assigned(pjglib_convert_stream2stream); - end - else - DLLLoaded := False; -end; - -procedure Deinit; -begin - if not DLLLoaded then - exit; - FreeLibrary(DLLHandle); -end; - -initialization - -Init; - -finalization - -Deinit; - -end. diff --git a/imports/TTADLL.pas b/imports/TTADLL.pas deleted file mode 100644 index 12ba17e..0000000 --- a/imports/TTADLL.pas +++ /dev/null @@ -1,53 +0,0 @@ -unit TTADLL; - -interface - -uses - WinAPI.Windows, - System.SysUtils, System.Classes; - -var - tta_encode: function(const src: Pointer; srcSize: PInteger; dst: Pointer; - dstCapacity: PInteger): Boolean cdecl; - tta_decode: function(const src: Pointer; srcSize: Integer; dst: Pointer; - dstCapacity: PInteger): Boolean cdecl; - tta_getsize: function(const src: Pointer; srcSize: Integer; - headerSize: PInteger): Integer cdecl; - DLLLoaded: Boolean = False; - -implementation - -var - DLLHandle: THandle; - -procedure Init; -begin - DLLHandle := LoadLibrary(PChar(ExtractFilePath(ParamStr(0)) + - 'libtta_dll.dll')); - if DLLHandle >= 32 then - begin - @tta_encode := GetProcAddress(DLLHandle, 'encode'); - @tta_decode := GetProcAddress(DLLHandle, 'decode'); - @tta_getsize := GetProcAddress(DLLHandle, 'getsize'); - DLLLoaded := Assigned(tta_encode) and Assigned(tta_decode); - end - else - DLLLoaded := False; -end; - -procedure Deinit; -begin - if not DLLLoaded then - exit; - FreeLibrary(DLLHandle); -end; - -initialization - -Init; - -finalization - -Deinit; - -end. diff --git a/io/IODecode.pas b/io/IODecode.pas index a82389b..4ee9769 100644 --- a/io/IODecode.pas +++ b/io/IODecode.pas @@ -128,7 +128,7 @@ begin '_' + Random($7FFFFFFF).ToHexString + XTOOL_MAPSUF2)), LFilename); try Move(SS2.Memory^, (PByte(SS1.Memory) + LEntry.Position)^, - Min(SS2.Size, LEntry.Size)); + LEntry.Size); finally SS2.Free; end; diff --git a/precompressor/PrecompCrypto.pas b/precompressor/PrecompCrypto.pas index 26a198c..a16fb9b 100644 --- a/precompressor/PrecompCrypto.pas +++ b/precompressor/PrecompCrypto.pas @@ -91,7 +91,7 @@ begin DI2.OldSize := SI.NewSize; DI2.NewSize := SI.NewSize; Funcs^.LogScan1(CryptoCodecs[GetBits(SI.Option, 0, 5)], SI.Position, - SI.OldSize, -1); + SI.OldSize, SI.NewSize); Add(Instance, @SI, DI1.Codec, @DI2); end; end; @@ -108,10 +108,10 @@ begin exit; if (Res > 0) and (StreamInfo^.OldSize > 0) then begin - Output(Instance, Input, StreamInfo^.OldSize); StreamInfo^.NewSize := StreamInfo^.OldSize; + Output(Instance, Input, StreamInfo^.OldSize); Funcs^.LogScan2(CryptoCodecs[GetBits(StreamInfo^.Option, 0, 5)], - StreamInfo^.OldSize, -1); + StreamInfo^.OldSize, StreamInfo^.NewSize); Result := True; end; end; @@ -143,7 +143,7 @@ begin end; Result := True; Funcs^.LogProcess(CryptoCodecs[GetBits(StreamInfo^.Option, 0, 5)], nil, - StreamInfo^.OldSize, -1, -1, Result); + StreamInfo^.OldSize, StreamInfo^.NewSize, StreamInfo^.OldSize, Result); end; end; @@ -175,7 +175,7 @@ begin Output(Instance, Input, StreamInfo.OldSize); Result := True; Funcs^.LogRestore(CryptoCodecs[GetBits(StreamInfo.Option, 0, 5)], nil, - StreamInfo.OldSize, -1, -1, Result); + StreamInfo.OldSize, StreamInfo.NewSize, StreamInfo.OldSize, Result); end; end; diff --git a/precompressor/PrecompEXE.pas b/precompressor/PrecompEXE.pas index 9dfca77..3c9cd8a 100644 --- a/precompressor/PrecompEXE.pas +++ b/precompressor/PrecompEXE.pas @@ -589,7 +589,7 @@ begin StreamInfo^.OldSize); Funcs^.LogProcess(PChar(Codec.Names[X]), nil, StreamInfo^.OldSize, StreamInfo^.NewSize, Res1, Result); - if (Result = False) and (DIFF_TOLERANCE > 0) then + if Result = False then begin Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1)); diff --git a/precompressor/PrecompLZ4.pas b/precompressor/PrecompLZ4.pas index ff9005a..f51b8b0 100644 --- a/precompressor/PrecompLZ4.pas +++ b/precompressor/PrecompLZ4.pas @@ -24,14 +24,12 @@ const L_MAXSIZE = 16 * 1024 * 1024; L_BLOCKSIZE = 0; L_BLOCKDEPENDENCY = 0; - L_ACCELERATION = 1; var SOList: array of array [0 .. CODEC_COUNT - 1] of TSOList; CodecAvailable, CodecEnabled: TArray; LBlockSize: Integer = L_BLOCKSIZE; LBlockDependency: Integer = L_BLOCKDEPENDENCY; - LAcceleration: Integer = L_ACCELERATION; function LZ4Init(Command: PChar; Count: Integer; Funcs: PPrecompFuncs): Boolean; var @@ -59,8 +57,6 @@ begin if (CompareText(S, LZ4Codecs[LZ4_CODEC]) = 0) and LZ4DLL.DLLLoaded then begin CodecEnabled[LZ4_CODEC] := True; - if Funcs^.GetParam(Command, X, 'a') <> '' then - LAcceleration := StrToInt(Funcs^.GetParam(Command, X, 'a')); end else if (CompareText(S, LZ4Codecs[LZ4HC_CODEC]) = 0) and LZ4DLL.DLLLoaded then @@ -90,7 +86,7 @@ begin if SOList[X, LZ4_CODEC].Count = 0 then SOList[X, LZ4_CODEC].Update([1]); SetLength(Options, 0); - for I := 2 to 12 do + for I := 3 to 12 do Insert(I, Options, Length(Options)); for X := Low(SOList) to High(SOList) do if SOList[X, LZ4HC_CODEC].Count = 0 then @@ -122,7 +118,6 @@ begin Option^ := 0; SetBits(Option^, LBlockSize, 12, 2); SetBits(Option^, LBlockDependency, 14, 1); - SetBits(Option^, LAcceleration, 15, 7); I := 0; while Funcs^.GetCodec(Command, I, False) <> '' do begin @@ -130,8 +125,6 @@ begin if (CompareText(S, LZ4Codecs[LZ4_CODEC]) = 0) and LZ4DLL.DLLLoaded then begin SetBits(Option^, LZ4_CODEC, 0, 5); - if Funcs^.GetParam(Command, I, 'a') <> '' then - SetBits(Option^, StrToInt(Funcs^.GetParam(Command, I, 'a')), 15, 7); Result := True; end else if (CompareText(S, LZ4Codecs[LZ4HC_CODEC]) = 0) and LZ4DLL.DLLLoaded @@ -195,7 +188,6 @@ begin SetBits(SI.Option, X, 0, 5); SetBits(SI.Option, LBlockSize, 12, 2); SetBits(SI.Option, LBlockDependency, 14, 1); - SetBits(SI.Option, LAcceleration, 15, 7); if System.Pos(SPrecompSep2, DI1.Codec) > 0 then SI.Status := TStreamStatus.Predicted else @@ -238,8 +230,8 @@ begin end; if Res > StreamInfo^.OldSize then begin - Output(Instance, Buffer, Res); StreamInfo^.NewSize := Res; + Output(Instance, Buffer, Res); Funcs^.LogScan2(LZ4Codecs[GetBits(StreamInfo^.Option, 0, 5)], StreamInfo^.OldSize, StreamInfo^.NewSize); Result := True; @@ -269,13 +261,12 @@ begin if StreamInfo^.Status = TStreamStatus.Predicted then if GetBits(StreamInfo^.Option, 5, 7) <> I then continue; - Params := ''; case X of LZ4_CODEC: begin - Params := 'a' + GetBits(StreamInfo^.Option, 15, 7).ToString; - Res1 := LZ4_compress_fast(NewInput, Buffer, StreamInfo^.NewSize, Y, - GetBits(StreamInfo^.Option, 15, 7)); + Params := ''; + Res1 := LZ4_compress_default(NewInput, Buffer, + StreamInfo^.NewSize, Y); end; LZ4HC_CODEC: begin @@ -305,7 +296,7 @@ begin break; end; if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or - (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then + (SOList[Instance][X].Count = 1)) then begin Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1)); Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1, @@ -341,16 +332,14 @@ begin X := GetBits(StreamInfo.Option, 0, 5); if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then exit; - Params := ''; Buffer := Funcs^.Allocator(Instance, LZ4F_compressFrameBound(StreamInfo.NewSize, nil)); case X of LZ4_CODEC: begin - Params := 'a' + GetBits(StreamInfo.Option, 15, 7).ToString; - Res1 := LZ4_compress_fast(Input, Buffer, StreamInfo.NewSize, - LZ4F_compressFrameBound(StreamInfo.NewSize, nil), - GetBits(StreamInfo.Option, 15, 7)); + Params := ''; + Res1 := LZ4_compress_default(Input, Buffer, StreamInfo.NewSize, + LZ4F_compressFrameBound(StreamInfo.NewSize, nil)); end; LZ4HC_CODEC: begin diff --git a/precompressor/PrecompLZO.pas b/precompressor/PrecompLZO.pas index f090292..472fbcf 100644 --- a/precompressor/PrecompLZO.pas +++ b/precompressor/PrecompLZO.pas @@ -272,8 +272,8 @@ begin end; if Res > StreamInfo^.OldSize then begin - Output(Instance, Buffer, Res); StreamInfo^.NewSize := Res; + Output(Instance, Buffer, Res); Funcs^.LogScan2(LZOCodecs[GetBits(StreamInfo^.Option, 0, 5)], StreamInfo^.OldSize, StreamInfo^.NewSize); Result := True; @@ -301,7 +301,6 @@ begin if StreamInfo^.Status = TStreamStatus.Predicted then if GetBits(StreamInfo^.Option, 5, 7) <> I then continue; - Params := ''; Res1 := StreamInfo^.NewSize; case X of LZO1X_CODEC: @@ -327,7 +326,7 @@ begin break; end; if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or - (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then + (SOList[Instance][X].Count = 1)) then begin Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1)); Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1, @@ -362,7 +361,6 @@ begin X := GetBits(StreamInfo.Option, 0, 5); if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then exit; - Params := ''; Buffer := Funcs^.Allocator(Instance, StreamInfo.NewSize); Res1 := StreamInfo.NewSize; case X of diff --git a/precompressor/PrecompMain.pas b/precompressor/PrecompMain.pas index aa53630..b7a1cba 100644 --- a/precompressor/PrecompMain.pas +++ b/precompressor/PrecompMain.pas @@ -7,7 +7,7 @@ interface uses Threading, Utils, SynCommons, ParseClass, ParseExpr, PrecompUtils, PrecompCrypto, PrecompZLib, PrecompLZ4, PrecompLZO, PrecompZSTD, - PrecompOodle, PrecompMedia, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE, + PrecompOodle, PrecompINI, PrecompSearch, PrecompDLL, PrecompEXE, WinAPI.Windows, WinAPI.ShlObj, System.SysUtils, System.Classes, System.SyncObjs, System.Math, System.Types, System.StrUtils, System.RTLConsts, System.TimeSpan, System.Diagnostics, @@ -24,7 +24,8 @@ type ChunkSize, Threads: Integer; Depth: Integer; LowMem: Boolean; - DBaseFile, ExtractDir: String; + DBaseFile: String; + DedupFile: String; end; PDecodeOptions = ^TDecodeOptions; @@ -33,6 +34,7 @@ type Method: String; ChunkCount, Threads: Integer; Depth: Integer; + DedupFile: String; DedupSysMem, DedupGPUMem: Int64; end; @@ -51,10 +53,10 @@ function PrecompReadFuture(Index: Integer; Position: NativeInt; Buffer: Pointer; procedure PrecompLogScan1(Codec: PChar; Position: Int64; InSize, OutSize: Integer)cdecl; procedure PrecompLogScan2(Codec: PChar; InSize, OutSize: Integer)cdecl; -procedure PrecompLogProcess(Codec, Method: PChar; Size1, Size2, Size3: Integer; - Status: Boolean)cdecl; -procedure PrecompLogRestore(Codec, Method: PChar; Size1, Size2, Size3: Integer; - Status: Boolean)cdecl; +procedure PrecompLogProcess(Codec, Method: PChar; + OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl; +procedure PrecompLogRestore(Codec, Method: PChar; + OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl; procedure PrecompLogPatch1(OldSize, NewSize, PatchSize: Integer; Status: Boolean)cdecl; procedure PrecompLogPatch2(OldSize, NewSize, PatchSize: Integer; @@ -99,11 +101,10 @@ var IntArray: array [0 .. 1] of Int64; Codecs: array of TPrecompressor; DBFile: String = ''; - ExtDir: String = ''; UseDB: Boolean = False; + DupFile: String = ''; StoreDD: Boolean = False; - VERBOSE: Boolean = False; - EXTRACT: Boolean = False; + DupGUID: TGUID; DupSysMem: Int64 = 0; EncInfo: TEncInfo; ConTask: TTask; @@ -130,14 +131,11 @@ begin WriteLn(ErrOutput, ''); WriteLn(ErrOutput, 'Advanced parameters:'); WriteLn(ErrOutput, - ' --dbase=# - use database (#=filename to save db, optional)'); + ' --dbase=# - use database (#=filename to save db, optional)'); WriteLn(ErrOutput, - ' --dedup=# - use stream deduplication (#=filename to save db, optional)'); + ' --dedup=# - use stream deduplication (#=filename to save db, optional)'); WriteLn(ErrOutput, - ' --mem=# - deduplication ram usage limit (#=size) [75p]'); - WriteLn(ErrOutput, - ' --diff=# - set xdelta threshold to accept streams [5p]'); - WriteLn(ErrOutput, ' --extract=# - extract streams to directory path'); + ' --mem=# - deduplication ram usage limit (#=size) [75p]'); WriteLn(ErrOutput, ''); end; @@ -184,18 +182,16 @@ begin if Options.DBaseFile <> '' then UseDB := True; StoreDD := ArgParse.AsBoolean('--dedup'); + Options.DedupFile := ArgParse.AsString('--dedup='); S := ArgParse.AsString('--diff=', 0, '5p'); S := ReplaceText(S, 'p', '%'); DIFF_TOLERANCE := Max(0.00, ExpParse.Evaluate(S)); VERBOSE := ArgParse.AsBoolean('--verbose'); - Options.ExtractDir := ArgParse.AsString('--extract='); - if Options.ExtractDir <> '' then - EXTRACT := DirectoryExists(Options.ExtractDir); finally ArgParse.Free; ExpParse.Free; end; - if VERBOSE or EXTRACT then + if VERBOSE then Options.Threads := 1; end; @@ -214,6 +210,7 @@ begin S := ReplaceText(S, 'p', '%'); S := ReplaceText(S, '%', '%*' + CPUCount.ToString); Options.Threads := Max(1, Round(ExpParse.Evaluate(S))); + Options.DedupFile := ArgParse.AsString('--dedup='); S := ArgParse.AsString('--mem=', 0, '75p'); S := ReplaceText(S, 'KB', '* 1024^1'); S := ReplaceText(S, 'MB', '* 1024^2'); @@ -270,13 +267,14 @@ type StrIdx: TArray; end; + TDupRec = record + Dict: TSynDictionary; + Index: Integer; + end; + var - DBInfo: TArray>; - DBCount: TArray; - DDInfo: TArray>; - DDCount1: TArray; - DDList1: TArray; - DDIndex: Integer; + Database: TSynDictionary; + Duplicates1: array [0 .. 1] of TDupRec; ComVars1: TArray; Tasks: TArray; CurCodec: TArray; @@ -287,7 +285,6 @@ var Scanned1, Scanned2, Processed: TArray; LogInt: Integer; LogInt64: Int64; - LogPtr: Pointer; procedure CodecInit(Count: Integer; Method: String); var @@ -306,7 +303,6 @@ begin Insert(PrecompLZO.Codec, Codecs, Length(Codecs)); Insert(PrecompZSTD.Codec, Codecs, Length(Codecs)); Insert(PrecompOodle.Codec, Codecs, Length(Codecs)); - Insert(PrecompMedia.Codec, Codecs, Length(Codecs)); for X := Low(Codecs) to High(Codecs) do for Y := Low(Codecs[X].Names) to High(Codecs[X].Names) do Insert(Codecs[X].Names[Y], List, Length(List)); @@ -398,101 +394,64 @@ end; procedure PrecompLogScan1(Codec: PChar; Position: Int64; InSize, OutSize: Integer); -var - S: String; begin if not VERBOSE then exit; with ComVars1[CurDepth[0]] do begin - if OutSize < 0 then - S := '(%d)' - else - S := '(%d >> %d)'; if (OutSize > 0) and (Position < DataStore.Size(0)) and (MemOutput1[0].Position - CurPos1[0] = OutSize) then - WriteLn(ErrOutput, Format('[%d] Actual %s stream found at %s ' + S, + WriteLn(ErrOutput, Format('[%d] Actual %s stream found at %s (%d >> %d)', [CurDepth[0], Codec, (DataStore.Position(0) + Position).ToHexString, InSize, OutSize])) else - WriteLn(ErrOutput, Format('[%d] Possible %s stream located at %s ' + S, + WriteLn(ErrOutput, + Format('[%d] Possible %s stream located at %s (%d >> %d)', [CurDepth[0], Codec, (DataStore.Position(0) + Position).ToHexString, InSize, OutSize])); end; end; procedure PrecompLogScan2(Codec: PChar; InSize, OutSize: Integer); +begin + if not VERBOSE then + exit; + WriteLn(ErrOutput, Format('[%d] Confirmed %s stream at %s (%d >> %d)', + [CurDepth[0], Codec, LogInt64.ToHexString, InSize, OutSize])); +end; + +procedure PrecompLogProcess(Codec, Method: PChar; + OriginalSize, InSize, OutSize: Integer; Status: Boolean); var S: String; begin if not VERBOSE then exit; - if OutSize < 0 then - S := '(%d)' + if Status then + S := '[%d] Processed %s stream at %s (%d >> %d >> %d)' + + IfThen(String(Method) <> '', ' using %s', '') + ' successfully' else - S := '(%d >> %d)'; - WriteLn(ErrOutput, Format('[%d] Confirmed %s stream at %s ' + S, - [CurDepth[0], Codec, LogInt64.ToHexString, InSize, OutSize])); + S := '[%d] Processing %s stream at %s (%d >> %d >> %d)' + + IfThen(String(Method) <> '', ' using %s', '') + ' has failed'; + WriteLn(ErrOutput, Format(S, [CurDepth[0], Codec, LogInt64.ToHexString, + OriginalSize, InSize, OutSize, Method])); end; -procedure PrecompLogProcess(Codec, Method: PChar; Size1, Size2, Size3: Integer; - Status: Boolean); +procedure PrecompLogRestore(Codec, Method: PChar; + OriginalSize, InSize, OutSize: Integer; Status: Boolean); var - S1, S2: String; -begin - if VERBOSE then - begin - if Size2 < 0 then - S1 := '(%d)' - else if Size3 < 0 then - S1 := '(%d >> %d)' - else - S1 := '(%d >> %d >> %d)'; - if Status then - S2 := '[%d] Processed %s stream at %s ' + S1 + - IfThen(String(Method) <> '', ' using ' + String(Method), '') + - ' successfully' - else - S2 := '[%d] Processing %s stream at %s ' + S1 + - IfThen(String(Method) <> '', ' using ' + String(Method), '') + - ' has failed'; - WriteLn(ErrOutput, Format(S2, [CurDepth[0], Codec, LogInt64.ToHexString, - Size1, Size2, Size3])); - end; - if EXTRACT and (CurDepth[0] = 0) then - begin - S1 := '%s_%s.raw'; - with TFileStream.Create(ExtDir + Format(S1, [LogInt64.ToHexString, Codec]), - fmCreate) do - try - WriteBuffer(LogPtr^, Size1); - finally - Free; - end; - end; -end; - -procedure PrecompLogRestore(Codec, Method: PChar; Size1, Size2, Size3: Integer; - Status: Boolean); -var - S1, S2: String; + S: String; begin if not VERBOSE then exit; - if Size2 < 0 then - S1 := '(%d)' - else if Size3 < 0 then - S1 := '(%d >> %d)' - else - S1 := '(%d >> %d >> %d)'; if Status then - S2 := '[%d] Restored %s stream at %s ' + S1 + IfThen(String(Method) <> '', - ' using ' + String(Method), '') + ' successfully' + S := '[%d] Restored %s stream at %s (%d >> %d >> %d)' + + IfThen(String(Method) <> '', ' using %s', '') + ' successfully' else - S2 := '[%d] Restoring %s stream at %s ' + S1 + IfThen(String(Method) <> '', - ' using ' + String(Method), '') + ' has failed'; - WriteLn(ErrOutput, Format(S2, [CurDepth[0], Codec, LogInt64.ToHexString, - Size1, Size2, Size3])); + S := '[%d] Restoring %s stream at %s (%d >> %d >> %d)' + + IfThen(String(Method) <> '', ' using %s', '') + ' has failed'; + WriteLn(ErrOutput, Format(S, [CurDepth[0], Codec, LogInt64.ToHexString, + OriginalSize, InSize, OutSize, Method])); end; procedure PrecompLogPatch1(OldSize, NewSize, PatchSize: Integer; @@ -594,9 +553,7 @@ begin begin MemOutput1[Instance].Position := CurPos1[Instance]; exit; - end - else if LCodec = CurCodec[Instance] then - LOption := Info^.Option; + end; end else begin @@ -659,130 +616,53 @@ begin CurTransfer[Instance] := String(Codec); end; -function CheckDB(StreamInfo: TEncodeSI; Database: PDatabase): Boolean; +function CheckDB(Dictionary: TSynDictionary; const StreamInfo: TEncodeSI; + var Database: TDatabase): Boolean; var - A: Word; - I: Integer; - LCount: Integer; - DB: PDatabase; + DBKey: Int64; begin Result := False; - A := LongRec(StreamInfo.Checksum).Lo; - AtomicExchange(LCount, DBCount[A]); - for I := 0 to LCount - 1 do - begin - DB := @DBInfo[A, I]; - if (DB^.Size = StreamInfo.OldSize) and (DB^.Checksum = StreamInfo.Checksum) - then - begin - if Assigned(Database) then - Move(DB^, Database^, SizeOf(TDatabase)); - Result := True; - break; - end; - end; + Int64Rec(DBKey).Lo := StreamInfo.Checksum; + Int64Rec(DBKey).Hi := StreamInfo.OldSize; + Result := Dictionary.FindAndCopy(DBKey, Database); end; -procedure AddDB(StreamInfo: TEncodeSI); +procedure AddDB(Dictionary: TSynDictionary; const StreamInfo: TEncodeSI; + const Database: TDatabase); var - A: Word; - I: Integer; - DB: TDatabase; + DBKey: Int64; begin - A := LongRec(StreamInfo.Checksum).Lo; - if not CheckDB(StreamInfo, nil) then - begin - GlobalSync.Acquire; - try - DB.Size := StreamInfo.OldSize; - DB.Codec := StreamInfo.Codec; - DB.Option := StreamInfo.Option; - DB.Checksum := StreamInfo.Checksum; - DB.Status := StreamInfo.Status; - Insert(DB, DBInfo[A], Length(DBInfo[A])); - Inc(DBCount[A]); - finally - GlobalSync.Release; - end; - end; + Int64Rec(DBKey).Lo := StreamInfo.Checksum; + Int64Rec(DBKey).Hi := StreamInfo.OldSize; + Dictionary.AddOrUpdate(DBKey, Database); end; -function CheckDD(StreamInfo: TEncodeSI; Database: PDuplicate1; - Index: PInteger): Boolean; +function CheckDup(var DupRec: TDupRec; const StreamInfo: TEncodeSI; + var StreamKey, DupCount: Integer): Boolean; var - A: Word; - I: Integer; - LCount: Integer; - DD: PDuplicate1; + DupKey: Int64; + DupInfo: PDuplicate; + DupAdded: Boolean; begin Result := False; - A := LongRec(StreamInfo.Checksum).Lo; - LCount := DDCount1[A]; - for I := 0 to LCount - 1 do + Inc(DupRec.Index); + Int64Rec(DupKey).Lo := StreamInfo.Checksum; + Int64Rec(DupKey).Hi := StreamInfo.OldSize; + DupInfo := DupRec.Dict.FindValueOrAdd(DupKey, DupAdded); + if not DupAdded then begin - DD := @DDInfo[A, I]; - if (DD^.Size = StreamInfo.OldSize) and (DD^.Checksum = StreamInfo.Checksum) - then - begin - if Assigned(Database) then - Move(DD^, Database^, SizeOf(TDuplicate1)); - if Assigned(Index) then - Index^ := I; - Result := True; - break; - end; - end; -end; - -function FindDD(StreamInfo: TEncodeSI; Index, Count: PInteger): Boolean; -var - A: Word; - I: Integer; - DD: PDuplicate1; -begin - Result := False; - if CheckDD(StreamInfo, nil, @I) then - begin - A := LongRec(StreamInfo.Checksum).Lo; - DD := @DDInfo[A, I]; - if Assigned(Index) then - Index^ := DD^.Index; - if Assigned(Count) then - Count^ := DD^.Count; - Result := True; - end; -end; - -function FindOrAddDD(StreamInfo: TEncodeSI; Index, Count: PInteger): Boolean; -var - A: Word; - I: Integer; - DD: TDuplicate1; - I64: Int64; -begin - Result := False; - Inc(DDIndex); - A := LongRec(StreamInfo.Checksum).Lo; - if not CheckDD(StreamInfo, nil, @I) then - begin - DD.Size := StreamInfo.OldSize; - DD.Checksum := StreamInfo.Checksum; - DD.Index := DDIndex; - DD.Count := 0; - I := Length(DDInfo[A]); - Insert(DD, DDInfo[A], I); - Int64Rec(I64).Words[0] := A; - Int64Rec(I64).Hi := DDCount1[A]; - Insert(I64, DDList1, Length(DDList1)); - Inc(DDCount1[A]); Result := True; + Inc(DupInfo^.Count); + StreamKey := DupInfo^.Index; + DupCount := DupInfo^.Count; end else - Inc(DDInfo[A, I].Count); - if Assigned(Index) then - Index^ := DDInfo[A, I].Index; - if Assigned(Count) then - Count^ := DDInfo[A, I].Count; + begin + DupInfo^.Count := 0; + DupInfo^.Index := DupRec.Index; + StreamKey := -1; + DupCount := 0; + end; end; procedure Scan1(Index, Depth: Integer); @@ -849,8 +729,6 @@ begin X := DataStore.ActualSize(Index) - NativeInt(SI2.Position - DataStore.Position(Index)); LogInt64 := SI2.Position; - LogPtr := PByte(DataStore.Slot(Index).Memory) + - NativeInt(SI2.Position - DataStore.Position(Index)); if (SI1.OldSize <= X) and Codecs[SI2.Codec].Scan2(Index, Depth, PByte(DataStore.Slot(Index).Memory) + NativeInt(SI2.Position - DataStore.Position(Index)), X, @SI1, @J, @@ -952,11 +830,10 @@ begin SI1.Resource := SI2.Resource; SI1.Option := SI2.Option; SI1.Status := SI2.Status; - LogInt64 := DataStore.Position(ThreadIndex) + SI2.ActualPosition; - LogPtr := PByte(DataStore.Slot(ThreadIndex).Memory) + SI2.ActualPosition; + LogInt64 := DataStore.Position(0) + SI2.ActualPosition; if UseDB and (SI2.Codec > 2) then begin - DBBool := CheckDB(SI2, @DBTyp); + DBBool := CheckDB(Database, SI2, DBTyp); if DBBool and (SI2.Codec = DBTyp.Codec) then begin if DBTyp.Status = TStreamStatus.Invalid then @@ -964,7 +841,7 @@ begin else begin SI1.Option := DBTyp.Option; - SI1.Status := TStreamStatus.Database; + SI1.Status := TStreamStatus.Predicted; end; end; end; @@ -1005,14 +882,13 @@ begin if UseDB then if not DBBool then begin + DBTyp.Codec := SI2.Codec; + DBTyp.Option := SI1.Option; if Result then - begin - SI2.Option := SI1.Option; - SI2.Status := TStreamStatus.Predicted - end + DBTyp.Status := TStreamStatus.Predicted else - SI2.Status := TStreamStatus.Invalid; - AddDB(SI2); + DBTyp.Status := TStreamStatus.Invalid; + AddDB(Database, SI2, DBTyp); end; if Result then begin @@ -1157,9 +1033,10 @@ procedure EncInit(Input, Output: TStream; Options: PEncodeOptions); var UI32: UInt32; I, J, K: Integer; - W: Word; Bytes: TBytes; NI: NativeInt; + DBKey: Int64; + DBTyp: TDatabase; S: String; DupMethod: Boolean; begin @@ -1169,21 +1046,15 @@ begin ThreadSync[I] := TCriticalSection.Create; I := XTOOL_PRECOMP; Output.WriteBuffer(I, I.Size); - if UseDB then + CreateGUID(DupGUID); + Output.WriteBuffer(DupGUID, SizeOf(TGUID)); + Database := TSynDictionary.Create(TypeInfo(TInt64DynArray), + TypeInfo(TDatabaseDynArray)); + for I := Low(Duplicates1) to High(Duplicates1) do begin - SetLength(DBInfo, $10000); - SetLength(DBCount, $10000); - for I := Low(DBInfo) to High(DBInfo) do - DBCount[I] := 0; - end; - if StoreDD then - begin - SetLength(DDInfo, $10000); - SetLength(DDCount1, $10000); - SetLength(DDList1, 0); - for I := Low(DDInfo) to High(DDInfo) do - DDCount1[I] := 0; - DDIndex := -1; + Duplicates1[I].Dict := TSynDictionary.Create(TypeInfo(TInt64DynArray), + TypeInfo(TDuplicateDynArray)); + Duplicates1[I].Index := -1; end; SetLength(Tasks, Options^.Threads); SetLength(CurCodec, Options^.Threads); @@ -1255,6 +1126,7 @@ begin end; CodecInit(Options^.Threads, Options^.Method); DBFile := Options^.DBaseFile; + DupFile := Options^.DedupFile; if FileExists(ExtractFilePath(Utils.GetModuleName) + DBFile) then begin with TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + DBFile, @@ -1268,19 +1140,16 @@ begin end; with WorkStream[0] do begin - Position := 0; - while Position < Size do + J := PInteger(Memory)^; + for I := 0 to J - 1 do begin - ReadBuffer(W, W.Size); - ReadBuffer(J, J.Size); - DBCount[W] := J; - SetLength(DBInfo[W], J); - for K := 0 to J - 1 do - ReadBuffer(DBInfo[W, K], SizeOf(TDatabase)); + NI := Integer.Size + (I * (SizeOf(Int64) + SizeOf(TDatabase))); + DBKey := PInt64(PByte(Memory) + NI)^; + DBTyp := PDatabase(PByte(Memory) + NI + SizeOf(Int64))^; + Database.Add(DBKey, DBTyp); end; end; end; - ExtDir := IncludeTrailingBackSlash(Options^.ExtractDir); Output.WriteBuffer(Options^.Depth, Options^.Depth.Size); S := ''; I := 0; @@ -1368,6 +1237,9 @@ begin Tasks[I].Free; WorkStream[I].Free; end; + Database.Free; + for I := Low(Duplicates1) to High(Duplicates1) do + Duplicates1[I].Dict.Free; FreeResources; GlobalSync.Free; for I := Low(ThreadSync) to High(ThreadSync) do @@ -1393,19 +1265,19 @@ var StreamCount: Integer; BlockSize: Int64; UI32: UInt32; - I, J, K, X: Integer; - W: Word; - I64: Int64; + I, J, X: Integer; LastStream, LastPos: Int64; LastIndex: Integer; CurrSize: Cardinal; DupBool: Boolean; - DupIdx1, DupIdx2, DupCount: Integer; - DupTyp: TDuplicate2; + DupKey, DupCount: Integer; + DBKey: Int64; + DBTyp: TDatabase; + DupTyp: TDuplicate; begin if (Depth = 0) then begin - if StoreDD then + if (DupFile = '') and StoreDD then TempOutput := TPrecompVMStream.Create else TempOutput := Output; @@ -1413,7 +1285,6 @@ begin else TempOutput := Output; Result := False; - DupIdx1 := 0; with ComVars1[Depth] do begin LastStream := 0; @@ -1510,15 +1381,16 @@ begin begin Inc(StreamCount); DupBool := False; - if (Depth = 0) and StoreDD then - DupBool := not FindOrAddDD(StreamInfo, @DupIdx2, @DupCount); + if (Depth = 0) and ((DupFile <> '') or StoreDD) then + DupBool := CheckDup(Duplicates1[0], StreamInfo, DupKey, + DupCount); if DupBool then begin - if DupCount = 1 then + if DupCount = 2 then Inc(EncInfo.DecMem2, StreamInfo.OldSize); FillChar(StreamHeader, SizeOf(TStreamHeader), 0); StreamHeader.Kind := DUPLICATED_STREAM; - StreamHeader.Option := DupIdx2; + StreamHeader.Option := DupKey; end else begin @@ -1555,19 +1427,19 @@ begin Result := True else if Depth > 0 then exit; - I64 := MemStream[I].Position; MemStream[I].Position := 0; MemStream[I].WriteBuffer(StreamCount, StreamCount.Size); MemStream[I].WriteBuffer(BlockSize, BlockSize.Size); - TempOutput.WriteBuffer(MemStream[I].Memory^, I64); + TempOutput.WriteBuffer(MemStream[I].Memory^, MemStream[I].Position + + StreamCount * SizeOf(TStreamHeader)); InfoStore1[I].Index := LastIndex; J := InfoStore1[I].Get(StreamInfo); while J >= 0 do begin DupBool := False; - if (Depth = 0) and StoreDD then - DupBool := FindDD(StreamInfo, @DupIdx2, @DupCount); - if (DupBool = False) or (DupIdx1 = DupIdx2) then + if (Depth = 0) and ((DupFile <> '') or StoreDD) then + DupBool := CheckDup(Duplicates1[1], StreamInfo, DupKey, DupCount); + if not DupBool then begin if StreamInfo.ExtSize < 0 then begin @@ -1593,7 +1465,6 @@ begin StreamInfo.ExtSize.Size); end; end; - Inc(DupIdx1); if Succ(J - LastIndex) = StreamCount then break; J := InfoStore1[I].Get(StreamInfo); @@ -1648,16 +1519,14 @@ begin with WorkStream[0] do begin Position := 0; - for W := 0 to $10000 - 1 do + J := Database.Count; + WriteBuffer(J, J.Size); + for I := 0 to J - 1 do begin - J := DBCount[I]; - if J > 0 then - begin - WriteBuffer(W, W.Size); - WriteBuffer(J, J.Size); - for K := 0 to J - 1 do - WriteBuffer(DBInfo[W, K], SizeOf(TDatabase)); - end; + DBKey := PInt64(Database.Keys.ElemPtr(I))^; + WriteBuffer(DBKey, SizeOf(Int64)); + DBTyp := PDatabase(Database.Values.ElemPtr(I))^; + WriteBuffer(DBTyp, SizeOf(TDatabase)); end; end; with TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + DBFile, @@ -1667,27 +1536,40 @@ begin Free; end; end; - if StoreDD then + if (DupFile <> '') or StoreDD then begin + for I := Duplicates1[0].Dict.Count - 1 downto 0 do + begin + if PDuplicate(Duplicates1[0].Dict.Values.ElemPtr(I))^.Count < 1 then + Duplicates1[0].Dict.DeleteAt(I); + end; with WorkStream[0] do begin Position := 0; - UI32 := 0; - for I := Low(DDList1) to High(DDList1) do + WriteBuffer(DupGUID, SizeOf(TGUID)); + Duplicates1[0].Dict.Values.Sort(DuplicateSortCompare); + J := Duplicates1[0].Dict.Count; + WriteBuffer(J, J.Size); + for I := 0 to J - 1 do begin - J := Int64Rec(DDList1[I]).Words[0]; - X := Int64Rec(DDList1[I]).Hi; - if DDInfo[J, X].Count > 0 then - begin - DupTyp.Index := DDInfo[J, X].Index; - DupTyp.Count := DDInfo[J, X].Count; - WriteBuffer(DupTyp, SizeOf(TDuplicate2)); - Inc(UI32); - end; + DupTyp := PDuplicate(Duplicates1[0].Dict.Values.ElemPtr(I))^; + WriteBuffer(DupTyp, SizeOf(TDuplicate)); end; end; - Output.WriteBuffer(UI32, UI32.Size); - Output.WriteBuffer(WorkStream[0].Memory^, WorkStream[0].Position); + if DupFile <> '' then + begin + with TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + DupFile, + FSMode(FileExists(ExtractFilePath(Utils.GetModuleName) + DupFile))) do + begin + Position := Size; + WriteBuffer(WorkStream[0].Memory^, WorkStream[0].Position); + end; + end + else + Output.WriteBuffer(WorkStream[0].Memory^, WorkStream[0].Position); + end; + if (DupFile = '') and StoreDD then + begin Output.CopyFrom(TempOutput, 0); TempOutput.Free; end; @@ -1751,9 +1633,10 @@ var NStream: TArrayStream; DataMgr: TDataManager; ComVars2: TArray; - DDList2: TArray; - DDCount2: Integer; - DDIndex1, DDIndex2: Integer; + Duplicates2: TSynDictionary; + DupIdx1: Integer; + DupIdx2: TArray; + DupBool: TArray; BlockPos: Int64; procedure PrecompOutput2(Instance: Integer; const Buffer: Pointer; @@ -1761,9 +1644,8 @@ procedure PrecompOutput2(Instance: Integer; const Buffer: Pointer; begin with ComVars2[CurDepth[Instance]] do DecOutput[Instance].WriteBuffer(Buffer^, Size); - if StoreDD and (CurDepth[Instance] = 0) then - if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index)) then - DataMgr.Write(DDIndex1, Buffer, Size); + if (CurDepth[Instance] = 0) and (DupBool[Instance]) then + DataMgr.Write(DupIdx2[Instance], Buffer^, Size); end; procedure PrecompOutput3(Instance: Integer; const Buffer: Pointer; @@ -1771,6 +1653,8 @@ procedure PrecompOutput3(Instance: Integer; const Buffer: Pointer; begin with ComVars2[CurDepth[Instance]] do MemOutput1[Instance].WriteBuffer(Buffer^, Size); + if (CurDepth[Instance] = 0) and (DupBool[Instance]) then + DataMgr.Write(DupIdx2[Instance], Buffer^, Size); end; procedure Restore(MT: Boolean; Index, Depth: Integer); @@ -1791,11 +1675,18 @@ begin while X < StreamCount[Index]^ do begin SH := PStreamHeader(MemStream1[Index].Memory) + X; + if (Depth = 0) then + begin + DupIdx2[Index] := DupIdx1 + X; + DupBool[Index] := Duplicates2.FindAndCopy(DupIdx2[Index], Y); + if DupBool[Index] then + DataMgr.Add(DupIdx2[Index], SH^.OldSize, Y); + end; if MT then begin LOutput := @PrecompOutput3; Pos := StreamInfo[Index]^.Pos[X]; - X64 := Pos + Max(SH^.OldSize, SH^.NewSize); + X64 := Pos + SH^.NewSize; while (BlockPos < X64) do begin if IsErrored(Tasks) or (BlockPos < 0) then @@ -1806,13 +1697,6 @@ begin end else begin - if StoreDD and (Depth = 0) then - begin - Inc(DDIndex1); - if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index)) - then - DataMgr.Add(DDIndex1, SH^.OldSize, DDList2[DDIndex2].Count); - end; LOutput := @PrecompOutput2; DecInput[Index].ReadBuffer(UI32, UI32.Size); if UI32 > 0 then @@ -1856,12 +1740,9 @@ begin end; CurCodec[Index] := SH^.Codec; CurDepth[Index] := Depth; - Y := GetBits(SI.Option, 0, 5); - if not InRange(Y, 0, Pred(Length(Codecs[SH^.Codec].Names))) then - Y := 0; if (Codecs[SH^.Codec].Restore(Index, Depth, Ptr1, Ptr2, SI, LOutput, @PrecompFunctions) = False) then - raise Exception.CreateFmt(SPrecompError3, [Codecs[SH^.Codec].Names[Y]]); + raise Exception.CreateFmt(SPrecompError3, [Codecs[SH^.Codec].Names[0]]); NStream.Update(0, CalcSysMem); if MT then begin @@ -1870,13 +1751,7 @@ begin StreamInfo[Index]^.Completed[X] := True; end else - begin - if StoreDD and (Depth = 0) then - if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index)) - then - Inc(DDIndex2); Inc(Pos, SH^.NewSize); - end; X := AtomicIncrement(StreamIdx[Index]^); end; end; @@ -1897,16 +1772,24 @@ var I, J: Integer; Bytes: TBytes; UI32: UInt32; - DupTyp: TDuplicate1; - LResData: TResData; + DupTyp: TDuplicate; + LStream: TStream; + LGUID: TGUID; + LResData: PResData; begin GlobalSync := TCriticalSection.Create; SetLength(ThreadSync, Options^.Threads); for I := Low(ThreadSync) to High(ThreadSync) do ThreadSync[I] := TCriticalSection.Create; DupSysMem := Options^.DedupSysMem; - NStream.Add(TypeInfo(TMemoryStream) { , CalcSysMem } ); - // NStream.Add(TypeInfo(TPrecompVMStream)); + NStream.Add(TypeInfo(TMemoryStream), CalcSysMem); + NStream.Add(TypeInfo(TPrecompVMStream)); + Duplicates2 := TSynDictionary.Create(TypeInfo(TIntegerDynArray), + TypeInfo(TIntegerDynArray)); + DupIdx1 := 0; + SetLength(DupIdx2, Options^.Threads); + SetLength(DupBool, Options^.Threads); + Input.ReadBuffer(DupGUID, SizeOf(TGUID)); Input.ReadBuffer(Options^.Depth, Options^.Depth.Size); Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size); SetLength(Bytes, LongRec(I).Bytes[0]); @@ -1915,14 +1798,15 @@ begin Input.ReadBuffer(I, I.Size); for J := 0 to I - 1 do begin + New(LResData); Input.ReadBuffer(LongRec(I).Bytes[0], LongRec(I).Bytes[0].Size); SetLength(Bytes, LongRec(I).Bytes[0]); Input.ReadBuffer(Bytes[0], LongRec(I).Bytes[0]); - LResData.Name := StringOf(Bytes); - Input.ReadBuffer(LResData.Size, LResData.Size.Size); - GetMem(LResData.Data, LResData.Size); - Input.ReadBuffer(LResData.Data^, LResData.Size); - Insert(LResData, Resources, Length(Resources)); + LResData^.Name := StringOf(Bytes); + Input.ReadBuffer(LResData^.Size, LResData^.Size.Size); + GetMem(LResData^.Data, LResData^.Size); + Input.ReadBuffer(LResData^.Data^, LResData^.Size); + Insert(LResData^, Resources, Length(Resources)); end; SetLength(Tasks, Options^.Threads); SetLength(CurCodec, Options^.Threads); @@ -1976,18 +1860,45 @@ begin end; end; Input.ReadBuffer(StoreDD, StoreDD.Size); - UI32 := 0; - if StoreDD then + if StoreDD or FileExists(ExtractFilePath(Utils.GetModuleName) + + Options^.DedupFile) then begin - Input.ReadBuffer(UI32, UI32.Size); - SetLength(DDList2, UI32); - DDCount2 := UI32; - for I := Low(DDList2) to High(DDList2) do - Input.ReadBuffer(DDList2[I], SizeOf(TDuplicate2)); - DDIndex1 := -1; - DDIndex2 := 0; + if StoreDD then + LStream := Input + else + begin + LStream := TFileStream.Create(ExtractFilePath(Utils.GetModuleName) + + Options^.DedupFile, fmShareDenyNone); + LStream.Position := 0; + end; + while True do + begin + LStream.ReadBuffer(LGUID, SizeOf(TGUID)); + LStream.ReadBuffer(J, J.Size); + I := J * SizeOf(TDuplicate); + if CompareMem(@DupGUID, @LGUID, SizeOf(TGUID)) then + begin + if WorkStream[0].Size < I then + WorkStream[0].Size := I; + LStream.ReadBuffer(WorkStream[0].Memory^, I); + for I := 0 to J - 1 do + begin + DupTyp := (PDuplicate(WorkStream[0].Memory) + I)^; + Duplicates2.Add(DupTyp.Index, DupTyp.Count); + end; + break; + end + else if StoreDD then + raise EReadError.CreateRes(@SInvalidProperty) + else + LStream.Seek(I, TSeekOrigin.soCurrent); + if StoreDD or (LStream.Position >= LStream.Size) then + break; + end; + if not StoreDD then + LStream.Free; end; - DataMgr := TDataManager.Create(NStream); + DataMgr := TDataManager.Create(NStream, Duplicates2.Count); end; procedure DecFree; @@ -2022,6 +1933,7 @@ begin WorkStream[I].Free; end; DataMgr.Free; + Duplicates2.Free; FreeResources; GlobalSync.Free; for I := Low(ThreadSync) to High(ThreadSync) do @@ -2069,17 +1981,8 @@ begin Inc(CurrPos, Max(StreamHeader^.OldSize, StreamHeader^.NewSize)); end; end; - if (Depth = 0) and (Length(Tasks) > 1) and (StreamCount[Index]^ > 1) - then - begin - if MemInput[Index].Size < CurrPos then - MemInput[Index].Size := CurrPos; - end - else - begin - if MemInput[Index].Size < BlockSize then - MemInput[Index].Size := BlockSize; - end; + if MemInput[Index].Size < BlockSize then + MemInput[Index].Size := BlockSize; MemInput[Index].Position := 0; StreamIdx[Index]^ := -1; if (Depth = 0) and (Length(Tasks) > 1) and (StreamCount[Index]^ > 1) @@ -2093,6 +1996,9 @@ begin for J := 0 to StreamCount[Index]^ - 1 do begin StreamHeader := PStreamHeader(MemStream1[Index].Memory) + J; + MemInput[Index].Size := Max(MemInput[Index].Size, + StreamInfo[Index]^.Pos[J] + Max(StreamHeader^.OldSize, + StreamHeader^.NewSize)); MemInput[Index].Position := StreamInfo[Index]^.Pos[J]; if CopyStream(DecInput[Index], MemInput[Index], StreamHeader^.NewSize) <> StreamHeader^.NewSize then @@ -2120,26 +2026,12 @@ begin if IsErrored(Tasks) then for I := Low(Tasks) to High(Tasks) do Tasks[I].RaiseLastError; - if StoreDD and (Depth = 0) then - begin - Inc(DDIndex1); - if ((DDIndex2 < DDCount2) and (DDIndex1 = DDList2[DDIndex2].Index)) - then - begin - DataMgr.Add(DDIndex1, StreamHeader^.OldSize, - DDList2[DDIndex2].Count); - DataMgr.Write(DDIndex1, - (PByte(MemInput[Index].Memory) + StreamInfo[Index]^.Pos[J]), - StreamHeader^.OldSize); - Inc(DDIndex2); - end; - end; if StreamHeader^.Kind and DUPLICATED_STREAM = DUPLICATED_STREAM then DataMgr.CopyData(StreamHeader^.Option, DecOutput[Index]) else DecOutput[Index].WriteBuffer ((PByte(MemInput[Index].Memory) + StreamInfo[Index]^.Pos[J])^, - StreamHeader^.OldSize); + (PStreamHeader(MemStream1[Index].Memory) + J)^.OldSize); end; WaitForAll(Tasks); end @@ -2149,6 +2041,8 @@ begin DecInput[Index].ReadBuffer(UI32, UI32.Size); if UI32 > 0 then CopyStreamEx(DecInput[Index], DecOutput[Index], UI32); + if Depth = 0 then + Inc(DupIdx1, StreamCount[Index]^); DecInput[Index].ReadBuffer(StreamCount[Index]^, StreamCount[Index]^.Size); end; end; diff --git a/precompressor/PrecompMedia.pas b/precompressor/PrecompMedia.pas deleted file mode 100644 index 5c7c616..0000000 --- a/precompressor/PrecompMedia.pas +++ /dev/null @@ -1,1005 +0,0 @@ -unit PrecompMedia; - -{$POINTERMATH ON} - -interface - -uses - BrunsliDLL, FLACDLL, PackJPGDLL, JoJpegDLL, XDeltaDLL, - Utils, - PrecompUtils, - System.SysUtils, System.Classes, System.Math; - -var - Codec: TPrecompressor; - -implementation - -const - MediaCodecs: array of PChar = ['flac', 'packjpg', 'brunsli', 'jojpeg']; - CODEC_COUNT = 4; - FLAC_CODEC = 0; - PACKJPG_CODEC = 1; - BRUNSLI_CODEC = 2; - JOJPEG_CODEC = 3; - -const - FLAC_LEVEL = 5; - J_WORKMEM = 262144; - -type - PFlacEncCD = ^TFlacEncCD; - - TFlacEncCD = record - Output: TMemoryStreamEx; - Error: Boolean; - end; - - PFlacDecCD = ^TFlacDecCD; - - TFlacDecCD = record - Input, Output: TMemoryStreamEx; - end; - -var - cctx, dctx: array of Pointer; - ccd: array of TFlacEncCD; - dcd: array of TFlacDecCD; - JJInst: TArray; - FlacLevel: Integer = FLAC_LEVEL; - CodecAvailable, CodecEnabled: TArray; - RestoreOut: TArray<_PrecompOutput>; - RestoreSize: TArray; - -const - RIFF_SIGN = $46464952; - WAVE_SIGN = $45564157; - fmt_SIGN = $20746D66; - data_SIGN = $61746164; - - WAVE_FORMAT_PCM = 1; - WAVE_FORMAT_EXTENSIBLE = $FFFE; - -type - PRIFF_hdr = ^TRIFF_hdr; - - TRIFF_hdr = packed record - chunk_id, chunk_size, form_type: Cardinal; - end; - - PWAVE_hdr = ^TWAVE_hdr; - - TWAVE_hdr = packed record - audio_format, num_channels: Word; - sample_rate, byte_rate: Cardinal; - block_align, bits_per_sample: Word; - end; - - PWAVE_subchunk_hdr = ^TWAVE_subchunk_hdr; - - TWAVE_subchunk_hdr = packed record - subchunk_id, subchunk_size: Cardinal; - end; - - PWAVE_subformat = ^TWAVE_subformat; - - TWAVE_subformat = packed record - f1: Cardinal; - f2, f3: Word; - f4: array [0 .. 7] of Byte; - end; - - PWAVE_ext_hdr = ^TWAVE_ext_hdr; - - TWAVE_ext_hdr = packed record - cb_size, valid_bits: Word; - ch_mask: Cardinal; - est: TWAVE_subformat; - end; - -function GetWAVEInfo(InBuff: PByte; InSize: NativeInt; wave_hdr: PWAVE_hdr; - chunk_size, header_size: PCardinal): Boolean; -var - Pos, LastPos: NativeInt; - riffhdr: TRIFF_hdr; - subchunk_hdr: TWAVE_subchunk_hdr; - wave_hdr_ex: TWAVE_ext_hdr; -begin - Result := False; - Pos := 0; - if InSize - Pos < SizeOf(TRIFF_hdr) then - exit; - Move((InBuff + Pos)^, riffhdr, SizeOf(TRIFF_hdr)); - Inc(Pos, SizeOf(TRIFF_hdr)); - if (riffhdr.chunk_id <> RIFF_SIGN) or (riffhdr.form_type <> WAVE_SIGN) then - exit; - while True do - begin - if InSize - Pos < SizeOf(TWAVE_subchunk_hdr) then - exit; - Move((InBuff + Pos)^, subchunk_hdr, SizeOf(TWAVE_subchunk_hdr)); - Inc(Pos, SizeOf(TWAVE_subchunk_hdr)); - if subchunk_hdr.subchunk_id = fmt_SIGN then - begin - LastPos := Pos; - if InSize - Pos < SizeOf(TWAVE_hdr) then - exit; - Move((InBuff + Pos)^, wave_hdr^, SizeOf(TWAVE_hdr)); - Inc(Pos, SizeOf(TWAVE_hdr)); - if wave_hdr^.audio_format = WAVE_FORMAT_EXTENSIBLE then - begin - if InSize - Pos < SizeOf(TWAVE_ext_hdr) then - exit; - Move((InBuff + Pos)^, wave_hdr_ex, SizeOf(TWAVE_ext_hdr)); - Inc(Pos, SizeOf(TWAVE_ext_hdr)); - wave_hdr^.audio_format := wave_hdr_ex.est.f1; - end - else if wave_hdr^.audio_format <> WAVE_FORMAT_PCM then - exit; - Pos := LastPos + subchunk_hdr.subchunk_size; - end - else if subchunk_hdr.subchunk_id = data_SIGN then - begin - Result := True; - chunk_size^ := subchunk_hdr.subchunk_size; - header_size^ := Pos; - exit; - end - else - Inc(Pos, subchunk_hdr.subchunk_size); - end; -end; - -function FlacEncoderWriteCB(encoder: Pointer; const Buffer; bytes: NativeUInt; - samples, current_frame: Cardinal; client_data: PFlacEncCD): Integer cdecl; -begin - try - client_data^.Output.WriteBuffer(Buffer, bytes); - except - client_data^.Error := True; - end; - Result := 0; -end; - -function FlacDecoderReadCB(decoder: Pointer; var Buffer; bytes: PNativeUInt; - client_data: PFlacDecCD): Integer cdecl; -begin - bytes^ := client_data^.Input.Read(Buffer, bytes^); - if bytes^ > 0 then - Result := 0 - else - Result := 1; -end; - -function FlacDecoderWriteCB(decoder: Pointer; frame: PFLAC__Frame; - Buffer: Pointer; client_data: PFlacDecCD): Integer cdecl; -var - I, J, X: Integer; -begin - for I := 0 to frame^.header.blocksize - 1 do - for J := 0 to frame^.header.channels - 1 do - begin - X := PInteger(PNativeInt(Buffer)[J])[I]; - client_data^.Output.WriteBuffer(X, frame^.header.bits_per_sample div 8); - end; - Result := 0; -end; - -procedure FlacDecoderErrorCB(decoder: Pointer; Status: Integer; - client_data: Pointer)cdecl; -begin - -end; - -function FlacEncode(ctx: Pointer; cd: PFlacEncCD; InBuff: PByte; - InSize: Integer; OutBuff: PByte; OutSize: Integer; - Level: Integer = 5): Integer; -const - READSIZE = 1000; -var - I, J, X: Integer; - Ptr: PByte; - wave_hdr: TWAVE_hdr; - byte_per_sample: Word; - data_size, hdr_size, smp_size: Cardinal; - ok: Boolean; - pcm: TArray; -begin - Result := 0; - ok := False; - if GetWAVEInfo(InBuff, InSize, @wave_hdr, @data_size, @hdr_size) then - begin - ok := True; - byte_per_sample := (wave_hdr.bits_per_sample + 7) div 8; - smp_size := wave_hdr.num_channels * byte_per_sample; - ok := ok and FLAC__stream_encoder_set_verify(ctx, True); - ok := ok and FLAC__stream_encoder_set_compression_level(ctx, Level); - ok := ok and FLAC__stream_encoder_set_channels(ctx, wave_hdr.num_channels); - ok := ok and FLAC__stream_encoder_set_bits_per_sample(ctx, - wave_hdr.bits_per_sample); - ok := ok and FLAC__stream_encoder_set_sample_rate(ctx, - wave_hdr.sample_rate); - end; - if ok then - try - cd^.Error := False; - cd^.Output.Update(OutBuff, OutSize); - cd^.Output.Size := 0; - if FLAC__stream_encoder_init_stream(ctx, @FlacEncoderWriteCB, nil, nil, - nil, cd) = 0 then - begin - Ptr := InBuff + hdr_size; - SetLength(pcm, READSIZE * wave_hdr.num_channels); - while True do - begin - X := 0; - for I := 0 to READSIZE - 1 do - begin - if NativeInt(Ptr + smp_size - InBuff) > hdr_size + data_size then - break; - Inc(X); - for J := 0 to wave_hdr.num_channels - 1 do - begin - if NativeInt(Ptr - InBuff) > hdr_size + data_size then - break; - pcm[I * wave_hdr.num_channels + J] := - GetBits(PInteger(Ptr)^, 0, Pred(wave_hdr.bits_per_sample)) - - IfThen(GetBits(PInteger(Ptr)^, Pred(wave_hdr.bits_per_sample), - 1) = 0, 0, 1 shl Pred(wave_hdr.bits_per_sample)); - Inc(Ptr, byte_per_sample); - end; - end; - ok := ok and FLAC__stream_encoder_process_interleaved(ctx, pcm[0], X); - if (NativeInt(Ptr + smp_size - InBuff) > hdr_size + data_size) or - (ok = False) then - break; - end; - end; - finally - FLAC__stream_encoder_finish(ctx); - if ok and (cd^.Error = False) then - Result := cd^.Output.Size; - end; -end; - -function FlacDecode(ctx: Pointer; cd: PFlacDecCD; InBuff: PByte; - InSize: Integer; OutBuff: PByte; OutSize: Integer): Integer; -var - ok: Boolean; -begin - Result := 0; - ok := False; - cd^.Input.Update(InBuff, InSize); - cd^.Input.Position := 0; - cd^.Input.Size := InSize; - cd^.Output.Update(OutBuff, OutSize); - cd^.Output.Size := 0; - try - if FLAC__stream_decoder_init_stream(ctx, @FlacDecoderReadCB, nil, nil, nil, - nil, @FlacDecoderWriteCB, nil, @FlacDecoderErrorCB, cd) = 0 then - begin - ok := True; - ok := ok and FLAC__stream_decoder_process_until_end_of_stream(ctx); - end; - finally - FLAC__stream_decoder_finish(ctx); - if ok then - Result := cd^.Output.Size; - end; -end; - -const - JPG_HEADER = $D8FF; - JPG_FOOTER = $D9FF; - -function GetJPEGInfo(InBuff: PByte; InSize: NativeInt; ImageSize: PCardinal; - IsProgressive: PBoolean): Boolean; -// https://github.com/schnaader/precomp-cpp/blob/master/precomp.cpp -var - Pos: NativeInt; - hasQuantTable: Boolean; - done, found: Boolean; - length: Integer; - isMarker: Boolean; - bytesRead: Cardinal; -begin - Result := False; - Pos := 0; - done := False; - found := False; - if (PWord(InBuff)^ = JPG_HEADER) and ((InBuff + 2)^ = $FF) then - begin - Pos := 3; - if (InBuff + Pos)^ in [$C0, $C2, $C4, $DB .. $FE] then - begin - hasQuantTable := (InBuff + Pos)^ = $DB; - IsProgressive^ := (InBuff + Pos)^ = $C2; - Pos := 2; - repeat - if (Pos + 5 > InSize) or ((InBuff + Pos)[0] <> $FF) then - break; - length := Integer((InBuff + Pos)[2]) * 256 + Integer((InBuff + Pos)[3]); - case (InBuff + Pos)[1] of - $DB: - begin - if (length <= 262) and (((length - 2) mod 65) = 0) and - ((InBuff + Pos)[4] <= 3) then - begin - hasQuantTable := True; - Inc(Pos, length + 2); - end - else - done := True; - end; - $C4: - begin - done := (((InBuff + Pos)[4] and $F) > 3) or - (((InBuff + Pos)[4] shr 4) > 1); - Inc(Pos, length + 2); - end; - $DA: - begin - found := hasQuantTable; - Inc(Pos, length + 2); - end; - $D9: - done := True; - $C2: - begin - IsProgressive^ := True; - Inc(Pos, length + 2); - end; - $C0: - begin - done := (InBuff + Pos)[4] <> $08; - Inc(Pos, length + 2); - end; - else - Inc(Pos, length + 2); - end; - until done; - if found then - while Pos < InSize - 1 do - begin - if PWord(InBuff + Pos)^ = JPG_FOOTER then - begin - ImageSize^ := Pos + 2; - Result := True; - break; - end; - Inc(Pos); - end; - end; - end; -end; - -function BrunsliWriter(cd: Pointer; data: Pointer; Size: NativeUInt) - : Integer cdecl; -begin - RestoreOut[PInteger(cd)^](PInteger(cd)^, data, Size); - Inc(RestoreSize[PInteger(cd)^], Size); - Result := Size; -end; - -function MediaInit(Command: PChar; Count: Integer; - Funcs: PPrecompFuncs): Boolean; -var - I: Integer; - Options: TArray; - S: String; - X, Y: Integer; -begin - Result := True; - SetLength(RestoreOut, Count); - SetLength(RestoreSize, Count); - for X := Low(CodecAvailable) to High(CodecAvailable) do - begin - CodecAvailable[X] := False; - CodecEnabled[X] := False; - end; - CodecAvailable[FLAC_CODEC] := FLACDLL.DLLLoaded; - CodecAvailable[PACKJPG_CODEC] := PackJPGDLL.DLLLoaded; - CodecAvailable[BRUNSLI_CODEC] := BrunsliDLL.DLLLoaded; - CodecAvailable[JOJPEG_CODEC] := JoJpegDLL.DLLLoaded; - X := 0; - while Funcs^.GetCodec(Command, X, False) <> '' do - begin - S := Funcs^.GetCodec(Command, X, False); - if (CompareText(S, MediaCodecs[FLAC_CODEC]) = 0) and FLACDLL.DLLLoaded then - begin - CodecEnabled[FLAC_CODEC] := True; - if Funcs^.GetParam(Command, X, 'l') <> '' then - FlacLevel := StrToInt(Funcs^.GetParam(Command, X, 'l')); - end - else if (CompareText(S, MediaCodecs[PACKJPG_CODEC]) = 0) and PackJPGDLL.DLLLoaded - then - CodecEnabled[PACKJPG_CODEC] := True - else if (CompareText(S, MediaCodecs[BRUNSLI_CODEC]) = 0) and BrunsliDLL.DLLLoaded - then - CodecEnabled[BRUNSLI_CODEC] := True - else if (CompareText(S, MediaCodecs[JOJPEG_CODEC]) = 0) and JoJpegDLL.DLLLoaded - then - CodecEnabled[JOJPEG_CODEC] := True;; - Inc(X); - end; - if CodecAvailable[FLAC_CODEC] then - begin - SetLength(cctx, Count); - SetLength(dctx, Count); - SetLength(ccd, Count); - SetLength(dcd, Count); - for X := Low(cctx) to High(cctx) do - begin - cctx[X] := FLAC__stream_encoder_new; - dctx[X] := FLAC__stream_decoder_new; - ccd[X].Output := TMemoryStreamEx.Create(False); - dcd[X].Input := TMemoryStreamEx.Create(False); - dcd[X].Output := TMemoryStreamEx.Create(False); - end; - end; - if CodecAvailable[JOJPEG_CODEC] then - begin - SetLength(JJInst, Count); - for X := Low(JJInst) to High(JJInst) do - JJInst[X] := GetMemory(jojpeg_Size); - end; -end; - -procedure MediaFree(Funcs: PPrecompFuncs); -var - X: Integer; -begin - if CodecAvailable[FLAC_CODEC] then - begin - for X := Low(cctx) to High(cctx) do - begin - FLAC__stream_encoder_delete(cctx[X]); - FLAC__stream_decoder_delete(dctx[X]); - ccd[X].Output.Free; - dcd[X].Input.Free; - dcd[X].Output.Free; - end; - end; - if CodecAvailable[JOJPEG_CODEC] then - for X := Low(JJInst) to High(JJInst) do - FreeMemory(JJInst[X]); -end; - -function MediaParse(Command: PChar; Option: PInteger; - Funcs: PPrecompFuncs): Boolean; -var - S: String; - I: Integer; -begin - Result := False; - Option^ := 0; - I := 0; - while Funcs^.GetCodec(Command, I, False) <> '' do - begin - S := Funcs^.GetCodec(Command, I, False); - if (CompareText(S, MediaCodecs[FLAC_CODEC]) = 0) and FLACDLL.DLLLoaded then - begin - SetBits(Option^, FLAC_CODEC, 0, 5); - Result := True; - end - else if (CompareText(S, MediaCodecs[PACKJPG_CODEC]) = 0) and PackJPGDLL.DLLLoaded - then - begin - SetBits(Option^, PACKJPG_CODEC, 0, 5); - Result := True; - end - else if (CompareText(S, MediaCodecs[BRUNSLI_CODEC]) = 0) and BrunsliDLL.DLLLoaded - then - begin - SetBits(Option^, BRUNSLI_CODEC, 0, 5); - Result := True; - end - else if (CompareText(S, MediaCodecs[JOJPEG_CODEC]) = 0) and JoJpegDLL.DLLLoaded - then - begin - SetBits(Option^, JOJPEG_CODEC, 0, 5); - Result := True; - end; - Inc(I); - end; -end; - -procedure MediaScan1(Instance, Depth: Integer; Input: PByte; - Size, SizeEx: NativeInt; Output: _PrecompOutput; Add: _PrecompAdd; - Funcs: PPrecompFuncs); -var - Buffer: PByte; - Pos: NativeInt; - X, Y, Z: Integer; - SI: _StrInfo1; - DI1, DI2: TDepthInfo; - DS: TPrecompStr; - wave_hdr: TWAVE_hdr; - data_size, hdr_size: Cardinal; - progressive: Boolean; -begin - DI1 := Funcs^.GetDepthInfo(Instance); - DS := Funcs^.GetCodec(DI1.Codec, 0, False); - if DS <> '' then - begin - X := IndexTextW(@DS[0], MediaCodecs); - if (X < 0) or (DI1.OldSize <> SizeEx) then - exit; - if not CodecAvailable[X] then - exit; - Y := DI1.OldSize; - case X of - FLAC_CODEC: - begin - if GetWAVEInfo(Input, Y, @wave_hdr, @data_size, @hdr_size) then - Y := data_size + hdr_size - else - Y := 0; - end; - PACKJPG_CODEC, BRUNSLI_CODEC, JOJPEG_CODEC: - begin - if GetJPEGInfo(Input, Y, @data_size, @progressive) then - Y := data_size - else - Y := 0; - end; - else - Y := 0; - end; - if Y > 0 then - begin - Z := data_size; - if X = FLAC_CODEC then - begin - Buffer := Funcs^.Allocator(Instance, hdr_size); - Move(Input^, Buffer^, hdr_size); - Inc(PInteger(Buffer)^); - Output(Instance, @hdr_size, hdr_size.Size); - Output(Instance, Buffer, hdr_size); - Output(Instance, Input + hdr_size, Z); - Inc(Z, hdr_size.Size); - Inc(Z, hdr_size); - end - else - Output(Instance, Input, Z); - SI.Position := 0; - SI.OldSize := Y; - SI.NewSize := Z; - SI.Option := 0; - SetBits(SI.Option, X, 0, 5); - if System.Pos(SPrecompSep2, DI1.Codec) > 0 then - SI.Status := TStreamStatus.Predicted - else - SI.Status := TStreamStatus.None; - DS := Funcs^.GetDepthCodec(DI1.Codec); - Move(DS[0], DI2.Codec, SizeOf(DI2.Codec)); - DI2.OldSize := SI.NewSize; - DI2.NewSize := SI.NewSize; - Funcs^.LogScan1(MediaCodecs[GetBits(SI.Option, 0, 5)], SI.Position, - SI.OldSize, -1); - Add(Instance, @SI, DI1.Codec, @DI2); - end; - exit; - end; - if BoolArray(CodecEnabled, False) then - exit; - Pos := 0; - while Pos < Size do - begin - if CodecEnabled[FLAC_CODEC] and (PCardinal(Input + Pos)^ = RIFF_SIGN) then - begin - Y := SizeEx - Pos; - if GetWAVEInfo(Input + Pos, Y, @wave_hdr, @data_size, @hdr_size) then - Y := data_size + hdr_size - else - Y := 0; - if Y > 0 then - begin - Z := data_size; - Buffer := Funcs^.Allocator(Instance, hdr_size); - Move((Input + Pos)^, Buffer^, hdr_size); - Inc(PInteger(Buffer)^); - Output(Instance, @hdr_size, hdr_size.Size); - Output(Instance, Buffer, hdr_size); - Output(Instance, Input + Pos + hdr_size, Z); - Inc(Z, hdr_size.Size); - Inc(Z, hdr_size); - SI.Position := Pos; - SI.OldSize := Y; - SI.NewSize := Z; - SI.Option := 0; - SetBits(SI.Option, FLAC_CODEC, 0, 5); - SI.Status := TStreamStatus.None; - Funcs^.LogScan1(MediaCodecs[GetBits(SI.Option, 0, 5)], SI.Position, - SI.OldSize, -1); - Add(Instance, @SI, nil, nil); - Inc(Pos, SI.OldSize); - continue; - end; - end; - if (CodecEnabled[PACKJPG_CODEC] or CodecEnabled[BRUNSLI_CODEC] or - CodecEnabled[JOJPEG_CODEC]) and (PWord(Input + Pos)^ = JPG_HEADER) then - begin - Y := SizeEx - Pos; - if GetJPEGInfo(Input + Pos, Y, @data_size, @progressive) then - Y := data_size - else - Y := 0; - if Y > 0 then - begin - Z := data_size; - Output(Instance, Input + Pos, Z); - SI.Position := Pos; - SI.OldSize := Y; - SI.NewSize := Z; - SI.Option := 0; - if CodecEnabled[BRUNSLI_CODEC] then - SetBits(SI.Option, BRUNSLI_CODEC, 0, 5) - else if CodecEnabled[JOJPEG_CODEC] then - SetBits(SI.Option, JOJPEG_CODEC, 0, 5) - else if CodecEnabled[PACKJPG_CODEC] then - SetBits(SI.Option, PACKJPG_CODEC, 0, 5); - SI.Status := TStreamStatus.None; - Funcs^.LogScan1(MediaCodecs[GetBits(SI.Option, 0, 5)], SI.Position, - SI.OldSize, -1); - Add(Instance, @SI, nil, nil); - Inc(Pos, SI.OldSize); - continue; - end; - end; - Inc(Pos); - end; -end; - -function MediaScan2(Instance, Depth: Integer; Input: Pointer; Size: NativeInt; - StreamInfo: PStrInfo2; Offset: PInteger; Output: _PrecompOutput; - Funcs: PPrecompFuncs): Boolean; -var - Buffer: PByte; - X, Y, Z: Integer; - wave_hdr: TWAVE_hdr; - data_size, hdr_size: Cardinal; - progressive: Boolean; -begin - Result := False; - X := GetBits(StreamInfo^.Option, 0, 5); - if StreamInfo^.OldSize <= 0 then - exit; - Y := StreamInfo^.OldSize; - case X of - FLAC_CODEC: - begin - if GetWAVEInfo(Input, Y, @wave_hdr, @data_size, @hdr_size) then - Y := data_size + hdr_size - else - Y := 0; - end; - PACKJPG_CODEC, BRUNSLI_CODEC, JOJPEG_CODEC: - begin - if GetJPEGInfo(Input, Y, @data_size, @progressive) then - Y := data_size - else - Y := 0; - end; - else - Y := 0; - end; - if Y > 0 then - begin - Z := data_size; - if X = FLAC_CODEC then - begin - Buffer := Funcs^.Allocator(Instance, hdr_size); - Move(Input^, Buffer^, hdr_size); - Inc(PInteger(Buffer)^); - Output(Instance, @hdr_size, hdr_size.Size); - Output(Instance, Buffer, hdr_size); - Output(Instance, PByte(Input) + hdr_size, Z); - Inc(Z, hdr_size.Size); - Inc(Z, hdr_size); - end - else - Output(Instance, Input, Z); - StreamInfo^.NewSize := Z; - Funcs^.LogScan2(MediaCodecs[GetBits(StreamInfo^.Option, 0, 5)], - StreamInfo^.OldSize, -1); - Result := True; - end; -end; - -function MediaProcess(Instance, Depth: Integer; OldInput, NewInput: Pointer; - StreamInfo: PStrInfo2; Output: _PrecompOutput; Funcs: PPrecompFuncs): Boolean; -var - Buffer, Ptr: PByte; - Params: String; - ctx: Pointer; - I, J: Integer; - X, Y: Integer; - Res: Cardinal; - Res1, Res2: Integer; -begin - Result := False; - X := GetBits(StreamInfo^.Option, 0, 5); - if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then - exit; - Params := ''; - case X of - FLAC_CODEC: - begin - Params := 'l' + FlacLevel.ToString; - Y := Integer.Size + PInteger(NewInput)^; - Res := StreamInfo.NewSize - Y; - Res := FlacEncode(cctx[Instance], @ccd[Instance], OldInput, - StreamInfo^.OldSize, PByte(NewInput) + Y, Res, FlacLevel); - if (Res > 0) and (Res + Y < StreamInfo^.NewSize) then - begin - StreamInfo^.NewSize := Res + Y; - Result := True; - end; - end; - PACKJPG_CODEC: - begin - Buffer := nil; - Res := StreamInfo.OldSize; - pjglib_init_streams(OldInput, pjglib_memory, StreamInfo^.OldSize, - Buffer, pjglib_memory); - if pjglib_convert_stream2mem(@Buffer, @Res, nil) and - (Res < StreamInfo^.NewSize) then - begin - Move(Buffer^, NewInput^, Res); - StreamInfo^.NewSize := Res; - Result := True; - end; - ShowMessage(''); - end; - BRUNSLI_CODEC: - begin - ctx := brunsli_alloc_JPEGData; - try - if brunsli_ReadJpeg(ctx, OldInput, StreamInfo^.OldSize) = Integer(True) - then - begin - Res := brunsli_GetMaximumEncodedSize(ctx); - Buffer := Funcs^.Allocator(Instance, Res); - Res := brunsli_EncodeJpeg(ctx, Buffer, Res); - if Res < StreamInfo^.NewSize then - begin - Move(Buffer^, NewInput^, Res); - StreamInfo^.NewSize := Res; - Result := True; - end; - end; - finally - brunsli_free_JPEGData(ctx); - end; - end; - JOJPEG_CODEC: - begin - ctx := JJInst[Instance]; - Buffer := Funcs^.Allocator(Instance, J_WORKMEM * 2); - I := 0; - J := 0; - Y := 0; - jojpeg_Init(ctx, jojpeg_Compress); - try - jojpeg_Addbuf(ctx, jojpeg_Compress, Buffer, J_WORKMEM, - jojpeg_enc_Output1); - jojpeg_Addbuf(ctx, jojpeg_Compress, Buffer + J_WORKMEM, J_WORKMEM, - jojpeg_enc_Output2); - while True do - begin - Res1 := jojpeg_Loop(ctx, jojpeg_Compress); - if (Res1 = jojpeg_enc_Input) then - begin - Res2 := Min(StreamInfo^.OldSize - I, J_WORKMEM); - jojpeg_Addbuf(ctx, jojpeg_Compress, PByte(OldInput) + I, Res2, - jojpeg_enc_Input); - Inc(I, Res2); - end; - if (Res1 in [0, jojpeg_enc_Output1]) then - begin - Res2 := jojpeg_Getvalue(ctx, jojpeg_Compress, jojpeg_enc_Output1); - Move(Buffer^, (PByte(NewInput) + J)^, Res2); - Inc(J, Res2); - jojpeg_Addbuf(ctx, jojpeg_Compress, Buffer, J_WORKMEM, - jojpeg_enc_Output1); - end; - if (Res1 = jojpeg_enc_Output2) or (Res1 = 0) then - begin - Res2 := jojpeg_Getvalue(ctx, jojpeg_Compress, jojpeg_enc_Output2); - Output(Instance, Buffer + J_WORKMEM, Res2); - Inc(Y, Res2); - jojpeg_Addbuf(ctx, jojpeg_Compress, Buffer + J_WORKMEM, J_WORKMEM, - jojpeg_enc_Output2); - if Res1 = 0 then - break; - end; - end; - if I = StreamInfo^.OldSize then - begin - StreamInfo^.NewSize := J; - Result := True; - end; - finally - jojpeg_Quit(ctx, jojpeg_Compress); - end; - end; - end; - Funcs^.LogProcess(MediaCodecs[GetBits(StreamInfo^.Option, 0, 5)], - PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, -1, Result); -end; - -function MediaRestore(Instance, Depth: Integer; Input, InputExt: Pointer; - StreamInfo: _StrInfo3; Output: _PrecompOutput; Funcs: PPrecompFuncs): Boolean; -var - Buffer: PByte; - Params: String; - ctx, writer: Pointer; - I, J: Integer; - X, Y: Integer; - Res: Cardinal; - Res1, Res2: Integer; -begin - Result := False; - X := GetBits(StreamInfo.Option, 0, 5); - if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then - exit; - Params := ''; - case X of - FLAC_CODEC: - begin - Buffer := Funcs^.Allocator(Instance, StreamInfo.OldSize); - Y := Integer.Size + PInteger(Input)^; - Res := StreamInfo.OldSize; - Res := FlacDecode(dctx[Instance], @dcd[Instance], PByte(Input) + Y, - StreamInfo.NewSize - Y, Buffer + PInteger(Input)^, Res); - if Res > 0 then - begin - Move((PByte(Input) + Integer.Size)^, Buffer^, PInteger(Input)^); - Inc(Res, PInteger(Input)^); - Dec(PInteger(Buffer)^); - Result := True; - end; - if Result then - Output(Instance, Buffer, Res); - end; - PACKJPG_CODEC: - begin - Buffer := nil; - pjglib_init_streams(Input, pjglib_memory, StreamInfo.NewSize, Buffer, - pjglib_memory); - Res := StreamInfo.OldSize; - Result := pjglib_convert_stream2mem(@Buffer, @Res, nil); - if Result then - Output(Instance, Buffer, Res); - end; - BRUNSLI_CODEC: - begin - RestoreOut[Instance] := Output; - RestoreSize[Instance] := 0; - Res := RestoreSize[Instance]; - ctx := brunsli_alloc_JPEGData; - try - if brunsli_DecodeJpeg(ctx, Input, StreamInfo.NewSize) = BRUNSLI_OK - then - begin - writer := brunsli_alloc_JPEGOutput(BrunsliWriter, @Instance); - try - Result := Boolean(brunsli_WriteJpeg(ctx, writer)); - finally - brunsli_free_JPEGOutput(writer); - end; - end; - finally - brunsli_free_JPEGData(ctx); - end; - Res := RestoreSize[Instance]; - end; - JOJPEG_CODEC: - begin - ctx := JJInst[Instance]; - Buffer := Funcs^.Allocator(Instance, J_WORKMEM); - I := 0; - J := 0; - Y := 0; - jojpeg_Init(ctx, jojpeg_Decompress); - try - jojpeg_Addbuf(ctx, jojpeg_Decompress, Buffer, J_WORKMEM, - jojpeg_dec_Output); - while True do - begin - Res1 := jojpeg_Loop(ctx, jojpeg_Decompress); - if (Res1 = jojpeg_dec_Input1) then - begin - Res2 := Min(StreamInfo.NewSize - I, J_WORKMEM); - jojpeg_Addbuf(ctx, jojpeg_Decompress, PByte(Input) + I, Res2, - jojpeg_dec_Input1); - Inc(I, Res2); - end; - if (Res1 = jojpeg_dec_Input2) then - begin - Res2 := Min(StreamInfo.ExtSize - J, J_WORKMEM); - jojpeg_Addbuf(ctx, jojpeg_Decompress, PByte(InputExt) + J, Res2, - jojpeg_dec_Input2); - Inc(J, Res2); - end; - if (Res1 in [0, jojpeg_dec_Output]) then - begin - Res2 := jojpeg_Getvalue(ctx, jojpeg_Decompress, - jojpeg_dec_Output); - Output(Instance, Buffer, Res2); - Inc(Y, Res2); - jojpeg_Addbuf(ctx, jojpeg_Decompress, Buffer, J_WORKMEM, - jojpeg_dec_Output); - if Res1 = 0 then - break; - end; - end; - { while True do - begin - Res1 := jojpeg_Loop(ctx, jojpeg_Decompress); - if Res1 = 0 then - break; - if (Res1 = jojpeg_dec_Input1) then - begin - Res2 := Min(StreamInfo.NewSize - I, J_WORKMEM); - jojpeg_Addbuf(ctx, jojpeg_Decompress, PByte(Input) + I, Res2, - jojpeg_dec_Input1); - Inc(I, Res2); - end; - if (Res1 = jojpeg_dec_Input2) then - begin - Res2 := Min(StreamInfo.ExtSize - J, J_WORKMEM); - jojpeg_Addbuf(ctx, jojpeg_Decompress, PByte(InputExt) + J, Res2, - jojpeg_dec_Input2); - Inc(J, Res2); - end; - if (Res1 = jojpeg_dec_Output) then - begin - Output(Instance, Buffer, J_WORKMEM); - Inc(Y, J_WORKMEM); - jojpeg_Addbuf(ctx, jojpeg_Decompress, Buffer, J_WORKMEM, - jojpeg_dec_Output); - end; - end; - if (Y <> StreamInfo.OldSize) and (StreamInfo.OldSize - Y < J_WORKMEM) - then - begin - Res2 := StreamInfo.OldSize - Y; - Output(Instance, Buffer, Res2); - Inc(Y, Res2); - end; } - Res := Y; - Result := Y = StreamInfo.OldSize; - finally - jojpeg_Quit(ctx, jojpeg_Decompress); - end; - end; - end; - Funcs^.LogRestore(MediaCodecs[GetBits(StreamInfo.Option, 0, 5)], - PChar(Params), StreamInfo.NewSize, Res, -1, Result); -end; - -var - I: Integer; - -initialization - -Codec.Names := []; -for I := Low(MediaCodecs) to High(MediaCodecs) do -begin - Codec.Names := Codec.Names + [MediaCodecs[I]]; - StockMethods.Add(MediaCodecs[I]); -end; -Codec.Initialised := False; -Codec.Init := @MediaInit; -Codec.Free := @MediaFree; -Codec.Parse := @MediaParse; -Codec.Scan1 := @MediaScan1; -Codec.Scan2 := @MediaScan2; -Codec.Process := @MediaProcess; -Codec.Restore := @MediaRestore; -SetLength(CodecAvailable, length(Codec.Names)); -SetLength(CodecEnabled, length(Codec.Names)); - -end. diff --git a/precompressor/PrecompOodle.pas b/precompressor/PrecompOodle.pas index 7950c03..53710e3 100644 --- a/precompressor/PrecompOodle.pas +++ b/precompressor/PrecompOodle.pas @@ -681,8 +681,8 @@ begin if CustomLZ_Decompress0(Input, Buffer, StreamInfo^.OldSize, OodleSI.DSize, Res) then begin - Output(Instance, Buffer, Res); StreamInfo^.NewSize := Res; + Output(Instance, Buffer, Res); Funcs^.LogScan2(OodleCodecs[GetBits(StreamInfo^.Option, 0, 5)], StreamInfo^.OldSize, StreamInfo^.NewSize); Result := True; @@ -730,7 +730,7 @@ begin break; end; if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or - (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then + (SOList[Instance][X].Count = 1)) then begin Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1)); Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1, diff --git a/precompressor/PrecompUtils.pas b/precompressor/PrecompUtils.pas index 377e612..72a4e6e 100644 --- a/precompressor/PrecompUtils.pas +++ b/precompressor/PrecompUtils.pas @@ -18,7 +18,7 @@ resourcestring SPrecompSep4 = '/'; const - SuccessStatus = 4; + SuccessStatus = 3; DEFAULT_STREAM = 0; EXTENDED_STREAM = 1; @@ -32,7 +32,7 @@ type TPrecompStr = array [0 .. 255] of Char; - TStreamStatus = (None, Invalid, Predicted, Database); + TStreamStatus = (None, Invalid, Predicted); PDepthInfo = ^TDepthInfo; @@ -182,10 +182,10 @@ type LogScan1: procedure(Codec: PChar; Position: Int64; InSize, OutSize: Integer)cdecl; LogScan2: procedure(Codec: PChar; InSize, OutSize: Integer)cdecl; // 35 - LogProcess: procedure(Codec, Method: PChar; Size1, Size2, Size3: Integer; - Status: Boolean)cdecl; - LogRestore: procedure(Codec, Method: PChar; Size1, Size2, Size3: Integer; - Status: Boolean)cdecl; + LogProcess: procedure(Codec, Method: PChar; + OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl; + LogRestore: procedure(Codec, Method: PChar; + OriginalSize, InSize, OutSize: Integer; Status: Boolean)cdecl; LogPatch1: procedure(OldSize, NewSize, PatchSize: Integer; Status: Boolean)cdecl; LogPatch2: procedure(OldSize, NewSize, PatchSize: Integer; @@ -243,29 +243,22 @@ type PDatabase = ^TDatabase; - TDatabase = packed record - Size: Integer; + TDatabase = record Codec: Byte; - Option: Integer; - Checksum: Cardinal; Status: TStreamStatus; + Option: Integer; end; - PDuplicate1 = ^TDuplicate1; + TDatabaseDynArray = TArray; - TDuplicate1 = packed record - Size: Integer; - Checksum: Cardinal; + PDuplicate = ^TDuplicate; + + TDuplicate = record Index: Integer; Count: Integer; end; - PDuplicate2 = ^TDuplicate2; - - TDuplicate2 = packed record - Index: Integer; - Count: Integer; - end; + TDuplicateDynArray = TArray; TPrecompVMStream = class(TStream) private const @@ -295,6 +288,8 @@ type Size: Integer; end; +function DuplicateSortCompare(const Left, Right): Integer; + procedure AddMethod(Method: String); procedure ClearMethods; @@ -363,6 +358,7 @@ function PrecompAcceptPatch(OldSize, NewSize, PatchSize: Integer) var PrecompFunctions: _PrecompFuncs; DIFF_TOLERANCE: Single = 0.05; + VERBOSE: Boolean = False; EncodeSICmp: TEncodeSIComparer; FutureSICmp: TFutureSIComparer; StockMethods, ExternalMethods: TStringList; @@ -384,6 +380,11 @@ begin Result := Integer(CompareValue(Left.Position, Right.Position)); end; +function DuplicateSortCompare(const Left, Right): Integer; +begin + Result := TDuplicate(Left).Index - TDuplicate(Right).Index; +end; + procedure AddMethod(Method: String); begin if (StockMethods.IndexOf(Method) < 0) and (ExternalMethods.IndexOf(Method) < 0) @@ -855,7 +856,7 @@ var begin Result := 0; if xd3_encode(OldBuff, OldSize, NewBuff, NewSize, PatchBuff, @Res, PatchSize, - Integer(XD3_NOCOMPRESS)) = 0 then + 0) = 0 then Result := Res; // MakeDiff(OldBuff, NewBuff, PatchBuff, OldSize, NewSize, Result); end; @@ -868,7 +869,7 @@ var begin Result := 0; if xd3_decode(PatchBuff, PatchSize, OldBuff, OldSize, NewBuff, @Res, NewSize, - Integer(XD3_NOCOMPRESS)) = 0 then + 0) = 0 then Result := Res; // MakePatch(OldBuff, PatchBuff, NewBuff, OldSize, PatchSize, Result); end; diff --git a/precompressor/PrecompZLib.pas b/precompressor/PrecompZLib.pas index 5ae50ee..82b44ce 100644 --- a/precompressor/PrecompZLib.pas +++ b/precompressor/PrecompZLib.pas @@ -255,8 +255,7 @@ begin CodecAvailable[ZLIB_CODEC] := ZLibDLL.DLLLoaded; CodecAvailable[REFLATE_CODEC] := ReflateDLL.DLLLoaded; CodecAvailable[PREFLATE_CODEC] := PreflateDLL.DLLLoaded; - CodecAvailable[PNG_CODEC] := ZLibDLL.DLLLoaded or ReflateDLL.DLLLoaded or - PreflateDLL.DLLLoaded; + CodecAvailable[PNG_CODEC] := True; X := 0; while Funcs^.GetCodec(Command, X, False) <> '' do begin @@ -434,13 +433,12 @@ var begin DI1 := Funcs^.GetDepthInfo(Instance); DS := Funcs^.GetCodec(DI1.Codec, 0, False); - X := -1; if DS <> '' then begin X := IndexTextW(@DS[0], ZlibCodecs); if (X < 0) or (DI1.OldSize <> SizeEx) then exit; - if not CodecAvailable[X] then + if CodecAvailable[X] then exit; end else if BoolArray(CodecEnabled, False) then @@ -569,7 +567,7 @@ begin if (I = ZLIB_CODEC) and (WinBits = 0) then SetBits(SI.Option, 1, 12, 3); SetBits(SI.Option, I, 0, 5); - if CodecEnabled[I] or (I = X) then + if CodecEnabled[I] then begin DS := Funcs^.GetDepthCodec(DI1.Codec); Move(DS[0], DI2.Codec, SizeOf(DI2.Codec)); @@ -634,14 +632,18 @@ function ZlibProcess(Instance, Depth: Integer; OldInput, NewInput: Pointer; function IsValidLevel(CLevel, ZLevel: Integer): Boolean; begin - Result := False; case CLevel of 1, 6: - Result := CLevel = ZLevel; + if CLevel = ZLevel then + Result := True; 2 .. 5: - Result := ZLevel = 5; + if ZLevel = 5 then + Result := True; 7 .. 9: - Result := ZLevel = 9; + if ZLevel = 9 then + Result := True; + else + Result := False; end; end; @@ -662,7 +664,6 @@ begin if not X in [PNG_CODEC] then if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then exit; - Params := ''; case X of ZLIB_CODEC: begin @@ -672,7 +673,7 @@ begin begin L := I div 10; M := I mod 10; - if StreamInfo^.Status >= TStreamStatus.Predicted then + if StreamInfo^.Status = TStreamStatus.Predicted then begin if InRange(GetBits(StreamInfo^.Option, 5, 7), 1, 9) then begin @@ -683,40 +684,36 @@ begin begin if GetBits(StreamInfo^.Option, 5, 7) <> I then continue; - if StreamInfo^.Status = TStreamStatus.Database then + { I := GetBits(StreamInfo^.Option, 5, 7); + SOList[Instance][ZLIB_CODEC].Add(I); Result := True; + break; } end; end; Params := 'l' + I.ToString + ':' + 'w' + (GetBits(StreamInfo^.Option, 12, 3) + 8).ToString; ZStream := @ZStream1[Instance, L, M, GetBits(StreamInfo^.Option, 12, 3)]; - if not Result then - begin - ZStream^.next_in := NewInput; - ZStream^.avail_in := StreamInfo^.NewSize; - deflateReset(ZStream^); - repeat - ZStream^.next_out := Buffer; - ZStream^.avail_out := Z_BLKSIZE; - Res1 := deflate(ZStream^, Z_FINISH); - if Res1 < 0 then - raise EZCompressionError.Create(string(_z_errmsg[2 - Res1])) - at ReturnAddress; - Res2 := Z_BLKSIZE - ZStream^.avail_out; - Verified := CompareMem(PByte(OldInput) + ZStream^.total_out - - Res2, Buffer, Res2); - if not Verified then - break; - until (ZStream^.avail_in = 0) and (ZStream^.avail_out > 0); - end - else - ZStream.total_out := StreamInfo^.OldSize; + ZStream^.next_in := NewInput; + ZStream^.avail_in := StreamInfo^.NewSize; + deflateReset(ZStream^); + repeat + ZStream^.next_out := Buffer; + ZStream^.avail_out := Z_BLKSIZE; + Res1 := deflate(ZStream^, Z_FINISH); + if Res1 < 0 then + raise EZCompressionError.Create(string(_z_errmsg[2 - Res1])) + at ReturnAddress; + Res2 := Z_BLKSIZE - ZStream^.avail_out; + Verified := CompareMem(PByte(OldInput) + ZStream^.total_out - Res2, + Buffer, Res2); + if not Verified then + break; + until (ZStream^.avail_in = 0) and (ZStream^.avail_out > 0); Funcs^.LogProcess(ZlibCodecs[GetBits(StreamInfo^.Option, 0, 5)], PChar(Params), StreamInfo^.OldSize, StreamInfo^.NewSize, - ZStream^.total_out, (Result = True) or - (Verified and (Res1 = Z_STREAM_END))); - if (Result = True) or (Verified and (Res1 = Z_STREAM_END)) then + ZStream^.total_out, Verified and (Res1 = Z_STREAM_END)); + if Verified and (Res1 = Z_STREAM_END) then begin SetBits(StreamInfo^.Option, I, 5, 7); SOList[Instance][ZLIB_CODEC].Add(I); @@ -742,7 +739,7 @@ begin Buffer := Funcs^.Allocator(Instance, R_WORKMEM * 2); J := 0; HR := RefInst1[Instance]; - if StreamInfo^.Status >= TStreamStatus.Predicted then + if StreamInfo^.Status = TStreamStatus.Predicted then L := GetBits(StreamInfo^.Option, 5, 7) else L := RLevel; @@ -807,6 +804,7 @@ begin PNG_CODEC: begin Buffer := Funcs^.Allocator(Instance, StreamInfo^.OldSize); + Params := ''; if DecodePNG(NewInput, Buffer, StreamInfo^.OldSize) then Result := CompareMem(OldInput, Buffer, StreamInfo^.OldSize); Funcs^.LogProcess(ZlibCodecs[GetBits(StreamInfo^.Option, 0, 5)], @@ -833,7 +831,6 @@ begin if not X in [PNG_CODEC] then if BoolArray(CodecAvailable, False) or (CodecAvailable[X] = False) then exit; - Params := ''; case X of ZLIB_CODEC: begin @@ -918,6 +915,7 @@ begin PNG_CODEC: begin Buffer := Funcs^.Allocator(Instance, StreamInfo.OldSize); + Params := ''; if DecodePNG(Input, Buffer, StreamInfo.OldSize) then begin Output(Instance, Buffer, StreamInfo.OldSize); diff --git a/precompressor/PrecompZSTD.pas b/precompressor/PrecompZSTD.pas index c4bd9d1..8615493 100644 --- a/precompressor/PrecompZSTD.pas +++ b/precompressor/PrecompZSTD.pas @@ -233,8 +233,8 @@ begin end; if Res > StreamInfo^.OldSize then begin - Output(Instance, Buffer, Res); StreamInfo^.NewSize := Res; + Output(Instance, Buffer, Res); Funcs^.LogScan2(ZSTDCodecs[GetBits(StreamInfo^.Option, 0, 5)], StreamInfo^.OldSize, StreamInfo^.NewSize); Result := True; @@ -265,7 +265,6 @@ begin if StreamInfo^.Status = TStreamStatus.Predicted then if GetBits(StreamInfo^.Option, 5, 7) <> I then continue; - Params := ''; case X of ZSTD_CODEC: begin @@ -308,7 +307,7 @@ begin if Res1 < 0 then exit; if (Result = False) and ((StreamInfo^.Status = TStreamStatus.Predicted) or - (SOList[Instance][X].Count = 1)) and (DIFF_TOLERANCE > 0) then + (SOList[Instance][X].Count = 1)) then begin Buffer := Funcs^.Allocator(Instance, Res1 + Max(StreamInfo^.OldSize, Res1)); Res2 := PrecompEncodePatch(OldInput, StreamInfo^.OldSize, Buffer, Res1, diff --git a/resources/Win32/fast-lzma2.dll b/resources/Win32/fast-lzma2.dll deleted file mode 100644 index c166eb4..0000000 Binary files a/resources/Win32/fast-lzma2.dll and /dev/null differ diff --git a/resources/Win64/fast-lzma2.dll b/resources/Win64/fast-lzma2.dll deleted file mode 100644 index 397329f..0000000 Binary files a/resources/Win64/fast-lzma2.dll and /dev/null differ diff --git a/xtool.dpr b/xtool.dpr index 9dc958e..bbe9006 100644 --- a/xtool.dpr +++ b/xtool.dpr @@ -52,13 +52,9 @@ uses oObjects in 'contrib\ParseExpression\oObjects.pas', ParseClass in 'contrib\ParseExpression\ParseClass.pas', ParseExpr in 'contrib\ParseExpression\ParseExpr.pas', - BrunsliDLL in 'imports\BrunsliDLL.pas', - FLACDLL in 'imports\FLACDLL.pas', - JoJpegDLL in 'imports\JoJpegDLL.pas', LZ4DLL in 'imports\LZ4DLL.pas', LZODLL in 'imports\LZODLL.pas', OodleDLL in 'imports\OodleDLL.pas', - PackJPGDLL in 'imports\PackJPGDLL.pas', PreflateDLL in 'imports\PreflateDLL.pas', ReflateDLL in 'imports\ReflateDLL.pas', XDeltaDLL in 'imports\XDeltaDLL.pas', @@ -72,7 +68,6 @@ uses PrecompLZ4 in 'precompressor\PrecompLZ4.pas', PrecompLZO in 'precompressor\PrecompLZO.pas', PrecompZSTD in 'precompressor\PrecompZSTD.pas', - PrecompMedia in 'precompressor\PrecompMedia.pas', PrecompOodle in 'precompressor\PrecompOodle.pas', PrecompINI in 'precompressor\PrecompINI.pas', PrecompSearch in 'precompressor\PrecompSearch.pas', @@ -87,8 +82,7 @@ uses IOPatch in 'io\IOPatch.pas', IOExecute in 'io\IOExecute.pas', IODecode in 'io\IODecode.pas', - IOUtils in 'io\IOUtils.pas', - LZMADLL in 'imports\LZMADLL.pas'; + IOUtils in 'io\IOUtils.pas'; {$SETPEFLAGS IMAGE_FILE_LARGE_ADDRESS_AWARE or IMAGE_FILE_RELOCS_STRIPPED} @@ -147,24 +141,265 @@ begin Result := THandleStream.Create(GetStdHandle(STD_INPUT_HANDLE)) else if Pos('://', Input) > 0 then Result := TDownloadStream.Create(Input) - else if FileExists(Input) then - Result := TFileStream.Create(Input, fmShareDenyNone) else - Result := TDirInputStream.Create(Input); + Result := TFileStream.Create(Input, fmShareDenyNone); end; -function GetOutStream(Output: string; MultiInput: Boolean = False): TStream; +function GetOutStream(Output: string): TStream; begin - if (Output = '') then - Result := TNullStream.Create - else if DirectoryExists(Output) then - Result := TDirOutputStream.Create(Output) - else if (Output = '-') then + if (Output = '-') or (Output = '') then Result := THandleStream.Create(GetStdHandle(STD_OUTPUT_HANDLE)) else Result := TFileStream.Create(Output, fmCreate); end; +{ changelog + ES_R34 (0.5.3) + - added png stream preprocessor + - removed grittibanzli codec (since nobody uses it) + + ES_R33 (0.5.2) + - added IO functions (archive, execute) + - fixed issue in patch io function + - removed compression on patch diff files + + ES_R32 (0.5.1) + - added IO functions (find, extract, patch) + - generate database feature and IO functions now can search for streams larger than chunk size + + ES_R31 (0.5.0) + - added IO functions (erase, replace) + - fixed external executable support bugs + + ES_R30 (0.4.8) + - fixed issue with storing incorrect recompression information when stream patching is performed + + ES_R29 (0.4.7) + - updated oodle scanner + - updated external executable support + - updated configuration based plugin support to add depth information + - updated verbose mode + + ES_R28 (0.4.6) + - generate database feature fixed + - fixed external executable support issues + - fixed lz4f level setting bug + + ES_R28 (0.4.5) + - removed leviathan codec restriction + + ES_R27 (0.4.4) + - fixed issue of lz4 codec loading incorrect library + - fixed issue with handling endianess via configuration based plugins + - updated framework of library based plugins + + ES_R26 (0.4.3) + - added verbose mode + - added feature that allows you to enforce a different library to be loaded + - fixed issues related to imperfect stream patching + - fixed issues with old libraries with missing functions that cause xtool to crash on startup + - updated oodle codec + - updated reflate codec + - updated zstd codec + + ES_R25 (0.4.2) + - removed debugging code from encryption and executable codec + - fixed issue with depth when using search codec + - fixed external executable support issues + + ES_R24 (0.4.1) + - fixed issue of status not reporting when encoding + - added depth method support for search support + - fixed zlib encoding issues for different window bits + - fixed zlib memory leak issue + - updated all internal codecs to support information relayed by external codecs + - updated lz4f codec and removed temporarily removed support for universal scanning + - added option to change recompression level to be used by reflate + - updated external executable support + - generate database feature currently bugged, wait for next update + - search database structure changed, older database files will no longer work with newer releases + + ES_R23 (0.4.0) + - project made open source + - added external executable support + - added generate database feature + - fixed search support bug + + ES_R22 (0.3.22) + - updated search support (speed improvements) + - updated command line parser + - added partial universal scanner for lzo1x streams + - added universal scanner for lz4f streams + - fixed issue with configuration files failing to execute without conditions + + ES_R21 (0.3.21) + - updated search support + + ES_R20 (0.3.20) + - fixed library support bug + - x86 build discontinued (has bugs from nowhere) + + ES_R19 (0.3.19) + - updated lzo codec + + ES_R18 (0.3.18) + - fixed depth bug + - fixed library plugin bugs + + ES_R17 (0.3.17) + - fixed multi-threading bug + + ES_R16 (0.3.16) + - minor bug fixes + + ES_R15 (0.3.15) + - converted library support to unicode (don't know why I used ansi in the first place) + - added library support functions + - added rc4 encryption support + + ES_R14 (0.3.14) + - fixed library support bug + - updated library structure + + ES_R13 (0.3.13) + - updated lz4 codec + - updated library structure + - updated depth info functions + - updated depth feature + + ES_R12 (0.3.12) + - added depth info functions + - added support for oodle 2.9.0+ functions + - fixed data patching bug + - updated oodle codec + - updated command line parser + + ES_R11 (0.3.11) + - fixed x86 build bugs + - fixed config multi-threading bug + - fixed resource management bug + - fixed deduplication bug + + ES_R10 (0.3.10) + - minor bug fixes + - added diff tolerance parameter (--diff=) + - fixed plugin database bug + - fixed lz4 codec bug + - updated oodle codec + - updated library structure + - added resource management + - added direct use encryption codecs + - added embedded deduplication feature (--dedup) [makes temps during encoding] + + ES_R9 (0.3.9) + - fixed future stream bug + + ES_R8 (0.3.8) + - fixed command line parser bug + - updated library support + + ES_R7 (0.3.7) + - updated library structure + + ES_R6 (0.3.6) + - updated oodle codec (fixed more lzna bugs) + + ES_R5 (0.3.5) + - updated oodle codec (fixed lzna bug) + - added custom method configuration + + ES_R4 (0.3.4) + - fixed bug depthing + + ES_R3 (0.3.3) + - updated lz4 codec + - updated library support + + ES_R2 (0.3.2) + - improved depthing + - updated library support + - fixed zstd codec issues + - removed fast memory + + ES_R1 (0.3.1) + - updated library support + - updated command line parser + - included x86 build + - fixed depthing issues + + 2012_R2 (0.2.14) + - added library support + - added compress, decompress, encrypt, decrypt, hash, delta functions (used by library) + - added lzo codec placeholders + - fixed oodle bug + - fixed lz4 bug + - removed libdunia codec + + 2012_R1 (0.2.13) + - added oo2ext* dll support + - updated search support + + 2011_R1 (0.2.12) + - added temporary libdunia codec + + 2010_R5 (0.2.11) + - fixed search/config support bug + + 2010_R4 (0.2.10) + - updated search/config support + + 2010_R3 (0.2.9) + - added database search + - updated zlib scanner + - fixed reflate bug + - fixed 2GB memory limit + + 2010_R2 (0.2.8) + - fixed zstd codec + + 2010_R1 (0.2.7) + - added zstd codec + - added lz4, lz4hc, lzna, mermaid, selkie, hydra, leviathan codec placeholders + - added configuration support + - added xdelta support to handle crc mismatch streams + + 2009_R3 (0.2.6) + - documentation added + + 2009_R2 (0.2.5) + - added kraken codec + - fixed depthing issues + + 2009_R1 (0.2.4) + - added reflate forced verification + - updated deflate scanner + - fixed depthing issues + - fixed low memory mode issues + - fixed hanging issues when encoding + + 2008_R3 (0.2.3) + - fixed deduplication memory calculation error + - added virtual memory support for deduplication + - added --mem=# parameter to control deduplication memory usage + + 2008_R2 (0.2.2) + - fixed command line parser + - updated deflate scanner + - added stream deduplication + - added stream database + - added decompression memory limiter + - added grittibanzli (also handles deflate stream but slow af) + + 2008_R1 (0.2.1) + - initial release + + changelog } + +procedure EncodePNG(Input: PByte; Output: PByte); +begin + +end; + const BufferSize = 1048576; @@ -225,10 +460,12 @@ begin PrecompMain.PrintHelp else begin - Input := TBufferedStream.Create(GetInStream(ParamArgSafe(1, 0)), True, + while Length(ParamArg[1]) < 2 do + Insert('-', ParamArg[1], Length(ParamArg[1])); + Input := TBufferedStream.Create(GetInStream(ParamArg[1, 0]), True, + BufferSize); + Output := TBufferedStream.Create(GetOutStream(ParamArg[1, 1]), False, BufferSize); - Output := TBufferedStream.Create(GetOutStream(ParamArgSafe(1, 1)), - False, BufferSize); try PrecompMain.Parse(ParamArg[0], PrecompEnc); PrecompMain.Encode(Input, Output, PrecompEnc); @@ -313,6 +550,8 @@ begin IOArchive.PrintHelp else begin + while Length(ParamArg[1]) < 2 do + Insert('-', ParamArg[1], Length(ParamArg[1])); SetLength(StrArray, 0); for I := 0 to High(ParamArg[1]) - 1 do Insert(ParamArg[1, I], StrArray, Length(StrArray)); @@ -330,6 +569,8 @@ begin IOExecute.PrintHelp else begin + while Length(ParamArg[1]) < 2 do + Insert('-', ParamArg[1], Length(ParamArg[1])); SetLength(StrArray, 0); for I := 2 to High(ParamArg[1]) do Insert(ParamArg[1, I], StrArray, Length(StrArray)); diff --git a/xtool.dproj b/xtool.dproj index 194f8e7..9936163 100644 --- a/xtool.dproj +++ b/xtool.dproj @@ -111,13 +111,9 @@ - - - - @@ -131,7 +127,6 @@ - @@ -147,7 +142,6 @@ - ResourceItem RCDATA @@ -158,7 +152,6 @@ RCDATA XDELTA64_DLL - Base @@ -196,12 +189,6 @@ true - - - .\ - true - - .\