146 lines
3.6 KiB
ObjectPascal
146 lines
3.6 KiB
ObjectPascal
/// answer to http://stackoverflow.com/q/39685770/458259
|
|
program ORMFastMultiInsertWithIndex;
|
|
|
|
{
|
|
insert 1,000,000 rows via the ORM in a SQlite3 in-memory database,
|
|
and search by Key using an index (only 1024 diverse values of Key)
|
|
- note that two similar SELECT are executed, to show two ORM methods
|
|
|
|
with index defined before insertion:
|
|
INSERT 1000000 rows in 6.71s
|
|
SELECT 1000000 rows per Key index in 1.15s
|
|
|
|
with index created after insertion:
|
|
INSERT 1000000 rows in 2.91s
|
|
CREATE INDEX 1000000 in 1.28s
|
|
SELECT 1000000 rows per Key index in 1.15s
|
|
|
|
without the index:
|
|
INSERT 1000000 rows in 2.94s
|
|
SELECT 1000000 rows per Key index in 129.27s
|
|
|
|
So for huge data set, an index is worth it, and better done AFTER insertion!
|
|
|
|
}
|
|
|
|
{$APPTYPE CONSOLE}
|
|
|
|
uses
|
|
{$I SynDprUses.inc} // use FastMM4 on older Delphi, or set FPC threads
|
|
SysUtils,
|
|
SynCommons,
|
|
SynSQLite3,
|
|
SynSQLite3Static,
|
|
mORMot,
|
|
mORMotSQLite3;
|
|
|
|
type
|
|
TSQLEntry = class(TSQLRecord)
|
|
private
|
|
fKey: RawUTF8;
|
|
fValue: RawUTF8;
|
|
published
|
|
property Key: RawUTF8 read fKey write fKey;
|
|
property Value: RawUTF8 read fValue write fValue;
|
|
end;
|
|
|
|
const
|
|
// used to create some fake data, with some multiple occurences of Key
|
|
COUNT = 1000000; // one million rows insertion !
|
|
UNIQUE_KEY = 1024; // should be a power of two
|
|
|
|
procedure Process;
|
|
|
|
var
|
|
db: TSQLRestServerDB;
|
|
|
|
procedure DoInsert;
|
|
var i: integer;
|
|
rec: TSQLEntry;
|
|
batch: TSQLRestBatch;
|
|
begin
|
|
rec := TSQLEntry.Create;
|
|
batch := TSQLRestBatch.Create(db,TSQLEntry,10000,[boExtendedJSON]);
|
|
try
|
|
write('batch:');
|
|
for i := 0 to COUNT-1 do begin
|
|
// here we fill with some data
|
|
rec.Key := FormatUTF8('KEY%',[i and pred(UNIQUE_KEY)]);
|
|
rec.Value := FormatUTF8('VALUE%',[i]);
|
|
batch.Add(rec,true);
|
|
if i and $2ffff = $2ffff then begin
|
|
write(' send');
|
|
db.batchSend(batch);
|
|
batch.Reset;
|
|
end;
|
|
end;
|
|
db.BatchSend(batch);
|
|
finally
|
|
batch.Free;
|
|
rec.Free;
|
|
end;
|
|
end;
|
|
|
|
procedure DoSelect;
|
|
var i, total: integer;
|
|
rec: TSQLEntry;
|
|
key: RawUTF8;
|
|
values: TRawUTF8DynArray;
|
|
begin
|
|
total := 0;
|
|
rec := TSQLEntry.Create;
|
|
try
|
|
for i := 0 to pred(UNIQUE_KEY) do begin
|
|
key := FormatUTF8('KEY%',[i]);
|
|
// 1. first way: get all values at once
|
|
db.OneFieldValues(TSQLEntry,'Value',FormatUTF8('Key=?',[],[key]),values);
|
|
// here values[] contains all "Value" column associated with this key
|
|
// 2. second way: use ORM
|
|
rec.FillPrepare(db,'Key=?',[],[key],'Value');
|
|
while rec.FillOne do begin
|
|
// here you have rec.Value filled
|
|
assert(rec.Value=values[rec.FillCurrentRow-2]);
|
|
inc(total);
|
|
end;
|
|
end;
|
|
assert(total=COUNT);
|
|
finally
|
|
rec.Free;
|
|
end;
|
|
end;
|
|
|
|
var timer: TPrecisionTimer;
|
|
begin
|
|
if true then
|
|
db := TSQLRestServerDB.CreateWithOwnModel([TSQLEntry],SQLITE_MEMORY_DATABASE_NAME)
|
|
else begin
|
|
deletefile('test.db');
|
|
db := TSQLRestServerDB.CreateWithOwnModel([TSQLEntry],'test.db');
|
|
end;
|
|
try
|
|
db.DB.LockingMode := lmExclusive;
|
|
db.DB.Synchronous := smOff;
|
|
db.CreateMissingTables;
|
|
write('INSERT ', COUNT, ' rows');
|
|
timer.Start;
|
|
DoInsert;
|
|
writeln(' in ',timer.Stop);
|
|
write('CREATE INDEX ', COUNT);
|
|
timer.Start;
|
|
db.CreateSQLIndex(TSQLEntry,'Key',false); // faster done after insertion
|
|
writeln(' in ',timer.Stop);
|
|
write('SELECT ', COUNT, ' rows per Key index');
|
|
timer.Start;
|
|
DoSelect;
|
|
writeln(' in ',timer.Stop);
|
|
finally
|
|
db.Free;
|
|
end;
|
|
end;
|
|
|
|
begin
|
|
Process;
|
|
writeln('Press any key');
|
|
readln;
|
|
end.
|