ActorsParent.cpp 799 KB
Newer Older
1
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
3
4
5
6
7
8
9
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "ActorsParent.h"

#include <algorithm>
10
#include <numeric>
11
#include <stdint.h>  // UINTPTR_MAX, uintptr_t
12
13
14
15
16
17
18
#include "FileInfo.h"
#include "FileManager.h"
#include "IDBObjectStore.h"
#include "IDBTransaction.h"
#include "IndexedDatabase.h"
#include "IndexedDatabaseInlines.h"
#include "IndexedDatabaseManager.h"
19
#include "InitializedOnce.h"
20
21
22
23
24
25
#include "js/StructuredClone.h"
#include "js/Value.h"
#include "jsapi.h"
#include "KeyPath.h"
#include "mozilla/Attributes.h"
#include "mozilla/AutoRestore.h"
26
#include "mozilla/Casting.h"
27
#include "mozilla/CheckedInt.h"
28
#include "mozilla/CycleCollectedJSRuntime.h"
29
#include "mozilla/EndianUtils.h"
30
#include "mozilla/ErrorNames.h"
31
32
33
#include "mozilla/LazyIdleThread.h"
#include "mozilla/Maybe.h"
#include "mozilla/Preferences.h"
34
#include "mozilla/Result.h"
35
36
#include "mozilla/SnappyCompressOutputStream.h"
#include "mozilla/SnappyUncompressInputStream.h"
37
38
#include "mozilla/StaticPtr.h"
#include "mozilla/storage.h"
39
#include "mozilla/Telemetry.h"
40
#include "mozilla/Unused.h"
41
#include "mozilla/UniquePtrExtensions.h"
42
#include "mozilla/dom/ContentParent.h"
43
#include "mozilla/dom/File.h"
44
#include "mozilla/dom/FileBlobImpl.h"
45
#include "mozilla/dom/StructuredCloneTags.h"
46
#include "mozilla/dom/BrowserParent.h"
47
#include "mozilla/dom/filehandle/ActorsParent.h"
48
49
50
#include "mozilla/dom/indexedDB/PBackgroundIDBCursorParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBDatabaseParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBDatabaseFileParent.h"
51
#include "mozilla/dom/indexedDB/PBackgroundIDBDatabaseRequestParent.h"
52
53
54
55
56
#include "mozilla/dom/indexedDB/PBackgroundIDBFactoryParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBFactoryRequestParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBRequestParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBTransactionParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBVersionChangeTransactionParent.h"
57
#include "mozilla/dom/indexedDB/PBackgroundIndexedDBUtilsParent.h"
58
#include "mozilla/dom/IPCBlobInputStreamParent.h"
59
#include "mozilla/dom/IPCBlobUtils.h"
60
#include "mozilla/dom/quota/CheckedUnsafePtr.h"
61
62
#include "mozilla/dom/quota/Client.h"
#include "mozilla/dom/quota/FileStreams.h"
63
#include "mozilla/dom/quota/OriginScope.h"
64
#include "mozilla/dom/quota/QuotaCommon.h"
65
66
67
68
69
70
71
#include "mozilla/dom/quota/QuotaManager.h"
#include "mozilla/dom/quota/UsageInfo.h"
#include "mozilla/ipc/BackgroundParent.h"
#include "mozilla/ipc/BackgroundUtils.h"
#include "mozilla/ipc/InputStreamParams.h"
#include "mozilla/ipc/InputStreamUtils.h"
#include "mozilla/ipc/PBackground.h"
72
#include "mozilla/ipc/PBackgroundParent.h"
73
#include "mozilla/Scoped.h"
74
#include "mozilla/storage/Variant.h"
75
#include "nsAutoPtr.h"
76
77
78
79
80
#include "nsCharSeparatedTokenizer.h"
#include "nsClassHashtable.h"
#include "nsCOMPtr.h"
#include "nsDataHashtable.h"
#include "nsEscape.h"
81
#include "nsExceptionHandler.h"
82
83
#include "nsHashKeys.h"
#include "nsNetUtil.h"
84
#include "nsIAsyncInputStream.h"
85
#include "nsISimpleEnumerator.h"
86
87
88
#include "nsIEventTarget.h"
#include "nsIFile.h"
#include "nsIFileURL.h"
89
#include "nsIFileProtocolHandler.h"
90
#include "nsIInputStream.h"
91
#include "nsIInterfaceRequestor.h"
92
93
#include "nsInterfaceHashtable.h"
#include "nsIOutputStream.h"
94
#include "nsIPipe.h"
95
#include "nsIPrincipal.h"
96
#include "nsIScriptSecurityManager.h"
97
98
99
#include "nsISupports.h"
#include "nsISupportsImpl.h"
#include "nsISupportsPriority.h"
100
101
#include "nsIThread.h"
#include "nsITimer.h"
102
#include "nsIURI.h"
103
#include "nsIURIMutator.h"
104
105
#include "nsNetUtil.h"
#include "nsPrintfCString.h"
106
#include "nsQueryObject.h"
107
#include "nsRefPtrHashtable.h"
108
#include "nsStreamUtils.h"
109
#include "nsString.h"
110
#include "nsStringStream.h"
111
#include "nsThreadPool.h"
112
113
114
115
#include "nsThreadUtils.h"
#include "nsXPCOMCID.h"
#include "PermissionRequestBase.h"
#include "ProfilerHelpers.h"
116
117
#include "prsystem.h"
#include "prtime.h"
118
119
#include "ReportInternalError.h"
#include "snappy/snappy.h"
120

121
122
123
#define DISABLE_ASSERTS_FOR_FUZZING 0

#if DISABLE_ASSERTS_FOR_FUZZING
124
125
126
#  define ASSERT_UNLESS_FUZZING(...) \
    do {                             \
    } while (0)
127
#else
128
#  define ASSERT_UNLESS_FUZZING(...) MOZ_ASSERT(false, __VA_ARGS__)
129
130
#endif

131
132
#define IDB_DEBUG_LOG(_args) \
  MOZ_LOG(IndexedDatabaseManager::GetLoggingModule(), LogLevel::Debug, _args)
133

134
#if defined(MOZ_WIDGET_ANDROID)
135
#  define IDB_MOBILE
136
137
138
#endif

namespace mozilla {
139

140
MOZ_TYPE_SPECIFIC_SCOPED_POINTER_TEMPLATE(ScopedPRFileDesc, PRFileDesc,
141
142
                                          PR_Close);

143
144
145
146
147
namespace dom {
namespace indexedDB {

using namespace mozilla::dom::quota;
using namespace mozilla::ipc;
148
using mozilla::dom::quota::Client;
149

150
151
namespace {

152
class ConnectionPool;
153
154
155
156
class Cursor;
class Database;
struct DatabaseActorInfo;
class DatabaseFile;
157
class DatabaseLoggingInfo;
158
class DatabaseMaintenance;
159
class Factory;
160
class Maintenance;
161
class MutableFile;
162
163
class OpenDatabaseOp;
class TransactionBase;
164
class TransactionDatabaseOperationBase;
165
class VersionChangeTransaction;
166
167
template <bool StatementHasIndexKeyBindings>
struct ValueCursorTypeTraits;
168
169
170
171
172
173
174

/*******************************************************************************
 * Constants
 ******************************************************************************/

// If JS_STRUCTURED_CLONE_VERSION changes then we need to update our major
// schema version.
175
static_assert(JS_STRUCTURED_CLONE_VERSION == 8,
176
177
178
              "Need to update the major schema version.");

// Major schema version. Bump for almost everything.
179
const uint32_t kMajorSchemaVersion = 26;
180
181
182
183
184
185
186
187
188
189
190
191
192
193

// Minor schema version. Should almost always be 0 (maybe bump on release
// branches if we have to).
const uint32_t kMinorSchemaVersion = 0;

// The schema version we store in the SQLite database is a (signed) 32-bit
// integer. The major version is left-shifted 4 bits so the max value is
// 0xFFFFFFF. The minor version occupies the lower 4 bits and its max is 0xF.
static_assert(kMajorSchemaVersion <= 0xFFFFFFF,
              "Major version needs to fit in 28 bits.");
static_assert(kMinorSchemaVersion <= 0xF,
              "Minor version needs to fit in 4 bits.");

const int32_t kSQLiteSchemaVersion =
194
    int32_t((kMajorSchemaVersion << 4) + kMinorSchemaVersion);
195
196
197

const int32_t kStorageProgressGranularity = 1000;

198
199
200
201
202
// Changing the value here will override the page size of new databases only.
// A journal mode change and VACUUM are needed to change existing databases, so
// the best way to do that is to use the schema version upgrade mechanism.
const uint32_t kSQLitePageSizeOverride =
#ifdef IDB_MOBILE
203
    2048;
204
#else
205
    4096;
206
207
208
#endif

static_assert(kSQLitePageSizeOverride == /* mozStorage default */ 0 ||
209
210
211
                  (kSQLitePageSizeOverride % 2 == 0 &&
                   kSQLitePageSizeOverride >= 512 &&
                   kSQLitePageSizeOverride <= 65536),
212
213
214
215
              "Must be 0 (disabled) or a power of 2 between 512 and 65536!");

// Set to -1 to use SQLite's default, 0 to disable, or some positive number to
// enforce a custom limit.
216
const int32_t kMaxWALPages = 5000;  // 20MB on desktop, 10MB on mobile.
217
218
219
220
221

// Set to some multiple of the page size to grow the database in larger chunks.
const uint32_t kSQLiteGrowthIncrement = kSQLitePageSizeOverride * 2;

static_assert(kSQLiteGrowthIncrement >= 0 &&
222
223
                  kSQLiteGrowthIncrement % kSQLitePageSizeOverride == 0 &&
                  kSQLiteGrowthIncrement < uint32_t(INT32_MAX),
224
225
              "Must be 0 (disabled) or a positive multiple of the page size!");

226
227
228
229
230
231
232
233
234
235
236
237
238
239
// The maximum number of threads that can be used for database activity at a
// single time.
const uint32_t kMaxConnectionThreadCount = 20;

static_assert(kMaxConnectionThreadCount, "Must have at least one thread!");

// The maximum number of threads to keep when idle. Threads that become idle in
// excess of this number will be shut down immediately.
const uint32_t kMaxIdleConnectionThreadCount = 2;

static_assert(kMaxConnectionThreadCount >= kMaxIdleConnectionThreadCount,
              "Idle thread limit must be less than total thread limit!");

// The length of time that database connections will be held open after all
240
// transactions have completed before doing idle maintenance.
241
const uint32_t kConnectionIdleMaintenanceMS = 2 * 1000;  // 2 seconds
242
243

// The length of time that database connections will be held open after all
244
// transactions and maintenance have completed.
245
const uint32_t kConnectionIdleCloseMS = 10 * 1000;  // 10 seconds
246
247

// The length of time that idle threads will stay alive before being shut down.
248
const uint32_t kConnectionThreadIdleMS = 30 * 1000;  // 30 seconds
249

250
#define SAVEPOINT_CLAUSE "SAVEPOINT sp;"
251
252
253

const uint32_t kFileCopyBufferSize = 32768;

254
constexpr auto kJournalDirectoryName = NS_LITERAL_STRING("journals");
255

256
257
258
259
260
constexpr auto kFileManagerDirectoryNameSuffix = NS_LITERAL_STRING(".files");
constexpr auto kSQLiteSuffix = NS_LITERAL_STRING(".sqlite");
constexpr auto kSQLiteJournalSuffix = NS_LITERAL_STRING(".sqlite-journal");
constexpr auto kSQLiteSHMSuffix = NS_LITERAL_STRING(".sqlite-shm");
constexpr auto kSQLiteWALSuffix = NS_LITERAL_STRING(".sqlite-wal");
261

262
263
const char kPrefFileHandleEnabled[] = "dom.fileHandle.enabled";

264
265
266
267
268
269
270
271
272
273
274
275
276
277
constexpr auto kPermissionStringBase = NS_LITERAL_CSTRING("indexedDB-chrome-");
constexpr auto kPermissionReadSuffix = NS_LITERAL_CSTRING("-read");
constexpr auto kPermissionWriteSuffix = NS_LITERAL_CSTRING("-write");

// The following constants define all names of binding parameters in statements,
// where they are bound by name. This should include all parameter names which
// are bound by name. Binding may be done by index when the statement definition
// and binding are done in the same local scope, and no other reasons prevent
// using the indexes (e.g. multiple statement variants with differing number or
// order of parameters). Neither the styles of specifying parameter names
// (literally vs. via these constants) nor the binding styles (by index vs. by
// name) should not be mixed for the same statement. The decision must be made
// for each statement based on the proximity of statement and binding calls.
constexpr auto kStmtParamNameCurrentKey = NS_LITERAL_CSTRING("current_key");
278
279
280
constexpr auto kStmtParamNameRangeBound = NS_LITERAL_CSTRING("range_bound");
constexpr auto kStmtParamNameObjectStorePosition =
    NS_LITERAL_CSTRING("object_store_position");
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
constexpr auto kStmtParamNameLowerKey = NS_LITERAL_CSTRING("lower_key");
constexpr auto kStmtParamNameUpperKey = NS_LITERAL_CSTRING("upper_key");
constexpr auto kStmtParamNameKey = NS_LITERAL_CSTRING("key");
constexpr auto kStmtParamNameObjectStoreId =
    NS_LITERAL_CSTRING("object_store_id");
constexpr auto kStmtParamNameIndexId = NS_LITERAL_CSTRING("index_id");
// TODO: Maybe the uses of kStmtParamNameId should be replaced by more
// specific constants such as kStmtParamNameObjectStoreId.
constexpr auto kStmtParamNameId = NS_LITERAL_CSTRING("id");
constexpr auto kStmtParamNameValue = NS_LITERAL_CSTRING("value");
constexpr auto kStmtParamNameObjectDataKey =
    NS_LITERAL_CSTRING("object_data_key");
constexpr auto kStmtParamNameIndexDataValues =
    NS_LITERAL_CSTRING("index_data_values");
constexpr auto kStmtParamNameData = NS_LITERAL_CSTRING("data");
constexpr auto kStmtParamNameFileIds = NS_LITERAL_CSTRING("file_ids");
constexpr auto kStmtParamNameValueLocale = NS_LITERAL_CSTRING("value_locale");
298
constexpr auto kStmtParamNameLimit = NS_LITERAL_CSTRING("limit");
299
300
301
302
303
304

// The following constants define some names of columns in tables, which are
// referred to in remote locations, e.g. in calls to
// GetBindingClauseForKeyRange.
constexpr auto kColumnNameKey = NS_LITERAL_CSTRING("key");
constexpr auto kColumnNameValue = NS_LITERAL_CSTRING("value");
305
constexpr auto kColumnNameAliasSortKey = NS_LITERAL_CSTRING("sort_column");
306
307
308

// SQL fragments used at multiple locations.
constexpr auto kOpenLimit = NS_LITERAL_CSTRING(" LIMIT ");
309

310
311
312
313
314
315
316
// The deletion marker file is created before RemoveDatabaseFilesAndDirectory
// begins deleting a database. It is removed as the last step of deletion. If a
// deletion marker file is found when initializing the origin, the deletion
// routine is run again to ensure that the database and all of its related files
// are removed. The primary goal of this mechanism is to avoid situations where
// a database has been partially deleted, leading to inconsistent state for the
// origin.
317
318
constexpr auto kIdbDeletionMarkerFilePrefix =
    NS_LITERAL_STRING("idb-deleting-");
319

320
321
const uint32_t kDeleteTimeoutMs = 1000;

322
323
324
325
326
327
328
329
330
331
332
333
334
335
/**
 * Automatically crash the browser if IndexedDB shutdown takes this long.  We've
 * chosen a value that is longer than the value for QuotaManager shutdown timer
 * which is currently set to 30 seconds.  We've also chosen a value that is long
 * long enough that it is unlikely for the problem to be falsely triggered by
 * slow system I/O.  We've also chosen a value long enough so that automated
 * tests should time out and fail if IndexedDB shutdown hangs.  Also, this value
 * is long enough so that testers can notice the IndexedDB shutdown hang; we
 * want to know about the hangs, not hide them.  On the other hand this value is
 * less than 60 seconds which is used by nsTerminator to crash a hung main
 * process.
 */
#define SHUTDOWN_TIMEOUT_MS 50000

336
337
338
339
340
#ifdef DEBUG

const int32_t kDEBUGThreadPriority = nsISupportsPriority::PRIORITY_NORMAL;
const uint32_t kDEBUGThreadSleepMS = 0;

341
const int32_t kDEBUGTransactionThreadPriority =
342
    nsISupportsPriority::PRIORITY_NORMAL;
343
344
const uint32_t kDEBUGTransactionThreadSleepMS = 0;

345
346
347
348
349
350
#endif

/*******************************************************************************
 * Metadata classes
 ******************************************************************************/

351
352
353
// Can be instantiated either on the QuotaManager IO thread or on a
// versionchange transaction thread. These threads can never race so this is
// totally safe.
354
struct FullIndexMetadata {
355
356
  IndexMetadata mCommonMetadata = {0,     nsString(), KeyPath(0), nsCString(),
                                   false, false,      false};
357

358
  FlippedOnce<false> mDeleted;
359
360
361

  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(FullIndexMetadata)

362
 private:
363
  ~FullIndexMetadata() = default;
364
365
366
367
};

typedef nsRefPtrHashtable<nsUint64HashKey, FullIndexMetadata> IndexTable;

368
369
370
// Can be instantiated either on the QuotaManager IO thread or on a
// versionchange transaction thread. These threads can never race so this is
// totally safe.
371
struct FullObjectStoreMetadata {
372
  ObjectStoreMetadata mCommonMetadata = {0, nsString(), KeyPath(0), false};
373
374
375
  IndexTable mIndexes;

  // These two members are only ever touched on a transaction thread!
376
377
  int64_t mNextAutoIncrementId = 0;
  int64_t mCommittedAutoIncrementId = 0;
378

379
  FlippedOnce<false> mDeleted;
380
381
382

  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(FullObjectStoreMetadata);

383
  bool HasLiveIndexes() const;
384

385
 private:
386
  ~FullObjectStoreMetadata() = default;
387
388
389
};

typedef nsRefPtrHashtable<nsUint64HashKey, FullObjectStoreMetadata>
390
    ObjectStoreTable;
391

392
struct FullDatabaseMetadata {
393
394
395
396
397
398
399
400
  DatabaseMetadata mCommonMetadata;
  nsCString mDatabaseId;
  nsString mFilePath;
  ObjectStoreTable mObjectStores;

  int64_t mNextObjectStoreId;
  int64_t mNextIndexId;

401
 public:
402
  explicit FullDatabaseMetadata(const DatabaseMetadata& aCommonMetadata)
403
404
405
      : mCommonMetadata(aCommonMetadata),
        mNextObjectStoreId(0),
        mNextIndexId(0) {
406
407
408
409
410
    AssertIsOnBackgroundThread();
  }

  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(FullDatabaseMetadata)

411
  already_AddRefed<FullDatabaseMetadata> Duplicate() const;
412

413
 private:
414
  ~FullDatabaseMetadata() = default;
415
416
417
};

template <class MetadataType>
418
class MOZ_STACK_CLASS MetadataNameOrIdMatcher final {
419
420
421
422
  typedef MetadataNameOrIdMatcher<MetadataType> SelfType;

  const int64_t mId;
  const nsString mName;
423
  RefPtr<MetadataType> mMetadata;
424
425
  bool mCheckName;

426
 public:
427
  template <class Enumerable>
428
429
  static MetadataType* Match(const Enumerable& aEnumerable, uint64_t aId,
                             const nsAString& aName) {
430
431
432
433
    AssertIsOnBackgroundThread();
    MOZ_ASSERT(aId);

    SelfType closure(aId, aName);
434
    MatchHelper(aEnumerable, &closure);
435
436
437
438
439

    return closure.mMetadata;
  }

  template <class Enumerable>
440
  static MetadataType* Match(const Enumerable& aEnumerable, uint64_t aId) {
441
442
443
444
    AssertIsOnBackgroundThread();
    MOZ_ASSERT(aId);

    SelfType closure(aId);
445
    MatchHelper(aEnumerable, &closure);
446
447
448
449

    return closure.mMetadata;
  }

450
 private:
451
  MetadataNameOrIdMatcher(const int64_t& aId, const nsAString& aName)
452
      : mId(aId), mName(aName), mMetadata(nullptr), mCheckName(true) {
453
454
455
456
457
    AssertIsOnBackgroundThread();
    MOZ_ASSERT(aId);
  }

  explicit MetadataNameOrIdMatcher(const int64_t& aId)
458
      : mId(aId), mMetadata(nullptr), mCheckName(false) {
459
460
461
462
    AssertIsOnBackgroundThread();
    MOZ_ASSERT(aId);
  }

463
  template <class Enumerable>
464
  static void MatchHelper(const Enumerable& aEnumerable, SelfType* aClosure) {
465
466
467
    AssertIsOnBackgroundThread();
    MOZ_ASSERT(aClosure);

468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
    for (auto iter = aEnumerable.ConstIter(); !iter.Done(); iter.Next()) {
#ifdef DEBUG
      const uint64_t key = iter.Key();
#endif
      MetadataType* value = iter.UserData();
      MOZ_ASSERT(key != 0);
      MOZ_ASSERT(value);

      if (!value->mDeleted &&
          (aClosure->mId == value->mCommonMetadata.id() ||
           (aClosure->mCheckName &&
            aClosure->mName == value->mCommonMetadata.name()))) {
        aClosure->mMetadata = value;
        break;
      }
483
484
485
486
    }
  }
};

487
struct IndexDataValue final {
488
  int64_t mIndexId;
489
490
  Key mPosition;
  Key mLocaleAwarePosition;
491
492
  bool mUnique;

493
  IndexDataValue() : mIndexId(0), mUnique(false) {
494
495
496
    MOZ_COUNT_CTOR(IndexDataValue);
  }

497
498
  explicit IndexDataValue(const IndexDataValue& aOther)
      : mIndexId(aOther.mIndexId),
499
500
        mPosition(aOther.mPosition),
        mLocaleAwarePosition(aOther.mLocaleAwarePosition),
501
        mUnique(aOther.mUnique) {
502
    MOZ_ASSERT(!aOther.mPosition.IsUnset());
503
504
505
506

    MOZ_COUNT_CTOR(IndexDataValue);
  }

507
508
509
  IndexDataValue(int64_t aIndexId, bool aUnique, const Key& aPosition)
      : mIndexId(aIndexId), mPosition(aPosition), mUnique(aUnique) {
    MOZ_ASSERT(!aPosition.IsUnset());
510
511
512
513

    MOZ_COUNT_CTOR(IndexDataValue);
  }

514
515
516
517
518
519
520
  IndexDataValue(int64_t aIndexId, bool aUnique, const Key& aPosition,
                 const Key& aLocaleAwarePosition)
      : mIndexId(aIndexId),
        mPosition(aPosition),
        mLocaleAwarePosition(aLocaleAwarePosition),
        mUnique(aUnique) {
    MOZ_ASSERT(!aPosition.IsUnset());
521
522
523
524

    MOZ_COUNT_CTOR(IndexDataValue);
  }

525
  ~IndexDataValue() { MOZ_COUNT_DTOR(IndexDataValue); }
526

527
  bool operator==(const IndexDataValue& aOther) const {
528
529
530
    if (mIndexId != aOther.mIndexId) {
      return false;
    }
531
532
    if (mLocaleAwarePosition.IsUnset()) {
      return mPosition == aOther.mPosition;
533
    }
534
    return mLocaleAwarePosition == aOther.mLocaleAwarePosition;
535
536
  }

537
  bool operator<(const IndexDataValue& aOther) const {
538
    if (mIndexId == aOther.mIndexId) {
539
540
      if (mLocaleAwarePosition.IsUnset()) {
        return mPosition < aOther.mPosition;
541
      }
542
      return mLocaleAwarePosition < aOther.mLocaleAwarePosition;
543
544
545
546
547
548
    }

    return mIndexId < aOther.mIndexId;
  }
};

549
550
551
552
/*******************************************************************************
 * SQLite functions
 ******************************************************************************/

553
554
constexpr int32_t MakeSchemaVersion(uint32_t aMajorSchemaVersion,
                                    uint32_t aMinorSchemaVersion) {
555
556
557
  return int32_t((aMajorSchemaVersion << 4) + aMinorSchemaVersion);
}

558
559
560
561
// WARNING: the hash function used for the database name must not change.
// That's why this function exists separately from mozilla::HashString(), even
// though it is (at the time of writing) equivalent. See bug 780408 and bug
// 940315 for details.
562
563
564
uint32_t HashName(const nsAString& aName) {
  struct Helper {
    static uint32_t RotateBitsLeft32(uint32_t aValue, uint8_t aBits) {
565
566
567
568
569
570
571
      MOZ_ASSERT(aBits < 32);
      return (aValue << aBits) | (aValue >> (32 - aBits));
    }
  };

  static const uint32_t kGoldenRatioU32 = 0x9e3779b9u;

572
573
574
575
576
  return std::accumulate(aName.BeginReading(), aName.EndReading(), uint32_t(0),
                         [](uint32_t hash, char16_t ch) {
                           return kGoldenRatioU32 *
                                  (Helper::RotateBitsLeft32(hash, 5) ^ ch);
                         });
577
578
}

579
nsresult ClampResultCode(nsresult aResultCode) {
580
581
582
583
584
585
586
587
588
589
590
591
  if (NS_SUCCEEDED(aResultCode) ||
      NS_ERROR_GET_MODULE(aResultCode) == NS_ERROR_MODULE_DOM_INDEXEDDB) {
    return aResultCode;
  }

  switch (aResultCode) {
    case NS_ERROR_FILE_NO_DEVICE_SPACE:
      return NS_ERROR_DOM_INDEXEDDB_QUOTA_ERR;
    case NS_ERROR_STORAGE_CONSTRAINT:
      return NS_ERROR_DOM_INDEXEDDB_CONSTRAINT_ERR;
    default:
#ifdef DEBUG
592
593
      nsPrintfCString message("Converting non-IndexedDB error code (0x%" PRIX32
                              ") to "
594
                              "NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR",
595
                              static_cast<uint32_t>(aResultCode));
596
597
      NS_WARNING(message.get());
#else
598
        ;
599
600
601
602
603
604
605
#endif
  }

  IDB_REPORT_INTERNAL_ERR();
  return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
}

606
607
void GetDatabaseFilename(const nsAString& aName,
                         nsAutoString& aDatabaseFilename) {
608
609
  MOZ_ASSERT(aDatabaseFilename.IsEmpty());

610
611
  // WARNING: do not change this hash function. See the comment in HashName()
  // for details.
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
  aDatabaseFilename.AppendInt(HashName(aName));

  nsCString escapedName;
  if (!NS_Escape(NS_ConvertUTF16toUTF8(aName), escapedName, url_XPAlphas)) {
    MOZ_CRASH("Can't escape database name!");
  }

  const char* forwardIter = escapedName.BeginReading();
  const char* backwardIter = escapedName.EndReading() - 1;

  nsAutoCString substring;
  while (forwardIter <= backwardIter && substring.Length() < 21) {
    if (substring.Length() % 2) {
      substring.Append(*backwardIter--);
    } else {
      substring.Append(*forwardIter++);
    }
  }

  aDatabaseFilename.AppendASCII(substring.get(), substring.Length());
}

634
uint32_t CompressedByteCountForNumber(uint64_t aNumber) {
635
636
637
638
639
640
641
642
643
  // All bytes have 7 bits available.
  uint32_t count = 1;
  while ((aNumber >>= 7)) {
    count++;
  }

  return count;
}

644
uint32_t CompressedByteCountForIndexId(int64_t aIndexId) {
645
646
  MOZ_ASSERT(aIndexId);
  MOZ_ASSERT(UINT64_MAX - uint64_t(aIndexId) >= uint64_t(aIndexId),
647
             "Overflow!");
648
649
650
651

  return CompressedByteCountForNumber(uint64_t(aIndexId * 2));
}

652
void WriteCompressedNumber(uint64_t aNumber, uint8_t** aIterator) {
653
654
655
656
657
658
  MOZ_ASSERT(aIterator);
  MOZ_ASSERT(*aIterator);

  uint8_t*& buffer = *aIterator;

#ifdef DEBUG
659
  const uint8_t* const bufferStart = buffer;
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
  const uint64_t originalNumber = aNumber;
#endif

  while (true) {
    uint64_t shiftedNumber = aNumber >> 7;
    if (shiftedNumber) {
      *buffer++ = uint8_t(0x80 | (aNumber & 0x7f));
      aNumber = shiftedNumber;
    } else {
      *buffer++ = uint8_t(aNumber);
      break;
    }
  }

  MOZ_ASSERT(buffer > bufferStart);
  MOZ_ASSERT(uint32_t(buffer - bufferStart) ==
676
             CompressedByteCountForNumber(originalNumber));
677
678
}

679
uint64_t ReadCompressedNumber(const uint8_t** aIterator, const uint8_t* aEnd) {
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
  MOZ_ASSERT(aIterator);
  MOZ_ASSERT(*aIterator);
  MOZ_ASSERT(aEnd);
  MOZ_ASSERT(*aIterator < aEnd);

  const uint8_t*& buffer = *aIterator;

  uint8_t shiftCounter = 0;
  uint64_t result = 0;

  while (true) {
    MOZ_ASSERT(shiftCounter <= 56, "Shifted too many bits!");

    result += (uint64_t(*buffer & 0x7f) << shiftCounter);
    shiftCounter += 7;

    if (!(*buffer++ & 0x80)) {
      break;
    }

    if (NS_WARN_IF(buffer == aEnd)) {
      MOZ_ASSERT(false);
      break;
    }
  }

  return result;
}

709
710
void WriteCompressedIndexId(int64_t aIndexId, bool aUnique,
                            uint8_t** aIterator) {
711
712
713
714
715
716
717
718
719
720
  MOZ_ASSERT(aIndexId);
  MOZ_ASSERT(UINT64_MAX - uint64_t(aIndexId) >= uint64_t(aIndexId),
             "Overflow!");
  MOZ_ASSERT(aIterator);
  MOZ_ASSERT(*aIterator);

  const uint64_t indexId = (uint64_t(aIndexId * 2) | (aUnique ? 1 : 0));
  WriteCompressedNumber(indexId, aIterator);
}

721
722
void ReadCompressedIndexId(const uint8_t** aIterator, const uint8_t* aEnd,
                           int64_t* aIndexId, bool* aUnique) {
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
  MOZ_ASSERT(aIterator);
  MOZ_ASSERT(*aIterator);
  MOZ_ASSERT(aIndexId);
  MOZ_ASSERT(aUnique);

  uint64_t indexId = ReadCompressedNumber(aIterator, aEnd);

  if (indexId % 2) {
    *aUnique = true;
    indexId--;
  } else {
    *aUnique = false;
  }

  MOZ_ASSERT(UINT64_MAX / 2 >= uint64_t(indexId), "Bad index id!");

  *aIndexId = int64_t(indexId / 2);
}

// static
743
744
745
746
nsresult MakeCompressedIndexDataValues(
    const FallibleTArray<IndexDataValue>& aIndexValues,
    UniqueFreePtr<uint8_t>& aCompressedIndexDataValues,
    uint32_t* aCompressedIndexDataValuesLength) {
747
748
749
750
751
  MOZ_ASSERT(!NS_IsMainThread());
  MOZ_ASSERT(!IsOnBackgroundThread());
  MOZ_ASSERT(!aCompressedIndexDataValues);
  MOZ_ASSERT(aCompressedIndexDataValuesLength);

752
  AUTO_PROFILER_LABEL("MakeCompressedIndexDataValues", DOM);
753
754
755
756
757
758
759
760
761
762

  const uint32_t arrayLength = aIndexValues.Length();
  if (!arrayLength) {
    *aCompressedIndexDataValuesLength = 0;
    return NS_OK;
  }

  // First calculate the size of the final buffer.
  uint32_t blobDataLength = 0;

763
  for (const IndexDataValue& info : aIndexValues) {
764
765
    const nsCString& keyBuffer = info.mPosition.GetBuffer();
    const nsCString& sortKeyBuffer = info.mLocaleAwarePosition.GetBuffer();
766
    const uint32_t keyBufferLength = keyBuffer.Length();
767
    const uint32_t sortKeyBufferLength = sortKeyBuffer.Length();
768
769
770

    MOZ_ASSERT(!keyBuffer.IsEmpty());

771
    const CheckedUint32 infoLength =
772
773
774
775
        CheckedUint32(CompressedByteCountForIndexId(info.mIndexId)) +
        CompressedByteCountForNumber(keyBufferLength) +
        CompressedByteCountForNumber(sortKeyBufferLength) + keyBufferLength +
        sortKeyBufferLength;
776
777
778
779
780
    // Don't let |infoLength| overflow.
    if (NS_WARN_IF(!infoLength.isValid())) {
      IDB_REPORT_INTERNAL_ERR();
      return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
    }
781
782

    // Don't let |blobDataLength| overflow.
783
    if (NS_WARN_IF(UINT32_MAX - infoLength.value() < blobDataLength)) {
784
785
786
787
      IDB_REPORT_INTERNAL_ERR();
      return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
    }

788
    blobDataLength += infoLength.value();
789
790
  }

791
  UniqueFreePtr<uint8_t> blobData(
792
      static_cast<uint8_t*>(malloc(blobDataLength)));
793
794
795
796
797
798
799
  if (NS_WARN_IF(!blobData)) {
    IDB_REPORT_INTERNAL_ERR();
    return NS_ERROR_OUT_OF_MEMORY;
  }

  uint8_t* blobDataIter = blobData.get();

800
  for (const IndexDataValue& info : aIndexValues) {
801
802
    const nsCString& keyBuffer = info.mPosition.GetBuffer();
    const nsCString& sortKeyBuffer = info.mLocaleAwarePosition.GetBuffer();
803
    const uint32_t keyBufferLength = keyBuffer.Length();
804
    const uint32_t sortKeyBufferLength = sortKeyBuffer.Length();
805
806

    WriteCompressedIndexId(info.mIndexId, info.mUnique, &blobDataIter);
807
    WriteCompressedNumber(keyBufferLength, &blobDataIter);
808
809
810

    memcpy(blobDataIter, keyBuffer.get(), keyBufferLength);
    blobDataIter += keyBufferLength;
811
812
813
814
815

    WriteCompressedNumber(sortKeyBufferLength, &blobDataIter);

    memcpy(blobDataIter, sortKeyBuffer.get(), sortKeyBufferLength);
    blobDataIter += sortKeyBufferLength;
816
817
818
819
820
821
822
823
824
825
  }

  MOZ_ASSERT(blobDataIter == blobData.get() + blobDataLength);

  aCompressedIndexDataValues.swap(blobData);
  *aCompressedIndexDataValuesLength = uint32_t(blobDataLength);

  return NS_OK;
}

826
827
828
nsresult ReadCompressedIndexDataValuesFromBlob(
    const uint8_t* aBlobData, uint32_t aBlobDataLength,
    nsTArray<IndexDataValue>& aIndexValues) {
829
830
831
832
833
834
  MOZ_ASSERT(!NS_IsMainThread());
  MOZ_ASSERT(!IsOnBackgroundThread());
  MOZ_ASSERT(aBlobData);
  MOZ_ASSERT(aBlobDataLength);
  MOZ_ASSERT(aIndexValues.IsEmpty());

835
  AUTO_PROFILER_LABEL("ReadCompressedIndexDataValuesFromBlob", DOM);
836

837
838
839
840
841
  if (uintptr_t(aBlobData) > UINTPTR_MAX - aBlobDataLength) {
    IDB_REPORT_INTERNAL_ERR();
    return NS_ERROR_FILE_CORRUPTED;
  }

842
  const uint8_t* blobDataIter = aBlobData;
843
  const uint8_t* const blobDataEnd = aBlobData + aBlobDataLength;
844
845
846
847
848
849
850
851
852
853
854
855
856

  while (blobDataIter < blobDataEnd) {
    int64_t indexId;
    bool unique;
    ReadCompressedIndexId(&blobDataIter, blobDataEnd, &indexId, &unique);

    if (NS_WARN_IF(blobDataIter == blobDataEnd)) {
      IDB_REPORT_INTERNAL_ERR();
      return NS_ERROR_FILE_CORRUPTED;
    }

    // Read key buffer length.
    const uint64_t keyBufferLength =
857
        ReadCompressedNumber(&blobDataIter, blobDataEnd);
858
859
860

    if (NS_WARN_IF(blobDataIter == blobDataEnd) ||
        NS_WARN_IF(keyBufferLength > uint64_t(UINT32_MAX)) ||
861
862
        NS_WARN_IF(keyBufferLength > uintptr_t(blobDataEnd)) ||
        NS_WARN_IF(blobDataIter > blobDataEnd - keyBufferLength)) {
863
864
865
866
867
868
869
870
      IDB_REPORT_INTERNAL_ERR();
      return NS_ERROR_FILE_CORRUPTED;
    }

    nsCString keyBuffer(reinterpret_cast<const char*>(blobDataIter),
                        uint32_t(keyBufferLength));
    blobDataIter += keyBufferLength;

871
872
873
874
    IndexDataValue idv(indexId, unique, Key(keyBuffer));

    // Read sort key buffer length.
    const uint64_t sortKeyBufferLength =
875
        ReadCompressedNumber(&blobDataIter, blobDataEnd);
876
877
878
879

    if (sortKeyBufferLength > 0) {
      if (NS_WARN_IF(blobDataIter == blobDataEnd) ||
          NS_WARN_IF(sortKeyBufferLength > uint64_t(UINT32_MAX)) ||
880
881
          NS_WARN_IF(sortKeyBufferLength > uintptr_t(blobDataEnd)) ||
          NS_WARN_IF(blobDataIter > blobDataEnd - sortKeyBufferLength)) {
882
883
884
885
886
887
888
889
        IDB_REPORT_INTERNAL_ERR();
        return NS_ERROR_FILE_CORRUPTED;
      }

      nsCString sortKeyBuffer(reinterpret_cast<const char*>(blobDataIter),
                              uint32_t(sortKeyBufferLength));
      blobDataIter += sortKeyBufferLength;

890
      idv.mLocaleAwarePosition = Key(sortKeyBuffer);
891
892
893
    }

    if (NS_WARN_IF(!aIndexValues.InsertElementSorted(idv, fallible))) {
894
895
896
897
898
899
900
901
902
903
904
905
      IDB_REPORT_INTERNAL_ERR();
      return NS_ERROR_OUT_OF_MEMORY;
    }
  }

  MOZ_ASSERT(blobDataIter == blobDataEnd);

  return NS_OK;
}

// static
template <typename T>
906
907
nsresult ReadCompressedIndexDataValuesFromSource(
    T* aSource, uint32_t aColumnIndex, nsTArray<IndexDataValue>& aIndexValues) {
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
  MOZ_ASSERT(!NS_IsMainThread());
  MOZ_ASSERT(!IsOnBackgroundThread());
  MOZ_ASSERT(aSource);
  MOZ_ASSERT(aIndexValues.IsEmpty());

  int32_t columnType;
  nsresult rv = aSource->GetTypeOfIndex(aColumnIndex, &columnType);
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

  if (columnType == mozIStorageStatement::VALUE_TYPE_NULL) {
    return NS_OK;
  }

  MOZ_ASSERT(columnType == mozIStorageStatement::VALUE_TYPE_BLOB);

  const uint8_t* blobData;
  uint32_t blobDataLength;
  rv = aSource->GetSharedBlob(aColumnIndex, &blobDataLength, &blobData);
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

  if (NS_WARN_IF(!blobDataLength)) {
    IDB_REPORT_INTERNAL_ERR();
    return NS_ERROR_FILE_CORRUPTED;
  }

937
  rv = ReadCompressedIndexDataValuesFromBlob(blobData, blobDataLength,
938
939
940
941
942
943
944
945
                                             aIndexValues);
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

  return NS_OK;
}

946
947
948
949
nsresult ReadCompressedIndexDataValues(mozIStorageStatement* aStatement,
                                       uint32_t aColumnIndex,
                                       nsTArray<IndexDataValue>& aIndexValues) {
  return ReadCompressedIndexDataValuesFromSource(aStatement, aColumnIndex,
950
951
952
                                                 aIndexValues);
}

953
954
955
956
nsresult ReadCompressedIndexDataValues(mozIStorageValueArray* aValues,
                                       uint32_t aColumnIndex,
                                       nsTArray<IndexDataValue>& aIndexValues) {
  return ReadCompressedIndexDataValuesFromSource(aValues, aColumnIndex,
957
958
959
                                                 aIndexValues);
}

960
nsresult CreateFileTables(mozIStorageConnection* aConnection) {
961
962
963
  AssertIsOnIOThread();
  MOZ_ASSERT(aConnection);

964
  AUTO_PROFILER_LABEL("CreateFileTables", DOM);
965
966

  // Table `file`
967
968
969
970
971
  nsresult rv = aConnection->ExecuteSimpleSQL(
      NS_LITERAL_CSTRING("CREATE TABLE file ("
                         "id INTEGER PRIMARY KEY, "
                         "refcount INTEGER NOT NULL"
                         ");"));
972
973
974
975
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

976
977
978
979
980
981
982
983
  rv = aConnection->ExecuteSimpleSQL(
      NS_LITERAL_CSTRING("CREATE TRIGGER object_data_insert_trigger "
                         "AFTER INSERT ON object_data "
                         "FOR EACH ROW "
                         "WHEN NEW.file_ids IS NOT NULL "
                         "BEGIN "
                         "SELECT update_refcount(NULL, NEW.file_ids); "
                         "END;"));
984
985
986
987
988
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

  rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
989
990
991
992
993
      "CREATE TRIGGER object_data_update_trigger "
      "AFTER UPDATE OF file_ids ON object_data "
      "FOR EACH ROW "
      "WHEN OLD.file_ids IS NOT NULL OR NEW.file_ids IS NOT NULL "
      "BEGIN "
994
      "SELECT update_refcount(OLD.file_ids, NEW.file_ids); "
995
      "END;"));
996
997
998
999
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

1000
  rv = aConnection->ExecuteSimpleSQL(