ActorsParent.cpp

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "ActorsParent.h"

#include <algorithm>
#include <numeric>
#include <stdint.h>  // UINTPTR_MAX, uintptr_t
#include "FileInfo.h"
#include "FileManager.h"
#include "IDBObjectStore.h"
#include "IDBTransaction.h"
#include "IndexedDatabase.h"
#include "IndexedDatabaseInlines.h"
#include "IndexedDatabaseManager.h"
#include "InitializedOnce.h"
#include "js/StructuredClone.h"
#include "js/Value.h"
#include "jsapi.h"
#include "KeyPath.h"
#include "mozilla/Attributes.h"
#include "mozilla/AutoRestore.h"
#include "mozilla/Casting.h"
#include "mozilla/CheckedInt.h"
#include "mozilla/CycleCollectedJSRuntime.h"
#include "mozilla/EndianUtils.h"
#include "mozilla/ErrorNames.h"
#include "mozilla/LazyIdleThread.h"
#include "mozilla/Maybe.h"
#include "mozilla/Preferences.h"
#include "mozilla/Result.h"
#include "mozilla/SnappyCompressOutputStream.h"
#include "mozilla/SnappyUncompressInputStream.h"
#include "mozilla/StaticPtr.h"
#include "mozilla/storage.h"
#include "mozilla/Telemetry.h"
#include "mozilla/Unused.h"
#include "mozilla/UniquePtrExtensions.h"
#include "mozilla/dom/ContentParent.h"
#include "mozilla/dom/File.h"
#include "mozilla/dom/FileBlobImpl.h"
#include "mozilla/dom/StructuredCloneTags.h"
#include "mozilla/dom/BrowserParent.h"
#include "mozilla/dom/filehandle/ActorsParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBCursorParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBDatabaseParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBDatabaseFileParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBDatabaseRequestParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBFactoryParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBFactoryRequestParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBRequestParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBTransactionParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIDBVersionChangeTransactionParent.h"
#include "mozilla/dom/indexedDB/PBackgroundIndexedDBUtilsParent.h"
#include "mozilla/dom/IPCBlobInputStreamParent.h"
#include "mozilla/dom/IPCBlobUtils.h"
#include "mozilla/dom/quota/CheckedUnsafePtr.h"
#include "mozilla/dom/quota/Client.h"
#include "mozilla/dom/quota/FileStreams.h"
#include "mozilla/dom/quota/OriginScope.h"
#include "mozilla/dom/quota/QuotaCommon.h"
#include "mozilla/dom/quota/QuotaManager.h"
#include "mozilla/dom/quota/UsageInfo.h"
#include "mozilla/ipc/BackgroundParent.h"
#include "mozilla/ipc/BackgroundUtils.h"
#include "mozilla/ipc/InputStreamParams.h"
#include "mozilla/ipc/InputStreamUtils.h"
#include "mozilla/ipc/PBackground.h"
#include "mozilla/ipc/PBackgroundParent.h"
#include "mozilla/Scoped.h"
#include "mozilla/storage/Variant.h"
#include "nsAutoPtr.h"
#include "nsCharSeparatedTokenizer.h"
#include "nsClassHashtable.h"
#include "nsCOMPtr.h"
#include "nsDataHashtable.h"
#include "nsEscape.h"
#include "nsExceptionHandler.h"
#include "nsHashKeys.h"
#include "nsNetUtil.h"
#include "nsIAsyncInputStream.h"
#include "nsISimpleEnumerator.h"
#include "nsIEventTarget.h"
#include "nsIFile.h"
#include "nsIFileURL.h"
#include "nsIFileProtocolHandler.h"
#include "nsIInputStream.h"
#include "nsIInterfaceRequestor.h"
#include "nsInterfaceHashtable.h"
#include "nsIOutputStream.h"
#include "nsIPipe.h"
#include "nsIPrincipal.h"
#include "nsIScriptSecurityManager.h"
#include "nsISupports.h"
#include "nsISupportsImpl.h"
#include "nsISupportsPriority.h"
#include "nsIThread.h"
#include "nsITimer.h"
#include "nsIURI.h"
#include "nsIURIMutator.h"
#include "nsNetUtil.h"
#include "nsPrintfCString.h"
#include "nsQueryObject.h"
#include "nsRefPtrHashtable.h"
#include "nsStreamUtils.h"
#include "nsString.h"
#include "nsStringStream.h"
#include "nsThreadPool.h"
#include "nsThreadUtils.h"
#include "nsXPCOMCID.h"
#include "PermissionRequestBase.h"
#include "ProfilerHelpers.h"
#include "prsystem.h"
#include "prtime.h"
#include "ReportInternalError.h"
#include "snappy/snappy.h"

#define DISABLE_ASSERTS_FOR_FUZZING 0

#if DISABLE_ASSERTS_FOR_FUZZING
#  define ASSERT_UNLESS_FUZZING(...) \
    do {                             \
    } while (0)
#else
#  define ASSERT_UNLESS_FUZZING(...) MOZ_ASSERT(false, __VA_ARGS__)
#endif

#define IDB_DEBUG_LOG(_args) \
  MOZ_LOG(IndexedDatabaseManager::GetLoggingModule(), LogLevel::Debug, _args)

#if defined(MOZ_WIDGET_ANDROID)
#  define IDB_MOBILE
#endif

namespace mozilla {

MOZ_TYPE_SPECIFIC_SCOPED_POINTER_TEMPLATE(ScopedPRFileDesc, PRFileDesc,
                                          PR_Close);

namespace dom {
namespace indexedDB {

using namespace mozilla::dom::quota;
using namespace mozilla::ipc;
using mozilla::dom::quota::Client;

namespace {

class ConnectionPool;
class Cursor;
class Database;
struct DatabaseActorInfo;
class DatabaseFile;
class DatabaseLoggingInfo;
class DatabaseMaintenance;
class Factory;
class Maintenance;
class MutableFile;
class OpenDatabaseOp;
class TransactionBase;
class TransactionDatabaseOperationBase;
class VersionChangeTransaction;
template <bool StatementHasIndexKeyBindings>
struct ValueCursorTypeTraits;

/*******************************************************************************
 * Constants
 ******************************************************************************/

// If JS_STRUCTURED_CLONE_VERSION changes then we need to update our major
// schema version.
static_assert(JS_STRUCTURED_CLONE_VERSION == 8,
              "Need to update the major schema version.");

// Major schema version. Bump for almost everything.
const uint32_t kMajorSchemaVersion = 26;

// Minor schema version. Should almost always be 0 (maybe bump on release
// branches if we have to).
const uint32_t kMinorSchemaVersion = 0;

// The schema version we store in the SQLite database is a (signed) 32-bit
// integer. The major version is left-shifted 4 bits so the max value is
// 0xFFFFFFF. The minor version occupies the lower 4 bits and its max is 0xF.
static_assert(kMajorSchemaVersion <= 0xFFFFFFF,
              "Major version needs to fit in 28 bits.");
static_assert(kMinorSchemaVersion <= 0xF,
              "Minor version needs to fit in 4 bits.");

const int32_t kSQLiteSchemaVersion =
    int32_t((kMajorSchemaVersion << 4) + kMinorSchemaVersion);

const int32_t kStorageProgressGranularity = 1000;

// Changing the value here will override the page size of new databases only.
// A journal mode change and VACUUM are needed to change existing databases, so
// the best way to do that is to use the schema version upgrade mechanism.
const uint32_t kSQLitePageSizeOverride =
#ifdef IDB_MOBILE
    2048;
#else
    4096;
#endif

static_assert(kSQLitePageSizeOverride == /* mozStorage default */ 0 ||
                  (kSQLitePageSizeOverride % 2 == 0 &&
                   kSQLitePageSizeOverride >= 512 &&
                   kSQLitePageSizeOverride <= 65536),
              "Must be 0 (disabled) or a power of 2 between 512 and 65536!");

// Set to -1 to use SQLite's default, 0 to disable, or some positive number to
// enforce a custom limit.
const int32_t kMaxWALPages = 5000;  // 20MB on desktop, 10MB on mobile.

// Set to some multiple of the page size to grow the database in larger chunks.
const uint32_t kSQLiteGrowthIncrement = kSQLitePageSizeOverride * 2;

static_assert(kSQLiteGrowthIncrement >= 0 &&
                  kSQLiteGrowthIncrement % kSQLitePageSizeOverride == 0 &&
                  kSQLiteGrowthIncrement < uint32_t(INT32_MAX),
              "Must be 0 (disabled) or a positive multiple of the page size!");

// The maximum number of threads that can be used for database activity at a
// single time.
const uint32_t kMaxConnectionThreadCount = 20;

static_assert(kMaxConnectionThreadCount, "Must have at least one thread!");

// The maximum number of threads to keep when idle. Threads that become idle in
// excess of this number will be shut down immediately.
const uint32_t kMaxIdleConnectionThreadCount = 2;

static_assert(kMaxConnectionThreadCount >= kMaxIdleConnectionThreadCount,
              "Idle thread limit must be less than total thread limit!");

// The length of time that database connections will be held open after all
// transactions have completed before doing idle maintenance.
const uint32_t kConnectionIdleMaintenanceMS = 2 * 1000;  // 2 seconds

// The length of time that database connections will be held open after all
// transactions and maintenance have completed.
const uint32_t kConnectionIdleCloseMS = 10 * 1000;  // 10 seconds

// The length of time that idle threads will stay alive before being shut down.
const uint32_t kConnectionThreadIdleMS = 30 * 1000;  // 30 seconds

#define SAVEPOINT_CLAUSE "SAVEPOINT sp;"

const uint32_t kFileCopyBufferSize = 32768;

constexpr auto kJournalDirectoryName = NS_LITERAL_STRING("journals");

constexpr auto kFileManagerDirectoryNameSuffix = NS_LITERAL_STRING(".files");
constexpr auto kSQLiteSuffix = NS_LITERAL_STRING(".sqlite");
constexpr auto kSQLiteJournalSuffix = NS_LITERAL_STRING(".sqlite-journal");
constexpr auto kSQLiteSHMSuffix = NS_LITERAL_STRING(".sqlite-shm");
constexpr auto kSQLiteWALSuffix = NS_LITERAL_STRING(".sqlite-wal");

const char kPrefFileHandleEnabled[] = "dom.fileHandle.enabled";

constexpr auto kPermissionStringBase = NS_LITERAL_CSTRING("indexedDB-chrome-");
constexpr auto kPermissionReadSuffix = NS_LITERAL_CSTRING("-read");
constexpr auto kPermissionWriteSuffix = NS_LITERAL_CSTRING("-write");

// The following constants define all names of binding parameters in statements,
// where they are bound by name. This should include all parameter names which
// are bound by name. Binding may be done by index when the statement definition
// and binding are done in the same local scope, and no other reasons prevent
// using the indexes (e.g. multiple statement variants with differing number or
// order of parameters). Neither the styles of specifying parameter names
// (literally vs. via these constants) nor the binding styles (by index vs. by
// name) should not be mixed for the same statement. The decision must be made
// for each statement based on the proximity of statement and binding calls.
constexpr auto kStmtParamNameCurrentKey = NS_LITERAL_CSTRING("current_key");
constexpr auto kStmtParamNameRangeBound = NS_LITERAL_CSTRING("range_bound");
constexpr auto kStmtParamNameObjectStorePosition =
    NS_LITERAL_CSTRING("object_store_position");
constexpr auto kStmtParamNameLowerKey = NS_LITERAL_CSTRING("lower_key");
constexpr auto kStmtParamNameUpperKey = NS_LITERAL_CSTRING("upper_key");
constexpr auto kStmtParamNameKey = NS_LITERAL_CSTRING("key");
constexpr auto kStmtParamNameObjectStoreId =
    NS_LITERAL_CSTRING("object_store_id");
constexpr auto kStmtParamNameIndexId = NS_LITERAL_CSTRING("index_id");
// TODO: Maybe the uses of kStmtParamNameId should be replaced by more
// specific constants such as kStmtParamNameObjectStoreId.
constexpr auto kStmtParamNameId = NS_LITERAL_CSTRING("id");
constexpr auto kStmtParamNameValue = NS_LITERAL_CSTRING("value");
constexpr auto kStmtParamNameObjectDataKey =
    NS_LITERAL_CSTRING("object_data_key");
constexpr auto kStmtParamNameIndexDataValues =
    NS_LITERAL_CSTRING("index_data_values");
constexpr auto kStmtParamNameData = NS_LITERAL_CSTRING("data");
constexpr auto kStmtParamNameFileIds = NS_LITERAL_CSTRING("file_ids");
constexpr auto kStmtParamNameValueLocale = NS_LITERAL_CSTRING("value_locale");
constexpr auto kStmtParamNameLimit = NS_LITERAL_CSTRING("limit");

// The following constants define some names of columns in tables, which are
// referred to in remote locations, e.g. in calls to
// GetBindingClauseForKeyRange.
constexpr auto kColumnNameKey = NS_LITERAL_CSTRING("key");
constexpr auto kColumnNameValue = NS_LITERAL_CSTRING("value");
constexpr auto kColumnNameAliasSortKey = NS_LITERAL_CSTRING("sort_column");

// SQL fragments used at multiple locations.
constexpr auto kOpenLimit = NS_LITERAL_CSTRING(" LIMIT ");

// The deletion marker file is created before RemoveDatabaseFilesAndDirectory
// begins deleting a database. It is removed as the last step of deletion. If a
// deletion marker file is found when initializing the origin, the deletion
// routine is run again to ensure that the database and all of its related files
// are removed. The primary goal of this mechanism is to avoid situations where
// a database has been partially deleted, leading to inconsistent state for the
// origin.
constexpr auto kIdbDeletionMarkerFilePrefix =
    NS_LITERAL_STRING("idb-deleting-");

const uint32_t kDeleteTimeoutMs = 1000;

/**
 * Automatically crash the browser if IndexedDB shutdown takes this long.  We've
 * chosen a value that is longer than the value for QuotaManager shutdown timer
 * which is currently set to 30 seconds.  We've also chosen a value that is long
 * long enough that it is unlikely for the problem to be falsely triggered by
 * slow system I/O.  We've also chosen a value long enough so that automated
 * tests should time out and fail if IndexedDB shutdown hangs.  Also, this value
 * is long enough so that testers can notice the IndexedDB shutdown hang; we
 * want to know about the hangs, not hide them.  On the other hand this value is
 * less than 60 seconds which is used by nsTerminator to crash a hung main
 * process.
 */
#define SHUTDOWN_TIMEOUT_MS 50000

#ifdef DEBUG

const int32_t kDEBUGThreadPriority = nsISupportsPriority::PRIORITY_NORMAL;
const uint32_t kDEBUGThreadSleepMS = 0;

const int32_t kDEBUGTransactionThreadPriority =
    nsISupportsPriority::PRIORITY_NORMAL;
const uint32_t kDEBUGTransactionThreadSleepMS = 0;

#endif

/*******************************************************************************
 * Metadata classes
 ******************************************************************************/

// Can be instantiated either on the QuotaManager IO thread or on a
// versionchange transaction thread. These threads can never race so this is
// totally safe.
struct FullIndexMetadata {
  IndexMetadata mCommonMetadata = {0,     nsString(), KeyPath(0), nsCString(),
                                   false, false,      false};

  FlippedOnce<false> mDeleted;

  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(FullIndexMetadata)

 private:
  ~FullIndexMetadata() = default;
};

typedef nsRefPtrHashtable<nsUint64HashKey, FullIndexMetadata> IndexTable;

// Can be instantiated either on the QuotaManager IO thread or on a
// versionchange transaction thread. These threads can never race so this is
// totally safe.
struct FullObjectStoreMetadata {
  ObjectStoreMetadata mCommonMetadata = {0, nsString(), KeyPath(0), false};
  IndexTable mIndexes;

  // These two members are only ever touched on a transaction thread!
  int64_t mNextAutoIncrementId = 0;
  int64_t mCommittedAutoIncrementId = 0;

  FlippedOnce<false> mDeleted;

  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(FullObjectStoreMetadata);

  bool HasLiveIndexes() const;

 private:
  ~FullObjectStoreMetadata() = default;
};

typedef nsRefPtrHashtable<nsUint64HashKey, FullObjectStoreMetadata>
    ObjectStoreTable;

struct FullDatabaseMetadata {
  DatabaseMetadata mCommonMetadata;
  nsCString mDatabaseId;
  nsString mFilePath;
  ObjectStoreTable mObjectStores;

  int64_t mNextObjectStoreId;
  int64_t mNextIndexId;

 public:
  explicit FullDatabaseMetadata(const DatabaseMetadata& aCommonMetadata)
      : mCommonMetadata(aCommonMetadata),
        mNextObjectStoreId(0),
        mNextIndexId(0) {
    AssertIsOnBackgroundThread();
  }

  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(FullDatabaseMetadata)

  already_AddRefed<FullDatabaseMetadata> Duplicate() const;

 private:
  ~FullDatabaseMetadata() = default;
};

template <class MetadataType>
class MOZ_STACK_CLASS MetadataNameOrIdMatcher final {
  typedef MetadataNameOrIdMatcher<MetadataType> SelfType;

  const int64_t mId;
  const nsString mName;
  RefPtr<MetadataType> mMetadata;
  bool mCheckName;

 public:
  template <class Enumerable>
  static MetadataType* Match(const Enumerable& aEnumerable, uint64_t aId,
                             const nsAString& aName) {
    AssertIsOnBackgroundThread();
    MOZ_ASSERT(aId);

    SelfType closure(aId, aName);
    MatchHelper(aEnumerable, &closure);

    return closure.mMetadata;
  }

  template <class Enumerable>
  static MetadataType* Match(const Enumerable& aEnumerable, uint64_t aId) {
    AssertIsOnBackgroundThread();
    MOZ_ASSERT(aId);

    SelfType closure(aId);
    MatchHelper(aEnumerable, &closure);

    return closure.mMetadata;
  }

 private:
  MetadataNameOrIdMatcher(const int64_t& aId, const nsAString& aName)
      : mId(aId), mName(aName), mMetadata(nullptr), mCheckName(true) {
    AssertIsOnBackgroundThread();
    MOZ_ASSERT(aId);
  }

  explicit MetadataNameOrIdMatcher(const int64_t& aId)
      : mId(aId), mMetadata(nullptr), mCheckName(false) {
    AssertIsOnBackgroundThread();
    MOZ_ASSERT(aId);
  }

  template <class Enumerable>
  static void MatchHelper(const Enumerable& aEnumerable, SelfType* aClosure) {
    AssertIsOnBackgroundThread();
    MOZ_ASSERT(aClosure);

    for (auto iter = aEnumerable.ConstIter(); !iter.Done(); iter.Next()) {
#ifdef DEBUG
      const uint64_t key = iter.Key();
#endif
      MetadataType* value = iter.UserData();
      MOZ_ASSERT(key != 0);
      MOZ_ASSERT(value);

      if (!value->mDeleted &&
          (aClosure->mId == value->mCommonMetadata.id() ||
           (aClosure->mCheckName &&
            aClosure->mName == value->mCommonMetadata.name()))) {
        aClosure->mMetadata = value;
        break;
      }
    }
  }
};

struct IndexDataValue final {
  int64_t mIndexId;
  Key mPosition;
  Key mLocaleAwarePosition;
  bool mUnique;

  IndexDataValue() : mIndexId(0), mUnique(false) {
    MOZ_COUNT_CTOR(IndexDataValue);
  }

  explicit IndexDataValue(const IndexDataValue& aOther)
      : mIndexId(aOther.mIndexId),
        mPosition(aOther.mPosition),
        mLocaleAwarePosition(aOther.mLocaleAwarePosition),
        mUnique(aOther.mUnique) {
    MOZ_ASSERT(!aOther.mPosition.IsUnset());

    MOZ_COUNT_CTOR(IndexDataValue);
  }

  IndexDataValue(int64_t aIndexId, bool aUnique, const Key& aPosition)
      : mIndexId(aIndexId), mPosition(aPosition), mUnique(aUnique) {
    MOZ_ASSERT(!aPosition.IsUnset());

    MOZ_COUNT_CTOR(IndexDataValue);
  }

  IndexDataValue(int64_t aIndexId, bool aUnique, const Key& aPosition,
                 const Key& aLocaleAwarePosition)
      : mIndexId(aIndexId),
        mPosition(aPosition),
        mLocaleAwarePosition(aLocaleAwarePosition),
        mUnique(aUnique) {
    MOZ_ASSERT(!aPosition.IsUnset());

    MOZ_COUNT_CTOR(IndexDataValue);
  }

  ~IndexDataValue() { MOZ_COUNT_DTOR(IndexDataValue); }

  bool operator==(const IndexDataValue& aOther) const {
    if (mIndexId != aOther.mIndexId) {
      return false;
    }
    if (mLocaleAwarePosition.IsUnset()) {
      return mPosition == aOther.mPosition;
    }
    return mLocaleAwarePosition == aOther.mLocaleAwarePosition;
  }

  bool operator<(const IndexDataValue& aOther) const {
    if (mIndexId == aOther.mIndexId) {
      if (mLocaleAwarePosition.IsUnset()) {
        return mPosition < aOther.mPosition;
      }
      return mLocaleAwarePosition < aOther.mLocaleAwarePosition;
    }

    return mIndexId < aOther.mIndexId;
  }
};

/*******************************************************************************
 * SQLite functions
 ******************************************************************************/

constexpr int32_t MakeSchemaVersion(uint32_t aMajorSchemaVersion,
                                    uint32_t aMinorSchemaVersion) {
  return int32_t((aMajorSchemaVersion << 4) + aMinorSchemaVersion);
}

// WARNING: the hash function used for the database name must not change.
// That's why this function exists separately from mozilla::HashString(), even
// though it is (at the time of writing) equivalent. See bug 780408 and bug
// 940315 for details.
uint32_t HashName(const nsAString& aName) {
  struct Helper {
    static uint32_t RotateBitsLeft32(uint32_t aValue, uint8_t aBits) {
      MOZ_ASSERT(aBits < 32);
      return (aValue << aBits) | (aValue >> (32 - aBits));
    }
  };

  static const uint32_t kGoldenRatioU32 = 0x9e3779b9u;

  return std::accumulate(aName.BeginReading(), aName.EndReading(), uint32_t(0),
                         [](uint32_t hash, char16_t ch) {
                           return kGoldenRatioU32 *
                                  (Helper::RotateBitsLeft32(hash, 5) ^ ch);
                         });
}

nsresult ClampResultCode(nsresult aResultCode) {
  if (NS_SUCCEEDED(aResultCode) ||
      NS_ERROR_GET_MODULE(aResultCode) == NS_ERROR_MODULE_DOM_INDEXEDDB) {
    return aResultCode;
  }

  switch (aResultCode) {
    case NS_ERROR_FILE_NO_DEVICE_SPACE:
      return NS_ERROR_DOM_INDEXEDDB_QUOTA_ERR;
    case NS_ERROR_STORAGE_CONSTRAINT:
      return NS_ERROR_DOM_INDEXEDDB_CONSTRAINT_ERR;
    default:
#ifdef DEBUG
      nsPrintfCString message("Converting non-IndexedDB error code (0x%" PRIX32
                              ") to "
                              "NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR",
                              static_cast<uint32_t>(aResultCode));
      NS_WARNING(message.get());
#else
        ;
#endif
  }

  IDB_REPORT_INTERNAL_ERR();
  return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
}

void GetDatabaseFilename(const nsAString& aName,
                         nsAutoString& aDatabaseFilename) {
  MOZ_ASSERT(aDatabaseFilename.IsEmpty());

  // WARNING: do not change this hash function. See the comment in HashName()
  // for details.
  aDatabaseFilename.AppendInt(HashName(aName));

  nsCString escapedName;
  if (!NS_Escape(NS_ConvertUTF16toUTF8(aName), escapedName, url_XPAlphas)) {
    MOZ_CRASH("Can't escape database name!");
  }

  const char* forwardIter = escapedName.BeginReading();
  const char* backwardIter = escapedName.EndReading() - 1;

  nsAutoCString substring;
  while (forwardIter <= backwardIter && substring.Length() < 21) {
    if (substring.Length() % 2) {
      substring.Append(*backwardIter--);
    } else {
      substring.Append(*forwardIter++);
    }
  }

  aDatabaseFilename.AppendASCII(substring.get(), substring.Length());
}

uint32_t CompressedByteCountForNumber(uint64_t aNumber) {
  // All bytes have 7 bits available.
  uint32_t count = 1;
  while ((aNumber >>= 7)) {
    count++;
  }

  return count;
}

uint32_t CompressedByteCountForIndexId(int64_t aIndexId) {
  MOZ_ASSERT(aIndexId);
  MOZ_ASSERT(UINT64_MAX - uint64_t(aIndexId) >= uint64_t(aIndexId),
             "Overflow!");

  return CompressedByteCountForNumber(uint64_t(aIndexId * 2));
}

void WriteCompressedNumber(uint64_t aNumber, uint8_t** aIterator) {
  MOZ_ASSERT(aIterator);
  MOZ_ASSERT(*aIterator);

  uint8_t*& buffer = *aIterator;

#ifdef DEBUG
  const uint8_t* const bufferStart = buffer;
  const uint64_t originalNumber = aNumber;
#endif

  while (true) {
    uint64_t shiftedNumber = aNumber >> 7;
    if (shiftedNumber) {
      *buffer++ = uint8_t(0x80 | (aNumber & 0x7f));
      aNumber = shiftedNumber;
    } else {
      *buffer++ = uint8_t(aNumber);
      break;
    }
  }

  MOZ_ASSERT(buffer > bufferStart);
  MOZ_ASSERT(uint32_t(buffer - bufferStart) ==
             CompressedByteCountForNumber(originalNumber));
}

uint64_t ReadCompressedNumber(const uint8_t** aIterator, const uint8_t* aEnd) {
  MOZ_ASSERT(aIterator);
  MOZ_ASSERT(*aIterator);
  MOZ_ASSERT(aEnd);
  MOZ_ASSERT(*aIterator < aEnd);

  const uint8_t*& buffer = *aIterator;

  uint8_t shiftCounter = 0;
  uint64_t result = 0;

  while (true) {
    MOZ_ASSERT(shiftCounter <= 56, "Shifted too many bits!");

    result += (uint64_t(*buffer & 0x7f) << shiftCounter);
    shiftCounter += 7;

    if (!(*buffer++ & 0x80)) {
      break;
    }

    if (NS_WARN_IF(buffer == aEnd)) {
      MOZ_ASSERT(false);
      break;
    }
  }

  return result;
}

void WriteCompressedIndexId(int64_t aIndexId, bool aUnique,
                            uint8_t** aIterator) {
  MOZ_ASSERT(aIndexId);
  MOZ_ASSERT(UINT64_MAX - uint64_t(aIndexId) >= uint64_t(aIndexId),
             "Overflow!");
  MOZ_ASSERT(aIterator);
  MOZ_ASSERT(*aIterator);

  const uint64_t indexId = (uint64_t(aIndexId * 2) | (aUnique ? 1 : 0));
  WriteCompressedNumber(indexId, aIterator);
}

void ReadCompressedIndexId(const uint8_t** aIterator, const uint8_t* aEnd,
                           int64_t* aIndexId, bool* aUnique) {
  MOZ_ASSERT(aIterator);
  MOZ_ASSERT(*aIterator);
  MOZ_ASSERT(aIndexId);
  MOZ_ASSERT(aUnique);

  uint64_t indexId = ReadCompressedNumber(aIterator, aEnd);

  if (indexId % 2) {
    *aUnique = true;
    indexId--;
  } else {
    *aUnique = false;
  }

  MOZ_ASSERT(UINT64_MAX / 2 >= uint64_t(indexId), "Bad index id!");

  *aIndexId = int64_t(indexId / 2);
}

// static
nsresult MakeCompressedIndexDataValues(
    const FallibleTArray<IndexDataValue>& aIndexValues,
    UniqueFreePtr<uint8_t>& aCompressedIndexDataValues,
    uint32_t* aCompressedIndexDataValuesLength) {
  MOZ_ASSERT(!NS_IsMainThread());
  MOZ_ASSERT(!IsOnBackgroundThread());
  MOZ_ASSERT(!aCompressedIndexDataValues);
  MOZ_ASSERT(aCompressedIndexDataValuesLength);

  AUTO_PROFILER_LABEL("MakeCompressedIndexDataValues", DOM);

  const uint32_t arrayLength = aIndexValues.Length();
  if (!arrayLength) {
    *aCompressedIndexDataValuesLength = 0;
    return NS_OK;
  }

  // First calculate the size of the final buffer.
  uint32_t blobDataLength = 0;

  for (const IndexDataValue& info : aIndexValues) {
    const nsCString& keyBuffer = info.mPosition.GetBuffer();
    const nsCString& sortKeyBuffer = info.mLocaleAwarePosition.GetBuffer();
    const uint32_t keyBufferLength = keyBuffer.Length();
    const uint32_t sortKeyBufferLength = sortKeyBuffer.Length();

    MOZ_ASSERT(!keyBuffer.IsEmpty());

    const CheckedUint32 infoLength =
        CheckedUint32(CompressedByteCountForIndexId(info.mIndexId)) +
        CompressedByteCountForNumber(keyBufferLength) +
        CompressedByteCountForNumber(sortKeyBufferLength) + keyBufferLength +
        sortKeyBufferLength;
    // Don't let |infoLength| overflow.
    if (NS_WARN_IF(!infoLength.isValid())) {
      IDB_REPORT_INTERNAL_ERR();
      return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
    }

    // Don't let |blobDataLength| overflow.
    if (NS_WARN_IF(UINT32_MAX - infoLength.value() < blobDataLength)) {
      IDB_REPORT_INTERNAL_ERR();
      return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
    }

    blobDataLength += infoLength.value();
  }

  UniqueFreePtr<uint8_t> blobData(
      static_cast<uint8_t*>(malloc(blobDataLength)));
  if (NS_WARN_IF(!blobData)) {
    IDB_REPORT_INTERNAL_ERR();
    return NS_ERROR_OUT_OF_MEMORY;
  }

  uint8_t* blobDataIter = blobData.get();

  for (const IndexDataValue& info : aIndexValues) {
    const nsCString& keyBuffer = info.mPosition.GetBuffer();
    const nsCString& sortKeyBuffer = info.mLocaleAwarePosition.GetBuffer();
    const uint32_t keyBufferLength = keyBuffer.Length();
    const uint32_t sortKeyBufferLength = sortKeyBuffer.Length();

    WriteCompressedIndexId(info.mIndexId, info.mUnique, &blobDataIter);
    WriteCompressedNumber(keyBufferLength, &blobDataIter);

    memcpy(blobDataIter, keyBuffer.get(), keyBufferLength);
    blobDataIter += keyBufferLength;

    WriteCompressedNumber(sortKeyBufferLength, &blobDataIter);

    memcpy(blobDataIter, sortKeyBuffer.get(), sortKeyBufferLength);
    blobDataIter += sortKeyBufferLength;
  }

  MOZ_ASSERT(blobDataIter == blobData.get() + blobDataLength);

  aCompressedIndexDataValues.swap(blobData);
  *aCompressedIndexDataValuesLength = uint32_t(blobDataLength);

  return NS_OK;
}

nsresult ReadCompressedIndexDataValuesFromBlob(
    const uint8_t* aBlobData, uint32_t aBlobDataLength,
    nsTArray<IndexDataValue>& aIndexValues) {
  MOZ_ASSERT(!NS_IsMainThread());
  MOZ_ASSERT(!IsOnBackgroundThread());
  MOZ_ASSERT(aBlobData);
  MOZ_ASSERT(aBlobDataLength);
  MOZ_ASSERT(aIndexValues.IsEmpty());

  AUTO_PROFILER_LABEL("ReadCompressedIndexDataValuesFromBlob", DOM);

  if (uintptr_t(aBlobData) > UINTPTR_MAX - aBlobDataLength) {
    IDB_REPORT_INTERNAL_ERR();
    return NS_ERROR_FILE_CORRUPTED;
  }

  const uint8_t* blobDataIter = aBlobData;
  const uint8_t* const blobDataEnd = aBlobData + aBlobDataLength;

  while (blobDataIter < blobDataEnd) {
    int64_t indexId;
    bool unique;
    ReadCompressedIndexId(&blobDataIter, blobDataEnd, &indexId, &unique);

    if (NS_WARN_IF(blobDataIter == blobDataEnd)) {
      IDB_REPORT_INTERNAL_ERR();
      return NS_ERROR_FILE_CORRUPTED;
    }

    // Read key buffer length.
    const uint64_t keyBufferLength =
        ReadCompressedNumber(&blobDataIter, blobDataEnd);

    if (NS_WARN_IF(blobDataIter == blobDataEnd) ||
        NS_WARN_IF(keyBufferLength > uint64_t(UINT32_MAX)) ||
        NS_WARN_IF(keyBufferLength > uintptr_t(blobDataEnd)) ||
        NS_WARN_IF(blobDataIter > blobDataEnd - keyBufferLength)) {
      IDB_REPORT_INTERNAL_ERR();
      return NS_ERROR_FILE_CORRUPTED;
    }

    nsCString keyBuffer(reinterpret_cast<const char*>(blobDataIter),
                        uint32_t(keyBufferLength));
    blobDataIter += keyBufferLength;

    IndexDataValue idv(indexId, unique, Key(keyBuffer));

    // Read sort key buffer length.
    const uint64_t sortKeyBufferLength =
        ReadCompressedNumber(&blobDataIter, blobDataEnd);

    if (sortKeyBufferLength > 0) {
      if (NS_WARN_IF(blobDataIter == blobDataEnd) ||
          NS_WARN_IF(sortKeyBufferLength > uint64_t(UINT32_MAX)) ||
          NS_WARN_IF(sortKeyBufferLength > uintptr_t(blobDataEnd)) ||
          NS_WARN_IF(blobDataIter > blobDataEnd - sortKeyBufferLength)) {
        IDB_REPORT_INTERNAL_ERR();
        return NS_ERROR_FILE_CORRUPTED;
      }

      nsCString sortKeyBuffer(reinterpret_cast<const char*>(blobDataIter),
                              uint32_t(sortKeyBufferLength));
      blobDataIter += sortKeyBufferLength;

      idv.mLocaleAwarePosition = Key(sortKeyBuffer);
    }

    if (NS_WARN_IF(!aIndexValues.InsertElementSorted(idv, fallible))) {
      IDB_REPORT_INTERNAL_ERR();
      return NS_ERROR_OUT_OF_MEMORY;
    }
  }

  MOZ_ASSERT(blobDataIter == blobDataEnd);

  return NS_OK;
}

// static
template <typename T>
nsresult ReadCompressedIndexDataValuesFromSource(
    T* aSource, uint32_t aColumnIndex, nsTArray<IndexDataValue>& aIndexValues) {
  MOZ_ASSERT(!NS_IsMainThread());
  MOZ_ASSERT(!IsOnBackgroundThread());
  MOZ_ASSERT(aSource);
  MOZ_ASSERT(aIndexValues.IsEmpty());

  int32_t columnType;
  nsresult rv = aSource->GetTypeOfIndex(aColumnIndex, &columnType);
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

  if (columnType == mozIStorageStatement::VALUE_TYPE_NULL) {
    return NS_OK;
  }

  MOZ_ASSERT(columnType == mozIStorageStatement::VALUE_TYPE_BLOB);

  const uint8_t* blobData;
  uint32_t blobDataLength;
  rv = aSource->GetSharedBlob(aColumnIndex, &blobDataLength, &blobData);
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

  if (NS_WARN_IF(!blobDataLength)) {
    IDB_REPORT_INTERNAL_ERR();
    return NS_ERROR_FILE_CORRUPTED;
  }

  rv = ReadCompressedIndexDataValuesFromBlob(blobData, blobDataLength,
                                             aIndexValues);
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

  return NS_OK;
}

nsresult ReadCompressedIndexDataValues(mozIStorageStatement* aStatement,
                                       uint32_t aColumnIndex,
                                       nsTArray<IndexDataValue>& aIndexValues) {
  return ReadCompressedIndexDataValuesFromSource(aStatement, aColumnIndex,
                                                 aIndexValues);
}

nsresult ReadCompressedIndexDataValues(mozIStorageValueArray* aValues,
                                       uint32_t aColumnIndex,
                                       nsTArray<IndexDataValue>& aIndexValues) {
  return ReadCompressedIndexDataValuesFromSource(aValues, aColumnIndex,
                                                 aIndexValues);
}

nsresult CreateFileTables(mozIStorageConnection* aConnection) {
  AssertIsOnIOThread();
  MOZ_ASSERT(aConnection);

  AUTO_PROFILER_LABEL("CreateFileTables", DOM);

  // Table `file`
  nsresult rv = aConnection->ExecuteSimpleSQL(
      NS_LITERAL_CSTRING("CREATE TABLE file ("
                         "id INTEGER PRIMARY KEY, "
                         "refcount INTEGER NOT NULL"
                         ");"));
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

  rv = aConnection->ExecuteSimpleSQL(
      NS_LITERAL_CSTRING("CREATE TRIGGER object_data_insert_trigger "
                         "AFTER INSERT ON object_data "
                         "FOR EACH ROW "
                         "WHEN NEW.file_ids IS NOT NULL "
                         "BEGIN "
                         "SELECT update_refcount(NULL, NEW.file_ids); "
                         "END;"));
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

  rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
      "CREATE TRIGGER object_data_update_trigger "
      "AFTER UPDATE OF file_ids ON object_data "
      "FOR EACH ROW "
      "WHEN OLD.file_ids IS NOT NULL OR NEW.file_ids IS NOT NULL "
      "BEGIN "
      "SELECT update_refcount(OLD.file_ids, NEW.file_ids); "
      "END;"));
  if (NS_WARN_IF(NS_FAILED(rv))) {
    return rv;
  }

  rv = aConnection->ExecuteSimpleSQL(