gh-15085: Import mochitests for remote settings (gh-15086)

.gitattributes

@@ -5,4 +5,4 @@
 *.patch linguist-language=C++
 *.d.ts linguist-language=TypeScript
 
-src/zen/tests/*.js linguist-language=Test
+src/zen/tests/* linguist-language=C++

src/testing/xpcshell/runxpcshelltests-py.patch

@@ -0,0 +1,14 @@
+diff --git a/testing/xpcshell/runxpcshelltests.py b/testing/xpcshell/runxpcshelltests.py
+index 403401613189ffb758037850dc6bbb1d55f194ab..cad28e245155eb54775e551885532026e56c5fa9 100755
+--- a/testing/xpcshell/runxpcshelltests.py
++++ b/testing/xpcshell/runxpcshelltests.py
+@@ -2275,6 +2275,9 @@ class XPCShellTests:
+         appDirKey = None
+         if "appname" in self.mozInfo:
+             appDirKey = self.mozInfo["appname"] + "-appdir"
++            if appDirKey == "zen-appdir":
++                # Inherited manifests use firefox-appdir.
++                appDirKey = "firefox-appdir"
+ 
+         # We have to do this before we run tests that depend on having the node
+         # http/2 server.

src/zen/space-routing/ZenSpaceRoutingManager.sys.mjs

@@ -27,7 +27,7 @@ class nsZenSpaceRoutingManager {
     const element = window.MozXULElement.parseXULToFragment(`
         <menuseparator/>
         <menuitem id="context_zen-add-domain-to-routing"
-                  data-lazy-l10n-id="tab-context-zen-add-domain-to-sr"
+                  data-l10n-id="tab-context-zen-add-domain-to-sr"
                   data-l10n-args='{"tabCount": 1}'/>
       `);
     window.document.getElementById("context_undoCloseTab").after(element);

src/zen/tests/manifest.toml

@@ -13,10 +13,26 @@ disable = [
   "browser_readerMode_colorSchemePref.js",
 ]
 
+[remote-settings]
+source = "services/settings/test/unit"
+is_direct_path = true
+xpcshell = true
+
+[remote-settings.replace-manifest]
+"../../../" = "../../../../services/"
+
 [safebrowsing]
 source = "browser/components/safebrowsing/content/test"
 is_direct_path = true
 
+[services-crypto]
+source = "services/crypto/tests/unit"
+is_direct_path = true
+xpcshell = true
+
+[services-crypto.replace-manifest]
+"../../../" = "../../../../services/"
+
 [shell]
 source = "browser/components/shell/test"
 is_direct_path = true
@@ -25,6 +41,15 @@ disable = [
   "browser_setDesktopBackgroundPreview.js",
 ]
 
+[sync]
+source = "services/sync/tests/unit"
+is_direct_path = true
+xpcshell = true
+
+[sync.replace-manifest]
+"../../../" = "../../../../services/"
+'"identity.fxaccounts.enabled=true"' = '"identity.fxaccounts.enabled=true", "browser.formfill.enable=true"'
+
 [tabMediaIndicator]
 source = "browser/components/tabbrowser/test/browser/tabMediaIndicator"
 is_direct_path = true

src/zen/tests/mochitests/moz.build

@@ -15,4 +15,7 @@ BROWSER_CHROME_MANIFESTS += [
 	"tooltiptext/browser.toml",
 ]
 XPCSHELL_TESTS_MANIFESTS += [
+	"remote-settings/xpcshell.toml",
+	"services-crypto/xpcshell.toml",
+	"sync/xpcshell.toml",
 ]

src/zen/tests/mochitests/remote-settings/head_settings.js

@@ -0,0 +1,86 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/* import-globals-from ../../../common/tests/unit/head_global.js */
+/* import-globals-from ../../../common/tests/unit/head_helpers.js */
+
+"use strict";
+
+ChromeUtils.defineESModuleGetters(this, {
+  AppConstants: "resource://gre/modules/AppConstants.sys.mjs",
+  Database: "resource://services-settings/Database.sys.mjs",
+  RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
+  RemoteSettingsClient:
+    "resource://services-settings/RemoteSettingsClient.sys.mjs",
+  RemoteSettingsWorker:
+    "resource://services-settings/RemoteSettingsWorker.sys.mjs",
+  setTimeout: "resource://gre/modules/Timer.sys.mjs",
+  SharedUtils: "resource://services-settings/SharedUtils.sys.mjs",
+  SyncHistory: "resource://services-settings/SyncHistory.sys.mjs",
+  TelemetryTestUtils: "resource://testing-common/TelemetryTestUtils.sys.mjs",
+  TestUtils: "resource://testing-common/TestUtils.sys.mjs",
+  UptakeTelemetry: "resource://services-settings/UptakeTelemetry.sys.mjs",
+  Utils: "resource://services-settings/Utils.sys.mjs",
+});
+
+const IS_ANDROID = AppConstants.platform == "android";
+
+function arrayEqual(a, b) {
+  return JSON.stringify(a) == JSON.stringify(b);
+}
+
+add_setup(function () {
+  Services.fog.initializeFOG();
+});
+
+function enableUptakeMetric() {
+  Services.fog.applyServerKnobsConfig(
+    JSON.stringify({
+      metrics_enabled: {
+        "uptake.remotecontent.result.uptake_remotesettings": true,
+      },
+    })
+  );
+}
+
+function assertTelemetryEvents(expectedEvents) {
+  if (IS_ANDROID) {
+    /*
+      Since `Services.fog.applyServerKnobsConfig()` has no effect on Android
+      https://searchfox.org/firefox-main/rev/e28b34ab33dbf49364999070168cbb7e11e8e5bd/toolkit/components/glean/xpcom/FOG.cpp#379-390
+      we skip assertions on events on this platform.
+      Alternatively we could have disabled the whole tests suites or introduced `if IS_ANDROID` on each
+      test involving telemetry, but this seems the most reasonable way.
+      See Bug 2028765 and Bug 2042499
+    */
+    Assert.ok(
+      true,
+      "Glean Uptake Telemetry assertions are ALWAYS true on Android"
+    );
+    return;
+  }
+  const events =
+    Glean.uptakeRemotecontentResult.uptakeRemotesettings.testGetValue() ?? [];
+  const receivedValues = events.map(e => e.extra.value);
+  Assert.equal(
+    events.length,
+    expectedEvents.length,
+    `number of uptake events (${receivedValues})`
+  );
+  for (let i = 0; i < expectedEvents.length; i++) {
+    for (const [key, expected] of Object.entries(expectedEvents[i])) {
+      if (typeof expected === "function") {
+        Assert.ok(
+          expected(events[i].extra[key]),
+          `event[${i}].extra.${key} passes validator`
+        );
+      } else {
+        Assert.equal(
+          events[i].extra[key],
+          expected,
+          `event[${i}].extra.${key}`
+        );
+      }
+    }
+  }
+}

src/zen/tests/mochitests/remote-settings/test_attachments_downloader.js

@@ -0,0 +1,896 @@
+const { Downloader } = ChromeUtils.importESModule(
+  "resource://services-settings/Attachments.sys.mjs"
+);
+
+const RECORD = {
+  id: "1f3a0802-648d-11ea-bd79-876a8b69c377",
+  attachment: {
+    hash: "f41ed47d0f43325c9f089d03415c972ce1d3f1ecab6e4d6260665baf3db3ccee",
+    size: 1597,
+    filename: "test_file.pem",
+    location:
+      "main-workspace/some-collection/65650a0f-7c22-4c10-9744-2d67e301f5f4.pem",
+    mimetype: "application/x-pem-file",
+  },
+};
+
+const RECORD_OF_DUMP = {
+  id: "filename-of-dump.txt",
+  attachment: {
+    filename: "filename-of-dump.txt",
+    hash: "4c46ef7e4f1951d210fe54c21e07c09bab265fd122580083ed1d6121547a8c6b",
+    size: 25,
+  },
+  last_modified: 1234567,
+  some_key: "some metadata",
+};
+
+let downloader;
+let server;
+
+add_setup(() => {
+  server = new HttpServer();
+  server.start(-1);
+  registerCleanupFunction(() => server.stop(() => {}));
+
+  server.registerDirectory(
+    "/cdn/main-workspace/some-collection/",
+    do_get_file("test_attachments_downloader")
+  );
+  server.registerDirectory(
+    "/cdn/bundles/",
+    do_get_file("test_attachments_downloader")
+  );
+
+  // For this test, we are using a server other than production. Force
+  // LOAD_DUMPS to true so that we can still load attachments from dumps.
+  delete Utils.LOAD_DUMPS;
+  Utils.LOAD_DUMPS = true;
+});
+
+async function clear_state() {
+  Services.fog.testResetFOG();
+  enableUptakeMetric();
+
+  Services.prefs.setStringPref(
+    "services.settings.server",
+    `http://localhost:${server.identity.primaryPort}/v1`
+  );
+
+  downloader = new Downloader("main", "some-collection");
+  downloader.cache = {};
+  const memCacheImpl = {
+    get: async id => {
+      return downloader.cache[id];
+    },
+    set: async (id, obj) => {
+      downloader.cache[id] = obj;
+    },
+    setMultiple: async idsObjs => {
+      idsObjs.forEach(([id, obj]) => (downloader.cache[id] = obj));
+    },
+    delete: async id => {
+      delete downloader.cache[id];
+    },
+    deleteMultiple: async ids => {
+      ids.forEach(id => delete downloader.cache[id]);
+    },
+    hasData: async () => {
+      return !!Object.keys(downloader.cache).length;
+    },
+  };
+  // The download() method requires a cacheImpl, but the Downloader
+  // class does not have one. Define a dummy no-op one.
+  Object.defineProperty(downloader, "cacheImpl", {
+    value: memCacheImpl,
+    // Writable to allow specific tests to override cacheImpl.
+    writable: true,
+  });
+  await downloader.deleteDownloaded(RECORD);
+
+  server.registerPathHandler("/v1/", (request, response) => {
+    response.write(
+      JSON.stringify({
+        capabilities: {
+          attachments: {
+            base_url: `http://localhost:${server.identity.primaryPort}/cdn/`,
+          },
+        },
+      })
+    );
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setStatusLine(null, 200, "OK");
+  });
+
+  // For tests that use a real client and DB cache, clear the local DB too.
+  const client = RemoteSettings("some-collection");
+  await client.db.clear();
+  await client.db.pruneAttachments([]);
+}
+add_task(clear_state);
+
+add_task(
+  async function test_download_throws_server_info_error_if_invalid_response() {
+    server.registerPathHandler("/v1/", (request, response) => {
+      response.write("{bad json content");
+      response.setHeader("Content-Type", "application/json; charset=UTF-8");
+      response.setStatusLine(null, 200, "OK");
+    });
+
+    let error;
+    try {
+      await downloader.download(RECORD);
+    } catch (e) {
+      error = e;
+    }
+
+    Assert.ok(error instanceof Downloader.ServerInfoError);
+  }
+);
+add_task(clear_state);
+
+add_task(async function test_download_is_retried_3_times_if_download_fails() {
+  const record = {
+    id: "abc",
+    attachment: {
+      ...RECORD.attachment,
+      location: "404-error.pem",
+    },
+  };
+
+  let called = 0;
+  const _fetchAttachment = downloader._fetchAttachment;
+  downloader._fetchAttachment = async url => {
+    called++;
+    return _fetchAttachment(url);
+  };
+
+  let error;
+  try {
+    await downloader.download(record);
+  } catch (e) {
+    error = e;
+  }
+
+  Assert.equal(called, 4); // 1 + 3 retries
+  Assert.ok(error instanceof Downloader.DownloadError);
+});
+add_task(clear_state);
+
+add_task(async function test_download_as_bytes() {
+  const bytes = await downloader.downloadAsBytes(RECORD);
+
+  // See *.pem file in tests data.
+  Assert.greater(
+    bytes.byteLength,
+    1500,
+    `Wrong bytes size: ${bytes.byteLength}`
+  );
+});
+add_task(clear_state);
+
+add_task(async function test_download_is_retried_3_times_if_content_fails() {
+  const record = {
+    id: "abc",
+    attachment: {
+      ...RECORD.attachment,
+      hash: "always-wrong",
+    },
+  };
+  let called = 0;
+  downloader._fetchAttachment = async () => {
+    called++;
+    return new ArrayBuffer();
+  };
+
+  let error;
+  try {
+    await downloader.download(record);
+  } catch (e) {
+    error = e;
+  }
+
+  Assert.equal(called, 4); // 1 + 3 retries
+  Assert.ok(error instanceof Downloader.BadContentError);
+});
+add_task(clear_state);
+
+add_task(async function test_delete_all() {
+  const client = RemoteSettings("some-collection");
+  await client.db.create(RECORD);
+  await downloader.download(RECORD);
+
+  await client.attachments.deleteAll();
+
+  Assert.ok(!(await client.attachments.cacheImpl.get(RECORD.id)));
+});
+add_task(clear_state);
+
+add_task(async function test_downloader_reports_download_errors() {
+  const client = RemoteSettings("some-collection");
+
+  const record = {
+    attachment: {
+      ...RECORD.attachment,
+      location: "404-error.pem",
+    },
+  };
+
+  try {
+    await client.attachments.download(record, { retry: 0 });
+  } catch (e) {}
+
+  assertTelemetryEvents([
+    {
+      value: UptakeTelemetry.STATUS.DOWNLOAD_START,
+      source: client.identifier,
+    },
+    {
+      value: UptakeTelemetry.STATUS.DOWNLOAD_ERROR,
+      source: client.identifier,
+    },
+  ]);
+});
+add_task(clear_state);
+
+add_task(async function test_downloader_reports_offline_error() {
+  const backupOffline = Services.io.offline;
+  Services.io.offline = true;
+
+  try {
+    const client = RemoteSettings("some-collection");
+    const record = {
+      attachment: {
+        ...RECORD.attachment,
+        location: "will-try-and-fail.pem",
+      },
+    };
+    try {
+      await client.attachments.download(record, { retry: 0 });
+    } catch (e) {}
+
+    assertTelemetryEvents([
+      {
+        value: UptakeTelemetry.STATUS.DOWNLOAD_START,
+        source: client.identifier,
+      },
+      {
+        value: UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR,
+        source: client.identifier,
+      },
+    ]);
+  } finally {
+    Services.io.offline = backupOffline;
+  }
+});
+add_task(clear_state);
+
+// Common code for test_download_cache_hit and test_download_cache_corruption.
+async function doTestDownloadCacheImpl({
+  simulateCorruption,
+  expectedReads = 1,
+  expectedWrites = 1,
+  downloadOptions = {},
+}) {
+  let readCount = 0;
+  let writeCount = 0;
+  const cacheImpl = {
+    async get(attachmentId) {
+      Assert.equal(attachmentId, RECORD.id, "expected attachmentId");
+      ++readCount;
+      if (simulateCorruption) {
+        throw new Error("Simulation of corrupted cache (read)");
+      }
+    },
+    async set(attachmentId, attachment) {
+      Assert.equal(attachmentId, RECORD.id, "expected attachmentId");
+      Assert.deepEqual(attachment.record, RECORD, "expected record");
+      ++writeCount;
+      if (simulateCorruption) {
+        throw new Error("Simulation of corrupted cache (write)");
+      }
+    },
+    async delete() {},
+  };
+  Object.defineProperty(downloader, "cacheImpl", { value: cacheImpl });
+
+  let downloadResult = await downloader.download(RECORD, downloadOptions);
+  Assert.equal(downloadResult._source, "remote_match", "expected source");
+  Assert.equal(downloadResult.buffer.byteLength, 1597, "expected result");
+  Assert.equal(readCount, expectedReads, "expected cache read attempts");
+  Assert.equal(writeCount, expectedWrites, "expected cache write attempts");
+}
+
+add_task(async function test_download_cache_hit() {
+  await doTestDownloadCacheImpl({ simulateCorruption: false });
+});
+add_task(clear_state);
+
+// Verify that the downloader works despite a broken cache implementation.
+add_task(async function test_download_cache_corruption() {
+  await doTestDownloadCacheImpl({ simulateCorruption: true });
+});
+add_task(clear_state);
+
+add_task(async function test_download_with_cache_enabled() {
+  await doTestDownloadCacheImpl({
+    simulateCorruption: false,
+    downloadOptions: {
+      cacheResult: true,
+    },
+  });
+});
+add_task(clear_state);
+
+add_task(async function test_download_with_cache_disabled() {
+  await doTestDownloadCacheImpl({
+    simulateCorruption: false,
+    expectedWrites: 0,
+    downloadOptions: {
+      cacheResult: false,
+    },
+  });
+});
+add_task(clear_state);
+
+add_task(async function test_download_cached() {
+  const client = RemoteSettings("main", "some-collection");
+  const attachmentId = "dummy filename";
+  const badRecord = {
+    attachment: {
+      ...RECORD.attachment,
+      hash: "non-matching hash",
+      location: "non-existing-location-should-fail.bin",
+    },
+  };
+  async function downloadWithCache(record, options) {
+    options = { ...options, useCache: true };
+    return client.attachments.download(record, options);
+  }
+  function checkInfo(downloadResult, expectedSource, msg) {
+    Assert.deepEqual(
+      downloadResult.record,
+      RECORD,
+      `${msg} : expected identical record`
+    );
+    // Simple check: assume that content is identical if the size matches.
+    Assert.equal(
+      downloadResult.buffer.byteLength,
+      RECORD.attachment.size,
+      `${msg} : expected buffer`
+    );
+    Assert.equal(
+      downloadResult._source,
+      expectedSource,
+      `${msg} : expected source of the result`
+    );
+  }
+
+  await Assert.rejects(
+    downloadWithCache(null, { attachmentId }),
+    /DownloadError: Could not download dummy filename/,
+    "Download without record or cache should fail."
+  );
+
+  // Populate cache.
+  const info1 = await downloadWithCache(RECORD, { attachmentId });
+  checkInfo(info1, "remote_match", "first time download");
+
+  await Assert.rejects(
+    downloadWithCache(null, { attachmentId }),
+    /DownloadError: Could not download dummy filename/,
+    "Download without record still fails even if there is a cache."
+  );
+
+  await Assert.rejects(
+    downloadWithCache(badRecord, { attachmentId }),
+    /DownloadError: Could not download .*non-existing-location-should-fail.bin/,
+    "Download with non-matching record still fails even if there is a cache."
+  );
+
+  // Download from cache.
+  const info2 = await downloadWithCache(RECORD, { attachmentId });
+  checkInfo(info2, "cache_match", "download matching record from cache");
+
+  const info3 = await downloadWithCache(RECORD, {
+    attachmentId,
+    fallbackToCache: true,
+  });
+  checkInfo(info3, "cache_match", "fallbackToCache accepts matching record");
+
+  const info4 = await downloadWithCache(null, {
+    attachmentId,
+    fallbackToCache: true,
+  });
+  checkInfo(info4, "cache_fallback", "fallbackToCache accepts null record");
+
+  const info5 = await downloadWithCache(badRecord, {
+    attachmentId,
+    fallbackToCache: true,
+  });
+  checkInfo(info5, "cache_fallback", "fallbackToCache ignores bad record");
+
+  // Bye bye cache.
+  await client.attachments.deleteDownloaded({ id: attachmentId });
+  await Assert.rejects(
+    downloadWithCache(null, { attachmentId, fallbackToCache: true }),
+    /DownloadError: Could not download dummy filename/,
+    "Download without cache should fail again."
+  );
+  await Assert.rejects(
+    downloadWithCache(badRecord, { attachmentId, fallbackToCache: true }),
+    /DownloadError: Could not download .*non-existing-location-should-fail.bin/,
+    "Download should fail to fall back to a download of a non-existing record"
+  );
+});
+add_task(clear_state);
+
+add_task(async function test_download_from_dump() {
+  const client = RemoteSettings("dump-collection", {
+    bucketName: "dump-bucket",
+  });
+
+  // Temporarily replace the resource:-URL with another resource:-URL.
+  const orig_RESOURCE_BASE_URL = Downloader._RESOURCE_BASE_URL;
+  Downloader._RESOURCE_BASE_URL = "resource://rs-downloader-test";
+  const resProto = Services.io
+    .getProtocolHandler("resource")
+    .QueryInterface(Ci.nsIResProtocolHandler);
+  resProto.setSubstitution(
+    "rs-downloader-test",
+    Services.io.newFileURI(do_get_file("test_attachments_downloader"))
+  );
+
+  function checkInfo(result, expectedSource, expectedRecord = RECORD_OF_DUMP) {
+    Assert.equal(
+      new TextDecoder().decode(new Uint8Array(result.buffer)),
+      "This would be a RS dump.\n",
+      "expected content from dump"
+    );
+    Assert.deepEqual(result.record, expectedRecord, "expected record for dump");
+    Assert.equal(result._source, expectedSource, "expected source of dump");
+  }
+
+  // If record matches, should happen before network request.
+  const dump1 = await client.attachments.download(RECORD_OF_DUMP, {
+    // Note: attachmentId not set, so should fall back to record.id.
+    fallbackToDump: true,
+  });
+  checkInfo(dump1, "dump_match");
+
+  // If no record given, should try network first, but then fall back to dump.
+  const dump2 = await client.attachments.download(null, {
+    attachmentId: RECORD_OF_DUMP.id,
+    fallbackToDump: true,
+  });
+  checkInfo(dump2, "dump_fallback");
+
+  // Fill the cache with the same data as the dump for the next part.
+  await client.db.saveAttachment(RECORD_OF_DUMP.id, {
+    record: RECORD_OF_DUMP,
+    blob: new Blob([dump1.buffer]),
+  });
+  // The dump should take precedence over the cache.
+  const dump3 = await client.attachments.download(RECORD_OF_DUMP, {
+    fallbackToCache: true,
+    fallbackToDump: true,
+  });
+  checkInfo(dump3, "dump_match");
+
+  // When the record is not given, the dump takes precedence over the cache
+  // as a fallback (when the cache and dump are identical).
+  const dump4 = await client.attachments.download(null, {
+    attachmentId: RECORD_OF_DUMP.id,
+    fallbackToCache: true,
+    fallbackToDump: true,
+  });
+  checkInfo(dump4, "dump_fallback");
+
+  // Store a record in the cache that is newer than the dump.
+  const RECORD_NEWER_THAN_DUMP = {
+    ...RECORD_OF_DUMP,
+    last_modified: RECORD_OF_DUMP.last_modified + 1,
+  };
+  await client.db.saveAttachment(RECORD_OF_DUMP.id, {
+    record: RECORD_NEWER_THAN_DUMP,
+    blob: new Blob([dump1.buffer]),
+  });
+
+  // When the record is not given, use the cache if it has a more recent record.
+  const dump5 = await client.attachments.download(null, {
+    attachmentId: RECORD_OF_DUMP.id,
+    fallbackToCache: true,
+    fallbackToDump: true,
+  });
+  checkInfo(dump5, "cache_fallback", RECORD_NEWER_THAN_DUMP);
+
+  // When a record is given, use whichever that has the matching last_modified.
+  const dump6 = await client.attachments.download(RECORD_OF_DUMP, {
+    fallbackToCache: true,
+    fallbackToDump: true,
+  });
+  checkInfo(dump6, "dump_match");
+  const dump7 = await client.attachments.download(RECORD_NEWER_THAN_DUMP, {
+    fallbackToCache: true,
+    fallbackToDump: true,
+  });
+  checkInfo(dump7, "cache_match", RECORD_NEWER_THAN_DUMP);
+
+  await client.attachments.deleteDownloaded(RECORD_OF_DUMP);
+
+  await Assert.rejects(
+    client.attachments.download(null, {
+      attachmentId: "filename-without-meta.txt",
+      fallbackToDump: true,
+    }),
+    /DownloadError: Could not download filename-without-meta.txt/,
+    "Cannot download dump that lacks a .meta.json file"
+  );
+
+  await Assert.rejects(
+    client.attachments.download(null, {
+      attachmentId: "filename-without-content.txt",
+      fallbackToDump: true,
+    }),
+    /Could not download resource:\/\/rs-downloader-test\/settings\/dump-bucket\/dump-collection\/filename-without-content\.txt(?!\.meta\.json)/,
+    "Cannot download dump that is missing, despite the existing .meta.json"
+  );
+
+  // Restore, just in case.
+  Downloader._RESOURCE_BASE_URL = orig_RESOURCE_BASE_URL;
+  resProto.setSubstitution("rs-downloader-test", null);
+});
+// Not really needed because the last test doesn't modify the main collection,
+// but added for consistency with other tests tasks around here.
+add_task(clear_state);
+
+add_task(
+  async function test_download_from_dump_fails_when_load_dumps_is_false() {
+    const client = RemoteSettings("dump-collection", {
+      bucketName: "dump-bucket",
+    });
+
+    // Temporarily replace the resource:-URL with another resource:-URL.
+    const orig_RESOURCE_BASE_URL = Downloader._RESOURCE_BASE_URL;
+    Downloader._RESOURCE_BASE_URL = "resource://rs-downloader-test";
+    const resProto = Services.io
+      .getProtocolHandler("resource")
+      .QueryInterface(Ci.nsIResProtocolHandler);
+    resProto.setSubstitution(
+      "rs-downloader-test",
+      Services.io.newFileURI(do_get_file("test_attachments_downloader"))
+    );
+
+    function checkInfo(
+      result,
+      expectedSource,
+      expectedRecord = RECORD_OF_DUMP
+    ) {
+      Assert.equal(
+        new TextDecoder().decode(new Uint8Array(result.buffer)),
+        "This would be a RS dump.\n",
+        "expected content from dump"
+      );
+      Assert.deepEqual(
+        result.record,
+        expectedRecord,
+        "expected record for dump"
+      );
+      Assert.equal(result._source, expectedSource, "expected source of dump");
+    }
+
+    // Download the dump so that we can use it to fill the cache.
+    const dump1 = await client.attachments.download(RECORD_OF_DUMP, {
+      // Note: attachmentId not set, so should fall back to record.id.
+      fallbackToDump: true,
+    });
+    checkInfo(dump1, "dump_match");
+
+    // Fill the cache with the same data as the dump for the next part.
+    await client.db.saveAttachment(RECORD_OF_DUMP.id, {
+      record: RECORD_OF_DUMP,
+      blob: new Blob([dump1.buffer]),
+    });
+
+    // Now turn off loading dumps, and check we no longer load from the dump,
+    // but use the cache instead.
+    Utils.LOAD_DUMPS = false;
+
+    const dump2 = await client.attachments.download(RECORD_OF_DUMP, {
+      // Note: attachmentId not set, so should fall back to record.id.
+      fallbackToDump: true,
+    });
+    checkInfo(dump2, "cache_match");
+
+    // When the record is not given, the dump would take precedence over the
+    // cache but we have disabled dumps, so we should load from the cache.
+    const dump4 = await client.attachments.download(null, {
+      attachmentId: RECORD_OF_DUMP.id,
+      fallbackToCache: true,
+      fallbackToDump: true,
+    });
+    checkInfo(dump4, "cache_fallback");
+
+    // Restore, just in case.
+    Utils.LOAD_DUMPS = true;
+    Downloader._RESOURCE_BASE_URL = orig_RESOURCE_BASE_URL;
+    resProto.setSubstitution("rs-downloader-test", null);
+  }
+);
+
+add_task(async function test_attachment_get() {
+  // Since get() is largely a wrapper around the same code as download(),
+  // we only test a couple of parts to check it functions as expected, and
+  // rely on the download() testing for the rest.
+
+  await Assert.rejects(
+    downloader.get(RECORD),
+    /NotFoundError: Could not find /,
+    "get() fails when there is no local cache nor dump"
+  );
+
+  const client = RemoteSettings("dump-collection", {
+    bucketName: "dump-bucket",
+  });
+
+  // Temporarily replace the resource:-URL with another resource:-URL.
+  const orig_RESOURCE_BASE_URL = Downloader._RESOURCE_BASE_URL;
+  Downloader._RESOURCE_BASE_URL = "resource://rs-downloader-test";
+  const resProto = Services.io
+    .getProtocolHandler("resource")
+    .QueryInterface(Ci.nsIResProtocolHandler);
+  resProto.setSubstitution(
+    "rs-downloader-test",
+    Services.io.newFileURI(do_get_file("test_attachments_downloader"))
+  );
+
+  function checkInfo(result, expectedSource, expectedRecord = RECORD_OF_DUMP) {
+    Assert.equal(
+      new TextDecoder().decode(new Uint8Array(result.buffer)),
+      "This would be a RS dump.\n",
+      "expected content from dump"
+    );
+    Assert.deepEqual(result.record, expectedRecord, "expected record for dump");
+    Assert.equal(result._source, expectedSource, "expected source of dump");
+  }
+
+  // When a record is given, use whichever that has the matching last_modified.
+  const dump = await client.attachments.get(RECORD_OF_DUMP);
+  checkInfo(dump, "dump_match");
+
+  await client.attachments.deleteDownloaded(RECORD_OF_DUMP);
+
+  await Assert.rejects(
+    client.attachments.get(null, {
+      attachmentId: "filename-without-meta.txt",
+      fallbackToDump: true,
+    }),
+    /NotFoundError: Could not find filename-without-meta.txt in cache or dump/,
+    "Cannot download dump that lacks a .meta.json file"
+  );
+
+  await Assert.rejects(
+    client.attachments.get(null, {
+      attachmentId: "filename-without-content.txt",
+      fallbackToDump: true,
+    }),
+    /Could not download resource:\/\/rs-downloader-test\/settings\/dump-bucket\/dump-collection\/filename-without-content\.txt(?!\.meta\.json)/,
+    "Cannot download dump that is missing, despite the existing .meta.json"
+  );
+
+  // Restore, just in case.
+  Downloader._RESOURCE_BASE_URL = orig_RESOURCE_BASE_URL;
+  resProto.setSubstitution("rs-downloader-test", null);
+});
+// Not really needed because the last test doesn't modify the main collection,
+// but added for consistency with other tests tasks around here.
+add_task(clear_state);
+
+add_task(async function test_obsolete_attachments_are_pruned() {
+  const RECORD2 = {
+    ...RECORD,
+    id: "another-id",
+  };
+  const client = RemoteSettings("some-collection");
+  // Store records and related attachments directly in the cache.
+  await client.db.importChanges({}, 42, [RECORD, RECORD2], { clear: true });
+  await client.db.saveAttachment(RECORD.id, {
+    record: RECORD,
+    blob: new Blob(["123"]),
+  });
+  await client.db.saveAttachment("custom-id", {
+    record: RECORD2,
+    blob: new Blob(["456"]),
+  });
+  // Store an extraneous cached attachment.
+  await client.db.saveAttachment("bar", {
+    record: { id: "bar" },
+    blob: new Blob(["789"]),
+  });
+
+  const recordAttachment = await client.attachments.cacheImpl.get(RECORD.id);
+  Assert.equal(
+    await recordAttachment.blob.text(),
+    "123",
+    "Record has a cached attachment"
+  );
+  const record2Attachment = await client.attachments.cacheImpl.get("custom-id");
+  Assert.equal(
+    await record2Attachment.blob.text(),
+    "456",
+    "Record 2 has a cached attachment"
+  );
+  const { blob: cachedExtra } = await client.attachments.cacheImpl.get("bar");
+  Assert.equal(await cachedExtra.text(), "789", "There is an extra attachment");
+
+  await client.attachments.prune([]);
+
+  Assert.ok(
+    await client.attachments.cacheImpl.get(RECORD.id),
+    "Record attachment was kept"
+  );
+  Assert.ok(
+    await client.attachments.cacheImpl.get("custom-id"),
+    "Record 2 attachment was kept"
+  );
+  Assert.ok(
+    !(await client.attachments.cacheImpl.get("bar")),
+    "Extra was deleted"
+  );
+});
+add_task(clear_state);
+
+add_task(
+  async function test_obsolete_attachments_listed_as_excluded_are_not_pruned() {
+    const client = RemoteSettings("some-collection");
+    // Store records and related attachments directly in the cache.
+    await client.db.importChanges({}, 42, [], { clear: true });
+    await client.db.saveAttachment(RECORD.id, {
+      record: RECORD,
+      blob: new Blob(["123"]),
+    });
+
+    const recordAttachment = await client.attachments.cacheImpl.get(RECORD.id);
+    Assert.equal(
+      await recordAttachment.blob.text(),
+      "123",
+      "Record has a cached attachment"
+    );
+
+    await client.attachments.prune([RECORD.id]);
+
+    Assert.ok(
+      await client.attachments.cacheImpl.get(RECORD.id),
+      "Record attachment was kept"
+    );
+  }
+);
+
+add_task(clear_state);
+
+add_task(async function test_cacheAll_happy_path() {
+  // verify bundle is downloaded succesfully
+  const allSuccess = await downloader.cacheAll();
+  Assert.ok(
+    allSuccess,
+    "Attachments cacheAll succesfully downloaded a bundle and saved all attachments"
+  );
+
+  // verify accuracy of attachments downloaded
+  Assert.equal(
+    downloader.cache["1"].record.title,
+    "test1",
+    "Test record 1 meta content appears accurate."
+  );
+  Assert.equal(
+    await downloader.cache["1"].blob.text(),
+    "test1\n",
+    "Test file 1 content is accurate."
+  );
+  Assert.equal(
+    downloader.cache["2"].record.title,
+    "test2",
+    "Test record 2 meta content appears accurate."
+  );
+  Assert.equal(
+    await downloader.cache["2"].blob.text(),
+    "test2\n",
+    "Test file 2 content is accurate."
+  );
+});
+
+add_task(async function test_cacheAll_using_real_db() {
+  const client = RemoteSettings("some-collection");
+
+  const allSuccess = await client.attachments.cacheAll();
+
+  Assert.ok(
+    allSuccess,
+    "Attachments cacheAll succesfully downloaded a bundle and saved all attachments"
+  );
+
+  Assert.equal(
+    (await client.attachments.cacheImpl.get("2")).record.title,
+    "test2",
+    "Test record 2 meta content appears accurate."
+  );
+  Assert.equal(
+    await (await client.attachments.cacheImpl.get("2")).blob.text(),
+    "test2\n",
+    "Test file 2 content is accurate."
+  );
+});
+
+add_task(clear_state);
+
+add_task(async function test_cacheAll_skips_with_existing_data() {
+  downloader.cache = {
+    1: "1",
+  };
+  const allSuccess = await downloader.cacheAll();
+  Assert.equal(
+    allSuccess,
+    null,
+    "Attachments cacheAll skips downloads if data already exists"
+  );
+});
+
+add_task(async function test_cacheAll_does_not_skip_if_force_is_true() {
+  downloader.cache = {
+    1: "1",
+  };
+  const allSuccess = await downloader.cacheAll(true);
+  Assert.equal(
+    allSuccess,
+    true,
+    "Attachments cacheAll does not skip downloads if force is true"
+  );
+});
+
+add_task(clear_state);
+
+add_task(async function test_cacheAll_failed_request() {
+  downloader.bucketName = "fake-bucket";
+  downloader.collectionName = "fake-collection";
+  const allSuccess = await downloader.cacheAll();
+  Assert.equal(
+    allSuccess,
+    false,
+    "Attachments cacheAll request failed to download a bundle and returned false"
+  );
+});
+
+add_task(clear_state);
+
+add_task(async function test_cacheAll_failed_unzip() {
+  downloader.bucketName = "error-bucket";
+  downloader.collectionName = "bad-zip";
+  const allSuccess = await downloader.cacheAll();
+  Assert.equal(
+    allSuccess,
+    false,
+    "Attachments cacheAll request failed to extract a bundle and returned false"
+  );
+});
+
+add_task(clear_state);
+
+add_task(async function test_cacheAll_failed_save() {
+  const client = RemoteSettings("some-collection");
+
+  const backup = client.db.saveAttachments;
+  client.db.saveAttachments = () => {
+    throw new Error("boom");
+  };
+
+  const allSuccess = await client.attachments.cacheAll();
+
+  Assert.equal(
+    allSuccess,
+    false,
+    "Attachments cacheAll failed to save entries in DB and returned false"
+  );
+  client.db.saveAttachments = backup;
+});
+
+add_task(clear_state);

src/zen/tests/mochitests/remote-settings/test_attachments_downloader/65650a0f-7c22-4c10-9744-2d67e301f5f4.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEbjCCA1agAwIBAgIQBg3WwdBnkBtUdfz/wp4xNzANBgkqhkiG9w0BAQsFADBa
+MQswCQYDVQQGEwJJRTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJl
+clRydXN0MSIwIAYDVQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTE1
+MTAxNDEyMDAwMFoXDTIwMTAxNDEyMDAwMFowbzELMAkGA1UEBhMCVVMxCzAJBgNV
+BAgTAkNBMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRkwFwYDVQQKExBDbG91ZEZs
+YXJlLCBJbmMuMSAwHgYDVQQDExdDbG91ZEZsYXJlIEluYyBSU0EgQ0EtMTCCASIw
+DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJGiNOIE4s0M4wdhDeV9aMfAYY9l
+yG9cfGQqt7a5UgrRA81bi4istCyhzfzRWUW+NAmf6X2HEnA3xLI1M+pH/xEbk9pw
+jc8/1CPy9jUjBwb89zt5PWh2I1KxZVg/Bnx2yYdVcKTUMKt0GLDXfZXN+RYZHJQo
+lDlzjH5xV0IpDMv/FsMEZWcfx1JorBf08bRnRVkl9RY00y2ujVr+492ze+zYQ9s7
+HcidpR+7ret3jzLSvojsaA5+fOaCG0ctVJcLfnkQ5lWR95ByBdO1NapfqZ1+kmCL
+3baVSeUpYQriBwznxfLuGs8POo4QdviYVtSPBWjOEfb+o1c6Mbo8p4noFzUCAwEA
+AaOCARkwggEVMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMDQG
+CCsGAQUFBwEBBCgwJjAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu
+Y29tMDoGA1UdHwQzMDEwL6AtoCuGKWh0dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9P
+bW5pcm9vdDIwMjUuY3JsMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIB
+FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMB0GA1UdDgQWBBSRBYrfTCLG
+bYuUTBZFfu5vAvu3wDAfBgNVHSMEGDAWgBTlnVkwgkdYzKz6CFQ2hns6tQRN8DAN
+BgkqhkiG9w0BAQsFAAOCAQEAVJle3ar9NSnTrLAhgfkcpClIY6/kabDIEa8cOnu1
+SOXf4vbtZakSmmIbFbmYDUGIU5XwwVdF/FKNzNBRf9G4EL/S0NXytBKj4A34UGQA
+InaV+DgVLzCifN9cAHi8EFEAfbglUvPvLPFXF0bwffElYm7QBSiHYSZmfOKLCyiv
+3zlQsf7ozNBAxfbmnRMRSUBcIhRwnaFoFgDs7yU6R1Yk4pO7eMgWpdPGhymDTIvv
+RnauKStzKsAli9i5hQ4nTDITUpMAmeJoXodgwRkC3Civw32UR2rxObIyxPpbfODb
+sZKNGO9K5Sjj6turB1zwbd2wI8MhtUCY9tGmSYhe7G6Bkw==
+-----END CERTIFICATE-----

src/zen/tests/mochitests/remote-settings/test_attachments_downloader/error-bucket--bad-zip.zip

@@ -0,0 +1 @@
+not a zip

src/zen/tests/mochitests/remote-settings/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt

@@ -0,0 +1 @@
+This would be a RS dump.

src/zen/tests/mochitests/remote-settings/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt.meta.json

@@ -0,0 +1,10 @@
+{
+  "id": "filename-of-dump.txt",
+  "attachment": {
+    "filename": "filename-of-dump.txt",
+    "hash": "4c46ef7e4f1951d210fe54c21e07c09bab265fd122580083ed1d6121547a8c6b",
+    "size": 25
+  },
+  "last_modified": 1234567,
+  "some_key": "some metadata"
+}

src/zen/tests/mochitests/remote-settings/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-content.txt.meta.json

@@ -0,0 +1,8 @@
+{
+  "fyi": "This .meta.json file describes an attachment, but that attachment is missing.",
+  "attachment": {
+    "filename": "filename-without-content.txt",
+    "hash": "...",
+    "size": "..."
+  }
+}

src/zen/tests/mochitests/remote-settings/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-meta.txt

@@ -0,0 +1 @@
+The filename-without-meta.txt.meta.json file is missing.

src/zen/tests/mochitests/remote-settings/test_remote_settings_dump_lastmodified.js

@@ -0,0 +1,55 @@
+"use strict";
+
+async function getLocalDumpLastModified(bucket, collection) {
+  let res;
+  try {
+    res = await fetch(
+      `resource://app/defaults/settings/${bucket}/${collection}.json`
+    );
+  } catch (e) {
+    return -1;
+  }
+  const { timestamp } = await res.json();
+  Assert.greaterOrEqual(
+    timestamp,
+    0,
+    `${bucket}/${collection} dump has timestamp`
+  );
+  return timestamp;
+}
+
+add_task(async function lastModified_of_non_existing_dump() {
+  ok(!Utils._dumpStats, "_dumpStats not initialized");
+  equal(
+    await Utils.getLocalDumpLastModified("did not", "exist"),
+    -1,
+    "A non-existent dump has value -1"
+  );
+  ok(Utils._dumpStats, "_dumpStats was initialized");
+
+  ok("did not/exist" in Utils._dumpStats, "cached non-existing dump result");
+  delete Utils._dumpStats["did not/exist"];
+});
+
+add_task(async function lastModified_summary_is_correct() {
+  ok(!!Object.keys(Utils._dumpStats).length, "Contains summary of dumps");
+
+  let checked = 0;
+  for (let [identifier, lastModified] of Object.entries(Utils._dumpStats)) {
+    let [bucket, collection] = identifier.split("/");
+    let actual = await getLocalDumpLastModified(bucket, collection);
+    if (actual < 0) {
+      info(`${identifier} has no dump, skip.`);
+      continue;
+    }
+    info(`Checking correctness of ${identifier}`);
+    equal(
+      await Utils.getLocalDumpLastModified(bucket, collection),
+      lastModified,
+      `Expected last_modified value for ${identifier}`
+    );
+    equal(lastModified, actual, `last_modified should match collection`);
+    checked++;
+  }
+  Assert.greater(checked, 0, "At least one dump was packaged and checked.");
+});

src/zen/tests/mochitests/remote-settings/test_remote_settings_jexl_filters.js

@@ -0,0 +1,210 @@
+let client;
+
+async function createRecords(records) {
+  await client.db.importChanges(
+    {},
+    42,
+    records.map((record, i) => ({
+      id: `record-${i}`,
+      ...record,
+    })),
+    {
+      clear: true,
+    }
+  );
+}
+
+add_setup(() => {
+  client = RemoteSettings("some-key");
+});
+
+add_task(async function test_returns_all_without_target() {
+  await createRecords([
+    {
+      passwordSelector: "#pass-signin",
+    },
+    {
+      filter_expression: null,
+    },
+    {
+      filter_expression: "",
+    },
+  ]);
+
+  const list = await client.get();
+  equal(list.length, 3);
+});
+
+add_task(async function test_filters_can_be_disabled() {
+  const c = RemoteSettings("no-jexl", { filterCreator: null });
+  await c.db.importChanges({}, 42, [
+    {
+      id: "abc",
+      filter_expression: "1 == 2",
+    },
+  ]);
+
+  const list = await c.get();
+  equal(list.length, 1);
+});
+
+add_task(async function test_returns_entries_where_jexl_is_true() {
+  await createRecords([
+    {
+      willMatch: true,
+      filter_expression: "1",
+    },
+    {
+      willMatch: true,
+      filter_expression: "[42]",
+    },
+    {
+      willMatch: true,
+      filter_expression: "1 == 2 || 1 == 1",
+    },
+    {
+      willMatch: true,
+      filter_expression: 'env.appinfo.ID == "xpcshell@tests.mozilla.org"',
+    },
+    {
+      willMatch: false,
+      filter_expression: "env.version == undefined",
+    },
+    {
+      willMatch: true,
+      filter_expression: "env.unknown == undefined",
+    },
+    {
+      willMatch: false,
+      filter_expression: "1 == 2",
+    },
+  ]);
+
+  const list = await client.get();
+  equal(list.length, 5);
+  ok(list.every(e => e.willMatch));
+});
+
+add_task(async function test_ignores_entries_where_jexl_is_invalid() {
+  await createRecords([
+    {
+      filter_expression: "true === true", // JavaScript Error: "Invalid expression token: ="
+    },
+    {
+      filter_expression: "Objects.keys({}) == []", // Token ( (openParen) unexpected in expression
+    },
+  ]);
+
+  const list = await client.get();
+  equal(list.length, 0);
+});
+
+add_task(async function test_support_of_date_filters() {
+  await createRecords([
+    {
+      willMatch: true,
+      filter_expression: '"1982-05-08"|date < "2016-03-22"|date',
+    },
+    {
+      willMatch: false,
+      filter_expression: '"2000-01-01"|date < "1970-01-01"|date',
+    },
+  ]);
+
+  const list = await client.get();
+  equal(list.length, 1);
+  ok(list.every(e => e.willMatch));
+});
+
+add_task(async function test_support_of_preferences_filters() {
+  await createRecords([
+    {
+      willMatch: true,
+      filter_expression: '"services.settings.last_etag"|preferenceValue == 42',
+    },
+    {
+      willMatch: true,
+      filter_expression:
+        '"services.settings.poll_interval"|preferenceExists == true',
+    },
+    {
+      willMatch: true,
+      filter_expression:
+        '"services.settings.poll_interval"|preferenceIsUserSet == false',
+    },
+    {
+      willMatch: true,
+      filter_expression:
+        '"services.settings.last_etag"|preferenceIsUserSet == true',
+    },
+  ]);
+
+  // Set a pref for the user.
+  Services.prefs.setIntPref("services.settings.last_etag", 42);
+
+  const list = await client.get();
+  equal(list.length, 4);
+  ok(list.every(e => e.willMatch));
+});
+
+add_task(async function test_support_of_intersect_operator() {
+  await createRecords([
+    {
+      willMatch: true,
+      filter_expression: '{foo: 1, bar: 2}|keys intersect ["foo"]',
+    },
+    {
+      willMatch: true,
+      filter_expression: '(["a", "b"] intersect ["a", 1, 4]) == "a"',
+    },
+    {
+      willMatch: false,
+      filter_expression: '(["a", "b"] intersect [3, 1, 4]) == "c"',
+    },
+    {
+      willMatch: true,
+      filter_expression: `
+      [1, 2, 3]
+        intersect
+      [3, 4, 5]
+    `,
+    },
+  ]);
+
+  const list = await client.get();
+  equal(list.length, 3);
+  ok(list.every(e => e.willMatch));
+});
+
+add_task(async function test_support_of_samples() {
+  await createRecords([
+    {
+      willMatch: true,
+      filter_expression: '"always-true"|stableSample(1)',
+    },
+    {
+      willMatch: false,
+      filter_expression: '"always-false"|stableSample(0)',
+    },
+    {
+      willMatch: true,
+      filter_expression: '"turns-to-true-0"|stableSample(0.5)',
+    },
+    {
+      willMatch: false,
+      filter_expression: '"turns-to-false-1"|stableSample(0.5)',
+    },
+    {
+      willMatch: true,
+      filter_expression: '"turns-to-true-0"|bucketSample(0, 50, 100)',
+    },
+    {
+      willMatch: false,
+      filter_expression: '"turns-to-false-1"|bucketSample(0, 50, 100)',
+    },
+  ]);
+
+  const list = await client.get();
+  equal(list.length, 3);
+  ok(list.every(e => e.willMatch));
+});

src/zen/tests/mochitests/remote-settings/test_remote_settings_offline.js

@@ -0,0 +1,139 @@
+// A collection with a dump that's packaged on all builds where this test runs,
+// including on Android at mobile/android/installer/package-manifest.in
+const TEST_BUCKET = "main";
+const TEST_COLLECTION = "password-recipes";
+
+let client;
+let DUMP_RECORDS;
+let DUMP_LAST_MODIFIED;
+
+add_setup(async () => {
+  // "services.settings.server" pref is not set.
+  // Test defaults to an unreachable server,
+  // and will only load from the dump if any.
+
+  client = new RemoteSettingsClient(TEST_COLLECTION, {
+    bucketName: TEST_BUCKET,
+  });
+
+  const dump = await SharedUtils.loadJSONDump(TEST_BUCKET, TEST_COLLECTION);
+  DUMP_RECORDS = dump.data;
+  DUMP_LAST_MODIFIED = dump.timestamp;
+
+  // Dumps are fetched via the following, which sorts the records, newest first.
+  // https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
+  equal(
+    DUMP_LAST_MODIFIED,
+    DUMP_RECORDS[0].last_modified,
+    "records in dump ought to be sorted by last_modified"
+  );
+});
+
+async function importData(records) {
+  await RemoteSettingsWorker._execute("_test_only_import", [
+    TEST_BUCKET,
+    TEST_COLLECTION,
+    records,
+    records[0]?.last_modified || 0,
+  ]);
+}
+
+async function clear_state() {
+  await client.db.clear();
+}
+
+add_task(async function test_load_from_dump_when_offline() {
+  // Baseline: verify that the collection is empty at first,
+  // but non-empty after loading from the dump.
+  const beforeTimestamp = await client.db.getLastModified();
+  equal(beforeTimestamp, null, "collection empty when offline");
+
+  // should import from dump since collection was not initialized.
+  const after = await client.get();
+  equal(after.length, DUMP_RECORDS.length, "collection loaded from dump");
+  equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "dump's timestamp");
+});
+add_task(clear_state);
+
+add_task(async function test_optional_skip_dump_after_empty_import() {
+  // clear_state should have wiped the database.
+  const beforeTimestamp = await client.db.getLastModified();
+  equal(beforeTimestamp, null, "collection empty after clearing");
+
+  // Verify that the dump is not imported again by client.get()
+  // when the database is initialized with an empty dump
+  // with `loadDumpIfNewer` disabled.
+  await importData([]); // <-- Empty set of records.
+
+  const after = await client.get({ loadDumpIfNewer: false });
+  equal(after.length, 0, "collection still empty due to import");
+  equal(await client.getLastModified(), 0, "Empty dump has no timestamp");
+});
+add_task(clear_state);
+
+add_task(async function test_optional_skip_dump_after_non_empty_import() {
+  await importData([{ last_modified: 1234, id: "dummy" }]);
+
+  const after = await client.get({ loadDumpIfNewer: false });
+  equal(after.length, 1, "Imported dummy data");
+  equal(await client.getLastModified(), 1234, "Expected timestamp of import");
+
+  await importData([]);
+  const after2 = await client.get({ loadDumpIfNewer: false });
+  equal(after2.length, 0, "Previous data wiped on duplicate import");
+  equal(await client.getLastModified(), 0, "Timestamp of empty collection");
+});
+add_task(clear_state);
+
+add_task(async function test_load_dump_after_empty_import() {
+  await importData([]); // <-- Empty set of records, i.e. last_modified = 0.
+
+  const after = await client.get();
+  equal(after.length, DUMP_RECORDS.length, "Imported dump");
+  equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "dump's timestamp");
+});
+add_task(clear_state);
+
+add_task(async function test_load_dump_after_non_empty_import() {
+  // Dump is updated regularly, verify that the dump matches our expectations
+  // before running the test.
+  Assert.greater(
+    DUMP_LAST_MODIFIED,
+    1234,
+    "Assuming dump to be newer than dummy 1234"
+  );
+
+  await importData([{ last_modified: 1234, id: "dummy" }]);
+
+  const after = await client.get();
+  equal(after.length, DUMP_RECORDS.length, "Imported dump");
+  equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "dump's timestamp");
+});
+add_task(clear_state);
+
+add_task(async function test_load_dump_after_import_from_broken_distro() {
+  // Dump is updated regularly, verify that the dump matches our expectations
+  // before running the test.
+  Assert.greater(
+    DUMP_LAST_MODIFIED,
+    1234,
+    "Assuming dump to be newer than dummy 1234"
+  );
+
+  // No last_modified time.
+  await importData([{ id: "dummy" }]);
+
+  const after = await client.get();
+  equal(after.length, DUMP_RECORDS.length, "Imported dump");
+  equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "dump's timestamp");
+});
+add_task(clear_state);
+
+add_task(async function test_skip_dump_if_same_last_modified() {
+  await importData([{ last_modified: DUMP_LAST_MODIFIED, id: "dummy" }]);
+
+  const after = await client.get();
+  equal(after.length, 1, "Not importing dump when time matches");
+  equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "Same timestamp");
+});
+add_task(clear_state);

src/zen/tests/mochitests/remote-settings/test_remote_settings_older_than_local.js

@@ -0,0 +1,149 @@
+const PREF_SETTINGS_SERVER = "services.settings.server";
+
+let server;
+let client;
+
+async function clear_state() {
+  await client.db.clear();
+}
+
+function setJSONResponse(response, changeset) {
+  response.setStatusLine(null, 200, "OK");
+  response.setHeader("Content-Type", "application/json; charset=UTF-8");
+  response.setHeader("Date", new Date().toUTCString());
+  response.write(JSON.stringify(changeset));
+}
+
+add_setup(() => {
+  Services.prefs.setStringPref("services.settings.loglevel", "debug");
+
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  Services.prefs.setStringPref(
+    PREF_SETTINGS_SERVER,
+    `http://localhost:${server.identity.primaryPort}/v1`
+  );
+
+  client = RemoteSettings("some-cid");
+
+  const bodies = {
+    // First successful sync at timestamp=333
+    "/v1/buckets/main/collections/some-cid/changeset?_expected=333": {
+      timestamp: 333,
+      metadata: {
+        signatures: [{ signature: "abc", x5u: "data:text/plain;base64,pem" }],
+      },
+      changes: [
+        { id: "rid2", last_modified: 22 },
+        { id: "rid1", last_modified: 11 },
+      ],
+    },
+    // Client fetches with _expected=222&_since=333
+    "/v1/buckets/main/collections/some-cid/changeset?_expected=222&_since=333":
+      {
+        timestamp: 222,
+        metadata: {
+          signatures: [{ signature: "ghi", x5u: "data:text/plain;base64,pem" }],
+        },
+        changes: [{ id: "rid1", last_modified: 11 }],
+      },
+    // Client refetches full collection with _expected=222 after signature fails.
+    "/v1/buckets/main/collections/some-cid/changeset?_expected=222": {
+      timestamp: 222,
+      metadata: {
+        signatures: [{ signature: "ghi", x5u: "data:text/plain;base64,pem" }],
+      },
+      changes: [{ id: "rid1", last_modified: 11 }],
+    },
+  };
+  const handler = (request, response) => {
+    const body = bodies[`${request.path}?${request.queryString}`];
+    if (!body) {
+      const err = new Error(
+        `Unexpected request ${request.path}?${request.queryString}`
+      );
+      console.error(err);
+      throw err;
+    }
+    setJSONResponse(response, body);
+  };
+  Object.keys(bodies).map(path =>
+    server.registerPathHandler(path.split("?")[0], handler)
+  );
+
+  // In this test suite, the only valid data is the one at timestamp=222 or timestamp=333, but not a mix of the two.
+  let backup = client._verifier;
+  client._verifier = {
+    asyncVerifyContentSignature: serialized => {
+      return (
+        serialized ==
+          '{"data":[{"id":"rid2","last_modified":22}],"last_modified":"444"}' ||
+        serialized ==
+          '{"data":[{"id":"rid1","last_modified":11},{"id":"rid2","last_modified":22}],"last_modified":"333"}' ||
+        serialized ==
+          '{"data":[{"id":"rid1","last_modified":11}],"last_modified":"222"}'
+      );
+    },
+  };
+
+  registerCleanupFunction(() => {
+    client._verifier = backup;
+    server.stop(() => {});
+    Services.prefs.clearUserPref(PREF_SETTINGS_SERVER);
+
+    Services.prefs.clearUserPref("services.settings.loglevel");
+  });
+});
+
+add_task(clear_state);
+
+add_task(async function test_ignores_if_local_signature_is_valid() {
+  await client.maybeSync(333);
+  Assert.deepEqual(
+    [
+      await client.db.getLastModified(),
+      (await client.get()).map(({ id }) => id),
+    ],
+    [333, ["rid1", "rid2"]]
+  );
+
+  await client.maybeSync(222);
+
+  // The client should detect that timestamp is older,
+  // but it ignores it because local data is good.
+  Assert.deepEqual(
+    [
+      await client.db.getLastModified(),
+      (await client.get()).map(({ id }) => id),
+    ],
+    [333, ["rid1", "rid2"]]
+  );
+});
+add_task(clear_state);
+
+add_task(async function test_uses_old_data_if_local_signature_is_invalid() {
+  // Import some data in the local DB. Signature will be bad.
+  await client.db.importChanges({}, 333, [{ id: "myid", last_modified: 1234 }]);
+  Assert.deepEqual(
+    [
+      await client.db.getLastModified(),
+      (await client.get()).map(({ id }) => id),
+    ],
+    [333, ["myid"]]
+  );
+
+  await client.maybeSync(222);
+
+  // The client should detect that timestamp is older,
+  // but will replace local data with this old data.
+  Assert.deepEqual(
+    [
+      await client.db.getLastModified(),
+      (await client.get()).map(({ id }) => id),
+    ],
+    [222, ["rid1"]]
+  );
+});
+add_task(clear_state);

src/zen/tests/mochitests/remote-settings/test_remote_settings_recover_broken.js

@@ -0,0 +1,138 @@
+const PREF_SETTINGS_SERVER = "services.settings.server";
+const CHANGES_PATH = "/v1" + Utils.CHANGES_PATH;
+const BROKEN_SYNC_THRESHOLD = 10; // See default pref value
+
+let server;
+let client;
+let maybeSyncBackup;
+
+async function clear_state() {
+  // Disable logging output.
+  Services.prefs.setStringPref("services.settings.loglevel", "critical");
+  // Pull data from the test server.
+  Services.prefs.setStringPref(
+    PREF_SETTINGS_SERVER,
+    `http://localhost:${server.identity.primaryPort}/v1`
+  );
+
+  // Clear sync history.
+  await new SyncHistory("").clear();
+
+  // Simulate a response whose ETag gets incremented on each call
+  // (in order to generate several history entries, indexed by timestamp).
+  let timestamp = 1337;
+  server.registerPathHandler(CHANGES_PATH, (request, response) => {
+    response.setStatusLine(null, 200, "OK");
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setHeader("Date", new Date(1000000).toUTCString());
+    response.setHeader("ETag", `"${timestamp}"`);
+    response.write(
+      JSON.stringify({
+        timestamp,
+        changes: [
+          {
+            last_modified: ++timestamp,
+            bucket: "main",
+            collection: "desktop-manager",
+          },
+        ],
+      })
+    );
+  });
+
+  // Restore original maybeSync() method between each test.
+  client.maybeSync = maybeSyncBackup;
+}
+
+function run_test() {
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  client = RemoteSettings("desktop-manager");
+  maybeSyncBackup = client.maybeSync;
+
+  run_next_test();
+
+  registerCleanupFunction(() => {
+    server.stop(() => {});
+    // Restore original maybeSync() method when test suite is done.
+    client.maybeSync = maybeSyncBackup;
+  });
+}
+
+add_task(clear_state);
+
+add_task(async function test_db_is_destroyed_when_sync_is_broken() {
+  // Simulate a successful sync.
+  client.maybeSync = async () => {
+    // Store some data in local DB.
+    await client.db.importChanges({}, 1515, []);
+  };
+  await RemoteSettings.pollChanges({ trigger: "timer" });
+
+  // Register a client with a failing sync method.
+  client.maybeSync = () => {
+    throw new RemoteSettingsClient.InvalidSignatureError(
+      "main/desktop-manager"
+    );
+  };
+
+  // Now obtain several failures in a row.
+  for (var i = 0; i < BROKEN_SYNC_THRESHOLD; i++) {
+    try {
+      await RemoteSettings.pollChanges({ trigger: "timer" });
+    } catch (e) {}
+  }
+
+  // Synchronization is in broken state.
+  Assert.equal(
+    await client.db.getLastModified(),
+    1515,
+    "Local DB was not destroyed yet"
+  );
+
+  // Synchronize again. Broken state will be detected.
+  try {
+    await RemoteSettings.pollChanges({ trigger: "timer" });
+  } catch (e) {}
+
+  // DB was destroyed.
+  Assert.equal(
+    await client.db.getLastModified(),
+    null,
+    "Local DB was destroyed"
+  );
+});
+
+add_task(clear_state);
+
+add_task(async function test_db_is_not_destroyed_when_state_is_server_error() {
+  // Since we don't mock the server endpoints to obtain the changeset of this
+  // collection, the call to `maybeSync()` will fail with network errors.
+
+  // Store some data in local DB.
+  await client.db.importChanges({}, 1515, []);
+
+  // Now obtain several failures in a row.
+  let lastError;
+  for (var i = 0; i < BROKEN_SYNC_THRESHOLD + 1; i++) {
+    try {
+      await RemoteSettings.pollChanges({ trigger: "timer" });
+    } catch (e) {
+      lastError = e;
+    }
+  }
+  Assert.ok(
+    /Cannot parse server content/.test(lastError.message),
+    "Error is about server"
+  );
+  // DB was not destroyed.
+  Assert.equal(
+    await client.db.getLastModified(),
+    1515,
+    "Local DB was not destroyed"
+  );
+});
+
+add_task(clear_state);

src/zen/tests/mochitests/remote-settings/test_remote_settings_release_prefs.js

@@ -0,0 +1,252 @@
+"use strict";
+
+var nextUniqId = 0;
+function getNewUtils() {
+  const { Utils } = ChromeUtils.importESModule(
+    `resource://services-settings/Utils.sys.mjs?_${++nextUniqId}`
+  );
+  return Utils;
+}
+
+// A collection with a dump that's packaged on all builds where this test runs,
+// including on Android at mobile/android/installer/package-manifest.in
+const TEST_BUCKET = "main";
+const TEST_COLLECTION = "password-recipes";
+
+async function importData(records) {
+  await RemoteSettingsWorker._execute("_test_only_import", [
+    TEST_BUCKET,
+    TEST_COLLECTION,
+    records,
+    records[0]?.last_modified || 0,
+  ]);
+}
+
+async function clear_state() {
+  Services.env.set("MOZ_REMOTE_SETTINGS_DEVTOOLS", "0");
+  Services.prefs.clearUserPref("services.settings.server");
+  Services.prefs.clearUserPref("services.settings.preview_enabled");
+}
+
+add_setup(async function () {
+  // Set this env vars in order to test the code path where the
+  // server URL can only be overridden from Dev Tools.
+  // See `isRunningTests` in `services/settings/Utils.sys.mjs`.
+  const before = Services.env.get("MOZ_DISABLE_NONLOCAL_CONNECTIONS");
+  Services.env.set("MOZ_DISABLE_NONLOCAL_CONNECTIONS", "0");
+
+  registerCleanupFunction(() => {
+    clear_state();
+    Services.env.set("MOZ_DISABLE_NONLOCAL_CONNECTIONS", before);
+  });
+});
+
+add_task(clear_state);
+
+add_task(
+  {
+    skip_if: () => !AppConstants.RELEASE_OR_BETA,
+  },
+  async function test_server_url_cannot_be_toggled_in_release() {
+    Services.prefs.setStringPref(
+      "services.settings.server",
+      "http://localhost:8888/v1"
+    );
+
+    const Utils = getNewUtils();
+
+    Assert.equal(
+      Utils.SERVER_URL,
+      AppConstants.REMOTE_SETTINGS_SERVER_URLS[0],
+      "Server url pref was not read in release"
+    );
+  }
+);
+
+add_task(
+  {
+    skip_if: () => AppConstants.RELEASE_OR_BETA,
+  },
+  async function test_server_url_cannot_be_toggled_in_dev_nightly() {
+    Services.prefs.setStringPref(
+      "services.settings.server",
+      "http://localhost:8888/v1"
+    );
+
+    const Utils = getNewUtils();
+
+    Assert.notEqual(
+      Utils.SERVER_URL,
+      AppConstants.REMOTE_SETTINGS_SERVER_URLS[0],
+      "Server url pref was read in nightly/dev"
+    );
+  }
+);
+add_task(clear_state);
+
+add_task(
+  {
+    skip_if: () => !AppConstants.RELEASE_OR_BETA,
+  },
+  async function test_preview_mode_cannot_be_toggled_in_release() {
+    Services.prefs.setBoolPref("services.settings.preview_enabled", true);
+
+    const Utils = getNewUtils();
+
+    Assert.ok(!Utils.PREVIEW_MODE, "Preview mode pref was not read in release");
+  }
+);
+add_task(clear_state);
+
+add_task(
+  {
+    skip_if: () => AppConstants.RELEASE_OR_BETA,
+  },
+  async function test_preview_mode_cannot_be_toggled_in_dev_nightly() {
+    Services.prefs.setBoolPref("services.settings.preview_enabled", true);
+
+    const Utils = getNewUtils();
+
+    Assert.ok(Utils.PREVIEW_MODE, "Preview mode pref is read in dev/nightly");
+  }
+);
+add_task(clear_state);
+
+add_task(
+  {
+    skip_if: () => !AppConstants.RELEASE_OR_BETA,
+  },
+  async function test_load_dumps_will_always_be_loaded_in_release() {
+    Services.prefs.setStringPref(
+      "services.settings.server",
+      "http://localhost:8888/v1"
+    );
+
+    const Utils = getNewUtils();
+
+    Assert.equal(
+      Utils.SERVER_URL,
+      AppConstants.REMOTE_SETTINGS_SERVER_URLS[0],
+      "Server url pref was not read"
+    );
+    Assert.ok(Utils.LOAD_DUMPS, "Dumps will always be loaded");
+  }
+);
+
+add_task(
+  {
+    skip_if: () => AppConstants.RELEASE_OR_BETA,
+  },
+  async function test_load_dumps_can_be_disabled_in_dev_nightly() {
+    Services.prefs.setStringPref(
+      "services.settings.server",
+      "http://localhost:8888/v1"
+    );
+
+    const Utils = getNewUtils();
+
+    Assert.notEqual(
+      Utils.SERVER_URL,
+      AppConstants.REMOTE_SETTINGS_SERVER_URLS[0],
+      "Server url pref was read"
+    );
+    Assert.ok(!Utils.LOAD_DUMPS, "Dumps are not loaded if server is not prod");
+  }
+);
+add_task(clear_state);
+
+add_task(
+  async function test_server_url_can_be_changed_in_all_versions_if_running_for_devtools() {
+    Services.env.set("MOZ_REMOTE_SETTINGS_DEVTOOLS", "1");
+    Services.prefs.setStringPref(
+      "services.settings.server",
+      "http://localhost:8888/v1"
+    );
+
+    const Utils = getNewUtils();
+
+    Assert.notEqual(
+      Utils.SERVER_URL,
+      AppConstants.REMOTE_SETTINGS_SERVER_URLS[0],
+      "Server url pref was read"
+    );
+  }
+);
+add_task(clear_state);
+
+add_task(
+  async function test_preview_mode_can_be_changed_in_all_versions_if_running_for_devtools() {
+    Services.env.set("MOZ_REMOTE_SETTINGS_DEVTOOLS", "1");
+    Services.prefs.setBoolPref("services.settings.preview_enabled", true);
+
+    const Utils = getNewUtils();
+
+    Assert.ok(Utils.PREVIEW_MODE, "Preview mode pref was read");
+  }
+);
+add_task(clear_state);
+
+add_task(
+  async function test_dumps_are_not_loaded_if_server_is_not_prod_if_running_for_devtools() {
+    Services.env.set("MOZ_REMOTE_SETTINGS_DEVTOOLS", "1");
+    Services.prefs.setStringPref(
+      "services.settings.server",
+      "http://localhost:8888/v1"
+    );
+
+    const Utils = getNewUtils();
+    Assert.ok(!Utils.LOAD_DUMPS, "Dumps won't be loaded");
+
+    // The section below ensures that the LOAD_DUMPS flag properly takes effect.
+    // The client is set up here rather than add_setup to avoid triggering the
+    // lazy getters that are behind the global Utils.LOAD_DUMPS. If they are
+    // triggered too early, then they will potentially cache different values
+    // for the server urls and environment variables and this test then won't be
+    // testing what we expect it to.
+
+    let client = new RemoteSettingsClient(TEST_COLLECTION);
+
+    const dump = await SharedUtils.loadJSONDump(TEST_BUCKET, TEST_COLLECTION);
+    let DUMP_LAST_MODIFIED = dump.timestamp;
+
+    // Dump is updated regularly, verify that the dump matches our expectations
+    // before running the test.
+    Assert.greater(
+      DUMP_LAST_MODIFIED,
+      1234,
+      "Assuming dump to be newer than dummy 1234"
+    );
+
+    await client.db.clear();
+    await importData([{ last_modified: 1234, id: "dummy" }]);
+
+    const after = await client.get();
+    Assert.deepEqual(
+      after,
+      [{ last_modified: 1234, id: "dummy" }],
+      "Should have kept the original import"
+    );
+    Assert.equal(
+      await client.getLastModified(),
+      1234,
+      "Should have kept the import's timestamp"
+    );
+    await client.db.clear();
+  }
+);
+add_task(clear_state);
+
+add_task(
+  async function test_dumps_are_loaded_if_server_is_prod_if_running_for_devtools() {
+    Services.env.set("MOZ_REMOTE_SETTINGS_DEVTOOLS", "1");
+    Services.prefs.setStringPref(
+      "services.settings.server",
+      AppConstants.REMOTE_SETTINGS_SERVER_URLS[0]
+    );
+
+    const Utils = getNewUtils();
+
+    Assert.ok(Utils.LOAD_DUMPS, "dumps are loaded if prod");
+  }
+);
+add_task(clear_state);

src/zen/tests/mochitests/remote-settings/test_remote_settings_signatures.js

@@ -0,0 +1,952 @@
+/* import-globals-from ../../../common/tests/unit/head_helpers.js */
+"use strict";
+
+const PREF_SETTINGS_SERVER = "services.settings.server";
+const SIGNER_NAME = "onecrl.content-signature.mozilla.org";
+
+const CERT_DIR = "test_remote_settings_signatures/";
+const CHAIN_FILES = ["collection_signing_ee.pem", "collection_signing_int.pem"];
+
+function getFileData(file) {
+  const stream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance(
+    Ci.nsIFileInputStream
+  );
+  stream.init(file, -1, 0, 0);
+  const data = NetUtil.readInputStreamToString(stream, stream.available());
+  stream.close();
+  return data;
+}
+
+function getCertChain() {
+  const chain = [];
+  for (let file of CHAIN_FILES) {
+    chain.push(getFileData(do_get_file(CERT_DIR + file)));
+  }
+  return chain.join("\n");
+}
+
+let server;
+let client;
+
+add_setup(() => {
+  // Signature verification is enabled by default. We use a custom signer
+  // because these tests were originally written for OneCRL.
+  client = RemoteSettings("signed", { signerName: SIGNER_NAME });
+
+  Services.prefs.setStringPref("services.settings.loglevel", "debug");
+
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  registerCleanupFunction(() => {
+    Services.prefs.clearUserPref("services.settings.loglevel");
+    Services.prefs.clearUserPref(PREF_SETTINGS_SERVER);
+    server.stop(() => {});
+  });
+});
+
+add_task(async function test_check_signatures() {
+  // First, perform a signature verification with known data and signature
+  // to ensure things are working correctly
+  let verifier = Cc[
+    "@mozilla.org/security/contentsignatureverifier;1"
+  ].createInstance(Ci.nsIContentSignatureVerifier);
+
+  const emptyData = "[]";
+  const emptySignature =
+    "p384ecdsa=zbugm2FDitsHwk5-IWsas1PpWwY29f0Fg5ZHeqD8fzep7AVl2vfcaHA7LdmCZ28qZLOioGKvco3qT117Q4-HlqFTJM7COHzxGyU2MMJ0ZTnhJrPOC1fP3cVQjU1PTWi9";
+
+  ok(
+    await verifier.asyncVerifyContentSignature(
+      emptyData,
+      emptySignature,
+      getCertChain(),
+      SIGNER_NAME,
+      Ci.nsIX509CertDB.AppXPCShellRoot
+    )
+  );
+
+  const collectionData =
+    '[{"details":{"bug":"https://bugzilla.mozilla.org/show_bug.cgi?id=1155145","created":"2016-01-18T14:43:37Z","name":"GlobalSign certs","who":".","why":"."},"enabled":true,"id":"97fbf7c4-3ef2-f54f-0029-1ba6540c63ea","issuerName":"MHExKDAmBgNVBAMTH0dsb2JhbFNpZ24gUm9vdFNpZ24gUGFydG5lcnMgQ0ExHTAbBgNVBAsTFFJvb3RTaWduIFBhcnRuZXJzIENBMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMQswCQYDVQQGEwJCRQ==","last_modified":2000,"serialNumber":"BAAAAAABA/A35EU="},{"details":{"bug":"https://bugzilla.mozilla.org/show_bug.cgi?id=1155145","created":"2016-01-18T14:48:11Z","name":"GlobalSign certs","who":".","why":"."},"enabled":true,"id":"e3bd531e-1ee4-7407-27ce-6fdc9cecbbdc","issuerName":"MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB","last_modified":3000,"serialNumber":"BAAAAAABI54PryQ="}]';
+  const collectionSignature =
+    "p384ecdsa=f4pA2tYM5jQgWY6YUmhUwQiBLj6QO5sHLD_5MqLePz95qv-7cNCuQoZnPQwxoptDtW8hcWH3kLb0quR7SB-r82gkpR9POVofsnWJRA-ETb0BcIz6VvI3pDT49ZLlNg3p";
+
+  ok(
+    await verifier.asyncVerifyContentSignature(
+      collectionData,
+      collectionSignature,
+      getCertChain(),
+      SIGNER_NAME,
+      Ci.nsIX509CertDB.AppXPCShellRoot
+    )
+  );
+});
+
+add_task(async function test_bad_signature_does_not_lead_to_empty_list() {
+  Services.prefs.setStringPref(
+    PREF_SETTINGS_SERVER,
+    `http://localhost:${server.identity.primaryPort}/v1`
+  );
+  const x5u = `http://localhost:${server.identity.primaryPort}/x5u.pem`;
+
+  const networkCalls = [];
+
+  server.registerPathHandler(
+    "/v1/buckets/monitor/collections/changes/changeset",
+    (request, response) => {
+      response.write(
+        JSON.stringify({
+          changes: [
+            {
+              bucket: "main",
+              collection: "no-dump-no-local-data",
+              last_modified: 42,
+            },
+          ],
+        })
+      );
+      response.setHeader("Content-Type", "application/json; charset=UTF-8");
+      response.setStatusLine(null, 200, "OK");
+    }
+  );
+  server.registerPathHandler(
+    "/v1/buckets/monitor/collections/changes/changeset",
+    (request, response) => {
+      networkCalls.push(request);
+      response.write(
+        JSON.stringify({
+          changes: [
+            {
+              bucket: "main",
+              collection: "no-dump-no-local-data",
+              last_modified: 42,
+            },
+          ],
+        })
+      );
+      response.setHeader("Content-Type", "application/json; charset=UTF-8");
+      response.setStatusLine(null, 200, "OK");
+    }
+  );
+  server.registerPathHandler(
+    "/v1/buckets/main/collections/no-dump-no-local-data/changeset",
+    (request, response) => {
+      response.write(
+        JSON.stringify({
+          timestamp: 42,
+          changes: [],
+          metadata: {
+            signatures: [
+              {
+                signature: "bad-signature",
+                x5u,
+              },
+            ],
+          },
+        })
+      );
+      response.setHeader("Content-Type", "application/json; charset=UTF-8");
+      response.setStatusLine(null, 200, "OK");
+    }
+  );
+  server.registerPathHandler("/x5u.pem", (request, response) => {
+    response.write(getCertChain()); // At least cert will be valid.
+    response.setHeader("Content-Type", "text/plain; charset=UTF-8");
+    response.setStatusLine(null, 200, "OK");
+  });
+
+  const clientEmpty = RemoteSettings("no-dump-no-local-data");
+  clientEmpty.verifySignature = true; // default
+
+  // Check that client.get() will initiate a sync,
+  // and that it will throw since the signature is bad,
+  // and not return an empty list (`emptyListFallback: false`)
+  let error;
+  try {
+    await clientEmpty.get({
+      emptyListFallback: false,
+      syncIfEmpty: true, // default value
+    });
+  } catch (exc) {
+    error = exc;
+  }
+  equal(error.name, "InvalidSignatureError");
+
+  // Even running client.sync() will throw and won't leave
+  // anything in the database.
+  error = null;
+  try {
+    await clientEmpty.sync();
+  } catch (exc) {
+    error = exc;
+  }
+  equal(error.name, "InvalidSignatureError");
+  equal(await clientEmpty.db.getLastModified(), null);
+
+  // Call .get() again will initiate another sync.
+  networkCalls.length = 0;
+  try {
+    await clientEmpty.get({
+      emptyListFallback: false,
+      syncIfEmpty: true, // default value
+    });
+  } catch (exc) {
+    error = exc;
+  }
+  Assert.greater(networkCalls.length, 0, "Network calls were made");
+  equal(error.name, "InvalidSignatureError");
+});
+
+add_task(async function test_check_synchronization_with_signatures() {
+  const port = server.identity.primaryPort;
+
+  const x5u = `http://localhost:${port}/test_remote_settings_signatures/test_cert_chain.pem`;
+
+  // Telemetry reports.
+  const TELEMETRY_SOURCE = client.identifier;
+
+  function registerHandlers(responses) {
+    function handleResponse(serverTimeMillis, request, response) {
+      const key = `${request.method}:${request.path}?${request.queryString}`;
+      const available = responses[key];
+      const sampled = available.length > 1 ? available.shift() : available[0];
+      if (!sampled) {
+        do_throw(
+          `unexpected ${request.method} request for ${request.path}?${request.queryString}`
+        );
+      }
+
+      response.setStatusLine(
+        null,
+        sampled.status.status,
+        sampled.status.statusText
+      );
+      // send the headers
+      for (let headerLine of sampled.sampleHeaders) {
+        let headerElements = headerLine.split(":");
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+
+      // set the server date
+      response.setHeader("Date", new Date(serverTimeMillis).toUTCString());
+
+      response.write(sampled.responseBody);
+    }
+
+    for (let key of Object.keys(responses)) {
+      const keyParts = key.split(":");
+      const valueParts = keyParts[1].split("?");
+      const path = valueParts[0];
+
+      server.registerPathHandler(path, handleResponse.bind(null, 2000));
+    }
+  }
+
+  // set up prefs so the kinto updater talks to the test server
+  Services.prefs.setStringPref(
+    PREF_SETTINGS_SERVER,
+    `http://localhost:${server.identity.primaryPort}/v1`
+  );
+
+  // These are records we'll use in the test collections
+  const RECORD1 = {
+    details: {
+      bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
+      created: "2016-01-18T14:43:37Z",
+      name: "GlobalSign certs",
+      who: ".",
+      why: ".",
+    },
+    enabled: true,
+    id: "97fbf7c4-3ef2-f54f-0029-1ba6540c63ea",
+    issuerName:
+      "MHExKDAmBgNVBAMTH0dsb2JhbFNpZ24gUm9vdFNpZ24gUGFydG5lcnMgQ0ExHTAbBgNVBAsTFFJvb3RTaWduIFBhcnRuZXJzIENBMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMQswCQYDVQQGEwJCRQ==",
+    last_modified: 2000,
+    serialNumber: "BAAAAAABA/A35EU=",
+  };
+
+  const RECORD2 = {
+    details: {
+      bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
+      created: "2016-01-18T14:48:11Z",
+      name: "GlobalSign certs",
+      who: ".",
+      why: ".",
+    },
+    enabled: true,
+    id: "e3bd531e-1ee4-7407-27ce-6fdc9cecbbdc",
+    issuerName:
+      "MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB",
+    last_modified: 3000,
+    serialNumber: "BAAAAAABI54PryQ=",
+  };
+
+  const RECORD3 = {
+    details: {
+      bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
+      created: "2016-01-18T14:48:11Z",
+      name: "GlobalSign certs",
+      who: ".",
+      why: ".",
+    },
+    enabled: true,
+    id: "c7c49b69-a4ab-418e-92a9-e1961459aa7f",
+    issuerName:
+      "MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB",
+    last_modified: 4000,
+    serialNumber: "BAAAAAABI54PryQ=",
+  };
+
+  const RECORD1_DELETION = {
+    deleted: true,
+    enabled: true,
+    id: "97fbf7c4-3ef2-f54f-0029-1ba6540c63ea",
+    last_modified: 3500,
+  };
+
+  // Check that a signature on an empty collection is OK
+  // We need to set up paths on the HTTP server to return specific data from
+  // specific paths for each test. Here we prepare data for each response.
+
+  // A cert chain response (this the cert chain that contains the signing
+  // cert, the root and any intermediates in between). This is used in each
+  // sync.
+  const RESPONSE_CERT_CHAIN = {
+    comment: "RESPONSE_CERT_CHAIN",
+    sampleHeaders: ["Content-Type: text/plain; charset=UTF-8"],
+    status: { status: 200, statusText: "OK" },
+    responseBody: getCertChain(),
+  };
+
+  // A server settings response. This is used in each sync.
+  const RESPONSE_SERVER_SETTINGS = {
+    comment: "RESPONSE_SERVER_SETTINGS",
+    sampleHeaders: [
+      "Access-Control-Allow-Origin: *",
+      "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+      "Content-Type: application/json; charset=UTF-8",
+      "Server: waitress",
+    ],
+    status: { status: 200, statusText: "OK" },
+    responseBody: JSON.stringify({
+      settings: {
+        batch_max_requests: 25,
+      },
+      url: `http://localhost:${port}/v1/`,
+      documentation: "https://kinto.readthedocs.org/",
+      version: "1.5.1",
+      commit: "cbc6f58",
+      hello: "kinto",
+    }),
+  };
+
+  // This is the initial, empty state of the collection. This is only used
+  // for the first sync.
+  const RESPONSE_EMPTY_INITIAL = {
+    comment: "RESPONSE_EMPTY_INITIAL",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      'ETag: "1000"',
+    ],
+    status: { status: 200, statusText: "OK" },
+    responseBody: JSON.stringify({
+      timestamp: 1000,
+      metadata: {
+        signatures: [
+          {
+            x5u,
+            signature:
+              "vxuAg5rDCB-1pul4a91vqSBQRXJG_j7WOYUTswxRSMltdYmbhLRH8R8brQ9YKuNDF56F-w6pn4HWxb076qgKPwgcEBtUeZAO_RtaHXRkRUUgVzAr86yQL4-aJTbv3D6u",
+          },
+        ],
+      },
+      changes: [],
+    }),
+  };
+
+  // Here, we map request method and path to the available responses
+  const emptyCollectionResponses = {
+    "GET:/test_remote_settings_signatures/test_cert_chain.pem?": [
+      RESPONSE_CERT_CHAIN,
+    ],
+    "GET:/v1/?": [RESPONSE_SERVER_SETTINGS],
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=1000": [
+      RESPONSE_EMPTY_INITIAL,
+    ],
+  };
+
+  //
+  // 1.
+  // - collection: undefined -> []
+  // - timestamp: undefined -> 1000
+  //
+
+  // .. and use this map to register handlers for each path
+  registerHandlers(emptyCollectionResponses);
+
+  // Clear events snapshot.
+  Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_ALL_CHANNELS, true);
+  Services.fog.testResetFOG();
+  enableUptakeMetric();
+
+  // With all of this set up, we attempt a sync. This will resolve if all is
+  // well and throw if something goes wrong.
+  await client.maybeSync(1000);
+
+  equal((await client.get()).length, 0);
+
+  assertTelemetryEvents([
+    {
+      value: UptakeTelemetry.STATUS.SYNC_START,
+      source: TELEMETRY_SOURCE,
+      trigger: "manual",
+    },
+    {
+      value: UptakeTelemetry.STATUS.SUCCESS,
+      source: TELEMETRY_SOURCE,
+      trigger: "manual",
+    },
+  ]);
+
+  //
+  // 2.
+  // - collection: [] -> [RECORD2, RECORD1]
+  // - timestamp: 1000 -> 3000
+  //
+  // Check that some additions (2 records) to the collection have a valid
+  // signature.
+
+  // This response adds two entries (RECORD1 and RECORD2) to the collection
+  const RESPONSE_TWO_ADDED = {
+    comment: "RESPONSE_TWO_ADDED",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      'ETag: "3000"',
+    ],
+    status: { status: 200, statusText: "OK" },
+    responseBody: JSON.stringify({
+      timestamp: 3000,
+      metadata: {
+        signatures: [
+          {
+            x5u,
+            signature:
+              "dwhJeypadNIyzGj3QdI0KMRTPnHhFPF_j73mNrsPAHKMW46S2Ftf4BzsPMvPMB8h0TjDus13wo_R4l432DHe7tYyMIWXY0PBeMcoe5BREhFIxMxTsh9eGVXBD1e3UwRy",
+          },
+        ],
+      },
+      changes: [RECORD2, RECORD1],
+    }),
+  };
+
+  const twoItemsResponses = {
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=3000&_since=1000":
+      [RESPONSE_TWO_ADDED],
+  };
+  registerHandlers(twoItemsResponses);
+  await client.maybeSync(3000);
+
+  equal((await client.get()).length, 2);
+
+  //
+  // 3.
+  // - collection: [RECORD2, RECORD1] -> [RECORD2, RECORD3]
+  // - timestamp: 3000 -> 4000
+  //
+  // Check the collection with one addition and one removal has a valid
+  // signature
+  const THREE_ITEMS_SIG =
+    "MIEmNghKnkz12UodAAIc3q_Y4a3IJJ7GhHF4JYNYmm8avAGyPM9fYU7NzVo94pzjotG7vmtiYuHyIX2rTHTbT587w0LdRWxipgFd_PC1mHiwUyjFYNqBBG-kifYk7kEw";
+
+  // Remove RECORD1, add RECORD3
+  const RESPONSE_ONE_ADDED_ONE_REMOVED = {
+    comment: "RESPONSE_ONE_ADDED_ONE_REMOVED ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      'ETag: "4000"',
+    ],
+    status: { status: 200, statusText: "OK" },
+    responseBody: JSON.stringify({
+      timestamp: 4000,
+      metadata: {
+        signatures: [
+          {
+            x5u,
+            signature: THREE_ITEMS_SIG,
+          },
+        ],
+      },
+      changes: [RECORD3, RECORD1_DELETION],
+    }),
+  };
+
+  const oneAddedOneRemovedResponses = {
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=4000&_since=3000":
+      [RESPONSE_ONE_ADDED_ONE_REMOVED],
+  };
+  registerHandlers(oneAddedOneRemovedResponses);
+  await client.maybeSync(4000);
+
+  equal((await client.get()).length, 2);
+
+  //
+  // 4.
+  // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3]
+  // - timestamp: 4000 -> 4100
+  //
+  // Check the signature is still valid with no operation (no changes)
+
+  // Leave the collection unchanged
+  const RESPONSE_EMPTY_NO_UPDATE = {
+    comment: "RESPONSE_EMPTY_NO_UPDATE ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      'ETag: "4000"',
+    ],
+    status: { status: 200, statusText: "OK" },
+    responseBody: JSON.stringify({
+      timestamp: 4000,
+      metadata: {
+        signatures: [
+          {
+            x5u,
+            signature: THREE_ITEMS_SIG,
+          },
+        ],
+      },
+      changes: [],
+    }),
+  };
+
+  const noOpResponses = {
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=4100&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE],
+  };
+  registerHandlers(noOpResponses);
+  await client.maybeSync(4100);
+
+  equal((await client.get()).length, 2);
+
+  //
+  // 5.
+  // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3]
+  // - timestamp: 4000 -> 5000
+  //
+  // Check the collection is reset when the signature is invalid.
+  // Client will:
+  //   - Fetch metadata (with bad signature)
+  //   - Perform the sync (fetch empty changes)
+  //   - Refetch the metadata and the whole collection
+  //   - Validate signature successfully, but with no changes to emit.
+
+  const RESPONSE_COMPLETE_INITIAL = {
+    comment: "RESPONSE_COMPLETE_INITIAL ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      'ETag: "4000"',
+    ],
+    status: { status: 200, statusText: "OK" },
+    responseBody: JSON.stringify({
+      timestamp: 4000,
+      metadata: {
+        signatures: [
+          {
+            x5u,
+            signature: THREE_ITEMS_SIG,
+          },
+        ],
+      },
+      changes: [RECORD2, RECORD3],
+    }),
+  };
+
+  const RESPONSE_EMPTY_NO_UPDATE_BAD_SIG = {
+    ...RESPONSE_EMPTY_NO_UPDATE,
+    responseBody: JSON.stringify({
+      timestamp: 4000,
+      metadata: {
+        signatures: [
+          {
+            x5u,
+            signature: "aW52YWxpZCBzaWduYXR1cmUK",
+          },
+        ],
+      },
+      changes: [],
+    }),
+  };
+
+  const badSigGoodSigResponses = {
+    // The first collection state is the three item collection (since
+    // there was sync with no updates before) - but, since the signature is wrong,
+    // another request will be made...
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE_BAD_SIG],
+    // Subsequent signature returned is a valid one for the three item
+    // collection.
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000": [
+      RESPONSE_COMPLETE_INITIAL,
+    ],
+  };
+
+  registerHandlers(badSigGoodSigResponses);
+  Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_ALL_CHANNELS, true);
+  Services.fog.testResetFOG();
+  enableUptakeMetric();
+
+  let syncEventSent = false;
+  client.on("sync", () => {
+    syncEventSent = true;
+  });
+
+  await client.maybeSync(5000);
+
+  equal((await client.get()).length, 2);
+
+  // since we only fixed the signature, and no data was changed, the sync event
+  // was not sent.
+  equal(syncEventSent, false);
+
+  // ensure that the failure count is incremented for a succesful sync with an
+  // (initial) bad signature - only SERVICES_SETTINGS_SYNC_SIG_FAIL should
+  // increment.
+  assertTelemetryEvents([
+    {
+      value: UptakeTelemetry.STATUS.SYNC_START,
+      source: TELEMETRY_SOURCE,
+      trigger: "manual",
+    },
+    {
+      value: UptakeTelemetry.STATUS.SIGNATURE_ERROR,
+      source: TELEMETRY_SOURCE,
+      trigger: "manual",
+    },
+  ]);
+
+  //
+  // 6.
+  // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3]
+  // - timestamp: 4000 -> 5000
+  //
+  // Check the collection is reset when the signature is invalid.
+  // Client will:
+  //   - Fetch metadata (with bad signature)
+  //   - Perform the sync (fetch empty changes)
+  //   - Refetch the whole collection and metadata
+  //   - Sync will be no-op since local is equal to server, no changes to emit.
+
+  const badSigGoodOldResponses = {
+    // The first collection state is the current state (since there's no update
+    // - but, since the signature is wrong, another request will be made)
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE_BAD_SIG],
+    // The next request is for the full collection. This will be
+    // checked against the valid signature and last_modified times will be
+    // compared. Sync should be a no-op, even though the signature is good,
+    // because the local collection is newer.
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000": [
+      RESPONSE_EMPTY_INITIAL,
+    ],
+  };
+
+  // ensure our collection hasn't been replaced with an older, empty one
+  equal((await client.get()).length, 2, "collection was restored");
+
+  registerHandlers(badSigGoodOldResponses);
+
+  syncEventSent = false;
+  client.on("sync", () => {
+    syncEventSent = true;
+  });
+
+  await client.maybeSync(5000);
+
+  // Local data was unchanged, since it was never than the one returned by the server,
+  // thus the sync event is not sent.
+  equal(syncEventSent, false, "event was not sent");
+
+  //
+  // 7.
+  // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3]
+  // - timestamp: 4000 -> 5000
+  //
+  // Check that a tampered local DB will be overwritten and
+  // sync event contain the appropriate data.
+
+  const RESPONSE_COMPLETE_BAD_SIG = {
+    ...RESPONSE_EMPTY_NO_UPDATE,
+    responseBody: JSON.stringify({
+      timestamp: 5000,
+      metadata: {
+        signatures: [
+          {
+            x5u,
+            signature: "aW52YWxpZCBzaWduYXR1cmUK",
+          },
+        ],
+      },
+      changes: [RECORD2, RECORD3],
+    }),
+  };
+
+  const badLocalContentGoodSigResponses = {
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000&_since=3900":
+      [RESPONSE_COMPLETE_BAD_SIG],
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000": [
+      RESPONSE_COMPLETE_INITIAL,
+    ],
+  };
+
+  registerHandlers(badLocalContentGoodSigResponses);
+
+  // we create a local state manually here, in order to test that the sync event data
+  // properly contains created, updated, and deleted records.
+  // the local DB contains same id as RECORD2 and a fake record.
+  // the final server collection contains RECORD2 and RECORD3
+  const localId = "0602b1b2-12ab-4d3a-b6fb-593244e7b035";
+  await client.db.importChanges(
+    { signatures: [{ x5u, signature: "abc" }] },
+    3900,
+    [
+      { ...RECORD2, last_modified: 1234567890, serialNumber: "abc" },
+      { id: localId },
+    ],
+    {
+      clear: true,
+    }
+  );
+
+  let syncData = null;
+  client.on("sync", ({ data }) => {
+    syncData = data;
+  });
+
+  // Clear events snapshot.
+  Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_ALL_CHANNELS, true);
+  Services.fog.testResetFOG();
+  enableUptakeMetric();
+
+  // Events telemetry is sampled on released, use fake channel.
+  await client.maybeSync(5000);
+
+  // We should report a corruption_error.
+  assertTelemetryEvents([
+    {
+      value: UptakeTelemetry.STATUS.SYNC_START,
+      source: client.identifier,
+      trigger: "manual",
+    },
+    {
+      value: UptakeTelemetry.STATUS.CORRUPTION_ERROR,
+      source: client.identifier,
+      trigger: "manual",
+      duration: d => parseInt(d) > 0,
+    },
+  ]);
+
+  // The local data was corrupted, and the Telemetry status reflects it.
+  // But the sync overwrote the bad data and was eventually a success.
+  // Since local data was replaced, we use records IDs to determine
+  // what was created and deleted. And bad local data will appear
+  // in the sync event as deleted.
+  equal(syncData.current.length, 2);
+  equal(syncData.created.length, 1);
+  equal(syncData.created[0].id, RECORD3.id);
+  equal(syncData.updated.length, 1);
+  equal(syncData.updated[0].old.serialNumber, "abc");
+  equal(syncData.updated[0].new.serialNumber, RECORD2.serialNumber);
+  equal(syncData.deleted.length, 1);
+  equal(syncData.deleted[0].id, localId);
+
+  //
+  // 8.
+  // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3] (unchanged because of error)
+  // - timestamp: 4000 -> 6000
+  //
+  // Check that a failing signature throws after retry, and that sync changes
+  // are not applied.
+
+  const RESPONSE_ONLY_RECORD4_BAD_SIG = {
+    comment: "Create RECORD4",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      'ETag: "6000"',
+    ],
+    status: { status: 200, statusText: "OK" },
+    responseBody: JSON.stringify({
+      timestamp: 6000,
+      metadata: {
+        signatures: [
+          {
+            x5u,
+            signature: "aaaaaaaaaaaaaaaaaaaaaaaa", // sig verifier wants proper length or will crash.
+          },
+        ],
+      },
+      changes: [
+        {
+          id: "f765df30-b2f1-42f6-9803-7bd5a07b5098",
+          last_modified: 6000,
+        },
+      ],
+    }),
+  };
+  const RESPONSE_EMPTY_NO_UPDATE_BAD_SIG_6000 = {
+    ...RESPONSE_EMPTY_NO_UPDATE,
+    responseBody: JSON.stringify({
+      timestamp: 6000,
+      metadata: {
+        signatures: [
+          {
+            x5u,
+            signature: "aW52YWxpZCBzaWduYXR1cmUK",
+          },
+        ],
+      },
+      changes: [],
+    }),
+  };
+  const allBadSigResponses = {
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=6000&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE_BAD_SIG_6000],
+    "GET:/v1/buckets/main/collections/signed/changeset?_expected=6000": [
+      RESPONSE_ONLY_RECORD4_BAD_SIG,
+    ],
+  };
+
+  // Reset telemetry capture.
+  Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_ALL_CHANNELS, true);
+  Services.fog.testResetFOG();
+  enableUptakeMetric();
+
+  registerHandlers(allBadSigResponses);
+  await Assert.rejects(
+    client.maybeSync(6000),
+    RemoteSettingsClient.InvalidSignatureError,
+    "Sync failed as expected (bad signature after retry)"
+  );
+
+  // Ensure that the failure is reflected in the accumulated telemetry:
+  assertTelemetryEvents([
+    {
+      value: UptakeTelemetry.STATUS.SYNC_START,
+      source: TELEMETRY_SOURCE,
+      trigger: "manual",
+    },
+    {
+      value: UptakeTelemetry.STATUS.SIGNATURE_RETRY_ERROR,
+      source: TELEMETRY_SOURCE,
+      trigger: "manual",
+    },
+  ]);
+
+  // When signature fails after retry, the local data present before sync
+  // should be maintained (if its signature is valid).
+  ok(
+    arrayEqual(
+      (await client.get()).map(r => r.id),
+      [RECORD3.id, RECORD2.id]
+    ),
+    "Local records were not changed"
+  );
+  // And local data should still be valid.
+  await client.get({ verifySignature: true }); // Not raising.
+
+  //
+  // 9.
+  // - collection: [RECORD2, RECORD3] -> [] (cleared)
+  // - timestamp: 4000 -> 6000
+  //
+  // Check that local data is cleared during sync if signature is not valid.
+
+  await client.db.create({
+    id: "c6b19c67-2e0e-4a82-b7f7-1777b05f3e81",
+    last_modified: 42,
+    tampered: true,
+  });
+
+  await Assert.rejects(
+    client.maybeSync(6000),
+    RemoteSettingsClient.InvalidSignatureError,
+    "Sync failed as expected (bad signature after retry)"
+  );
+
+  // Since local data was tampered, it was cleared.
+  equal((await client.get()).length, 0, "Local database is now empty.");
+
+  //
+  // 10.
+  // - collection: [RECORD2, RECORD3] -> [] (cleared)
+  // - timestamp: 4000 -> 6000
+  //
+  // Check that local data is cleared during sync if signature is not valid.
+
+  await client.db.create({
+    id: "c6b19c67-2e0e-4a82-b7f7-1777b05f3e81",
+    last_modified: 42,
+    tampered: true,
+  });
+
+  await Assert.rejects(
+    client.maybeSync(6000),
+    RemoteSettingsClient.InvalidSignatureError,
+    "Sync failed as expected (bad signature after retry)"
+  );
+  // Since local data was tampered, it was cleared.
+  equal((await client.get()).length, 0, "Local database is now empty.");
+
+  //
+  // 11.
+  // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3]
+  // - timestamp: 4000 -> 6000
+  //
+  // Check that local data is restored if signature was valid before sync.
+  const sigCalls = [];
+  let i = 0;
+  client._verifier = {
+    async asyncVerifyContentSignature(serialized) {
+      sigCalls.push(serialized);
+      console.log(`verify call ${i}`);
+      return [
+        false, // After importing changes.
+        true, // When checking previous local data.
+        false, // Still fail after retry.
+        true, // When checking previous local data again.
+      ][i++];
+    },
+  };
+  // Create an extra record. It will have a valid signature locally
+  // thanks to the verifier mock.
+  await client.db.importChanges(
+    {
+      signatures: [{ x5u, signature: "aa" }],
+    },
+    4000,
+    [
+      {
+        id: "extraId",
+        last_modified: 42,
+      },
+    ]
+  );
+
+  equal((await client.get()).length, 1);
+
+  // Now sync, but importing changes will have failing signature,
+  // and so will retry (see `sigResults`).
+  await Assert.rejects(
+    client.maybeSync(6000),
+    RemoteSettingsClient.InvalidSignatureError,
+    "Sync failed as expected (bad signature after retry)"
+  );
+  equal(i, 4, "sync has retried as expected");
+
+  // Make sure that we retried on a blank DB. The extra record should
+  // have been deleted when we validated the signature the second time.
+  // Since local data was tampered, it was cleared.
+  ok(/extraId/.test(sigCalls[0]), "extra record when importing changes");
+  ok(/extraId/.test(sigCalls[1]), "extra record when checking local");
+  ok(!/extraId/.test(sigCalls[2]), "db was flushed before retry");
+  ok(/extraId/.test(sigCalls[3]), "when checking local after retry");
+});

src/zen/tests/mochitests/remote-settings/test_remote_settings_signatures/collection_signing_ee.pem

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICdTCCAV2gAwIBAgIUdVZo8xeazUY3Fzm6cy61HaSjFJUwDQYJKoZIhvcNAQEL
+BQAwIzEhMB8GA1UEAwwYY29sbGVjdGlvbi1zaWduZXItaW50LUNBMCIYDzIwMjQx
+MTI3MDAwMDAwWhgPMjAyNzAyMDUwMDAwMDBaMCYxJDAiBgNVBAMMG2NvbGxlY3Rp
+b24tc2lnbmVyLWVlLWludC1DQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABKFockM2
+K1x7GInzeRVGFaHHP7SN7oY+AikV22COJS3ktxMtqM6Y6DFTTmqcDAsJyNY5regy
+BuW6gTRzoR+jMOBdqMluQ4P+J4c9qXEDviiIz/AC8Fr3Gh/dzIN0qm6pzqNIMEYw
+EwYDVR0lBAwwCgYIKwYBBQUHAwMwLwYDVR0RBCgwJoIkb25lY3JsLmNvbnRlbnQt
+c2lnbmF0dXJlLm1vemlsbGEub3JnMA0GCSqGSIb3DQEBCwUAA4IBAQBVHMPKgXKX
+EVVTDRw8lF0MQBHUPA2+4f9JDsaq5AexGpFqfDzFobrzZnzKQyaODeoalhASraWs
+GvhXrXFeSkSLqNpw657JXuIvVOULJjr2h9uA9N837KgFosvLLIIoKs1zeXG0LSYy
+NSJBSme+n8ETV2d8RJ/w575rhnheDz6GcVH0hduvQR0g7dg8ZMtdfi16sjKaJRup
+coH4N8t4YhLqM4PyrEATZyTEW5jOCKCb91FWgPu0ovfQz+t3jjS7TPpUa3IYJS8V
+eFUHXUAXFFgD0xCl7P/V+kopWSijZHT8mwnPfn2z9EmV5ijioIsw2MF+FACkPWa1
+o2NzujCdDAEl
+-----END CERTIFICATE-----

src/zen/tests/mochitests/remote-settings/test_remote_settings_signatures/collection_signing_ee.pem.certspec

@@ -0,0 +1,5 @@
+issuer:collection-signer-int-CA
+subject:collection-signer-ee-int-CA
+subjectKey:secp384r1
+extension:extKeyUsage:codeSigning
+extension:subjectAlternativeName:onecrl.content-signature.mozilla.org

src/zen/tests/mochitests/remote-settings/test_remote_settings_signatures/collection_signing_int.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDAzCCAeugAwIBAgIUf3MHKqXo3U9dG7tkPLy8t7D+1o4wDQYJKoZIhvcNAQEL
+BQAwKTEnMCUGA1UEAwweeHBjc2hlbGwgc2lnbmVkIGFwcHMgdGVzdCByb290MCIY
+DzIwMjQxMTI3MDAwMDAwWhgPMjAyNzAyMDUwMDAwMDBaMCMxITAfBgNVBAMMGGNv
+bGxlY3Rpb24tc2lnbmVyLWludC1DQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALqIUahEjhbWQf1utogGNhA9PBPZ6uQ1SrTs9WhXbCR7wcclqODYH72x
+nAabbhqG8mvir1p1a2pkcQh6pVqnRYf3HNUknAJ+zUP8HmnQOCApk6sgw0nk27lM
+wmtsDu0Vgg/xfq1pGrHTAjqLKkHup3DgDw2N/WYLK7AkkqR9uYhheZCxV5A90jvF
+4LhIH6g304hD7ycW2FW3ZlqqfgKQLzp7EIAGJMwcbJetlmFbt+KWEsB1MaMMkd20
+yvf8rR0l0wnvuRcOp2jhs3svIm9p47SKlWEd7ibWJZ2rkQhONsscJAQsvxaLL+Xx
+j5kXMbiz/kkj+nJRxDHVA6zaGAo17Y0CAwEAAaMlMCMwDAYDVR0TBAUwAwEB/zAT
+BgNVHSUEDDAKBggrBgEFBQcDAzANBgkqhkiG9w0BAQsFAAOCAQEAjJAZX8SZOh1H
+xLqKHgt2EiFloIW3k3aAnTFQPAEUp+IBXjJdA8cFzEDSHo0dTr8jBhR8V1gwy8+Z
+1i6loA0rgJUdDhUP24SqbfsdF6kcltlBfjR4lBfbZq0BClMiNDL2ZOQS4neF15fN
+WXUv2UE9ooKNUjgIzXHbU/ykbwTKB7Bmsh+r5CYs9LMNF5P/QDAlDNwvZglxiyyq
+61fi06lnBXff0zObSdV/8Wm0PQyHEBfNY3L94wuIFxGsX0YngO7CCtaIFIeymzob
+8SvH4W0vfeqYzDFM4PfgFbDs4KnAzJS+WwWq2HRDsHMZTDZVVXVA61LXYhFEOam1
+ASCDwhnClA==
+-----END CERTIFICATE-----

src/zen/tests/mochitests/remote-settings/test_remote_settings_signatures/collection_signing_int.pem.certspec

@@ -0,0 +1,4 @@
+issuer:xpcshell signed apps test root
+subject:collection-signer-int-CA
+extension:basicConstraints:cA,
+extension:extKeyUsage:codeSigning

src/zen/tests/mochitests/remote-settings/test_remote_settings_startup_bundle.js

@@ -0,0 +1,152 @@
+const { Database } = ChromeUtils.importESModule(
+  "resource://services-settings/Database.sys.mjs"
+);
+const { RemoteSettings } = ChromeUtils.importESModule(
+  "resource://services-settings/remote-settings.sys.mjs"
+);
+
+let server;
+let startupClients;
+
+add_setup(async () => {
+  // Disable signature verification of collections listed in test startup.json.mzlz4
+  // since their metadata will eventually expire.
+  startupClients = [
+    "message-groups",
+    "nimbus-desktop-experiments",
+    "search-categorization",
+    "tracking-protection-lists",
+    "query-stripping",
+    "cfr",
+  ].map(id => {
+    const c = RemoteSettings(id);
+    c.verifySignature = false;
+    return c;
+  });
+
+  // Setup an HTTP server to serve the 'startup.json.mzlz4' bundle.
+  server = new HttpServer();
+  server.start(-1);
+  server.registerDirectory(
+    "/cdn/bundles/",
+    do_get_file("test_remote_settings_startup_bundle")
+  );
+  server.registerPathHandler("/v1/", (request, response) => {
+    response.write(
+      JSON.stringify({
+        capabilities: {
+          attachments: {
+            base_url: `http://localhost:${server.identity.primaryPort}/cdn/`,
+          },
+        },
+      })
+    );
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setStatusLine(null, 200, "OK");
+  });
+  Services.prefs.setStringPref(
+    "services.settings.server",
+    `http://localhost:${server.identity.primaryPort}/v1`
+  );
+  registerCleanupFunction(() => {
+    server.stop(() => {});
+    Services.prefs.clearUserPref("services.settings.loglevel");
+  });
+});
+
+async function clear_state() {
+  await Database.destroy();
+  RemoteSettings._ongoingExtractBundlePromise = null;
+}
+
+add_task(async function test_bundle_is_pulled_when_get_needs_sync() {
+  const client = startupClients[0];
+
+  Assert.ok(
+    !(await Utils.hasLocalDump(client.bucketName, client.collectionName)),
+    "Client has no packaged dump"
+  );
+
+  const records = await client.get();
+
+  Assert.equal(records.length, 6, "Records were read from startup bundle");
+});
+add_task(clear_state);
+
+add_task(
+  async function test_signature_of_extracted_data_from_bundle_is_verified() {
+    const c = RemoteSettings("tracking-protection-lists"); // part of startup.json.mzlz4
+    c.verifySignature = true;
+
+    let called = null;
+    c.validateCollectionSignature = (records, timestamp, metadata) => {
+      called = { records, timestamp, metadata };
+    };
+
+    await c.get();
+
+    Assert.ok(!!called.records.length);
+    Assert.greaterOrEqual(called.timestamp, 1694684362860);
+    Assert.ok(called.metadata.flags.includes("startup"));
+  }
+);
+add_task(clear_state);
+
+add_task(async function test_bundle_is_not_importent_when_signature_fails() {
+  const c = RemoteSettings("tracking-protection-lists"); // part of startup.json.mzlz4
+  c.verifySignature = true;
+
+  let called = false;
+  c.validateCollectionSignature = () => {
+    called = true;
+    throw new Error("boom!");
+  };
+
+  let error;
+  try {
+    await c.get({ emptyListFallback: false });
+    Assert.ok(false, ".get() should fail without network");
+  } catch (e) {
+    error = e;
+  }
+  Assert.ok(called, "Signature was verified");
+  Assert.equal(
+    error.name,
+    "UnknownCollectionError",
+    ".get() fails without network and with bad startup bundle"
+  );
+});
+add_task(clear_state);
+
+add_task(async function test_sync_occurs_if_collection_not_part_of_bundle() {
+  const c = RemoteSettings("foo");
+
+  let error;
+  try {
+    await c.get({ emptyListFallback: false });
+    Assert.ok(false, ".get() should fail when bundle disabled");
+  } catch (e) {
+    error = e;
+  }
+  Assert.equal(
+    error.name,
+    "UnknownCollectionError",
+    ".get() fails to sync data"
+  );
+});
+add_task(clear_state);
+
+add_task(async function test_several_clients_wait_for_bundle() {
+  // several clients calling .get() in parallel will all take content from bundle.
+  const results = await Promise.allSettled(
+    startupClients.map(c => c.get({ emptyListFallback: false }))
+  );
+
+  Assert.deepEqual(
+    [6, 70, 5, "UnknownCollectionError", 3, 11],
+    results.map(({ status, value, reason }) =>
+      status == "fulfilled" ? value.length : reason.name
+    )
+  );
+});
+add_task(clear_state);

src/zen/tests/mochitests/remote-settings/test_remote_settings_sync_history.js

@@ -0,0 +1,65 @@
+"use strict";
+
+async function clear_state() {
+  await new SyncHistory("").clear();
+}
+add_task(clear_state);
+
+add_task(async function test_entries_are_stored_by_source() {
+  const history = new SyncHistory();
+  await history.store(42, "success", { pi: "3.14" });
+  // Check that history is isolated by source.
+  await new SyncHistory("main/cfr").store(88, "error");
+
+  const l = await history.list();
+
+  Assert.deepEqual(l, [
+    {
+      timestamp: 42,
+      status: "success",
+      infos: { pi: "3.14" },
+      datetime: new Date(42),
+    },
+  ]);
+});
+add_task(clear_state);
+
+add_task(
+  async function test_old_entries_are_removed_keep_fixed_size_per_source() {
+    const history = new SyncHistory("settings-sync", { size: 3 });
+    const anotherHistory = await new SyncHistory("main/cfr");
+
+    await history.store(42, "success");
+    await history.store(41, "sync_error");
+    await history.store(43, "up_to_date");
+
+    let l = await history.list();
+    Assert.equal(l.length, 3);
+
+    await history.store(44, "success");
+    await anotherHistory.store(44, "success");
+
+    l = await history.list();
+    Assert.equal(l.length, 3);
+    Assert.ok(!l.map(e => e.timestamp).includes(41));
+
+    l = await anotherHistory.list();
+    Assert.equal(l.length, 1);
+  }
+);
+add_task(clear_state);
+
+add_task(async function test_entries_are_sorted_by_timestamp_desc() {
+  const history = new SyncHistory("settings-sync");
+  await history.store(42, "success");
+  await history.store(41, "sync_error");
+  await history.store(44, "up_to_date");
+
+  const l = await history.list();
+
+  Assert.deepEqual(
+    l.map(e => e.timestamp),
+    [44, 42, 41]
+  );
+});
+add_task(clear_state);

src/zen/tests/mochitests/remote-settings/test_remote_settings_utils.js

@@ -0,0 +1,299 @@
+const BinaryOutputStream = Components.Constructor(
+  "@mozilla.org/binaryoutputstream;1",
+  "nsIBinaryOutputStream",
+  "setOutputStream"
+);
+
+const server = new HttpServer();
+server.start(-1);
+registerCleanupFunction(() => server.stop(() => {}));
+const SERVER_BASE_URL = `http://localhost:${server.identity.primaryPort}`;
+
+const proxyServer = new HttpServer();
+proxyServer.identity.add("http", "localhost", server.identity.primaryPort);
+proxyServer.start(-1);
+registerCleanupFunction(() => proxyServer.stop(() => {}));
+const PROXY_PORT = proxyServer.identity.primaryPort;
+
+// A sequence of bytes that would become garbage if it were to be read as UTF-8:
+// - 0xEF 0xBB 0xBF is a byte order mark.
+// - 0xC0 on its own is invalid (it's the first byte of a 2-byte encoding).
+const INVALID_UTF_8_BYTES = [0xef, 0xbb, 0xbf, 0xc0];
+
+server.registerPathHandler("/binary.dat", (request, response) => {
+  response.setStatusLine(null, 201, "StatusLineHere");
+  response.setHeader("headerName", "HeaderValue: HeaderValueEnd");
+  let binaryOut = new BinaryOutputStream(response.bodyOutputStream);
+  binaryOut.writeByteArray([0xef, 0xbb, 0xbf, 0xc0]);
+});
+
+// HTTPS requests are proxied with CONNECT, but our test server is HTTP,
+// which means that the proxy will receive GET http://localhost:port.
+var proxiedCount = 0;
+proxyServer.registerPrefixHandler("/", (request, response) => {
+  ++proxiedCount;
+  Assert.equal(request.path, "/binary.dat", `Proxy request ${proxiedCount}`);
+  // Close connection without sending any response.
+  response.seizePower();
+  response.finish();
+});
+
+add_task(async function test_utils_fetch_binary() {
+  let res = await Utils.fetch(`${SERVER_BASE_URL}/binary.dat`);
+
+  Assert.equal(res.status, 201, "res.status");
+  Assert.equal(res.statusText, "StatusLineHere", "res.statusText");
+  Assert.equal(
+    res.headers.get("headerName"),
+    "HeaderValue: HeaderValueEnd",
+    "Utils.fetch should return the header"
+  );
+
+  Assert.deepEqual(
+    Array.from(new Uint8Array(await res.arrayBuffer())),
+    INVALID_UTF_8_BYTES,
+    "Binary response body should be returned as is"
+  );
+});
+
+add_task(async function test_utils_fetch_binary_as_text() {
+  let res = await Utils.fetch(`${SERVER_BASE_URL}/binary.dat`);
+  Assert.deepEqual(
+    Array.from(await res.text(), c => c.charCodeAt(0)),
+    [65533],
+    "Interpreted as UTF-8, the response becomes garbage"
+  );
+});
+
+add_task(async function test_utils_fetch_binary_as_json() {
+  let res = await Utils.fetch(`${SERVER_BASE_URL}/binary.dat`);
+  await Assert.rejects(
+    res.json(),
+    /SyntaxError: JSON.parse: unexpected character/,
+    "Binary data is invalid JSON"
+  );
+});
+
+add_task(async function test_utils_fetch_has_conservative() {
+  let channelPromise = TestUtils.topicObserved("http-on-modify-request");
+  await Utils.fetch(`${SERVER_BASE_URL}/binary.dat`);
+
+  let channel = (await channelPromise)[0].QueryInterface(Ci.nsIHttpChannel);
+
+  Assert.equal(channel.URI.spec, `${SERVER_BASE_URL}/binary.dat`, "URL OK");
+
+  let internalChannel = channel.QueryInterface(Ci.nsIHttpChannelInternal);
+  Assert.ok(internalChannel.beConservative, "beConservative flag is set");
+});
+
+add_task(async function test_utils_fetch_has_conservative() {
+  let channelPromise = TestUtils.topicObserved("http-on-modify-request");
+  await Utils.fetch(`${SERVER_BASE_URL}/binary.dat`);
+
+  let channel = (await channelPromise)[0].QueryInterface(Ci.nsIHttpChannel);
+
+  Assert.equal(channel.URI.spec, `${SERVER_BASE_URL}/binary.dat`, "URL OK");
+
+  let internalChannel = channel.QueryInterface(Ci.nsIHttpChannelInternal);
+  Assert.ok(internalChannel.beConservative, "beConservative flag is set");
+});
+
+add_task(async function test_utils_fetch_with_bad_proxy() {
+  Services.prefs.setIntPref("network.proxy.type", 1);
+  Services.prefs.setStringPref("network.proxy.http", "127.0.0.1");
+  Services.prefs.setIntPref("network.proxy.http_port", PROXY_PORT);
+  Services.prefs.setBoolPref("network.proxy.allow_hijacking_localhost", true);
+
+  // The URL that we're going to request.
+  const DESTINATION_URL = `${SERVER_BASE_URL}/binary.dat`;
+
+  Assert.equal(proxiedCount, 0, "Proxy not used yet");
+  {
+    info("Bad proxy, default prefs");
+    let res = await Utils.fetch(DESTINATION_URL);
+    Assert.equal(res.status, 201, "Bypassed bad proxy");
+    // 10 instead of 1 because of reconnect attempts after a dropped request.
+    Assert.equal(proxiedCount, 10, "Proxy was used by HttpChannel");
+  }
+
+  // Disables the failover logic from HttpChannel.
+  Services.prefs.setBoolPref("network.proxy.failover_direct", false);
+  proxiedCount = 0;
+  {
+    info("Bad proxy, disabled network.proxy.failover_direct");
+    let res = await Utils.fetch(DESTINATION_URL);
+    Assert.equal(res.status, 201, "Bypassed bad proxy");
+    // 10 instead of 1 because of reconnect attempts after a dropped request.
+    Assert.equal(proxiedCount, 10, "Proxy was used by ServiceRequest");
+  }
+
+  proxiedCount = 0;
+  {
+    info("Using internal option of Utils.fetch: bypassProxy=true");
+    let res = await Utils.fetch(DESTINATION_URL, { bypassProxy: true });
+    Assert.equal(res.status, 201, "Bypassed bad proxy");
+    Assert.equal(proxiedCount, 0, "Not using proxy when bypassProxy=true");
+  }
+
+  // Disables the failover logic from ServiceRequest/Utils.fetch
+  Services.prefs.setBoolPref("network.proxy.allow_bypass", false);
+  proxiedCount = 0;
+
+  info("Bad proxy, disabled network.proxy.allow_bypass");
+  await Assert.rejects(
+    Utils.fetch(DESTINATION_URL),
+    /NetworkError/,
+    "Bad proxy request should fail without failover"
+  );
+  // 10 instead of 1 because of reconnect attempts after a dropped request.
+  Assert.equal(proxiedCount, 10, "Attempted to use proxy again");
+
+  Services.prefs.clearUserPref("network.proxy.type");
+  Services.prefs.clearUserPref("network.proxy.http");
+  Services.prefs.clearUserPref("network.proxy.http_port");
+  Services.prefs.clearUserPref("network.proxy.allow_hijacking_localhost");
+  Services.prefs.clearUserPref("network.proxy.failover_direct");
+  Services.prefs.clearUserPref("network.proxy.allow_bypass");
+});
+
+add_task(async function test_base_attachment_url_depends_on_server() {
+  Services.prefs.setStringPref(
+    "services.settings.server",
+    `http://localhost:${server.identity.primaryPort}/v1`
+  );
+  server.registerPathHandler("/v1/", (request, response) => {
+    response.write(
+      JSON.stringify({
+        capabilities: {
+          attachments: {
+            base_url: "http://default-url.com/",
+          },
+        },
+      })
+    );
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setStatusLine(null, 200, "OK");
+  });
+  const before = await Utils.baseAttachmentsURL();
+  Assert.equal(before, "http://default-url.com/");
+
+  Services.prefs.setStringPref(
+    "services.settings.server",
+    `http://localhost:${server.identity.primaryPort}/v2`
+  );
+
+  Assert.equal(
+    Services.prefs.getStringPref("services.settings.server"),
+    Utils.SERVER_URL
+  );
+
+  server.registerPathHandler("/v2/", (request, response) => {
+    response.write(
+      JSON.stringify({
+        capabilities: {
+          attachments: {
+            base_url: "http://some-cdn-url.org/",
+          },
+        },
+      })
+    );
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setStatusLine(null, 200, "OK");
+  });
+
+  const after = await Utils.baseAttachmentsURL();
+  Assert.equal(after, "http://some-cdn-url.org/");
+});
+
+add_task(async function test_base_attachment_url_reads_from_prefs() {
+  Services.prefs.setStringPref(
+    "services.settings.base_attachments_url",
+    `${Utils.SERVER_URL}|https://other/`
+  );
+  Assert.equal(await Utils.baseAttachmentsURL(), "https://other/");
+});
+
+add_task(async function test_base_attachment_url_is_robust_to_bad_saved_data() {
+  server.registerPathHandler("/v1/", (request, response) => {
+    response.write(
+      JSON.stringify({
+        capabilities: {
+          attachments: {
+            base_url: "http://some-cdn-url.org/",
+          },
+        },
+      })
+    );
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setStatusLine(null, 200, "OK");
+  });
+
+  for (const badValue of [
+    "",
+    "|",
+    Utils.SERVER_URL,
+    `${Utils.SERVER_URL}|https://other`, // Missing trailing slash.
+    `${Utils.SERVER_URL}|https://other/path`, // Missing trailing slash.
+  ]) {
+    Services.prefs.setStringPref(
+      "services.settings.base_attachments_url",
+      badValue
+    );
+    Assert.equal(
+      await Utils.baseAttachmentsURL(),
+      "http://some-cdn-url.org/",
+      "Fallsback to server's"
+    );
+  }
+});
+
+add_task(async function test_base_attachment_url_does_not_retry_by_default() {
+  Services.prefs.clearUserPref("services.settings.base_attachments_url");
+  Services.prefs.setStringPref(
+    "services.settings.server",
+    `http://localhost:${server.identity.primaryPort}/v1`
+  );
+  let requests = 0;
+  server.registerPathHandler("/v1/", (request, response) => {
+    requests++;
+    response.setStatusLine(null, 503, "Service Unavailable");
+  });
+  try {
+    await Utils.baseAttachmentsURL({ retryWaitMsec: 10 });
+    Assert.ok(false, "should throw on error");
+  } catch (e) {}
+  Assert.equal(requests, 1);
+});
+
+add_task(async function test_base_attachment_url_can_retry() {
+  Services.prefs.clearUserPref("services.settings.base_attachments_url");
+  Services.prefs.setStringPref(
+    "services.settings.server",
+    `http://localhost:${server.identity.primaryPort}/v1`
+  );
+  let requests = 0;
+  server.registerPathHandler("/v1/", (request, response) => {
+    requests++;
+    if (requests < 4) {
+      response.setStatusLine(null, 503, "Service Unavailable");
+      return;
+    }
+
+    response.write(
+      JSON.stringify({
+        capabilities: {
+          attachments: {
+            base_url: "http://some-cdn-url.org/",
+          },
+        },
+      })
+    );
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setStatusLine(null, 200, "OK");
+  });
+  Assert.equal(
+    await Utils.baseAttachmentsURL({ retries: 3, retryWaitMsec: 10 }),
+    "http://some-cdn-url.org/"
+  );
+  Assert.equal(requests, 4);
+});

src/zen/tests/mochitests/remote-settings/test_remote_settings_utils_telemetry.js

@@ -0,0 +1,79 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const server = new HttpServer();
+server.start(-1);
+registerCleanupFunction(() => server.stop(() => {}));
+const SERVER_BASE_URL = `http://localhost:${server.identity.primaryPort}`;
+
+const proxyServer = new HttpServer();
+proxyServer.start(-1);
+const PROXY_PORT = proxyServer.identity.primaryPort;
+proxyServer.stop();
+
+server.registerPathHandler("/destination", (request, response) => {
+  response.setStatusLine(null, 412);
+});
+
+function assertBypassTelemetryEvents(expectedEvents) {
+  const events = Glean.serviceRequest.bypassProxyInfo.testGetValue() ?? [];
+  Assert.equal(events.length, expectedEvents.length);
+  for (let i = 0; i < expectedEvents.length; i++) {
+    Assert.deepEqual(events[i].extra, {
+      value: expectedEvents[i].value,
+      ...expectedEvents[i].extra,
+    });
+  }
+  Services.fog.testResetFOG();
+}
+
+add_task(async function test_telemetry() {
+  const DESTINATION_URL = `${SERVER_BASE_URL}/destination`;
+
+  {
+    let res = await Utils.fetch(DESTINATION_URL);
+    Assert.equal(res.status, 412, "fetch without proxy succeeded");
+  }
+  assertBypassTelemetryEvents([]);
+
+  Services.prefs.setIntPref("network.proxy.type", 1);
+  Services.prefs.setStringPref("network.proxy.http", "127.0.0.1");
+  Services.prefs.setIntPref("network.proxy.http_port", PROXY_PORT);
+  Services.prefs.setBoolPref("network.proxy.allow_hijacking_localhost", true);
+
+  {
+    let res = await Utils.fetch(DESTINATION_URL);
+    Assert.equal(res.status, 412, "fetch with broken proxy succeeded");
+  }
+  // Note: failover handled by HttpChannel, hence no failover here.
+  assertBypassTelemetryEvents([]);
+
+  // Disable HttpChannel failover in favor of Utils.fetch's implementation.
+  Services.prefs.setBoolPref("network.proxy.failover_direct", false);
+  {
+    let res = await Utils.fetch(DESTINATION_URL);
+    Assert.equal(res.status, 412, "fetch succeeded with bypassProxy feature");
+  }
+  assertBypassTelemetryEvents([
+    {
+      category: "service_request",
+      method: "bypass",
+      object: "proxy_info",
+      value: "remote-settings",
+      extra: {
+        source: "prefs",
+        type: "manual",
+      },
+    },
+  ]);
+
+  Services.prefs.setBoolPref("network.proxy.allow_bypass", false);
+  await Assert.rejects(
+    Utils.fetch(DESTINATION_URL),
+    /NetworkError/,
+    "Request without failover fails"
+  );
+  assertBypassTelemetryEvents([]);
+});

src/zen/tests/mochitests/remote-settings/test_remote_settings_worker.js

@@ -0,0 +1,117 @@
+add_task(async function test_canonicaljson() {
+  const records = [
+    { id: "1", title: "title 1" },
+    { id: "2", title: "title 2" },
+  ];
+  const timestamp = 42;
+
+  const serialized = await RemoteSettingsWorker.canonicalStringify(
+    records,
+    timestamp
+  );
+
+  Assert.equal(
+    serialized,
+    '{"data":[{"id":"1","title":"title 1"},{"id":"2","title":"title 2"}],"last_modified":"42"}'
+  );
+});
+
+add_task(async function test_import_json_dump_into_idb() {
+  if (IS_ANDROID) {
+    // Skip test: we don't ship remote settings dumps on Android (see package-manifest).
+    return;
+  }
+  const client = new RemoteSettingsClient("language-dictionaries");
+  const before = await client.db.getLastModified();
+  Assert.equal(before, null);
+
+  await RemoteSettingsWorker.importJSONDump("main", "language-dictionaries");
+
+  const after = await client.get({ syncIfEmpty: false });
+  Assert.ok(!!after.length);
+  let lastModifiedStamp = await client.getLastModified();
+
+  Assert.equal(
+    lastModifiedStamp,
+    Math.max(...after.map(record => record.last_modified)),
+    "Should have correct last modified timestamp"
+  );
+
+  // Force a DB close for shutdown so we can delete the DB later.
+  Database._shutdownHandler();
+});
+
+add_task(async function test_throws_error_if_worker_fails() {
+  let error;
+  try {
+    await RemoteSettingsWorker.canonicalStringify(null, 42);
+  } catch (e) {
+    error = e;
+  }
+  Assert.equal(error.message.endsWith("records is null"), true);
+});
+
+add_task(async function test_throws_error_if_worker_fails_async() {
+  if (IS_ANDROID) {
+    // Skip test: we don't ship dump, so importJSONDump() is no-op.
+    return;
+  }
+  // Delete the Remote Settings database, and try to import a dump.
+  // This is not supported, and the error thrown asynchronously in the worker
+  // should be reported to the caller.
+  await new Promise((resolve, reject) => {
+    const request = indexedDB.deleteDatabase("remote-settings");
+    request.onsuccess = () => resolve();
+    request.onblocked = () => reject(new Error("Cannot delete DB"));
+    request.onerror = event => reject(event.target.error);
+  });
+  let error;
+  try {
+    await RemoteSettingsWorker.importJSONDump("main", "language-dictionaries");
+  } catch (e) {
+    error = e;
+  }
+  Assert.ok(/IndexedDB: Error accessing remote-settings/.test(error.message));
+});
+
+add_task(async function test_throws_error_if_worker_crashes() {
+  // This simulates a crash at the worker level (not within a promise).
+  let error;
+  try {
+    await RemoteSettingsWorker._execute("unknown_method");
+  } catch (e) {
+    error = e;
+  }
+  Assert.equal(error.message, "TypeError: Agent[method] is not a function");
+});
+
+add_task(async function test_stops_worker_after_timeout() {
+  // Change the idle time.
+  Services.prefs.setIntPref(
+    "services.settings.worker_idle_max_milliseconds",
+    1
+  );
+  // Run a task:
+  let serialized = await RemoteSettingsWorker.canonicalStringify([], 42);
+  Assert.equal(serialized, '{"data":[],"last_modified":"42"}', "API works.");
+  // Check that the worker gets stopped now the task is done:
+  await TestUtils.waitForCondition(() => !RemoteSettingsWorker.worker);
+  // Ensure the worker stays alive for 10 minutes instead:
+  Services.prefs.setIntPref(
+    "services.settings.worker_idle_max_milliseconds",
+    600000
+  );
+  // Run another task:
+  serialized = await RemoteSettingsWorker.canonicalStringify([], 42);
+  Assert.equal(
+    serialized,
+    '{"data":[],"last_modified":"42"}',
+    "API still works."
+  );
+  Assert.ok(RemoteSettingsWorker.worker, "Worker should stay alive a bit.");
+
+  // Clear the pref.
+  Services.prefs.clearUserPref(
+    "services.settings.worker_idle_max_milliseconds"
+  );
+});

src/zen/tests/mochitests/remote-settings/test_shutdown_handling.js

@@ -0,0 +1,132 @@
+/* Any copyright is dedicated to the Public Domain.
+http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_setup(() => {
+  Services.prefs.setStringPref("services.settings.loglevel", "debug");
+  registerCleanupFunction(() => {
+    Services.prefs.clearUserPref("services.settings.loglevel");
+  });
+});
+
+add_task(async function test_shutdown_abort_after_start() {
+  // Start a forever transaction:
+  let counter = 0;
+  let transactionStarted;
+  let startedPromise = new Promise(r => {
+    transactionStarted = r;
+  });
+  let promise = Database._executeIDB(
+    "records",
+    store => {
+      // Signal we've started.
+      transactionStarted();
+      function makeRequest() {
+        if (++counter > 1000) {
+          Assert.ok(
+            false,
+            "We ran 1000 requests and didn't get aborted, what?"
+          );
+          return;
+        }
+        dump("Making request " + counter + "\n");
+        const request = store
+          .index("cid")
+          .openCursor(IDBKeyRange.only("foopydoo/foo"));
+        request.onsuccess = () => {
+          makeRequest();
+        };
+      }
+      makeRequest();
+    },
+    { mode: "readonly" }
+  );
+
+  // Wait for the transaction to start.
+  await startedPromise;
+
+  Database._shutdownHandler(); // should abort the readonly transaction.
+
+  let rejection;
+  await promise.catch(e => {
+    rejection = e;
+  });
+  ok(rejection, "Promise should have rejected.");
+
+  // Now clear the shutdown flag and rejection error:
+  Database._cancelShutdown();
+  rejection = null;
+});
+
+add_task(async function test_shutdown_immediate_abort() {
+  // Now abort directly from the successful request.
+  let promise = Database._executeIDB(
+    "records",
+    store => {
+      let request = store
+        .index("cid")
+        .openCursor(IDBKeyRange.only("foopydoo/foo"));
+      request.onsuccess = () => {
+        // Abort immediately.
+        Database._shutdownHandler();
+        request = store
+          .index("cid")
+          .openCursor(IDBKeyRange.only("foopydoo/foo"));
+        Assert.ok(false, "IndexedDB allowed opening a cursor after aborting?!");
+      };
+    },
+    { mode: "readonly" }
+  );
+
+  let rejection;
+  // Wait for the abort
+  await promise.catch(e => {
+    rejection = e;
+  });
+  ok(rejection, "Directly aborted promise should also have rejected.");
+  // Now clear the shutdown flag and rejection error:
+  Database._cancelShutdown();
+});
+
+add_task(async function test_shutdown_worker() {
+  let client = new RemoteSettingsClient("language-dictionaries");
+  const before = await client.db.getLastModified();
+  Assert.equal(before, null);
+
+  let records = [{}];
+  let importPromise = RemoteSettingsWorker._execute(
+    "_test_only_import",
+    ["main", "language-dictionaries", records, 0],
+    { mustComplete: true }
+  );
+  let stringifyPromise = RemoteSettingsWorker.canonicalStringify(
+    [],
+    [],
+    Date.now()
+  );
+  // Change the idle time so we shut the worker down even though we can't
+  // set gShutdown from outside of the worker management code.
+  Services.prefs.setIntPref(
+    "services.settings.worker_idle_max_milliseconds",
+    1
+  );
+  RemoteSettingsWorker._abortCancelableRequests();
+  await Assert.rejects(
+    stringifyPromise,
+    /Shutdown/,
+    "Should have aborted the stringify request at shutdown."
+  );
+  await Assert.rejects(
+    importPromise,
+    /shutting down/,
+    "Ensure imports get aborted during shutdown"
+  );
+  const after = await client.db.getLastModified();
+  Assert.equal(after, null);
+  await TestUtils.waitForCondition(() => !RemoteSettingsWorker.worker);
+  Assert.ok(
+    !RemoteSettingsWorker.worker,
+    "Worker should have been terminated."
+  );
+});

src/zen/tests/mochitests/remote-settings/test_uptake_telemetry.js

@@ -0,0 +1,68 @@
+const { UptakeTelemetry } = ChromeUtils.importESModule(
+  "resource://services-settings/UptakeTelemetry.sys.mjs"
+);
+
+add_setup(function () {
+  Services.fog.initializeFOG();
+});
+
+function enableUptakeMetric() {
+  Services.fog.applyServerKnobsConfig(
+    JSON.stringify({
+      metrics_enabled: {
+        "uptake.remotecontent.result.uptake_remotesettings": true,
+      },
+    })
+  );
+}
+
+add_task(async function test_unknown_status_is_not_reported() {
+  Services.fog.testResetFOG();
+  enableUptakeMetric();
+
+  try {
+    await UptakeTelemetry.report("unknown-status", { source: "update-source" });
+  } catch (e) {}
+
+  Assert.equal(
+    null,
+    Glean.uptakeRemotecontentResult.uptakeRemotesettings.testGetValue()
+  );
+});
+
+add_task(async function test_age_is_converted_to_string_and_reported() {
+  Services.fog.testResetFOG();
+  enableUptakeMetric();
+  const status = UptakeTelemetry.STATUS.SUCCESS;
+  const age = 42;
+
+  await UptakeTelemetry.report(status, { source: "s", age });
+
+  const events =
+    Glean.uptakeRemotecontentResult.uptakeRemotesettings.testGetValue();
+  Assert.equal(1, events.length);
+  Assert.deepEqual(events[0].extra, {
+    value: status,
+    source: "s",
+    age: `${age}`,
+  });
+});
+
+add_task(async function test_each_status_can_be_caught_in_snapshot() {
+  Services.fog.testResetFOG();
+  enableUptakeMetric();
+  const source = "some-source";
+
+  for (const status of Object.values(UptakeTelemetry.STATUS)) {
+    await UptakeTelemetry.report(status, { source });
+  }
+
+  const events =
+    Glean.uptakeRemotecontentResult.uptakeRemotesettings.testGetValue();
+  for (const status of Object.values(UptakeTelemetry.STATUS)) {
+    Assert.ok(
+      events.some(e => e.extra.value === status && e.extra.source === source),
+      `check events for ${status}`
+    );
+  }
+});

src/zen/tests/mochitests/remote-settings/xpcshell.toml

@@ -0,0 +1,50 @@
+[DEFAULT]
+head = "../../../../services/common/tests/unit/head_global.js ../../../../services/common/tests/unit/head_helpers.js head_settings.js"
+firefox-appdir = "browser"
+tags = "remote-settings"
+support-files = ["test_remote_settings_signatures/**"]
+skip-if = [
+  "appname == 'thunderbird'", # Bug 1662758 - these tests don't pass if default bucket isn't "main".
+]
+
+["test_attachments_downloader.js"]
+support-files = ["test_attachments_downloader/**"]
+
+["test_remote_settings.js"]
+
+["test_remote_settings_dump_lastmodified.js"]
+
+["test_remote_settings_jexl_filters.js"]
+
+["test_remote_settings_offline.js"]
+
+["test_remote_settings_older_than_local.js"]
+
+["test_remote_settings_poll.js"]
+
+["test_remote_settings_recover_broken.js"]
+
+["test_remote_settings_release_prefs.js"]
+
+["test_remote_settings_signatures.js"]
+
+["test_remote_settings_startup_bundle.js"]
+support-files = ["test_remote_settings_startup_bundle/**"]
+
+["test_remote_settings_sync_history.js"]
+
+["test_remote_settings_utils.js"]
+
+["test_remote_settings_utils_telemetry.js"]
+skip-if = [
+  "os == 'android' && os_version == '14' && arch == 'x86_64'", # Bug 1739463
+]
+
+["test_remote_settings_worker.js"]
+
+["test_shutdown_handling.js"]
+
+["test_uptake_telemetry.js"]
+skip-if = [
+  "os == 'android'", # Bug 2042499
+]

src/zen/tests/mochitests/services-crypto/head_helpers.js

@@ -0,0 +1,68 @@
+/* import-globals-from ../../../common/tests/unit/head_helpers.js */
+
+var { XPCOMUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+try {
+  // In the context of xpcshell tests, there won't be a default AppInfo
+  // eslint-disable-next-line mozilla/use-services
+  Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULAppInfo);
+} catch (ex) {
+  // Make sure to provide the right OS so crypto loads the right binaries
+  var OS = "XPCShell";
+  if (mozinfo.os == "win") {
+    OS = "WINNT";
+  } else if (mozinfo.os == "mac") {
+    OS = "Darwin";
+  } else {
+    OS = "Linux";
+  }
+
+  const { updateAppInfo } = ChromeUtils.importESModule(
+    "resource://testing-common/AppInfo.sys.mjs"
+  );
+  updateAppInfo({
+    name: "XPCShell",
+    ID: "{3e3ba16c-1675-4e88-b9c8-afef81b3d2ef}",
+    version: "1",
+    platformVersion: "",
+    OS,
+  });
+}
+
+function base64UrlDecode(s) {
+  s = s.replace(/-/g, "+");
+  s = s.replace(/_/g, "/");
+
+  // Replace padding if it was stripped by the sender.
+  // See http://tools.ietf.org/html/rfc4648#section-4
+  switch (s.length % 4) {
+    case 0:
+      break; // No pad chars in this case
+    case 2:
+      s += "==";
+      break; // Two pad chars
+    case 3:
+      s += "=";
+      break; // One pad char
+    default:
+      throw new Error("Illegal base64url string!");
+  }
+
+  // With correct padding restored, apply the standard base64 decoder
+  return atob(s);
+}
+
+/**
+ * Print some debug message to the console. All arguments will be printed,
+ * separated by spaces.
+ *
+ * @param [arg0, arg1, arg2, ...]
+ *        Any number of arguments to print out
+ * Usage: _("Hello World") -> prints "Hello World"
+ * Usage: _(1, 2, 3) -> prints "1 2 3"
+ */
+var _ = function () {
+  print(Array.from(arguments).join(" "));
+};

src/zen/tests/mochitests/services-crypto/test_crypto_crypt.js

@@ -0,0 +1,226 @@
+const { WeaveCrypto } = ChromeUtils.importESModule(
+  "moz-src:///services/crypto/modules/WeaveCrypto.sys.mjs"
+);
+
+var cryptoSvc = new WeaveCrypto();
+
+add_task(async function test_key_memoization() {
+  let cryptoGlobal = cryptoSvc._getCrypto();
+  let oldImport = cryptoGlobal.subtle.importKey;
+  if (!oldImport) {
+    _("Couldn't swizzle crypto.subtle.importKey; returning.");
+    return;
+  }
+
+  let iv = cryptoSvc.generateRandomIV();
+  let key = await cryptoSvc.generateRandomKey();
+  let c = 0;
+  cryptoGlobal.subtle.importKey = function (
+    format,
+    keyData,
+    algo,
+    extractable,
+    usages
+  ) {
+    c++;
+    return oldImport.call(
+      cryptoGlobal.subtle,
+      format,
+      keyData,
+      algo,
+      extractable,
+      usages
+    );
+  };
+
+  // Encryption should cause a single counter increment.
+  Assert.equal(c, 0);
+  let cipherText = await cryptoSvc.encrypt("Hello, world.", key, iv);
+  Assert.equal(c, 1);
+  cipherText = await cryptoSvc.encrypt("Hello, world.", key, iv);
+  Assert.equal(c, 1);
+
+  // ... as should decryption.
+  await cryptoSvc.decrypt(cipherText, key, iv);
+  await cryptoSvc.decrypt(cipherText, key, iv);
+  await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(c, 2);
+
+  // Un-swizzle.
+  cryptoGlobal.subtle.importKey = oldImport;
+});
+
+// Just verify that it gets populated with the correct bytes.
+add_task(async function test_makeUint8Array() {
+  ChromeUtils.importESModule("resource://gre/modules/ctypes.sys.mjs");
+
+  let item1 = cryptoSvc.makeUint8Array("abcdefghi", false);
+  Assert.ok(item1);
+  for (let i = 0; i < 8; ++i) {
+    Assert.equal(item1[i], "abcdefghi".charCodeAt(i));
+  }
+});
+
+add_task(async function test_encrypt_decrypt() {
+  // First, do a normal run with expected usage... Generate a random key and
+  // iv, encrypt and decrypt a string.
+  var iv = cryptoSvc.generateRandomIV();
+  Assert.equal(iv.length, 24);
+
+  var key = await cryptoSvc.generateRandomKey();
+  Assert.equal(key.length, 44);
+
+  var mySecret = "bacon is a vegetable";
+  var cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  Assert.equal(cipherText.length, 44);
+
+  var clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(clearText.length, 20);
+
+  // Did the text survive the encryption round-trip?
+  Assert.equal(clearText, mySecret);
+  Assert.notEqual(cipherText, mySecret); // just to be explicit
+
+  // Do some more tests with a fixed key/iv, to check for reproducable results.
+  key = "St1tFCor7vQEJNug/465dQ==";
+  iv = "oLjkfrLIOnK2bDRvW4kXYA==";
+
+  _("Testing small IV.");
+  mySecret = "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=";
+  let shortiv = "YWJj";
+  let err;
+  try {
+    await cryptoSvc.encrypt(mySecret, key, shortiv);
+  } catch (ex) {
+    err = ex;
+  }
+  Assert.ok(!!err);
+
+  _("Testing long IV.");
+  let longiv = "gsgLRDaxWvIfKt75RjuvFWERt83FFsY2A0TW+0b2iVk=";
+  try {
+    await cryptoSvc.encrypt(mySecret, key, longiv);
+  } catch (ex) {
+    err = ex;
+  }
+  Assert.ok(!!err);
+
+  // Test small input sizes
+  mySecret = "";
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "OGQjp6mK1a3fs9k9Ml4L3w==");
+  Assert.equal(clearText, mySecret);
+
+  mySecret = "x";
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "96iMl4vhOxFUW/lVHHzVqg==");
+  Assert.equal(clearText, mySecret);
+
+  mySecret = "xx";
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "olpPbETRYROCSqFWcH2SWg==");
+  Assert.equal(clearText, mySecret);
+
+  mySecret = "xxx";
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "rRbpHGyVSZizLX/x43Wm+Q==");
+  Assert.equal(clearText, mySecret);
+
+  mySecret = "xxxx";
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "HeC7miVGDcpxae9RmiIKAw==");
+  Assert.equal(clearText, mySecret);
+
+  // Test non-ascii input
+  // ("testuser1" using similar-looking glyphs)
+  mySecret = String.fromCharCode(355, 277, 349, 357, 533, 537, 101, 345, 185);
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "Pj4ixByXoH3SU3JkOXaEKPgwRAWplAWFLQZkpJd5Kr4=");
+  Assert.equal(clearText, mySecret);
+
+  // Tests input spanning a block boundary (AES block size is 16 bytes)
+  mySecret = "123456789012345";
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "e6c5hwphe45/3VN/M0bMUA==");
+  Assert.equal(clearText, mySecret);
+
+  mySecret = "1234567890123456";
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "V6aaOZw8pWlYkoIHNkhsP1JOIQF87E2vTUvBUQnyV04=");
+  Assert.equal(clearText, mySecret);
+
+  mySecret = "12345678901234567";
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "V6aaOZw8pWlYkoIHNkhsP5GvxWJ9+GIAS6lXw+5fHTI=");
+  Assert.equal(clearText, mySecret);
+
+  key = "iz35tuIMq4/H+IYw2KTgow==";
+  iv = "TJYrvva2KxvkM8hvOIvWp3==";
+  mySecret = "i like pie";
+
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "DLGx8BWqSCLGG7i/xwvvxg==");
+  Assert.equal(clearText, mySecret);
+
+  key = "c5hG3YG+NC61FFy8NOHQak1ZhMEWO79bwiAfar2euzI=";
+  iv = "gsgLRDaxWvIfKt75RjuvFW==";
+  mySecret = "i like pie";
+
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = await cryptoSvc.decrypt(cipherText, key, iv);
+  Assert.equal(cipherText, "o+ADtdMd8ubzNWurS6jt0Q==");
+  Assert.equal(clearText, mySecret);
+
+  key = "St1tFCor7vQEJNug/465dQ==";
+  iv = "oLjkfrLIOnK2bDRvW4kXYA==";
+  mySecret = "does thunder read testcases?";
+  cipherText = await cryptoSvc.encrypt(mySecret, key, iv);
+  Assert.equal(cipherText, "T6fik9Ros+DB2ablH9zZ8FWZ0xm/szSwJjIHZu7sjPs=");
+
+  var badkey = "badkeybadkeybadkeybadk==";
+  var badiv = "badivbadivbadivbadivbad=";
+  var badcipher = "crapinputcrapinputcrapinputcrapinputcrapinp=";
+  var failure;
+
+  try {
+    failure = false;
+    clearText = await cryptoSvc.decrypt(cipherText, badkey, iv);
+  } catch (e) {
+    failure = true;
+  }
+  Assert.ok(failure);
+
+  try {
+    failure = false;
+    clearText = await cryptoSvc.decrypt(cipherText, key, badiv);
+  } catch (e) {
+    failure = true;
+  }
+  Assert.ok(failure);
+
+  try {
+    failure = false;
+    clearText = await cryptoSvc.decrypt(cipherText, badkey, badiv);
+  } catch (e) {
+    failure = true;
+  }
+  Assert.ok(failure);
+
+  try {
+    failure = false;
+    clearText = await cryptoSvc.decrypt(badcipher, key, iv);
+  } catch (e) {
+    failure = true;
+  }
+  Assert.ok(failure);
+});

src/zen/tests/mochitests/services-crypto/test_crypto_random.js

@@ -0,0 +1,52 @@
+const { WeaveCrypto } = ChromeUtils.importESModule(
+  "moz-src:///services/crypto/modules/WeaveCrypto.sys.mjs"
+);
+
+var cryptoSvc = new WeaveCrypto();
+
+add_task(async function test_crypto_random() {
+  if (this.gczeal) {
+    _("Running crypto random tests with gczeal(2).");
+    gczeal(2);
+  }
+
+  // Test salt generation.
+  var salt;
+
+  salt = cryptoSvc.generateRandomBytes(0);
+  Assert.equal(salt.length, 0);
+  salt = cryptoSvc.generateRandomBytes(1);
+  Assert.equal(salt.length, 4);
+  salt = cryptoSvc.generateRandomBytes(2);
+  Assert.equal(salt.length, 4);
+  salt = cryptoSvc.generateRandomBytes(3);
+  Assert.equal(salt.length, 4);
+  salt = cryptoSvc.generateRandomBytes(4);
+  Assert.equal(salt.length, 8);
+  salt = cryptoSvc.generateRandomBytes(8);
+  Assert.equal(salt.length, 12);
+
+  // sanity check to make sure salts seem random
+  var salt2 = cryptoSvc.generateRandomBytes(8);
+  Assert.equal(salt2.length, 12);
+  Assert.notEqual(salt, salt2);
+
+  salt = cryptoSvc.generateRandomBytes(1024);
+  Assert.equal(salt.length, 1368);
+  salt = cryptoSvc.generateRandomBytes(16);
+  Assert.equal(salt.length, 24);
+
+  // Test random key generation
+  var keydata, keydata2, iv;
+
+  keydata = await cryptoSvc.generateRandomKey();
+  Assert.equal(keydata.length, 44);
+  keydata2 = await cryptoSvc.generateRandomKey();
+  Assert.notEqual(keydata, keydata2); // sanity check for randomness
+  iv = cryptoSvc.generateRandomIV();
+  Assert.equal(iv.length, 24);
+
+  if (this.gczeal) {
+    gczeal(0);
+  }
+});

src/zen/tests/mochitests/services-crypto/test_jwcrypto.js

@@ -0,0 +1,51 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+ChromeUtils.defineESModuleGetters(this, {
+  jwcrypto: "moz-src:///services/crypto/modules/jwcrypto.sys.mjs",
+});
+
+// Enable logging from jwcrypto.sys.mjs.
+Services.prefs.setStringPref("services.crypto.jwcrypto.log.level", "Debug");
+
+add_task(async function test_jwe_roundtrip_ecdh_es_encryption() {
+  const plaintext = crypto.getRandomValues(new Uint8Array(123));
+  const remoteKey = await crypto.subtle.generateKey(
+    {
+      name: "ECDH",
+      namedCurve: "P-256",
+    },
+    true,
+    ["deriveKey"]
+  );
+  const remoteJWK = await crypto.subtle.exportKey("jwk", remoteKey.publicKey);
+  delete remoteJWK.key_ops;
+  const jwe = await jwcrypto.generateJWE(remoteJWK, plaintext);
+  const decrypted = await jwcrypto.decryptJWE(jwe, remoteKey.privateKey);
+  Assert.deepEqual(plaintext, decrypted);
+});
+
+add_task(async function test_jwe_header_includes_key_id() {
+  const plaintext = crypto.getRandomValues(new Uint8Array(123));
+  const remoteKey = await crypto.subtle.generateKey(
+    {
+      name: "ECDH",
+      namedCurve: "P-256",
+    },
+    true,
+    ["deriveKey"]
+  );
+  const remoteJWK = await crypto.subtle.exportKey("jwk", remoteKey.publicKey);
+  delete remoteJWK.key_ops;
+  remoteJWK.kid = "key identifier";
+  const jwe = await jwcrypto.generateJWE(remoteJWK, plaintext);
+  let [header /* other items deliberately ignored */] = jwe.split(".");
+  header = JSON.parse(
+    new TextDecoder().decode(
+      ChromeUtils.base64URLDecode(header, { padding: "reject" })
+    )
+  );
+  Assert.equal(header.kid, "key identifier");
+});

src/zen/tests/mochitests/services-crypto/test_utils_hawk.js

@@ -0,0 +1,346 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { CryptoUtils } = ChromeUtils.importESModule(
+  "moz-src:///services/crypto/modules/utils.sys.mjs"
+);
+
+function run_test() {
+  initTestLogging();
+
+  run_next_test();
+}
+
+add_task(async function test_hawk() {
+  let compute = CryptoUtils.computeHAWK;
+
+  let method = "POST";
+  let ts = 1353809207;
+  let nonce = "Ygvqdz";
+
+  let credentials = {
+    id: "123456",
+    key: "2983d45yun89q",
+  };
+
+  let uri_https = CommonUtils.makeURI(
+    "https://example.net/somewhere/over/the/rainbow"
+  );
+  let opts = {
+    credentials,
+    ext: "Bazinga!",
+    ts,
+    nonce,
+    payload: "something to write about",
+    contentType: "text/plain",
+  };
+
+  let result = await compute(uri_https, method, opts);
+  Assert.equal(
+    result.field,
+    'Hawk id="123456", ts="1353809207", nonce="Ygvqdz", ' +
+      'hash="2QfCt3GuY9HQnHWyWD3wX68ZOKbynqlfYmuO2ZBRqtY=", ' +
+      'ext="Bazinga!", ' +
+      'mac="q1CwFoSHzPZSkbIvl0oYlD+91rBUEvFk763nMjMndj8="'
+  );
+  Assert.equal(result.artifacts.ts, ts);
+  Assert.equal(result.artifacts.nonce, nonce);
+  Assert.equal(result.artifacts.method, method);
+  Assert.equal(result.artifacts.resource, "/somewhere/over/the/rainbow");
+  Assert.equal(result.artifacts.host, "example.net");
+  Assert.equal(result.artifacts.port, 443);
+  Assert.equal(
+    result.artifacts.hash,
+    "2QfCt3GuY9HQnHWyWD3wX68ZOKbynqlfYmuO2ZBRqtY="
+  );
+  Assert.equal(result.artifacts.ext, "Bazinga!");
+
+  let opts_noext = {
+    credentials,
+    ts,
+    nonce,
+    payload: "something to write about",
+    contentType: "text/plain",
+  };
+  result = await compute(uri_https, method, opts_noext);
+  Assert.equal(
+    result.field,
+    'Hawk id="123456", ts="1353809207", nonce="Ygvqdz", ' +
+      'hash="2QfCt3GuY9HQnHWyWD3wX68ZOKbynqlfYmuO2ZBRqtY=", ' +
+      'mac="HTgtd0jPI6E4izx8e4OHdO36q00xFCU0FolNq3RiCYs="'
+  );
+  Assert.equal(result.artifacts.ts, ts);
+  Assert.equal(result.artifacts.nonce, nonce);
+  Assert.equal(result.artifacts.method, method);
+  Assert.equal(result.artifacts.resource, "/somewhere/over/the/rainbow");
+  Assert.equal(result.artifacts.host, "example.net");
+  Assert.equal(result.artifacts.port, 443);
+  Assert.equal(
+    result.artifacts.hash,
+    "2QfCt3GuY9HQnHWyWD3wX68ZOKbynqlfYmuO2ZBRqtY="
+  );
+
+  /* Leaving optional fields out should work, although of course then we can't
+   * assert much about the resulting hashes. The resulting header should look
+   * roughly like:
+   * Hawk id="123456", ts="1378764955", nonce="QkynqsrS44M=", mac="/C5NsoAs2fVn+d/I5wMfwe2Gr1MZyAJ6pFyDHG4Gf9U="
+   */
+
+  result = await compute(uri_https, method, { credentials });
+  let fields = result.field.split(" ");
+  Assert.equal(fields[0], "Hawk");
+  Assert.equal(fields[1], 'id="123456",'); // from creds.id
+  Assert.ok(fields[2].startsWith('ts="'));
+  /* The HAWK spec calls for seconds-since-epoch, not ms-since-epoch.
+   * Warning: this test will fail in the year 33658, and for time travellers
+   * who journey earlier than 2001. Please plan accordingly. */
+  Assert.greater(result.artifacts.ts, 1000 * 1000 * 1000);
+  Assert.less(result.artifacts.ts, 1000 * 1000 * 1000 * 1000);
+  Assert.ok(fields[3].startsWith('nonce="'));
+  Assert.equal(fields[3].length, 'nonce="12345678901=",'.length);
+  Assert.equal(result.artifacts.nonce.length, "12345678901=".length);
+
+  let result2 = await compute(uri_https, method, { credentials });
+  Assert.notEqual(result.artifacts.nonce, result2.artifacts.nonce);
+
+  /* Using an upper-case URI hostname shouldn't affect the hash. */
+
+  let uri_https_upper = CommonUtils.makeURI(
+    "https://EXAMPLE.NET/somewhere/over/the/rainbow"
+  );
+  result = await compute(uri_https_upper, method, opts);
+  Assert.equal(
+    result.field,
+    'Hawk id="123456", ts="1353809207", nonce="Ygvqdz", ' +
+      'hash="2QfCt3GuY9HQnHWyWD3wX68ZOKbynqlfYmuO2ZBRqtY=", ' +
+      'ext="Bazinga!", ' +
+      'mac="q1CwFoSHzPZSkbIvl0oYlD+91rBUEvFk763nMjMndj8="'
+  );
+
+  /* Using a lower-case method name shouldn't affect the hash. */
+  result = await compute(uri_https_upper, method.toLowerCase(), opts);
+  Assert.equal(
+    result.field,
+    'Hawk id="123456", ts="1353809207", nonce="Ygvqdz", ' +
+      'hash="2QfCt3GuY9HQnHWyWD3wX68ZOKbynqlfYmuO2ZBRqtY=", ' +
+      'ext="Bazinga!", ' +
+      'mac="q1CwFoSHzPZSkbIvl0oYlD+91rBUEvFk763nMjMndj8="'
+  );
+
+  /* The localtimeOffsetMsec field should be honored. HAWK uses this to
+   * compensate for clock skew between client and server: if the request is
+   * rejected with a timestamp out-of-range error, the error includes the
+   * server's time, and the client computes its clock offset and tries again.
+   * Clients can remember this offset for a while.
+   */
+
+  result = await compute(uri_https, method, {
+    credentials,
+    now: 1378848968650,
+  });
+  Assert.equal(result.artifacts.ts, 1378848968);
+
+  result = await compute(uri_https, method, {
+    credentials,
+    now: 1378848968650,
+    localtimeOffsetMsec: 1000 * 1000,
+  });
+  Assert.equal(result.artifacts.ts, 1378848968 + 1000);
+
+  /* Search/query-args in URIs should be included in the hash. */
+  let makeURI = CommonUtils.makeURI;
+  result = await compute(makeURI("http://example.net/path"), method, opts);
+  Assert.equal(result.artifacts.resource, "/path");
+  Assert.equal(
+    result.artifacts.mac,
+    "WyKHJjWaeYt8aJD+H9UeCWc0Y9C+07ooTmrcrOW4MPI="
+  );
+
+  result = await compute(makeURI("http://example.net/path/"), method, opts);
+  Assert.equal(result.artifacts.resource, "/path/");
+  Assert.equal(
+    result.artifacts.mac,
+    "xAYp2MgZQFvTKJT9u8nsvMjshCRRkuaeYqQbYSFp9Qw="
+  );
+
+  result = await compute(
+    makeURI("http://example.net/path?query=search"),
+    method,
+    opts
+  );
+  Assert.equal(result.artifacts.resource, "/path?query=search");
+  Assert.equal(
+    result.artifacts.mac,
+    "C06a8pip2rA4QkBiosEmC32WcgFcW/R5SQC6kUWyqho="
+  );
+
+  /* Test handling of the payload, which is supposed to be a bytestring
+  (String with codepoints from U+0000 to U+00FF, pre-encoded). */
+
+  result = await compute(makeURI("http://example.net/path"), method, {
+    credentials,
+    ts: 1353809207,
+    nonce: "Ygvqdz",
+  });
+  Assert.equal(result.artifacts.hash, undefined);
+  Assert.equal(
+    result.artifacts.mac,
+    "S3f8E4hAURAqJxOlsYugkPZxLoRYrClgbSQ/3FmKMbY="
+  );
+
+  // Empty payload changes nothing.
+  result = await compute(makeURI("http://example.net/path"), method, {
+    credentials,
+    ts: 1353809207,
+    nonce: "Ygvqdz",
+    payload: null,
+  });
+  Assert.equal(result.artifacts.hash, undefined);
+  Assert.equal(
+    result.artifacts.mac,
+    "S3f8E4hAURAqJxOlsYugkPZxLoRYrClgbSQ/3FmKMbY="
+  );
+
+  result = await compute(makeURI("http://example.net/path"), method, {
+    credentials,
+    ts: 1353809207,
+    nonce: "Ygvqdz",
+    payload: "hello",
+  });
+  Assert.equal(
+    result.artifacts.hash,
+    "uZJnFj0XVBA6Rs1hEvdIDf8NraM0qRNXdFbR3NEQbVA="
+  );
+  Assert.equal(
+    result.artifacts.mac,
+    "pLsHHzngIn5CTJhWBtBr+BezUFvdd/IadpTp/FYVIRM="
+  );
+
+  // update, utf-8 payload
+  result = await compute(makeURI("http://example.net/path"), method, {
+    credentials,
+    ts: 1353809207,
+    nonce: "Ygvqdz",
+    payload: "andré@example.org", // non-ASCII
+  });
+  Assert.equal(
+    result.artifacts.hash,
+    "66DiyapJ0oGgj09IXWdMv8VCg9xk0PL5RqX7bNnQW2k="
+  );
+  Assert.equal(
+    result.artifacts.mac,
+    "2B++3x5xfHEZbPZGDiK3IwfPZctkV4DUr2ORg1vIHvk="
+  );
+
+  /* If "hash" is provided, "payload" is ignored. */
+  result = await compute(makeURI("http://example.net/path"), method, {
+    credentials,
+    ts: 1353809207,
+    nonce: "Ygvqdz",
+    hash: "66DiyapJ0oGgj09IXWdMv8VCg9xk0PL5RqX7bNnQW2k=",
+    payload: "something else",
+  });
+  Assert.equal(
+    result.artifacts.hash,
+    "66DiyapJ0oGgj09IXWdMv8VCg9xk0PL5RqX7bNnQW2k="
+  );
+  Assert.equal(
+    result.artifacts.mac,
+    "2B++3x5xfHEZbPZGDiK3IwfPZctkV4DUr2ORg1vIHvk="
+  );
+
+  // the payload "hash" is also non-urlsafe base64 (+/)
+  result = await compute(makeURI("http://example.net/path"), method, {
+    credentials,
+    ts: 1353809207,
+    nonce: "Ygvqdz",
+    payload: "something else",
+  });
+  Assert.equal(
+    result.artifacts.hash,
+    "lERFXr/IKOaAoYw+eBseDUSwmqZTX0uKZpcWLxsdzt8="
+  );
+  Assert.equal(
+    result.artifacts.mac,
+    "jiZuhsac35oD7IdcblhFncBr8tJFHcwWLr8NIYWr9PQ="
+  );
+
+  /* Test non-ascii hostname. HAWK (via the node.js "url" module) punycodes
+   * "ëxample.net" into "xn--xample-ova.net" before hashing. I still think
+   * punycode was a bad joke that got out of the lab and into a spec.
+   */
+
+  result = await compute(makeURI("http://ëxample.net/path"), method, {
+    credentials,
+    ts: 1353809207,
+    nonce: "Ygvqdz",
+  });
+  Assert.equal(
+    result.artifacts.mac,
+    "pILiHl1q8bbNQIdaaLwAFyaFmDU70MGehFuCs3AA5M0="
+  );
+  Assert.equal(result.artifacts.host, "xn--xample-ova.net");
+
+  result = await compute(makeURI("http://example.net/path"), method, {
+    credentials,
+    ts: 1353809207,
+    nonce: "Ygvqdz",
+    ext: 'backslash=\\ quote=" EOF',
+  });
+  Assert.equal(
+    result.artifacts.mac,
+    "BEMW76lwaJlPX4E/dajF970T6+GzWvaeyLzUt8eOTOc="
+  );
+  Assert.equal(
+    result.field,
+    'Hawk id="123456", ts="1353809207", nonce="Ygvqdz", ext="backslash=\\\\ quote=\\" EOF", mac="BEMW76lwaJlPX4E/dajF970T6+GzWvaeyLzUt8eOTOc="'
+  );
+
+  result = await compute(makeURI("http://example.net:1234/path"), method, {
+    credentials,
+    ts: 1353809207,
+    nonce: "Ygvqdz",
+  });
+  Assert.equal(
+    result.artifacts.mac,
+    "6D3JSFDtozuq8QvJTNUc1JzeCfy6h5oRvlhmSTPv6LE="
+  );
+  Assert.equal(
+    result.field,
+    'Hawk id="123456", ts="1353809207", nonce="Ygvqdz", mac="6D3JSFDtozuq8QvJTNUc1JzeCfy6h5oRvlhmSTPv6LE="'
+  );
+
+  /* HAWK (the node.js library) uses a URL parser which stores the "port"
+   * field as a string, but makeURI() gives us an integer. So we'll diverge
+   * on ports with a leading zero. This test vector would fail on the node.js
+   * library (HAWK-1.1.1), where they get a MAC of
+   * "T+GcAsDO8GRHIvZLeepSvXLwDlFJugcZroAy9+uAtcw=". I think HAWK should be
+   * updated to do what we do here, so port="01234" should get the same hash
+   * as port="1234".
+   */
+  result = await compute(makeURI("http://example.net:01234/path"), method, {
+    credentials,
+    ts: 1353809207,
+    nonce: "Ygvqdz",
+  });
+  Assert.equal(
+    result.artifacts.mac,
+    "6D3JSFDtozuq8QvJTNUc1JzeCfy6h5oRvlhmSTPv6LE="
+  );
+  Assert.equal(
+    result.field,
+    'Hawk id="123456", ts="1353809207", nonce="Ygvqdz", mac="6D3JSFDtozuq8QvJTNUc1JzeCfy6h5oRvlhmSTPv6LE="'
+  );
+});
+
+add_test(function test_strip_header_attributes() {
+  let strip = CryptoUtils.stripHeaderAttributes;
+
+  Assert.equal(strip(undefined), "");
+  Assert.equal(strip("text/plain"), "text/plain");
+  Assert.equal(strip("TEXT/PLAIN"), "text/plain");
+  Assert.equal(strip("  text/plain  "), "text/plain");
+  Assert.equal(strip("text/plain ; charset=utf-8  "), "text/plain");
+
+  run_next_test();
+});

src/zen/tests/mochitests/services-crypto/test_utils_httpmac.js

@@ -0,0 +1,73 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { CryptoUtils } = ChromeUtils.importESModule(
+  "moz-src:///services/crypto/modules/utils.sys.mjs"
+);
+
+add_test(function setup() {
+  initTestLogging();
+  run_next_test();
+});
+
+add_task(async function test_sha1() {
+  _("Ensure HTTP MAC SHA1 generation works as expected.");
+
+  let id = "vmo1txkttblmn51u2p3zk2xiy16hgvm5ok8qiv1yyi86ffjzy9zj0ez9x6wnvbx7";
+  let key = "b8u1cc5iiio5o319og7hh8faf2gi5ym4aq0zwf112cv1287an65fudu5zj7zo7dz";
+  let ts = 1329181221;
+  let method = "GET";
+  let nonce = "wGX71";
+  let uri = CommonUtils.makeURI("http://10.250.2.176/alias/");
+
+  let result = await CryptoUtils.computeHTTPMACSHA1(id, key, method, uri, {
+    ts,
+    nonce,
+  });
+
+  Assert.equal(btoa(result.mac), "jzh5chjQc2zFEvLbyHnPdX11Yck=");
+
+  Assert.equal(
+    result.getHeader(),
+    'MAC id="vmo1txkttblmn51u2p3zk2xiy16hgvm5ok8qiv1yyi86ffjzy9zj0ez9x6wnvbx7", ' +
+      'ts="1329181221", nonce="wGX71", mac="jzh5chjQc2zFEvLbyHnPdX11Yck="'
+  );
+
+  let ext = "EXTRA DATA; foo,bar=1";
+
+  result = await CryptoUtils.computeHTTPMACSHA1(id, key, method, uri, {
+    ts,
+    nonce,
+    ext,
+  });
+  Assert.equal(btoa(result.mac), "bNf4Fnt5k6DnhmyipLPkuZroH68=");
+  Assert.equal(
+    result.getHeader(),
+    'MAC id="vmo1txkttblmn51u2p3zk2xiy16hgvm5ok8qiv1yyi86ffjzy9zj0ez9x6wnvbx7", ' +
+      'ts="1329181221", nonce="wGX71", mac="bNf4Fnt5k6DnhmyipLPkuZroH68=", ' +
+      'ext="EXTRA DATA; foo,bar=1"'
+  );
+});
+
+add_task(async function test_nonce_length() {
+  _("Ensure custom nonce lengths are honoured.");
+
+  function get_mac(length) {
+    let uri = CommonUtils.makeURI("http://example.com/");
+    return CryptoUtils.computeHTTPMACSHA1("foo", "bar", "GET", uri, {
+      nonce_bytes: length,
+    });
+  }
+
+  let result = await get_mac(12);
+  Assert.equal(12, atob(result.nonce).length);
+
+  result = await get_mac(2);
+  Assert.equal(2, atob(result.nonce).length);
+
+  result = await get_mac(0);
+  Assert.equal(8, atob(result.nonce).length);
+
+  result = await get_mac(-1);
+  Assert.equal(8, atob(result.nonce).length);
+});

src/zen/tests/mochitests/services-crypto/xpcshell.toml

@@ -0,0 +1,20 @@
+[DEFAULT]
+head = "head_helpers.js ../../../../services/common/tests/unit/head_helpers.js"
+firefox-appdir = "browser"
+support-files = ["!/services/common/tests/unit/head_helpers.js"]
+
+["test_crypto_crypt.js"]
+
+["test_crypto_random.js"]
+run-if = [
+  "appname != 'thunderbird'",
+]
+
+["test_jwcrypto.js"]
+run-if = [
+  "appname != 'thunderbird'",
+]
+
+["test_utils_hawk.js"]
+
+["test_utils_httpmac.js"]

src/zen/tests/mochitests/sync/addon1-search.json

@@ -0,0 +1,21 @@
+{
+  "next": null,
+  "results": [
+    {
+      "name": "Non-Restartless Test Extension",
+      "type": "extension",
+      "guid": "addon1@tests.mozilla.org",
+      "current_version": {
+        "version": "1.0",
+        "files": [
+          {
+            "platform": "all",
+            "size": 485,
+            "url": "http://127.0.0.1:8888/addon1.xpi"
+          }
+        ]
+      },
+      "last_updated": "2011-09-05T20:42:09Z"
+    }
+  ]
+}

src/zen/tests/mochitests/sync/bootstrap1-search.json

@@ -0,0 +1,21 @@
+{
+  "next": null,
+  "results": [
+    {
+      "name": "Restartless Test Extension",
+      "type": "extension",
+      "guid": "bootstrap1@tests.mozilla.org",
+      "current_version": {
+        "version": "1.0",
+        "files": [
+          {
+            "platform": "all",
+            "size": 485,
+            "url": "http://127.0.0.1:8888/bootstrap1.xpi"
+          }
+        ]
+      },
+      "last_updated": "2011-09-05T20:42:09Z"
+    }
+  ]
+}

src/zen/tests/mochitests/sync/head_appinfo.js

@@ -0,0 +1,58 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/* import-globals-from ../../../common/tests/unit/head_helpers.js */
+
+var { XPCOMUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+// Required to avoid failures.
+do_get_profile();
+
+// Init FormHistoryStartup and pretend we opened a profile.
+var fhs = Cc["@mozilla.org/satchel/form-history-startup;1"].getService(
+  Ci.nsIObserver
+);
+fhs.observe(null, "profile-after-change", null);
+
+// An app is going to have some prefs set which xpcshell tests don't.
+Services.prefs.setStringPref(
+  "identity.sync.tokenserver.uri",
+  "http://token-server"
+);
+
+// Make sure to provide the right OS so crypto loads the right binaries
+function getOS() {
+  switch (mozinfo.os) {
+    case "win":
+      return "WINNT";
+    case "mac":
+      return "Darwin";
+    default:
+      return "Linux";
+  }
+}
+
+const { updateAppInfo } = ChromeUtils.importESModule(
+  "resource://testing-common/AppInfo.sys.mjs"
+);
+updateAppInfo({
+  name: "XPCShell",
+  ID: "xpcshell@tests.mozilla.org",
+  version: "1",
+  platformVersion: "",
+  OS: getOS(),
+});
+
+// Register resource aliases. Normally done in SyncComponents.manifest.
+function addResourceAlias() {
+  const resProt = Services.io
+    .getProtocolHandler("resource")
+    .QueryInterface(Ci.nsIResProtocolHandler);
+  for (let s of ["common", "sync", "crypto"]) {
+    let uri = Services.io.newURI("resource://gre/modules/services-" + s + "/");
+    resProt.setSubstitution("services-" + s, uri);
+  }
+}
+addResourceAlias();

src/zen/tests/mochitests/sync/head_errorhandler_common.js

@@ -0,0 +1,195 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/* import-globals-from head_appinfo.js */
+/* import-globals-from ../../../common/tests/unit/head_helpers.js */
+/* import-globals-from head_helpers.js */
+/* import-globals-from head_http_server.js */
+
+// This file expects Service to be defined in the global scope when EHTestsCommon
+// is used (from service.js).
+/* global Service */
+
+var { Changeset, EngineManager, Store, SyncEngine, Tracker, LegacyTracker } =
+  ChromeUtils.importESModule("resource://services-sync/engines.sys.mjs");
+var {
+  ABORT_SYNC_COMMAND,
+  CLIENT_NOT_CONFIGURED,
+  CREDENTIALS_CHANGED,
+  DEFAULT_DOWNLOAD_BATCH_SIZE,
+  DEFAULT_GUID_FETCH_BATCH_SIZE,
+  DEFAULT_KEYBUNDLE_NAME,
+  DEVICE_TYPE_DESKTOP,
+  DEVICE_TYPE_MOBILE,
+  ENGINE_APPLY_FAIL,
+  ENGINE_BATCH_INTERRUPTED,
+  ENGINE_DOWNLOAD_FAIL,
+  ENGINE_SUCCEEDED,
+  ENGINE_UNKNOWN_FAIL,
+  ENGINE_UPLOAD_FAIL,
+  HMAC_EVENT_INTERVAL,
+  IDLE_OBSERVER_BACK_DELAY,
+  LOGIN_FAILED,
+  LOGIN_FAILED_INVALID_PASSPHRASE,
+  LOGIN_FAILED_LOGIN_REJECTED,
+  LOGIN_FAILED_NETWORK_ERROR,
+  LOGIN_FAILED_NO_PASSPHRASE,
+  LOGIN_FAILED_NO_USERNAME,
+  LOGIN_FAILED_SERVER_ERROR,
+  LOGIN_SUCCEEDED,
+  MASTER_PASSWORD_LOCKED,
+  MASTER_PASSWORD_LOCKED_RETRY_INTERVAL,
+  MAXIMUM_BACKOFF_INTERVAL,
+  MAX_ERROR_COUNT_BEFORE_BACKOFF,
+  MAX_HISTORY_DOWNLOAD,
+  MAX_HISTORY_UPLOAD,
+  METARECORD_DOWNLOAD_FAIL,
+  MINIMUM_BACKOFF_INTERVAL,
+  MULTI_DEVICE_THRESHOLD,
+  NO_SYNC_NODE_FOUND,
+  NO_SYNC_NODE_INTERVAL,
+  OVER_QUOTA,
+  PREFS_BRANCH,
+  RESPONSE_OVER_QUOTA,
+  SCORE_INCREMENT_MEDIUM,
+  SCORE_INCREMENT_SMALL,
+  SCORE_INCREMENT_XLARGE,
+  SCORE_UPDATE_DELAY,
+  SERVER_MAINTENANCE,
+  SINGLE_USER_THRESHOLD,
+  SQLITE_MAX_VARIABLE_NUMBER,
+  STATUS_DISABLED,
+  STATUS_OK,
+  STORAGE_VERSION,
+  SYNC_FAILED,
+  SYNC_FAILED_PARTIAL,
+  SYNC_KEY_DECODED_LENGTH,
+  SYNC_KEY_ENCODED_LENGTH,
+  SYNC_SUCCEEDED,
+  URI_LENGTH_MAX,
+  VERSION_OUT_OF_DATE,
+  WEAVE_VERSION,
+  kFirefoxShuttingDown,
+  kFirstSyncChoiceNotMade,
+  kSyncBackoffNotMet,
+  kSyncMasterPasswordLocked,
+  kSyncNetworkOffline,
+  kSyncNotConfigured,
+  kSyncWeaveDisabled,
+} = ChromeUtils.importESModule("resource://services-sync/constants.sys.mjs");
+var { BulkKeyBundle, SyncKeyBundle } = ChromeUtils.importESModule(
+  "resource://services-sync/keys.sys.mjs"
+);
+
+// Common code for test_errorhandler_{1,2}.js -- pulled out to make it less
+// monolithic and take less time to execute.
+const EHTestsCommon = {
+  service_unavailable(request, response) {
+    let body = "Service Unavailable";
+    response.setStatusLine(request.httpVersion, 503, "Service Unavailable");
+    response.setHeader("Retry-After", "42");
+    response.bodyOutputStream.write(body, body.length);
+  },
+
+  async sync_httpd_setup() {
+    let clientsEngine = Service.clientsEngine;
+    let clientsSyncID = await clientsEngine.resetLocalSyncID();
+    let catapultEngine = Service.engineManager.get("catapult");
+    let catapultSyncID = await catapultEngine.resetLocalSyncID();
+    let global = new ServerWBO("global", {
+      syncID: Service.syncID,
+      storageVersion: STORAGE_VERSION,
+      engines: {
+        clients: { version: clientsEngine.version, syncID: clientsSyncID },
+        catapult: { version: catapultEngine.version, syncID: catapultSyncID },
+      },
+    });
+    let clientsColl = new ServerCollection({}, true);
+
+    // Tracking info/collections.
+    let collectionsHelper = track_collections_helper();
+    let upd = collectionsHelper.with_updated_collection;
+
+    let handler_401 = httpd_handler(401, "Unauthorized");
+    return httpd_setup({
+      // Normal server behaviour.
+      "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()),
+      "/1.1/johndoe/info/collections": collectionsHelper.handler,
+      "/1.1/johndoe/storage/crypto/keys": upd(
+        "crypto",
+        new ServerWBO("keys").handler()
+      ),
+      "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()),
+
+      // Credentials are wrong or node reallocated.
+      "/1.1/janedoe/storage/meta/global": handler_401,
+      "/1.1/janedoe/info/collections": handler_401,
+
+      // Maintenance or overloaded (503 + Retry-After) at info/collections.
+      "/1.1/broken.info/info/collections": EHTestsCommon.service_unavailable,
+
+      // Maintenance or overloaded (503 + Retry-After) at meta/global.
+      "/1.1/broken.meta/storage/meta/global": EHTestsCommon.service_unavailable,
+      "/1.1/broken.meta/info/collections": collectionsHelper.handler,
+
+      // Maintenance or overloaded (503 + Retry-After) at crypto/keys.
+      "/1.1/broken.keys/storage/meta/global": upd("meta", global.handler()),
+      "/1.1/broken.keys/info/collections": collectionsHelper.handler,
+      "/1.1/broken.keys/storage/crypto/keys": EHTestsCommon.service_unavailable,
+
+      // Maintenance or overloaded (503 + Retry-After) at wiping collection.
+      "/1.1/broken.wipe/info/collections": collectionsHelper.handler,
+      "/1.1/broken.wipe/storage/meta/global": upd("meta", global.handler()),
+      "/1.1/broken.wipe/storage/crypto/keys": upd(
+        "crypto",
+        new ServerWBO("keys").handler()
+      ),
+      "/1.1/broken.wipe/storage": EHTestsCommon.service_unavailable,
+      "/1.1/broken.wipe/storage/clients": upd("clients", clientsColl.handler()),
+      "/1.1/broken.wipe/storage/catapult": EHTestsCommon.service_unavailable,
+    });
+  },
+
+  CatapultEngine: (function () {
+    function CatapultEngine() {
+      SyncEngine.call(this, "Catapult", Service);
+    }
+    CatapultEngine.prototype = {
+      exception: null, // tests fill this in
+      async _sync() {
+        if (this.exception) {
+          throw this.exception;
+        }
+      },
+    };
+    Object.setPrototypeOf(CatapultEngine.prototype, SyncEngine.prototype);
+
+    return CatapultEngine;
+  })(),
+
+  async generateCredentialsChangedFailure() {
+    // Make sync fail due to changed credentials. We simply re-encrypt
+    // the keys with a different Sync Key, without changing the local one.
+    let newSyncKeyBundle = new BulkKeyBundle("crypto");
+    await newSyncKeyBundle.generateRandom();
+    let keys = Service.collectionKeys.asWBO();
+    await keys.encrypt(newSyncKeyBundle);
+    return keys.upload(Service.resource(Service.cryptoKeysURL));
+  },
+
+  async setUp(server) {
+    syncTestLogging();
+    await configureIdentity({ username: "johndoe" }, server);
+    return EHTestsCommon.generateAndUploadKeys();
+  },
+
+  async generateAndUploadKeys() {
+    await generateNewKeys(Service.collectionKeys);
+    let serverKeys = Service.collectionKeys.asWBO("crypto", "keys");
+    await serverKeys.encrypt(Service.identity.syncKeyBundle);
+    let response = await serverKeys.upload(
+      Service.resource(Service.cryptoKeysURL)
+    );
+    return response.success;
+  },
+};

src/zen/tests/mochitests/sync/head_helpers.js

@@ -0,0 +1,621 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/* import-globals-from head_appinfo.js */
+/* import-globals-from ../../../common/tests/unit/head_helpers.js */
+/* import-globals-from head_errorhandler_common.js */
+/* import-globals-from head_http_server.js */
+
+// This file expects Service to be defined in the global scope when EHTestsCommon
+// is used (from service.js).
+/* global Service */
+
+var { AddonTestUtils, MockAsyncShutdown } = ChromeUtils.importESModule(
+  "resource://testing-common/AddonTestUtils.sys.mjs"
+);
+var { Async } = ChromeUtils.importESModule(
+  "resource://services-common/async.sys.mjs"
+);
+var { CommonUtils } = ChromeUtils.importESModule(
+  "resource://services-common/utils.sys.mjs"
+);
+var { PlacesTestUtils } = ChromeUtils.importESModule(
+  "resource://testing-common/PlacesTestUtils.sys.mjs"
+);
+var { sinon } = ChromeUtils.importESModule(
+  "resource://testing-common/Sinon.sys.mjs"
+);
+var { SerializableSet, Svc, Utils, getChromeWindow } =
+  ChromeUtils.importESModule("resource://services-sync/util.sys.mjs");
+var { XPCOMUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+var { PlacesUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/PlacesUtils.sys.mjs"
+);
+var { PlacesSyncUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/PlacesSyncUtils.sys.mjs"
+);
+var { ObjectUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/ObjectUtils.sys.mjs"
+);
+var {
+  MockFxaStorageManager,
+  SyncTestingInfrastructure,
+  configureFxAccountIdentity,
+  configureIdentity,
+  encryptPayload,
+  getLoginTelemetryScalar,
+  makeFxAccountsInternalMock,
+  makeIdentityConfig,
+  promiseNamedTimer,
+  promiseZeroTimer,
+  sumHistogram,
+  syncTestLogging,
+  waitForZeroTimer,
+} = ChromeUtils.importESModule(
+  "resource://testing-common/services/sync/utils.sys.mjs"
+);
+ChromeUtils.defineESModuleGetters(this, {
+  AddonManager: "resource://gre/modules/AddonManager.sys.mjs",
+});
+
+add_setup(async function head_setup() {
+  // Initialize logging. This will sometimes be reset by a pref reset,
+  // so it's also called as part of SyncTestingInfrastructure().
+  syncTestLogging();
+  // If a test imports Service, make sure it is initialized first.
+  if (typeof Service !== "undefined") {
+    await Service.promiseInitialized;
+  }
+});
+
+// This is needed for loadAddonTestFunctions().
+var gGlobalScope = this;
+
+function ExtensionsTestPath(path) {
+  if (path[0] != "/") {
+    throw Error("Path must begin with '/': " + path);
+  }
+
+  return "../../../../toolkit/mozapps/extensions/test/xpcshell" + path;
+}
+
+function webExtensionsTestPath(path) {
+  if (path[0] != "/") {
+    throw Error("Path must begin with '/': " + path);
+  }
+
+  return "../../../../toolkit/components/extensions/test/xpcshell" + path;
+}
+
+/**
+ * Loads the WebExtension test functions by importing its test file.
+ */
+function loadWebExtensionTestFunctions() {
+  /* import-globals-from ../../../../toolkit/components/extensions/test/xpcshell/head_sync.js */
+  const path = webExtensionsTestPath("/head_sync.js");
+  let file = do_get_file(path);
+  let uri = Services.io.newFileURI(file);
+  Services.scriptloader.loadSubScript(uri.spec, gGlobalScope);
+}
+
+/**
+ * Installs an add-on from an addonInstall
+ *
+ * @param  install addonInstall instance to install
+ */
+async function installAddonFromInstall(install) {
+  await install.install();
+
+  Assert.notEqual(null, install.addon);
+  Assert.notEqual(null, install.addon.syncGUID);
+
+  return install.addon;
+}
+
+/**
+ * Convenience function to install an add-on from the extensions unit tests.
+ *
+ * @param  file
+ *         Add-on file to install.
+ * @param  reconciler
+ *         addons reconciler, if passed we will wait on the events to be
+ *         processed before resolving
+ * @return addon object that was installed
+ */
+async function installAddon(file, reconciler = null) {
+  let install = await AddonManager.getInstallForFile(file);
+  Assert.notEqual(null, install);
+  const addon = await installAddonFromInstall(install);
+  if (reconciler) {
+    await reconciler.queueCaller.promiseCallsComplete();
+  }
+  return addon;
+}
+
+/**
+ * Convenience function to uninstall an add-on.
+ *
+ * @param addon
+ *        Addon instance to uninstall
+ * @param reconciler
+ *        addons reconciler, if passed we will wait on the events to be
+ *        processed before resolving
+ */
+async function uninstallAddon(addon, reconciler = null) {
+  const uninstallPromise = new Promise(res => {
+    let listener = {
+      onUninstalled(uninstalled) {
+        if (uninstalled.id == addon.id) {
+          AddonManager.removeAddonListener(listener);
+          res(uninstalled);
+        }
+      },
+    };
+    AddonManager.addAddonListener(listener);
+  });
+  addon.uninstall();
+  await uninstallPromise;
+  if (reconciler) {
+    await reconciler.queueCaller.promiseCallsComplete();
+  }
+}
+
+async function generateNewKeys(collectionKeys, collections = null) {
+  let wbo = await collectionKeys.generateNewKeysWBO(collections);
+  let modified = new_timestamp();
+  collectionKeys.setContents(wbo.cleartext, modified);
+}
+
+// Helpers for testing open tabs.
+// These reflect part of the internal structure of TabEngine,
+// and stub part of Service.wm.
+
+function mockGetTabState(tab) {
+  return tab;
+}
+
+function mockGetOrderedNonPrivateWindows(urls) {
+  let tabs = [];
+  let win = {
+    gBrowser: {
+      tabs,
+    },
+  };
+
+  let lastAccessed = 2000;
+  for (let url of urls) {
+    tabs.push({
+      linkedBrowser: {
+        currentURI: Services.io.newURI(url),
+        contentTitle: "title",
+      },
+      lastAccessed,
+    });
+    lastAccessed += 1000;
+  }
+
+  return [win];
+}
+
+// Helper function to get the sync telemetry and add the typically used test
+// engine names to its list of allowed engines.
+function get_sync_test_telemetry() {
+  let { SyncTelemetry } = ChromeUtils.importESModule(
+    "resource://services-sync/telemetry.sys.mjs"
+  );
+  SyncTelemetry.tryRefreshDevices = function () {};
+  let testEngines = ["rotary", "steam", "sterling", "catapult", "nineties"];
+  for (let engineName of testEngines) {
+    SyncTelemetry.allowedEngines.add(engineName);
+  }
+  SyncTelemetry.submissionInterval = -1;
+  return SyncTelemetry;
+}
+
+function assert_valid_ping(record) {
+  if (record && (record.why != "shutdown" || !!record.syncs.length)) {
+    record.syncs.forEach(p => {
+      lessOrEqual(p.when, Date.now());
+    });
+  }
+}
+
+function assert_success_sync(record) {
+  ok(!record.failureReason, JSON.stringify(record.failureReason));
+  equal(undefined, record.status);
+  greater(record.engines.length, 0);
+  for (let e of record.engines) {
+    ok(!e.failureReason);
+    equal(undefined, e.status);
+    if (e.validation) {
+      equal(undefined, e.validation.problems);
+      equal(undefined, e.validation.failureReason);
+    }
+    if (e.outgoing) {
+      for (let o of e.outgoing) {
+        equal(undefined, o.failed);
+        notEqual(undefined, o.sent);
+      }
+    }
+    if (e.incoming) {
+      equal(undefined, e.incoming.failed);
+      equal(undefined, e.incoming.newFailed);
+      notEqual(undefined, e.incoming.applied || e.incoming.reconciled);
+    }
+  }
+}
+
+// Asserts that `ping` is a ping that doesn't contain any failure information
+function assert_success_ping(ping) {
+  ok(!!ping);
+  assert_valid_ping(ping);
+  ping.syncs.forEach(assert_success_sync);
+}
+
+// Hooks into telemetry to validate all pings after calling.
+function validate_all_future_pings() {
+  let telem = get_sync_test_telemetry();
+  telem.submit = assert_valid_ping;
+}
+
+function wait_for_pings(expectedPings) {
+  return new Promise(resolve => {
+    let telem = get_sync_test_telemetry();
+    let oldSubmit = telem.submit;
+    let pings = [];
+    telem.submit = function (record) {
+      pings.push(record);
+      if (pings.length == expectedPings) {
+        telem.submit = oldSubmit;
+        resolve(pings);
+      }
+    };
+  });
+}
+
+async function wait_for_ping(callback, allowErrorPings, getFullPing = false) {
+  let pingsPromise = wait_for_pings(1);
+  await callback();
+  let [record] = await pingsPromise;
+  if (allowErrorPings) {
+    assert_valid_ping(record);
+  } else {
+    assert_success_ping(record);
+  }
+  if (getFullPing) {
+    return record;
+  }
+  equal(record.syncs.length, 1);
+  return record.syncs[0];
+}
+
+// Perform a sync and validate all telemetry caused by the sync. If fnValidate
+// is null, we just check the ping records success. If fnValidate is specified,
+// then the sync must have recorded just a single sync, and that sync will be
+// passed to the function to be checked.
+async function sync_and_validate_telem(
+  fnValidate = null,
+  wantFullPing = false
+) {
+  let numErrors = 0;
+  let telem = get_sync_test_telemetry();
+  let oldSubmit = telem.submit;
+  try {
+    telem.submit = function (record) {
+      // This is called via an observer, so failures here don't cause the test
+      // to fail :(
+      try {
+        // All pings must be valid.
+        assert_valid_ping(record);
+        if (fnValidate) {
+          // for historical reasons most of these callbacks expect a "sync"
+          // record, not the entire ping.
+          if (wantFullPing) {
+            fnValidate(record);
+          } else {
+            Assert.equal(record.syncs.length, 1);
+            fnValidate(record.syncs[0]);
+          }
+        } else {
+          // no validation function means it must be a "success" ping.
+          assert_success_ping(record);
+        }
+      } catch (ex) {
+        print("Failure in ping validation callback", ex, "\n", ex.stack);
+        numErrors += 1;
+      }
+    };
+    await Service.sync();
+    Assert.equal(numErrors, 0, "There were telemetry validation errors");
+  } finally {
+    telem.submit = oldSubmit;
+  }
+}
+
+// Used for the (many) cases where we do a 'partial' sync, where only a single
+// engine is actually synced, but we still want to ensure we're generating a
+// valid ping. Returns a promise that resolves to the ping, or rejects with the
+// thrown error after calling an optional callback.
+async function sync_engine_and_validate_telem(
+  engine,
+  allowErrorPings,
+  onError,
+  wantFullPing = false
+) {
+  let telem = get_sync_test_telemetry();
+  let caughtError = null;
+  // Clear out status, so failures from previous syncs won't show up in the
+  // telemetry ping.
+  let { Status } = ChromeUtils.importESModule(
+    "resource://services-sync/status.sys.mjs"
+  );
+  Status._engines = {};
+  Status.partial = false;
+  // Ideally we'd clear these out like we do with engines, (probably via
+  // Status.resetSync()), but this causes *numerous* tests to fail, so we just
+  // assume that if no failureReason or engine failures are set, and the
+  // status properties are the same as they were initially, that it's just
+  // a leftover.
+  // This is only an issue since we're triggering the sync of just one engine,
+  // without doing any other parts of the sync.
+  let initialServiceStatus = Status._service;
+  let initialSyncStatus = Status._sync;
+
+  let oldSubmit = telem.submit;
+  let submitPromise = new Promise((resolve, reject) => {
+    telem.submit = function (ping) {
+      telem.submit = oldSubmit;
+      ping.syncs.forEach(record => {
+        if (record && record.status) {
+          // did we see anything to lead us to believe that something bad actually happened
+          let realProblem =
+            record.failureReason ||
+            record.engines.some(e => {
+              if (e.failureReason || e.status) {
+                return true;
+              }
+              if (e.outgoing && e.outgoing.some(o => o.failed > 0)) {
+                return true;
+              }
+              return e.incoming && e.incoming.failed;
+            });
+          if (!realProblem) {
+            // no, so if the status is the same as it was initially, just assume
+            // that its leftover and that we can ignore it.
+            if (record.status.sync && record.status.sync == initialSyncStatus) {
+              delete record.status.sync;
+            }
+            if (
+              record.status.service &&
+              record.status.service == initialServiceStatus
+            ) {
+              delete record.status.service;
+            }
+            if (!record.status.sync && !record.status.service) {
+              delete record.status;
+            }
+          }
+        }
+      });
+      if (allowErrorPings) {
+        assert_valid_ping(ping);
+      } else {
+        assert_success_ping(ping);
+      }
+      equal(ping.syncs.length, 1);
+      if (caughtError) {
+        if (onError) {
+          onError(ping.syncs[0], ping);
+        }
+        reject(caughtError);
+      } else if (wantFullPing) {
+        resolve(ping);
+      } else {
+        resolve(ping.syncs[0]);
+      }
+    };
+  });
+  // neuter the scheduler as it interacts badly with some of the tests - the
+  // engine being synced usually isn't the registered engine, so we see
+  // scored incremented and not removed, which schedules unexpected syncs.
+  let oldObserve = Service.scheduler.observe;
+  Service.scheduler.observe = () => {};
+  try {
+    Svc.Obs.notify("weave:service:sync:start");
+    try {
+      await engine.sync();
+    } catch (e) {
+      caughtError = e;
+    }
+    if (caughtError) {
+      Svc.Obs.notify("weave:service:sync:error", caughtError);
+    } else {
+      Svc.Obs.notify("weave:service:sync:finish");
+    }
+  } finally {
+    Service.scheduler.observe = oldObserve;
+  }
+  return submitPromise;
+}
+
+// Returns a promise that resolves once the specified observer notification
+// has fired.
+function promiseOneObserver(topic) {
+  return new Promise(resolve => {
+    let observer = function (subject, data) {
+      Svc.Obs.remove(topic, observer);
+      resolve({ subject, data });
+    };
+    Svc.Obs.add(topic, observer);
+  });
+}
+
+async function registerRotaryEngine() {
+  let { RotaryEngine } = ChromeUtils.importESModule(
+    "resource://testing-common/services/sync/rotaryengine.sys.mjs"
+  );
+  await Service.engineManager.clear();
+
+  await Service.engineManager.register(RotaryEngine);
+  let engine = Service.engineManager.get("rotary");
+  let syncID = await engine.resetLocalSyncID();
+  engine.enabled = true;
+
+  return { engine, syncID, tracker: engine._tracker };
+}
+
+// Set the validation prefs to attempt validation every time to avoid non-determinism.
+function enableValidationPrefs(engines = ["bookmarks"]) {
+  for (let engine of engines) {
+    Svc.PrefBranch.setIntPref(`engine.${engine}.validation.interval`, 0);
+    Svc.PrefBranch.setIntPref(
+      `engine.${engine}.validation.percentageChance`,
+      100
+    );
+    Svc.PrefBranch.setIntPref(`engine.${engine}.validation.maxRecords`, -1);
+    Svc.PrefBranch.setBoolPref(`engine.${engine}.validation.enabled`, true);
+  }
+}
+
+async function serverForEnginesWithKeys(users, engines, callback) {
+  // Generate and store a fake default key bundle to avoid resetting the client
+  // before the first sync.
+  let wbo = await Service.collectionKeys.generateNewKeysWBO();
+  let modified = new_timestamp();
+  Service.collectionKeys.setContents(wbo.cleartext, modified);
+
+  let allEngines = [Service.clientsEngine].concat(engines);
+
+  let globalEngines = {};
+  for (let engine of allEngines) {
+    let syncID = await engine.resetLocalSyncID();
+    globalEngines[engine.name] = { version: engine.version, syncID };
+  }
+
+  let contents = {
+    meta: {
+      global: {
+        syncID: Service.syncID,
+        storageVersion: STORAGE_VERSION,
+        engines: globalEngines,
+      },
+    },
+    crypto: {
+      keys: encryptPayload(wbo.cleartext),
+    },
+  };
+  for (let engine of allEngines) {
+    contents[engine.name] = {};
+  }
+
+  return serverForUsers(users, contents, callback);
+}
+
+async function serverForFoo(engine, callback) {
+  // The bookmarks engine *always* tracks changes, meaning we might try
+  // and sync due to the bookmarks we ourselves create! Worse, because we
+  // do an engine sync only, there's no locking - so we end up with multiple
+  // syncs running. Neuter that by making the threshold very large.
+  Service.scheduler.syncThreshold = 10000000;
+  return serverForEnginesWithKeys({ foo: "password" }, engine, callback);
+}
+
+// Places notifies history observers asynchronously, so `addVisits` might return
+// before the tracker receives the notification. This helper registers an
+// observer that resolves once the expected notification fires.
+async function promiseVisit(expectedType, expectedURI) {
+  return new Promise(resolve => {
+    function done(type, uri) {
+      if (uri == expectedURI.spec && type == expectedType) {
+        PlacesObservers.removeListener(
+          ["page-visited", "page-removed"],
+          observer.handlePlacesEvents
+        );
+        resolve();
+      }
+    }
+    let observer = {
+      handlePlacesEvents(events) {
+        Assert.equal(events.length, 1);
+
+        if (events[0].type === "page-visited") {
+          done("added", events[0].url);
+        } else if (events[0].type === "page-removed") {
+          Assert.ok(events[0].isRemovedFromStore);
+          done("removed", events[0].url);
+        }
+      },
+    };
+    PlacesObservers.addListener(
+      ["page-visited", "page-removed"],
+      observer.handlePlacesEvents
+    );
+  });
+}
+
+async function addVisit(
+  suffix,
+  referrer = null,
+  transition = PlacesUtils.history.TRANSITION_LINK
+) {
+  let uriString = "http://getfirefox.com/" + suffix;
+  let uri = CommonUtils.makeURI(uriString);
+  _("Adding visit for URI " + uriString);
+
+  let visitAddedPromise = promiseVisit("added", uri);
+  await PlacesTestUtils.addVisits({
+    uri,
+    visitDate: Date.now() * 1000,
+    transition,
+    referrer,
+  });
+  await visitAddedPromise;
+
+  return uri;
+}
+
+function bookmarkNodesToInfos(nodes) {
+  return nodes.map(node => {
+    let info = {
+      guid: node.guid,
+      index: node.index,
+    };
+    if (node.children) {
+      info.children = bookmarkNodesToInfos(node.children);
+    }
+    return info;
+  });
+}
+
+async function assertBookmarksTreeMatches(rootGuid, expected, message) {
+  let root = await PlacesUtils.promiseBookmarksTree(rootGuid, {
+    includeItemIds: true,
+  });
+  let actual = bookmarkNodesToInfos(root.children);
+
+  if (!ObjectUtils.deepEqual(actual, expected)) {
+    _(`Expected structure for ${rootGuid}`, JSON.stringify(expected));
+    _(`Actual structure for ${rootGuid}`, JSON.stringify(actual));
+    throw new Assert.constructor.AssertionError({ actual, expected, message });
+  }
+}
+
+function add_bookmark_test(task) {
+  const { BookmarksEngine } = ChromeUtils.importESModule(
+    "resource://services-sync/engines/bookmarks.sys.mjs"
+  );
+
+  add_task(async function () {
+    _(`Running bookmarks test ${task.name}`);
+    let engine = new BookmarksEngine(Service);
+    await engine.initialize();
+    await engine._resetClient();
+    try {
+      await task(engine);
+    } finally {
+      await engine.finalize();
+    }
+  });
+}

src/zen/tests/mochitests/sync/missing-sourceuri.json

@@ -0,0 +1,20 @@
+{
+  "next": null,
+  "results": [
+    {
+      "name": "Restartless Test Extension",
+      "type": "extension",
+      "guid": "missing-sourceuri@tests.mozilla.org",
+      "current_version": {
+        "version": "1.0",
+        "files": [
+          {
+            "platform": "all",
+            "size": 485
+          }
+        ]
+      },
+      "last_updated": "2011-09-05T20:42:09Z"
+    }
+  ]
+}

src/zen/tests/mochitests/sync/missing-xpi-search.json

@@ -0,0 +1,21 @@
+{
+  "next": null,
+  "results": [
+    {
+      "name": "Non-Restartless Test Extension",
+      "type": "extension",
+      "guid": "missing-xpi@tests.mozilla.org",
+      "current_version": {
+        "version": "1.0",
+        "files": [
+          {
+            "platform": "all",
+            "size": 485,
+            "url": "http://127.0.0.1:8888/THIS_DOES_NOT_EXIST.xpi"
+          }
+        ]
+      },
+      "last_updated": "2011-09-05T20:42:09Z"
+    }
+  ]
+}

src/zen/tests/mochitests/sync/prefs_test_prefs_store.js

@@ -0,0 +1,47 @@
+// This is a "preferences" file used by test_prefs_store.js
+
+/* global pref, user_pref */
+
+// The prefs that control what should be synced.
+// Most of these are "default" prefs, so the value itself will not sync.
+pref("services.sync.prefs.sync.testing.int", true);
+pref("services.sync.prefs.sync.testing.string", true);
+pref("services.sync.prefs.sync.testing.bool", true);
+pref("services.sync.prefs.sync.testing.dont.change", true);
+// This is a default pref, but has the special "sync-seen" pref.
+pref("services.sync.prefs.sync.testing.seen", true);
+pref("services.sync.prefs.sync-seen.testing.seen", false);
+
+// this one is a user pref, so it *will* sync.
+user_pref("services.sync.prefs.sync.testing.turned.off", false);
+pref("services.sync.prefs.sync.testing.nonexistent", true);
+pref("services.sync.prefs.sync.testing.default", true);
+pref("services.sync.prefs.sync.testing.synced.url", true);
+// We shouldn't sync the URL, or the flag that says we should sync the pref
+// (otherwise some other client might overwrite our local value).
+user_pref("services.sync.prefs.sync.testing.unsynced.url", true);
+
+// The preference values - these are all user_prefs, otherwise their value
+// will not be synced.
+user_pref("testing.int", 123);
+user_pref("testing.string", "ohai");
+user_pref("testing.bool", true);
+user_pref("testing.dont.change", "Please don't change me.");
+user_pref("testing.turned.off", "I won't get synced.");
+user_pref("testing.not.turned.on", "I won't get synced either!");
+// Some url we don't want to sync
+user_pref(
+  "testing.unsynced.url",
+  "moz-extension://d5d31b00-b944-4afb-bd3d-d0326551a0ae"
+);
+user_pref("testing.synced.url", "https://www.example.com");
+
+// A pref that exists but still has the default value - will be synced with
+// null as the value.
+pref("testing.default", "I'm the default value");
+
+// A pref that has the default value - it will start syncing as soon as
+// we see a change, even if the change is to the default.
+pref("testing.seen", "the value");
+
+// A pref that shouldn't be synced

src/zen/tests/mochitests/sync/rewrite-search.json

@@ -0,0 +1,21 @@
+{
+  "next": null,
+  "results": [
+    {
+      "name": "Rewrite Test Extension",
+      "type": "extension",
+      "guid": "rewrite@tests.mozilla.org",
+      "current_version": {
+        "version": "1.0",
+        "files": [
+          {
+            "platform": "all",
+            "size": 485,
+            "url": "http://127.0.0.1:8888/require.xpi?src=api"
+          }
+        ]
+      },
+      "last_updated": "2011-09-05T20:42:09Z"
+    }
+  ]
+}

src/zen/tests/mochitests/sync/systemaddon-search.json

@@ -0,0 +1,21 @@
+{
+  "next": null,
+  "results": [
+    {
+      "name": "System Add-on Test",
+      "type": "extension",
+      "guid": "system1@tests.mozilla.org",
+      "current_version": {
+        "version": "1.0",
+        "files": [
+          {
+            "platform": "all",
+            "size": 999,
+            "url": "http://127.0.0.1:8888/system.xpi"
+          }
+        ]
+      },
+      "last_updated": "2011-09-05T20:42:09Z"
+    }
+  ]
+}

src/zen/tests/mochitests/sync/test_412.js

@@ -0,0 +1,60 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { RotaryEngine } = ChromeUtils.importESModule(
+  "resource://testing-common/services/sync/rotaryengine.sys.mjs"
+);
+
+add_task(async function test_412_not_treated_as_failure() {
+  await Service.engineManager.register(RotaryEngine);
+  let engine = Service.engineManager.get("rotary");
+
+  let server = await serverForFoo(engine);
+
+  await SyncTestingInfrastructure(server);
+  await generateNewKeys(Service.collectionKeys);
+
+  // add an item to the server to the first sync advances lastModified.
+  let collection = server.getCollection("foo", "rotary");
+  let payload = encryptPayload({
+    id: "existing",
+    something: "existing record",
+  });
+  collection.insert("existing", payload);
+
+  let promiseObserved = promiseOneObserver("weave:engine:sync:finish");
+  try {
+    // Do sync.
+    _("initial sync to initialize the world");
+    await Service.sync();
+
+    // create a new record that should be uploaded and arrange for our lastSync
+    // timestamp to be wrong so we get a 412.
+    engine._store.items = { new: "new record" };
+    await engine._tracker.addChangedID("new", 0);
+
+    let saw412 = false;
+    let _uploadOutgoing = engine._uploadOutgoing;
+    engine._uploadOutgoing = async () => {
+      let lastSync = await engine.getLastSync();
+      await engine.setLastSync(lastSync - 2);
+      try {
+        await _uploadOutgoing.call(engine);
+      } catch (ex) {
+        saw412 = ex.status == 412;
+        throw ex;
+      }
+    };
+    _("Second sync - expecting a 412");
+    await Service.sync();
+    await promiseObserved;
+    ok(saw412, "did see a 412 error");
+    // But service status should be OK as the 412 shouldn't be treated as an error.
+    equal(Service.status.service, STATUS_OK);
+  } finally {
+    await promiseStopServer(server);
+  }
+});

src/zen/tests/mochitests/sync/test_addon_utils.js

@@ -0,0 +1,156 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const { AddonUtils } = ChromeUtils.importESModule(
+  "resource://services-sync/addonutils.sys.mjs"
+);
+
+const HTTP_PORT = 8888;
+const SERVER_ADDRESS = "http://127.0.0.1:8888";
+
+Services.prefs.setStringPref(
+  "extensions.getAddons.get.url",
+  SERVER_ADDRESS + "/search/guid:%IDS%"
+);
+
+AddonTestUtils.init(this);
+AddonTestUtils.createAppInfo(
+  "xpcshell@tests.mozilla.org",
+  "XPCShell",
+  "1",
+  "1.9.2"
+);
+
+add_task(async function setup() {
+  await AddonTestUtils.promiseStartupManager();
+});
+
+function createAndStartHTTPServer(port = HTTP_PORT) {
+  try {
+    let server = new HttpServer();
+
+    server.registerFile(
+      "/search/guid:missing-sourceuri%40tests.mozilla.org",
+      do_get_file("missing-sourceuri.json")
+    );
+
+    server.registerFile(
+      "/search/guid:rewrite%40tests.mozilla.org",
+      do_get_file("rewrite-search.json")
+    );
+
+    server.start(port);
+
+    return server;
+  } catch (ex) {
+    _("Got exception starting HTTP server on port " + port);
+    _("Error: " + Log.exceptionStr(ex));
+    do_throw(ex);
+  }
+  return null; /* not hit, but keeps eslint happy! */
+}
+
+function run_test() {
+  syncTestLogging();
+
+  run_next_test();
+}
+
+add_task(async function test_handle_empty_source_uri() {
+  _("Ensure that search results without a sourceURI are properly ignored.");
+
+  let server = createAndStartHTTPServer();
+
+  const ID = "missing-sourceuri@tests.mozilla.org";
+
+  const result = await AddonUtils.installAddons([
+    { id: ID, requireSecureURI: false },
+  ]);
+
+  Assert.ok("installedIDs" in result);
+  Assert.equal(0, result.installedIDs.length);
+
+  Assert.ok("skipped" in result);
+  Assert.ok(result.skipped.includes(ID));
+
+  await promiseStopServer(server);
+});
+
+add_test(function test_ignore_untrusted_source_uris() {
+  _("Ensures that source URIs from insecure schemes are rejected.");
+
+  const bad = [
+    "http://example.com/foo.xpi",
+    "ftp://example.com/foo.xpi",
+    "silly://example.com/foo.xpi",
+  ];
+
+  const good = ["https://example.com/foo.xpi"];
+
+  for (let s of bad) {
+    let sourceURI = Services.io.newURI(s);
+    let addon = { sourceURI, name: "bad", id: "bad" };
+
+    let canInstall = AddonUtils.canInstallAddon(addon);
+    Assert.ok(!canInstall, "Correctly rejected a bad URL");
+  }
+
+  for (let s of good) {
+    let sourceURI = Services.io.newURI(s);
+    let addon = { sourceURI, name: "good", id: "good" };
+
+    let canInstall = AddonUtils.canInstallAddon(addon);
+    Assert.ok(canInstall, "Correctly accepted a good URL");
+  }
+  run_next_test();
+});
+
+add_task(async function test_source_uri_rewrite() {
+  _("Ensure that a 'src=api' query string is rewritten to 'src=sync'");
+
+  // This tests for conformance with bug 708134 so server-side metrics aren't
+  // skewed.
+
+  // We resort to monkeypatching because of the API design.
+  let oldFunction =
+    Object.getPrototypeOf(AddonUtils).installAddonFromSearchResult;
+
+  let installCalled = false;
+  Object.getPrototypeOf(AddonUtils).installAddonFromSearchResult =
+    async function testInstallAddon(addon) {
+      Assert.equal(
+        SERVER_ADDRESS + "/require.xpi?src=sync",
+        addon.sourceURI.spec
+      );
+
+      installCalled = true;
+
+      const install = await AddonUtils.getInstallFromSearchResult(addon);
+      Assert.equal(
+        SERVER_ADDRESS + "/require.xpi?src=sync",
+        install.sourceURI.spec
+      );
+      Assert.deepEqual(
+        install.installTelemetryInfo,
+        { source: "sync" },
+        "Got the expected installTelemetryInfo"
+      );
+
+      return { id: addon.id, addon, install };
+    };
+
+  let server = createAndStartHTTPServer();
+
+  let installOptions = {
+    id: "rewrite@tests.mozilla.org",
+    requireSecureURI: false,
+  };
+  await AddonUtils.installAddons([installOptions]);
+
+  Assert.ok(installCalled);
+  Object.getPrototypeOf(AddonUtils).installAddonFromSearchResult = oldFunction;
+
+  await promiseStopServer(server);
+});

src/zen/tests/mochitests/sync/test_addons_engine.js

@@ -0,0 +1,277 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const { AddonManager } = ChromeUtils.importESModule(
+  "resource://gre/modules/AddonManager.sys.mjs"
+);
+const { CHANGE_INSTALLED } = ChromeUtils.importESModule(
+  "resource://services-sync/addonsreconciler.sys.mjs"
+);
+const { AddonsEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/addons.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+Services.prefs.setStringPref(
+  "extensions.getAddons.get.url",
+  "http://localhost:8888/search/guid:%IDS%"
+);
+Services.prefs.setBoolPref("extensions.install.requireSecureOrigin", false);
+
+let engine;
+let syncID;
+let reconciler;
+let tracker;
+
+AddonTestUtils.init(this);
+
+const ADDON_ID = "addon1@tests.mozilla.org";
+const XPI = AddonTestUtils.createTempWebExtensionFile({
+  manifest: {
+    name: "Test 1",
+    description: "Test Description",
+    browser_specific_settings: { gecko: { id: ADDON_ID } },
+  },
+});
+
+async function resetReconciler() {
+  reconciler._addons = {};
+  reconciler._changes = [];
+
+  await reconciler.saveState();
+
+  await tracker.clearChangedIDs();
+}
+
+add_task(async function setup() {
+  AddonTestUtils.createAppInfo(
+    "xpcshell@tests.mozilla.org",
+    "XPCShell",
+    "1",
+    "1.9.2"
+  );
+  AddonTestUtils.overrideCertDB();
+  await AddonTestUtils.promiseStartupManager();
+
+  await Service.engineManager.register(AddonsEngine);
+  engine = Service.engineManager.get("addons");
+  syncID = await engine.resetLocalSyncID();
+  reconciler = engine._reconciler;
+  tracker = engine._tracker;
+
+  reconciler.startListening();
+
+  // Don't flush to disk in the middle of an event listener!
+  // This causes test hangs on WinXP.
+  reconciler._shouldPersist = false;
+
+  await resetReconciler();
+});
+
+// This is a basic sanity test for the unit test itself. If this breaks, the
+// add-ons API likely changed upstream.
+add_task(async function test_addon_install() {
+  _("Ensure basic add-on APIs work as expected.");
+
+  let install = await AddonManager.getInstallForFile(XPI);
+  Assert.notEqual(install, null);
+  Assert.equal(install.type, "extension");
+  Assert.equal(install.name, "Test 1");
+
+  await resetReconciler();
+});
+
+add_task(async function test_find_dupe() {
+  _("Ensure the _findDupe() implementation is sane.");
+
+  // This gets invoked at the top of sync, which is bypassed by this
+  // test, so we do it manually.
+  await engine._refreshReconcilerState();
+
+  let addon = await installAddon(XPI, reconciler);
+
+  let record = {
+    id: Utils.makeGUID(),
+    addonID: ADDON_ID,
+    enabled: true,
+    applicationID: Services.appinfo.ID,
+    source: "amo",
+  };
+
+  let dupe = await engine._findDupe(record);
+  Assert.equal(addon.syncGUID, dupe);
+
+  record.id = addon.syncGUID;
+  dupe = await engine._findDupe(record);
+  Assert.equal(null, dupe);
+
+  await uninstallAddon(addon, reconciler);
+  await resetReconciler();
+});
+
+add_task(async function test_get_changed_ids() {
+  let timerPrecision = Services.prefs.getBoolPref(
+    "privacy.reduceTimerPrecision"
+  );
+  Services.prefs.setBoolPref("privacy.reduceTimerPrecision", false);
+
+  registerCleanupFunction(function () {
+    Services.prefs.setBoolPref("privacy.reduceTimerPrecision", timerPrecision);
+  });
+
+  _("Ensure getChangedIDs() has the appropriate behavior.");
+
+  _("Ensure getChangedIDs() returns an empty object by default.");
+  let changes = await engine.getChangedIDs();
+  Assert.equal("object", typeof changes);
+  Assert.equal(0, Object.keys(changes).length);
+
+  _("Ensure tracker changes are populated.");
+  let now = new Date();
+  let changeTime = now.getTime() / 1000;
+  let guid1 = Utils.makeGUID();
+  await tracker.addChangedID(guid1, changeTime);
+
+  changes = await engine.getChangedIDs();
+  Assert.equal("object", typeof changes);
+  Assert.equal(1, Object.keys(changes).length);
+  Assert.ok(guid1 in changes);
+  Assert.equal(changeTime, changes[guid1]);
+
+  await tracker.clearChangedIDs();
+
+  _("Ensure reconciler changes are populated.");
+  let addon = await installAddon(XPI, reconciler);
+  await tracker.clearChangedIDs(); // Just in case.
+  changes = await engine.getChangedIDs();
+  Assert.equal("object", typeof changes);
+  Assert.equal(1, Object.keys(changes).length);
+  Assert.ok(addon.syncGUID in changes);
+  _(
+    "Change time: " + changeTime + ", addon change: " + changes[addon.syncGUID]
+  );
+  Assert.greaterOrEqual(changes[addon.syncGUID], changeTime);
+
+  let oldTime = changes[addon.syncGUID];
+  let guid2 = addon.syncGUID;
+  await uninstallAddon(addon, reconciler);
+  changes = await engine.getChangedIDs();
+  Assert.equal(1, Object.keys(changes).length);
+  Assert.ok(guid2 in changes);
+  Assert.greater(changes[guid2], oldTime);
+
+  _("Ensure non-syncable add-ons aren't picked up by reconciler changes.");
+  reconciler._addons = {};
+  reconciler._changes = [];
+  let record = {
+    id: "DUMMY",
+    guid: Utils.makeGUID(),
+    enabled: true,
+    installed: true,
+    modified: new Date(),
+    type: "UNSUPPORTED",
+    scope: 0,
+    foreignInstall: false,
+  };
+  reconciler.addons.DUMMY = record;
+  await reconciler._addChange(record.modified, CHANGE_INSTALLED, record);
+
+  changes = await engine.getChangedIDs();
+  _(JSON.stringify(changes));
+  Assert.equal(0, Object.keys(changes).length);
+
+  await resetReconciler();
+});
+
+add_task(async function test_disabled_install_semantics() {
+  _("Ensure that syncing a disabled add-on preserves proper state.");
+
+  // This is essentially a test for bug 712542, which snuck into the original
+  // add-on sync drop. It ensures that when an add-on is installed that the
+  // disabled state and incoming syncGUID is preserved, even on the next sync.
+  const USER = "foo";
+  const PASSWORD = "password";
+
+  let server = new SyncServer();
+  server.start();
+  await SyncTestingInfrastructure(server, USER, PASSWORD);
+
+  await generateNewKeys(Service.collectionKeys);
+
+  let contents = {
+    meta: {
+      global: { engines: { addons: { version: engine.version, syncID } } },
+    },
+    crypto: {},
+    addons: {},
+  };
+
+  server.registerUser(USER, "password");
+  server.createContents(USER, contents);
+
+  let amoServer = new HttpServer();
+  amoServer.registerFile(
+    "/search/guid:addon1%40tests.mozilla.org",
+    do_get_file("addon1-search.json")
+  );
+
+  amoServer.registerFile("/addon1.xpi", XPI);
+  amoServer.start(8888);
+
+  // Insert an existing record into the server.
+  let id = Utils.makeGUID();
+  let now = Date.now() / 1000;
+
+  let record = encryptPayload({
+    id,
+    applicationID: Services.appinfo.ID,
+    addonID: ADDON_ID,
+    enabled: false,
+    deleted: false,
+    source: "amo",
+  });
+  let wbo = new ServerWBO(id, record, now - 2);
+  server.insertWBO(USER, "addons", wbo);
+
+  _("Performing sync of add-ons engine.");
+  await engine._sync();
+
+  // At this point the non-restartless extension should be staged for install.
+
+  // Don't need this server any more.
+  await promiseStopServer(amoServer);
+
+  // We ensure the reconciler has recorded the proper ID and enabled state.
+  let addon = reconciler.getAddonStateFromSyncGUID(id);
+  Assert.notEqual(null, addon);
+  Assert.equal(false, addon.enabled);
+
+  // We fake an app restart and perform another sync, just to make sure things
+  // are sane.
+  await AddonTestUtils.promiseRestartManager();
+
+  let collection = server.getCollection(USER, "addons");
+  engine.lastModified = collection.timestamp;
+  await engine._sync();
+
+  // The client should not upload a new record. The old record should be
+  // retained and unmodified.
+  Assert.equal(1, collection.count());
+
+  let payload = collection.payloads()[0];
+  Assert.notEqual(null, collection.wbo(id));
+  Assert.equal(ADDON_ID, payload.addonID);
+  Assert.ok(!payload.enabled);
+
+  await promiseStopServer(server);
+});
+
+add_test(function cleanup() {
+  // There's an xpcom-shutdown hook for this, but let's give this a shot.
+  reconciler.stopListening();
+  run_next_test();
+});

src/zen/tests/mochitests/sync/test_addons_reconciler.js

@@ -0,0 +1,209 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const { AddonsReconciler, CHANGE_INSTALLED, CHANGE_UNINSTALLED } =
+  ChromeUtils.importESModule(
+    "resource://services-sync/addonsreconciler.sys.mjs"
+  );
+const { AddonsEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/addons.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+AddonTestUtils.init(this);
+AddonTestUtils.createAppInfo(
+  "xpcshell@tests.mozilla.org",
+  "XPCShell",
+  "1",
+  "1.9.2"
+);
+AddonTestUtils.overrideCertDB();
+
+const ADDON_ID = "addon1@tests.mozilla.org";
+const XPI = AddonTestUtils.createTempWebExtensionFile({
+  manifest: {
+    name: "Test 1",
+    description: "Test Description",
+    browser_specific_settings: { gecko: { id: ADDON_ID } },
+  },
+});
+
+function makeAddonsReconciler() {
+  const log = Service.engineManager.get("addons")._log;
+  const queueCaller = Async.asyncQueueCaller(log);
+  return new AddonsReconciler(queueCaller);
+}
+
+add_task(async function setup() {
+  await AddonTestUtils.promiseStartupManager();
+  Svc.PrefBranch.setBoolPref("engine.addons", true);
+  await Service.engineManager.register(AddonsEngine);
+});
+
+add_task(async function test_defaults() {
+  _("Ensure new objects have reasonable defaults.");
+
+  let reconciler = makeAddonsReconciler();
+  await reconciler.ensureStateLoaded();
+
+  Assert.ok(!reconciler._listening);
+  Assert.equal("object", typeof reconciler.addons);
+  Assert.equal(0, Object.keys(reconciler.addons).length);
+  Assert.equal(0, reconciler._changes.length);
+  Assert.equal(0, reconciler._listeners.length);
+});
+
+add_task(async function test_load_state_empty_file() {
+  _("Ensure loading from a missing file results in defaults being set.");
+
+  let reconciler = makeAddonsReconciler();
+  await reconciler.ensureStateLoaded();
+
+  let loaded = await reconciler.loadState();
+  Assert.ok(!loaded);
+
+  Assert.equal("object", typeof reconciler.addons);
+  Assert.equal(0, Object.keys(reconciler.addons).length);
+  Assert.equal(0, reconciler._changes.length);
+});
+
+add_task(async function test_install_detection() {
+  _("Ensure that add-on installation results in appropriate side-effects.");
+
+  let reconciler = makeAddonsReconciler();
+  await reconciler.ensureStateLoaded();
+  reconciler.startListening();
+
+  let before = new Date();
+  let addon = await installAddon(XPI);
+  let after = new Date();
+
+  Assert.equal(1, Object.keys(reconciler.addons).length);
+  Assert.ok(addon.id in reconciler.addons);
+  let record = reconciler.addons[ADDON_ID];
+
+  const KEYS = [
+    "id",
+    "guid",
+    "enabled",
+    "installed",
+    "modified",
+    "type",
+    "scope",
+    "foreignInstall",
+  ];
+  for (let key of KEYS) {
+    Assert.ok(key in record);
+    Assert.notEqual(null, record[key]);
+  }
+
+  Assert.equal(addon.id, record.id);
+  Assert.equal(addon.syncGUID, record.guid);
+  Assert.ok(record.enabled);
+  Assert.ok(record.installed);
+  Assert.ok(record.modified >= before && record.modified <= after);
+  Assert.equal("extension", record.type);
+  Assert.ok(!record.foreignInstall);
+
+  Assert.equal(1, reconciler._changes.length);
+  let change = reconciler._changes[0];
+  Assert.ok(change[0] >= before && change[1] <= after);
+  Assert.equal(CHANGE_INSTALLED, change[1]);
+  Assert.equal(addon.id, change[2]);
+
+  await uninstallAddon(addon);
+});
+
+add_task(async function test_uninstall_detection() {
+  _("Ensure that add-on uninstallation results in appropriate side-effects.");
+
+  let reconciler = makeAddonsReconciler();
+  await reconciler.ensureStateLoaded();
+  reconciler.startListening();
+
+  reconciler._addons = {};
+  reconciler._changes = [];
+
+  let addon = await installAddon(XPI);
+  let id = addon.id;
+
+  reconciler._changes = [];
+  await uninstallAddon(addon, reconciler);
+
+  Assert.equal(1, Object.keys(reconciler.addons).length);
+  Assert.ok(id in reconciler.addons);
+
+  let record = reconciler.addons[id];
+  Assert.ok(!record.installed);
+
+  Assert.equal(1, reconciler._changes.length);
+  let change = reconciler._changes[0];
+  Assert.equal(CHANGE_UNINSTALLED, change[1]);
+  Assert.equal(id, change[2]);
+});
+
+add_task(async function test_load_state_future_version() {
+  _("Ensure loading a file from a future version results in no data loaded.");
+
+  const FILENAME = "TEST_LOAD_STATE_FUTURE_VERSION";
+
+  let reconciler = makeAddonsReconciler();
+  await reconciler.ensureStateLoaded();
+
+  // First we populate our new file.
+  let state = { version: 100, addons: { foo: {} }, changes: [[1, 1, "foo"]] };
+
+  // jsonSave() expects an object with ._log, so we give it a reconciler
+  // instance.
+  await Utils.jsonSave(FILENAME, reconciler, state);
+
+  let loaded = await reconciler.loadState(FILENAME);
+  Assert.ok(!loaded);
+
+  Assert.equal("object", typeof reconciler.addons);
+  Assert.equal(0, Object.keys(reconciler.addons).length);
+  Assert.equal(0, reconciler._changes.length);
+});
+
+add_task(async function test_prune_changes_before_date() {
+  _("Ensure that old changes are pruned properly.");
+
+  let reconciler = makeAddonsReconciler();
+  await reconciler.ensureStateLoaded();
+  reconciler._changes = [];
+
+  let now = new Date();
+  const HOUR_MS = 1000 * 60 * 60;
+
+  _("Ensure pruning an empty changes array works.");
+  reconciler.pruneChangesBeforeDate(now);
+  Assert.equal(0, reconciler._changes.length);
+
+  let old = new Date(now.getTime() - HOUR_MS);
+  let young = new Date(now.getTime() - 1000);
+  reconciler._changes.push([old, CHANGE_INSTALLED, "foo"]);
+  reconciler._changes.push([young, CHANGE_INSTALLED, "bar"]);
+  Assert.equal(2, reconciler._changes.length);
+
+  _("Ensure pruning with an old time won't delete anything.");
+  let threshold = new Date(old.getTime() - 1);
+  reconciler.pruneChangesBeforeDate(threshold);
+  Assert.equal(2, reconciler._changes.length);
+
+  _("Ensure pruning a single item works.");
+  threshold = new Date(young.getTime() - 1000);
+  reconciler.pruneChangesBeforeDate(threshold);
+  Assert.equal(1, reconciler._changes.length);
+  Assert.notEqual(undefined, reconciler._changes[0]);
+  Assert.equal(young, reconciler._changes[0][0]);
+  Assert.equal("bar", reconciler._changes[0][2]);
+
+  _("Ensure pruning all changes works.");
+  reconciler._changes.push([old, CHANGE_INSTALLED, "foo"]);
+  reconciler.pruneChangesBeforeDate(now);
+  Assert.equal(0, reconciler._changes.length);
+});

src/zen/tests/mochitests/sync/test_addons_store.js

@@ -0,0 +1,933 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const { AddonsEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/addons.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { SyncedRecordsTelemetry } = ChromeUtils.importESModule(
+  "resource://services-sync/telemetry.sys.mjs"
+);
+
+const HTTP_PORT = 8888;
+
+Services.prefs.setStringPref(
+  "extensions.getAddons.get.url",
+  "http://localhost:8888/search/guid:%IDS%"
+);
+// Note that all compat-override URLs currently 404, but that's OK - the main
+// thing is to avoid us hitting the real AMO.
+Services.prefs.setStringPref(
+  "extensions.getAddons.compatOverides.url",
+  "http://localhost:8888/compat-override/guid:%IDS%"
+);
+Services.prefs.setBoolPref("extensions.install.requireSecureOrigin", false);
+Services.prefs.setBoolPref("extensions.checkUpdateSecurity", false);
+
+AddonTestUtils.init(this);
+AddonTestUtils.createAppInfo(
+  "xpcshell@tests.mozilla.org",
+  "XPCShell",
+  "1",
+  "1.9.2"
+);
+AddonTestUtils.overrideCertDB();
+
+Services.prefs.setBoolPref("extensions.experiments.enabled", true);
+
+const SYSTEM_ADDON_ID = "system1@tests.mozilla.org";
+const THEME_ID = "synctheme@tests.mozilla.org";
+
+add_setup(async function setupBuiltInAddon() {
+  // Enable SCOPE_APPLICATION for builtin testing.  Default in tests is only SCOPE_PROFILE.
+  let scopes = AddonManager.SCOPE_PROFILE | AddonManager.SCOPE_APPLICATION;
+  Services.prefs.setIntPref("extensions.enabledScopes", scopes);
+
+  const addon_version = "1.0";
+  const addon_res_url_path = "test-builtin-addon";
+
+  let xpi = await AddonTestUtils.createTempWebExtensionFile({
+    manifest: {
+      version: addon_version,
+      browser_specific_settings: { gecko: { id: SYSTEM_ADDON_ID } },
+    },
+  });
+
+  // The built-in location requires a resource: URL that maps to a
+  // jar: or file: URL.  This would typically be something bundled
+  // into omni.ja but for testing we just use a temp file.
+  let base = Services.io.newURI(`jar:file:${xpi.path}!/`);
+  let resProto = Services.io
+    .getProtocolHandler("resource")
+    .QueryInterface(Ci.nsIResProtocolHandler);
+  resProto.setSubstitution(addon_res_url_path, base);
+  let builtins = [
+    {
+      addon_id: SYSTEM_ADDON_ID,
+      addon_version,
+      res_url: `resource://${addon_res_url_path}/`,
+    },
+  ];
+  await AddonTestUtils.overrideBuiltIns({ builtins });
+  await AddonTestUtils.promiseStartupManager();
+});
+
+const ID1 = "addon1@tests.mozilla.org";
+const ID2 = "addon2@tests.mozilla.org";
+const ID3 = "addon3@tests.mozilla.org";
+
+const ADDONS = {
+  test_addon1: {
+    manifest: {
+      browser_specific_settings: {
+        gecko: {
+          id: ID1,
+          update_url: "http://example.com/data/test_install.json",
+        },
+      },
+    },
+  },
+
+  test_addon2: {
+    manifest: {
+      browser_specific_settings: { gecko: { id: ID2 } },
+    },
+  },
+
+  test_addon3: {
+    manifest: {
+      browser_specific_settings: {
+        gecko: {
+          id: ID3,
+          strict_max_version: "0",
+        },
+      },
+    },
+  },
+};
+
+const SEARCH_RESULT = {
+  next: null,
+  results: [
+    {
+      name: "Test Extension",
+      type: "extension",
+      guid: "addon1@tests.mozilla.org",
+      current_version: {
+        version: "1.0",
+        files: [
+          {
+            platform: "all",
+            size: 485,
+            url: "http://localhost:8888/addon1.xpi",
+          },
+        ],
+      },
+      last_updated: "2018-10-27T04:12:00.826Z",
+    },
+  ],
+};
+
+const MISSING_SEARCH_RESULT = {
+  next: null,
+  results: [
+    {
+      name: "Test",
+      type: "extension",
+      guid: "missing-xpi@tests.mozilla.org",
+      current_version: {
+        version: "1.0",
+        files: [
+          {
+            platform: "all",
+            size: 123,
+            url: "http://localhost:8888/THIS_DOES_NOT_EXIST.xpi",
+          },
+        ],
+      },
+    },
+  ],
+};
+
+const AMOSIGNED_SHA1_SEARCH_RESULT = {
+  next: null,
+  results: [
+    {
+      name: "Test Extension",
+      type: "extension",
+      guid: "amosigned-xpi@tests.mozilla.org",
+      current_version: {
+        version: "2.1",
+        files: [
+          {
+            platform: "all",
+            size: 4287,
+            url: "http://localhost:8888/amosigned-sha1only.xpi",
+          },
+        ],
+      },
+      last_updated: "2024-03-21T16:00:06.640Z",
+    },
+  ],
+};
+
+const XPIS = {};
+for (let [name, files] of Object.entries(ADDONS)) {
+  XPIS[name] = AddonTestUtils.createTempWebExtensionFile(files);
+}
+
+let engine;
+let store;
+let reconciler;
+
+const proxyService = Cc[
+  "@mozilla.org/network/protocol-proxy-service;1"
+].getService(Ci.nsIProtocolProxyService);
+
+const proxyFilter = {
+  proxyInfo: proxyService.newProxyInfo(
+    "http",
+    "localhost",
+    HTTP_PORT,
+    "",
+    "",
+    0,
+    4096,
+    null
+  ),
+
+  applyFilter(channel, defaultProxyInfo, callback) {
+    if (channel.URI.host === "example.com") {
+      callback.onProxyFilterResult(this.proxyInfo);
+    } else {
+      callback.onProxyFilterResult(defaultProxyInfo);
+    }
+  },
+};
+
+proxyService.registerChannelFilter(proxyFilter, 0);
+registerCleanupFunction(() => {
+  proxyService.unregisterChannelFilter(proxyFilter);
+});
+
+/**
+ * Create a AddonsRec for this application with the fields specified.
+ *
+ * @param  id       Sync GUID of record
+ * @param  addonId  ID of add-on
+ * @param  enabled  Boolean whether record is enabled
+ * @param  deleted  Boolean whether record was deleted
+ */
+function createRecordForThisApp(id, addonId, enabled, deleted) {
+  return {
+    id,
+    addonID: addonId,
+    enabled,
+    deleted: !!deleted,
+    applicationID: Services.appinfo.ID,
+    source: "amo",
+  };
+}
+
+function createAndStartHTTPServer(port) {
+  try {
+    let server = new HttpServer();
+
+    server.registerPathHandler(
+      "/search/guid:addon1%40tests.mozilla.org",
+      (req, resp) => {
+        resp.setHeader("Content-type", "application/json", true);
+        resp.write(JSON.stringify(SEARCH_RESULT));
+      }
+    );
+    server.registerPathHandler(
+      "/search/guid:missing-xpi%40tests.mozilla.org",
+      (req, resp) => {
+        resp.setHeader("Content-type", "application/json", true);
+        resp.write(JSON.stringify(MISSING_SEARCH_RESULT));
+      }
+    );
+    server.registerFile("/addon1.xpi", XPIS.test_addon1);
+
+    server.registerPathHandler(
+      "/search/guid:amosigned-xpi%40tests.mozilla.org",
+      (req, resp) => {
+        resp.setHeader("Content-type", "application/json", true);
+        resp.write(JSON.stringify(AMOSIGNED_SHA1_SEARCH_RESULT));
+      }
+    );
+    server.registerFile(
+      "/amosigned-sha1only.xpi",
+      do_get_file("amosigned-sha1only.xpi")
+    );
+
+    server.start(port);
+
+    return server;
+  } catch (ex) {
+    _("Got exception starting HTTP server on port " + port);
+    _("Error: " + Log.exceptionStr(ex));
+    do_throw(ex);
+  }
+  return null; /* not hit, but keeps eslint happy! */
+}
+
+// A helper function to ensure that the reconciler's current view of the addon
+// is the same as the addon itself. If it's not, then the reconciler missed a
+// change, and is likely to re-upload the addon next sync because of the change
+// it missed.
+async function checkReconcilerUpToDate(addon) {
+  let stateBefore = Object.assign({}, store.reconciler.addons[addon.id]);
+  await store.reconciler.rectifyStateFromAddon(addon);
+  let stateAfter = store.reconciler.addons[addon.id];
+  deepEqual(stateBefore, stateAfter);
+}
+
+add_setup(async function setup() {
+  await Service.engineManager.register(AddonsEngine);
+  engine = Service.engineManager.get("addons");
+  store = engine._store;
+  reconciler = engine._reconciler;
+
+  reconciler.startListening();
+
+  // Don't flush to disk in the middle of an event listener!
+  // This causes test hangs on WinXP.
+  reconciler._shouldPersist = false;
+});
+
+add_task(async function test_remove() {
+  _("Ensure removing add-ons from deleted records works.");
+
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+  let record = createRecordForThisApp(addon.syncGUID, ID1, true, true);
+  let countTelemetry = new SyncedRecordsTelemetry();
+  let failed = await store.applyIncomingBatch([record], countTelemetry);
+  Assert.equal(0, failed.length);
+  Assert.equal(null, countTelemetry.failedReasons);
+  Assert.equal(0, countTelemetry.incomingCounts.failed);
+
+  let newAddon = await AddonManager.getAddonByID(ID1);
+  Assert.equal(null, newAddon);
+});
+
+add_task(async function test_apply_enabled() {
+  let countTelemetry = new SyncedRecordsTelemetry();
+  _("Ensures that changes to the userEnabled flag apply.");
+
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+  Assert.ok(addon.isActive);
+  Assert.ok(!addon.userDisabled);
+
+  _("Ensure application of a disable record works as expected.");
+  let records = [];
+  records.push(createRecordForThisApp(addon.syncGUID, ID1, false, false));
+
+  let [failed] = await Promise.all([
+    store.applyIncomingBatch(records, countTelemetry),
+    AddonTestUtils.promiseAddonEvent("onDisabled"),
+  ]);
+  Assert.equal(0, failed.length);
+  Assert.equal(0, countTelemetry.incomingCounts.failed);
+  addon = await AddonManager.getAddonByID(ID1);
+  Assert.ok(addon.userDisabled);
+  await checkReconcilerUpToDate(addon);
+  records = [];
+
+  _("Ensure enable record works as expected.");
+  records.push(createRecordForThisApp(addon.syncGUID, ID1, true, false));
+  [failed] = await Promise.all([
+    store.applyIncomingBatch(records, countTelemetry),
+    AddonTestUtils.promiseWebExtensionStartup(ID1),
+  ]);
+  Assert.equal(0, failed.length);
+  Assert.equal(0, countTelemetry.incomingCounts.failed);
+  addon = await AddonManager.getAddonByID(ID1);
+  Assert.ok(!addon.userDisabled);
+  await checkReconcilerUpToDate(addon);
+  records = [];
+
+  _("Ensure enabled state updates don't apply if the ignore pref is set.");
+  records.push(createRecordForThisApp(addon.syncGUID, ID1, false, false));
+  Svc.PrefBranch.setBoolPref("addons.ignoreUserEnabledChanges", true);
+  failed = await store.applyIncomingBatch(records, countTelemetry);
+  Assert.equal(0, failed.length);
+  Assert.equal(0, countTelemetry.incomingCounts.failed);
+  addon = await AddonManager.getAddonByID(ID1);
+  Assert.ok(!addon.userDisabled);
+  records = [];
+
+  await uninstallAddon(addon, reconciler);
+  Svc.PrefBranch.clearUserPref("addons.ignoreUserEnabledChanges");
+});
+
+add_task(async function test_apply_enabled_appDisabled() {
+  _(
+    "Ensures that changes to the userEnabled flag apply when the addon is appDisabled."
+  );
+
+  // this addon is appDisabled by default.
+  let addon = await installAddon(XPIS.test_addon3);
+  Assert.ok(addon.appDisabled);
+  Assert.ok(!addon.isActive);
+  Assert.ok(!addon.userDisabled);
+
+  _("Ensure application of a disable record works as expected.");
+  store.reconciler.pruneChangesBeforeDate(Date.now() + 10);
+  store.reconciler._changes = [];
+  let records = [];
+  let countTelemetry = new SyncedRecordsTelemetry();
+  records.push(createRecordForThisApp(addon.syncGUID, ID3, false, false));
+  let failed = await store.applyIncomingBatch(records, countTelemetry);
+  Assert.equal(0, failed.length);
+  Assert.equal(0, countTelemetry.incomingCounts.failed);
+  addon = await AddonManager.getAddonByID(ID3);
+  Assert.ok(addon.userDisabled);
+  await checkReconcilerUpToDate(addon);
+  records = [];
+
+  _("Ensure enable record works as expected.");
+  records.push(createRecordForThisApp(addon.syncGUID, ID3, true, false));
+  failed = await store.applyIncomingBatch(records, countTelemetry);
+  Assert.equal(0, failed.length);
+  Assert.equal(0, countTelemetry.incomingCounts.failed);
+  addon = await AddonManager.getAddonByID(ID3);
+  Assert.ok(!addon.userDisabled);
+  await checkReconcilerUpToDate(addon);
+  records = [];
+
+  await uninstallAddon(addon, reconciler);
+});
+
+add_task(async function test_ignore_different_appid() {
+  _(
+    "Ensure that incoming records with a different application ID are ignored."
+  );
+
+  // We test by creating a record that should result in an update.
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+  Assert.ok(!addon.userDisabled);
+
+  let record = createRecordForThisApp(addon.syncGUID, ID1, false, false);
+  record.applicationID = "FAKE_ID";
+  let countTelemetry = new SyncedRecordsTelemetry();
+  let failed = await store.applyIncomingBatch([record], countTelemetry);
+  Assert.equal(0, failed.length);
+
+  let newAddon = await AddonManager.getAddonByID(ID1);
+  Assert.ok(!newAddon.userDisabled);
+
+  await uninstallAddon(addon, reconciler);
+});
+
+add_task(async function test_ignore_unknown_source() {
+  _("Ensure incoming records with unknown source are ignored.");
+
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+
+  let record = createRecordForThisApp(addon.syncGUID, ID1, false, false);
+  record.source = "DUMMY_SOURCE";
+  let countTelemetry = new SyncedRecordsTelemetry();
+  let failed = await store.applyIncomingBatch([record], countTelemetry);
+  Assert.equal(0, failed.length);
+
+  let newAddon = await AddonManager.getAddonByID(ID1);
+  Assert.ok(!newAddon.userDisabled);
+
+  await uninstallAddon(addon, reconciler);
+});
+
+add_task(async function test_apply_uninstall() {
+  _("Ensures that uninstalling an add-on from a record works.");
+
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+
+  let records = [];
+  let countTelemetry = new SyncedRecordsTelemetry();
+  records.push(createRecordForThisApp(addon.syncGUID, ID1, true, true));
+  let failed = await store.applyIncomingBatch(records, countTelemetry);
+  Assert.equal(0, failed.length);
+  Assert.equal(0, countTelemetry.incomingCounts.failed);
+
+  addon = await AddonManager.getAddonByID(ID1);
+  Assert.equal(null, addon);
+});
+
+add_task(async function test_addon_syncability() {
+  _("Ensure isAddonSyncable functions properly.");
+
+  Svc.PrefBranch.setStringPref(
+    "addons.trustedSourceHostnames",
+    "addons.mozilla.org,other.example.com"
+  );
+
+  Assert.ok(!(await store.isAddonSyncable(null)));
+
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+  Assert.ok(await store.isAddonSyncable(addon));
+
+  let dummy = {};
+  const KEYS = [
+    "id",
+    "syncGUID",
+    "type",
+    "scope",
+    "foreignInstall",
+    "isSyncable",
+  ];
+  for (let k of KEYS) {
+    dummy[k] = addon[k];
+  }
+
+  Assert.ok(await store.isAddonSyncable(dummy));
+
+  dummy.type = "UNSUPPORTED";
+  Assert.ok(!(await store.isAddonSyncable(dummy)));
+  dummy.type = addon.type;
+
+  dummy.scope = 0;
+  Assert.ok(!(await store.isAddonSyncable(dummy)));
+  dummy.scope = addon.scope;
+
+  dummy.isSyncable = false;
+  Assert.ok(!(await store.isAddonSyncable(dummy)));
+  dummy.isSyncable = addon.isSyncable;
+
+  dummy.foreignInstall = true;
+  Assert.ok(!(await store.isAddonSyncable(dummy)));
+  dummy.foreignInstall = false;
+
+  await uninstallAddon(addon, reconciler);
+
+  Assert.ok(!store.isSourceURITrusted(null));
+
+  let trusted = [
+    "https://addons.mozilla.org/foo",
+    "https://other.example.com/foo",
+  ];
+
+  let untrusted = [
+    "http://addons.mozilla.org/foo", // non-https
+    "ftps://addons.mozilla.org/foo", // non-https
+    "https://untrusted.example.com/foo", // non-trusted hostname`
+  ];
+
+  for (let uri of trusted) {
+    Assert.ok(store.isSourceURITrusted(Services.io.newURI(uri)));
+  }
+
+  for (let uri of untrusted) {
+    Assert.ok(!store.isSourceURITrusted(Services.io.newURI(uri)));
+  }
+
+  Svc.PrefBranch.setStringPref("addons.trustedSourceHostnames", "");
+  for (let uri of trusted) {
+    Assert.ok(!store.isSourceURITrusted(Services.io.newURI(uri)));
+  }
+
+  Svc.PrefBranch.setStringPref(
+    "addons.trustedSourceHostnames",
+    "addons.mozilla.org"
+  );
+  Assert.ok(
+    store.isSourceURITrusted(
+      Services.io.newURI("https://addons.mozilla.org/foo")
+    )
+  );
+
+  Svc.PrefBranch.clearUserPref("addons.trustedSourceHostnames");
+});
+
+add_task(async function test_get_all_ids() {
+  _("Ensures that getAllIDs() returns an appropriate set.");
+
+  _("Installing two addons.");
+  // XXX - this test seems broken - at this point, before we've installed the
+  // addons below, store.getAllIDs() returns all addons installed by previous
+  // tests, even though those tests uninstalled the addon.
+  // So if any tests above ever add a new addon ID, they are going to need to
+  // be added here too.
+  // Assert.equal(0, Object.keys(store.getAllIDs()).length);
+  let addon1 = await installAddon(XPIS.test_addon1, reconciler);
+  let addon2 = await installAddon(XPIS.test_addon2, reconciler);
+  let addon3 = await installAddon(XPIS.test_addon3, reconciler);
+
+  _("Ensure they're syncable.");
+  Assert.ok(await store.isAddonSyncable(addon1));
+  Assert.ok(await store.isAddonSyncable(addon2));
+  Assert.ok(await store.isAddonSyncable(addon3));
+
+  let ids = await store.getAllIDs();
+
+  Assert.equal("object", typeof ids);
+  Assert.equal(3, Object.keys(ids).length);
+  Assert.ok(addon1.syncGUID in ids);
+  Assert.ok(addon2.syncGUID in ids);
+  Assert.ok(addon3.syncGUID in ids);
+
+  await uninstallAddon(addon1, reconciler);
+  await uninstallAddon(addon2, reconciler);
+  await uninstallAddon(addon3, reconciler);
+});
+
+add_task(async function test_change_item_id() {
+  _("Ensures that changeItemID() works properly.");
+
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+
+  let oldID = addon.syncGUID;
+  let newID = Utils.makeGUID();
+
+  await store.changeItemID(oldID, newID);
+
+  let newAddon = await AddonManager.getAddonByID(ID1);
+  Assert.notEqual(null, newAddon);
+  Assert.equal(newID, newAddon.syncGUID);
+
+  await uninstallAddon(newAddon, reconciler);
+});
+
+add_task(async function test_create() {
+  _("Ensure creating/installing an add-on from a record works.");
+
+  let server = createAndStartHTTPServer(HTTP_PORT);
+
+  let guid = Utils.makeGUID();
+  let record = createRecordForThisApp(guid, ID1, true, false);
+  let countTelemetry = new SyncedRecordsTelemetry();
+  let failed = await store.applyIncomingBatch([record], countTelemetry);
+  Assert.equal(0, failed.length);
+
+  let newAddon = await AddonManager.getAddonByID(ID1);
+  Assert.notEqual(null, newAddon);
+  Assert.equal(guid, newAddon.syncGUID);
+  Assert.ok(!newAddon.userDisabled);
+
+  await uninstallAddon(newAddon, reconciler);
+
+  await promiseStopServer(server);
+});
+
+add_task(async function test_weak_signature_restrictions() {
+  _("Ensure installing add-ons with a weak signature fails when restricted.");
+
+  // Ensure restrictions on weak signatures are enabled (this should be removed when
+  // the new behavior is riding the train).
+  const resetWeakSignaturePref =
+    AddonTestUtils.setWeakSignatureInstallAllowed(false);
+  const server = createAndStartHTTPServer(HTTP_PORT);
+  const ID_TEST_SHA1 = "amosigned-xpi@tests.mozilla.org";
+
+  const guidKO = Utils.makeGUID();
+  const guidOK = Utils.makeGUID();
+  const recordKO = createRecordForThisApp(guidKO, ID_TEST_SHA1, true, false);
+  const recordOK = createRecordForThisApp(guidOK, ID1, true, false);
+  const countTelemetry = new SyncedRecordsTelemetry();
+
+  let failed;
+
+  const { messages } = await AddonTestUtils.promiseConsoleOutput(async () => {
+    failed = await store.applyIncomingBatch(
+      [recordKO, recordOK],
+      countTelemetry
+    );
+  });
+
+  Assert.equal(
+    1,
+    failed.length,
+    "Expect only 1 on the two synced add-ons to fail"
+  );
+
+  resetWeakSignaturePref();
+
+  let addonKO = await AddonManager.getAddonByID(ID_TEST_SHA1);
+  Assert.equal(null, addonKO, `Expect ${ID_TEST_SHA1} to NOT be installed`);
+  let addonOK = await AddonManager.getAddonByID(ID1);
+  Assert.notEqual(null, addonOK, `Expect ${ID1} to be installed`);
+
+  await uninstallAddon(addonOK, reconciler);
+  await promiseStopServer(server);
+
+  AddonTestUtils.checkMessages(messages, {
+    expected: [
+      {
+        message:
+          /Download of .*\/amosigned-sha1only.xpi failed: install rejected due to the package not including a strong cryptographic signature/,
+      },
+    ],
+  });
+});
+
+add_task(async function test_create_missing_search() {
+  _("Ensures that failed add-on searches are handled gracefully.");
+
+  let server = createAndStartHTTPServer(HTTP_PORT);
+
+  // The handler for this ID is not installed, so a search should 404.
+  const id = "missing@tests.mozilla.org";
+  let guid = Utils.makeGUID();
+  let record = createRecordForThisApp(guid, id, true, false);
+  let countTelemetry = new SyncedRecordsTelemetry();
+  let failed = await store.applyIncomingBatch([record], countTelemetry);
+  Assert.equal(1, failed.length);
+  Assert.equal(guid, failed[0]);
+  Assert.equal(
+    countTelemetry.incomingCounts.failedReasons[0].name,
+    "GET <URL> failed (status 404)"
+  );
+  Assert.equal(countTelemetry.incomingCounts.failedReasons[0].count, 1);
+
+  let addon = await AddonManager.getAddonByID(id);
+  Assert.equal(null, addon);
+
+  await promiseStopServer(server);
+});
+
+add_task(async function test_create_bad_install() {
+  _("Ensures that add-ons without a valid install are handled gracefully.");
+
+  let server = createAndStartHTTPServer(HTTP_PORT);
+
+  // The handler returns a search result but the XPI will 404.
+  const id = "missing-xpi@tests.mozilla.org";
+  let guid = Utils.makeGUID();
+  let record = createRecordForThisApp(guid, id, true, false);
+  let countTelemetry = new SyncedRecordsTelemetry();
+  /* let failed = */ await store.applyIncomingBatch([record], countTelemetry);
+  // This addon had no source URI so was skipped - but it's not treated as
+  // failure.
+  // XXX - this test isn't testing what we thought it was. Previously the addon
+  // was not being installed due to requireSecureURL checking *before* we'd
+  // attempted to get the XPI.
+  // With requireSecureURL disabled we do see a download failure, but the addon
+  // *does* get added to |failed|.
+  // FTR: onDownloadFailed() is called with ERROR_NETWORK_FAILURE, so it's going
+  // to be tricky to distinguish a 404 from other transient network errors
+  // where we do want the addon to end up in |failed|.
+  // This is being tracked in bug 1284778.
+  // Assert.equal(0, failed.length);
+
+  let addon = await AddonManager.getAddonByID(id);
+  Assert.equal(null, addon);
+
+  await promiseStopServer(server);
+});
+
+add_task(async function test_ignore_system() {
+  _("Ensure we ignore system addons");
+  // Our system addon should not appear in getAllIDs
+  await engine._refreshReconcilerState();
+  let num = 0;
+  let ids = await store.getAllIDs();
+  for (let guid in ids) {
+    num += 1;
+    let addon = reconciler.getAddonStateFromSyncGUID(guid);
+    Assert.notEqual(addon.id, SYSTEM_ADDON_ID);
+  }
+  Assert.greater(num, 1, "should have seen at least one.");
+});
+
+add_task(async function test_incoming_system() {
+  _("Ensure we handle incoming records that refer to a system addon");
+  // eg, loop initially had a normal addon but it was then "promoted" to be a
+  // system addon but wanted to keep the same ID. The server record exists due
+  // to this.
+
+  // before we start, ensure the system addon isn't disabled.
+  Assert.ok(!(await AddonManager.getAddonByID(SYSTEM_ADDON_ID).userDisabled));
+
+  // Now simulate an incoming record with the same ID as the system addon,
+  // but flagged as disabled - it should not be applied.
+  let server = createAndStartHTTPServer(HTTP_PORT);
+  // We make the incoming record flag the system addon as disabled - it should
+  // be ignored.
+  let guid = Utils.makeGUID();
+  let record = createRecordForThisApp(guid, SYSTEM_ADDON_ID, false, false);
+  let countTelemetry = new SyncedRecordsTelemetry();
+  let failed = await store.applyIncomingBatch([record], countTelemetry);
+  Assert.equal(0, failed.length);
+
+  // The system addon should still not be userDisabled.
+  Assert.ok(!(await AddonManager.getAddonByID(SYSTEM_ADDON_ID).userDisabled));
+
+  await promiseStopServer(server);
+});
+
+add_task(async function test_wipe() {
+  _("Ensures that wiping causes add-ons to be uninstalled.");
+
+  await installAddon(XPIS.test_addon1, reconciler);
+
+  await store.wipe();
+
+  let addon = await AddonManager.getAddonByID(ID1);
+  Assert.equal(null, addon);
+});
+
+add_task(async function test_wipe_and_install() {
+  _("Ensure wipe followed by install works.");
+
+  // This tests the reset sync flow where remote data is replaced by local. The
+  // receiving client will see a wipe followed by a record which should undo
+  // the wipe.
+  let installed = await installAddon(XPIS.test_addon1, reconciler);
+
+  let record = createRecordForThisApp(installed.syncGUID, ID1, true, false);
+
+  await store.wipe();
+
+  let deleted = await AddonManager.getAddonByID(ID1);
+  Assert.equal(null, deleted);
+
+  // Re-applying the record can require re-fetching the XPI.
+  let server = createAndStartHTTPServer(HTTP_PORT);
+
+  await store.applyIncoming(record);
+
+  let fetched = await AddonManager.getAddonByID(record.addonID);
+  Assert.ok(!!fetched);
+
+  // wipe again to we are left with a clean slate.
+  await store.wipe();
+
+  await promiseStopServer(server);
+});
+
+// STR for what this is testing:
+// * Either:
+//   * Install then remove an addon, then delete addons.json from the profile
+//     or corrupt it (in which case the addon manager will remove it)
+//   * Install then remove an addon while addon caching is disabled, then
+//     re-enable addon caching.
+// * Install the same addon in a different profile, sync it.
+// * Sync this profile
+// Before bug 1467904, the addon would fail to install because this profile
+// has a copy of the addon in our addonsreconciler.json, but the addon manager
+// does *not* have a copy in its cache, and repopulating that cache would not
+// re-add it as the addon is no longer installed locally.
+add_task(async function test_incoming_reconciled_but_not_cached() {
+  _(
+    "Ensure we handle incoming records our reconciler has but the addon cache does not"
+  );
+
+  // Make sure addon is not installed.
+  let addon = await AddonManager.getAddonByID(ID1);
+  Assert.equal(null, addon);
+
+  Services.prefs.setBoolPref("extensions.getAddons.cache.enabled", false);
+
+  addon = await installAddon(XPIS.test_addon1, reconciler);
+  Assert.notEqual(await AddonManager.getAddonByID(ID1), null);
+  await uninstallAddon(addon, reconciler);
+
+  Services.prefs.setBoolPref("extensions.getAddons.cache.enabled", true);
+
+  // now pretend it is incoming.
+  let server = createAndStartHTTPServer(HTTP_PORT);
+  let guid = Utils.makeGUID();
+  let record = createRecordForThisApp(guid, ID1, true, false);
+  let countTelemetry = new SyncedRecordsTelemetry();
+  let failed = await store.applyIncomingBatch([record], countTelemetry);
+  Assert.equal(0, failed.length);
+
+  Assert.notEqual(await AddonManager.getAddonByID(ID1), null);
+
+  await promiseStopServer(server);
+});
+
+// Helper for testing theme-specific addons
+function makeThemeSearchResult(id) {
+  return {
+    next: null,
+    results: [
+      {
+        name: "Sync Theme",
+        type: "theme",
+        guid: id,
+        current_version: {
+          version: "1.0",
+          files: [
+            {
+              platform: "all",
+              size: 1234,
+              url: `http://localhost:${HTTP_PORT}/synctheme.xpi`,
+            },
+          ],
+        },
+        last_updated: "2025-03-01T00:00:00.000Z",
+      },
+    ],
+  };
+}
+
+/**
+ * Incoming theme add-on record should
+ *   – install the theme
+ *   – enable it immediately
+ *   – clear the hand-off pref
+ */
+add_task(async function test_incoming_theme_gets_enabled() {
+  const xpiTheme = AddonTestUtils.createTempWebExtensionFile({
+    manifest: {
+      manifest_version: 2,
+      name: "Sync Theme",
+      version: "1.0",
+      applications: { gecko: { id: THEME_ID } },
+      theme: { colors: { frame: "#000000", tab_background_text: "#ffffff" } },
+    },
+  });
+
+  const server = createAndStartHTTPServer(HTTP_PORT);
+  server.registerFile("/synctheme.xpi", xpiTheme);
+  server.registerPathHandler(
+    `/search/guid:${encodeURIComponent(THEME_ID)}`,
+    (req, resp) => {
+      resp.setHeader("Content-Type", "application/json", true);
+      resp.write(JSON.stringify(makeThemeSearchResult(THEME_ID)));
+    }
+  );
+
+  // Pretend Prefs‑engine has just synced a new activeThemeID
+  Services.prefs.setStringPref("extensions.pendingActiveThemeID", THEME_ID);
+
+  // Feed an add-on record into the Addons engine.
+  const guid = Utils.makeGUID();
+  const record = createRecordForThisApp(guid, THEME_ID, true, false);
+  const telem = new SyncedRecordsTelemetry();
+
+  const onStartup = AddonTestUtils.promiseWebExtensionStartup(THEME_ID);
+  const failed = await store.applyIncomingBatch([record], telem);
+
+  Assert.equal(0, failed.length, "No records should fail to apply");
+  await onStartup;
+
+  const theme = await AddonManager.getAddonByID(THEME_ID);
+  Assert.ok(theme, "Theme is installed");
+  Assert.ok(theme.isActive, "Theme is active");
+  Assert.ok(!theme.userDisabled, "Theme is not user-disabled");
+
+  Assert.equal(
+    Services.prefs.getPrefType("extensions.pendingActiveThemeID"),
+    Ci.nsIPrefBranch.PREF_INVALID,
+    "Hand-off pref was cleared"
+  );
+
+  // Clean-up
+  await uninstallAddon(theme, reconciler);
+  await promiseStopServer(server);
+});
+
+// NOTE: The test above must be the last test run due to the addon cache
+// being trashed. It is probably possible to fix that by running, eg,
+// AddonRespository.backgroundUpdateCheck() to rebuild the cache, but that
+// requires implementing more AMO functionality in our test server
+
+add_task(async function cleanup() {
+  // There's an xpcom-shutdown hook for this, but let's give this a shot.
+  reconciler.stopListening();
+});

src/zen/tests/mochitests/sync/test_addons_tracker.js

@@ -0,0 +1,174 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const { AddonsEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/addons.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+AddonTestUtils.init(this);
+AddonTestUtils.createAppInfo(
+  "xpcshell@tests.mozilla.org",
+  "XPCShell",
+  "1",
+  "1.9.2"
+);
+AddonTestUtils.overrideCertDB();
+
+Services.prefs.setBoolPref("extensions.experiments.enabled", true);
+
+Svc.PrefBranch.setBoolPref("engine.addons", true);
+
+let reconciler;
+let tracker;
+
+const addon1ID = "addon1@tests.mozilla.org";
+
+const ADDONS = {
+  test_addon1: {
+    manifest: {
+      browser_specific_settings: { gecko: { id: addon1ID } },
+    },
+  },
+};
+
+const XPIS = {};
+
+async function cleanup() {
+  tracker.stop();
+
+  tracker.resetScore();
+  await tracker.clearChangedIDs();
+
+  reconciler._addons = {};
+  reconciler._changes = [];
+  await reconciler.saveState();
+}
+
+add_task(async function setup() {
+  await AddonTestUtils.promiseStartupManager();
+  for (let [name, data] of Object.entries(ADDONS)) {
+    XPIS[name] = AddonTestUtils.createTempWebExtensionFile(data);
+  }
+  await Service.engineManager.register(AddonsEngine);
+  let engine = Service.engineManager.get("addons");
+  reconciler = engine._reconciler;
+  tracker = engine._tracker;
+
+  await cleanup();
+});
+
+add_task(async function test_empty() {
+  _("Verify the tracker is empty to start with.");
+
+  Assert.equal(0, Object.keys(await tracker.getChangedIDs()).length);
+  Assert.equal(0, tracker.score);
+
+  await cleanup();
+});
+
+add_task(async function test_not_tracking() {
+  _("Ensures the tracker doesn't do anything when it isn't tracking.");
+
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+  await uninstallAddon(addon, reconciler);
+
+  Assert.equal(0, Object.keys(await tracker.getChangedIDs()).length);
+  Assert.equal(0, tracker.score);
+
+  await cleanup();
+});
+
+add_task(async function test_track_install() {
+  _("Ensure that installing an add-on notifies tracker.");
+
+  reconciler.startListening();
+
+  tracker.start();
+
+  Assert.equal(0, tracker.score);
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+  let changed = await tracker.getChangedIDs();
+
+  Assert.equal(1, Object.keys(changed).length);
+  Assert.ok(addon.syncGUID in changed);
+  Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score);
+
+  await uninstallAddon(addon, reconciler);
+  await cleanup();
+});
+
+add_task(async function test_track_uninstall() {
+  _("Ensure that uninstalling an add-on notifies tracker.");
+
+  reconciler.startListening();
+
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+  let guid = addon.syncGUID;
+  Assert.equal(0, tracker.score);
+
+  tracker.start();
+
+  await uninstallAddon(addon, reconciler);
+  let changed = await tracker.getChangedIDs();
+  Assert.equal(1, Object.keys(changed).length);
+  Assert.ok(guid in changed);
+  Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score);
+
+  await cleanup();
+});
+
+add_task(async function test_track_user_disable() {
+  _("Ensure that tracker sees disabling of add-on");
+
+  reconciler.startListening();
+
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+  Assert.ok(!addon.userDisabled);
+  Assert.ok(!addon.appDisabled);
+  Assert.ok(addon.isActive);
+
+  tracker.start();
+  Assert.equal(0, tracker.score);
+
+  _("Disabling add-on");
+  await addon.disable();
+  await reconciler.queueCaller.promiseCallsComplete();
+
+  let changed = await tracker.getChangedIDs();
+  Assert.equal(1, Object.keys(changed).length);
+  Assert.ok(addon.syncGUID in changed);
+  Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score);
+
+  await uninstallAddon(addon, reconciler);
+  await cleanup();
+});
+
+add_task(async function test_track_enable() {
+  _("Ensure that enabling a disabled add-on notifies tracker.");
+
+  reconciler.startListening();
+
+  let addon = await installAddon(XPIS.test_addon1, reconciler);
+  await addon.disable();
+  await Async.promiseYield();
+
+  Assert.equal(0, tracker.score);
+
+  tracker.start();
+  await addon.enable();
+  await Async.promiseYield();
+  await reconciler.queueCaller.promiseCallsComplete();
+
+  let changed = await tracker.getChangedIDs();
+  Assert.equal(1, Object.keys(changed).length);
+  Assert.ok(addon.syncGUID in changed);
+  Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score);
+
+  await uninstallAddon(addon, reconciler);
+  await cleanup();
+});

src/zen/tests/mochitests/sync/test_addons_validator.js

@@ -0,0 +1,65 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { AddonValidator } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/addons.sys.mjs"
+);
+
+function getDummyServerAndClient() {
+  return {
+    server: [
+      {
+        id: "1111",
+        applicationID: Services.appinfo.ID,
+        addonID: "synced-addon@example.com",
+        enabled: true,
+        source: "amo",
+        understood: true,
+      },
+    ],
+    client: [
+      {
+        syncGUID: "1111",
+        id: "synced-addon@example.com",
+        type: "extension",
+        isSystem: false,
+        isSyncable: true,
+      },
+      {
+        syncGUID: "2222",
+        id: "system-addon@example.com",
+        type: "extension",
+        isSystem: true,
+        isSyncable: false,
+      },
+      {
+        // Plugins don't have a `syncedGUID`, but we don't sync them, so we
+        // shouldn't report them as client duplicates.
+        id: "some-plugin",
+        type: "plugin",
+      },
+      {
+        id: "another-plugin",
+        type: "plugin",
+      },
+    ],
+  };
+}
+
+add_task(async function test_valid() {
+  let { server, client } = getDummyServerAndClient();
+  let validator = new AddonValidator({
+    _findDupe() {
+      return null;
+    },
+    isAddonSyncable(item) {
+      return item.type != "plugin";
+    },
+  });
+  let { problemData, clientRecords, records, deletedRecords } =
+    await validator.compareClientWithServer(client, server);
+  equal(clientRecords.length, 4);
+  equal(records.length, 1);
+  equal(deletedRecords.length, 0);
+  deepEqual(problemData, validator.emptyProblemData());
+});

src/zen/tests/mochitests/sync/test_bookmark_batch_fail.js

@@ -0,0 +1,25 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+_("Making sure a failing sync reports a useful error");
+// `Service` is used as a global in head_helpers.js.
+// eslint-disable-next-line no-unused-vars
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+add_bookmark_test(async function run_test(engine) {
+  await engine.initialize();
+  engine._syncStartup = async function () {
+    throw new Error("FAIL!");
+  };
+
+  try {
+    _("Try calling the sync that should throw right away");
+    await engine._sync();
+    do_throw("Should have failed sync!");
+  } catch (ex) {
+    _("Making sure what we threw ended up as the exception:", ex);
+    Assert.equal(ex.message, "FAIL!");
+  }
+});

src/zen/tests/mochitests/sync/test_bookmark_decline_undecline.js

@@ -0,0 +1,48 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+// A stored reference to the collection won't be valid after disabling.
+function getBookmarkWBO(server, guid) {
+  let coll = server.user("foo").collection("bookmarks");
+  if (!coll) {
+    return null;
+  }
+  return coll.wbo(guid);
+}
+
+add_task(async function test_decline_undecline() {
+  let engine = Service.engineManager.get("bookmarks");
+  let server = await serverForFoo(engine);
+  await SyncTestingInfrastructure(server);
+
+  try {
+    let { guid: bzGuid } = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+      url: "https://bugzilla.mozilla.org",
+      index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+      title: "bugzilla",
+    });
+
+    ok(!getBookmarkWBO(server, bzGuid), "Shouldn't have been uploaded yet");
+    await Service.sync();
+    ok(getBookmarkWBO(server, bzGuid), "Should be present on server");
+
+    engine.enabled = false;
+    await Service.sync();
+    ok(
+      !getBookmarkWBO(server, bzGuid),
+      "Shouldn't be present on server anymore"
+    );
+
+    engine.enabled = true;
+    await Service.sync();
+    ok(getBookmarkWBO(server, bzGuid), "Should be present on server again");
+  } finally {
+    await PlacesSyncUtils.bookmarks.reset();
+    await promiseStopServer(server);
+  }
+});

src/zen/tests/mochitests/sync/test_bookmark_order.js

@@ -0,0 +1,586 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+_(
+  "Making sure after processing incoming bookmarks, they show up in the right order"
+);
+const { Bookmark, BookmarkFolder } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/bookmarks.sys.mjs"
+);
+const { Weave } = ChromeUtils.importESModule(
+  "resource://services-sync/main.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+async function serverForFoo(engine) {
+  await generateNewKeys(Service.collectionKeys);
+
+  let clientsEngine = Service.clientsEngine;
+  let clientsSyncID = await clientsEngine.resetLocalSyncID();
+  let engineSyncID = await engine.resetLocalSyncID();
+  return serverForUsers(
+    { foo: "password" },
+    {
+      meta: {
+        global: {
+          syncID: Service.syncID,
+          storageVersion: STORAGE_VERSION,
+          engines: {
+            clients: {
+              version: clientsEngine.version,
+              syncID: clientsSyncID,
+            },
+            [engine.name]: {
+              version: engine.version,
+              syncID: engineSyncID,
+            },
+          },
+        },
+      },
+      crypto: {
+        keys: encryptPayload({
+          id: "keys",
+          // Generate a fake default key bundle to avoid resetting the client
+          // before the first sync.
+          default: [
+            await Weave.Crypto.generateRandomKey(),
+            await Weave.Crypto.generateRandomKey(),
+          ],
+        }),
+      },
+      [engine.name]: {},
+    }
+  );
+}
+
+async function resolveConflict(
+  engine,
+  collection,
+  timestamp,
+  buildTree,
+  message
+) {
+  let guids = {
+    // These items don't exist on the server.
+    fx: Utils.makeGUID(),
+    nightly: Utils.makeGUID(),
+    support: Utils.makeGUID(),
+    customize: Utils.makeGUID(),
+
+    // These exist on the server, but in a different order, and `res`
+    // has completely different children.
+    res: Utils.makeGUID(),
+    tb: Utils.makeGUID(),
+
+    // These don't exist locally.
+    bz: Utils.makeGUID(),
+    irc: Utils.makeGUID(),
+    mdn: Utils.makeGUID(),
+  };
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: guids.fx,
+        title: "Get Firefox!",
+        url: "http://getfirefox.com/",
+      },
+      {
+        guid: guids.res,
+        title: "Resources",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            guid: guids.nightly,
+            title: "Nightly",
+            url: "https://nightly.mozilla.org/",
+          },
+          {
+            guid: guids.support,
+            title: "Support",
+            url: "https://support.mozilla.org/",
+          },
+          {
+            guid: guids.customize,
+            title: "Customize",
+            url: "https://mozilla.org/firefox/customize/",
+          },
+        ],
+      },
+      {
+        title: "Get Thunderbird!",
+        guid: guids.tb,
+        url: "http://getthunderbird.com/",
+      },
+    ],
+  });
+
+  let serverRecords = [
+    {
+      id: "menu",
+      type: "folder",
+      title: "Bookmarks Menu",
+      parentid: "places",
+      children: [guids.tb, guids.res],
+    },
+    {
+      id: guids.tb,
+      type: "bookmark",
+      parentid: "menu",
+      bmkUri: "http://getthunderbird.com/",
+      title: "Get Thunderbird!",
+    },
+    {
+      id: guids.res,
+      type: "folder",
+      parentid: "menu",
+      title: "Resources",
+      children: [guids.irc, guids.bz, guids.mdn],
+    },
+    {
+      id: guids.bz,
+      type: "bookmark",
+      parentid: guids.res,
+      bmkUri: "https://bugzilla.mozilla.org/",
+      title: "Bugzilla",
+    },
+    {
+      id: guids.mdn,
+      type: "bookmark",
+      parentid: guids.res,
+      bmkUri: "https://developer.mozilla.org/",
+      title: "MDN",
+    },
+    {
+      id: guids.irc,
+      type: "bookmark",
+      parentid: guids.res,
+      bmkUri: "ircs://irc.mozilla.org/nightly",
+      title: "IRC",
+    },
+  ];
+  for (let record of serverRecords) {
+    collection.insert(record.id, encryptPayload(record), timestamp);
+  }
+
+  engine.lastModified = collection.timestamp;
+  await sync_engine_and_validate_telem(engine, false);
+
+  let expectedTree = buildTree(guids);
+  await assertBookmarksTreeMatches(
+    PlacesUtils.bookmarks.menuGuid,
+    expectedTree,
+    message
+  );
+}
+
+async function get_engine() {
+  return Service.engineManager.get("bookmarks");
+}
+
+add_task(async function test_local_order_newer() {
+  let engine = await get_engine();
+  let server = await serverForFoo(engine);
+  await SyncTestingInfrastructure(server);
+
+  try {
+    let collection = server.user("foo").collection("bookmarks");
+    let serverModified = Date.now() / 1000 - 120;
+    await resolveConflict(
+      engine,
+      collection,
+      serverModified,
+      guids => [
+        {
+          guid: guids.fx,
+          index: 0,
+        },
+        {
+          guid: guids.res,
+          index: 1,
+          children: [
+            {
+              guid: guids.nightly,
+              index: 0,
+            },
+            {
+              guid: guids.support,
+              index: 1,
+            },
+            {
+              guid: guids.customize,
+              index: 2,
+            },
+            {
+              guid: guids.irc,
+              index: 3,
+            },
+            {
+              guid: guids.bz,
+              index: 4,
+            },
+            {
+              guid: guids.mdn,
+              index: 5,
+            },
+          ],
+        },
+        {
+          guid: guids.tb,
+          index: 2,
+        },
+      ],
+      "Should use local order as base if remote is older"
+    );
+  } finally {
+    await engine.wipeClient();
+    await Service.startOver();
+    await promiseStopServer(server);
+  }
+});
+
+add_task(async function test_remote_order_newer() {
+  let engine = await get_engine();
+  let server = await serverForFoo(engine);
+  await SyncTestingInfrastructure(server);
+
+  try {
+    let collection = server.user("foo").collection("bookmarks");
+    let serverModified = Date.now() / 1000 + 120;
+    await resolveConflict(
+      engine,
+      collection,
+      serverModified,
+      guids => [
+        {
+          guid: guids.tb,
+          index: 0,
+        },
+        {
+          guid: guids.res,
+          index: 1,
+          children: [
+            {
+              guid: guids.irc,
+              index: 0,
+            },
+            {
+              guid: guids.bz,
+              index: 1,
+            },
+            {
+              guid: guids.mdn,
+              index: 2,
+            },
+            {
+              guid: guids.nightly,
+              index: 3,
+            },
+            {
+              guid: guids.support,
+              index: 4,
+            },
+            {
+              guid: guids.customize,
+              index: 5,
+            },
+          ],
+        },
+        {
+          guid: guids.fx,
+          index: 2,
+        },
+      ],
+      "Should use remote order as base if local is older"
+    );
+  } finally {
+    await engine.wipeClient();
+    await Service.startOver();
+    await promiseStopServer(server);
+  }
+});
+
+add_task(async function test_bookmark_order() {
+  let engine = await get_engine();
+  let store = engine._store;
+  _("Starting with a clean slate of no bookmarks");
+  await store.wipe();
+  await assertBookmarksTreeMatches(
+    "",
+    [
+      {
+        guid: PlacesUtils.bookmarks.menuGuid,
+        index: 0,
+      },
+      {
+        guid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+      },
+      {
+        // Index 2 is the tags root. (Root indices depend on the order of the
+        // `CreateRoot` calls in `Database::CreateBookmarkRoots`).
+        guid: PlacesUtils.bookmarks.unfiledGuid,
+        index: 3,
+      },
+      {
+        guid: PlacesUtils.bookmarks.mobileGuid,
+        index: 4,
+      },
+    ],
+    "clean slate"
+  );
+
+  function bookmark(name, parent) {
+    let bm = new Bookmark("http://weave.server/my-bookmark");
+    bm.id = name;
+    bm.title = name;
+    bm.bmkUri = "http://uri/";
+    bm.parentid = parent || "unfiled";
+    bm.tags = [];
+    return bm;
+  }
+
+  function folder(name, parent, children) {
+    let bmFolder = new BookmarkFolder("http://weave.server/my-bookmark-folder");
+    bmFolder.id = name;
+    bmFolder.title = name;
+    bmFolder.parentid = parent || "unfiled";
+    bmFolder.children = children;
+    return bmFolder;
+  }
+
+  async function apply(records) {
+    for (record of records) {
+      await store.applyIncoming(record);
+    }
+    await engine._apply();
+  }
+  let id10 = "10_aaaaaaaaa";
+  _("basic add first bookmark");
+  await apply([bookmark(id10, "")]);
+  await assertBookmarksTreeMatches(
+    "",
+    [
+      {
+        guid: PlacesUtils.bookmarks.menuGuid,
+        index: 0,
+      },
+      {
+        guid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+      },
+      {
+        guid: PlacesUtils.bookmarks.unfiledGuid,
+        index: 3,
+        children: [
+          {
+            guid: id10,
+            index: 0,
+          },
+        ],
+      },
+      {
+        guid: PlacesUtils.bookmarks.mobileGuid,
+        index: 4,
+      },
+    ],
+    "basic add first bookmark"
+  );
+  let id20 = "20_aaaaaaaaa";
+  _("basic append behind 10");
+  await apply([bookmark(id20, "")]);
+  await assertBookmarksTreeMatches(
+    "",
+    [
+      {
+        guid: PlacesUtils.bookmarks.menuGuid,
+        index: 0,
+      },
+      {
+        guid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+      },
+      {
+        guid: PlacesUtils.bookmarks.unfiledGuid,
+        index: 3,
+        children: [
+          {
+            guid: id10,
+            index: 0,
+          },
+          {
+            guid: id20,
+            index: 1,
+          },
+        ],
+      },
+      {
+        guid: PlacesUtils.bookmarks.mobileGuid,
+        index: 4,
+      },
+    ],
+    "basic append behind 10"
+  );
+
+  let id31 = "31_aaaaaaaaa";
+  let id30 = "f30_aaaaaaaa";
+  _("basic create in folder");
+  let b31 = bookmark(id31, id30);
+  let f30 = folder(id30, "", [id31]);
+  await apply([b31, f30]);
+  await assertBookmarksTreeMatches(
+    "",
+    [
+      {
+        guid: PlacesUtils.bookmarks.menuGuid,
+        index: 0,
+      },
+      {
+        guid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+      },
+      {
+        guid: PlacesUtils.bookmarks.unfiledGuid,
+        index: 3,
+        children: [
+          {
+            guid: id10,
+            index: 0,
+          },
+          {
+            guid: id20,
+            index: 1,
+          },
+          {
+            guid: id30,
+            index: 2,
+            children: [
+              {
+                guid: id31,
+                index: 0,
+              },
+            ],
+          },
+        ],
+      },
+      {
+        guid: PlacesUtils.bookmarks.mobileGuid,
+        index: 4,
+      },
+    ],
+    "basic create in folder"
+  );
+
+  let id41 = "41_aaaaaaaaa";
+  let id40 = "f40_aaaaaaaa";
+  _("insert missing parent -> append to unfiled");
+  await apply([bookmark(id41, id40)]);
+  await assertBookmarksTreeMatches(
+    "",
+    [
+      {
+        guid: PlacesUtils.bookmarks.menuGuid,
+        index: 0,
+      },
+      {
+        guid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+      },
+      {
+        guid: PlacesUtils.bookmarks.unfiledGuid,
+        index: 3,
+        children: [
+          {
+            guid: id10,
+            index: 0,
+          },
+          {
+            guid: id20,
+            index: 1,
+          },
+          {
+            guid: id30,
+            index: 2,
+            children: [
+              {
+                guid: id31,
+                index: 0,
+              },
+            ],
+          },
+          {
+            guid: id41,
+            index: 3,
+          },
+        ],
+      },
+      {
+        guid: PlacesUtils.bookmarks.mobileGuid,
+        index: 4,
+      },
+    ],
+    "insert missing parent -> append to unfiled"
+  );
+
+  let id42 = "42_aaaaaaaaa";
+
+  _("insert another missing parent -> append");
+  await apply([bookmark(id42, id40)]);
+  await assertBookmarksTreeMatches(
+    "",
+    [
+      {
+        guid: PlacesUtils.bookmarks.menuGuid,
+        index: 0,
+      },
+      {
+        guid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+      },
+      {
+        guid: PlacesUtils.bookmarks.unfiledGuid,
+        index: 3,
+        children: [
+          {
+            guid: id10,
+            index: 0,
+          },
+          {
+            guid: id20,
+            index: 1,
+          },
+          {
+            guid: id30,
+            index: 2,
+            children: [
+              {
+                guid: id31,
+                index: 0,
+              },
+            ],
+          },
+          {
+            guid: id41,
+            index: 3,
+          },
+          {
+            guid: id42,
+            index: 4,
+          },
+        ],
+      },
+      {
+        guid: PlacesUtils.bookmarks.mobileGuid,
+        index: 4,
+      },
+    ],
+    "insert another missing parent -> append"
+  );
+
+  await engine.wipeClient();
+  await Service.startOver();
+  await engine.finalize();
+});

src/zen/tests/mochitests/sync/test_bookmark_places_query_rewriting.js

@@ -0,0 +1,57 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+_("Rewrite place: URIs.");
+const { BookmarkQuery, BookmarkFolder } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/bookmarks.sys.mjs"
+);
+// `Service` is used as a global in head_helpers.js.
+// eslint-disable-next-line no-unused-vars
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+function makeTagRecord(id, uri) {
+  let tagRecord = new BookmarkQuery("bookmarks", id);
+  tagRecord.queryId = "MagicTags";
+  tagRecord.parentName = "Bookmarks Toolbar";
+  tagRecord.bmkUri = uri;
+  tagRecord.title = "tagtag";
+  tagRecord.folderName = "bar";
+  tagRecord.parentid = PlacesUtils.bookmarks.toolbarGuid;
+  return tagRecord;
+}
+
+add_bookmark_test(async function run_test(engine) {
+  let store = engine._store;
+
+  let toolbar = new BookmarkFolder("bookmarks", "toolbar");
+  toolbar.parentid = "places";
+  toolbar.children = ["abcdefabcdef"];
+
+  let uri = "place:folder=499&type=7&queryType=1";
+  let tagRecord = makeTagRecord("abcdefabcdef", uri);
+
+  _("Type: " + tagRecord.type);
+  _("Folder name: " + tagRecord.folderName);
+  await store.applyIncoming(toolbar);
+  await store.applyIncoming(tagRecord);
+  await engine._apply();
+
+  let insertedRecord = await store.createRecord("abcdefabcdef", "bookmarks");
+  Assert.equal(insertedRecord.bmkUri, "place:tag=bar");
+
+  _("... but not if the type is wrong.");
+  let wrongTypeURI = "place:folder=499&type=2&queryType=1";
+  let wrongTypeRecord = makeTagRecord("fedcbafedcba", wrongTypeURI);
+  await store.applyIncoming(wrongTypeRecord);
+  toolbar.children = ["fedcbafedcba"];
+  await store.applyIncoming(toolbar);
+  let expected = wrongTypeURI;
+  await engine._apply();
+  // the mirror appends a special param to these.
+  expected += "&excludeItems=1";
+
+  insertedRecord = await store.createRecord("fedcbafedcba", "bookmarks");
+  Assert.equal(insertedRecord.bmkUri, expected);
+});

src/zen/tests/mochitests/sync/test_bookmark_record.js

@@ -0,0 +1,64 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Bookmark, BookmarkQuery, PlacesItem } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/bookmarks.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+function prepareBookmarkItem(collection, id) {
+  let b = new Bookmark(collection, id);
+  b.cleartext.stuff = "my payload here";
+  return b;
+}
+
+add_task(async function test_bookmark_record() {
+  await configureIdentity();
+
+  await generateNewKeys(Service.collectionKeys);
+  let keyBundle = Service.identity.syncKeyBundle;
+
+  _("Creating a record");
+
+  let placesItem = new PlacesItem("bookmarks", "foo", "bookmark");
+  let bookmarkItem = prepareBookmarkItem("bookmarks", "foo");
+
+  _("Checking getTypeObject");
+  Assert.equal(placesItem.getTypeObject(placesItem.type), Bookmark);
+  Assert.equal(bookmarkItem.getTypeObject(bookmarkItem.type), Bookmark);
+
+  await bookmarkItem.encrypt(keyBundle);
+  _("Ciphertext is " + bookmarkItem.ciphertext);
+  Assert.notEqual(bookmarkItem.ciphertext, null);
+
+  _("Decrypting the record");
+
+  let payload = await bookmarkItem.decrypt(keyBundle);
+  Assert.equal(payload.stuff, "my payload here");
+  Assert.equal(bookmarkItem.getTypeObject(bookmarkItem.type), Bookmark);
+  Assert.notEqual(payload, bookmarkItem.payload); // wrap.data.payload is the encrypted one
+});
+
+add_task(async function test_query_foldername() {
+  // Bug 1443388
+  let checks = [
+    ["foo", "foo"],
+    ["", undefined],
+  ];
+  for (let [inVal, outVal] of checks) {
+    let bmk1 = new BookmarkQuery("bookmarks", Utils.makeGUID());
+    bmk1.fromSyncBookmark({
+      url: Services.io.newURI("https://example.com"),
+      folder: inVal,
+    });
+    Assert.strictEqual(bmk1.folderName, outVal);
+
+    // other direction
+    let bmk2 = new BookmarkQuery("bookmarks", Utils.makeGUID());
+    bmk2.folderName = inVal;
+    let record = bmk2.toSyncBookmark();
+    Assert.strictEqual(record.folder, outVal);
+  }
+});

src/zen/tests/mochitests/sync/test_bookmark_store.js

@@ -0,0 +1,486 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Bookmark, BookmarkFolder, BookmarkQuery, PlacesItem } =
+  ChromeUtils.importESModule(
+    "resource://services-sync/engines/bookmarks.sys.mjs"
+  );
+// `Service` is used as a global in head_helpers.js.
+// eslint-disable-next-line no-unused-vars
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+const BookmarksToolbarTitle = "toolbar";
+
+// apply some test records without going via a test server.
+async function apply_records(engine, records) {
+  for (record of records) {
+    await engine._store.applyIncoming(record);
+  }
+  await engine._apply();
+}
+
+add_bookmark_test(async function test_ignore_specials(engine) {
+  _("Ensure that we can't delete bookmark roots.");
+  let store = engine._store;
+
+  // Belt...
+  let record = new BookmarkFolder("bookmarks", "toolbar", "folder");
+  record.deleted = true;
+  Assert.notEqual(
+    null,
+    await PlacesTestUtils.promiseItemId(PlacesUtils.bookmarks.toolbarGuid)
+  );
+
+  await apply_records(engine, [record]);
+
+  // Ensure that the toolbar exists.
+  Assert.notEqual(
+    null,
+    await PlacesTestUtils.promiseItemId(PlacesUtils.bookmarks.toolbarGuid)
+  );
+
+  await apply_records(engine, [record]);
+
+  Assert.notEqual(
+    null,
+    await PlacesTestUtils.promiseItemId(PlacesUtils.bookmarks.toolbarGuid)
+  );
+  await store.wipe();
+});
+
+add_bookmark_test(async function test_bookmark_create(engine) {
+  let store = engine._store;
+
+  try {
+    _("Ensure the record isn't present yet.");
+    let item = await PlacesUtils.bookmarks.fetch({
+      url: "http://getfirefox.com/",
+    });
+    Assert.equal(null, item);
+
+    _("Let's create a new record.");
+    let fxrecord = new Bookmark("bookmarks", "get-firefox1");
+    fxrecord.bmkUri = "http://getfirefox.com/";
+    fxrecord.title = "Get Firefox!";
+    fxrecord.tags = ["firefox", "awesome", "browser"];
+    fxrecord.keyword = "awesome";
+    fxrecord.parentName = BookmarksToolbarTitle;
+    fxrecord.parentid = "toolbar";
+    await apply_records(engine, [fxrecord]);
+
+    _("Verify it has been created correctly.");
+    item = await PlacesUtils.bookmarks.fetch(fxrecord.id);
+    Assert.equal(item.type, PlacesUtils.bookmarks.TYPE_BOOKMARK);
+    Assert.equal(item.url.href, "http://getfirefox.com/");
+    Assert.equal(item.title, fxrecord.title);
+    Assert.equal(item.parentGuid, PlacesUtils.bookmarks.toolbarGuid);
+    let keyword = await PlacesUtils.keywords.fetch(fxrecord.keyword);
+    Assert.equal(keyword.url.href, "http://getfirefox.com/");
+
+    _(
+      "Have the store create a new record object. Verify that it has the same data."
+    );
+    let newrecord = await store.createRecord(fxrecord.id);
+    Assert.ok(newrecord instanceof Bookmark);
+    for (let property of [
+      "type",
+      "bmkUri",
+      "title",
+      "keyword",
+      "parentName",
+      "parentid",
+    ]) {
+      Assert.equal(newrecord[property], fxrecord[property]);
+    }
+    Assert.ok(Utils.deepEquals(newrecord.tags.sort(), fxrecord.tags.sort()));
+
+    _("The calculated sort index is based on frecency data.");
+    Assert.greaterOrEqual(newrecord.sortindex, 150);
+
+    _("Create a record with some values missing.");
+    let tbrecord = new Bookmark("bookmarks", "thunderbird1");
+    tbrecord.bmkUri = "http://getthunderbird.com/";
+    tbrecord.parentName = BookmarksToolbarTitle;
+    tbrecord.parentid = "toolbar";
+    await apply_records(engine, [tbrecord]);
+
+    _("Verify it has been created correctly.");
+    item = await PlacesUtils.bookmarks.fetch(tbrecord.id);
+    Assert.equal(item.type, PlacesUtils.bookmarks.TYPE_BOOKMARK);
+    Assert.equal(item.url.href, "http://getthunderbird.com/");
+    Assert.equal(item.title, "");
+    Assert.equal(item.parentGuid, PlacesUtils.bookmarks.toolbarGuid);
+    keyword = await PlacesUtils.keywords.fetch({
+      url: "http://getthunderbird.com/",
+    });
+    Assert.equal(null, keyword);
+  } finally {
+    _("Clean up.");
+    await store.wipe();
+  }
+});
+
+add_bookmark_test(async function test_bookmark_update(engine) {
+  let store = engine._store;
+
+  try {
+    _("Create a bookmark whose values we'll change.");
+    let bmk1 = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      url: "http://getfirefox.com/",
+      title: "Get Firefox!",
+    });
+    await PlacesUtils.keywords.insert({
+      url: "http://getfirefox.com/",
+      keyword: "firefox",
+    });
+
+    _("Update the record with some null values.");
+    let record = await store.createRecord(bmk1.guid);
+    record.title = null;
+    record.keyword = null;
+    record.tags = null;
+    await apply_records(engine, [record]);
+
+    _("Verify that the values have been cleared.");
+    let item = await PlacesUtils.bookmarks.fetch(bmk1.guid);
+    Assert.equal(item.title, "");
+    let keyword = await PlacesUtils.keywords.fetch({
+      url: "http://getfirefox.com/",
+    });
+    Assert.equal(null, keyword);
+  } finally {
+    _("Clean up.");
+    await store.wipe();
+  }
+});
+
+add_bookmark_test(async function test_bookmark_createRecord(engine) {
+  let store = engine._store;
+
+  try {
+    _("Create a bookmark without a title.");
+    let bmk1 = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      url: "http://getfirefox.com/",
+    });
+
+    _("Verify that the record is created accordingly.");
+    let record = await store.createRecord(bmk1.guid);
+    Assert.equal(record.title, "");
+    Assert.equal(record.keyword, null);
+  } finally {
+    _("Clean up.");
+    await store.wipe();
+  }
+});
+
+add_bookmark_test(async function test_folder_create(engine) {
+  let store = engine._store;
+
+  try {
+    _("Create a folder.");
+    let folder = new BookmarkFolder("bookmarks", "testfolder-1");
+    folder.parentName = BookmarksToolbarTitle;
+    folder.parentid = "toolbar";
+    folder.title = "Test Folder";
+    await apply_records(engine, [folder]);
+
+    _("Verify it has been created correctly.");
+    let item = await PlacesUtils.bookmarks.fetch(folder.id);
+    Assert.equal(item.type, PlacesUtils.bookmarks.TYPE_FOLDER);
+    Assert.equal(item.title, folder.title);
+    Assert.equal(item.parentGuid, PlacesUtils.bookmarks.toolbarGuid);
+
+    _(
+      "Have the store create a new record object. Verify that it has the same data."
+    );
+    let newrecord = await store.createRecord(folder.id);
+    Assert.ok(newrecord instanceof BookmarkFolder);
+    for (let property of ["title", "parentName", "parentid"]) {
+      Assert.equal(newrecord[property], folder[property]);
+    }
+
+    _("Folders have high sort index to ensure they're synced first.");
+    Assert.equal(newrecord.sortindex, 1000000);
+  } finally {
+    _("Clean up.");
+    await store.wipe();
+  }
+});
+
+add_bookmark_test(async function test_folder_createRecord(engine) {
+  let store = engine._store;
+
+  try {
+    _("Create a folder.");
+    let folder1 = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "Folder1",
+    });
+
+    _("Create two bookmarks in that folder without assigning them GUIDs.");
+    let bmk1 = await PlacesUtils.bookmarks.insert({
+      parentGuid: folder1.guid,
+      url: "http://getfirefox.com/",
+      title: "Get Firefox!",
+    });
+    let bmk2 = await PlacesUtils.bookmarks.insert({
+      parentGuid: folder1.guid,
+      url: "http://getthunderbird.com/",
+      title: "Get Thunderbird!",
+    });
+
+    _("Create a record for the folder and verify basic properties.");
+    let record = await store.createRecord(folder1.guid);
+    Assert.ok(record instanceof BookmarkFolder);
+    Assert.equal(record.title, "Folder1");
+    Assert.equal(record.parentid, "toolbar");
+    Assert.equal(record.parentName, BookmarksToolbarTitle);
+
+    _(
+      "Verify the folder's children. Ensures that the bookmarks were given GUIDs."
+    );
+    Assert.deepEqual(record.children, [bmk1.guid, bmk2.guid]);
+  } finally {
+    _("Clean up.");
+    await store.wipe();
+  }
+});
+
+add_bookmark_test(async function test_deleted(engine) {
+  let store = engine._store;
+
+  try {
+    _("Create a bookmark that will be deleted.");
+    let bmk1 = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      url: "http://getfirefox.com/",
+      title: "Get Firefox!",
+    });
+    // The engine needs to think we've previously synced it.
+    await PlacesTestUtils.markBookmarksAsSynced();
+
+    _("Delete the bookmark through the store.");
+    let record = new PlacesItem("bookmarks", bmk1.guid);
+    record.deleted = true;
+    await apply_records(engine, [record]);
+    _("Ensure it has been deleted.");
+    let item = await PlacesUtils.bookmarks.fetch(bmk1.guid);
+    let newrec = await store.createRecord(bmk1.guid);
+    Assert.equal(null, item);
+    Assert.equal(newrec.deleted, true);
+    _("Verify that the keyword has been cleared.");
+    let keyword = await PlacesUtils.keywords.fetch({
+      url: "http://getfirefox.com/",
+    });
+    Assert.equal(null, keyword);
+  } finally {
+    _("Clean up.");
+    await store.wipe();
+  }
+});
+
+add_bookmark_test(async function test_move_folder(engine) {
+  let store = engine._store;
+  store._childrenToOrder = {}; // *sob* - only needed for legacy.
+
+  try {
+    _("Create two folders and a bookmark in one of them.");
+    let folder1 = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "Folder1",
+    });
+    let folder2 = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: "Folder2",
+    });
+    let bmk = await PlacesUtils.bookmarks.insert({
+      parentGuid: folder1.guid,
+      url: "http://getfirefox.com/",
+      title: "Get Firefox!",
+    });
+    // add records to the store that represent the current state.
+    await apply_records(engine, [
+      await store.createRecord(folder1.guid),
+      await store.createRecord(folder2.guid),
+      await store.createRecord(bmk.guid),
+    ]);
+
+    _("Now simulate incoming records reparenting it.");
+    let bmkRecord = await store.createRecord(bmk.guid);
+    Assert.equal(bmkRecord.parentid, folder1.guid);
+    bmkRecord.parentid = folder2.guid;
+
+    let folder1Record = await store.createRecord(folder1.guid);
+    Assert.deepEqual(folder1Record.children, [bmk.guid]);
+    folder1Record.children = [];
+    let folder2Record = await store.createRecord(folder2.guid);
+    Assert.deepEqual(folder2Record.children, []);
+    folder2Record.children = [bmk.guid];
+
+    await apply_records(engine, [bmkRecord, folder1Record, folder2Record]);
+
+    _("Verify the new parent.");
+    let movedBmk = await PlacesUtils.bookmarks.fetch(bmk.guid);
+    Assert.equal(movedBmk.parentGuid, folder2.guid);
+  } finally {
+    _("Clean up.");
+    await store.wipe();
+  }
+});
+
+add_bookmark_test(async function test_move_order(engine) {
+  let store = engine._store;
+  let tracker = engine._tracker;
+
+  // Make sure the tracker is turned on.
+  tracker.start();
+  try {
+    _("Create two bookmarks");
+    let bmk1 = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      url: "http://getfirefox.com/",
+      title: "Get Firefox!",
+    });
+    let bmk2 = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      url: "http://getthunderbird.com/",
+      title: "Get Thunderbird!",
+    });
+
+    _("Verify order.");
+    let childIds =
+      await PlacesSyncUtils.bookmarks.fetchChildRecordIds("toolbar");
+    Assert.deepEqual(childIds, [bmk1.guid, bmk2.guid]);
+    let toolbar = await store.createRecord("toolbar");
+    Assert.deepEqual(toolbar.children, [bmk1.guid, bmk2.guid]);
+
+    _("Move bookmarks around.");
+    store._childrenToOrder = {};
+    toolbar.children = [bmk2.guid, bmk1.guid];
+    await apply_records(engine, [
+      toolbar,
+      await store.createRecord(bmk1.guid),
+      await store.createRecord(bmk2.guid),
+    ]);
+    delete store._childrenToOrder;
+
+    _("Verify new order.");
+    let newChildIds =
+      await PlacesSyncUtils.bookmarks.fetchChildRecordIds("toolbar");
+    Assert.deepEqual(newChildIds, [bmk2.guid, bmk1.guid]);
+  } finally {
+    await tracker.stop();
+    _("Clean up.");
+    await store.wipe();
+  }
+});
+
+// Tests Bug 806460, in which query records arrive with empty folder
+// names and missing bookmark URIs.
+add_bookmark_test(async function test_empty_query_doesnt_die(engine) {
+  let record = new BookmarkQuery("bookmarks", "8xoDGqKrXf1P");
+  record.folderName = "";
+  record.queryId = "";
+  record.parentName = "Toolbar";
+  record.parentid = "toolbar";
+
+  // These should not throw.
+  await apply_records(engine, [record]);
+
+  delete record.folderName;
+  await apply_records(engine, [record]);
+});
+
+add_bookmark_test(async function test_calculateIndex_for_invalid_url(engine) {
+  let store = engine._store;
+
+  let folderIndex = await store._calculateIndex({
+    type: "folder",
+  });
+  equal(folderIndex, 1000000, "Should use high sort index for folders");
+
+  let toolbarIndex = await store._calculateIndex({
+    parentid: "toolbar",
+  });
+  equal(toolbarIndex, 150, "Should bump sort index for toolbar bookmarks");
+
+  let validURLIndex = await store._calculateIndex({
+    bmkUri: "http://example.com/a",
+  });
+  greaterOrEqual(validURLIndex, 0, "Should use frecency for index");
+
+  let invalidURLIndex = await store._calculateIndex({
+    bmkUri: "!@#$%",
+  });
+  equal(invalidURLIndex, 0, "Should not throw for invalid URLs");
+});
+
+// Test that applying incoming records uses the server's modified timestamp
+// for the local lastModified, not the current time. This is important because
+// using the current time causes non-deterministic behavior in the merge
+// algorithm when comparing local and remote ages.
+add_bookmark_test(async function test_incoming_lastModified(engine) {
+  let store = engine._store;
+
+  // The server's modified timestamp (in seconds).
+  const serverModified = 1770000000;
+
+  _("Create a folder record with a specific server modified time.");
+  let folderRecord = new BookmarkFolder("bookmarks", "BBBBBBBBBBBB");
+  folderRecord.title = "Test Folder";
+  folderRecord.parentName = BookmarksToolbarTitle;
+  folderRecord.parentid = "toolbar";
+  folderRecord.children = ["AAAAAAAAAAAA"];
+  folderRecord.modified = serverModified;
+
+  _("Create a bookmark record with a specific server modified time.");
+  let bmkRecord = new Bookmark("bookmarks", "AAAAAAAAAAAA");
+  bmkRecord.bmkUri = "http://example.com/test";
+  bmkRecord.title = "Test Bookmark";
+  bmkRecord.parentName = "Test Folder";
+  bmkRecord.parentid = "BBBBBBBBBBBB";
+  bmkRecord.modified = serverModified;
+
+  await apply_records(engine, [folderRecord, bmkRecord]);
+
+  _("Verify the bookmark's lastModified matches the server timestamp.");
+  let item = await PlacesUtils.bookmarks.fetch({ guid: "AAAAAAAAAAAA" });
+  Assert.ok(item, "Bookmark should exist");
+  // The server timestamp is in seconds, lastModified is in milliseconds.
+  Assert.equal(
+    item.lastModified.getTime(),
+    serverModified * 1000,
+    "lastModified should match the server's modified timestamp"
+  );
+
+  _("Verify the folder's lastModified also matches.");
+  let folder = await PlacesUtils.bookmarks.fetch("BBBBBBBBBBBB");
+  Assert.ok(folder, "Folder should exist");
+  Assert.equal(
+    folder.lastModified.getTime(),
+    serverModified * 1000,
+    "Folder lastModified should match the server's modified timestamp"
+  );
+
+  _("Update the bookmark with a newer server timestamp.");
+  const newerServerModified = 1780000000;
+  bmkRecord.title = "Updated Test Bookmark";
+  bmkRecord.modified = newerServerModified;
+  await apply_records(engine, [bmkRecord]);
+
+  item = await PlacesUtils.bookmarks.fetch("AAAAAAAAAAAA");
+  Assert.equal(
+    item.lastModified.getTime(),
+    newerServerModified * 1000,
+    "lastModified should be updated to the newer server timestamp"
+  );
+
+  await store.wipe();
+});

src/zen/tests/mochitests/sync/test_bridged_engine.js

@@ -0,0 +1,230 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { BridgedEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/bridged_engine.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+add_task(async function test_interface() {
+  class TestBridge {
+    constructor() {
+      this.storageVersion = 2;
+      this.syncID = "syncID111111";
+      this.clear();
+    }
+
+    clear() {
+      this.lastSyncMillis = 0;
+      this.wasSyncStarted = false;
+      this.incomingEnvelopes = [];
+      this.uploadedIDs = [];
+      this.wasSyncFinished = false;
+      this.wasReset = false;
+      this.wasWiped = false;
+    }
+
+    // `mozIBridgedSyncEngine` methods.
+
+    lastSync() {
+      return this.lastSyncMillis;
+    }
+
+    setLastSync(millis) {
+      this.lastSyncMillis = millis;
+    }
+
+    resetSyncId() {
+      return this.syncID;
+    }
+
+    ensureCurrentSyncId(newSyncId) {
+      equal(newSyncId, this.syncID, "Local and new sync IDs should match");
+      return this.syncID;
+    }
+
+    syncStarted() {
+      this.wasSyncStarted = true;
+    }
+
+    storeIncoming(envelopes) {
+      this.incomingEnvelopes.push(...envelopes.map(r => JSON.parse(r)));
+    }
+
+    apply() {
+      let outgoingEnvelopes = [
+        {
+          id: "hanson",
+          data: {
+            plants: ["seed", "flower 💐", "rose"],
+            canYouTell: false,
+          },
+        },
+        {
+          id: "sheryl-crow",
+          data: {
+            today: "winding 🛣",
+            tomorrow: "winding 🛣",
+          },
+        },
+      ].map(cleartext =>
+        JSON.stringify({
+          id: cleartext.id,
+          payload: JSON.stringify(cleartext),
+        })
+      );
+      return outgoingEnvelopes;
+    }
+
+    setUploaded(millis, ids) {
+      this.uploadedIDs.push(...ids);
+    }
+
+    syncFinished() {
+      this.wasSyncFinished = true;
+    }
+
+    reset() {
+      this.clear();
+      this.wasReset = true;
+    }
+
+    wipe() {
+      this.clear();
+      this.wasWiped = true;
+    }
+  }
+
+  let bridge = new TestBridge();
+  let engine = new BridgedEngine("Nineties", Service);
+  engine._bridge = bridge;
+  engine.enabled = true;
+
+  let server = await serverForFoo(engine);
+  try {
+    await SyncTestingInfrastructure(server);
+
+    info("Add server records");
+    let foo = server.user("foo");
+    let collection = foo.collection("nineties");
+    let now = new_timestamp();
+    collection.insert(
+      "backstreet",
+      encryptPayload({
+        id: "backstreet",
+        data: {
+          say: "I want it that way",
+          when: "never",
+        },
+      }),
+      now
+    );
+    collection.insert(
+      "tlc",
+      encryptPayload({
+        id: "tlc",
+        data: {
+          forbidden: ["scrubs 🚫"],
+          numberAvailable: false,
+        },
+      }),
+      now + 5
+    );
+
+    info("Sync the engine");
+    // Advance the last sync time to skip the Backstreet Boys...
+    bridge.lastSyncMillis = 1000 * (now + 2);
+    await sync_engine_and_validate_telem(engine, false);
+
+    let metaGlobal = foo.collection("meta").wbo("global").get();
+    deepEqual(
+      JSON.parse(metaGlobal.payload).engines.nineties,
+      {
+        version: 2,
+        syncID: "syncID111111",
+      },
+      "Should write storage version and sync ID to m/g"
+    );
+
+    greater(bridge.lastSyncMillis, 0, "Should update last sync time");
+    ok(
+      bridge.wasSyncStarted,
+      "Should have started sync before storing incoming"
+    );
+    deepEqual(
+      bridge.incomingEnvelopes
+        .sort((a, b) => a.id.localeCompare(b.id))
+        .map(({ payload, ...envelope }) => ({
+          cleartextAsObject: JSON.parse(payload),
+          ...envelope,
+        })),
+      [
+        {
+          id: "tlc",
+          modified: now + 5,
+          cleartextAsObject: {
+            id: "tlc",
+            data: {
+              forbidden: ["scrubs 🚫"],
+              numberAvailable: false,
+            },
+          },
+        },
+      ],
+      "Should stage incoming records from server"
+    );
+    deepEqual(
+      bridge.uploadedIDs.sort(),
+      ["hanson", "sheryl-crow"],
+      "Should mark new local records as uploaded"
+    );
+    ok(bridge.wasSyncFinished, "Should have finished sync after uploading");
+
+    deepEqual(
+      collection.keys().sort(),
+      ["backstreet", "hanson", "sheryl-crow", "tlc"],
+      "Should have all records on server"
+    );
+    let expectedRecords = [
+      {
+        id: "sheryl-crow",
+        data: {
+          today: "winding 🛣",
+          tomorrow: "winding 🛣",
+        },
+      },
+      {
+        id: "hanson",
+        data: {
+          plants: ["seed", "flower 💐", "rose"],
+          canYouTell: false,
+        },
+      },
+    ];
+    for (let expected of expectedRecords) {
+      let actual = collection.cleartext(expected.id);
+      deepEqual(
+        actual,
+        expected,
+        `Should upload record ${expected.id} from bridged engine`
+      );
+    }
+
+    await engine.resetClient();
+    ok(bridge.wasReset, "Should reset local storage for bridge");
+
+    await engine.wipeClient();
+    ok(bridge.wasWiped, "Should wipe local storage for bridge");
+
+    await engine.resetSyncID();
+    ok(
+      !foo.collection("nineties"),
+      "Should delete server collection after resetting sync ID"
+    );
+  } finally {
+    await promiseStopServer(server);
+    await engine.finalize();
+  }
+});

src/zen/tests/mochitests/sync/test_clients_escape.js

@@ -0,0 +1,57 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+add_task(async function test_clients_escape() {
+  _("Set up test fixtures.");
+
+  await configureIdentity();
+  let keyBundle = Service.identity.syncKeyBundle;
+
+  let engine = Service.clientsEngine;
+
+  try {
+    _("Test that serializing client records results in uploadable ascii");
+    engine.localID = "ascii";
+    engine.localName = "wéävê";
+
+    _("Make sure we have the expected record");
+    let record = await engine._createRecord("ascii");
+    Assert.equal(record.id, "ascii");
+    Assert.equal(record.name, "wéävê");
+
+    _("Encrypting record...");
+    await record.encrypt(keyBundle);
+    _("Encrypted.");
+
+    let serialized = JSON.stringify(record);
+    let checkCount = 0;
+    _("Checking for all ASCII:", serialized);
+    for (let ch of serialized) {
+      let code = ch.charCodeAt(0);
+      _("Checking asciiness of '", ch, "'=", code);
+      Assert.less(code, 128);
+      checkCount++;
+    }
+
+    _("Processed", checkCount, "characters out of", serialized.length);
+    Assert.equal(checkCount, serialized.length);
+
+    _("Making sure the record still looks like it did before");
+    await record.decrypt(keyBundle);
+    Assert.equal(record.id, "ascii");
+    Assert.equal(record.name, "wéävê");
+
+    _("Sanity check that creating the record also gives the same");
+    record = await engine._createRecord("ascii");
+    Assert.equal(record.id, "ascii");
+    Assert.equal(record.name, "wéävê");
+  } finally {
+    for (const pref of Svc.PrefBranch.getChildList("")) {
+      Svc.PrefBranch.clearUserPref(pref);
+    }
+  }
+});

src/zen/tests/mochitests/sync/test_collection_getBatched.js

@@ -0,0 +1,187 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Collection, WBORecord } = ChromeUtils.importESModule(
+  "resource://services-sync/record.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+function recordRange(lim, offset, total) {
+  let res = [];
+  for (let i = offset; i < Math.min(lim + offset, total); ++i) {
+    res.push({ id: String(i), payload: "test:" + i });
+  }
+  return res;
+}
+
+function get_test_collection_info({
+  totalRecords,
+  batchSize,
+  lastModified,
+  throwAfter = Infinity,
+  interruptedAfter = Infinity,
+}) {
+  let coll = new Collection("http://example.com/test/", WBORecord, Service);
+  coll.full = true;
+  let requests = [];
+  let responses = [];
+  coll.get = async function () {
+    let limit = +this.limit;
+    let offset = 0;
+    if (this.offset) {
+      equal(this.offset.slice(0, 6), "foobar");
+      offset = +this.offset.slice(6);
+    }
+    requests.push({
+      limit,
+      offset,
+      spec: this.spec,
+      headers: Object.assign({}, this.headers),
+    });
+    if (--throwAfter === 0) {
+      throw new Error("Some Network Error");
+    }
+    let body = recordRange(limit, offset, totalRecords);
+    let response = {
+      obj: body,
+      success: true,
+      status: 200,
+      headers: {},
+    };
+    if (--interruptedAfter === 0) {
+      response.success = false;
+      response.status = 412;
+      response.body = "";
+    } else if (offset + limit < totalRecords) {
+      // Ensure we're treating this as an opaque string, since the docs say
+      // it might not be numeric.
+      response.headers["x-weave-next-offset"] = "foobar" + (offset + batchSize);
+    }
+    response.headers["x-last-modified"] = lastModified;
+    responses.push(response);
+    return response;
+  };
+  return { responses, requests, coll };
+}
+
+add_task(async function test_success() {
+  const totalRecords = 11;
+  const batchSize = 2;
+  const lastModified = "111111";
+  let { responses, requests, coll } = get_test_collection_info({
+    totalRecords,
+    batchSize,
+    lastModified,
+  });
+  let { response, records } = await coll.getBatched(batchSize);
+
+  equal(requests.length, Math.ceil(totalRecords / batchSize));
+
+  equal(records.length, totalRecords);
+  checkRecordsOrder(records);
+
+  // ensure we're returning the last response
+  equal(responses[responses.length - 1], response);
+
+  // check first separately since its a bit of a special case
+  ok(!requests[0].headers["x-if-unmodified-since"]);
+  ok(!requests[0].offset);
+  equal(requests[0].limit, batchSize);
+  let expectedOffset = 2;
+  for (let i = 1; i < requests.length; ++i) {
+    let req = requests[i];
+    equal(req.headers["x-if-unmodified-since"], lastModified);
+    equal(req.limit, batchSize);
+    if (i !== requests.length - 1) {
+      equal(req.offset, expectedOffset);
+    }
+
+    expectedOffset += batchSize;
+  }
+
+  // ensure we cleaned up anything that would break further
+  // use of this collection.
+  ok(!coll._headers["x-if-unmodified-since"]);
+  ok(!coll.offset);
+  ok(!coll.limit || coll.limit == Infinity);
+});
+
+add_task(async function test_total_limit() {
+  _("getBatched respects the (initial) value of the limit property");
+  const totalRecords = 100;
+  const recordLimit = 11;
+  const batchSize = 2;
+  const lastModified = "111111";
+  let { requests, coll } = get_test_collection_info({
+    totalRecords,
+    batchSize,
+    lastModified,
+  });
+  coll.limit = recordLimit;
+  let { records } = await coll.getBatched(batchSize);
+  checkRecordsOrder(records);
+
+  equal(requests.length, Math.ceil(recordLimit / batchSize));
+  equal(records.length, recordLimit);
+
+  for (let i = 0; i < requests.length; ++i) {
+    let req = requests[i];
+    if (i !== requests.length - 1) {
+      equal(req.limit, batchSize);
+    } else {
+      equal(req.limit, recordLimit % batchSize);
+    }
+  }
+
+  equal(coll._limit, recordLimit);
+});
+
+add_task(async function test_412() {
+  _("We shouldn't record records if we get a 412 in the middle of a batch");
+  const totalRecords = 11;
+  const batchSize = 2;
+  const lastModified = "111111";
+  let { responses, requests, coll } = get_test_collection_info({
+    totalRecords,
+    batchSize,
+    lastModified,
+    interruptedAfter: 3,
+  });
+  let { response, records } = await coll.getBatched(batchSize);
+
+  equal(requests.length, 3);
+  equal(records.length, 0); // we should not get any records
+
+  // ensure we're returning the last response
+  equal(responses[responses.length - 1], response);
+
+  ok(!response.success);
+  equal(response.status, 412);
+});
+
+add_task(async function test_get_throws() {
+  _("getBatched() should throw if a get() throws");
+  const totalRecords = 11;
+  const batchSize = 2;
+  const lastModified = "111111";
+  let { requests, coll } = get_test_collection_info({
+    totalRecords,
+    batchSize,
+    lastModified,
+    throwAfter: 3,
+  });
+
+  await Assert.rejects(coll.getBatched(batchSize), /Some Network Error/);
+
+  equal(requests.length, 3);
+});
+
+function checkRecordsOrder(records) {
+  ok(!!records.length);
+  for (let i = 0; i < records.length; i++) {
+    equal(records[i].id, String(i));
+    equal(records[i].payload, "test:" + i);
+  }
+}

src/zen/tests/mochitests/sync/test_collections_recovery.js

@@ -0,0 +1,95 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Verify that we wipe the server if we have to regenerate keys.
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+add_task(async function test_missing_crypto_collection() {
+  enableValidationPrefs();
+
+  let johnHelper = track_collections_helper();
+  let johnU = johnHelper.with_updated_collection;
+  let johnColls = johnHelper.collections;
+
+  let empty = false;
+  function maybe_empty(handler) {
+    return function (request, response) {
+      if (empty) {
+        let body = "{}";
+        response.setStatusLine(request.httpVersion, 200, "OK");
+        response.bodyOutputStream.write(body, body.length);
+      } else {
+        handler(request, response);
+      }
+    };
+  }
+
+  let handlers = {
+    "/1.1/johndoe/info/collections": maybe_empty(johnHelper.handler),
+    "/1.1/johndoe/storage/crypto/keys": johnU(
+      "crypto",
+      new ServerWBO("keys").handler()
+    ),
+    "/1.1/johndoe/storage/meta/global": johnU(
+      "meta",
+      new ServerWBO("global").handler()
+    ),
+  };
+  let collections = [
+    "clients",
+    "bookmarks",
+    "forms",
+    "history",
+    "passwords",
+    "prefs",
+    "tabs",
+  ];
+  // Disable addon sync because AddonManager won't be initialized here.
+  await Service.engineManager.unregister("addons");
+  await Service.engineManager.unregister("extension-storage");
+
+  for (let coll of collections) {
+    handlers["/1.1/johndoe/storage/" + coll] = johnU(
+      coll,
+      new ServerCollection({}, true).handler()
+    );
+  }
+  let server = httpd_setup(handlers);
+  await configureIdentity({ username: "johndoe" }, server);
+
+  try {
+    let fresh = 0;
+    let orig = Service._freshStart;
+    Service._freshStart = async function () {
+      _("Called _freshStart.");
+      await orig.call(Service);
+      fresh++;
+    };
+
+    _("Startup, no meta/global: freshStart called once.");
+    await sync_and_validate_telem();
+    Assert.equal(fresh, 1);
+    fresh = 0;
+
+    _("Regular sync: no need to freshStart.");
+    await Service.sync();
+    Assert.equal(fresh, 0);
+
+    _("Simulate a bad info/collections.");
+    delete johnColls.crypto;
+    await sync_and_validate_telem();
+    Assert.equal(fresh, 1);
+    fresh = 0;
+
+    _("Regular sync: no need to freshStart.");
+    await sync_and_validate_telem();
+    Assert.equal(fresh, 0);
+  } finally {
+    for (const pref of Svc.PrefBranch.getChildList("")) {
+      Svc.PrefBranch.clearUserPref(pref);
+    }
+    await promiseStopServer(server);
+  }
+});

src/zen/tests/mochitests/sync/test_corrupt_keys.js

@@ -0,0 +1,248 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Weave } = ChromeUtils.importESModule(
+  "resource://services-sync/main.sys.mjs"
+);
+const { HistoryEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/history.sys.mjs"
+);
+const { CryptoWrapper, WBORecord } = ChromeUtils.importESModule(
+  "resource://services-sync/record.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+add_task(async function test_locally_changed_keys() {
+  enableValidationPrefs();
+
+  let hmacErrorCount = 0;
+  function counting(f) {
+    return async function () {
+      hmacErrorCount++;
+      return f.call(this);
+    };
+  }
+
+  Service.handleHMACEvent = counting(Service.handleHMACEvent);
+
+  let server = new SyncServer();
+  let johndoe = server.registerUser("johndoe", "password");
+  johndoe.createContents({
+    meta: {},
+    crypto: {},
+    clients: {},
+  });
+  server.start();
+
+  try {
+    Svc.PrefBranch.setStringPref("registerEngines", "Tab");
+
+    await configureIdentity({ username: "johndoe" }, server);
+    // We aren't doing a .login yet, so fudge the cluster URL.
+    Service.clusterURL = Service.identity._token.endpoint;
+
+    await Service.engineManager.register(HistoryEngine);
+    // Disable addon sync because AddonManager won't be initialized here.
+    await Service.engineManager.unregister("addons");
+    await Service.engineManager.unregister("extension-storage");
+
+    async function corrupt_local_keys() {
+      Service.collectionKeys._default.keyPair = [
+        await Weave.Crypto.generateRandomKey(),
+        await Weave.Crypto.generateRandomKey(),
+      ];
+    }
+
+    _("Setting meta.");
+
+    // Bump version on the server.
+    let m = new WBORecord("meta", "global");
+    m.payload = {
+      syncID: "foooooooooooooooooooooooooo",
+      storageVersion: STORAGE_VERSION,
+    };
+    await m.upload(Service.resource(Service.metaURL));
+
+    _(
+      "New meta/global: " +
+        JSON.stringify(johndoe.collection("meta").wbo("global"))
+    );
+
+    // Upload keys.
+    await generateNewKeys(Service.collectionKeys);
+    let serverKeys = Service.collectionKeys.asWBO("crypto", "keys");
+    await serverKeys.encrypt(Service.identity.syncKeyBundle);
+    Assert.ok(
+      (await serverKeys.upload(Service.resource(Service.cryptoKeysURL))).success
+    );
+
+    // Check that login works.
+    Assert.ok(await Service.login());
+    Assert.ok(Service.isLoggedIn);
+
+    // Sync should upload records.
+    await sync_and_validate_telem();
+
+    // Tabs exist.
+    _("Tabs modified: " + johndoe.modified("tabs"));
+    Assert.greater(johndoe.modified("tabs"), 0);
+
+    // Let's create some server side history records.
+    let liveKeys = Service.collectionKeys.keyForCollection("history");
+    _("Keys now: " + liveKeys.keyPair);
+    let visitType = Ci.nsINavHistoryService.TRANSITION_LINK;
+    let history = johndoe.createCollection("history");
+    for (let i = 0; i < 5; i++) {
+      let id = "record-no--" + i;
+      let modified = Date.now() / 1000 - 60 * (i + 10);
+
+      let w = new CryptoWrapper("history", "id");
+      w.cleartext = {
+        id,
+        histUri: "http://foo/bar?" + id,
+        title: id,
+        sortindex: i,
+        visits: [{ date: (modified - 5) * 1000000, type: visitType }],
+        deleted: false,
+      };
+      await w.encrypt(liveKeys);
+
+      let payload = { ciphertext: w.ciphertext, IV: w.IV, hmac: w.hmac };
+      history.insert(id, payload, modified);
+    }
+
+    history.timestamp = Date.now() / 1000;
+    let old_key_time = johndoe.modified("crypto");
+    _("Old key time: " + old_key_time);
+
+    // Check that we can decrypt one.
+    let rec = new CryptoWrapper("history", "record-no--0");
+    await rec.fetch(
+      Service.resource(Service.storageURL + "history/record-no--0")
+    );
+    _(JSON.stringify(rec));
+    Assert.ok(!!(await rec.decrypt(liveKeys)));
+
+    Assert.equal(hmacErrorCount, 0);
+
+    // Fill local key cache with bad data.
+    await corrupt_local_keys();
+    _(
+      "Keys now: " + Service.collectionKeys.keyForCollection("history").keyPair
+    );
+
+    Assert.equal(hmacErrorCount, 0);
+
+    _("HMAC error count: " + hmacErrorCount);
+    // Now syncing should succeed, after one HMAC error.
+    await sync_and_validate_telem(ping => {
+      Assert.equal(
+        ping.engines.find(e => e.name == "history").incoming.applied,
+        5
+      );
+    });
+
+    Assert.equal(hmacErrorCount, 1);
+    _(
+      "Keys now: " + Service.collectionKeys.keyForCollection("history").keyPair
+    );
+
+    // And look! We downloaded history!
+    Assert.ok(
+      await PlacesUtils.history.hasVisits("http://foo/bar?record-no--0")
+    );
+    Assert.ok(
+      await PlacesUtils.history.hasVisits("http://foo/bar?record-no--1")
+    );
+    Assert.ok(
+      await PlacesUtils.history.hasVisits("http://foo/bar?record-no--2")
+    );
+    Assert.ok(
+      await PlacesUtils.history.hasVisits("http://foo/bar?record-no--3")
+    );
+    Assert.ok(
+      await PlacesUtils.history.hasVisits("http://foo/bar?record-no--4")
+    );
+    Assert.equal(hmacErrorCount, 1);
+
+    _("Busting some new server values.");
+    // Now what happens if we corrupt the HMAC on the server?
+    for (let i = 5; i < 10; i++) {
+      let id = "record-no--" + i;
+      let modified = 1 + Date.now() / 1000;
+
+      let w = new CryptoWrapper("history", "id");
+      w.cleartext = {
+        id,
+        histUri: "http://foo/bar?" + id,
+        title: id,
+        sortindex: i,
+        visits: [{ date: (modified - 5) * 1000000, type: visitType }],
+        deleted: false,
+      };
+      await w.encrypt(Service.collectionKeys.keyForCollection("history"));
+      w.hmac = w.hmac.toUpperCase();
+
+      let payload = { ciphertext: w.ciphertext, IV: w.IV, hmac: w.hmac };
+      history.insert(id, payload, modified);
+    }
+    history.timestamp = Date.now() / 1000;
+
+    _("Server key time hasn't changed.");
+    Assert.equal(johndoe.modified("crypto"), old_key_time);
+
+    _("Resetting HMAC error timer.");
+    Service.lastHMACEvent = 0;
+
+    _("Syncing...");
+    await sync_and_validate_telem(ping => {
+      Assert.equal(
+        ping.engines.find(e => e.name == "history").incoming.failed,
+        5
+      );
+    });
+
+    _(
+      "Keys now: " + Service.collectionKeys.keyForCollection("history").keyPair
+    );
+    _(
+      "Server keys have been updated, and we skipped over 5 more HMAC errors without adjusting history."
+    );
+    Assert.greater(johndoe.modified("crypto"), old_key_time);
+    Assert.equal(hmacErrorCount, 6);
+    Assert.equal(
+      false,
+      await PlacesUtils.history.hasVisits("http://foo/bar?record-no--5")
+    );
+    Assert.equal(
+      false,
+      await PlacesUtils.history.hasVisits("http://foo/bar?record-no--6")
+    );
+    Assert.equal(
+      false,
+      await PlacesUtils.history.hasVisits("http://foo/bar?record-no--7")
+    );
+    Assert.equal(
+      false,
+      await PlacesUtils.history.hasVisits("http://foo/bar?record-no--8")
+    );
+    Assert.equal(
+      false,
+      await PlacesUtils.history.hasVisits("http://foo/bar?record-no--9")
+    );
+  } finally {
+    for (const pref of Svc.PrefBranch.getChildList("")) {
+      Svc.PrefBranch.clearUserPref(pref);
+    }
+    await promiseStopServer(server);
+  }
+});
+
+function run_test() {
+  Log.repository.rootLogger.addAppender(new Log.DumpAppender());
+  validate_all_future_pings();
+
+  run_next_test();
+}

src/zen/tests/mochitests/sync/test_declined.js

@@ -0,0 +1,198 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { DeclinedEngines } = ChromeUtils.importESModule(
+  "resource://services-sync/stages/declined.sys.mjs"
+);
+const { EngineSynchronizer } = ChromeUtils.importESModule(
+  "resource://services-sync/stages/enginesync.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { Observers } = ChromeUtils.importESModule(
+  "resource://services-common/observers.sys.mjs"
+);
+
+function PetrolEngine() {}
+PetrolEngine.prototype.name = "petrol";
+
+function DieselEngine() {}
+DieselEngine.prototype.name = "diesel";
+
+function DummyEngine() {}
+DummyEngine.prototype.name = "dummy";
+
+function ActualEngine() {}
+ActualEngine.prototype.name = "actual";
+Object.setPrototypeOf(ActualEngine.prototype, SyncEngine.prototype);
+
+function getEngineManager() {
+  let manager = new EngineManager(Service);
+  Service.engineManager = manager;
+  manager._engines = {
+    petrol: new PetrolEngine(),
+    diesel: new DieselEngine(),
+    dummy: new DummyEngine(),
+    actual: new ActualEngine(),
+  };
+  return manager;
+}
+
+/**
+ * 'Fetch' a meta/global record that doesn't mention declined.
+ *
+ * Push it into the EngineSynchronizer to set enabled; verify that those are
+ * correct.
+ *
+ * Then push it into DeclinedEngines to set declined; verify that none are
+ * declined, and a notification is sent for our locally disabled-but-not-
+ * declined engines.
+ */
+add_task(async function testOldMeta() {
+  let meta = {
+    payload: {
+      engines: {
+        petrol: 1,
+        diesel: 2,
+        nonlocal: 3, // Enabled but not supported.
+      },
+    },
+  };
+
+  _("Record: " + JSON.stringify(meta));
+
+  let manager = getEngineManager();
+
+  // Update enabled from meta/global.
+  let engineSync = new EngineSynchronizer(Service);
+  await engineSync._updateEnabledFromMeta(meta, 3, manager);
+
+  Assert.ok(manager._engines.petrol.enabled, "'petrol' locally enabled.");
+  Assert.ok(manager._engines.diesel.enabled, "'diesel' locally enabled.");
+  Assert.ok(
+    !("nonlocal" in manager._engines),
+    "We don't know anything about the 'nonlocal' engine."
+  );
+  Assert.ok(!manager._engines.actual.enabled, "'actual' not locally enabled.");
+  Assert.ok(!manager.isDeclined("actual"), "'actual' not declined, though.");
+
+  let declinedEngines = new DeclinedEngines(Service);
+
+  function onNotDeclined(subject) {
+    Observers.remove("weave:engines:notdeclined", onNotDeclined);
+    Assert.ok(
+      subject.undecided.has("actual"),
+      "EngineManager observed that 'actual' was undecided."
+    );
+
+    let declined = manager.getDeclined();
+    _("Declined: " + JSON.stringify(declined));
+
+    Assert.ok(!meta.changed, "No need to upload a new meta/global.");
+    run_next_test();
+  }
+
+  Observers.add("weave:engines:notdeclined", onNotDeclined);
+
+  declinedEngines.updateDeclined(meta, manager);
+});
+
+/**
+ * 'Fetch' a meta/global that declines an engine we don't
+ * recognize. Ensure that we track that declined engine along
+ * with any we locally declined, and that the meta/global
+ * record is marked as changed and includes all declined
+ * engines.
+ */
+add_task(async function testDeclinedMeta() {
+  let meta = {
+    payload: {
+      engines: {
+        petrol: 1,
+        diesel: 2,
+        nonlocal: 3, // Enabled but not supported.
+      },
+      declined: ["nonexistent"], // Declined and not supported.
+    },
+  };
+
+  _("Record: " + JSON.stringify(meta));
+
+  let manager = getEngineManager();
+  manager._engines.petrol.enabled = true;
+  manager._engines.diesel.enabled = true;
+  manager._engines.dummy.enabled = true;
+  manager._engines.actual.enabled = false; // Disabled but not declined.
+
+  manager.decline(["localdecline"]); // Declined and not supported.
+
+  let declinedEngines = new DeclinedEngines(Service);
+
+  function onNotDeclined(subject) {
+    Observers.remove("weave:engines:notdeclined", onNotDeclined);
+    Assert.ok(
+      subject.undecided.has("actual"),
+      "EngineManager observed that 'actual' was undecided."
+    );
+
+    let declined = manager.getDeclined();
+    _("Declined: " + JSON.stringify(declined));
+
+    Assert.equal(
+      declined.indexOf("actual"),
+      -1,
+      "'actual' is locally disabled, but not marked as declined."
+    );
+
+    Assert.equal(
+      declined.indexOf("clients"),
+      -1,
+      "'clients' is enabled and not remotely declined."
+    );
+    Assert.equal(
+      declined.indexOf("petrol"),
+      -1,
+      "'petrol' is enabled and not remotely declined."
+    );
+    Assert.equal(
+      declined.indexOf("diesel"),
+      -1,
+      "'diesel' is enabled and not remotely declined."
+    );
+    Assert.equal(
+      declined.indexOf("dummy"),
+      -1,
+      "'dummy' is enabled and not remotely declined."
+    );
+
+    Assert.lessOrEqual(
+      0,
+      declined.indexOf("nonexistent"),
+      "'nonexistent' was declined on the server."
+    );
+
+    Assert.lessOrEqual(
+      0,
+      declined.indexOf("localdecline"),
+      "'localdecline' was declined locally."
+    );
+
+    // The meta/global is modified, too.
+    Assert.lessOrEqual(
+      0,
+      meta.payload.declined.indexOf("nonexistent"),
+      "meta/global's declined contains 'nonexistent'."
+    );
+    Assert.lessOrEqual(
+      0,
+      meta.payload.declined.indexOf("localdecline"),
+      "meta/global's declined contains 'localdecline'."
+    );
+    Assert.strictEqual(true, meta.changed, "meta/global was changed.");
+  }
+
+  Observers.add("weave:engines:notdeclined", onNotDeclined);
+
+  declinedEngines.updateDeclined(meta, manager);
+});

src/zen/tests/mochitests/sync/test_disconnect_shutdown.js

@@ -0,0 +1,104 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const { SyncDisconnect, SyncDisconnectInternal } = ChromeUtils.importESModule(
+  "resource://services-sync/SyncDisconnect.sys.mjs"
+);
+const { AsyncShutdown } = ChromeUtils.importESModule(
+  "resource://gre/modules/AsyncShutdown.sys.mjs"
+);
+const { PREF_LAST_FXA_USER_UID } = ChromeUtils.importESModule(
+  "resource://gre/modules/FxAccountsCommon.sys.mjs"
+);
+
+add_task(async function test_shutdown_blocker() {
+  let spySignout = sinon.stub(
+    SyncDisconnectInternal,
+    "doSyncAndAccountDisconnect"
+  );
+
+  // We don't need to check for the lock regularly as we end up aborting the wait.
+  SyncDisconnectInternal.lockRetryInterval = 1000;
+  // Force the retry count to a very large value - this test should never
+  // abort due to the retry count and we want the test to fail (aka timeout)
+  // should our abort code not work.
+  SyncDisconnectInternal.lockRetryCount = 10000;
+  // mock the "browser" sanitize function - it should not be called by
+  // this test.
+  let spyBrowser = sinon.stub(SyncDisconnectInternal, "doSanitizeBrowserData");
+  // mock Sync
+  let mockEngine1 = {
+    enabled: true,
+    name: "Test Engine 1",
+    wipeClient: sinon.spy(),
+  };
+  let mockEngine2 = {
+    enabled: false,
+    name: "Test Engine 2",
+    wipeClient: sinon.spy(),
+  };
+
+  // This weave mock never gives up the lock.
+  let Weave = {
+    Service: {
+      enabled: true,
+      lock: () => false, // so we never get the lock.
+      unlock: sinon.spy(),
+
+      engineManager: {
+        getAll: sinon.stub().returns([mockEngine1, mockEngine2]),
+      },
+      errorHandler: {
+        resetFileLog: sinon.spy(),
+      },
+    },
+  };
+  let weaveStub = sinon.stub(SyncDisconnectInternal, "getWeave");
+  weaveStub.returns(Weave);
+
+  Services.prefs.setStringPref(
+    PREF_LAST_FXA_USER_UID,
+    "dGVzdEBleGFtcGxlLmNvbQ=="
+  );
+
+  let promiseDisconnected = SyncDisconnect.disconnect(true);
+
+  // Pretend we hit the shutdown blocker.
+  info("simulating appShutdownConfirmed");
+  Services.prefs.setBoolPref("toolkit.asyncshutdown.testing", true);
+  AsyncShutdown.appShutdownConfirmed._trigger();
+  Services.prefs.clearUserPref("toolkit.asyncshutdown.testing");
+
+  info("waiting for disconnect to complete");
+  await promiseDisconnected;
+
+  Assert.ok(
+    !Services.prefs.prefHasUserValue(PREF_LAST_FXA_USER_UID),
+    "Should have reset different user warning pref"
+  );
+  Assert.equal(
+    Weave.Service.unlock.callCount,
+    0,
+    "should not have unlocked at the end"
+  );
+  Assert.ok(!Weave.Service.enabled, "Weave should be and remain disabled");
+  Assert.equal(
+    Weave.Service.errorHandler.resetFileLog.callCount,
+    1,
+    "should have reset the log"
+  );
+  Assert.equal(
+    mockEngine1.wipeClient.callCount,
+    1,
+    "enabled engine should have been wiped"
+  );
+  Assert.equal(
+    mockEngine2.wipeClient.callCount,
+    0,
+    "disabled engine should not have been wiped"
+  );
+  Assert.equal(spyBrowser.callCount, 1, "should not sanitize the browser");
+  Assert.equal(spySignout.callCount, 1, "should have signed out of FxA");
+});

src/zen/tests/mochitests/sync/test_engine.js

@@ -0,0 +1,246 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Observers } = ChromeUtils.importESModule(
+  "resource://services-common/observers.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+function SteamStore(engine) {
+  Store.call(this, "Steam", engine);
+  this.wasWiped = false;
+}
+SteamStore.prototype = {
+  async wipe() {
+    this.wasWiped = true;
+  },
+};
+Object.setPrototypeOf(SteamStore.prototype, Store.prototype);
+
+function SteamTracker(name, engine) {
+  LegacyTracker.call(this, name || "Steam", engine);
+}
+Object.setPrototypeOf(SteamTracker.prototype, LegacyTracker.prototype);
+
+function SteamEngine(name, service) {
+  SyncEngine.call(this, name, service);
+  this.wasReset = false;
+  this.wasSynced = false;
+}
+SteamEngine.prototype = {
+  _storeObj: SteamStore,
+  _trackerObj: SteamTracker,
+
+  async _resetClient() {
+    this.wasReset = true;
+  },
+
+  async _sync() {
+    this.wasSynced = true;
+  },
+};
+Object.setPrototypeOf(SteamEngine.prototype, SyncEngine.prototype);
+
+var engineObserver = {
+  topics: [],
+
+  observe(subject, topic, data) {
+    Assert.equal(data, "steam");
+    this.topics.push(topic);
+  },
+
+  reset() {
+    this.topics = [];
+  },
+};
+Observers.add("weave:engine:reset-client:start", engineObserver);
+Observers.add("weave:engine:reset-client:finish", engineObserver);
+Observers.add("weave:engine:wipe-client:start", engineObserver);
+Observers.add("weave:engine:wipe-client:finish", engineObserver);
+Observers.add("weave:engine:sync:start", engineObserver);
+Observers.add("weave:engine:sync:finish", engineObserver);
+
+async function cleanup(engine) {
+  for (const pref of Svc.PrefBranch.getChildList("")) {
+    Svc.PrefBranch.clearUserPref(pref);
+  }
+  engine.wasReset = false;
+  engine.wasSynced = false;
+  engineObserver.reset();
+  await engine._tracker.clearChangedIDs();
+  await engine.finalize();
+}
+
+add_task(async function test_members() {
+  _("Engine object members");
+  let engine = new SteamEngine("Steam", Service);
+  await engine.initialize();
+  Assert.equal(engine.Name, "Steam");
+  Assert.equal(engine.prefName, "steam");
+  Assert.ok(engine._store instanceof SteamStore);
+  Assert.ok(engine._tracker instanceof SteamTracker);
+});
+
+add_task(async function test_score() {
+  _("Engine.score corresponds to tracker.score and is readonly");
+  let engine = new SteamEngine("Steam", Service);
+  await engine.initialize();
+  Assert.equal(engine.score, 0);
+  engine._tracker.score += 5;
+  Assert.equal(engine.score, 5);
+
+  try {
+    engine.score = 10;
+  } catch (ex) {
+    // Setting an attribute that has a getter produces an error in
+    // Firefox <= 3.6 and is ignored in later versions.  Either way,
+    // the attribute's value won't change.
+  }
+  Assert.equal(engine.score, 5);
+});
+
+add_task(async function test_resetClient() {
+  _("Engine.resetClient calls _resetClient");
+  let engine = new SteamEngine("Steam", Service);
+  await engine.initialize();
+  Assert.ok(!engine.wasReset);
+
+  await engine.resetClient();
+  Assert.ok(engine.wasReset);
+  Assert.equal(engineObserver.topics[0], "weave:engine:reset-client:start");
+  Assert.equal(engineObserver.topics[1], "weave:engine:reset-client:finish");
+
+  await cleanup(engine);
+});
+
+add_task(async function test_invalidChangedIDs() {
+  _("Test that invalid changed IDs on disk don't end up live.");
+  let engine = new SteamEngine("Steam", Service);
+  await engine.initialize();
+  let tracker = engine._tracker;
+
+  await tracker._beforeSave();
+  await IOUtils.writeUTF8(tracker._storage.path, "5", {
+    tmpPath: tracker._storage.path + ".tmp",
+  });
+
+  ok(!tracker._storage.dataReady);
+  const changes = await tracker.getChangedIDs();
+  changes.placeholder = true;
+  deepEqual(
+    changes,
+    { placeholder: true },
+    "Accessing changed IDs should load changes from disk as a side effect"
+  );
+  ok(tracker._storage.dataReady);
+
+  Assert.ok(changes.placeholder);
+  await cleanup(engine);
+});
+
+add_task(async function test_wipeClient() {
+  _("Engine.wipeClient calls resetClient, wipes store, clears changed IDs");
+  let engine = new SteamEngine("Steam", Service);
+  await engine.initialize();
+  Assert.ok(!engine.wasReset);
+  Assert.ok(!engine._store.wasWiped);
+  Assert.ok(await engine._tracker.addChangedID("a-changed-id"));
+  let changes = await engine._tracker.getChangedIDs();
+  Assert.ok("a-changed-id" in changes);
+
+  await engine.wipeClient();
+  Assert.ok(engine.wasReset);
+  Assert.ok(engine._store.wasWiped);
+  changes = await engine._tracker.getChangedIDs();
+  Assert.equal(JSON.stringify(changes), "{}");
+  Assert.equal(engineObserver.topics[0], "weave:engine:wipe-client:start");
+  Assert.equal(engineObserver.topics[1], "weave:engine:reset-client:start");
+  Assert.equal(engineObserver.topics[2], "weave:engine:reset-client:finish");
+  Assert.equal(engineObserver.topics[3], "weave:engine:wipe-client:finish");
+
+  await cleanup(engine);
+});
+
+add_task(async function test_enabled() {
+  _("Engine.enabled corresponds to preference");
+  let engine = new SteamEngine("Steam", Service);
+  await engine.initialize();
+  try {
+    Assert.ok(!engine.enabled);
+    Svc.PrefBranch.setBoolPref("engine.steam", true);
+    Assert.ok(engine.enabled);
+
+    engine.enabled = false;
+    Assert.ok(!Svc.PrefBranch.getBoolPref("engine.steam"));
+  } finally {
+    await cleanup(engine);
+  }
+});
+
+add_task(async function test_sync() {
+  let engine = new SteamEngine("Steam", Service);
+  await engine.initialize();
+  try {
+    _("Engine.sync doesn't call _sync if it's not enabled");
+    Assert.ok(!engine.enabled);
+    Assert.ok(!engine.wasSynced);
+    await engine.sync();
+
+    Assert.ok(!engine.wasSynced);
+
+    _("Engine.sync calls _sync if it's enabled");
+    engine.enabled = true;
+
+    await engine.sync();
+    Assert.ok(engine.wasSynced);
+    Assert.equal(engineObserver.topics[0], "weave:engine:sync:start");
+    Assert.equal(engineObserver.topics[1], "weave:engine:sync:finish");
+  } finally {
+    await cleanup(engine);
+  }
+});
+
+add_task(async function test_disabled_no_track() {
+  _("When an engine is disabled, its tracker is not tracking.");
+  let engine = new SteamEngine("Steam", Service);
+  await engine.initialize();
+  let tracker = engine._tracker;
+  Assert.equal(engine, tracker.engine);
+
+  Assert.ok(!engine.enabled);
+  Assert.ok(!tracker._isTracking);
+  let changes = await tracker.getChangedIDs();
+  do_check_empty(changes);
+
+  Assert.ok(!tracker.engineIsEnabled());
+  Assert.ok(!tracker._isTracking);
+  changes = await tracker.getChangedIDs();
+  do_check_empty(changes);
+
+  let promisePrefChangeHandled = Promise.withResolvers();
+  const origMethod = tracker.onEngineEnabledChanged;
+  tracker.onEngineEnabledChanged = async (...args) => {
+    await origMethod.apply(tracker, args);
+    promisePrefChangeHandled.resolve();
+  };
+
+  engine.enabled = true; // Also enables the tracker automatically.
+  await promisePrefChangeHandled.promise;
+  Assert.ok(tracker._isTracking);
+  changes = await tracker.getChangedIDs();
+  do_check_empty(changes);
+
+  await tracker.addChangedID("abcdefghijkl");
+  changes = await tracker.getChangedIDs();
+  Assert.less(0, changes.abcdefghijkl);
+  promisePrefChangeHandled = Promise.withResolvers();
+  Svc.PrefBranch.setBoolPref("engine." + engine.prefName, false);
+  await promisePrefChangeHandled.promise;
+  Assert.ok(!tracker._isTracking);
+  changes = await tracker.getChangedIDs();
+  do_check_empty(changes);
+
+  await cleanup(engine);
+});

src/zen/tests/mochitests/sync/test_engine_abort.js

@@ -0,0 +1,79 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { WBORecord } = ChromeUtils.importESModule(
+  "resource://services-sync/record.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { RotaryEngine } = ChromeUtils.importESModule(
+  "resource://testing-common/services/sync/rotaryengine.sys.mjs"
+);
+
+add_task(async function test_processIncoming_abort() {
+  _(
+    "An abort exception, raised in applyIncoming, will abort _processIncoming."
+  );
+  let engine = new RotaryEngine(Service);
+
+  let collection = new ServerCollection();
+  let id = Utils.makeGUID();
+  let payload = encryptPayload({ id, denomination: "Record No. " + id });
+  collection.insert(id, payload);
+
+  let server = sync_httpd_setup({
+    "/1.1/foo/storage/rotary": collection.handler(),
+  });
+
+  await SyncTestingInfrastructure(server);
+  await generateNewKeys(Service.collectionKeys);
+
+  _("Create some server data.");
+  let syncID = await engine.resetLocalSyncID();
+  let meta_global = Service.recordManager.set(
+    engine.metaURL,
+    new WBORecord(engine.metaURL)
+  );
+  meta_global.payload.engines = { rotary: { version: engine.version, syncID } };
+  _("Fake applyIncoming to abort.");
+  engine._store.applyIncoming = async function () {
+    let ex = {
+      code: SyncEngine.prototype.eEngineAbortApplyIncoming,
+      cause: "Nooo",
+    };
+    _("Throwing: " + JSON.stringify(ex));
+    throw ex;
+  };
+
+  _("Trying _processIncoming. It will throw after aborting.");
+  let err;
+  try {
+    await engine._syncStartup();
+    await engine._processIncoming();
+  } catch (ex) {
+    err = ex;
+  }
+
+  Assert.equal(err, "Nooo");
+  err = undefined;
+
+  _("Trying engine.sync(). It will abort without error.");
+  try {
+    // This will quietly fail.
+    await engine.sync();
+  } catch (ex) {
+    err = ex;
+  }
+
+  Assert.equal(err, undefined);
+
+  await promiseStopServer(server);
+  for (const pref of Svc.PrefBranch.getChildList("")) {
+    Svc.PrefBranch.clearUserPref(pref);
+  }
+  Service.recordManager.clearCache();
+
+  await engine._tracker.clearChangedIDs();
+  await engine.finalize();
+});

src/zen/tests/mochitests/sync/test_engine_changes_during_sync.js

@@ -0,0 +1,611 @@
+const { FormHistory } = ChromeUtils.importESModule(
+  "resource://gre/modules/FormHistory.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { Bookmark, BookmarkFolder, BookmarkQuery } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/bookmarks.sys.mjs"
+);
+const { HistoryRec } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/history.sys.mjs"
+);
+const { FormRec } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/forms.sys.mjs"
+);
+const { LoginRec } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/passwords.sys.mjs"
+);
+const { PrefRec } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/prefs.sys.mjs"
+);
+
+const LoginInfo = Components.Constructor(
+  "@mozilla.org/login-manager/loginInfo;1",
+  Ci.nsILoginInfo,
+  "init"
+);
+
+/**
+ * We don't test the clients or tabs engines because neither has conflict
+ * resolution logic. The clients engine syncs twice per global sync, and
+ * custom conflict resolution logic for commands that doesn't use
+ * timestamps. Tabs doesn't have conflict resolution at all, since it's
+ * read-only.
+ */
+
+async function assertChildGuids(folderGuid, expectedChildGuids, message) {
+  let tree = await PlacesUtils.promiseBookmarksTree(folderGuid);
+  let childGuids = tree.children.map(child => child.guid);
+  deepEqual(childGuids, expectedChildGuids, message);
+}
+
+async function cleanup(engine, server) {
+  await engine._tracker.stop();
+  await engine._store.wipe();
+  for (const pref of Svc.PrefBranch.getChildList("")) {
+    Svc.PrefBranch.clearUserPref(pref);
+  }
+  Service.recordManager.clearCache();
+  await promiseStopServer(server);
+}
+
+add_task(async function test_history_change_during_sync() {
+  _("Ensure that we don't bump the score when applying history records.");
+
+  enableValidationPrefs();
+
+  let engine = Service.engineManager.get("history");
+  let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]);
+  await SyncTestingInfrastructure(server);
+  let collection = server.user("foo").collection("history");
+
+  // Override `uploadOutgoing` to insert a record while we're applying
+  // changes. The tracker should ignore this change.
+  let uploadOutgoing = engine._uploadOutgoing;
+  engine._uploadOutgoing = async function () {
+    engine._uploadOutgoing = uploadOutgoing;
+    try {
+      await uploadOutgoing.call(this);
+    } finally {
+      _("Inserting local history visit");
+      await addVisit("during_sync");
+      await engine._tracker.asyncObserver.promiseObserversComplete();
+    }
+  };
+
+  engine._tracker.start();
+
+  try {
+    let remoteRec = new HistoryRec("history", "UrOOuzE5QM-e");
+    remoteRec.histUri = "http://getfirefox.com/";
+    remoteRec.title = "Get Firefox!";
+    remoteRec.visits = [
+      {
+        date: PlacesUtils.toPRTime(Date.now()),
+        type: PlacesUtils.history.TRANSITION_TYPED,
+      },
+    ];
+    collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext));
+
+    await sync_engine_and_validate_telem(engine, true);
+    strictEqual(
+      Service.scheduler.globalScore,
+      0,
+      "Should not bump global score for visits added during sync"
+    );
+
+    equal(
+      collection.count(),
+      1,
+      "New local visit should not exist on server after first sync"
+    );
+
+    await sync_engine_and_validate_telem(engine, true);
+    strictEqual(
+      Service.scheduler.globalScore,
+      0,
+      "Should not bump global score during second history sync"
+    );
+
+    equal(
+      collection.count(),
+      2,
+      "New local visit should exist on server after second sync"
+    );
+  } finally {
+    engine._uploadOutgoing = uploadOutgoing;
+    await cleanup(engine, server);
+  }
+});
+
+add_task(async function test_passwords_change_during_sync() {
+  _("Ensure that we don't bump the score when applying passwords.");
+
+  enableValidationPrefs();
+
+  let engine = Service.engineManager.get("passwords");
+  let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]);
+  await SyncTestingInfrastructure(server);
+  let collection = server.user("foo").collection("passwords");
+
+  let uploadOutgoing = engine._uploadOutgoing;
+  engine._uploadOutgoing = async function () {
+    engine._uploadOutgoing = uploadOutgoing;
+    try {
+      await uploadOutgoing.call(this);
+    } finally {
+      _("Inserting local password");
+      let login = new LoginInfo(
+        "https://example.com",
+        "",
+        null,
+        "username",
+        "password",
+        "",
+        ""
+      );
+      await Services.logins.addLoginAsync(login);
+      await engine._tracker.asyncObserver.promiseObserversComplete();
+    }
+  };
+
+  engine._tracker.start();
+
+  try {
+    let remoteRec = new LoginRec(
+      "passwords",
+      "{765e3d6e-071d-d640-a83d-81a7eb62d3ed}"
+    );
+    remoteRec.formSubmitURL = "";
+    remoteRec.httpRealm = "";
+    remoteRec.hostname = "https://mozilla.org";
+    remoteRec.username = "username";
+    remoteRec.password = "sekrit";
+    remoteRec.timeCreated = Date.now();
+    remoteRec.timePasswordChanged = Date.now();
+    collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext));
+
+    await sync_engine_and_validate_telem(engine, true);
+    strictEqual(
+      Service.scheduler.globalScore,
+      0,
+      "Should not bump global score for passwords added during first sync"
+    );
+
+    equal(
+      collection.count(),
+      1,
+      "New local password should not exist on server after first sync"
+    );
+
+    await sync_engine_and_validate_telem(engine, true);
+    strictEqual(
+      Service.scheduler.globalScore,
+      0,
+      "Should not bump global score during second passwords sync"
+    );
+
+    equal(
+      collection.count(),
+      2,
+      "New local password should exist on server after second sync"
+    );
+  } finally {
+    engine._uploadOutgoing = uploadOutgoing;
+    await cleanup(engine, server);
+  }
+});
+
+add_task(async function test_prefs_change_during_sync() {
+  _("Ensure that we don't bump the score when applying prefs.");
+
+  const TEST_PREF = "test.duringSync";
+  // create a "control pref" for the pref we sync.
+  Services.prefs.setBoolPref("services.sync.prefs.sync.test.duringSync", true);
+
+  enableValidationPrefs();
+
+  let engine = Service.engineManager.get("prefs");
+  let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]);
+  await SyncTestingInfrastructure(server);
+  let collection = server.user("foo").collection("prefs");
+
+  let uploadOutgoing = engine._uploadOutgoing;
+  engine._uploadOutgoing = async function () {
+    engine._uploadOutgoing = uploadOutgoing;
+    try {
+      await uploadOutgoing.call(this);
+    } finally {
+      _("Updating local pref value");
+      // Change the value of a synced pref.
+      Services.prefs.setStringPref(TEST_PREF, "hello");
+      await engine._tracker.asyncObserver.promiseObserversComplete();
+    }
+  };
+
+  engine._tracker.start();
+
+  try {
+    // All synced prefs are stored in a single record, so we'll only ever
+    // have one record on the server. This test just checks that we don't
+    // track or upload prefs changed during the sync.
+    let guid = CommonUtils.encodeBase64URL(Services.appinfo.ID);
+    let remoteRec = new PrefRec("prefs", guid);
+    remoteRec.value = {
+      [TEST_PREF]: "world",
+    };
+    collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext));
+
+    await sync_engine_and_validate_telem(engine, true);
+    strictEqual(
+      Service.scheduler.globalScore,
+      0,
+      "Should not bump global score for prefs added during first sync"
+    );
+    let payloads = collection.payloads();
+    equal(
+      payloads.length,
+      1,
+      "Should not upload multiple prefs records after first sync"
+    );
+    equal(
+      payloads[0].value[TEST_PREF],
+      "world",
+      "Should not upload pref value changed during first sync"
+    );
+
+    await sync_engine_and_validate_telem(engine, true);
+    strictEqual(
+      Service.scheduler.globalScore,
+      0,
+      "Should not bump global score during second prefs sync"
+    );
+    payloads = collection.payloads();
+    equal(
+      payloads.length,
+      1,
+      "Should not upload multiple prefs records after second sync"
+    );
+    equal(
+      payloads[0].value[TEST_PREF],
+      "hello",
+      "Should upload changed pref value during second sync"
+    );
+  } finally {
+    engine._uploadOutgoing = uploadOutgoing;
+    await cleanup(engine, server);
+    Services.prefs.clearUserPref(TEST_PREF);
+  }
+});
+
+add_task(async function test_forms_change_during_sync() {
+  _("Ensure that we don't bump the score when applying form records.");
+
+  enableValidationPrefs();
+
+  let engine = Service.engineManager.get("forms");
+  let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]);
+  await SyncTestingInfrastructure(server);
+  let collection = server.user("foo").collection("forms");
+
+  let uploadOutgoing = engine._uploadOutgoing;
+  engine._uploadOutgoing = async function () {
+    engine._uploadOutgoing = uploadOutgoing;
+    try {
+      await uploadOutgoing.call(this);
+    } finally {
+      _("Inserting local form history entry");
+      await FormHistory.update([
+        {
+          op: "add",
+          fieldname: "favoriteDrink",
+          value: "cocoa",
+        },
+      ]);
+      await engine._tracker.asyncObserver.promiseObserversComplete();
+    }
+  };
+
+  engine._tracker.start();
+
+  try {
+    // Add an existing remote form history entry. We shouldn't bump the score when
+    // we apply this record.
+    let remoteRec = new FormRec("forms", "Tl9dHgmJSR6FkyxS");
+    remoteRec.name = "name";
+    remoteRec.value = "alice";
+    collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext));
+
+    await sync_engine_and_validate_telem(engine, true);
+    strictEqual(
+      Service.scheduler.globalScore,
+      0,
+      "Should not bump global score for forms added during first sync"
+    );
+
+    equal(
+      collection.count(),
+      1,
+      "New local form should not exist on server after first sync"
+    );
+
+    await sync_engine_and_validate_telem(engine, true);
+    strictEqual(
+      Service.scheduler.globalScore,
+      0,
+      "Should not bump global score during second forms sync"
+    );
+
+    equal(
+      collection.count(),
+      2,
+      "New local form should exist on server after second sync"
+    );
+  } finally {
+    engine._uploadOutgoing = uploadOutgoing;
+    await cleanup(engine, server);
+  }
+});
+
+add_task(async function test_bookmark_change_during_sync() {
+  _("Ensure that we track bookmark changes made during a sync.");
+
+  enableValidationPrefs();
+  let schedulerProto = Object.getPrototypeOf(Service.scheduler);
+  let syncThresholdDescriptor = Object.getOwnPropertyDescriptor(
+    schedulerProto,
+    "syncThreshold"
+  );
+  Object.defineProperty(Service.scheduler, "syncThreshold", {
+    // Trigger resync if any changes exist, rather than deciding based on the
+    // normal sync threshold.
+    get: () => 0,
+  });
+
+  let engine = Service.engineManager.get("bookmarks");
+  let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]);
+  await SyncTestingInfrastructure(server);
+
+  // Already-tracked bookmarks that shouldn't be uploaded during the first sync.
+  let bzBmk = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "https://bugzilla.mozilla.org/",
+    title: "Bugzilla",
+  });
+  _(`Bugzilla GUID: ${bzBmk.guid}`);
+
+  await PlacesTestUtils.setBookmarkSyncFields({
+    guid: bzBmk.guid,
+    syncChangeCounter: 0,
+    syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+  });
+
+  let collection = server.user("foo").collection("bookmarks");
+
+  let bmk3; // New child of Folder 1, created locally during sync.
+
+  let uploadOutgoing = engine._uploadOutgoing;
+  engine._uploadOutgoing = async function () {
+    engine._uploadOutgoing = uploadOutgoing;
+    try {
+      await uploadOutgoing.call(this);
+    } finally {
+      _("Inserting bookmark into local store");
+      bmk3 = await PlacesUtils.bookmarks.insert({
+        parentGuid: folder1.guid,
+        url: "https://mozilla.org/",
+        title: "Mozilla",
+      });
+      await engine._tracker.asyncObserver.promiseObserversComplete();
+    }
+  };
+
+  // New bookmarks that should be uploaded during the first sync.
+  let folder1 = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "Folder 1",
+  });
+  _(`Folder GUID: ${folder1.guid}`);
+
+  let tbBmk = await PlacesUtils.bookmarks.insert({
+    parentGuid: folder1.guid,
+    url: "http://getthunderbird.com/",
+    title: "Get Thunderbird!",
+  });
+  _(`Thunderbird GUID: ${tbBmk.guid}`);
+
+  engine._tracker.start();
+
+  try {
+    let bmk2_guid = "get-firefox1"; // New child of Folder 1, created remotely.
+    let folder2_guid = "folder2-1111"; // New folder, created remotely.
+    let tagQuery_guid = "tag-query111"; // New tag query child of Folder 2, created remotely.
+    let bmk4_guid = "example-org1"; // New tagged child of Folder 2, created remotely.
+    {
+      // An existing record changed on the server that should not trigger
+      // another sync when applied.
+      let remoteBzBmk = new Bookmark("bookmarks", bzBmk.guid);
+      remoteBzBmk.bmkUri = "https://bugzilla.mozilla.org/";
+      remoteBzBmk.description = "New description";
+      remoteBzBmk.title = "Bugzilla";
+      remoteBzBmk.tags = ["new", "tags"];
+      remoteBzBmk.parentName = "Bookmarks Menu";
+      remoteBzBmk.parentid = "menu";
+      collection.insert(bzBmk.guid, encryptPayload(remoteBzBmk.cleartext));
+
+      let remoteFolder = new BookmarkFolder("bookmarks", folder2_guid);
+      remoteFolder.title = "Folder 2";
+      remoteFolder.children = [bmk4_guid, tagQuery_guid];
+      remoteFolder.parentName = "Bookmarks Menu";
+      remoteFolder.parentid = "menu";
+      collection.insert(folder2_guid, encryptPayload(remoteFolder.cleartext));
+
+      let remoteFxBmk = new Bookmark("bookmarks", bmk2_guid);
+      remoteFxBmk.bmkUri = "http://getfirefox.com/";
+      remoteFxBmk.description = "Firefox is awesome.";
+      remoteFxBmk.title = "Get Firefox!";
+      remoteFxBmk.tags = ["firefox", "awesome", "browser"];
+      remoteFxBmk.keyword = "awesome";
+      remoteFxBmk.parentName = "Folder 1";
+      remoteFxBmk.parentid = folder1.guid;
+      collection.insert(bmk2_guid, encryptPayload(remoteFxBmk.cleartext));
+
+      // A tag query referencing a nonexistent tag folder, which we should
+      // create locally when applying the record.
+      let remoteTagQuery = new BookmarkQuery("bookmarks", tagQuery_guid);
+      remoteTagQuery.bmkUri = "place:type=7&folder=999";
+      remoteTagQuery.title = "Taggy tags";
+      remoteTagQuery.folderName = "taggy";
+      remoteTagQuery.parentName = "Folder 2";
+      remoteTagQuery.parentid = folder2_guid;
+      collection.insert(
+        tagQuery_guid,
+        encryptPayload(remoteTagQuery.cleartext)
+      );
+
+      // A bookmark that should appear in the results for the tag query.
+      let remoteTaggedBmk = new Bookmark("bookmarks", bmk4_guid);
+      remoteTaggedBmk.bmkUri = "https://example.org/";
+      remoteTaggedBmk.title = "Tagged bookmark";
+      remoteTaggedBmk.tags = ["taggy"];
+      remoteTaggedBmk.parentName = "Folder 2";
+      remoteTaggedBmk.parentid = folder2_guid;
+      collection.insert(bmk4_guid, encryptPayload(remoteTaggedBmk.cleartext));
+
+      collection.insert(
+        "toolbar",
+        encryptPayload({
+          id: "toolbar",
+          type: "folder",
+          title: "toolbar",
+          children: [folder1.guid],
+          parentName: "places",
+          parentid: "places",
+        })
+      );
+
+      collection.insert(
+        "menu",
+        encryptPayload({
+          id: "menu",
+          type: "folder",
+          title: "menu",
+          children: [bzBmk.guid, folder2_guid],
+          parentName: "places",
+          parentid: "places",
+        })
+      );
+
+      collection.insert(
+        folder1.guid,
+        encryptPayload({
+          id: folder1.guid,
+          type: "folder",
+          title: "Folder 1",
+          children: [bmk2_guid],
+          parentName: "toolbar",
+          parentid: "toolbar",
+        })
+      );
+    }
+
+    await assertChildGuids(
+      folder1.guid,
+      [tbBmk.guid],
+      "Folder should have 1 child before first sync"
+    );
+
+    let pingsPromise = wait_for_pings(2);
+
+    let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      Object.keys(changes).sort(),
+      [folder1.guid, tbBmk.guid, "menu", "mobile", "toolbar", "unfiled"].sort(),
+      "Should track bookmark and folder created before first sync"
+    );
+
+    // Unlike the tests above, we can't use `sync_engine_and_validate_telem`
+    // because the bookmarks engine will automatically schedule a follow-up
+    // sync for us.
+    _("Perform first sync and immediate follow-up sync");
+    Service.sync({ engines: ["bookmarks"] });
+
+    let pings = await pingsPromise;
+    equal(pings.length, 2, "Should submit two pings");
+    ok(
+      pings.every(p => {
+        assert_success_ping(p);
+        return p.syncs.length == 1;
+      }),
+      "Should submit 1 sync per ping"
+    );
+
+    strictEqual(
+      Service.scheduler.globalScore,
+      0,
+      "Should reset global score after follow-up sync"
+    );
+    ok(bmk3, "Should insert bookmark during first sync to simulate change");
+    ok(
+      collection.wbo(bmk3.guid),
+      "Changed bookmark should be uploaded after follow-up sync"
+    );
+
+    let bmk2 = await PlacesUtils.bookmarks.fetch({
+      guid: bmk2_guid,
+    });
+    ok(bmk2, "Remote bookmark should be applied during first sync");
+    {
+      // We only check child GUIDs, and not their order, because the exact
+      // order is an implementation detail.
+      let folder1Children = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+        folder1.guid
+      );
+      deepEqual(
+        folder1Children.sort(),
+        [bmk2_guid, tbBmk.guid, bmk3.guid].sort(),
+        "Folder 1 should have 3 children after first sync"
+      );
+    }
+    await assertChildGuids(
+      folder2_guid,
+      [bmk4_guid, tagQuery_guid],
+      "Folder 2 should have 2 children after first sync"
+    );
+    let taggedURIs = [];
+    await PlacesUtils.bookmarks.fetch({ tags: ["taggy"] }, b =>
+      taggedURIs.push(b.url)
+    );
+    equal(taggedURIs.length, 1, "Should have 1 tagged URI");
+    equal(
+      taggedURIs[0].href,
+      "https://example.org/",
+      "Synced tagged bookmark should appear in tagged URI list"
+    );
+
+    changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      changes,
+      {},
+      "Should have already uploaded changes in follow-up sync"
+    );
+
+    // First ping won't include validation data, since we've changed bookmarks
+    // and `canValidate` will indicate it can't proceed.
+    let engineData = pings.map(p => {
+      return p.syncs[0].engines.find(e => e.name == "bookmarks-buffered");
+    });
+    ok(engineData[0].validation, "Engine should validate after first sync");
+    ok(engineData[1].validation, "Engine should validate after second sync");
+  } finally {
+    Object.defineProperty(
+      schedulerProto,
+      "syncThreshold",
+      syncThresholdDescriptor
+    );
+    engine._uploadOutgoing = uploadOutgoing;
+    await cleanup(engine, server);
+  }
+});

src/zen/tests/mochitests/sync/test_enginemanager.js

@@ -0,0 +1,232 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+function PetrolEngine() {}
+PetrolEngine.prototype.name = "petrol";
+PetrolEngine.prototype.finalize = async function () {};
+
+function DieselEngine() {}
+DieselEngine.prototype.name = "diesel";
+DieselEngine.prototype.finalize = async function () {};
+
+function DummyEngine() {}
+DummyEngine.prototype.name = "dummy";
+DummyEngine.prototype.finalize = async function () {};
+
+class ActualEngine extends SyncEngine {
+  constructor(service) {
+    super("Actual", service);
+  }
+}
+
+add_task(async function test_basics() {
+  _("We start out with a clean slate");
+
+  let manager = new EngineManager(Service);
+
+  let engines = await manager.getAll();
+  Assert.equal(engines.length, 0);
+  Assert.equal(await manager.get("dummy"), undefined);
+
+  _("Register an engine");
+  await manager.register(DummyEngine);
+  let dummy = await manager.get("dummy");
+  Assert.ok(dummy instanceof DummyEngine);
+
+  engines = await manager.getAll();
+  Assert.equal(engines.length, 1);
+  Assert.equal(engines[0], dummy);
+
+  _("Register an already registered engine is ignored");
+  await manager.register(DummyEngine);
+  Assert.equal(await manager.get("dummy"), dummy);
+
+  _("Register multiple engines in one go");
+  await manager.register([PetrolEngine, DieselEngine]);
+  let petrol = await manager.get("petrol");
+  let diesel = await manager.get("diesel");
+  Assert.ok(petrol instanceof PetrolEngine);
+  Assert.ok(diesel instanceof DieselEngine);
+
+  engines = await manager.getAll();
+  Assert.equal(engines.length, 3);
+  Assert.notEqual(engines.indexOf(petrol), -1);
+  Assert.notEqual(engines.indexOf(diesel), -1);
+
+  _("Retrieve multiple engines in one go");
+  engines = await manager.get(["dummy", "diesel"]);
+  Assert.equal(engines.length, 2);
+  Assert.notEqual(engines.indexOf(dummy), -1);
+  Assert.notEqual(engines.indexOf(diesel), -1);
+
+  _("getEnabled() only returns enabled engines");
+  engines = await manager.getEnabled();
+  Assert.equal(engines.length, 0);
+
+  petrol.enabled = true;
+  engines = await manager.getEnabled();
+  Assert.equal(engines.length, 1);
+  Assert.equal(engines[0], petrol);
+
+  dummy.enabled = true;
+  diesel.enabled = true;
+  engines = await manager.getEnabled();
+  Assert.equal(engines.length, 3);
+
+  _("getEnabled() returns enabled engines in sorted order");
+  petrol.syncPriority = 1;
+  dummy.syncPriority = 2;
+  diesel.syncPriority = 3;
+
+  engines = await manager.getEnabled();
+
+  Assert.deepEqual(engines, [petrol, dummy, diesel]);
+
+  _("Changing the priorities should change the order in getEnabled()");
+
+  dummy.syncPriority = 4;
+
+  engines = await manager.getEnabled();
+
+  Assert.deepEqual(engines, [petrol, diesel, dummy]);
+
+  _("Unregister an engine by name");
+  await manager.unregister("dummy");
+  Assert.equal(await manager.get("dummy"), undefined);
+  engines = await manager.getAll();
+  Assert.equal(engines.length, 2);
+  Assert.equal(engines.indexOf(dummy), -1);
+
+  _("Unregister an engine by value");
+  // manager.unregister() checks for instanceof Engine, so let's make one:
+  await manager.register(ActualEngine);
+  let actual = await manager.get("actual");
+  Assert.ok(actual instanceof ActualEngine);
+  Assert.ok(actual instanceof SyncEngine);
+
+  await manager.unregister(actual);
+  Assert.equal(await manager.get("actual"), undefined);
+});
+
+class AutoEngine {
+  constructor(type) {
+    this.name = "automobile";
+    this.type = type;
+    this.initializeCalled = false;
+    this.finalizeCalled = false;
+    this.isActive = false;
+  }
+
+  async initialize() {
+    Assert.ok(!this.initializeCalled);
+    Assert.equal(AutoEngine.current, undefined);
+    this.initializeCalled = true;
+    this.isActive = true;
+    AutoEngine.current = this;
+  }
+
+  async finalize() {
+    Assert.equal(AutoEngine.current, this);
+    Assert.ok(!this.finalizeCalled);
+    Assert.ok(this.isActive);
+    this.finalizeCalled = true;
+    this.isActive = false;
+    AutoEngine.current = undefined;
+  }
+}
+
+class GasolineEngine extends AutoEngine {
+  constructor() {
+    super("gasoline");
+  }
+}
+
+class ElectricEngine extends AutoEngine {
+  constructor() {
+    super("electric");
+  }
+}
+
+add_task(async function test_alternates() {
+  let manager = new EngineManager(Service);
+  let engines = await manager.getAll();
+  Assert.equal(engines.length, 0);
+
+  const prefName = "services.sync.engines.automobile.electric";
+  Services.prefs.clearUserPref(prefName);
+
+  await manager.registerAlternatives(
+    "automobile",
+    prefName,
+    ElectricEngine,
+    GasolineEngine
+  );
+
+  let gasEngine = manager.get("automobile");
+  Assert.equal(gasEngine.type, "gasoline");
+
+  Assert.ok(gasEngine.isActive);
+  Assert.ok(gasEngine.initializeCalled);
+  Assert.ok(!gasEngine.finalizeCalled);
+  Assert.equal(AutoEngine.current, gasEngine);
+
+  _("Check that setting the controlling pref to false makes no difference");
+  Services.prefs.setBoolPref(prefName, false);
+  Assert.equal(manager.get("automobile"), gasEngine);
+  Assert.ok(gasEngine.isActive);
+  Assert.ok(gasEngine.initializeCalled);
+  Assert.ok(!gasEngine.finalizeCalled);
+
+  _("Even after the call to switchAlternatives");
+  await manager.switchAlternatives();
+  Assert.equal(manager.get("automobile"), gasEngine);
+  Assert.ok(gasEngine.isActive);
+  Assert.ok(gasEngine.initializeCalled);
+  Assert.ok(!gasEngine.finalizeCalled);
+
+  _("Set the pref to true, we still shouldn't switch yet");
+  Services.prefs.setBoolPref(prefName, true);
+  Assert.equal(manager.get("automobile"), gasEngine);
+  Assert.ok(gasEngine.isActive);
+  Assert.ok(gasEngine.initializeCalled);
+  Assert.ok(!gasEngine.finalizeCalled);
+
+  _("Now we expect to switch from gas to electric");
+  await manager.switchAlternatives();
+  let elecEngine = manager.get("automobile");
+  Assert.equal(elecEngine.type, "electric");
+  Assert.ok(elecEngine.isActive);
+  Assert.ok(elecEngine.initializeCalled);
+  Assert.ok(!elecEngine.finalizeCalled);
+  Assert.equal(AutoEngine.current, elecEngine);
+
+  Assert.ok(!gasEngine.isActive);
+  Assert.ok(gasEngine.finalizeCalled);
+
+  _("Switch back, and ensure we get a new instance that got initialized again");
+  Services.prefs.setBoolPref(prefName, false);
+  await manager.switchAlternatives();
+
+  // First make sure we deactivated the electric engine as we should
+  Assert.ok(!elecEngine.isActive);
+  Assert.ok(elecEngine.initializeCalled);
+  Assert.ok(elecEngine.finalizeCalled);
+
+  let newGasEngine = manager.get("automobile");
+  Assert.notEqual(newGasEngine, gasEngine);
+  Assert.equal(newGasEngine.type, "gasoline");
+
+  Assert.ok(newGasEngine.isActive);
+  Assert.ok(newGasEngine.initializeCalled);
+  Assert.ok(!newGasEngine.finalizeCalled);
+
+  _("Make sure unregister removes the alt info too");
+  await manager.unregister("automobile");
+  Assert.equal(manager.get("automobile"), null);
+  Assert.ok(newGasEngine.finalizeCalled);
+  Assert.deepEqual(Object.keys(manager._altEngineInfo), []);
+});

src/zen/tests/mochitests/sync/test_errorhandler_1.js

@@ -0,0 +1,335 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { Status } = ChromeUtils.importESModule(
+  "resource://services-sync/status.sys.mjs"
+);
+
+const fakeServer = new SyncServer();
+fakeServer.start();
+const fakeServerUrl = "http://localhost:" + fakeServer.port;
+
+registerCleanupFunction(function () {
+  return promiseStopServer(fakeServer).finally(() => {
+    for (const pref of Svc.PrefBranch.getChildList("")) {
+      Svc.PrefBranch.clearUserPref(pref);
+    }
+  });
+});
+
+let engine;
+add_task(async function setup() {
+  await Service.engineManager.clear();
+  await Service.engineManager.register(EHTestsCommon.CatapultEngine);
+  engine = Service.engineManager.get("catapult");
+});
+
+async function clean() {
+  let promiseLogReset = promiseOneObserver("weave:service:reset-file-log");
+  await Service.startOver();
+  await promiseLogReset;
+  Status.resetSync();
+  Status.resetBackoff();
+  // Move log levels back to trace (startOver will have reversed this), sicne
+  syncTestLogging();
+}
+
+add_task(async function test_401_logout() {
+  enableValidationPrefs();
+
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+
+  // By calling sync, we ensure we're logged in.
+  await sync_and_validate_telem();
+  Assert.equal(Status.sync, SYNC_SUCCEEDED);
+  Assert.ok(Service.isLoggedIn);
+
+  let promiseErrors = new Promise(res => {
+    Svc.Obs.add("weave:service:sync:error", onSyncError);
+    function onSyncError() {
+      _("Got weave:service:sync:error in first sync.");
+      Svc.Obs.remove("weave:service:sync:error", onSyncError);
+
+      // Wait for the automatic next sync.
+      Svc.Obs.add("weave:service:login:error", onLoginError);
+      function onLoginError() {
+        _("Got weave:service:login:error in second sync.");
+        Svc.Obs.remove("weave:service:login:error", onLoginError);
+        res();
+      }
+    }
+  });
+
+  // Make sync fail due to login rejected.
+  await configureIdentity({ username: "janedoe" }, server);
+  Service._updateCachedURLs();
+
+  _("Starting first sync.");
+  await sync_and_validate_telem(ping => {
+    deepEqual(ping.failureReason, { name: "httperror", code: 401 });
+  });
+  _("First sync done.");
+
+  await promiseErrors;
+  Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR);
+  Assert.ok(!Service.isLoggedIn);
+
+  // Clean up.
+  await Service.startOver();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_credentials_changed_logout() {
+  enableValidationPrefs();
+
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+
+  // By calling sync, we ensure we're logged in.
+  await sync_and_validate_telem();
+  Assert.equal(Status.sync, SYNC_SUCCEEDED);
+  Assert.ok(Service.isLoggedIn);
+
+  await EHTestsCommon.generateCredentialsChangedFailure();
+
+  await sync_and_validate_telem(ping => {
+    equal(ping.status.sync, CREDENTIALS_CHANGED);
+    deepEqual(ping.failureReason, {
+      name: "unexpectederror",
+      error: "Error: Aborting sync, remote setup failed",
+    });
+  });
+
+  Assert.equal(Status.sync, CREDENTIALS_CHANGED);
+  Assert.ok(!Service.isLoggedIn);
+
+  // Clean up.
+  await Service.startOver();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_login_non_network_error() {
+  enableValidationPrefs();
+
+  // Test non-network errors are reported
+  // when calling sync
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+  Service.identity._syncKeyBundle = null;
+
+  await Service.sync();
+  Assert.equal(Status.login, LOGIN_FAILED_NO_PASSPHRASE);
+
+  await clean();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_sync_non_network_error() {
+  enableValidationPrefs();
+
+  // Test non-network errors are reported
+  // when calling sync
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+
+  // By calling sync, we ensure we're logged in.
+  await Service.sync();
+  Assert.equal(Status.sync, SYNC_SUCCEEDED);
+  Assert.ok(Service.isLoggedIn);
+
+  await EHTestsCommon.generateCredentialsChangedFailure();
+
+  await sync_and_validate_telem(ping => {
+    equal(ping.status.sync, CREDENTIALS_CHANGED);
+    deepEqual(ping.failureReason, {
+      name: "unexpectederror",
+      error: "Error: Aborting sync, remote setup failed",
+    });
+  });
+
+  Assert.equal(Status.sync, CREDENTIALS_CHANGED);
+  // If we clean this tick, telemetry won't get the right error
+  await Async.promiseYield();
+  await clean();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_login_sync_network_error() {
+  enableValidationPrefs();
+
+  // Test network errors are reported when calling sync.
+  await configureIdentity({ username: "broken.wipe" });
+  Service.clusterURL = fakeServerUrl;
+
+  await Service.sync();
+  Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR);
+
+  await clean();
+});
+
+add_task(async function test_sync_network_error() {
+  enableValidationPrefs();
+
+  // Test network errors are reported when calling sync.
+  Services.io.offline = true;
+
+  await Service.sync();
+  Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
+
+  Services.io.offline = false;
+  await clean();
+});
+
+add_task(async function test_login_non_network_error() {
+  enableValidationPrefs();
+
+  // Test non-network errors are reported
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+  Service.identity._syncKeyBundle = null;
+
+  await Service.sync();
+  Assert.equal(Status.login, LOGIN_FAILED_NO_PASSPHRASE);
+
+  await clean();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_sync_non_network_error() {
+  enableValidationPrefs();
+
+  // Test non-network errors are reported
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+
+  // By calling sync, we ensure we're logged in.
+  await Service.sync();
+  Assert.equal(Status.sync, SYNC_SUCCEEDED);
+  Assert.ok(Service.isLoggedIn);
+
+  await EHTestsCommon.generateCredentialsChangedFailure();
+
+  await Service.sync();
+  Assert.equal(Status.sync, CREDENTIALS_CHANGED);
+
+  await clean();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_login_network_error() {
+  enableValidationPrefs();
+
+  await configureIdentity({ username: "johndoe" });
+  Service.clusterURL = fakeServerUrl;
+
+  // Test network errors are not reported.
+
+  await Service.sync();
+  Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR);
+
+  Services.io.offline = false;
+  await clean();
+});
+
+add_task(async function test_sync_network_error() {
+  enableValidationPrefs();
+
+  // Test network errors are not reported.
+  Services.io.offline = true;
+
+  await Service.sync();
+  Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
+
+  Services.io.offline = false;
+  await clean();
+});
+
+add_task(async function test_sync_server_maintenance_error() {
+  enableValidationPrefs();
+
+  // Test server maintenance errors are not reported.
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+
+  const BACKOFF = 42;
+  engine.enabled = true;
+  engine.exception = { status: 503, headers: { "retry-after": BACKOFF } };
+
+  Assert.equal(Status.service, STATUS_OK);
+
+  await sync_and_validate_telem(ping => {
+    equal(ping.status.sync, SERVER_MAINTENANCE);
+    deepEqual(ping.engines.find(e => e.failureReason).failureReason, {
+      name: "httperror",
+      code: 503,
+    });
+  });
+
+  Assert.equal(Status.service, SYNC_FAILED_PARTIAL);
+  Assert.equal(Status.sync, SERVER_MAINTENANCE);
+
+  await clean();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_info_collections_login_server_maintenance_error() {
+  enableValidationPrefs();
+
+  // Test info/collections server maintenance errors are not reported.
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+
+  await configureIdentity({ username: "broken.info" }, server);
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function observe(subject) {
+    Svc.Obs.remove("weave:service:backoff:interval", observe);
+    backoffInterval = subject;
+  });
+
+  Assert.ok(!Status.enforceBackoff);
+  Assert.equal(Status.service, STATUS_OK);
+
+  await Service.sync();
+
+  Assert.ok(Status.enforceBackoff);
+  Assert.equal(backoffInterval, 42);
+  Assert.equal(Status.service, LOGIN_FAILED);
+  Assert.equal(Status.login, SERVER_MAINTENANCE);
+
+  await clean();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_meta_global_login_server_maintenance_error() {
+  enableValidationPrefs();
+
+  // Test meta/global server maintenance errors are not reported.
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+
+  await configureIdentity({ username: "broken.meta" }, server);
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function observe(subject) {
+    Svc.Obs.remove("weave:service:backoff:interval", observe);
+    backoffInterval = subject;
+  });
+
+  Assert.ok(!Status.enforceBackoff);
+  Assert.equal(Status.service, STATUS_OK);
+
+  await Service.sync();
+
+  Assert.ok(Status.enforceBackoff);
+  Assert.equal(backoffInterval, 42);
+  Assert.equal(Status.service, LOGIN_FAILED);
+  Assert.equal(Status.login, SERVER_MAINTENANCE);
+
+  await clean();
+  await promiseStopServer(server);
+});

src/zen/tests/mochitests/sync/test_errorhandler_2.js

@@ -0,0 +1,529 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { Status } = ChromeUtils.importESModule(
+  "resource://services-sync/status.sys.mjs"
+);
+const { FileUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/FileUtils.sys.mjs"
+);
+
+const fakeServer = new SyncServer();
+fakeServer.start();
+
+registerCleanupFunction(function () {
+  return promiseStopServer(fakeServer).finally(() => {
+    for (const pref of Svc.PrefBranch.getChildList("")) {
+      Svc.PrefBranch.clearUserPref(pref);
+    }
+  });
+});
+
+const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"]);
+logsdir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
+
+function removeLogFiles() {
+  let entries = logsdir.directoryEntries;
+  while (entries.hasMoreElements()) {
+    let logfile = entries.getNext().QueryInterface(Ci.nsIFile);
+    logfile.remove(false);
+  }
+}
+
+function getLogFiles() {
+  let result = [];
+  let entries = logsdir.directoryEntries;
+  while (entries.hasMoreElements()) {
+    result.push(entries.getNext().QueryInterface(Ci.nsIFile));
+  }
+  return result;
+}
+
+let engine;
+add_task(async function setup() {
+  await Service.engineManager.clear();
+  await Service.engineManager.register(EHTestsCommon.CatapultEngine);
+  engine = Service.engineManager.get("catapult");
+});
+
+async function clean() {
+  let promiseLogReset = promiseOneObserver("weave:service:reset-file-log");
+  await Service.startOver();
+  await promiseLogReset;
+  Status.resetSync();
+  Status.resetBackoff();
+  removeLogFiles();
+  // Move log levels back to trace (startOver will have reversed this), sicne
+  syncTestLogging();
+}
+
+add_task(async function test_crypto_keys_login_server_maintenance_error() {
+  enableValidationPrefs();
+
+  Status.resetSync();
+  // Test crypto/keys server maintenance errors are not reported.
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+
+  await configureIdentity({ username: "broken.keys" }, server);
+
+  // Force re-download of keys
+  Service.collectionKeys.clear();
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function observe(subject) {
+    Svc.Obs.remove("weave:service:backoff:interval", observe);
+    backoffInterval = subject;
+  });
+
+  Assert.ok(!Status.enforceBackoff);
+  Assert.equal(Status.service, STATUS_OK);
+
+  let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+  await Service.sync();
+  await promiseObserved;
+
+  Assert.ok(Status.enforceBackoff);
+  Assert.equal(backoffInterval, 42);
+  Assert.equal(Status.service, LOGIN_FAILED);
+  Assert.equal(Status.login, SERVER_MAINTENANCE);
+
+  await clean();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_lastSync_not_updated_on_complete_failure() {
+  enableValidationPrefs();
+
+  // Test info/collections prolonged server maintenance errors are reported.
+  let server = await EHTestsCommon.sync_httpd_setup();
+  await EHTestsCommon.setUp(server);
+
+  await configureIdentity({ username: "johndoe" }, server);
+
+  // Do an initial sync that we expect to be successful.
+  let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+  await sync_and_validate_telem();
+  await promiseObserved;
+
+  Assert.equal(Status.service, STATUS_OK);
+  Assert.equal(Status.sync, SYNC_SUCCEEDED);
+
+  let lastSync = Svc.PrefBranch.getStringPref("lastSync");
+
+  Assert.ok(lastSync);
+
+  // Report server maintenance on info/collections requests
+  server.registerPathHandler(
+    "/1.1/johndoe/info/collections",
+    EHTestsCommon.service_unavailable
+  );
+
+  promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+  await sync_and_validate_telem(() => {});
+  await promiseObserved;
+
+  Assert.equal(Status.sync, SERVER_MAINTENANCE);
+  Assert.equal(Status.service, SYNC_FAILED);
+
+  // We shouldn't update lastSync on complete failure.
+  Assert.equal(lastSync, Svc.PrefBranch.getStringPref("lastSync"));
+
+  await clean();
+  await promiseStopServer(server);
+});
+
+add_task(
+  async function test_sync_syncAndReportErrors_server_maintenance_error() {
+    enableValidationPrefs();
+
+    // Test server maintenance errors are reported
+    // when calling syncAndReportErrors.
+    let server = await EHTestsCommon.sync_httpd_setup();
+    await EHTestsCommon.setUp(server);
+
+    const BACKOFF = 42;
+    engine.enabled = true;
+    engine.exception = { status: 503, headers: { "retry-after": BACKOFF } };
+
+    Assert.equal(Status.service, STATUS_OK);
+
+    let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+    await Service.sync();
+    await promiseObserved;
+
+    Assert.equal(Status.service, SYNC_FAILED_PARTIAL);
+    Assert.equal(Status.sync, SERVER_MAINTENANCE);
+
+    await clean();
+    await promiseStopServer(server);
+  }
+);
+
+add_task(
+  async function test_info_collections_login_syncAndReportErrors_server_maintenance_error() {
+    enableValidationPrefs();
+
+    // Test info/collections server maintenance errors are reported
+    // when calling syncAndReportErrors.
+    let server = await EHTestsCommon.sync_httpd_setup();
+    await EHTestsCommon.setUp(server);
+
+    await configureIdentity({ username: "broken.info" }, server);
+
+    let backoffInterval;
+    Svc.Obs.add("weave:service:backoff:interval", function observe(subject) {
+      Svc.Obs.remove("weave:service:backoff:interval", observe);
+      backoffInterval = subject;
+    });
+
+    Assert.ok(!Status.enforceBackoff);
+    Assert.equal(Status.service, STATUS_OK);
+
+    let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+    await Service.sync();
+    await promiseObserved;
+
+    Assert.ok(Status.enforceBackoff);
+    Assert.equal(backoffInterval, 42);
+    Assert.equal(Status.service, LOGIN_FAILED);
+    Assert.equal(Status.login, SERVER_MAINTENANCE);
+
+    await clean();
+    await promiseStopServer(server);
+  }
+);
+
+add_task(
+  async function test_meta_global_login_syncAndReportErrors_server_maintenance_error() {
+    enableValidationPrefs();
+
+    // Test meta/global server maintenance errors are reported
+    // when calling syncAndReportErrors.
+    let server = await EHTestsCommon.sync_httpd_setup();
+    await EHTestsCommon.setUp(server);
+
+    await configureIdentity({ username: "broken.meta" }, server);
+
+    let backoffInterval;
+    Svc.Obs.add("weave:service:backoff:interval", function observe(subject) {
+      Svc.Obs.remove("weave:service:backoff:interval", observe);
+      backoffInterval = subject;
+    });
+
+    Assert.ok(!Status.enforceBackoff);
+    Assert.equal(Status.service, STATUS_OK);
+
+    let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+    await Service.sync();
+    await promiseObserved;
+
+    Assert.ok(Status.enforceBackoff);
+    Assert.equal(backoffInterval, 42);
+    Assert.equal(Status.service, LOGIN_FAILED);
+    Assert.equal(Status.login, SERVER_MAINTENANCE);
+
+    await clean();
+    await promiseStopServer(server);
+  }
+);
+
+add_task(
+  async function test_download_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
+    enableValidationPrefs();
+
+    // Test crypto/keys server maintenance errors are reported
+    // when calling syncAndReportErrors.
+    let server = await EHTestsCommon.sync_httpd_setup();
+    await EHTestsCommon.setUp(server);
+
+    await configureIdentity({ username: "broken.keys" }, server);
+    // Force re-download of keys
+    Service.collectionKeys.clear();
+
+    let backoffInterval;
+    Svc.Obs.add("weave:service:backoff:interval", function observe(subject) {
+      Svc.Obs.remove("weave:service:backoff:interval", observe);
+      backoffInterval = subject;
+    });
+
+    Assert.ok(!Status.enforceBackoff);
+    Assert.equal(Status.service, STATUS_OK);
+
+    let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+    await Service.sync();
+    await promiseObserved;
+
+    Assert.ok(Status.enforceBackoff);
+    Assert.equal(backoffInterval, 42);
+    Assert.equal(Status.service, LOGIN_FAILED);
+    Assert.equal(Status.login, SERVER_MAINTENANCE);
+
+    await clean();
+    await promiseStopServer(server);
+  }
+);
+
+add_task(
+  async function test_upload_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
+    enableValidationPrefs();
+
+    // Test crypto/keys server maintenance errors are reported
+    // when calling syncAndReportErrors.
+    let server = await EHTestsCommon.sync_httpd_setup();
+
+    // Start off with an empty account, do not upload a key.
+    await configureIdentity({ username: "broken.keys" }, server);
+
+    let backoffInterval;
+    Svc.Obs.add("weave:service:backoff:interval", function observe(subject) {
+      Svc.Obs.remove("weave:service:backoff:interval", observe);
+      backoffInterval = subject;
+    });
+
+    Assert.ok(!Status.enforceBackoff);
+    Assert.equal(Status.service, STATUS_OK);
+
+    let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+    await Service.sync();
+    await promiseObserved;
+
+    Assert.ok(Status.enforceBackoff);
+    Assert.equal(backoffInterval, 42);
+    Assert.equal(Status.service, LOGIN_FAILED);
+    Assert.equal(Status.login, SERVER_MAINTENANCE);
+
+    await clean();
+    await promiseStopServer(server);
+  }
+);
+
+add_task(
+  async function test_wipeServer_login_syncAndReportErrors_server_maintenance_error() {
+    enableValidationPrefs();
+
+    // Test crypto/keys server maintenance errors are reported
+    // when calling syncAndReportErrors.
+    let server = await EHTestsCommon.sync_httpd_setup();
+
+    // Start off with an empty account, do not upload a key.
+    await configureIdentity({ username: "broken.wipe" }, server);
+
+    let backoffInterval;
+    Svc.Obs.add("weave:service:backoff:interval", function observe(subject) {
+      Svc.Obs.remove("weave:service:backoff:interval", observe);
+      backoffInterval = subject;
+    });
+
+    Assert.ok(!Status.enforceBackoff);
+    Assert.equal(Status.service, STATUS_OK);
+
+    let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+    await Service.sync();
+    await promiseObserved;
+
+    Assert.ok(Status.enforceBackoff);
+    Assert.equal(backoffInterval, 42);
+    Assert.equal(Status.service, LOGIN_FAILED);
+    Assert.equal(Status.login, SERVER_MAINTENANCE);
+
+    await clean();
+    await promiseStopServer(server);
+  }
+);
+
+add_task(
+  async function test_wipeRemote_syncAndReportErrors_server_maintenance_error() {
+    enableValidationPrefs();
+
+    // Test that we report prolonged server maintenance errors that occur whilst
+    // wiping all remote devices.
+    let server = await EHTestsCommon.sync_httpd_setup();
+
+    await configureIdentity({ username: "broken.wipe" }, server);
+    await EHTestsCommon.generateAndUploadKeys();
+
+    engine.exception = null;
+    engine.enabled = true;
+
+    let backoffInterval;
+    Svc.Obs.add("weave:service:backoff:interval", function observe(subject) {
+      Svc.Obs.remove("weave:service:backoff:interval", observe);
+      backoffInterval = subject;
+    });
+
+    Assert.ok(!Status.enforceBackoff);
+    Assert.equal(Status.service, STATUS_OK);
+
+    Svc.PrefBranch.setStringPref("firstSync", "wipeRemote");
+
+    let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+    await Service.sync();
+    await promiseObserved;
+
+    Assert.ok(Status.enforceBackoff);
+    Assert.equal(backoffInterval, 42);
+    Assert.equal(Status.service, SYNC_FAILED);
+    Assert.equal(Status.sync, SERVER_MAINTENANCE);
+    Assert.equal(Svc.PrefBranch.getStringPref("firstSync"), "wipeRemote");
+
+    await clean();
+    await promiseStopServer(server);
+  }
+);
+
+add_task(async function test_sync_engine_generic_fail() {
+  enableValidationPrefs();
+
+  equal(getLogFiles().length, 0);
+
+  let server = await EHTestsCommon.sync_httpd_setup();
+  engine.enabled = true;
+  engine.sync = async function sync() {
+    Svc.Obs.notify("weave:engine:sync:error", ENGINE_UNKNOWN_FAIL, "catapult");
+  };
+  let lastSync = Svc.PrefBranch.getStringPref("lastSync", null);
+  let log = Log.repository.getLogger("Sync.ErrorHandler");
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+
+  Assert.equal(Status.engines.catapult, undefined);
+
+  let promiseObserved = new Promise(res => {
+    Svc.Obs.add("weave:engine:sync:finish", function onEngineFinish() {
+      Svc.Obs.remove("weave:engine:sync:finish", onEngineFinish);
+
+      log.info("Adding reset-file-log observer.");
+      Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+        Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+        res();
+      });
+    });
+  });
+
+  Assert.ok(await EHTestsCommon.setUp(server));
+  await sync_and_validate_telem(ping => {
+    deepEqual(ping.status.service, SYNC_FAILED_PARTIAL);
+    deepEqual(ping.engines.find(e => e.status).status, ENGINE_UNKNOWN_FAIL);
+  });
+
+  await promiseObserved;
+
+  _("Status.engines: " + JSON.stringify(Status.engines));
+  Assert.equal(Status.engines.catapult, ENGINE_UNKNOWN_FAIL);
+  Assert.equal(Status.service, SYNC_FAILED_PARTIAL);
+
+  // lastSync should update on partial failure.
+  Assert.notEqual(lastSync, Svc.PrefBranch.getStringPref("lastSync"));
+
+  // Test Error log was written on SYNC_FAILED_PARTIAL.
+  let logFiles = getLogFiles();
+  equal(logFiles.length, 1);
+  Assert.ok(
+    logFiles[0].leafName.startsWith("error-sync-"),
+    logFiles[0].leafName
+  );
+
+  await clean();
+
+  await promiseStopServer(server);
+});
+
+add_task(async function test_logs_on_sync_error() {
+  enableValidationPrefs();
+
+  _(
+    "Ensure that an error is still logged when weave:service:sync:error " +
+      "is notified, despite shouldReportError returning false."
+  );
+
+  let log = Log.repository.getLogger("Sync.ErrorHandler");
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+  log.info("TESTING");
+
+  // Ensure that we report no error.
+  Status.login = MASTER_PASSWORD_LOCKED;
+
+  let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+  Svc.Obs.notify("weave:service:sync:error", {});
+  await promiseObserved;
+
+  // Test that error log was written.
+  let logFiles = getLogFiles();
+  equal(logFiles.length, 1);
+  Assert.ok(
+    logFiles[0].leafName.startsWith("error-sync-"),
+    logFiles[0].leafName
+  );
+
+  await clean();
+});
+
+add_task(async function test_logs_on_login_error() {
+  enableValidationPrefs();
+
+  _(
+    "Ensure that an error is still logged when weave:service:login:error " +
+      "is notified, despite shouldReportError returning false."
+  );
+
+  let log = Log.repository.getLogger("Sync.ErrorHandler");
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+  log.info("TESTING");
+
+  // Ensure that we report no error.
+  Status.login = MASTER_PASSWORD_LOCKED;
+
+  let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+  Svc.Obs.notify("weave:service:login:error", {});
+  await promiseObserved;
+
+  // Test that error log was written.
+  let logFiles = getLogFiles();
+  equal(logFiles.length, 1);
+  Assert.ok(
+    logFiles[0].leafName.startsWith("error-sync-"),
+    logFiles[0].leafName
+  );
+
+  await clean();
+});
+
+// This test should be the last one since it monkeypatches the engine object
+// and we should only have one engine object throughout the file (bug 629664).
+add_task(async function test_engine_applyFailed() {
+  enableValidationPrefs();
+
+  let server = await EHTestsCommon.sync_httpd_setup();
+
+  engine.enabled = true;
+  delete engine.exception;
+  engine.sync = async function sync() {
+    Svc.Obs.notify("weave:engine:sync:applied", { newFailed: 1 }, "catapult");
+  };
+
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+
+  let promiseObserved = promiseOneObserver("weave:service:reset-file-log");
+
+  Assert.equal(Status.engines.catapult, undefined);
+  Assert.ok(await EHTestsCommon.setUp(server));
+  await Service.sync();
+  await promiseObserved;
+
+  Assert.equal(Status.engines.catapult, ENGINE_APPLY_FAIL);
+  Assert.equal(Status.service, SYNC_FAILED_PARTIAL);
+
+  // Test Error log was written on SYNC_FAILED_PARTIAL.
+  let logFiles = getLogFiles();
+  equal(logFiles.length, 1);
+  Assert.ok(
+    logFiles[0].leafName.startsWith("error-sync-"),
+    logFiles[0].leafName
+  );
+
+  await clean();
+  await promiseStopServer(server);
+});

src/zen/tests/mochitests/sync/test_errorhandler_filelog.js

@@ -0,0 +1,473 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// `Service` is used as a global in head_helpers.js.
+// eslint-disable-next-line no-unused-vars
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { logManager } = ChromeUtils.importESModule(
+  "resource://gre/modules/FxAccountsCommon.sys.mjs"
+);
+const { FileUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/FileUtils.sys.mjs"
+);
+
+const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"]);
+logsdir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
+
+// Delay to wait before cleanup, to allow files to age.
+// This is so large because the file timestamp granularity is per-second, and
+// so otherwise we can end up with all of our files -- the ones we want to
+// keep, and the ones we want to clean up -- having the same modified time.
+const CLEANUP_DELAY = 2000;
+const DELAY_BUFFER = 500; // Buffer for timers on different OS platforms.
+
+function run_test() {
+  validate_all_future_pings();
+  run_next_test();
+}
+
+add_test(function test_noOutput() {
+  // Ensure that the log appender won't print anything.
+  logManager._fileAppender.level = Log.Level.Fatal + 1;
+
+  // Clear log output from startup.
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", false);
+  Svc.Obs.notify("weave:service:sync:finish");
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLogOuter() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLogOuter);
+    // Clear again without having issued any output.
+    Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", true);
+
+    Svc.Obs.add("weave:service:reset-file-log", function onResetFileLogInner() {
+      Svc.Obs.remove("weave:service:reset-file-log", onResetFileLogInner);
+
+      logManager._fileAppender.level = Log.Level.Trace;
+      for (const pref of Svc.PrefBranch.getChildList("")) {
+        Svc.PrefBranch.clearUserPref(pref);
+      }
+      run_next_test();
+    });
+
+    // Fake a successful sync.
+    Svc.Obs.notify("weave:service:sync:finish");
+  });
+});
+
+add_test(function test_logOnSuccess_false() {
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", false);
+
+  let log = Log.repository.getLogger("Sync.Test.FileLog");
+  log.info("this won't show up");
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+    // No log file was written.
+    Assert.ok(!logsdir.directoryEntries.hasMoreElements());
+
+    for (const pref of Svc.PrefBranch.getChildList("")) {
+      Svc.PrefBranch.clearUserPref(pref);
+    }
+    run_next_test();
+  });
+
+  // Fake a successful sync.
+  Svc.Obs.notify("weave:service:sync:finish");
+});
+
+function readFile(file, callback) {
+  NetUtil.asyncFetch(
+    {
+      uri: NetUtil.newURI(file),
+      loadUsingSystemPrincipal: true,
+    },
+    function (inputStream, statusCode) {
+      let data = NetUtil.readInputStreamToString(
+        inputStream,
+        inputStream.available()
+      );
+      callback(statusCode, data);
+    }
+  );
+}
+
+add_test(function test_logOnSuccess_true() {
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", true);
+
+  let log = Log.repository.getLogger("Sync.Test.FileLog");
+  const MESSAGE = "this WILL show up";
+  log.info(MESSAGE);
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+    // Exactly one log file was written.
+    let entries = logsdir.directoryEntries;
+    Assert.ok(entries.hasMoreElements());
+    let logfile = entries.getNext().QueryInterface(Ci.nsIFile);
+    Assert.equal(logfile.leafName.slice(-4), ".txt");
+    Assert.ok(logfile.leafName.startsWith("success-sync-"), logfile.leafName);
+    Assert.ok(!entries.hasMoreElements());
+
+    // Ensure the log message was actually written to file.
+    readFile(logfile, function (error, data) {
+      Assert.ok(Components.isSuccessCode(error));
+      Assert.notEqual(data.indexOf(MESSAGE), -1);
+
+      // Clean up.
+      try {
+        logfile.remove(false);
+      } catch (ex) {
+        dump("Couldn't delete file: " + ex.message + "\n");
+        // Stupid Windows box.
+      }
+
+      for (const pref of Svc.PrefBranch.getChildList("")) {
+        Svc.PrefBranch.clearUserPref(pref);
+      }
+      run_next_test();
+    });
+  });
+
+  // Fake a successful sync.
+  Svc.Obs.notify("weave:service:sync:finish");
+});
+
+add_test(function test_sync_error_logOnError_false() {
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", false);
+
+  let log = Log.repository.getLogger("Sync.Test.FileLog");
+  log.info("this won't show up");
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+    // No log file was written.
+    Assert.ok(!logsdir.directoryEntries.hasMoreElements());
+
+    for (const pref of Svc.PrefBranch.getChildList("")) {
+      Svc.PrefBranch.clearUserPref(pref);
+    }
+    run_next_test();
+  });
+
+  // Fake an unsuccessful sync.
+  Svc.Obs.notify("weave:service:sync:error");
+});
+
+add_test(function test_sync_error_logOnError_true() {
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+
+  let log = Log.repository.getLogger("Sync.Test.FileLog");
+  const MESSAGE = "this WILL show up";
+  log.info(MESSAGE);
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+    // Exactly one log file was written.
+    let entries = logsdir.directoryEntries;
+    Assert.ok(entries.hasMoreElements());
+    let logfile = entries.getNext().QueryInterface(Ci.nsIFile);
+    Assert.equal(logfile.leafName.slice(-4), ".txt");
+    Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName);
+    Assert.ok(!entries.hasMoreElements());
+
+    // Ensure the log message was actually written to file.
+    readFile(logfile, function (error, data) {
+      Assert.ok(Components.isSuccessCode(error));
+      Assert.notEqual(data.indexOf(MESSAGE), -1);
+
+      // Clean up.
+      try {
+        logfile.remove(false);
+      } catch (ex) {
+        dump("Couldn't delete file: " + ex.message + "\n");
+        // Stupid Windows box.
+      }
+
+      for (const pref of Svc.PrefBranch.getChildList("")) {
+        Svc.PrefBranch.clearUserPref(pref);
+      }
+      run_next_test();
+    });
+  });
+
+  // Fake an unsuccessful sync.
+  Svc.Obs.notify("weave:service:sync:error");
+});
+
+add_test(function test_login_error_logOnError_false() {
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", false);
+
+  let log = Log.repository.getLogger("Sync.Test.FileLog");
+  log.info("this won't show up");
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+    // No log file was written.
+    Assert.ok(!logsdir.directoryEntries.hasMoreElements());
+
+    for (const pref of Svc.PrefBranch.getChildList("")) {
+      Svc.PrefBranch.clearUserPref(pref);
+    }
+    run_next_test();
+  });
+
+  // Fake an unsuccessful login.
+  Svc.Obs.notify("weave:service:login:error");
+});
+
+add_test(function test_login_error_logOnError_true() {
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+
+  let log = Log.repository.getLogger("Sync.Test.FileLog");
+  const MESSAGE = "this WILL show up";
+  log.info(MESSAGE);
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+    // Exactly one log file was written.
+    let entries = logsdir.directoryEntries;
+    Assert.ok(entries.hasMoreElements());
+    let logfile = entries.getNext().QueryInterface(Ci.nsIFile);
+    Assert.equal(logfile.leafName.slice(-4), ".txt");
+    Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName);
+    Assert.ok(!entries.hasMoreElements());
+
+    // Ensure the log message was actually written to file.
+    readFile(logfile, function (error, data) {
+      Assert.ok(Components.isSuccessCode(error));
+      Assert.notEqual(data.indexOf(MESSAGE), -1);
+
+      // Clean up.
+      try {
+        logfile.remove(false);
+      } catch (ex) {
+        dump("Couldn't delete file: " + ex.message + "\n");
+        // Stupid Windows box.
+      }
+
+      for (const pref of Svc.PrefBranch.getChildList("")) {
+        Svc.PrefBranch.clearUserPref(pref);
+      }
+      run_next_test();
+    });
+  });
+
+  // Fake an unsuccessful login.
+  Svc.Obs.notify("weave:service:login:error");
+});
+
+add_test(function test_noNewFailed_noErrorLog() {
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", false);
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+    // No log file was written.
+    Assert.ok(!logsdir.directoryEntries.hasMoreElements());
+
+    for (const pref of Svc.PrefBranch.getChildList("")) {
+      Svc.PrefBranch.clearUserPref(pref);
+    }
+    run_next_test();
+  });
+  // failed is nonzero and newFailed is zero -- shouldn't write a log.
+  let count = {
+    applied: 8,
+    succeeded: 4,
+    failed: 5,
+    newFailed: 0,
+    reconciled: 4,
+  };
+  Svc.Obs.notify("weave:engine:sync:applied", count, "foobar-engine");
+  Svc.Obs.notify("weave:service:sync:finish");
+});
+
+add_test(function test_newFailed_errorLog() {
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", false);
+
+  let log = Log.repository.getLogger("Sync.Test.FileLog");
+  const MESSAGE = "this WILL show up 2";
+  log.info(MESSAGE);
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+    // Exactly one log file was written.
+    let entries = logsdir.directoryEntries;
+    Assert.ok(entries.hasMoreElements());
+    let logfile = entries.getNext().QueryInterface(Ci.nsIFile);
+    Assert.equal(logfile.leafName.slice(-4), ".txt");
+    Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName);
+    Assert.ok(!entries.hasMoreElements());
+
+    // Ensure the log message was actually written to file.
+    readFile(logfile, function (error, data) {
+      Assert.ok(Components.isSuccessCode(error));
+      Assert.notEqual(data.indexOf(MESSAGE), -1);
+
+      // Clean up.
+      try {
+        logfile.remove(false);
+      } catch (ex) {
+        dump("Couldn't delete file: " + ex.message + "\n");
+        // Stupid Windows box.
+      }
+
+      for (const pref of Svc.PrefBranch.getChildList("")) {
+        Svc.PrefBranch.clearUserPref(pref);
+      }
+      run_next_test();
+    });
+  });
+  // newFailed is nonzero -- should write a log.
+  let count = {
+    applied: 8,
+    succeeded: 4,
+    failed: 5,
+    newFailed: 4,
+    reconciled: 4,
+  };
+
+  Svc.Obs.notify("weave:engine:sync:applied", count, "foobar-engine");
+  Svc.Obs.notify("weave:service:sync:finish");
+});
+
+add_test(function test_errorLog_dumpAddons() {
+  Svc.PrefBranch.setStringPref("log.logger", "Trace");
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+    let entries = logsdir.directoryEntries;
+    Assert.ok(entries.hasMoreElements());
+    let logfile = entries.getNext().QueryInterface(Ci.nsIFile);
+    Assert.equal(logfile.leafName.slice(-4), ".txt");
+    Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName);
+    Assert.ok(!entries.hasMoreElements());
+
+    // Ensure we logged some addon list (which is probably empty)
+    readFile(logfile, function (error, data) {
+      Assert.ok(Components.isSuccessCode(error));
+      Assert.notEqual(data.indexOf("Addons installed"), -1);
+
+      // Clean up.
+      try {
+        logfile.remove(false);
+      } catch (ex) {
+        dump("Couldn't delete file: " + ex.message + "\n");
+        // Stupid Windows box.
+      }
+
+      for (const pref of Svc.PrefBranch.getChildList("")) {
+        Svc.PrefBranch.clearUserPref(pref);
+      }
+      run_next_test();
+    });
+  });
+
+  // Fake an unsuccessful sync.
+  Svc.Obs.notify("weave:service:sync:error");
+});
+
+// Check that error log files are deleted above an age threshold.
+add_test(async function test_logErrorCleanup_age() {
+  _("Beginning test_logErrorCleanup_age.");
+  let maxAge = CLEANUP_DELAY / 1000;
+  let oldLogs = [];
+  let numLogs = 10;
+  let errString = "some error log\n";
+
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+  Svc.PrefBranch.setIntPref("log.appender.file.maxErrorAge", maxAge);
+
+  _("Making some files.");
+  const logsDir = PathUtils.join(PathUtils.profileDir, "weave", "logs");
+  await IOUtils.makeDirectory(logsDir);
+  for (let i = 0; i < numLogs; i++) {
+    let now = Date.now();
+    let filename = "error-sync-" + now + "" + i + ".txt";
+    let newLog = new FileUtils.File(PathUtils.join(logsDir, filename));
+    let foStream = FileUtils.openFileOutputStream(newLog);
+    foStream.write(errString, errString.length);
+    foStream.close();
+    _("  > Created " + filename);
+    oldLogs.push(newLog.leafName);
+  }
+
+  Svc.Obs.add(
+    "services-tests:common:log-manager:cleanup-logs",
+    function onCleanupLogs() {
+      Svc.Obs.remove(
+        "services-tests:common:log-manager:cleanup-logs",
+        onCleanupLogs
+      );
+
+      // Only the newest created log file remains.
+      let entries = logsdir.directoryEntries;
+      Assert.ok(entries.hasMoreElements());
+      let logfile = entries.getNext().QueryInterface(Ci.nsIFile);
+      Assert.ok(
+        oldLogs.every(function (e) {
+          return e != logfile.leafName;
+        })
+      );
+      Assert.ok(!entries.hasMoreElements());
+
+      // Clean up.
+      try {
+        logfile.remove(false);
+      } catch (ex) {
+        dump("Couldn't delete file: " + ex.message + "\n");
+        // Stupid Windows box.
+      }
+
+      for (const pref of Svc.PrefBranch.getChildList("")) {
+        Svc.PrefBranch.clearUserPref(pref);
+      }
+      run_next_test();
+    }
+  );
+
+  let delay = CLEANUP_DELAY + DELAY_BUFFER;
+
+  _("Cleaning up logs after " + delay + "msec.");
+  CommonUtils.namedTimer(
+    function onTimer() {
+      Svc.Obs.notify("weave:service:sync:error");
+    },
+    delay,
+    this,
+    "cleanup-timer"
+  );
+});
+
+add_task(async function test_remove_log_on_startOver() {
+  Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true);
+
+  let log = Log.repository.getLogger("Sync.Test.FileLog");
+  const MESSAGE = "this WILL show up";
+  log.info(MESSAGE);
+
+  let promiseLogWritten = promiseOneObserver("weave:service:reset-file-log");
+  // Fake an unsuccessful sync.
+  Svc.Obs.notify("weave:service:sync:error");
+
+  await promiseLogWritten;
+  // Should have at least 1 log file.
+  let entries = logsdir.directoryEntries;
+  Assert.ok(entries.hasMoreElements());
+
+  // Fake a reset.
+  let promiseRemoved = promiseOneObserver("weave:service:remove-file-log");
+  Svc.Obs.notify("weave:service:start-over:finish");
+  await promiseRemoved;
+
+  // should be no files left.
+  Assert.ok(!logsdir.directoryEntries.hasMoreElements());
+});

src/zen/tests/mochitests/sync/test_errorhandler_sync_checkServerError.js

@@ -0,0 +1,294 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { Status } = ChromeUtils.importESModule(
+  "resource://services-sync/status.sys.mjs"
+);
+const { FakeCryptoService } = ChromeUtils.importESModule(
+  "resource://testing-common/services/sync/fakeservices.sys.mjs"
+);
+
+var engineManager = Service.engineManager;
+
+function CatapultEngine() {
+  SyncEngine.call(this, "Catapult", Service);
+}
+CatapultEngine.prototype = {
+  exception: null, // tests fill this in
+  async _sync() {
+    throw this.exception;
+  },
+};
+Object.setPrototypeOf(CatapultEngine.prototype, SyncEngine.prototype);
+
+async function sync_httpd_setup() {
+  let collectionsHelper = track_collections_helper();
+  let upd = collectionsHelper.with_updated_collection;
+
+  let catapultEngine = engineManager.get("catapult");
+  let syncID = await catapultEngine.resetLocalSyncID();
+  let engines = { catapult: { version: catapultEngine.version, syncID } };
+
+  // Track these using the collections helper, which keeps modified times
+  // up-to-date.
+  let clientsColl = new ServerCollection({}, true);
+  let keysWBO = new ServerWBO("keys");
+  let globalWBO = new ServerWBO("global", {
+    storageVersion: STORAGE_VERSION,
+    syncID: Utils.makeGUID(),
+    engines,
+  });
+
+  let handlers = {
+    "/1.1/johndoe/info/collections": collectionsHelper.handler,
+    "/1.1/johndoe/storage/meta/global": upd("meta", globalWBO.handler()),
+    "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()),
+    "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()),
+  };
+  return httpd_setup(handlers);
+}
+
+async function setUp(server) {
+  await configureIdentity({ username: "johndoe" }, server);
+  new FakeCryptoService();
+  syncTestLogging();
+}
+
+async function generateAndUploadKeys(server) {
+  await generateNewKeys(Service.collectionKeys);
+  let serverKeys = Service.collectionKeys.asWBO("crypto", "keys");
+  await serverKeys.encrypt(Service.identity.syncKeyBundle);
+  let res = Service.resource(
+    server.baseURI + "/1.1/johndoe/storage/crypto/keys"
+  );
+  return (await serverKeys.upload(res)).success;
+}
+
+add_task(async function setup() {
+  await engineManager.clear();
+  validate_all_future_pings();
+  await engineManager.register(CatapultEngine);
+});
+
+add_task(async function test_backoff500() {
+  enableValidationPrefs();
+
+  _("Test: HTTP 500 sets backoff status.");
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  let engine = engineManager.get("catapult");
+  engine.enabled = true;
+  engine.exception = { status: 500 };
+
+  try {
+    Assert.ok(!Status.enforceBackoff);
+
+    // Forcibly create and upload keys here -- otherwise we don't get to the 500!
+    Assert.ok(await generateAndUploadKeys(server));
+
+    await Service.login();
+    await Service.sync();
+    Assert.ok(Status.enforceBackoff);
+    Assert.equal(Status.sync, SYNC_SUCCEEDED);
+    Assert.equal(Status.service, SYNC_FAILED_PARTIAL);
+  } finally {
+    Status.resetBackoff();
+    await Service.startOver();
+  }
+  await promiseStopServer(server);
+});
+
+add_task(async function test_backoff503() {
+  enableValidationPrefs();
+
+  _(
+    "Test: HTTP 503 with Retry-After header leads to backoff notification and sets backoff status."
+  );
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  const BACKOFF = 42;
+  let engine = engineManager.get("catapult");
+  engine.enabled = true;
+  engine.exception = { status: 503, headers: { "retry-after": BACKOFF } };
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function (subject) {
+    backoffInterval = subject;
+  });
+
+  try {
+    Assert.ok(!Status.enforceBackoff);
+
+    Assert.ok(await generateAndUploadKeys(server));
+
+    await Service.login();
+    await Service.sync();
+
+    Assert.ok(Status.enforceBackoff);
+    Assert.equal(backoffInterval, BACKOFF);
+    Assert.equal(Status.service, SYNC_FAILED_PARTIAL);
+    Assert.equal(Status.sync, SERVER_MAINTENANCE);
+  } finally {
+    Status.resetBackoff();
+    Status.resetSync();
+    await Service.startOver();
+  }
+  await promiseStopServer(server);
+});
+
+add_task(async function test_overQuota() {
+  enableValidationPrefs();
+
+  _("Test: HTTP 400 with body error code 14 means over quota.");
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  let engine = engineManager.get("catapult");
+  engine.enabled = true;
+  engine.exception = {
+    status: 400,
+    toString() {
+      return "14";
+    },
+  };
+
+  try {
+    Assert.equal(Status.sync, SYNC_SUCCEEDED);
+
+    Assert.ok(await generateAndUploadKeys(server));
+
+    await Service.login();
+    await Service.sync();
+
+    Assert.equal(Status.sync, OVER_QUOTA);
+    Assert.equal(Status.service, SYNC_FAILED_PARTIAL);
+  } finally {
+    Status.resetSync();
+    await Service.startOver();
+  }
+  await promiseStopServer(server);
+});
+
+add_task(async function test_service_networkError() {
+  enableValidationPrefs();
+
+  _(
+    "Test: Connection refused error from Service.sync() leads to the right status code."
+  );
+  let server = await sync_httpd_setup();
+  await setUp(server);
+  await promiseStopServer(server);
+  // Provoke connection refused.
+  Service.clusterURL = "http://localhost:12345/";
+
+  try {
+    Assert.equal(Status.sync, SYNC_SUCCEEDED);
+
+    Service._loggedIn = true;
+    await Service.sync();
+
+    Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
+    Assert.equal(Status.service, SYNC_FAILED);
+  } finally {
+    Status.resetSync();
+    await Service.startOver();
+  }
+});
+
+add_task(async function test_service_offline() {
+  enableValidationPrefs();
+
+  _(
+    "Test: Wanting to sync in offline mode leads to the right status code but does not increment the ignorable error count."
+  );
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  await promiseStopServer(server);
+  Services.io.offline = true;
+  Services.prefs.setBoolPref("network.dns.offline-localhost", false);
+
+  try {
+    Assert.equal(Status.sync, SYNC_SUCCEEDED);
+
+    Service._loggedIn = true;
+    await Service.sync();
+
+    Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
+    Assert.equal(Status.service, SYNC_FAILED);
+  } finally {
+    Status.resetSync();
+    await Service.startOver();
+  }
+  Services.io.offline = false;
+  Services.prefs.clearUserPref("network.dns.offline-localhost");
+});
+
+add_task(async function test_engine_networkError() {
+  enableValidationPrefs();
+
+  _(
+    "Test: Network related exceptions from engine.sync() lead to the right status code."
+  );
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  let engine = engineManager.get("catapult");
+  engine.enabled = true;
+  engine.exception = Components.Exception(
+    "NS_ERROR_UNKNOWN_HOST",
+    Cr.NS_ERROR_UNKNOWN_HOST
+  );
+
+  try {
+    Assert.equal(Status.sync, SYNC_SUCCEEDED);
+
+    Assert.ok(await generateAndUploadKeys(server));
+
+    await Service.login();
+    await Service.sync();
+
+    Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
+    Assert.equal(Status.service, SYNC_FAILED_PARTIAL);
+  } finally {
+    Status.resetSync();
+    await Service.startOver();
+  }
+  await promiseStopServer(server);
+});
+
+add_task(async function test_resource_timeout() {
+  enableValidationPrefs();
+
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  let engine = engineManager.get("catapult");
+  engine.enabled = true;
+  // Resource throws this when it encounters a timeout.
+  engine.exception = Components.Exception(
+    "Aborting due to channel inactivity.",
+    Cr.NS_ERROR_NET_TIMEOUT
+  );
+
+  try {
+    Assert.equal(Status.sync, SYNC_SUCCEEDED);
+
+    Assert.ok(await generateAndUploadKeys(server));
+
+    await Service.login();
+    await Service.sync();
+
+    Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
+    Assert.equal(Status.service, SYNC_FAILED_PARTIAL);
+  } finally {
+    Status.resetSync();
+    await Service.startOver();
+  }
+  await promiseStopServer(server);
+});

src/zen/tests/mochitests/sync/test_extension_storage_engine.js

@@ -0,0 +1,259 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+ChromeUtils.defineESModuleGetters(this, {
+  Service: "resource://services-sync/service.sys.mjs",
+  extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.sys.mjs",
+});
+
+const { ExtensionStorageEngineBridge, ExtensionStorageEngineKinto } =
+  ChromeUtils.importESModule(
+    "resource://services-sync/engines/extension-storage.sys.mjs"
+  );
+
+Services.prefs.setStringPref("webextensions.storage.sync.log.level", "debug");
+
+add_task(async function test_switching_between_kinto_and_bridged() {
+  function assertUsingKinto(message) {
+    let kintoEngine = Service.engineManager.get("extension-storage");
+    Assert.ok(kintoEngine instanceof ExtensionStorageEngineKinto, message);
+  }
+  function assertUsingBridged(message) {
+    let bridgedEngine = Service.engineManager.get("extension-storage");
+    Assert.ok(bridgedEngine instanceof ExtensionStorageEngineBridge, message);
+  }
+
+  let isUsingKinto = Services.prefs.getBoolPref(
+    "webextensions.storage.sync.kinto",
+    false
+  );
+  if (isUsingKinto) {
+    assertUsingKinto("Should use Kinto engine before flipping pref");
+  } else {
+    assertUsingBridged("Should use bridged engine before flipping pref");
+  }
+
+  _("Flip pref");
+  Services.prefs.setBoolPref("webextensions.storage.sync.kinto", !isUsingKinto);
+  await Service.engineManager.switchAlternatives();
+
+  if (isUsingKinto) {
+    assertUsingBridged("Should use bridged engine after flipping pref");
+  } else {
+    assertUsingKinto("Should use Kinto engine after flipping pref");
+  }
+
+  _("Clean up");
+  Services.prefs.clearUserPref("webextensions.storage.sync.kinto");
+  await Service.engineManager.switchAlternatives();
+});
+
+add_task(async function test_enable() {
+  const PREF = "services.sync.engine.extension-storage.force";
+
+  let addonsEngine = Service.engineManager.get("addons");
+  let extensionStorageEngine = Service.engineManager.get("extension-storage");
+
+  try {
+    Assert.ok(
+      addonsEngine.enabled,
+      "Add-ons engine should be enabled by default"
+    );
+    Assert.ok(
+      extensionStorageEngine.enabled,
+      "Extension storage engine should be enabled by default"
+    );
+
+    addonsEngine.enabled = false;
+    Assert.ok(
+      !extensionStorageEngine.enabled,
+      "Disabling add-ons should disable extension storage"
+    );
+
+    extensionStorageEngine.enabled = true;
+    Assert.ok(
+      !extensionStorageEngine.enabled,
+      "Enabling extension storage without override pref shouldn't work"
+    );
+
+    Services.prefs.setBoolPref(PREF, true);
+    Assert.ok(
+      extensionStorageEngine.enabled,
+      "Setting override pref should enable extension storage"
+    );
+
+    extensionStorageEngine.enabled = false;
+    Assert.ok(
+      !extensionStorageEngine.enabled,
+      "Disabling extension storage engine with override pref should work"
+    );
+
+    extensionStorageEngine.enabled = true;
+    Assert.ok(
+      extensionStorageEngine.enabled,
+      "Enabling extension storage with override pref should work"
+    );
+  } finally {
+    addonsEngine.enabled = true;
+    Services.prefs.clearUserPref(PREF);
+  }
+});
+
+add_task(async function test_notifyPendingChanges() {
+  let engine = new ExtensionStorageEngineBridge(Service);
+  await engine.initialize();
+
+  let extension = { id: "ext-1" };
+  let expectedChange = {
+    a: "b",
+    c: "d",
+  };
+
+  let lastSync = 0;
+  let syncID = Utils.makeGUID();
+  let error = null;
+  engine._rustStore = {
+    getSyncedChanges() {
+      if (error) {
+        throw new Error(error.message);
+      } else {
+        return [
+          { extId: extension.id, changes: JSON.stringify(expectedChange) },
+        ];
+      }
+    },
+  };
+
+  engine._bridge = {
+    ensureCurrentSyncId(id) {
+      if (syncID != id) {
+        syncID = id;
+        lastSync = 0;
+      }
+      return id;
+    },
+    resetSyncId() {
+      return syncID;
+    },
+    syncStarted() {},
+    lastSync() {
+      return lastSync;
+    },
+    setLastSync(lastSyncMillis) {
+      lastSync = lastSyncMillis;
+    },
+    apply() {
+      return [];
+    },
+    setUploaded(_modified, _ids) {},
+    syncFinished() {},
+  };
+
+  let server = await serverForFoo(engine);
+
+  let actualChanges = [];
+  let listener = changes => actualChanges.push(changes);
+  extensionStorageSync.addOnChangedListener(extension, listener);
+
+  try {
+    await SyncTestingInfrastructure(server);
+
+    info("Sync engine; notify about changes");
+    await sync_engine_and_validate_telem(engine, false);
+    deepEqual(
+      actualChanges,
+      [expectedChange],
+      "Should notify about changes during sync"
+    );
+
+    error = new Error("oops!");
+    actualChanges = [];
+    await sync_engine_and_validate_telem(engine, false);
+    deepEqual(
+      actualChanges,
+      [],
+      "Should finish syncing even if notifying about changes fails"
+    );
+  } finally {
+    extensionStorageSync.removeOnChangedListener(extension, listener);
+    await promiseStopServer(server);
+    await engine.finalize();
+  }
+});
+
+// It's difficult to know what to test - there's already tests for the bridged
+// engine etc - so we just try and check that this engine conforms to the
+// mozIBridgedSyncEngine interface guarantees.
+add_task(async function test_engine() {
+  // Forcibly set the bridged engine in the engine manager. the reason we do
+  // this, unlike the other tests where we just create the engine, is so that
+  // telemetry can get at the engine's `overrideTelemetryName`, which it gets
+  // through the engine manager.
+  await Service.engineManager.unregister("extension-storage");
+  await Service.engineManager.register(ExtensionStorageEngineBridge);
+  let engine = Service.engineManager.get("extension-storage");
+  Assert.equal(engine.version, 1);
+
+  Assert.deepEqual(await engine.getSyncID(), null);
+  await engine.resetLocalSyncID();
+  Assert.notEqual(await engine.getSyncID(), null);
+
+  Assert.equal(await engine.getLastSync(), 0);
+  // lastSync is seconds on this side of the world, but milli-seconds on the other.
+  await engine.setLastSync(1234.567);
+  // should have 2 digit precision.
+  Assert.equal(await engine.getLastSync(), 1234.57);
+  await engine.setLastSync(0);
+
+  // Set some data.
+  await extensionStorageSync.set({ id: "ext-2" }, { ext_2_key: "ext_2_value" });
+  // Now do a sync with out regular test server.
+  let server = await serverForFoo(engine);
+  try {
+    await SyncTestingInfrastructure(server);
+
+    info("Add server records");
+    let foo = server.user("foo");
+    let collection = foo.collection("extension-storage");
+    let now = new_timestamp();
+
+    collection.insert(
+      "fakeguid0000",
+      encryptPayload({
+        id: "fakeguid0000",
+        extId: "ext-1",
+        data: JSON.stringify({ foo: "bar" }),
+      }),
+      now
+    );
+
+    info("Sync the engine");
+
+    let ping = await sync_engine_and_validate_telem(engine, false);
+    Assert.ok(ping.engines.find(e => e.name == "rust-webext-storage"));
+    Assert.equal(
+      ping.engines.find(e => e.name == "extension-storage"),
+      null
+    );
+
+    // We should have applied the data from the existing collection record.
+    Assert.deepEqual(await extensionStorageSync.get({ id: "ext-1" }, null), {
+      foo: "bar",
+    });
+
+    // should now be 2 records on the server.
+    let payloads = collection.payloads();
+    Assert.equal(payloads.length, 2);
+    // find the new one we wrote.
+    let newPayload =
+      payloads[0].id == "fakeguid0000" ? payloads[1] : payloads[0];
+    Assert.equal(newPayload.data, `{"ext_2_key":"ext_2_value"}`);
+    // should have updated the timestamp.
+    greater(await engine.getLastSync(), 0, "Should update last sync time");
+  } finally {
+    await promiseStopServer(server);
+    await engine.finalize();
+  }
+});

src/zen/tests/mochitests/sync/test_extension_storage_engine_kinto.js

@@ -0,0 +1,136 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Services.prefs.setBoolPref("webextensions.storage.sync.kinto", true);
+
+const { ExtensionStorageEngineKinto: ExtensionStorageEngine } =
+  ChromeUtils.importESModule(
+    "resource://services-sync/engines/extension-storage.sys.mjs"
+  );
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { extensionStorageSyncKinto: extensionStorageSync } =
+  ChromeUtils.importESModule(
+    "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs"
+  );
+
+let engine;
+
+function mock(options) {
+  let calls = [];
+  let ret = function () {
+    calls.push(arguments);
+    return options.returns;
+  };
+  let proto = {
+    get calls() {
+      return calls;
+    },
+  };
+  Object.setPrototypeOf(proto, Function.prototype);
+  Object.setPrototypeOf(ret, proto);
+  return ret;
+}
+
+function setSkipChance(v) {
+  Services.prefs.setIntPref(
+    "services.sync.extension-storage.skipPercentageChance",
+    v
+  );
+}
+
+add_task(async function setup() {
+  await Service.engineManager.register(ExtensionStorageEngine);
+  engine = Service.engineManager.get("extension-storage");
+  do_get_profile(); // so we can use FxAccounts
+  loadWebExtensionTestFunctions();
+  setSkipChance(0);
+});
+
+add_task(async function test_calling_sync_calls__sync() {
+  let oldSync = ExtensionStorageEngine.prototype._sync;
+  let syncMock = (ExtensionStorageEngine.prototype._sync = mock({
+    returns: true,
+  }));
+  try {
+    // I wanted to call the main sync entry point for the entire
+    // package, but that fails because it tries to sync ClientEngine
+    // first, which fails.
+    await engine.sync();
+  } finally {
+    ExtensionStorageEngine.prototype._sync = oldSync;
+  }
+  equal(syncMock.calls.length, 1);
+});
+
+add_task(async function test_sync_skip() {
+  try {
+    // Do a few times to ensure we aren't getting "lucky" WRT Math.random()
+    for (let i = 0; i < 10; ++i) {
+      setSkipChance(100);
+      engine._tracker._score = 0;
+      ok(
+        !engine.shouldSkipSync("user"),
+        "Should allow explicitly requested syncs"
+      );
+      ok(!engine.shouldSkipSync("startup"), "Should allow startup syncs");
+      ok(
+        engine.shouldSkipSync("schedule"),
+        "Should skip scheduled syncs if skipProbability is 100"
+      );
+      engine._tracker._score = MULTI_DEVICE_THRESHOLD;
+      ok(
+        !engine.shouldSkipSync("schedule"),
+        "should allow scheduled syncs if tracker score is high"
+      );
+      engine._tracker._score = 0;
+      setSkipChance(0);
+      ok(
+        !engine.shouldSkipSync("schedule"),
+        "Should allow scheduled syncs if probability is 0"
+      );
+    }
+  } finally {
+    engine._tracker._score = 0;
+    setSkipChance(0);
+  }
+});
+
+add_task(async function test_calling_wipeClient_calls_clearAll() {
+  let oldClearAll = extensionStorageSync.clearAll;
+  let clearMock = (extensionStorageSync.clearAll = mock({
+    returns: Promise.resolve(),
+  }));
+  try {
+    await engine.wipeClient();
+  } finally {
+    extensionStorageSync.clearAll = oldClearAll;
+  }
+  equal(clearMock.calls.length, 1);
+});
+
+add_task(async function test_calling_sync_calls_ext_storage_sync() {
+  const extension = { id: "my-extension" };
+  let oldSync = extensionStorageSync.syncAll;
+  let syncMock = (extensionStorageSync.syncAll = mock({
+    returns: Promise.resolve(),
+  }));
+  try {
+    await withContext(async function (context) {
+      // Set something so that everyone knows that we're using storage.sync
+      await extensionStorageSync.set(extension, { a: "b" }, context);
+      let ping = await sync_engine_and_validate_telem(engine, false);
+      Assert.ok(ping.engines.find(e => e.name == "extension-storage"));
+      Assert.equal(
+        ping.engines.find(e => e.name == "rust-webext-storage"),
+        null
+      );
+    });
+  } finally {
+    extensionStorageSync.syncAll = oldSync;
+  }
+  Assert.greaterOrEqual(syncMock.calls.length, 1);
+});

src/zen/tests/mochitests/sync/test_extension_storage_tracker_kinto.js

@@ -0,0 +1,44 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Services.prefs.setBoolPref("webextensions.storage.sync.kinto", true);
+
+const { ExtensionStorageEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/extension-storage.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { extensionStorageSyncKinto: extensionStorageSync } =
+  ChromeUtils.importESModule(
+    "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs"
+  );
+
+let engine;
+
+add_task(async function setup() {
+  await Service.engineManager.register(ExtensionStorageEngine);
+  engine = Service.engineManager.get("extension-storage");
+  do_get_profile(); // so we can use FxAccounts
+  loadWebExtensionTestFunctions();
+});
+
+add_task(async function test_changing_extension_storage_changes_score() {
+  const tracker = engine._tracker;
+  const extension = { id: "my-extension-id" };
+  tracker.start();
+  await withContext(async function (context) {
+    await extensionStorageSync.set(extension, { a: "b" }, context);
+  });
+  Assert.equal(tracker.score, SCORE_INCREMENT_MEDIUM);
+
+  tracker.resetScore();
+  await withContext(async function (context) {
+    await extensionStorageSync.remove(extension, "a", context);
+  });
+  Assert.equal(tracker.score, SCORE_INCREMENT_MEDIUM);
+
+  await tracker.stop();
+});

src/zen/tests/mochitests/sync/test_form_validator.js

@@ -0,0 +1,86 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { FormValidator } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/forms.sys.mjs"
+);
+
+function getDummyServerAndClient() {
+  return {
+    server: [
+      {
+        id: "11111",
+        guid: "11111",
+        name: "foo",
+        fieldname: "foo",
+        value: "bar",
+      },
+      {
+        id: "22222",
+        guid: "22222",
+        name: "foo2",
+        fieldname: "foo2",
+        value: "bar2",
+      },
+      {
+        id: "33333",
+        guid: "33333",
+        name: "foo3",
+        fieldname: "foo3",
+        value: "bar3",
+      },
+    ],
+    client: [
+      {
+        id: "11111",
+        guid: "11111",
+        name: "foo",
+        fieldname: "foo",
+        value: "bar",
+      },
+      {
+        id: "22222",
+        guid: "22222",
+        name: "foo2",
+        fieldname: "foo2",
+        value: "bar2",
+      },
+      {
+        id: "33333",
+        guid: "33333",
+        name: "foo3",
+        fieldname: "foo3",
+        value: "bar3",
+      },
+    ],
+  };
+}
+
+add_task(async function test_valid() {
+  let { server, client } = getDummyServerAndClient();
+  let validator = new FormValidator();
+  let { problemData, clientRecords, records, deletedRecords } =
+    await validator.compareClientWithServer(client, server);
+  equal(clientRecords.length, 3);
+  equal(records.length, 3);
+  equal(deletedRecords.length, 0);
+  deepEqual(problemData, validator.emptyProblemData());
+});
+
+add_task(async function test_formValidatorIgnoresMissingClients() {
+  // Since history form records are not deleted from the server, the
+  // |FormValidator| shouldn't set the |missingClient| flag in |problemData|.
+  let { server, client } = getDummyServerAndClient();
+  client.pop();
+
+  let validator = new FormValidator();
+  let { problemData, clientRecords, records, deletedRecords } =
+    await validator.compareClientWithServer(client, server);
+
+  equal(clientRecords.length, 2);
+  equal(records.length, 3);
+  equal(deletedRecords.length, 0);
+
+  let expected = validator.emptyProblemData();
+  deepEqual(problemData, expected);
+});

src/zen/tests/mochitests/sync/test_forms_store.js

@@ -0,0 +1,176 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+_(
+  "Make sure the form store follows the Store api and correctly accesses the backend form storage"
+);
+const { FormEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/forms.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { SyncedRecordsTelemetry } = ChromeUtils.importESModule(
+  "resource://services-sync/telemetry.sys.mjs"
+);
+
+add_task(async function run_test() {
+  let engine = new FormEngine(Service);
+  await engine.initialize();
+  let store = engine._store;
+
+  async function applyEnsureNoFailures(records) {
+    let countTelemetry = new SyncedRecordsTelemetry();
+    Assert.equal(
+      (await store.applyIncomingBatch(records, countTelemetry)).length,
+      0
+    );
+  }
+
+  _("Remove any existing entries");
+  await store.wipe();
+  if ((await store.getAllIDs()).length) {
+    do_throw("Shouldn't get any ids!");
+  }
+
+  _("Add a form entry");
+  await applyEnsureNoFailures([
+    {
+      id: Utils.makeGUID(),
+      name: "name!!",
+      value: "value??",
+    },
+  ]);
+
+  _("Should have 1 entry now");
+  let id = "";
+  for (let _id in await store.getAllIDs()) {
+    if (id == "") {
+      id = _id;
+    } else {
+      do_throw("Should have only gotten one!");
+    }
+  }
+  Assert.ok(store.itemExists(id));
+
+  _("Should be able to find this entry as a dupe");
+  Assert.equal(
+    await engine._findDupe({ name: "name!!", value: "value??" }),
+    id
+  );
+
+  let rec = await store.createRecord(id);
+  _("Got record for id", id, rec);
+  Assert.equal(rec.name, "name!!");
+  Assert.equal(rec.value, "value??");
+
+  _("Create a non-existent id for delete");
+  Assert.ok((await store.createRecord("deleted!!")).deleted);
+
+  _("Try updating.. doesn't do anything yet");
+  await store.update({});
+
+  _("Remove all entries");
+  await store.wipe();
+  if ((await store.getAllIDs()).length) {
+    do_throw("Shouldn't get any ids!");
+  }
+
+  _("Add another entry");
+  await applyEnsureNoFailures([
+    {
+      id: Utils.makeGUID(),
+      name: "another",
+      value: "entry",
+    },
+  ]);
+  id = "";
+  for (let _id in await store.getAllIDs()) {
+    if (id == "") {
+      id = _id;
+    } else {
+      do_throw("Should have only gotten one!");
+    }
+  }
+
+  _("Change the id of the new entry to something else");
+  await store.changeItemID(id, "newid");
+
+  _("Make sure it's there");
+  Assert.ok(store.itemExists("newid"));
+
+  _("Remove the entry");
+  await store.remove({
+    id: "newid",
+  });
+  if ((await store.getAllIDs()).length) {
+    do_throw("Shouldn't get any ids!");
+  }
+
+  _("Removing the entry again shouldn't matter");
+  await store.remove({
+    id: "newid",
+  });
+  if ((await store.getAllIDs()).length) {
+    do_throw("Shouldn't get any ids!");
+  }
+
+  _("Add another entry to delete using applyIncomingBatch");
+  let toDelete = {
+    id: Utils.makeGUID(),
+    name: "todelete",
+    value: "entry",
+  };
+  await applyEnsureNoFailures([toDelete]);
+  id = "";
+  for (let _id in await store.getAllIDs()) {
+    if (id == "") {
+      id = _id;
+    } else {
+      do_throw("Should have only gotten one!");
+    }
+  }
+  Assert.ok(store.itemExists(id));
+  // mark entry as deleted
+  toDelete.id = id;
+  toDelete.deleted = true;
+  await applyEnsureNoFailures([toDelete]);
+  if ((await store.getAllIDs()).length) {
+    do_throw("Shouldn't get any ids!");
+  }
+
+  _("Add an entry to wipe");
+  await applyEnsureNoFailures([
+    {
+      id: Utils.makeGUID(),
+      name: "towipe",
+      value: "entry",
+    },
+  ]);
+
+  await store.wipe();
+
+  if ((await store.getAllIDs()).length) {
+    do_throw("Shouldn't get any ids!");
+  }
+
+  _("Ensure we work if formfill is disabled.");
+  Services.prefs.setBoolPref("browser.formfill.enable", false);
+  try {
+    // a search
+    if ((await store.getAllIDs()).length) {
+      do_throw("Shouldn't get any ids!");
+    }
+    // an update.
+    await applyEnsureNoFailures([
+      {
+        id: Utils.makeGUID(),
+        name: "some",
+        value: "entry",
+      },
+    ]);
+  } finally {
+    Services.prefs.clearUserPref("browser.formfill.enable");
+    await store.wipe();
+  }
+});

src/zen/tests/mochitests/sync/test_forms_tracker.js

@@ -0,0 +1,78 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { FormEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/forms.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+add_task(async function run_test() {
+  _("Verify we've got an empty tracker to work with.");
+  let engine = new FormEngine(Service);
+  await engine.initialize();
+  let tracker = engine._tracker;
+
+  let changes = await tracker.getChangedIDs();
+  do_check_empty(changes);
+  Log.repository.rootLogger.addAppender(new Log.DumpAppender());
+
+  async function addEntry(name, value) {
+    await engine._store.create({ name, value });
+    await engine._tracker.asyncObserver.promiseObserversComplete();
+  }
+  async function removeEntry(name, value) {
+    let guid = await engine._findDupe({ name, value });
+    await engine._store.remove({ id: guid });
+    await engine._tracker.asyncObserver.promiseObserversComplete();
+  }
+
+  try {
+    _("Create an entry. Won't show because we haven't started tracking yet");
+    await addEntry("name", "John Doe");
+    changes = await tracker.getChangedIDs();
+    do_check_empty(changes);
+
+    _("Tell the tracker to start tracking changes.");
+    tracker.start();
+    await removeEntry("name", "John Doe");
+    await addEntry("email", "john@doe.com");
+    changes = await tracker.getChangedIDs();
+    do_check_attribute_count(changes, 2);
+
+    _("Notifying twice won't do any harm.");
+    tracker.start();
+    await addEntry("address", "Memory Lane");
+    changes = await tracker.getChangedIDs();
+    do_check_attribute_count(changes, 3);
+
+    _("Check that ignoreAll is respected");
+    await tracker.clearChangedIDs();
+    tracker.score = 0;
+    tracker.ignoreAll = true;
+    await addEntry("username", "johndoe123");
+    await addEntry("favoritecolor", "green");
+    await removeEntry("name", "John Doe");
+    tracker.ignoreAll = false;
+    changes = await tracker.getChangedIDs();
+    do_check_empty(changes);
+    equal(tracker.score, 0);
+
+    _("Let's stop tracking again.");
+    await tracker.clearChangedIDs();
+    await tracker.stop();
+    await removeEntry("address", "Memory Lane");
+    changes = await tracker.getChangedIDs();
+    do_check_empty(changes);
+
+    _("Notifying twice won't do any harm.");
+    await tracker.stop();
+    await removeEntry("email", "john@doe.com");
+    changes = await tracker.getChangedIDs();
+    do_check_empty(changes);
+  } finally {
+    _("Clean up.");
+    await engine._store.wipe();
+  }
+});

src/zen/tests/mochitests/sync/test_fxa_node_reassignment.js

@@ -0,0 +1,399 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+_("Test that node reassignment happens correctly using the FxA identity mgr.");
+// The node-reassignment logic is quite different for FxA than for the legacy
+// provider.  In particular, there's no special request necessary for
+// reassignment - it comes from the token server - so we need to ensure the
+// Fxa cluster manager grabs a new token.
+
+const { RESTRequest } = ChromeUtils.importESModule(
+  "resource://services-common/rest.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { Status } = ChromeUtils.importESModule(
+  "resource://services-sync/status.sys.mjs"
+);
+const { SyncAuthManager } = ChromeUtils.importESModule(
+  "resource://services-sync/sync_auth.sys.mjs"
+);
+
+add_task(async function setup() {
+  // Disables all built-in engines. Important for avoiding errors thrown by the
+  // add-ons engine.
+  await Service.engineManager.clear();
+
+  // Setup the sync auth manager.
+  Status.__authManager = Service.identity = new SyncAuthManager();
+});
+
+// API-compatible with SyncServer handler. Bind `handler` to something to use
+// as a ServerCollection handler.
+function handleReassign(handler, req, resp) {
+  resp.setStatusLine(req.httpVersion, 401, "Node reassignment");
+  resp.setHeader("Content-Type", "application/json");
+  let reassignBody = JSON.stringify({ error: "401inator in place" });
+  resp.bodyOutputStream.write(reassignBody, reassignBody.length);
+}
+
+var numTokenRequests = 0;
+
+function prepareServer(cbAfterTokenFetch) {
+  syncTestLogging();
+  let config = makeIdentityConfig({ username: "johndoe" });
+  // A server callback to ensure we don't accidentally hit the wrong endpoint
+  // after a node reassignment.
+  let callback = {
+    onRequest(req) {
+      let full = `${req.scheme}://${req.host}:${req.port}${req.path}`;
+      let expected = config.fxaccount.token.endpoint;
+      Assert.ok(
+        full.startsWith(expected),
+        `request made to ${full}, expected ${expected}`
+      );
+    },
+  };
+  Object.setPrototypeOf(callback, SyncServerCallback);
+  let server = new SyncServer(callback);
+  server.registerUser("johndoe");
+  server.start();
+
+  // Set the token endpoint for the initial token request that's done implicitly
+  // via configureIdentity.
+  config.fxaccount.token.endpoint = server.baseURI + "1.1/johndoe/";
+  // And future token fetches will do magic around numReassigns.
+  let numReassigns = 0;
+  return configureIdentity(config).then(() => {
+    Service.identity._tokenServerClient = {
+      getTokenUsingOAuth() {
+        return new Promise(res => {
+          // Build a new URL with trailing zeros for the SYNC_VERSION part - this
+          // will still be seen as equivalent by the test server, but different
+          // by sync itself.
+          numReassigns += 1;
+          let trailingZeros = new Array(numReassigns + 1).join("0");
+          let token = config.fxaccount.token;
+          token.endpoint = server.baseURI + "1.1" + trailingZeros + "/johndoe";
+          token.uid = config.username;
+          _(`test server saw token fetch - endpoint now ${token.endpoint}`);
+          numTokenRequests += 1;
+          res(token);
+          if (cbAfterTokenFetch) {
+            cbAfterTokenFetch();
+          }
+        });
+      },
+    };
+    return server;
+  });
+}
+
+function getReassigned() {
+  try {
+    return Services.prefs.getBoolPref("services.sync.lastSyncReassigned");
+  } catch (ex) {
+    if (ex.result != Cr.NS_ERROR_UNEXPECTED) {
+      do_throw(
+        "Got exception retrieving lastSyncReassigned: " + Log.exceptionStr(ex)
+      );
+    }
+  }
+  return false;
+}
+
+/**
+ * Make a test request to `url`, then watch the result of two syncs
+ * to ensure that a node request was made.
+ * Runs `between` between the two. This can be used to undo deliberate failure
+ * setup, detach observers, etc.
+ */
+async function syncAndExpectNodeReassignment(
+  server,
+  firstNotification,
+  between,
+  secondNotification,
+  url
+) {
+  _("Starting syncAndExpectNodeReassignment\n");
+  let deferred = Promise.withResolvers();
+  async function onwards() {
+    let numTokenRequestsBefore;
+    function onFirstSync() {
+      _("First sync completed.");
+      Svc.Obs.remove(firstNotification, onFirstSync);
+      Svc.Obs.add(secondNotification, onSecondSync);
+
+      Assert.equal(Service.clusterURL, "");
+
+      // Track whether we fetched a new token.
+      numTokenRequestsBefore = numTokenRequests;
+
+      // Allow for tests to clean up error conditions.
+      between();
+    }
+    function onSecondSync() {
+      _("Second sync completed.");
+      Svc.Obs.remove(secondNotification, onSecondSync);
+      Service.scheduler.clearSyncTriggers();
+
+      // Make absolutely sure that any event listeners are done with their work
+      // before we proceed.
+      waitForZeroTimer(function () {
+        _("Second sync nextTick.");
+        Assert.equal(
+          numTokenRequests,
+          numTokenRequestsBefore + 1,
+          "fetched a new token"
+        );
+        Service.startOver().then(() => {
+          server.stop(deferred.resolve);
+        });
+      });
+    }
+
+    Svc.Obs.add(firstNotification, onFirstSync);
+    await Service.sync();
+  }
+
+  // Make sure that we really do get a 401 (but we can only do that if we are
+  // already logged in, as the login process is what sets up the URLs)
+  if (Service.isLoggedIn) {
+    _("Making request to " + url + " which should 401");
+    let request = new RESTRequest(url);
+    await request.get();
+    Assert.equal(request.response.status, 401);
+    CommonUtils.nextTick(onwards);
+  } else {
+    _("Skipping preliminary validation check for a 401 as we aren't logged in");
+    CommonUtils.nextTick(onwards);
+  }
+  await deferred.promise;
+}
+
+// Check that when we sync we don't request a new token by default - our
+// test setup has configured the client with a valid token, and that token
+// should be used to form the cluster URL.
+add_task(async function test_single_token_fetch() {
+  enableValidationPrefs();
+
+  _("Test a normal sync only fetches 1 token");
+
+  let numTokenFetches = 0;
+
+  function afterTokenFetch() {
+    numTokenFetches++;
+  }
+
+  // Set the cluster URL to an "old" version - this is to ensure we don't
+  // use that old cached version for the first sync but prefer the value
+  // we got from the token (and as above, we are also checking we don't grab
+  // a new token). If the test actually attempts to connect to this URL
+  // it will crash.
+  Service.clusterURL = "http://example.com/";
+
+  let server = await prepareServer(afterTokenFetch);
+
+  Assert.ok(!Service.isLoggedIn, "not already logged in");
+  await Service.sync();
+  Assert.equal(Status.sync, SYNC_SUCCEEDED, "sync succeeded");
+  Assert.equal(numTokenFetches, 0, "didn't fetch a new token");
+  // A bit hacky, but given we know how prepareServer works we can deduce
+  // that clusterURL we expect.
+  let expectedClusterURL = server.baseURI + "1.1/johndoe/";
+  Assert.equal(Service.clusterURL, expectedClusterURL);
+  await Service.startOver();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_momentary_401_engine() {
+  enableValidationPrefs();
+
+  _("Test a failure for engine URLs that's resolved by reassignment.");
+  let server = await prepareServer();
+  let john = server.user("johndoe");
+
+  _("Enabling the Rotary engine.");
+  let { engine, syncID, tracker } = await registerRotaryEngine();
+
+  // We need the server to be correctly set up prior to experimenting. Do this
+  // through a sync.
+  let global = {
+    syncID: Service.syncID,
+    storageVersion: STORAGE_VERSION,
+    rotary: { version: engine.version, syncID },
+  };
+  john.createCollection("meta").insert("global", global);
+
+  _("First sync to prepare server contents.");
+  await Service.sync();
+
+  _("Setting up Rotary collection to 401.");
+  let rotary = john.createCollection("rotary");
+  let oldHandler = rotary.collectionHandler;
+  rotary.collectionHandler = handleReassign.bind(this, undefined);
+
+  // We want to verify that the clusterURL pref has been cleared after a 401
+  // inside a sync. Flag the Rotary engine to need syncing.
+  john.collection("rotary").timestamp += 1000;
+
+  function between() {
+    _("Undoing test changes.");
+    rotary.collectionHandler = oldHandler;
+
+    function onLoginStart() {
+      // lastSyncReassigned shouldn't be cleared until a sync has succeeded.
+      _("Ensuring that lastSyncReassigned is still set at next sync start.");
+      Svc.Obs.remove("weave:service:login:start", onLoginStart);
+      Assert.ok(getReassigned());
+    }
+
+    _("Adding observer that lastSyncReassigned is still set on login.");
+    Svc.Obs.add("weave:service:login:start", onLoginStart);
+  }
+
+  await syncAndExpectNodeReassignment(
+    server,
+    "weave:service:sync:finish",
+    between,
+    "weave:service:sync:finish",
+    Service.storageURL + "rotary"
+  );
+
+  await tracker.clearChangedIDs();
+  await Service.engineManager.unregister(engine);
+});
+
+// This test ends up being a failing info fetch *after we're already logged in*.
+add_task(async function test_momentary_401_info_collections_loggedin() {
+  enableValidationPrefs();
+
+  _(
+    "Test a failure for info/collections after login that's resolved by reassignment."
+  );
+  let server = await prepareServer();
+
+  _("First sync to prepare server contents.");
+  await Service.sync();
+
+  _("Arrange for info/collections to return a 401.");
+  let oldHandler = server.toplevelHandlers.info;
+  server.toplevelHandlers.info = handleReassign;
+
+  function undo() {
+    _("Undoing test changes.");
+    server.toplevelHandlers.info = oldHandler;
+  }
+
+  Assert.ok(Service.isLoggedIn, "already logged in");
+
+  await syncAndExpectNodeReassignment(
+    server,
+    "weave:service:sync:error",
+    undo,
+    "weave:service:sync:finish",
+    Service.infoURL
+  );
+});
+
+// This test ends up being a failing info fetch *before we're logged in*.
+// In this case we expect to recover during the login phase - so the first
+// sync succeeds.
+add_task(async function test_momentary_401_info_collections_loggedout() {
+  enableValidationPrefs();
+
+  _(
+    "Test a failure for info/collections before login that's resolved by reassignment."
+  );
+
+  let oldHandler;
+  let sawTokenFetch = false;
+
+  function afterTokenFetch() {
+    // After a single token fetch, we undo our evil handleReassign hack, so
+    // the next /info request returns the collection instead of a 401
+    server.toplevelHandlers.info = oldHandler;
+    sawTokenFetch = true;
+  }
+
+  let server = await prepareServer(afterTokenFetch);
+
+  // Return a 401 for the next /info request - it will be reset immediately
+  // after a new token is fetched.
+  oldHandler = server.toplevelHandlers.info;
+  server.toplevelHandlers.info = handleReassign;
+
+  Assert.ok(!Service.isLoggedIn, "not already logged in");
+
+  await Service.sync();
+  Assert.equal(Status.sync, SYNC_SUCCEEDED, "sync succeeded");
+  // sync was successful - check we grabbed a new token.
+  Assert.ok(sawTokenFetch, "a new token was fetched by this test.");
+  // and we are done.
+  await Service.startOver();
+  await promiseStopServer(server);
+});
+
+// This test ends up being a failing meta/global fetch *after we're already logged in*.
+add_task(async function test_momentary_401_storage_loggedin() {
+  enableValidationPrefs();
+
+  _(
+    "Test a failure for any storage URL after login that's resolved by" +
+      "reassignment."
+  );
+  let server = await prepareServer();
+
+  _("First sync to prepare server contents.");
+  await Service.sync();
+
+  _("Arrange for meta/global to return a 401.");
+  let oldHandler = server.toplevelHandlers.storage;
+  server.toplevelHandlers.storage = handleReassign;
+
+  function undo() {
+    _("Undoing test changes.");
+    server.toplevelHandlers.storage = oldHandler;
+  }
+
+  Assert.ok(Service.isLoggedIn, "already logged in");
+
+  await syncAndExpectNodeReassignment(
+    server,
+    "weave:service:sync:error",
+    undo,
+    "weave:service:sync:finish",
+    Service.storageURL + "meta/global"
+  );
+});
+
+// This test ends up being a failing meta/global fetch *before we've logged in*.
+add_task(async function test_momentary_401_storage_loggedout() {
+  enableValidationPrefs();
+
+  _(
+    "Test a failure for any storage URL before login, not just engine parts. " +
+      "Resolved by reassignment."
+  );
+  let server = await prepareServer();
+
+  // Return a 401 for all storage requests.
+  let oldHandler = server.toplevelHandlers.storage;
+  server.toplevelHandlers.storage = handleReassign;
+
+  function undo() {
+    _("Undoing test changes.");
+    server.toplevelHandlers.storage = oldHandler;
+  }
+
+  Assert.ok(!Service.isLoggedIn, "already logged in");
+
+  await syncAndExpectNodeReassignment(
+    server,
+    "weave:service:login:error",
+    undo,
+    "weave:service:sync:finish",
+    Service.storageURL + "meta/global"
+  );
+});

src/zen/tests/mochitests/sync/test_fxa_service_cluster.js

@@ -0,0 +1,58 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { initializeIdentityWithTokenServerResponse } =
+  ChromeUtils.importESModule(
+    "resource://testing-common/services/sync/fxa_utils.sys.mjs"
+  );
+
+add_task(async function test_findCluster() {
+  _("Test FxA _findCluster()");
+
+  _("_findCluster() throws on 500 errors.");
+  initializeIdentityWithTokenServerResponse({
+    status: 500,
+    headers: [],
+    body: "",
+  });
+
+  await Assert.rejects(
+    Service.identity._findCluster(),
+    /TokenServerClientServerError/
+  );
+
+  _("_findCluster() returns null on authentication errors.");
+  initializeIdentityWithTokenServerResponse({
+    status: 401,
+    headers: { "content-type": "application/json" },
+    body: "{}",
+  });
+
+  let cluster = await Service.identity._findCluster();
+  Assert.strictEqual(cluster, null);
+
+  _("_findCluster() works with correct tokenserver response.");
+  let endpoint = "http://example.com/something";
+  initializeIdentityWithTokenServerResponse({
+    status: 200,
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      api_endpoint: endpoint,
+      duration: 300,
+      id: "id",
+      key: "key",
+      uid: "uid",
+    }),
+  });
+
+  cluster = await Service.identity._findCluster();
+  // The cluster manager ensures a trailing "/"
+  Assert.strictEqual(cluster, endpoint + "/");
+
+  for (const pref of Svc.PrefBranch.getChildList("")) {
+    Svc.PrefBranch.clearUserPref(pref);
+  }
+});

src/zen/tests/mochitests/sync/test_history_engine.js

@@ -0,0 +1,429 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { HistoryEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/history.sys.mjs"
+);
+
+// Use only for rawAddVisit.
+XPCOMUtils.defineLazyServiceGetter(
+  this,
+  "asyncHistory",
+  "@mozilla.org/browser/history;1",
+  Ci.mozIAsyncHistory
+);
+async function rawAddVisit(id, uri, visitPRTime, transitionType) {
+  return new Promise(resolve => {
+    let results = [];
+    let handler = {
+      handleResult(result) {
+        results.push(result);
+      },
+      handleError(resultCode) {
+        do_throw(`updatePlaces gave error ${resultCode}!`);
+      },
+      handleCompletion(count) {
+        resolve({ results, count });
+      },
+    };
+    asyncHistory.updatePlaces(
+      [
+        {
+          guid: id,
+          uri: typeof uri == "string" ? CommonUtils.makeURI(uri) : uri,
+          visits: [{ visitDate: visitPRTime, transitionType }],
+        },
+      ],
+      handler
+    );
+  });
+}
+
+add_task(async function test_history_download_limit() {
+  let engine = new HistoryEngine(Service);
+  await engine.initialize();
+
+  let server = await serverForFoo(engine);
+  await SyncTestingInfrastructure(server);
+
+  let lastSync = new_timestamp();
+
+  let collection = server.user("foo").collection("history");
+  for (let i = 0; i < 15; i++) {
+    let id = "place" + i.toString(10).padStart(7, "0");
+    let wbo = new ServerWBO(
+      id,
+      encryptPayload({
+        id,
+        histUri: "http://example.com/" + i,
+        title: "Page " + i,
+        visits: [
+          {
+            date: Date.now() * 1000,
+            type: PlacesUtils.history.TRANSITIONS.TYPED,
+          },
+          {
+            date: Date.now() * 1000,
+            type: PlacesUtils.history.TRANSITIONS.LINK,
+          },
+        ],
+      }),
+      lastSync + 1 + i
+    );
+    wbo.sortindex = 15 - i;
+    collection.insertWBO(wbo);
+  }
+
+  // We have 15 records on the server since the last sync, but our download
+  // limit is 5 records at a time. We should eventually fetch all 15.
+  await engine.setLastSync(lastSync);
+  engine.downloadBatchSize = 4;
+  engine.downloadLimit = 5;
+
+  // Don't actually fetch any backlogged records, so that we can inspect
+  // the backlog between syncs.
+  engine.guidFetchBatchSize = 0;
+
+  let ping = await sync_engine_and_validate_telem(engine, false);
+  deepEqual(ping.engines[0].incoming, { applied: 5 });
+
+  let backlogAfterFirstSync = Array.from(engine.toFetch).sort();
+  deepEqual(backlogAfterFirstSync, [
+    "place0000000",
+    "place0000001",
+    "place0000002",
+    "place0000003",
+    "place0000004",
+    "place0000005",
+    "place0000006",
+    "place0000007",
+    "place0000008",
+    "place0000009",
+  ]);
+
+  // We should have fast-forwarded the last sync time.
+  equal(await engine.getLastSync(), lastSync + 15);
+
+  engine.lastModified = collection.modified;
+  ping = await sync_engine_and_validate_telem(engine, false);
+  ok(!ping.engines[0].incoming);
+
+  // After the second sync, our backlog still contains the same GUIDs: we
+  // weren't able to make progress on fetching them, since our
+  // `guidFetchBatchSize` is 0.
+  let backlogAfterSecondSync = Array.from(engine.toFetch).sort();
+  deepEqual(backlogAfterFirstSync, backlogAfterSecondSync);
+
+  // Now add a newer record to the server.
+  let newWBO = new ServerWBO(
+    "placeAAAAAAA",
+    encryptPayload({
+      id: "placeAAAAAAA",
+      histUri: "http://example.com/a",
+      title: "New Page A",
+      visits: [
+        {
+          date: Date.now() * 1000,
+          type: PlacesUtils.history.TRANSITIONS.TYPED,
+        },
+      ],
+    }),
+    lastSync + 20
+  );
+  newWBO.sortindex = -1;
+  collection.insertWBO(newWBO);
+
+  engine.lastModified = collection.modified;
+  ping = await sync_engine_and_validate_telem(engine, false);
+  deepEqual(ping.engines[0].incoming, { applied: 1 });
+
+  // Our backlog should remain the same.
+  let backlogAfterThirdSync = Array.from(engine.toFetch).sort();
+  deepEqual(backlogAfterSecondSync, backlogAfterThirdSync);
+
+  equal(await engine.getLastSync(), lastSync + 20);
+
+  // Bump the fetch batch size to let the backlog make progress. We should
+  // make 3 requests to fetch 5 backlogged GUIDs.
+  engine.guidFetchBatchSize = 2;
+
+  engine.lastModified = collection.modified;
+  ping = await sync_engine_and_validate_telem(engine, false);
+  deepEqual(ping.engines[0].incoming, { applied: 5 });
+
+  deepEqual(Array.from(engine.toFetch).sort(), [
+    "place0000005",
+    "place0000006",
+    "place0000007",
+    "place0000008",
+    "place0000009",
+  ]);
+
+  // Sync again to clear out the backlog.
+  engine.lastModified = collection.modified;
+  ping = await sync_engine_and_validate_telem(engine, false);
+  deepEqual(ping.engines[0].incoming, { applied: 5 });
+
+  deepEqual(Array.from(engine.toFetch), []);
+
+  await engine.wipeClient();
+  await engine.finalize();
+});
+
+add_task(async function test_history_visit_roundtrip() {
+  let engine = new HistoryEngine(Service);
+  await engine.initialize();
+  let server = await serverForFoo(engine);
+  await SyncTestingInfrastructure(server);
+
+  engine._tracker.start();
+
+  let id = "aaaaaaaaaaaa";
+  let oneHourMS = 60 * 60 * 1000;
+  // Insert a visit with a non-round microsecond timestamp (e.g. it's not evenly
+  // divisible by 1000). This will typically be the case for visits that occur
+  // during normal navigation.
+  let time = (Date.now() - oneHourMS) * 1000 + 555;
+  // We use the low level history api since it lets us provide microseconds
+  let { count } = await rawAddVisit(
+    id,
+    "https://www.example.com",
+    time,
+    PlacesUtils.history.TRANSITIONS.TYPED
+  );
+  equal(count, 1);
+  // Check that it was inserted and that we didn't round on the insert.
+  let visits = await PlacesSyncUtils.history.fetchVisitsForURL(
+    "https://www.example.com"
+  );
+  equal(visits.length, 1);
+  equal(visits[0].date, time);
+
+  let collection = server.user("foo").collection("history");
+
+  // Sync the visit up to the server.
+  await sync_engine_and_validate_telem(engine, false);
+
+  collection.updateRecord(
+    id,
+    cleartext => {
+      // Double-check that we didn't round the visit's timestamp to the nearest
+      // millisecond when uploading.
+      equal(cleartext.visits[0].date, time);
+      // Add a remote visit so that we get past the deepEquals check in reconcile
+      // (otherwise the history engine will skip applying this record). The
+      // contents of this visit don't matter, beyond the fact that it needs to
+      // exist.
+      cleartext.visits.push({
+        date: (Date.now() - oneHourMS / 2) * 1000,
+        type: PlacesUtils.history.TRANSITIONS.LINK,
+      });
+    },
+    new_timestamp() + 10
+  );
+
+  // Force a remote sync.
+  await engine.setLastSync(new_timestamp() - 30);
+  await sync_engine_and_validate_telem(engine, false);
+
+  // Make sure that we didn't duplicate the visit when inserting. (Prior to bug
+  // 1423395, we would insert a duplicate visit, where the timestamp was
+  // effectively `Math.round(microsecondTimestamp / 1000) * 1000`.)
+  visits = await PlacesSyncUtils.history.fetchVisitsForURL(
+    "https://www.example.com"
+  );
+  equal(visits.length, 2);
+
+  await engine.wipeClient();
+  await engine.finalize();
+});
+
+add_task(async function test_history_visit_dedupe_old() {
+  let engine = new HistoryEngine(Service);
+  await engine.initialize();
+  let server = await serverForFoo(engine);
+  await SyncTestingInfrastructure(server);
+
+  let initialVisits = Array.from({ length: 25 }, (_, index) => ({
+    transition: PlacesUtils.history.TRANSITION_LINK,
+    date: new Date(Date.UTC(2017, 10, 1 + index)),
+  }));
+  initialVisits.push({
+    transition: PlacesUtils.history.TRANSITION_LINK,
+    date: new Date(),
+  });
+  await PlacesUtils.history.insert({
+    url: "https://www.example.com",
+    visits: initialVisits,
+  });
+
+  let recentVisits = await PlacesSyncUtils.history.fetchVisitsForURL(
+    "https://www.example.com"
+  );
+  equal(recentVisits.length, 20);
+  let { visits: allVisits, guid } = await PlacesUtils.history.fetch(
+    "https://www.example.com",
+    {
+      includeVisits: true,
+    }
+  );
+  equal(allVisits.length, 26);
+
+  let collection = server.user("foo").collection("history");
+
+  await sync_engine_and_validate_telem(engine, false);
+
+  collection.updateRecord(
+    guid,
+    data => {
+      data.visits.push(
+        // Add a couple remote visit equivalent to some old visits we have already
+        {
+          date: Date.UTC(2017, 10, 1) * 1000, // Nov 1, 2017
+          type: PlacesUtils.history.TRANSITIONS.LINK,
+        },
+        {
+          date: Date.UTC(2017, 10, 2) * 1000, // Nov 2, 2017
+          type: PlacesUtils.history.TRANSITIONS.LINK,
+        },
+        // Add a couple new visits to make sure we are still applying them.
+        {
+          date: Date.UTC(2017, 11, 4) * 1000, // Dec 4, 2017
+          type: PlacesUtils.history.TRANSITIONS.LINK,
+        },
+        {
+          date: Date.UTC(2017, 11, 5) * 1000, // Dec 5, 2017
+          type: PlacesUtils.history.TRANSITIONS.LINK,
+        }
+      );
+    },
+    new_timestamp() + 10
+  );
+
+  await engine.setLastSync(new_timestamp() - 30);
+  await sync_engine_and_validate_telem(engine, false);
+
+  allVisits = (
+    await PlacesUtils.history.fetch("https://www.example.com", {
+      includeVisits: true,
+    })
+  ).visits;
+
+  equal(allVisits.length, 28);
+  ok(
+    allVisits.find(x => x.date.getTime() === Date.UTC(2017, 11, 4)),
+    "Should contain the Dec. 4th visit"
+  );
+  ok(
+    allVisits.find(x => x.date.getTime() === Date.UTC(2017, 11, 5)),
+    "Should contain the Dec. 5th visit"
+  );
+
+  await engine.wipeClient();
+  await engine.finalize();
+});
+
+add_task(async function test_history_unknown_fields() {
+  let engine = new HistoryEngine(Service);
+  await engine.initialize();
+  let server = await serverForFoo(engine);
+  await SyncTestingInfrastructure(server);
+
+  engine._tracker.start();
+
+  let id = "aaaaaaaaaaaa";
+  let oneHourMS = 60 * 60 * 1000;
+  // Insert a visit with a non-round microsecond timestamp (e.g. it's not evenly
+  // divisible by 1000). This will typically be the case for visits that occur
+  // during normal navigation.
+  let time = (Date.now() - oneHourMS) * 1000 + 555;
+  // We use the low level history api since it lets us provide microseconds
+  let { count } = await rawAddVisit(
+    id,
+    "https://www.example.com",
+    time,
+    PlacesUtils.history.TRANSITIONS.TYPED
+  );
+  equal(count, 1);
+
+  let collection = server.user("foo").collection("history");
+
+  // Sync the visit up to the server.
+  await sync_engine_and_validate_telem(engine, false);
+
+  collection.updateRecord(
+    id,
+    cleartext => {
+      equal(cleartext.visits[0].date, time);
+
+      // Add unknown fields to an instance of a visit
+      cleartext.visits.push({
+        date: (Date.now() - oneHourMS / 2) * 1000,
+        type: PlacesUtils.history.TRANSITIONS.LINK,
+        unknownVisitField: "an unknown field could show up in a visit!",
+      });
+      cleartext.title = "A page title";
+      // Add unknown fields to the payload for this URL
+      cleartext.unknownStrField = "an unknown str field";
+      cleartext.unknownObjField = { newField: "a field within an object" };
+    },
+    new_timestamp() + 10
+  );
+
+  // Force a remote sync.
+  await engine.setLastSync(new_timestamp() - 30);
+  await sync_engine_and_validate_telem(engine, false);
+
+  // Add a new visit to ensure we're actually putting things back on the server
+  let newTime = (Date.now() - oneHourMS) * 1000 + 555;
+  await rawAddVisit(
+    id,
+    "https://www.example.com",
+    newTime,
+    PlacesUtils.history.TRANSITIONS.LINK
+  );
+
+  // Sync again
+  await engine.setLastSync(new_timestamp() - 30);
+  await sync_engine_and_validate_telem(engine, false);
+
+  let placeInfo = await PlacesSyncUtils.history.fetchURLInfoForGuid(id);
+
+  // Found the place we're looking for
+  Assert.equal(placeInfo.title, "A page title");
+  Assert.equal(placeInfo.url, "https://www.example.com/");
+
+  // It correctly returns any unknownFields that might've been
+  // stored in the moz_places_extra table
+  deepEqual(JSON.parse(placeInfo.unknownFields), {
+    unknownStrField: "an unknown str field",
+    unknownObjField: { newField: "a field within an object" },
+  });
+
+  // Getting visits via SyncUtils also will return unknownFields
+  // via the moz_historyvisits_extra table
+  let visits = await PlacesSyncUtils.history.fetchVisitsForURL(
+    "https://www.example.com"
+  );
+  equal(visits.length, 3);
+
+  // fetchVisitsForURL is a sync method that gets called during upload
+  // so unknown field should already be at the top-level
+  deepEqual(
+    visits[0].unknownVisitField,
+    "an unknown field could show up in a visit!"
+  );
+
+  // Remote history record should have the fields back at the top level
+  let remotePlace = collection.payloads().find(rec => rec.id === id);
+  deepEqual(remotePlace.unknownStrField, "an unknown str field");
+  deepEqual(remotePlace.unknownObjField, {
+    newField: "a field within an object",
+  });
+
+  await engine.wipeClient();
+  await engine.finalize();
+});

src/zen/tests/mochitests/sync/test_history_store.js

@@ -0,0 +1,560 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { HistoryEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/history.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+const { SyncedRecordsTelemetry } = ChromeUtils.importESModule(
+  "resource://services-sync/telemetry.sys.mjs"
+);
+
+const TIMESTAMP1 = (Date.now() - 103406528) * 1000;
+const TIMESTAMP2 = (Date.now() - 6592903) * 1000;
+const TIMESTAMP3 = (Date.now() - 123894) * 1000;
+
+function isDateApproximately(actual, expected, skewMillis = 1000) {
+  let lowerBound = expected - skewMillis;
+  let upperBound = expected + skewMillis;
+  return actual >= lowerBound && actual <= upperBound;
+}
+
+let engine, store, fxuri, fxguid, tburi, tbguid;
+
+async function applyEnsureNoFailures(records) {
+  let countTelemetry = new SyncedRecordsTelemetry();
+  Assert.equal(
+    (await store.applyIncomingBatch(records, countTelemetry)).length,
+    0
+  );
+}
+
+add_task(async function setup() {
+  engine = new HistoryEngine(Service);
+  await engine.initialize();
+  store = engine._store;
+});
+
+add_task(async function test_store() {
+  _("Verify that we've got an empty store to work with.");
+  do_check_empty(await store.getAllIDs());
+
+  _("Let's create an entry in the database.");
+  fxuri = CommonUtils.makeURI("http://getfirefox.com/");
+
+  await PlacesTestUtils.addVisits({
+    uri: fxuri,
+    title: "Get Firefox!",
+    visitDate: TIMESTAMP1,
+  });
+  _("Verify that the entry exists.");
+  let ids = Object.keys(await store.getAllIDs());
+  Assert.equal(ids.length, 1);
+  fxguid = ids[0];
+  Assert.ok(await store.itemExists(fxguid));
+
+  _("If we query a non-existent record, it's marked as deleted.");
+  let record = await store.createRecord("non-existent");
+  Assert.ok(record.deleted);
+
+  _("Verify createRecord() returns a complete record.");
+  record = await store.createRecord(fxguid);
+  Assert.equal(record.histUri, fxuri.spec);
+  Assert.equal(record.title, "Get Firefox!");
+  Assert.equal(record.visits.length, 1);
+  Assert.equal(record.visits[0].date, TIMESTAMP1);
+  Assert.equal(record.visits[0].type, Ci.nsINavHistoryService.TRANSITION_LINK);
+
+  _("Let's modify the record and have the store update the database.");
+  let secondvisit = {
+    date: TIMESTAMP2,
+    type: Ci.nsINavHistoryService.TRANSITION_TYPED,
+  };
+  let onVisitObserved = PlacesTestUtils.waitForNotification(["page-visited"]);
+  let updatedRec = await store.createRecord(fxguid);
+  updatedRec.cleartext.title = "Hol Dir Firefox!";
+  updatedRec.cleartext.visits.push(secondvisit);
+  await applyEnsureNoFailures([updatedRec]);
+  await onVisitObserved;
+  let queryres = await PlacesUtils.history.fetch(fxuri.spec, {
+    includeVisits: true,
+  });
+  Assert.equal(queryres.title, "Hol Dir Firefox!");
+  Assert.deepEqual(queryres.visits, [
+    {
+      date: new Date(TIMESTAMP2 / 1000),
+      transition: Ci.nsINavHistoryService.TRANSITION_TYPED,
+    },
+    {
+      date: new Date(TIMESTAMP1 / 1000),
+      transition: Ci.nsINavHistoryService.TRANSITION_LINK,
+    },
+  ]);
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_store_create() {
+  _("Create a brand new record through the store.");
+  tbguid = Utils.makeGUID();
+  tburi = CommonUtils.makeURI("http://getthunderbird.com");
+  let onVisitObserved = PlacesTestUtils.waitForNotification(["page-visited"]);
+  let record = await store.createRecord(tbguid);
+  record.cleartext = {
+    id: tbguid,
+    histUri: tburi.spec,
+    title: "The bird is the word!",
+    visits: [
+      { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_TYPED },
+    ],
+  };
+  await applyEnsureNoFailures([record]);
+  await onVisitObserved;
+  Assert.ok(await store.itemExists(tbguid));
+  do_check_attribute_count(await store.getAllIDs(), 1);
+  let queryres = await PlacesUtils.history.fetch(tburi.spec, {
+    includeVisits: true,
+  });
+  Assert.equal(queryres.title, "The bird is the word!");
+  Assert.deepEqual(queryres.visits, [
+    {
+      date: new Date(TIMESTAMP3 / 1000),
+      transition: Ci.nsINavHistoryService.TRANSITION_TYPED,
+    },
+  ]);
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_null_title() {
+  _(
+    "Make sure we handle a null title gracefully (it can happen in some cases, e.g. for resource:// URLs)"
+  );
+  let resguid = Utils.makeGUID();
+  let resuri = CommonUtils.makeURI("unknown://title");
+  let record = await store.createRecord(resguid);
+  record.cleartext = {
+    id: resguid,
+    histUri: resuri.spec,
+    title: null,
+    visits: [
+      { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_TYPED },
+    ],
+  };
+  await applyEnsureNoFailures([record]);
+  do_check_attribute_count(await store.getAllIDs(), 1);
+
+  let queryres = await PlacesUtils.history.fetch(resuri.spec, {
+    includeVisits: true,
+  });
+  Assert.equal(queryres.title, "");
+  Assert.deepEqual(queryres.visits, [
+    {
+      date: new Date(TIMESTAMP3 / 1000),
+      transition: Ci.nsINavHistoryService.TRANSITION_TYPED,
+    },
+  ]);
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_invalid_records() {
+  _("Make sure we handle invalid URLs in places databases gracefully.");
+  await PlacesUtils.withConnectionWrapper(
+    "test_invalid_record",
+    async function (db) {
+      await db.execute(
+        "INSERT INTO moz_places " +
+          "(url, url_hash, title, rev_host, visit_count, last_visit_date) " +
+          "VALUES ('invalid-uri', hash('invalid-uri'), 'Invalid URI', '.', 1, " +
+          TIMESTAMP3 +
+          ")"
+      );
+      // Add the corresponding visit to retain database coherence.
+      await db.execute(
+        "INSERT INTO moz_historyvisits " +
+          "(place_id, visit_date, visit_type, session) " +
+          "VALUES ((SELECT id FROM moz_places WHERE url_hash = hash('invalid-uri') AND url = 'invalid-uri'), " +
+          TIMESTAMP3 +
+          ", " +
+          Ci.nsINavHistoryService.TRANSITION_TYPED +
+          ", 1)"
+      );
+    }
+  );
+  do_check_attribute_count(await store.getAllIDs(), 1);
+
+  _("Make sure we report records with invalid URIs.");
+  let invalid_uri_guid = Utils.makeGUID();
+  let countTelemetry = new SyncedRecordsTelemetry();
+  let failed = await store.applyIncomingBatch(
+    [
+      {
+        id: invalid_uri_guid,
+        histUri: ":::::::::::::::",
+        title: "Doesn't have a valid URI",
+        visits: [
+          { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_EMBED },
+        ],
+      },
+    ],
+    countTelemetry
+  );
+  Assert.equal(failed.length, 1);
+  Assert.equal(failed[0], invalid_uri_guid);
+  Assert.equal(
+    countTelemetry.incomingCounts.failedReasons[0].name,
+    "<URL> is not a valid URL."
+  );
+  Assert.equal(countTelemetry.incomingCounts.failedReasons[0].count, 1);
+
+  _("Make sure we handle records with invalid GUIDs gracefully (ignore).");
+  await applyEnsureNoFailures([
+    {
+      id: "invalid",
+      histUri: "http://invalid.guid/",
+      title: "Doesn't have a valid GUID",
+      visits: [
+        { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_EMBED },
+      ],
+    },
+  ]);
+
+  _(
+    "Make sure we handle records with invalid visit codes or visit dates, gracefully ignoring those visits."
+  );
+  let no_date_visit_guid = Utils.makeGUID();
+  let no_type_visit_guid = Utils.makeGUID();
+  let invalid_type_visit_guid = Utils.makeGUID();
+  let non_integer_visit_guid = Utils.makeGUID();
+  countTelemetry = new SyncedRecordsTelemetry();
+  failed = await store.applyIncomingBatch(
+    [
+      {
+        id: no_date_visit_guid,
+        histUri: "http://no.date.visit/",
+        title: "Visit has no date",
+        visits: [{ type: Ci.nsINavHistoryService.TRANSITION_EMBED }],
+      },
+      {
+        id: no_type_visit_guid,
+        histUri: "http://no.type.visit/",
+        title: "Visit has no type",
+        visits: [{ date: TIMESTAMP3 }],
+      },
+      {
+        id: invalid_type_visit_guid,
+        histUri: "http://invalid.type.visit/",
+        title: "Visit has invalid type",
+        visits: [
+          {
+            date: TIMESTAMP3,
+            type: Ci.nsINavHistoryService.TRANSITION_LINK - 1,
+          },
+        ],
+      },
+      {
+        id: non_integer_visit_guid,
+        histUri: "http://non.integer.visit/",
+        title: "Visit has non-integer date",
+        visits: [
+          { date: 1234.567, type: Ci.nsINavHistoryService.TRANSITION_EMBED },
+        ],
+      },
+    ],
+    countTelemetry
+  );
+  Assert.equal(failed.length, 0);
+
+  // Make sure we can apply tombstones (both valid and invalid)
+  countTelemetry = new SyncedRecordsTelemetry();
+  failed = await store.applyIncomingBatch(
+    [
+      { id: no_date_visit_guid, deleted: true },
+      { id: "not-a-valid-guid", deleted: true },
+    ],
+    countTelemetry
+  );
+  Assert.deepEqual(failed, ["not-a-valid-guid"]);
+  Assert.equal(
+    countTelemetry.incomingCounts.failedReasons[0].name,
+    "<URL> is not a valid URL."
+  );
+
+  _("Make sure we handle records with javascript: URLs gracefully.");
+  await applyEnsureNoFailures(
+    [
+      {
+        id: Utils.makeGUID(),
+        histUri: "javascript:''",
+        title: "javascript:''",
+        visits: [
+          { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_EMBED },
+        ],
+      },
+    ],
+    countTelemetry
+  );
+
+  _("Make sure we handle records without any visits gracefully.");
+  await applyEnsureNoFailures([
+    {
+      id: Utils.makeGUID(),
+      histUri: "http://getfirebug.com",
+      title: "Get Firebug!",
+      visits: [],
+    },
+  ]);
+});
+
+add_task(async function test_unknowingly_invalid_records() {
+  _("Make sure we handle rejection of records by places gracefully.");
+  let oldCAU = store._canAddURI;
+  store._canAddURI = () => true;
+  try {
+    _("Make sure that when places rejects this record we record it as failed");
+    let guid = Utils.makeGUID();
+    let countTelemetry = new SyncedRecordsTelemetry();
+    let invalidRecord = await store.createRecord(guid);
+    invalidRecord.cleartext = {
+      id: guid,
+      histUri: "javascript:''",
+      title: "javascript:''",
+      visits: [
+        {
+          date: TIMESTAMP3,
+          type: Ci.nsINavHistoryService.TRANSITION_EMBED,
+        },
+      ],
+    };
+    let result = await store.applyIncomingBatch(
+      [invalidRecord],
+      countTelemetry
+    );
+    deepEqual(result, [guid]);
+  } finally {
+    store._canAddURI = oldCAU;
+  }
+});
+
+add_task(async function test_clamp_visit_dates() {
+  let futureVisitTime = Date.now() + 5 * 60 * 1000;
+  let recentVisitTime = Date.now() - 5 * 60 * 1000;
+
+  let recordA = await store.createRecord("visitAAAAAAA");
+  recordA.cleartext = {
+    id: "visitAAAAAAA",
+    histUri: "http://example.com/a",
+    title: "A",
+    visits: [
+      {
+        date: "invalidDate",
+        type: Ci.nsINavHistoryService.TRANSITION_LINK,
+      },
+    ],
+  };
+  let recordB = await store.createRecord("visitBBBBBBB");
+  recordB.cleartext = {
+    id: "visitBBBBBBB",
+    histUri: "http://example.com/b",
+    title: "B",
+    visits: [
+      {
+        date: 100,
+        type: Ci.nsINavHistoryService.TRANSITION_TYPED,
+      },
+      {
+        date: 250,
+        type: Ci.nsINavHistoryService.TRANSITION_TYPED,
+      },
+      {
+        date: recentVisitTime * 1000,
+        type: Ci.nsINavHistoryService.TRANSITION_TYPED,
+      },
+    ],
+  };
+  let recordC = await store.createRecord("visitCCCCCCC");
+  recordC.cleartext = {
+    id: "visitCCCCCCC",
+    histUri: "http://example.com/c",
+    title: "D",
+    visits: [
+      {
+        date: futureVisitTime * 1000,
+        type: Ci.nsINavHistoryService.TRANSITION_BOOKMARK,
+      },
+    ],
+  };
+  let recordD = await store.createRecord("visitDDDDDDD");
+  recordD.cleartext = {
+    id: "visitDDDDDDD",
+    histUri: "http://example.com/d",
+    title: "D",
+    visits: [
+      {
+        date: recentVisitTime * 1000,
+        type: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+      },
+    ],
+  };
+  await applyEnsureNoFailures([recordA, recordB, recordC, recordD]);
+
+  let visitsForA = await PlacesSyncUtils.history.fetchVisitsForURL(
+    "http://example.com/a"
+  );
+  deepEqual(visitsForA, [], "Should ignore visits with invalid dates");
+
+  let visitsForB = await PlacesSyncUtils.history.fetchVisitsForURL(
+    "http://example.com/b"
+  );
+  deepEqual(
+    visitsForB,
+    [
+      {
+        date: recentVisitTime * 1000,
+        type: Ci.nsINavHistoryService.TRANSITION_TYPED,
+      },
+      {
+        // We should clamp visit dates older than original Mosaic release.
+        date: PlacesSyncUtils.bookmarks.EARLIEST_BOOKMARK_TIMESTAMP * 1000,
+        type: Ci.nsINavHistoryService.TRANSITION_TYPED,
+      },
+    ],
+    "Should record clamped visit and valid visit for B"
+  );
+
+  let visitsForC = await PlacesSyncUtils.history.fetchVisitsForURL(
+    "http://example.com/c"
+  );
+  equal(visitsForC.length, 1, "Should record clamped future visit for C");
+  let visitDateForC = PlacesUtils.toDate(visitsForC[0].date);
+  ok(
+    isDateApproximately(visitDateForC, Date.now()),
+    "Should clamp future visit date for C to now"
+  );
+
+  let visitsForD = await PlacesSyncUtils.history.fetchVisitsForURL(
+    "http://example.com/d"
+  );
+  deepEqual(
+    visitsForD,
+    [
+      {
+        date: recentVisitTime * 1000,
+        type: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+      },
+    ],
+    "Should not clamp valid visit dates"
+  );
+});
+
+add_task(async function test_remove() {
+  _("Remove an existent record and a non-existent from the store.");
+  await applyEnsureNoFailures([
+    { id: fxguid, deleted: true },
+    { id: Utils.makeGUID(), deleted: true },
+  ]);
+  Assert.equal(false, await store.itemExists(fxguid));
+  let queryres = await PlacesUtils.history.fetch(fxuri.spec, {
+    includeVisits: true,
+  });
+  Assert.equal(null, queryres);
+
+  _("Make sure wipe works.");
+  await store.wipe();
+  do_check_empty(await store.getAllIDs());
+  queryres = await PlacesUtils.history.fetch(fxuri.spec, {
+    includeVisits: true,
+  });
+  Assert.equal(null, queryres);
+  queryres = await PlacesUtils.history.fetch(tburi.spec, {
+    includeVisits: true,
+  });
+  Assert.equal(null, queryres);
+});
+
+add_task(async function test_chunking() {
+  let mvpi = store.MAX_VISITS_PER_INSERT;
+  store.MAX_VISITS_PER_INSERT = 3;
+  let checkChunks = function (input, expected) {
+    let chunks = Array.from(store._generateChunks(input));
+    deepEqual(chunks, expected);
+  };
+  try {
+    checkChunks([{ visits: ["x"] }], [[{ visits: ["x"] }]]);
+
+    // 3 should still be one chunk.
+    checkChunks([{ visits: ["x", "x", "x"] }], [[{ visits: ["x", "x", "x"] }]]);
+
+    // 4 should still be one chunk as we don't split individual records.
+    checkChunks(
+      [{ visits: ["x", "x", "x", "x"] }],
+      [[{ visits: ["x", "x", "x", "x"] }]]
+    );
+
+    // 4 in the first and 1 in the second should be 2 chunks.
+    checkChunks(
+      [{ visits: ["x", "x", "x", "x"] }, { visits: ["x"] }],
+      // expected
+      [[{ visits: ["x", "x", "x", "x"] }], [{ visits: ["x"] }]]
+    );
+
+    // we put multiple records into chunks
+    checkChunks(
+      [
+        { visits: ["x", "x"] },
+        { visits: ["x"] },
+        { visits: ["x"] },
+        { visits: ["x", "x"] },
+        { visits: ["x", "x", "x", "x"] },
+      ],
+      // expected
+      [
+        [{ visits: ["x", "x"] }, { visits: ["x"] }],
+        [{ visits: ["x"] }, { visits: ["x", "x"] }],
+        [{ visits: ["x", "x", "x", "x"] }],
+      ]
+    );
+  } finally {
+    store.MAX_VISITS_PER_INSERT = mvpi;
+  }
+});
+
+add_task(async function test_getAllIDs_filters_file_uris() {
+  let uri = CommonUtils.makeURI("file:///Users/eoger/tps/config.json");
+  let visitAddedPromise = promiseVisit("added", uri);
+  await PlacesTestUtils.addVisits({
+    uri,
+    visitDate: Date.now() * 1000,
+    transition: PlacesUtils.history.TRANSITION_LINK,
+  });
+  await visitAddedPromise;
+
+  do_check_attribute_count(await store.getAllIDs(), 0);
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_applyIncomingBatch_filters_file_uris() {
+  const guid = Utils.makeGUID();
+  let uri = CommonUtils.makeURI("file:///Users/eoger/tps/config.json");
+  await applyEnsureNoFailures([
+    {
+      id: guid,
+      histUri: uri.spec,
+      title: "TPS CONFIG",
+      visits: [
+        { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_TYPED },
+      ],
+    },
+  ]);
+  Assert.equal(false, await store.itemExists(guid));
+  let queryres = await PlacesUtils.history.fetch(uri.spec, {
+    includeVisits: true,
+  });
+  Assert.equal(null, queryres);
+});
+
+add_task(async function cleanup() {
+  _("Clean up.");
+  await PlacesUtils.history.clear();
+});

src/zen/tests/mochitests/sync/test_history_tracker.js

@@ -0,0 +1,251 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { PlacesDBUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/PlacesDBUtils.sys.mjs"
+);
+const { HistoryEngine } = ChromeUtils.importESModule(
+  "resource://services-sync/engines/history.sys.mjs"
+);
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+let engine;
+let tracker;
+
+add_task(async function setup() {
+  await Service.engineManager.clear();
+  await Service.engineManager.register(HistoryEngine);
+  engine = Service.engineManager.get("history");
+  tracker = engine._tracker;
+});
+
+async function verifyTrackerEmpty() {
+  let changes = await engine.pullNewChanges();
+  do_check_empty(changes);
+  equal(tracker.score, 0);
+}
+
+async function verifyTrackedCount(expected) {
+  let changes = await engine.pullNewChanges();
+  do_check_attribute_count(changes, expected);
+}
+
+async function verifyTrackedItems(tracked) {
+  let changes = await engine.pullNewChanges();
+  let trackedIDs = new Set(Object.keys(changes));
+  for (let guid of tracked) {
+    ok(guid in changes, `${guid} should be tracked`);
+    Assert.greater(changes[guid], 0, `${guid} should have a modified time`);
+    trackedIDs.delete(guid);
+  }
+  equal(
+    trackedIDs.size,
+    0,
+    `Unhandled tracked IDs: ${JSON.stringify(Array.from(trackedIDs))}`
+  );
+}
+
+async function resetTracker() {
+  await tracker.clearChangedIDs();
+  tracker.resetScore();
+}
+
+async function cleanup() {
+  await PlacesUtils.history.clear();
+  await resetTracker();
+  await tracker.stop();
+}
+
+add_task(async function test_empty() {
+  _("Verify we've got an empty, disabled tracker to work with.");
+  await verifyTrackerEmpty();
+  Assert.ok(!tracker._isTracking);
+
+  await cleanup();
+});
+
+add_task(async function test_not_tracking() {
+  _("Create history item. Won't show because we haven't started tracking yet");
+  await addVisit("not_tracking");
+  await verifyTrackerEmpty();
+
+  await cleanup();
+});
+
+add_task(async function test_start_tracking() {
+  _("Add hook for save completion.");
+  let savePromise = new Promise((resolve, reject) => {
+    let save = tracker._storage._save;
+    tracker._storage._save = async function () {
+      try {
+        await save.call(this);
+        resolve();
+      } catch (ex) {
+        reject(ex);
+      } finally {
+        tracker._storage._save = save;
+      }
+    };
+  });
+
+  _("Tell the tracker to start tracking changes.");
+  tracker.start();
+  let scorePromise = promiseOneObserver("weave:engine:score:updated");
+  await addVisit("start_tracking");
+  await scorePromise;
+
+  _("Score updated in test_start_tracking.");
+  await verifyTrackedCount(1);
+  Assert.equal(tracker.score, SCORE_INCREMENT_SMALL);
+
+  await savePromise;
+
+  _("changedIDs written to disk. Proceeding.");
+  await cleanup();
+});
+
+add_task(async function test_start_tracking_twice() {
+  _("Verifying preconditions.");
+  tracker.start();
+  await addVisit("start_tracking_twice1");
+  await verifyTrackedCount(1);
+  Assert.equal(tracker.score, SCORE_INCREMENT_SMALL);
+
+  _("Notifying twice won't do any harm.");
+  tracker.start();
+  let scorePromise = promiseOneObserver("weave:engine:score:updated");
+  await addVisit("start_tracking_twice2");
+  await scorePromise;
+
+  _("Score updated in test_start_tracking_twice.");
+  await verifyTrackedCount(2);
+  Assert.equal(tracker.score, 2 * SCORE_INCREMENT_SMALL);
+
+  await cleanup();
+});
+
+add_task(async function test_track_delete() {
+  _("Deletions are tracked.");
+
+  // This isn't present because we weren't tracking when it was visited.
+  await addVisit("track_delete");
+  let uri = CommonUtils.makeURI("http://getfirefox.com/track_delete");
+  let guid = await engine._store.GUIDForUri(uri.spec);
+  await verifyTrackerEmpty();
+
+  tracker.start();
+  let visitRemovedPromise = promiseVisit("removed", uri);
+  let scorePromise = promiseOneObserver("weave:engine:score:updated");
+  await PlacesUtils.history.remove(uri);
+  await Promise.all([scorePromise, visitRemovedPromise]);
+
+  await verifyTrackedItems([guid]);
+  Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE);
+
+  await cleanup();
+});
+
+add_task(async function test_dont_track_expiration() {
+  _("Expirations are not tracked.");
+  let uriToRemove = await addVisit("to_remove");
+  let guidToRemove = await engine._store.GUIDForUri(uriToRemove.spec);
+
+  await resetTracker();
+  await verifyTrackerEmpty();
+
+  tracker.start();
+  let visitRemovedPromise = promiseVisit("removed", uriToRemove);
+  let scorePromise = promiseOneObserver("weave:engine:score:updated");
+
+  // Observe expiration.
+  Services.obs.addObserver(function onExpiration(aSubject, aTopic) {
+    Services.obs.removeObserver(onExpiration, aTopic);
+    // Remove the remaining page to update its score.
+    PlacesUtils.history.remove(uriToRemove);
+  }, PlacesUtils.TOPIC_EXPIRATION_FINISHED);
+
+  // Force expiration of 1 entry.
+  Services.prefs.setIntPref("places.history.expiration.max_pages", 0);
+  Cc["@mozilla.org/places/expiration;1"]
+    .getService(Ci.nsIObserver)
+    .observe(null, "places-debug-start-expiration", 1);
+
+  await Promise.all([scorePromise, visitRemovedPromise]);
+  await verifyTrackedItems([guidToRemove]);
+
+  await cleanup();
+});
+
+add_task(async function test_stop_tracking() {
+  _("Let's stop tracking again.");
+  await tracker.stop();
+  await addVisit("stop_tracking");
+  await verifyTrackerEmpty();
+
+  await cleanup();
+});
+
+add_task(async function test_stop_tracking_twice() {
+  await tracker.stop();
+  await addVisit("stop_tracking_twice1");
+
+  _("Notifying twice won't do any harm.");
+  await tracker.stop();
+  await addVisit("stop_tracking_twice2");
+  await verifyTrackerEmpty();
+
+  await cleanup();
+});
+
+add_task(async function test_filter_file_uris() {
+  tracker.start();
+
+  let uri = CommonUtils.makeURI("file:///Users/eoger/tps/config.json");
+  let visitAddedPromise = promiseVisit("added", uri);
+  await PlacesTestUtils.addVisits({
+    uri,
+    visitDate: Date.now() * 1000,
+    transition: PlacesUtils.history.TRANSITION_LINK,
+  });
+  await visitAddedPromise;
+
+  await verifyTrackerEmpty();
+  await tracker.stop();
+  await cleanup();
+});
+
+add_task(async function test_filter_hidden() {
+  tracker.start();
+
+  _("Add visit; should be hidden by the redirect");
+  let hiddenURI = await addVisit("hidden");
+  let hiddenGUID = await engine._store.GUIDForUri(hiddenURI.spec);
+  _(`Hidden visit GUID: ${hiddenGUID}`);
+
+  _("Add redirect visit; should be tracked");
+  let trackedURI = await addVisit(
+    "redirect",
+    hiddenURI.spec,
+    PlacesUtils.history.TRANSITION_REDIRECT_PERMANENT
+  );
+  let trackedGUID = await engine._store.GUIDForUri(trackedURI.spec);
+  _(`Tracked visit GUID: ${trackedGUID}`);
+
+  _("Add visit for framed link; should be ignored");
+  let embedURI = await addVisit(
+    "framed_link",
+    null,
+    PlacesUtils.history.TRANSITION_FRAMED_LINK
+  );
+  let embedGUID = await engine._store.GUIDForUri(embedURI.spec);
+  _(`Framed link visit GUID: ${embedGUID}`);
+
+  _("Run Places maintenance to mark redirect visit as hidden");
+  await PlacesDBUtils.maintenanceOnIdle();
+
+  await verifyTrackedItems([trackedGUID]);
+
+  await cleanup();
+});

src/zen/tests/mochitests/sync/test_hmac_error.js

@@ -0,0 +1,250 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+// Track HMAC error counts.
+var hmacErrorCount = 0;
+(function () {
+  let hHE = Service.handleHMACEvent;
+  Service.handleHMACEvent = async function () {
+    hmacErrorCount++;
+    return hHE.call(Service);
+  };
+})();
+
+async function shared_setup() {
+  enableValidationPrefs();
+  syncTestLogging();
+
+  hmacErrorCount = 0;
+
+  let clientsEngine = Service.clientsEngine;
+  let clientsSyncID = await clientsEngine.resetLocalSyncID();
+
+  // Make sure RotaryEngine is the only one we sync.
+  let { engine, syncID, tracker } = await registerRotaryEngine();
+  await engine.setLastSync(123); // Needs to be non-zero so that tracker is queried.
+  engine._store.items = {
+    flying: "LNER Class A3 4472",
+    scotsman: "Flying Scotsman",
+  };
+  await tracker.addChangedID("scotsman", 0);
+  Assert.equal(1, Service.engineManager.getEnabled().length);
+
+  let engines = {
+    rotary: { version: engine.version, syncID },
+    clients: { version: clientsEngine.version, syncID: clientsSyncID },
+  };
+
+  // Common server objects.
+  let global = new ServerWBO("global", { engines });
+  let keysWBO = new ServerWBO("keys");
+  let rotaryColl = new ServerCollection({}, true);
+  let clientsColl = new ServerCollection({}, true);
+
+  return [engine, rotaryColl, clientsColl, keysWBO, global, tracker];
+}
+
+add_task(async function hmac_error_during_404() {
+  _("Attempt to replicate the HMAC error setup.");
+  let [engine, rotaryColl, clientsColl, keysWBO, global, tracker] =
+    await shared_setup();
+
+  // Hand out 404s for crypto/keys.
+  let keysHandler = keysWBO.handler();
+  let key404Counter = 0;
+  let keys404Handler = function (request, response) {
+    if (key404Counter > 0) {
+      let body = "Not Found";
+      response.setStatusLine(request.httpVersion, 404, body);
+      response.bodyOutputStream.write(body, body.length);
+      key404Counter--;
+      return;
+    }
+    keysHandler(request, response);
+  };
+
+  let collectionsHelper = track_collections_helper();
+  let upd = collectionsHelper.with_updated_collection;
+  let handlers = {
+    "/1.1/foo/info/collections": collectionsHelper.handler,
+    "/1.1/foo/storage/meta/global": upd("meta", global.handler()),
+    "/1.1/foo/storage/crypto/keys": upd("crypto", keys404Handler),
+    "/1.1/foo/storage/clients": upd("clients", clientsColl.handler()),
+    "/1.1/foo/storage/rotary": upd("rotary", rotaryColl.handler()),
+  };
+
+  let server = sync_httpd_setup(handlers);
+  // Do not instantiate SyncTestingInfrastructure; we need real crypto.
+  await configureIdentity({ username: "foo" }, server);
+  await Service.login();
+
+  try {
+    _("Syncing.");
+    await sync_and_validate_telem();
+
+    _(
+      "Partially resetting client, as if after a restart, and forcing redownload."
+    );
+    Service.collectionKeys.clear();
+    await engine.setLastSync(0); // So that we redownload records.
+    key404Counter = 1;
+    _("---------------------------");
+    await sync_and_validate_telem();
+    _("---------------------------");
+
+    // Two rotary items, one client record... no errors.
+    Assert.equal(hmacErrorCount, 0);
+  } finally {
+    await tracker.clearChangedIDs();
+    await Service.engineManager.unregister(engine);
+    for (const pref of Svc.PrefBranch.getChildList("")) {
+      Svc.PrefBranch.clearUserPref(pref);
+    }
+    Service.recordManager.clearCache();
+    await promiseStopServer(server);
+  }
+});
+
+add_task(async function hmac_error_during_node_reassignment() {
+  _("Attempt to replicate an HMAC error during node reassignment.");
+  let [engine, rotaryColl, clientsColl, keysWBO, global, tracker] =
+    await shared_setup();
+
+  let collectionsHelper = track_collections_helper();
+  let upd = collectionsHelper.with_updated_collection;
+
+  // We'll provide a 401 mid-way through the sync. This function
+  // simulates shifting to a node which has no data.
+  function on401() {
+    _("Deleting server data...");
+    global.delete();
+    rotaryColl.delete();
+    keysWBO.delete();
+    clientsColl.delete();
+    delete collectionsHelper.collections.rotary;
+    delete collectionsHelper.collections.crypto;
+    delete collectionsHelper.collections.clients;
+    _("Deleted server data.");
+  }
+
+  let should401 = false;
+  function upd401(coll, handler) {
+    return function (request, response) {
+      if (should401 && request.method != "DELETE") {
+        on401();
+        should401 = false;
+        let body = '"reassigned!"';
+        response.setStatusLine(request.httpVersion, 401, "Node reassignment.");
+        response.bodyOutputStream.write(body, body.length);
+        return;
+      }
+      handler(request, response);
+    };
+  }
+
+  let handlers = {
+    "/1.1/foo/info/collections": collectionsHelper.handler,
+    "/1.1/foo/storage/meta/global": upd("meta", global.handler()),
+    "/1.1/foo/storage/crypto/keys": upd("crypto", keysWBO.handler()),
+    "/1.1/foo/storage/clients": upd401("clients", clientsColl.handler()),
+    "/1.1/foo/storage/rotary": upd("rotary", rotaryColl.handler()),
+  };
+
+  let server = sync_httpd_setup(handlers);
+  // Do not instantiate SyncTestingInfrastructure; we need real crypto.
+  await configureIdentity({ username: "foo" }, server);
+
+  _("Syncing.");
+  // First hit of clients will 401. This will happen after meta/global and
+  // keys -- i.e., in the middle of the sync, but before RotaryEngine.
+  should401 = true;
+
+  // Use observers to perform actions when our sync finishes.
+  // This allows us to observe the automatic next-tick sync that occurs after
+  // an abort.
+  function onSyncError() {
+    do_throw("Should not get a sync error!");
+  }
+  let onSyncFinished = function () {};
+  let obs = {
+    observe: function observe(subject, topic) {
+      switch (topic) {
+        case "weave:service:sync:error":
+          onSyncError();
+          break;
+        case "weave:service:sync:finish":
+          onSyncFinished();
+          break;
+      }
+    },
+  };
+
+  Svc.Obs.add("weave:service:sync:finish", obs);
+  Svc.Obs.add("weave:service:sync:error", obs);
+
+  // This kicks off the actual test. Split into a function here to allow this
+  // source file to broadly follow actual execution order.
+  async function onwards() {
+    _("== Invoking first sync.");
+    await Service.sync();
+    _("We should not simultaneously have data but no keys on the server.");
+    let hasData = rotaryColl.wbo("flying") || rotaryColl.wbo("scotsman");
+    let hasKeys = keysWBO.modified;
+
+    _("We correctly handle 401s by aborting the sync and starting again.");
+    Assert.equal(!hasData, !hasKeys);
+
+    _("Be prepared for the second (automatic) sync...");
+  }
+
+  _("Make sure that syncing again causes recovery.");
+  let callbacksPromise = new Promise(resolve => {
+    onSyncFinished = function () {
+      _("== First sync done.");
+      _("---------------------------");
+      onSyncFinished = function () {
+        _("== Second (automatic) sync done.");
+        let hasData = rotaryColl.wbo("flying") || rotaryColl.wbo("scotsman");
+        let hasKeys = keysWBO.modified;
+        Assert.equal(!hasData, !hasKeys);
+
+        // Kick off another sync. Can't just call it, because we're inside the
+        // lock...
+        (async () => {
+          await Async.promiseYield();
+          _("Now a fresh sync will get no HMAC errors.");
+          _(
+            "Partially resetting client, as if after a restart, and forcing redownload."
+          );
+          Service.collectionKeys.clear();
+          await engine.setLastSync(0);
+          hmacErrorCount = 0;
+
+          onSyncFinished = async function () {
+            // Two rotary items, one client record... no errors.
+            Assert.equal(hmacErrorCount, 0);
+
+            Svc.Obs.remove("weave:service:sync:finish", obs);
+            Svc.Obs.remove("weave:service:sync:error", obs);
+
+            await tracker.clearChangedIDs();
+            await Service.engineManager.unregister(engine);
+            for (const pref of Svc.PrefBranch.getChildList("")) {
+              Svc.PrefBranch.clearUserPref(pref);
+            }
+            Service.recordManager.clearCache();
+            server.stop(resolve);
+          };
+
+          Service.sync();
+        })().catch(console.error);
+      };
+    };
+  });
+  await onwards();
+  await callbacksPromise;
+});

src/zen/tests/mochitests/sync/test_httpd_sync_server.js

@@ -0,0 +1,250 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_test(function test_creation() {
+  // Explicit callback for this one.
+  let server = new SyncServer(Object.create(SyncServerCallback));
+  Assert.ok(!!server); // Just so we have a check.
+  server.start(null, function () {
+    _("Started on " + server.port);
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_url_parsing() {
+  let server = new SyncServer();
+
+  // Check that we can parse a WBO URI.
+  let parts = server.pathRE.exec("/1.1/johnsmith/storage/crypto/keys");
+  let [all, version, username, first, rest] = parts;
+  Assert.equal(all, "/1.1/johnsmith/storage/crypto/keys");
+  Assert.equal(version, "1.1");
+  Assert.equal(username, "johnsmith");
+  Assert.equal(first, "storage");
+  Assert.equal(rest, "crypto/keys");
+  Assert.equal(null, server.pathRE.exec("/nothing/else"));
+
+  // Check that we can parse a collection URI.
+  parts = server.pathRE.exec("/1.1/johnsmith/storage/crypto");
+  [all, version, username, first, rest] = parts;
+  Assert.equal(all, "/1.1/johnsmith/storage/crypto");
+  Assert.equal(version, "1.1");
+  Assert.equal(username, "johnsmith");
+  Assert.equal(first, "storage");
+  Assert.equal(rest, "crypto");
+
+  // We don't allow trailing slash on storage URI.
+  parts = server.pathRE.exec("/1.1/johnsmith/storage/");
+  Assert.equal(parts, undefined);
+
+  // storage alone is a valid request.
+  parts = server.pathRE.exec("/1.1/johnsmith/storage");
+  [all, version, username, first, rest] = parts;
+  Assert.equal(all, "/1.1/johnsmith/storage");
+  Assert.equal(version, "1.1");
+  Assert.equal(username, "johnsmith");
+  Assert.equal(first, "storage");
+  Assert.equal(rest, undefined);
+
+  parts = server.storageRE.exec("storage");
+  let collection;
+  [all, , collection] = parts;
+  Assert.equal(all, "storage");
+  Assert.equal(collection, undefined);
+
+  run_next_test();
+});
+
+const { RESTRequest } = ChromeUtils.importESModule(
+  "resource://services-common/rest.sys.mjs"
+);
+function localRequest(server, path) {
+  _("localRequest: " + path);
+  let url = server.baseURI.substr(0, server.baseURI.length - 1) + path;
+  _("url: " + url);
+  return new RESTRequest(url);
+}
+
+add_task(async function test_basic_http() {
+  let server = new SyncServer();
+  server.registerUser("john", "password");
+  Assert.ok(server.userExists("john"));
+  server.start();
+  _("Started on " + server.port);
+
+  let req = localRequest(server, "/1.1/john/storage/crypto/keys");
+  _("req is " + req);
+  // Shouldn't reject, beyond that we don't care.
+  await req.get();
+
+  await promiseStopServer(server);
+});
+
+add_task(async function test_info_collections() {
+  let server = new SyncServer(Object.create(SyncServerCallback));
+  function responseHasCorrectHeaders(r) {
+    Assert.equal(r.status, 200);
+    Assert.equal(r.headers["content-type"], "application/json");
+    Assert.ok("x-weave-timestamp" in r.headers);
+  }
+
+  server.registerUser("john", "password");
+  server.start();
+
+  let req = localRequest(server, "/1.1/john/info/collections");
+  await req.get();
+  responseHasCorrectHeaders(req.response);
+  Assert.equal(req.response.body, "{}");
+
+  let putReq = localRequest(server, "/1.1/john/storage/crypto/keys");
+  let payload = JSON.stringify({ foo: "bar" });
+  let putResp = await putReq.put(payload);
+
+  responseHasCorrectHeaders(putResp);
+
+  let putResponseBody = putResp.body;
+  _("PUT response body: " + JSON.stringify(putResponseBody));
+
+  // When we PUT something to crypto/keys, "crypto" appears in the response.
+  req = localRequest(server, "/1.1/john/info/collections");
+
+  await req.get();
+  responseHasCorrectHeaders(req.response);
+  let expectedColl = server.getCollection("john", "crypto");
+  Assert.ok(!!expectedColl);
+  let modified = expectedColl.timestamp;
+  Assert.greater(modified, 0);
+  Assert.equal(putResponseBody, modified);
+  Assert.equal(JSON.parse(req.response.body).crypto, modified);
+
+  await promiseStopServer(server);
+});
+
+add_task(async function test_storage_request() {
+  let keysURL = "/1.1/john/storage/crypto/keys?foo=bar";
+  let foosURL = "/1.1/john/storage/crypto/foos";
+  let storageURL = "/1.1/john/storage";
+
+  let server = new SyncServer();
+  let creation = server.timestamp();
+  server.registerUser("john", "password");
+
+  server.createContents("john", {
+    crypto: { foos: { foo: "bar" } },
+  });
+  let coll = server.user("john").collection("crypto");
+  Assert.ok(!!coll);
+
+  _("We're tracking timestamps.");
+  Assert.greaterOrEqual(coll.timestamp, creation);
+
+  async function retrieveWBONotExists() {
+    let req = localRequest(server, keysURL);
+    let response = await req.get();
+    _("Body is " + response.body);
+    _("Modified is " + response.newModified);
+    Assert.equal(response.status, 404);
+    Assert.equal(response.body, "Not found");
+  }
+
+  async function retrieveWBOExists() {
+    let req = localRequest(server, foosURL);
+    let response = await req.get();
+    _("Body is " + response.body);
+    _("Modified is " + response.newModified);
+    let parsedBody = JSON.parse(response.body);
+    Assert.equal(parsedBody.id, "foos");
+    Assert.equal(parsedBody.modified, coll.wbo("foos").modified);
+    Assert.equal(JSON.parse(parsedBody.payload).foo, "bar");
+  }
+
+  async function deleteWBONotExists() {
+    let req = localRequest(server, keysURL);
+    server.callback.onItemDeleted = function () {
+      do_throw("onItemDeleted should not have been called.");
+    };
+
+    let response = await req.delete();
+
+    _("Body is " + response.body);
+    _("Modified is " + response.newModified);
+    Assert.equal(response.status, 200);
+    delete server.callback.onItemDeleted;
+  }
+
+  async function deleteWBOExists() {
+    let req = localRequest(server, foosURL);
+    server.callback.onItemDeleted = function (username, collection, wboID) {
+      _("onItemDeleted called for " + collection + "/" + wboID);
+      delete server.callback.onItemDeleted;
+      Assert.equal(username, "john");
+      Assert.equal(collection, "crypto");
+      Assert.equal(wboID, "foos");
+    };
+    await req.delete();
+    _("Body is " + req.response.body);
+    _("Modified is " + req.response.newModified);
+    Assert.equal(req.response.status, 200);
+  }
+
+  async function deleteStorage() {
+    _("Testing DELETE on /storage.");
+    let now = server.timestamp();
+    _("Timestamp: " + now);
+    let req = localRequest(server, storageURL);
+    await req.delete();
+
+    _("Body is " + req.response.body);
+    _("Modified is " + req.response.newModified);
+    let parsedBody = JSON.parse(req.response.body);
+    Assert.greaterOrEqual(parsedBody, now);
+    do_check_empty(server.users.john.collections);
+  }
+
+  async function getStorageFails() {
+    _("Testing that GET on /storage fails.");
+    let req = localRequest(server, storageURL);
+    await req.get();
+    Assert.equal(req.response.status, 405);
+    Assert.equal(req.response.headers.allow, "DELETE");
+  }
+
+  async function getMissingCollectionWBO() {
+    _("Testing that fetching a WBO from an on-existent collection 404s.");
+    let req = localRequest(server, storageURL + "/foobar/baz");
+    await req.get();
+    Assert.equal(req.response.status, 404);
+  }
+
+  server.start(null);
+
+  await retrieveWBONotExists();
+  await retrieveWBOExists();
+  await deleteWBOExists();
+  await deleteWBONotExists();
+  await getStorageFails();
+  await getMissingCollectionWBO();
+  await deleteStorage();
+
+  await promiseStopServer(server);
+});
+
+add_task(async function test_x_weave_records() {
+  let server = new SyncServer();
+  server.registerUser("john", "password");
+
+  server.createContents("john", {
+    crypto: { foos: { foo: "bar" }, bars: { foo: "baz" } },
+  });
+  server.start();
+
+  let wbo = localRequest(server, "/1.1/john/storage/crypto/foos");
+  await wbo.get();
+  Assert.equal(false, "x-weave-records" in wbo.response.headers);
+  let col = localRequest(server, "/1.1/john/storage/crypto");
+  await col.get();
+  // Collection fetches do.
+  Assert.equal(col.response.headers["x-weave-records"], "2");
+
+  await promiseStopServer(server);
+});

src/zen/tests/mochitests/sync/test_interval_triggers.js

@@ -0,0 +1,472 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Svc.PrefBranch.setStringPref("registerEngines", "");
+const { Service } = ChromeUtils.importESModule(
+  "resource://services-sync/service.sys.mjs"
+);
+
+let scheduler;
+let clientsEngine;
+
+async function sync_httpd_setup() {
+  let clientsSyncID = await clientsEngine.resetLocalSyncID();
+  let global = new ServerWBO("global", {
+    syncID: Service.syncID,
+    storageVersion: STORAGE_VERSION,
+    engines: {
+      clients: { version: clientsEngine.version, syncID: clientsSyncID },
+    },
+  });
+  let clientsColl = new ServerCollection({}, true);
+
+  // Tracking info/collections.
+  let collectionsHelper = track_collections_helper();
+  let upd = collectionsHelper.with_updated_collection;
+
+  return httpd_setup({
+    "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()),
+    "/1.1/johndoe/info/collections": collectionsHelper.handler,
+    "/1.1/johndoe/storage/crypto/keys": upd(
+      "crypto",
+      new ServerWBO("keys").handler()
+    ),
+    "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()),
+  });
+}
+
+async function setUp(server) {
+  syncTestLogging();
+  await configureIdentity({ username: "johndoe" }, server);
+  await generateNewKeys(Service.collectionKeys);
+  let serverKeys = Service.collectionKeys.asWBO("crypto", "keys");
+  await serverKeys.encrypt(Service.identity.syncKeyBundle);
+  await serverKeys.upload(Service.resource(Service.cryptoKeysURL));
+}
+
+add_task(async function setup() {
+  scheduler = Service.scheduler;
+  clientsEngine = Service.clientsEngine;
+
+  // Don't remove stale clients when syncing. This is a test-only workaround
+  // that lets us add clients directly to the store, without losing them on
+  // the next sync.
+  clientsEngine._removeRemoteClient = async () => {};
+});
+
+add_task(async function test_successful_sync_adjustSyncInterval() {
+  enableValidationPrefs();
+
+  _("Test successful sync calling adjustSyncInterval");
+  let syncSuccesses = 0;
+  function onSyncFinish() {
+    _("Sync success.");
+    syncSuccesses++;
+  }
+  Svc.Obs.add("weave:service:sync:finish", onSyncFinish);
+
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  // Confirm defaults
+  Assert.ok(!scheduler.idle);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+  Assert.ok(!scheduler.hasIncomingItems);
+
+  _("Test as long as numClients <= 1 our sync interval is SINGLE_USER.");
+  // idle == true && numClients <= 1 && hasIncomingItems == false
+  scheduler.idle = true;
+  await Service.sync();
+  Assert.equal(syncSuccesses, 1);
+  Assert.ok(scheduler.idle);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.ok(!scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+
+  // idle == false && numClients <= 1 && hasIncomingItems == false
+  scheduler.idle = false;
+  await Service.sync();
+  Assert.equal(syncSuccesses, 2);
+  Assert.ok(!scheduler.idle);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.ok(!scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+
+  // idle == false && numClients <= 1 && hasIncomingItems == true
+  scheduler.hasIncomingItems = true;
+  await Service.sync();
+  Assert.equal(syncSuccesses, 3);
+  Assert.ok(!scheduler.idle);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.ok(scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+
+  // idle == true && numClients <= 1 && hasIncomingItems == true
+  scheduler.idle = true;
+  await Service.sync();
+  Assert.equal(syncSuccesses, 4);
+  Assert.ok(scheduler.idle);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.ok(scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+
+  _(
+    "Test as long as idle && numClients > 1 our sync interval is idleInterval."
+  );
+  // idle == true && numClients > 1 && hasIncomingItems == true
+  await Service.clientsEngine._store.create({
+    id: "foo",
+    cleartext: { name: "bar", type: "mobile" },
+  });
+  await Service.sync();
+  Assert.equal(syncSuccesses, 5);
+  Assert.ok(scheduler.idle);
+  Assert.greater(scheduler.numClients, 1);
+  Assert.ok(scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.idleInterval);
+
+  // idle == true && numClients > 1 && hasIncomingItems == false
+  scheduler.hasIncomingItems = false;
+  await Service.sync();
+  Assert.equal(syncSuccesses, 6);
+  Assert.ok(scheduler.idle);
+  Assert.greater(scheduler.numClients, 1);
+  Assert.ok(!scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.idleInterval);
+
+  _("Test non-idle, numClients > 1, no incoming items => activeInterval.");
+  // idle == false && numClients > 1 && hasIncomingItems == false
+  scheduler.idle = false;
+  await Service.sync();
+  Assert.equal(syncSuccesses, 7);
+  Assert.ok(!scheduler.idle);
+  Assert.greater(scheduler.numClients, 1);
+  Assert.ok(!scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.activeInterval);
+
+  _("Test non-idle, numClients > 1, incoming items => immediateInterval.");
+  // idle == false && numClients > 1 && hasIncomingItems == true
+  scheduler.hasIncomingItems = true;
+  await Service.sync();
+  Assert.equal(syncSuccesses, 8);
+  Assert.ok(!scheduler.idle);
+  Assert.greater(scheduler.numClients, 1);
+  Assert.ok(!scheduler.hasIncomingItems); // gets reset to false
+  Assert.equal(scheduler.syncInterval, scheduler.immediateInterval);
+
+  Svc.Obs.remove("weave:service:sync:finish", onSyncFinish);
+  await Service.startOver();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_unsuccessful_sync_adjustSyncInterval() {
+  enableValidationPrefs();
+
+  _("Test unsuccessful sync calling adjustSyncInterval");
+
+  let syncFailures = 0;
+  function onSyncError() {
+    _("Sync error.");
+    syncFailures++;
+  }
+  Svc.Obs.add("weave:service:sync:error", onSyncError);
+
+  _("Test unsuccessful sync calls adjustSyncInterval");
+  // Force sync to fail.
+  Svc.PrefBranch.setStringPref("firstSync", "notReady");
+
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  // Confirm defaults
+  Assert.ok(!scheduler.idle);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+  Assert.ok(!scheduler.hasIncomingItems);
+
+  _("Test as long as numClients <= 1 our sync interval is SINGLE_USER.");
+  // idle == true && numClients <= 1 && hasIncomingItems == false
+  scheduler.idle = true;
+  await Service.sync();
+  Assert.equal(syncFailures, 1);
+  Assert.ok(scheduler.idle);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.ok(!scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+
+  // idle == false && numClients <= 1 && hasIncomingItems == false
+  scheduler.idle = false;
+  await Service.sync();
+  Assert.equal(syncFailures, 2);
+  Assert.ok(!scheduler.idle);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.ok(!scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+
+  // idle == false && numClients <= 1 && hasIncomingItems == true
+  scheduler.hasIncomingItems = true;
+  await Service.sync();
+  Assert.equal(syncFailures, 3);
+  Assert.ok(!scheduler.idle);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.ok(scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+
+  // idle == true && numClients <= 1 && hasIncomingItems == true
+  scheduler.idle = true;
+  await Service.sync();
+  Assert.equal(syncFailures, 4);
+  Assert.ok(scheduler.idle);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.ok(scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+
+  _(
+    "Test as long as idle && numClients > 1 our sync interval is idleInterval."
+  );
+  // idle == true && numClients > 1 && hasIncomingItems == true
+  Svc.PrefBranch.setIntPref("clients.devices.mobile", 2);
+  scheduler.updateClientMode();
+
+  await Service.sync();
+  Assert.equal(syncFailures, 5);
+  Assert.ok(scheduler.idle);
+  Assert.greater(scheduler.numClients, 1);
+  Assert.ok(scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.idleInterval);
+
+  // idle == true && numClients > 1 && hasIncomingItems == false
+  scheduler.hasIncomingItems = false;
+  await Service.sync();
+  Assert.equal(syncFailures, 6);
+  Assert.ok(scheduler.idle);
+  Assert.greater(scheduler.numClients, 1);
+  Assert.ok(!scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.idleInterval);
+
+  _("Test non-idle, numClients > 1, no incoming items => activeInterval.");
+  // idle == false && numClients > 1 && hasIncomingItems == false
+  scheduler.idle = false;
+  await Service.sync();
+  Assert.equal(syncFailures, 7);
+  Assert.ok(!scheduler.idle);
+  Assert.greater(scheduler.numClients, 1);
+  Assert.ok(!scheduler.hasIncomingItems);
+  Assert.equal(scheduler.syncInterval, scheduler.activeInterval);
+
+  _("Test non-idle, numClients > 1, incoming items => immediateInterval.");
+  // idle == false && numClients > 1 && hasIncomingItems == true
+  scheduler.hasIncomingItems = true;
+  await Service.sync();
+  Assert.equal(syncFailures, 8);
+  Assert.ok(!scheduler.idle);
+  Assert.greater(scheduler.numClients, 1);
+  Assert.ok(!scheduler.hasIncomingItems); // gets reset to false
+  Assert.equal(scheduler.syncInterval, scheduler.immediateInterval);
+
+  await Service.startOver();
+  Svc.Obs.remove("weave:service:sync:error", onSyncError);
+  await promiseStopServer(server);
+});
+
+add_task(async function test_back_triggers_sync() {
+  enableValidationPrefs();
+
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  // Single device: no sync triggered.
+  scheduler.idle = true;
+  scheduler.observe(
+    null,
+    "active",
+    Svc.PrefBranch.getIntPref("scheduler.idleTime")
+  );
+  Assert.ok(!scheduler.idle);
+
+  // Multiple devices: sync is triggered.
+  Svc.PrefBranch.setIntPref("clients.devices.mobile", 2);
+  scheduler.updateClientMode();
+
+  let promiseDone = promiseOneObserver("weave:service:sync:finish");
+
+  scheduler.idle = true;
+  scheduler.observe(
+    null,
+    "active",
+    Svc.PrefBranch.getIntPref("scheduler.idleTime")
+  );
+  Assert.ok(!scheduler.idle);
+  await promiseDone;
+
+  Service.recordManager.clearCache();
+  for (const pref of Svc.PrefBranch.getChildList("")) {
+    Svc.PrefBranch.clearUserPref(pref);
+  }
+  scheduler.setDefaults();
+  await clientsEngine.resetClient();
+
+  await Service.startOver();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_adjust_interval_on_sync_error() {
+  enableValidationPrefs();
+
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  let syncFailures = 0;
+  function onSyncError() {
+    _("Sync error.");
+    syncFailures++;
+  }
+  Svc.Obs.add("weave:service:sync:error", onSyncError);
+
+  _("Test unsuccessful sync updates client mode & sync intervals");
+  // Force a sync fail.
+  Svc.PrefBranch.setStringPref("firstSync", "notReady");
+
+  Assert.equal(syncFailures, 0);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+
+  Svc.PrefBranch.setIntPref("clients.devices.mobile", 2);
+  await Service.sync();
+
+  Assert.equal(syncFailures, 1);
+  Assert.greater(scheduler.numClients, 1);
+  Assert.equal(scheduler.syncInterval, scheduler.activeInterval);
+
+  Svc.Obs.remove("weave:service:sync:error", onSyncError);
+  await Service.startOver();
+  await promiseStopServer(server);
+});
+
+add_task(async function test_bug671378_scenario() {
+  enableValidationPrefs();
+
+  // Test scenario similar to bug 671378. This bug appeared when a score
+  // update occurred that wasn't large enough to trigger a sync so
+  // scheduleNextSync() was called without a time interval parameter,
+  // setting nextSync to a non-zero value and preventing the timer from
+  // being adjusted in the next call to scheduleNextSync().
+  let server = await sync_httpd_setup();
+  await setUp(server);
+
+  let syncSuccesses = 0;
+  function onSyncFinish() {
+    _("Sync success.");
+    syncSuccesses++;
+  }
+  Svc.Obs.add("weave:service:sync:finish", onSyncFinish);
+
+  // After first sync call, syncInterval & syncTimer are singleDeviceInterval.
+  await Service.sync();
+  Assert.equal(syncSuccesses, 1);
+  Assert.equal(false, scheduler.numClients > 1);
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+  Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval);
+
+  let promiseDone = new Promise(resolve => {
+    // Wrap scheduleNextSync so we are notified when it is finished.
+    scheduler._scheduleNextSync = scheduler.scheduleNextSync;
+    scheduler.scheduleNextSync = function () {
+      scheduler._scheduleNextSync();
+
+      // Check on sync:finish scheduleNextSync sets the appropriate
+      // syncInterval and syncTimer values.
+      if (syncSuccesses == 2) {
+        Assert.notEqual(scheduler.nextSync, 0);
+        Assert.equal(scheduler.syncInterval, scheduler.activeInterval);
+        Assert.lessOrEqual(scheduler.syncTimer.delay, scheduler.activeInterval);
+
+        scheduler.scheduleNextSync = scheduler._scheduleNextSync;
+        Svc.Obs.remove("weave:service:sync:finish", onSyncFinish);
+        Service.startOver().then(() => {
+          server.stop(resolve);
+        });
+      }
+    };
+  });
+
+  // Set nextSync != 0
+  // syncInterval still hasn't been set by call to updateClientMode.
+  // Explicitly trying to invoke scheduleNextSync during a sync
+  // (to immitate a score update that isn't big enough to trigger a sync).
+  Svc.Obs.add("weave:service:sync:start", function onSyncStart() {
+    // Wait for other sync:start observers to be called so that
+    // nextSync is set to 0.
+    CommonUtils.nextTick(function () {
+      Svc.Obs.remove("weave:service:sync:start", onSyncStart);
+
+      scheduler.scheduleNextSync();
+      Assert.notEqual(scheduler.nextSync, 0);
+      Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+      Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval);
+    });
+  });
+
+  await Service.clientsEngine._store.create({
+    id: "foo",
+    cleartext: { name: "bar", type: "mobile" },
+  });
+  await Service.sync();
+  await promiseDone;
+});
+
+add_task(async function test_adjust_timer_larger_syncInterval() {
+  _(
+    "Test syncInterval > current timout period && nextSync != 0, syncInterval is NOT used."
+  );
+  Svc.PrefBranch.setIntPref("clients.devices.mobile", 2);
+  scheduler.updateClientMode();
+  Assert.equal(scheduler.syncInterval, scheduler.activeInterval);
+
+  scheduler.scheduleNextSync();
+
+  // Ensure we have a small interval.
+  Assert.notEqual(scheduler.nextSync, 0);
+  Assert.equal(scheduler.syncTimer.delay, scheduler.activeInterval);
+
+  // Make interval large again
+  await clientsEngine._wipeClient();
+  Svc.PrefBranch.clearUserPref("clients.devices.mobile");
+  scheduler.updateClientMode();
+  Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval);
+
+  scheduler.scheduleNextSync();
+
+  // Ensure timer delay remains as the small interval.
+  Assert.notEqual(scheduler.nextSync, 0);
+  Assert.lessOrEqual(scheduler.syncTimer.delay, scheduler.activeInterval);
+
+  // SyncSchedule.
+  await Service.startOver();
+});
+
+add_task(async function test_adjust_timer_smaller_syncInterval() {
+  _(
+    "Test current timout > syncInterval period && nextSync != 0, syncInterval is used."
+  );
+  scheduler.scheduleNextSync();
+
+  // Ensure we have a large interval.
+  Assert.notEqual(scheduler.nextSync, 0);
+  Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval);
+
+  // Make interval smaller
+  Svc.PrefBranch.setIntPref("clients.devices.mobile", 2);
+  scheduler.updateClientMode();
+  Assert.equal(scheduler.syncInterval, scheduler.activeInterval);
+
+  scheduler.scheduleNextSync();
+
+  // Ensure smaller timer delay is used.
+  Assert.notEqual(scheduler.nextSync, 0);
+  Assert.lessOrEqual(scheduler.syncTimer.delay, scheduler.activeInterval);
+
+  // SyncSchedule.
+  await Service.startOver();
+});