diff --git a/config/default.json b/config/default.json index 103dfcadd..44f1dc4da 100644 --- a/config/default.json +++ b/config/default.json @@ -1,5 +1,6 @@ { "serverUrl": "https://textsecure-service-staging.whispersystems.org", + "storageUrl": "https://storage-staging.signal.org", "cdn": { "0": "https://cdn-staging.signal.org", "2": "https://cdn2-staging.signal.org" diff --git a/config/production.json b/config/production.json index d8a203932..ee0d3e144 100644 --- a/config/production.json +++ b/config/production.json @@ -1,5 +1,6 @@ { "serverUrl": "https://textsecure-service.whispersystems.org", + "storageUrl": "https://storage.signal.org", "cdn": { "0": "https://cdn.signal.org", "2": "https://cdn2.signal.org" diff --git a/js/background.js b/js/background.js index 2a1b421c3..2ab80c420 100644 --- a/js/background.js +++ b/js/background.js @@ -1675,6 +1675,8 @@ addQueuedEventListener('viewSync', onViewSync); addQueuedEventListener('messageRequestResponse', onMessageRequestResponse); addQueuedEventListener('profileKeyUpdate', onProfileKeyUpdate); + addQueuedEventListener('fetchLatest', onFetchLatestSync); + addQueuedEventListener('keys', onKeysSync); window.Signal.AttachmentDownloads.start({ getMessageReceiver: () => messageReceiver, @@ -1688,6 +1690,7 @@ if (connectCount === 1) { window.Signal.Stickers.downloadQueuedPacks(); + await window.textsecure.messaging.sendRequestKeySyncMessage(); } // On startup after upgrading to a new version, request a contact sync @@ -2728,6 +2731,45 @@ Whisper.ViewSyncs.onSync(sync); } + async function onFetchLatestSync(ev) { + ev.confirm(); + + const { eventType } = ev; + + const FETCH_LATEST_ENUM = textsecure.protobuf.SyncMessage.FetchLatest.Type; + + switch (eventType) { + case FETCH_LATEST_ENUM.LOCAL_PROFILE: + // Intentionally do nothing since we'll be receiving the storage manifest request + // and will update local profile along with that. + break; + case FETCH_LATEST_ENUM.STORAGE_MANIFEST: + window.log.info('onFetchLatestSync: fetching latest manifest'); + await window.Signal.Util.runStorageServiceSyncJob(); + break; + default: + window.log.info( + `onFetchLatestSync: Unknown type encountered ${eventType}` + ); + } + } + + async function onKeysSync(ev) { + ev.confirm(); + + const { storageServiceKey } = ev; + + if (storageServiceKey) { + window.log.info('onKeysSync: received keys'); + const storageServiceKeyBase64 = window.Signal.Crypto.arrayBufferToBase64( + storageServiceKey + ); + storage.put('storageKey', storageServiceKeyBase64); + + await window.Signal.Util.runStorageServiceSyncJob(); + } + } + async function onMessageRequestResponse(ev) { ev.confirm(); diff --git a/js/models/conversations.js b/js/models/conversations.js index 639f365c7..aaafd7f0b 100644 --- a/js/models/conversations.js +++ b/js/models/conversations.js @@ -39,6 +39,14 @@ writeNewAttachmentData, } = window.Signal.Migrations; const { addStickerPackReference } = window.Signal.Data; + const { + arrayBufferToBase64, + base64ToArrayBuffer, + deriveAccessKey, + getRandomBytes, + stringFromBytes, + verifyAccessKey, + } = window.Signal.Crypto; const COLORS = [ 'red', @@ -760,6 +768,8 @@ verified ); } + + return keyChange; }, sendVerifySyncMessage(e164, uuid, state) { // Because syncVerification sends a (null) message to the target of the verify and @@ -1754,11 +1764,7 @@ // If we've never fetched user's profile, we default to what we have if (sealedSender === SEALED_SENDER.UNKNOWN) { const info = { - accessKey: - accessKey || - window.Signal.Crypto.arrayBufferToBase64( - window.Signal.Crypto.getRandomBytes(16) - ), + accessKey: accessKey || arrayBufferToBase64(getRandomBytes(16)), // Indicates that a client is capable of receiving uuid-only messages. // Not used yet. uuidCapable, @@ -1777,9 +1783,7 @@ accessKey: accessKey && sealedSender === SEALED_SENDER.ENABLED ? accessKey - : window.Signal.Crypto.arrayBufferToBase64( - window.Signal.Crypto.getRandomBytes(16) - ), + : arrayBufferToBase64(getRandomBytes(16)), // Indicates that a client is capable of receiving uuid-only messages. // Not used yet. uuidCapable, @@ -2343,9 +2347,7 @@ }); } - const identityKey = window.Signal.Crypto.base64ToArrayBuffer( - profile.identityKey - ); + const identityKey = base64ToArrayBuffer(profile.identityKey); const changed = await textsecure.storage.protocol.saveIdentity( `${id}.1`, identityKey, @@ -2375,9 +2377,9 @@ sealedSender: SEALED_SENDER.UNRESTRICTED, }); } else if (accessKey && profile.unidentifiedAccess) { - const haveCorrectKey = await window.Signal.Crypto.verifyAccessKey( - window.Signal.Crypto.base64ToArrayBuffer(accessKey), - window.Signal.Crypto.base64ToArrayBuffer(profile.unidentifiedAccess) + const haveCorrectKey = await verifyAccessKey( + base64ToArrayBuffer(accessKey), + base64ToArrayBuffer(profile.unidentifiedAccess) ); if (haveCorrectKey) { @@ -2466,8 +2468,8 @@ } // decode - const keyBuffer = window.Signal.Crypto.base64ToArrayBuffer(key); - const data = window.Signal.Crypto.base64ToArrayBuffer(encryptedName); + const keyBuffer = base64ToArrayBuffer(key); + const data = base64ToArrayBuffer(encryptedName); // decrypt const { given, family } = await textsecure.crypto.decryptProfileName( @@ -2476,10 +2478,8 @@ ); // encode - const profileName = window.Signal.Crypto.stringFromBytes(given); - const profileFamilyName = family - ? window.Signal.Crypto.stringFromBytes(family) - : null; + const profileFamilyName = family ? stringFromBytes(family) : null; + const profileName = given ? stringFromBytes(given) : null; // set this.set({ profileName, profileFamilyName }); @@ -2494,7 +2494,7 @@ if (!key) { return; } - const keyBuffer = window.Signal.Crypto.base64ToArrayBuffer(key); + const keyBuffer = base64ToArrayBuffer(key); // decrypt const decrypted = await textsecure.crypto.decryptProfile( @@ -2577,15 +2577,9 @@ return; } - const profileKeyBuffer = window.Signal.Crypto.base64ToArrayBuffer( - profileKey - ); - const accessKeyBuffer = await window.Signal.Crypto.deriveAccessKey( - profileKeyBuffer - ); - const accessKey = window.Signal.Crypto.arrayBufferToBase64( - accessKeyBuffer - ); + const profileKeyBuffer = base64ToArrayBuffer(profileKey); + const accessKeyBuffer = await deriveAccessKey(profileKeyBuffer); + const accessKey = arrayBufferToBase64(accessKeyBuffer); this.set({ accessKey }); }, async deriveProfileKeyVersionIfNeeded() { diff --git a/libtextsecure/protobufs.js b/libtextsecure/protobufs.js index b9952d087..e67df0961 100644 --- a/libtextsecure/protobufs.js +++ b/libtextsecure/protobufs.js @@ -31,6 +31,7 @@ } loadProtoBufs('SignalService.proto'); + loadProtoBufs('SignalStorage.proto'); loadProtoBufs('SubProtocol.proto'); loadProtoBufs('DeviceMessages.proto'); loadProtoBufs('Stickers.proto'); diff --git a/main.js b/main.js index 0400a34e1..aa305c342 100644 --- a/main.js +++ b/main.js @@ -191,6 +191,7 @@ function prepareURL(pathSegments, moreKeys) { version: app.getVersion(), buildExpiration: config.get('buildExpiration'), serverUrl: config.get('serverUrl'), + storageUrl: config.get('storageUrl'), cdnUrl0: config.get('cdn').get('0'), cdnUrl2: config.get('cdn').get('2'), certificateAuthority: config.get('certificateAuthority'), diff --git a/preload.js b/preload.js index 56f64baca..98e9b4bbf 100644 --- a/preload.js +++ b/preload.js @@ -316,6 +316,7 @@ try { window.WebAPI = window.textsecure.WebAPI.initialize({ url: config.serverUrl, + storageUrl: config.storageUrl, cdnUrlObject: { '0': config.cdnUrl0, '2': config.cdnUrl2, diff --git a/protos/SignalService.proto b/protos/SignalService.proto index f111b2eae..1bc3abc60 100644 --- a/protos/SignalService.proto +++ b/protos/SignalService.proto @@ -24,7 +24,6 @@ message Envelope { optional bytes content = 8; // Contains an encrypted Content optional string serverGuid = 9; optional uint64 serverTimestamp = 10; - } message Content { @@ -185,10 +184,10 @@ message DataMessage { } message Sticker { - optional bytes packId = 1; - optional bytes packKey = 2; - optional uint32 stickerId = 3; - optional AttachmentPointer data = 4; + optional bytes packId = 1; + optional bytes packKey = 2; + optional uint32 stickerId = 3; + optional AttachmentPointer data = 4; } message Reaction { @@ -299,9 +298,9 @@ message SyncMessage { } message Blocked { - repeated string numbers = 1; - repeated string uuids = 3; - repeated bytes groupIds = 2; + repeated string numbers = 1; + repeated string uuids = 3; + repeated bytes groupIds = 2; } message Request { @@ -311,11 +310,16 @@ message SyncMessage { GROUPS = 2; BLOCKED = 3; CONFIGURATION = 4; + KEYS = 5; } optional Type type = 1; } + message Keys { + optional bytes storageService = 1; + } + message Read { optional string sender = 1; optional string senderUuid = 3; @@ -323,10 +327,11 @@ message SyncMessage { } message Configuration { - optional bool readReceipts = 1; - optional bool unidentifiedDeliveryIndicators = 2; - optional bool typingIndicators = 3; - optional bool linkPreviews = 4; + optional bool readReceipts = 1; + optional bool unidentifiedDeliveryIndicators = 2; + optional bool typingIndicators = 3; + optional bool linkPreviews = 4; + optional uint32 provisioningVersion = 5; } message StickerPackOperation { @@ -334,6 +339,7 @@ message SyncMessage { INSTALL = 0; REMOVE = 1; } + optional bytes packId = 1; optional bytes packKey = 2; optional Type type = 3; @@ -360,6 +366,16 @@ message SyncMessage { optional Type type = 4; } + message FetchLatest { + enum Type { + UNKNOWN = 0; + LOCAL_PROFILE = 1; + STORAGE_MANIFEST = 2; + } + + optional Type type = 1; + } + optional Sent sent = 1; optional Contacts contacts = 2; optional Groups groups = 3; @@ -371,6 +387,8 @@ message SyncMessage { optional bytes padding = 8; repeated StickerPackOperation stickerPackOperation = 10; optional ViewOnceOpen viewOnceOpen = 11; + optional FetchLatest fetchLatest = 12; + optional Keys keys = 13; optional MessageRequestResponse messageRequestResponse = 14; } diff --git a/protos/SignalStorage.proto b/protos/SignalStorage.proto new file mode 100644 index 000000000..8e4db5705 --- /dev/null +++ b/protos/SignalStorage.proto @@ -0,0 +1,102 @@ +package signalservice; + +option java_package = "org.whispersystems.signalservice.internal.storage"; +option java_outer_classname = "SignalStorageProtos"; + +message StorageManifest { + optional uint64 version = 1; + optional bytes value = 2; +} + +message StorageItem { + optional bytes key = 1; + optional bytes value = 2; +} + +message StorageItems { + repeated StorageItem items = 1; +} + +message ReadOperation { + repeated bytes readKey = 1; +} + +message WriteOperation { + optional StorageManifest manifest = 1; + repeated StorageItem insertItem = 2; + repeated bytes deleteKey = 3; + optional bool clearAll = 4; +} + +message ManifestRecord { + message Identifier { + enum Type { + UNKNOWN = 0; + CONTACT = 1; + GROUPV1 = 2; + GROUPV2 = 3; + ACCOUNT = 4; + } + + optional bytes raw = 1; + optional Type type = 2; + } + + optional uint64 version = 1; + repeated Identifier keys = 2; +} + +message StorageRecord { + oneof record { + ContactRecord contact = 1; + GroupV1Record groupV1 = 2; + GroupV2Record groupV2 = 3; + AccountRecord account = 4; + } +} + +message ContactRecord { + enum IdentityState { + DEFAULT = 0; + VERIFIED = 1; + UNVERIFIED = 2; + } + + optional string serviceUuid = 1; + optional string serviceE164 = 2; + optional bytes profileKey = 3; + optional bytes identityKey = 4; + optional IdentityState identityState = 5; + optional string givenName = 6; + optional string familyName = 7; + optional string username = 8; + optional bool blocked = 9; + optional bool whitelisted = 10; + optional bool archived = 11; +} + +message GroupV1Record { + optional bytes id = 1; + optional bool blocked = 2; + optional bool whitelisted = 3; + optional bool archived = 4; +} + +message GroupV2Record { + optional bytes masterKey = 1; + optional bool blocked = 2; + optional bool whitelisted = 3; + optional bool archived = 4; +} + +message AccountRecord { + optional bytes profileKey = 1; + optional string givenName = 2; + optional string familyName = 3; + optional string avatarUrl = 4; + optional bool noteToSelfArchived = 5; + optional bool readReceipts = 6; + optional bool sealedSenderIndicators = 7; + optional bool typingIndicators = 8; + optional bool linkPreviews = 9; +} diff --git a/sticker-creator/preload.js b/sticker-creator/preload.js index 1b77877f8..ea65fdf7a 100644 --- a/sticker-creator/preload.js +++ b/sticker-creator/preload.js @@ -34,6 +34,7 @@ const { initialize: initializeWebAPI } = require('../ts/textsecure/WebAPI'); const WebAPI = initializeWebAPI({ url: config.serverUrl, + storageUrl: config.storageUrl, cdnUrlObject: { '0': config.cdnUrl0, '2': config.cdnUrl2, diff --git a/ts/Crypto.ts b/ts/Crypto.ts index e91347e63..933d7e69f 100644 --- a/ts/Crypto.ts +++ b/ts/Crypto.ts @@ -184,6 +184,20 @@ export async function decryptFile( return decryptSymmetric(key, ciphertext); } +export async function deriveStorageManifestKey( + storageServiceKey: ArrayBuffer, + version: number +) { + return hmacSha256(storageServiceKey, bytesFromString(`Manifest_${version}`)); +} + +export async function deriveStorageItemKey( + storageServiceKey: ArrayBuffer, + itemID: string +) { + return hmacSha256(storageServiceKey, bytesFromString(`Item_${itemID}`)); +} + export async function deriveAccessKey(profileKey: ArrayBuffer) { const iv = getZeroes(12); const plaintext = getZeroes(16); diff --git a/ts/textsecure.d.ts b/ts/textsecure.d.ts index 3b926e1f3..bd2a0897a 100644 --- a/ts/textsecure.d.ts +++ b/ts/textsecure.d.ts @@ -25,6 +25,16 @@ export type UnprocessedType = { version: number; }; +export type StorageServiceCallOptionsType = { + credentials?: StorageServiceCredentials; + greaterThanVersion?: string; +}; + +export type StorageServiceCredentials = { + username: string; + password: string; +}; + export type TextSecureType = { createTaskWithTimeout: ( task: () => Promise, @@ -70,6 +80,14 @@ export type TextSecureType = { ) => Promise; }; messaging: { + getStorageCredentials: () => Promise; + getStorageManifest: ( + options: StorageServiceCallOptionsType + ) => Promise; + getStorageRecords: ( + data: ArrayBuffer, + options: StorageServiceCallOptionsType + ) => Promise; sendStickerPackSync: ( operations: Array<{ packId: string; @@ -125,18 +143,42 @@ export type StorageProtocolType = StorageType & { keyPair: KeyPairType, confirmed?: boolean ) => Promise; + loadIdentityKey: (identifier: string) => Promise; loadSignedPreKeys: () => Promise>; + processVerifiedMessage: ( + identifier: string, + verifiedStatus: number, + publicKey: ArrayBuffer + ) => Promise; saveIdentityWithAttributes: ( number: string, options: IdentityKeyRecord ) => Promise; + setVerified: ( + encodedAddress: string, + verifiedStatus: number, + publicKey?: ArrayBuffer + ) => Promise; removeSignedPreKey: (keyId: number) => Promise; removeAllData: () => Promise; }; // Protobufs -type ProtobufCollectionType = { +type StorageServiceProtobufTypes = { + AccountRecord: typeof AccountRecordClass; + ContactRecord: typeof ContactRecordClass; + GroupV1Record: typeof GroupV1RecordClass; + GroupV2Record: typeof GroupV2RecordClass; + ManifestRecord: typeof ManifestRecordClass; + ReadOperation: typeof ReadOperation; + StorageItem: typeof StorageItemClass; + StorageItems: typeof StorageItemsClass; + StorageManifest: typeof StorageManifest; + StorageRecord: typeof StorageRecordClass; +}; + +type ProtobufCollectionType = StorageServiceProtobufTypes & { AttachmentPointer: typeof AttachmentPointerClass; ContactDetails: typeof ContactDetailsClass; Content: typeof ContentClass; @@ -458,6 +500,33 @@ export declare namespace GroupDetailsClass { } } +declare enum ManifestType { + UNKNOWN, + CONTACT, + GROUPV1, + GROUPV2, + ACCOUNT, +} + +type ManifestRecordIdentifier = { + raw: ProtoBinaryType; + type: ManifestType; +}; + +export declare class ManifestRecordClass { + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => ManifestRecordClass; + + static Identifier: { + Type: typeof ManifestType; + }; + + version: ProtoBigNumberType; + keys: ManifestRecordIdentifier[]; +} + export declare class NullMessageClass { static decode: ( data: ArrayBuffer | ByteBufferClass, @@ -526,6 +595,127 @@ export declare namespace ReceiptMessageClass { } } +// Storage Service related types + +declare class StorageManifest { + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => StorageManifest; + + version?: ProtoBigNumberType | null; + value?: ByteBufferClass | null; +} + +export declare class StorageRecordClass { + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => StorageRecordClass; + + contact?: ContactRecordClass | null; + groupV1?: GroupV1RecordClass | null; + groupV2?: GroupV2RecordClass | null; + account?: AccountRecordClass | null; +} + +export declare class StorageItemClass { + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => StorageItemClass; + + key?: ByteBufferClass | null; + value?: ByteBufferClass | null; +} + +export declare class StorageItemsClass { + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => StorageItemsClass; + + items?: StorageItemClass[] | null; +} + +export declare enum ContactRecordIdentityState { + DEFAULT = 0, + VERIFIED = 1, + UNVERIFIED = 2, +} + +export declare class ContactRecordClass { + static IdentityState: typeof ContactRecordIdentityState; + + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => ContactRecordClass; + + serviceUuid?: string | null; + serviceE164?: string | null; + profileKey?: ByteBufferClass | null; + identityKey?: ByteBufferClass | null; + identityState?: ContactRecordIdentityState | null; + givenName?: string | null; + familyName?: string | null; + username?: string | null; + blocked?: boolean | null; + whitelisted?: boolean | null; + archived?: boolean | null; +} + +export declare class GroupV1RecordClass { + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => GroupV1RecordClass; + + id?: ByteBufferClass | null; + blocked?: boolean | null; + whitelisted?: boolean | null; + archived?: boolean | null; +} + +export declare class GroupV2RecordClass { + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => GroupV2RecordClass; + + masterKey?: ByteBufferClass | null; + blocked?: boolean | null; + whitelisted?: boolean | null; + archived?: boolean | null; +} + +export declare class AccountRecordClass { + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => AccountRecordClass; + + profileKey?: ByteBufferClass | null; + givenName?: string | null; + familyName?: string | null; + avatarUrl?: string | null; + noteToSelfArchived?: boolean | null; + readReceipts?: boolean | null; + sealedSenderIndicators?: boolean | null; + typingIndicators?: boolean | null; + linkPreviews?: boolean | null; +} + +declare class ReadOperation { + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => ReadOperation; + + readKey: ArrayBuffer[] | ByteBufferClass[]; + toArrayBuffer: () => ArrayBuffer; +} + export declare class SyncMessageClass { static decode: ( data: ArrayBuffer | ByteBufferClass, @@ -544,6 +734,8 @@ export declare class SyncMessageClass { stickerPackOperation?: Array; viewOnceOpen?: SyncMessageClass.ViewOnceOpen; messageRequestResponse?: SyncMessageClass.MessageRequestResponse; + fetchLatest?: SyncMessageClass.FetchLatest; + keys?: SyncMessageClass.Keys; } // Note: we need to use namespaces to express nested classes in Typescript @@ -595,6 +787,12 @@ export declare namespace SyncMessageClass { senderUuid?: string; timestamp?: ProtoBinaryType; } + class FetchLatest { + type?: number; + } + class Keys { + storageService?: ByteBufferClass; + } class MessageRequestResponse { threadE164?: string; @@ -612,6 +810,7 @@ export declare namespace SyncMessageClass.Request { static CONFIGURATION: number; static CONTACTS: number; static GROUPS: number; + static KEYS: number; } } diff --git a/ts/textsecure/MessageReceiver.ts b/ts/textsecure/MessageReceiver.ts index 2a1aff9b5..059b0276a 100644 --- a/ts/textsecure/MessageReceiver.ts +++ b/ts/textsecure/MessageReceiver.ts @@ -42,6 +42,7 @@ declare global { data?: any; deliveryReceipt?: any; error?: any; + eventType?: string | number; groupDetails?: any; groupId?: string; messageRequestResponseType?: number; @@ -56,6 +57,7 @@ declare global { stickerPacks?: any; threadE164?: string; threadUuid?: string; + storageServiceKey?: ArrayBuffer; timestamp?: any; typing?: any; verified?: any; @@ -1437,6 +1439,10 @@ class MessageReceiverInner extends EventTarget { envelope, syncMessage.messageRequestResponse ); + } else if (syncMessage.fetchLatest) { + return this.handleFetchLatest(envelope, syncMessage.fetchLatest); + } else if (syncMessage.keys) { + return this.handleKeys(envelope, syncMessage.keys); } this.removeFromCache(envelope); @@ -1490,6 +1496,29 @@ class MessageReceiverInner extends EventTarget { ['threadUuid'], 'MessageReceiver::handleMessageRequestResponse' ); + } + async handleFetchLatest( + envelope: EnvelopeClass, + sync: SyncMessageClass.FetchLatest + ) { + window.log.info('got fetch latest sync message'); + + const ev = new Event('fetchLatest'); + ev.confirm = this.removeFromCache.bind(this, envelope); + ev.eventType = sync.type; + + return this.dispatchAndWait(ev); + } + async handleKeys(envelope: EnvelopeClass, sync: SyncMessageClass.Keys) { + window.log.info('got keys sync message'); + + if (!sync.storageService) { + return; + } + + const ev = new Event('keys'); + ev.confirm = this.removeFromCache.bind(this, envelope); + ev.storageServiceKey = sync.storageService.toArrayBuffer(); return this.dispatchAndWait(ev); } diff --git a/ts/textsecure/SendMessage.ts b/ts/textsecure/SendMessage.ts index 80f9ad9d4..19c1425e2 100644 --- a/ts/textsecure/SendMessage.ts +++ b/ts/textsecure/SendMessage.ts @@ -12,6 +12,8 @@ import { CallingMessageClass, ContentClass, DataMessageClass, + StorageServiceCallOptionsType, + StorageServiceCredentials, } from '../textsecure.d'; import { MessageError, SignedPreKeyRotationError } from './Errors'; @@ -810,6 +812,33 @@ export default class MessageSender { return Promise.resolve(); } + async sendRequestKeySyncMessage(options: SendOptionsType) { + const myUuid = window.textsecure.storage.user.getUuid(); + const myNumber = window.textsecure.storage.user.getNumber(); + const myDevice = window.textsecure.storage.user.getDeviceId(); + + if (myDevice === 1 || myDevice === '1') { + return; + } + + const request = new window.textsecure.protobuf.SyncMessage.Request(); + request.type = window.textsecure.protobuf.SyncMessage.Request.Type.KEYS; + + const syncMessage = this.createSyncMessage(); + syncMessage.request = request; + const contentMessage = new window.textsecure.protobuf.Content(); + contentMessage.syncMessage = syncMessage; + + const silent = true; + await this.sendIndividualProto( + myUuid || myNumber, + contentMessage, + Date.now(), + silent, + options + ); + } + async sendTypingMessage( options: { recipientId: string; @@ -1630,4 +1659,21 @@ export default class MessageSender { async makeProxiedRequest(url: string, options?: ProxiedRequestOptionsType) { return this.server.makeProxiedRequest(url, options); } + + async getStorageCredentials(): Promise { + return this.server.getStorageCredentials(); + } + + async getStorageManifest( + options: StorageServiceCallOptionsType + ): Promise { + return this.server.getStorageManifest(options); + } + + async getStorageRecords( + data: ArrayBuffer, + options: StorageServiceCallOptionsType + ): Promise { + return this.server.getStorageRecords(data, options); + } } diff --git a/ts/textsecure/WebAPI.ts b/ts/textsecure/WebAPI.ts index a4bc5644c..b8c9db7f6 100644 --- a/ts/textsecure/WebAPI.ts +++ b/ts/textsecure/WebAPI.ts @@ -10,6 +10,12 @@ import { getRandomValue } from '../Crypto'; import PQueue from 'p-queue'; import { v4 as getGuid } from 'uuid'; +import { + StorageServiceCallOptionsType, + StorageServiceCredentials, + TextSecureType, +} from '../textsecure.d'; + // tslint:disable no-bitwise function _btoa(str: any) { @@ -196,6 +202,7 @@ function getContentType(response: Response) { } type HeaderListType = { [name: string]: string }; +type HTTPCodeType = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH'; type PromiseAjaxOptionsType = { accessKey?: string; @@ -212,7 +219,7 @@ type PromiseAjaxOptionsType = { responseType?: 'json' | 'arraybuffer' | 'arraybufferwithdetails'; stack?: string; timeout?: number; - type: 'GET' | 'POST' | 'PUT' | 'DELETE'; + type: HTTPCodeType; unauthenticated?: boolean; user?: string; validateResponse?: any; @@ -479,13 +486,17 @@ const URL_CALLS = { getIceServers: 'v1/accounts/turn', attachmentId: 'v2/attachments/form/upload', deliveryCert: 'v1/certificate/delivery', - supportUnauthenticatedDelivery: 'v1/devices/unauthenticated_delivery', - registerCapabilities: 'v1/devices/capabilities', devices: 'v1/devices', keys: 'v2/keys', messages: 'v1/messages', profile: 'v1/profile', + registerCapabilities: 'v1/devices/capabilities', signed: 'v2/keys/signed', + storageManifest: 'v1/storage/manifest', + storageModify: 'v1/storage/', + storageRead: 'v1/storage/read', + storageToken: 'v1/storage/auth', + supportUnauthenticatedDelivery: 'v1/devices/unauthenticated_delivery', getStickerPackUpload: 'v1/sticker/pack/form', whoami: 'v1/accounts/whoami', config: 'v1/config', @@ -493,6 +504,7 @@ const URL_CALLS = { type InitializeOptionsType = { url: string; + storageUrl: string; cdnUrlObject: { readonly '0': string; readonly [propName: string]: string; @@ -513,12 +525,18 @@ type MessageType = any; type AjaxOptionsType = { accessKey?: string; call: keyof typeof URL_CALLS; - httpType: 'GET' | 'POST' | 'PUT' | 'DELETE'; + contentType?: string; + data?: ArrayBuffer | Buffer | string; + host?: string; + httpType: HTTPCodeType; jsonData?: any; + password?: string; responseType?: 'json' | 'arraybuffer' | 'arraybufferwithdetails'; + schema?: any; timeout?: number; unauthenticated?: boolean; urlParameters?: string; + username?: string; validateResponse?: any; }; @@ -571,6 +589,9 @@ export type WebAPIType = { getSenderCertificate: (withUuid?: boolean) => Promise; getSticker: (packId: string, stickerId: string) => Promise; getStickerPackManifest: (packId: string) => Promise; + getStorageCredentials: TextSecureType['messaging']['getStorageCredentials']; + getStorageManifest: TextSecureType['messaging']['getStorageManifest']; + getStorageRecords: TextSecureType['messaging']['getStorageRecords']; makeProxiedRequest: ( targetUrl: string, options?: ProxiedRequestOptionsType @@ -650,6 +671,7 @@ export type ProxiedRequestOptionsType = { // tslint:disable-next-line max-func-body-length export function initialize({ url, + storageUrl, cdnUrlObject, certificateAuthority, contentProxyUrl, @@ -659,6 +681,9 @@ export function initialize({ if (!is.string(url)) { throw new Error('WebAPI.initialize: Invalid server url'); } + if (!is.string(storageUrl)) { + throw new Error('WebAPI.initialize: Invalid storageUrl'); + } if (!is.object(cdnUrlObject)) { throw new Error('WebAPI.initialize: Invalid cdnUrlObject'); } @@ -713,6 +738,9 @@ export function initialize({ getSenderCertificate, getSticker, getStickerPackManifest, + getStorageCredentials, + getStorageManifest, + getStorageRecords, makeProxiedRequest, putAttachment, registerCapabilities, @@ -737,16 +765,16 @@ export function initialize({ return _outerAjax(null, { certificateAuthority, - contentType: 'application/json; charset=utf-8', - data: param.jsonData && _jsonThing(param.jsonData), - host: url, - password, + contentType: param.contentType || 'application/json; charset=utf-8', + data: param.data || (param.jsonData && _jsonThing(param.jsonData)), + host: param.host || url, + password: param.password || password, path: URL_CALLS[param.call] + param.urlParameters, proxyUrl, responseType: param.responseType, timeout: param.timeout, type: param.httpType, - user: username, + user: param.username || username, validateResponse: param.validateResponse, version, unauthenticated: param.unauthenticated, @@ -821,6 +849,50 @@ export function initialize({ }); } + async function getStorageCredentials(): Promise { + return _ajax({ + call: 'storageToken', + httpType: 'GET', + responseType: 'json', + schema: { username: 'string', password: 'string' }, + }); + } + + async function getStorageManifest( + options: StorageServiceCallOptionsType = {} + ): Promise { + const { credentials, greaterThanVersion } = options; + + return _ajax({ + call: 'storageManifest', + contentType: 'application/x-protobuf', + host: storageUrl, + httpType: 'GET', + responseType: 'arraybuffer', + urlParameters: greaterThanVersion + ? `/version/${greaterThanVersion}` + : '', + ...credentials, + }); + } + + async function getStorageRecords( + data: ArrayBuffer, + options: StorageServiceCallOptionsType = {} + ): Promise { + const { credentials } = options; + + return _ajax({ + call: 'storageRead', + contentType: 'application/x-protobuf', + data, + host: storageUrl, + httpType: 'PUT', + responseType: 'arraybuffer', + ...credentials, + }); + } + async function registerSupportForUnauthenticatedDelivery() { return _ajax({ call: 'supportUnauthenticatedDelivery', diff --git a/ts/util/index.ts b/ts/util/index.ts index 880bbe926..93fc170da 100644 --- a/ts/util/index.ts +++ b/ts/util/index.ts @@ -15,6 +15,7 @@ import { isFileDangerous } from './isFileDangerous'; import { makeLookup } from './makeLookup'; import { migrateColor } from './migrateColor'; import { missingCaseError } from './missingCaseError'; +import { runStorageServiceSyncJob } from './storageService'; import * as zkgroup from './zkgroup'; export { @@ -33,5 +34,6 @@ export { migrateColor, missingCaseError, Registration, + runStorageServiceSyncJob, zkgroup, }; diff --git a/ts/util/lint/exceptions.json b/ts/util/lint/exceptions.json index 1aa742977..bf189f3ce 100644 --- a/ts/util/lint/exceptions.json +++ b/ts/util/lint/exceptions.json @@ -223,7 +223,7 @@ "rule": "jQuery-wrap(", "path": "js/models/conversations.js", "line": " await wrap(", - "lineNumber": 644, + "lineNumber": 652, "reasonCategory": "falseMatch", "updated": "2020-06-09T20:26:46.515Z" }, @@ -11799,7 +11799,7 @@ "rule": "jQuery-wrap(", "path": "ts/textsecure/SendMessage.ts", "line": " return window.dcodeIO.ByteBuffer.wrap(string, 'hex').toArrayBuffer();", - "lineNumber": 30, + "lineNumber": 32, "reasonCategory": "falseMatch", "updated": "2020-05-28T18:08:02.658Z" }, @@ -11807,7 +11807,7 @@ "rule": "jQuery-wrap(", "path": "ts/textsecure/SendMessage.ts", "line": " return window.dcodeIO.ByteBuffer.wrap(string, 'base64').toArrayBuffer();", - "lineNumber": 33, + "lineNumber": 35, "reasonCategory": "falseMatch", "updated": "2020-05-28T18:08:02.658Z" }, @@ -11875,4 +11875,4 @@ "reasonCategory": "falseMatch", "updated": "2020-04-05T23:45:16.746Z" } -] +] \ No newline at end of file diff --git a/ts/util/storageService.ts b/ts/util/storageService.ts new file mode 100644 index 000000000..ab8a009de --- /dev/null +++ b/ts/util/storageService.ts @@ -0,0 +1,428 @@ +/* tslint:disable no-backbone-get-set-outside-model */ +import _ from 'lodash'; +import PQueue from 'p-queue'; + +import Crypto from '../textsecure/Crypto'; +import { + arrayBufferToBase64, + base64ToArrayBuffer, + constantTimeEqual, + deriveStorageItemKey, + deriveStorageManifestKey, +} from '../Crypto'; +import { + AccountRecordClass, + ContactRecordClass, + GroupV1RecordClass, + ManifestRecordClass, + StorageItemClass, +} from '../textsecure.d'; +import { ConversationType } from '../window.d'; + +function fromRecordVerified(verified: number): number { + const VERIFIED_ENUM = window.textsecure.storage.protocol.VerifiedStatus; + const STATE_ENUM = window.textsecure.protobuf.ContactRecord.IdentityState; + + switch (verified) { + case STATE_ENUM.VERIFIED: + return VERIFIED_ENUM.VERIFIED; + case STATE_ENUM.UNVERIFIED: + return VERIFIED_ENUM.UNVERIFIED; + default: + return VERIFIED_ENUM.DEFAULT; + } +} + +async function fetchManifest(manifestVersion: string) { + window.log.info('storageService.fetchManifest'); + try { + const credentials = await window.textsecure.messaging.getStorageCredentials(); + window.storage.put('storageCredentials', credentials); + + const manifestBinary = await window.textsecure.messaging.getStorageManifest( + { + credentials, + greaterThanVersion: manifestVersion, + } + ); + const encryptedManifest = window.textsecure.protobuf.StorageManifest.decode( + manifestBinary + ); + + // If we don't get a value we're assuming we're receiving a 204 + // it would be nice to get an actual e.code 204 and check against that. + if (!encryptedManifest.value || !encryptedManifest.version) { + window.log.info('storageService.fetchManifest: nothing changed'); + return; + } + + const storageKeyBase64 = window.storage.get('storageKey'); + const storageKey = base64ToArrayBuffer(storageKeyBase64); + const storageManifestKey = await deriveStorageManifestKey( + storageKey, + encryptedManifest.version.toNumber() + ); + + const decryptedManifest = await Crypto.decryptProfile( + encryptedManifest.value.toArrayBuffer(), + storageManifestKey + ); + + return window.textsecure.protobuf.ManifestRecord.decode(decryptedManifest); + } catch (err) { + window.log.error(`storageService.fetchManifest: ${err}`); + + if (err.code === 404) { + // No manifest exists, we create one + return { version: 0, keys: [] }; + } else if (err.code === 204) { + // noNewerManifest we're ok + return; + } + + throw err; + } +} + +async function mergeGroupV1Record( + storageID: string, + groupV1Record: GroupV1RecordClass +): Promise { + window.log.info(`storageService.mergeGroupV1Record: merging ${storageID}`); + + if (!groupV1Record.id) { + window.log.info( + `storageService.mergeGroupV1Record: no ID for ${storageID}` + ); + return; + } + + const conversation = await window.ConversationController.getOrCreateAndWait( + groupV1Record.id.toBinary(), + 'group' + ); + + conversation.set({ + isArchived: Boolean(groupV1Record.archived), + storageID, + }); + + window.Signal.Data.updateConversation(conversation.attributes); + + window.log.info(`storageService.mergeGroupV1Record: merged ${storageID}`); +} + +async function mergeContactRecord( + storageID: string, + contactRecord: ContactRecordClass +): Promise { + window.normalizeUuids( + contactRecord, + ['serviceUuid'], + 'storageService.mergeContactRecord' + ); + + if (!contactRecord.serviceE164) { + window.log.info( + `storageService.mergeContactRecord: no E164 for ${storageID}, uuid: ${contactRecord.serviceUuid}. Dropping record` + ); + return; + } + + const id = contactRecord.serviceE164 || contactRecord.serviceUuid; + + if (!id) { + window.log.info( + `storageService.mergeContactRecord: no ID for ${storageID}` + ); + return; + } + + window.log.info(`storageService.mergeContactRecord: merging ${storageID}`); + + const conversation = await window.ConversationController.getOrCreateAndWait( + id, + 'private' + ); + + if (contactRecord.blocked === true) { + window.storage.addBlockedNumber(conversation.id); + } else if (contactRecord.blocked === false) { + window.storage.removeBlockedNumber(conversation.id); + } + + const verified = contactRecord.identityState + ? fromRecordVerified(contactRecord.identityState) + : window.textsecure.storage.protocol.VerifiedStatus.DEFAULT; + + conversation.set({ + isArchived: Boolean(contactRecord.archived), + profileFamilyName: contactRecord.familyName, + profileKey: contactRecord.profileKey + ? arrayBufferToBase64(contactRecord.profileKey.toArrayBuffer()) + : null, + profileName: contactRecord.givenName, + profileSharing: Boolean(contactRecord.whitelisted), + storageID, + verified, + }); + + if ( + contactRecord.serviceUuid && + (!conversation.get('uuid') || + conversation.get('uuid') !== contactRecord.serviceUuid) + ) { + window.log.info( + `storageService.mergeContactRecord: updating UUID ${storageID}` + ); + conversation.set({ uuid: contactRecord.serviceUuid }); + } + + if (contactRecord.serviceE164 && !conversation.get('e164')) { + window.log.info( + `storageService.mergeContactRecord: updating E164 ${storageID}` + ); + conversation.set({ e164: contactRecord.serviceE164 }); + } + + const identityKey = await window.textsecure.storage.protocol.loadIdentityKey( + conversation.id + ); + + const identityKeyChanged = + identityKey && contactRecord.identityKey + ? !constantTimeEqual( + identityKey, + contactRecord.identityKey.toArrayBuffer() + ) + : false; + + if (identityKeyChanged && contactRecord.identityKey) { + await window.textsecure.storage.protocol.processVerifiedMessage( + conversation.id, + verified, + contactRecord.identityKey.toArrayBuffer() + ); + } else if (conversation.get('verified')) { + await window.textsecure.storage.protocol.setVerified( + conversation.id, + verified + ); + } + + window.Signal.Data.updateConversation(conversation.attributes); + + window.log.info(`storageService.mergeContactRecord: merged ${storageID}`); +} + +async function mergeAccountRecord( + storageID: string, + accountRecord: AccountRecordClass +): Promise { + window.log.info(`storageService.mergeAccountRecord: merging ${storageID}`); + + const { + profileKey, + linkPreviews, + readReceipts, + sealedSenderIndicators, + typingIndicators, + } = accountRecord; + + window.storage.put('read-receipt-setting', readReceipts); + + if (typeof sealedSenderIndicators === 'boolean') { + window.storage.put('sealedSenderIndicators', sealedSenderIndicators); + } + + if (typeof typingIndicators === 'boolean') { + window.storage.put('typingIndicators', typingIndicators); + } + + if (typeof linkPreviews === 'boolean') { + window.storage.put('linkPreviews', linkPreviews); + } + + if (profileKey) { + window.storage.put('profileKey', profileKey.toArrayBuffer()); + } + + window.log.info( + `storageService.mergeAccountRecord: merged settings ${storageID}` + ); + + const ourID = window.ConversationController.getOurConversationId(); + + if (!ourID) { + return; + } + + const conversation = await window.ConversationController.getOrCreateAndWait( + ourID, + 'private' + ); + + conversation.set({ + profileFamilyName: accountRecord.familyName, + profileKey: accountRecord.profileKey + ? arrayBufferToBase64(accountRecord.profileKey.toArrayBuffer()) + : null, + profileName: accountRecord.givenName, + storageID, + }); + + window.Signal.Data.updateConversation(conversation.attributes); + + window.log.info( + `storageService.mergeAccountRecord: merged profile ${storageID}` + ); +} + +// tslint:disable-next-line max-func-body-length +async function processManifest( + manifest: ManifestRecordClass +): Promise { + const credentials = window.storage.get('storageCredentials'); + const storageKeyBase64 = window.storage.get('storageKey'); + const storageKey = base64ToArrayBuffer(storageKeyBase64); + + const remoteKeysTypeMap = new Map(); + manifest.keys.forEach(key => { + remoteKeysTypeMap.set( + arrayBufferToBase64(key.raw.toArrayBuffer()), + key.type + ); + }); + + const localKeys = window + .getConversations() + .map((conversation: ConversationType) => conversation.get('storageID')) + .filter(Boolean); + window.log.info( + `storageService.processManifest localKeys.length ${localKeys.length}` + ); + + const remoteKeys = Array.from(remoteKeysTypeMap.keys()); + + const remoteOnly = remoteKeys.filter( + (key: string) => !localKeys.includes(key) + ); + + window.log.info( + `storageService.processManifest remoteOnly.length ${remoteOnly.length}` + ); + + const readOperation = new window.textsecure.protobuf.ReadOperation(); + readOperation.readKey = remoteOnly.map(base64ToArrayBuffer); + + const storageItemsBuffer = await window.textsecure.messaging.getStorageRecords( + readOperation.toArrayBuffer(), + { + credentials, + } + ); + + const storageItems = window.textsecure.protobuf.StorageItems.decode( + storageItemsBuffer + ); + + if (!storageItems.items) { + return false; + } + + const queue = new PQueue({ concurrency: 4 }); + + const mergedItems = storageItems.items.map( + (storageRecordWrapper: StorageItemClass) => async () => { + const { key, value: storageItemCiphertext } = storageRecordWrapper; + + if (!key || !storageItemCiphertext) { + return; + } + + const base64ItemID = arrayBufferToBase64(key.toArrayBuffer()); + + const storageItemKey = await deriveStorageItemKey( + storageKey, + base64ItemID + ); + + const storageItemPlaintext = await Crypto.decryptProfile( + storageItemCiphertext.toArrayBuffer(), + storageItemKey + ); + const storageRecord = window.textsecure.protobuf.StorageRecord.decode( + storageItemPlaintext + ); + + const itemType = remoteKeysTypeMap.get(base64ItemID); + + const ITEM_TYPE = + window.textsecure.protobuf.ManifestRecord.Identifier.Type; + + try { + if (itemType === ITEM_TYPE.UNKNOWN) { + window.log.info('storageService.processManifest: Unknown item type'); + } else if (itemType === ITEM_TYPE.CONTACT && storageRecord.contact) { + await mergeContactRecord(base64ItemID, storageRecord.contact); + } else if (itemType === ITEM_TYPE.GROUPV1 && storageRecord.groupV1) { + await mergeGroupV1Record(base64ItemID, storageRecord.groupV1); + } else if (itemType === ITEM_TYPE.GROUPV2 && storageRecord.groupV2) { + window.log.info( + 'storageService.processManifest: Skipping GroupV2 item' + ); + } else if (itemType === ITEM_TYPE.ACCOUNT && storageRecord.account) { + await mergeAccountRecord(base64ItemID, storageRecord.account); + } + } catch (err) { + window.log.error( + `storageService.processManifest: merging record failed ${base64ItemID}` + ); + } + } + ); + + try { + await queue.addAll(mergedItems); + await queue.onEmpty(); + return true; + } catch (err) { + window.log.error('storageService.processManifest: merging failed'); + return false; + } +} + +export async function runStorageServiceSyncJob() { + const localManifestVersion = '0'; // window.storage.get('manifestVersion') || 0; + + let manifest; + try { + manifest = await fetchManifest(localManifestVersion); + + // Guarding against no manifests being returned, everything should be ok + if (!manifest) { + return; + } + } catch (err) { + // We are supposed to retry here if it's a retryable error + window.log.error( + `storageService.runStorageServiceSyncJob: failed! ${ + err && err.stack ? err.stack : String(err) + }` + ); + return; + } + + const version = manifest.version.toNumber(); + + window.log.info( + `runStorageServiceSyncJob: manifest versions - previous: ${localManifestVersion}, current: ${version}` + ); + + const shouldUpdateVersion = await processManifest(manifest); + + if (shouldUpdateVersion) { + return; + window.storage.put('manifestVersion', version); + } +} diff --git a/ts/window.d.ts b/ts/window.d.ts index 8b56c5eab..22d9f423a 100644 --- a/ts/window.d.ts +++ b/ts/window.d.ts @@ -6,12 +6,13 @@ import { SignalProtocolAddressClass, StorageType, } from './libsignal.d'; -import { TextSecureType } from './textsecure.d'; +import { ContactRecordIdentityState, TextSecureType } from './textsecure.d'; import { WebAPIConnectType } from './textsecure/WebAPI'; import { CallingClass, CallHistoryDetailsType } from './services/calling'; import * as Crypto from './Crypto'; import { ColorType, LocalizerType } from './types/Util'; import { SendOptionsType } from './textsecure/SendMessage'; +import Data from './sql/Client'; type TaskResultType = any; @@ -49,11 +50,15 @@ declare global { put: (key: string, value: any) => void; remove: (key: string) => void; get: (key: string) => T | undefined; + addBlockedNumber: (number: string) => void; + isBlocked: (number: string) => boolean; + removeBlockedNumber: (number: string) => void; }; textsecure: TextSecureType; Signal: { Crypto: typeof Crypto; + Data: typeof Data; Metadata: { SecretSessionCipher: typeof SecretSessionCipherClass; createCertificateValidator: ( @@ -77,7 +82,25 @@ declare global { } } +export type ConversationAttributes = { + e164?: string | null; + isArchived?: boolean; + profileFamilyName?: string | null; + profileKey?: string | null; + profileName?: string | null; + profileSharing?: boolean; + name?: string; + storageID?: string; + uuid?: string | null; + verified?: number; +}; + export type ConversationType = { + attributes: ConversationAttributes; + fromRecordVerified: ( + verified: ContactRecordIdentityState + ) => ContactRecordIdentityState; + set: (props: Partial) => void; updateE164: (e164?: string) => void; updateUuid: (uuid?: string) => void; id: string; @@ -109,6 +132,7 @@ export type ConversationControllerType = { ) => ConversationType; getConversationId: (identifier: string) => string | null; ensureContactIds: (o: { e164?: string; uuid?: string }) => string; + getOurConversationId: () => string | null; prepareForSend: ( id: string, options: Object @@ -117,6 +141,7 @@ export type ConversationControllerType = { sendOptions: Object; }; get: (identifier: string) => null | ConversationType; + map: (mapFn: (conversation: ConversationType) => any) => any; }; export type DCodeIOType = { @@ -161,6 +186,7 @@ export class ByteBufferClass { static wrap: (value: any, type?: string) => ByteBufferClass; toString: (type: string) => string; toArrayBuffer: () => ArrayBuffer; + toBinary: () => string; slice: (start: number, end?: number) => ByteBufferClass; append: (data: ArrayBuffer) => void; limit: number;