diff --git a/zeppelin-web-angular/e2e/models/notebook-offline-recovery-page.ts b/zeppelin-web-angular/e2e/models/notebook-offline-recovery-page.ts new file mode 100644 index 00000000000..b4ecd4b0671 --- /dev/null +++ b/zeppelin-web-angular/e2e/models/notebook-offline-recovery-page.ts @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import type { Locator, Page } from '@playwright/test'; +import { NotebookParagraphPage } from './notebook-paragraph-page'; + +interface ConnectivityProbe { + documentId: string; + attempts: number[]; + sockets: WebSocket[]; + maxOpen: number; + maxActive: number; + offlineAt: number | null; + onlineAt: number | null; +} + +export interface ConnectivitySnapshot extends Omit { + online: boolean; + open: number; + active: number; + socketStates: number[]; +} + +declare global { + interface Window { + notebookConnectivityProbe: ConnectivityProbe; + } +} + +export class NotebookOfflineRecoveryPage { + readonly editorText: Locator; + readonly reconnecting: Locator; + private readonly paragraph: NotebookParagraphPage; + + constructor(readonly page: Page) { + this.paragraph = new NotebookParagraphPage(page); + this.editorText = this.paragraph.editorViewLines; + this.reconnecting = page.getByText('Connecting WebSocket ...', { exact: true }); + } + + async install(): Promise { + await this.page.addInitScript(() => { + const NativeWebSocket = window.WebSocket; + const probe: ConnectivityProbe = { + documentId: crypto.randomUUID(), + attempts: [], + sockets: [], + maxOpen: 0, + maxActive: 0, + offlineAt: null, + onlineAt: null + }; + window.notebookConnectivityProbe = probe; + const recordConcurrency = () => { + probe.maxOpen = Math.max(probe.maxOpen, probe.sockets.filter(socket => socket.readyState === 1).length); + probe.maxActive = Math.max(probe.maxActive, probe.sockets.filter(socket => socket.readyState !== 3).length); + }; + window.addEventListener('offline', () => (probe.offlineAt = performance.now())); + window.addEventListener('online', () => (probe.onlineAt = performance.now())); + window.WebSocket = class extends NativeWebSocket { + constructor(url: string | URL, protocols?: string | string[]) { + const isNotebookSocket = new URL(String(url), window.location.href).pathname.endsWith('/ws'); + if (isNotebookSocket) probe.attempts.push(performance.now()); + super(url, protocols); + if (!isNotebookSocket) return; + probe.sockets.push(this); + recordConcurrency(); + this.addEventListener('open', recordConcurrency); + this.addEventListener('close', recordConcurrency); + } + }; + }); + } + + async setOffline(offline: boolean): Promise { + await this.page.context().setOffline(offline); + if (this.page.context().browser()?.browserType().name() === 'firefox') { + // Firefox emulates network state without emitting window connectivity events. + // Supply those inputs explicitly; Chromium and WebKit retain native event delivery. + await this.page.evaluate(offline => { + window.dispatchEvent(new Event(offline ? 'offline' : 'online')); + }, offline); + } + } + + async snapshot(): Promise { + return this.page.evaluate(() => { + const { sockets, ...probe } = window.notebookConnectivityProbe; + return { + ...probe, + online: navigator.onLine, + open: sockets.filter(socket => socket.readyState === WebSocket.OPEN).length, + active: sockets.filter(socket => socket.readyState !== WebSocket.CLOSED).length, + socketStates: sockets.map(socket => socket.readyState) + }; + }); + } + + async observe(transition: 'offlineAt' | 'onlineAt', duration: number): Promise { + const remaining = await this.page.evaluate( + ({ transition, duration }) => { + const started = window.notebookConnectivityProbe[transition]; + if (started === null) throw new Error(`No ${transition} event was observed`); + return Math.max(0, started + duration - performance.now()); + }, + { transition, duration } + ); + // JUSTIFIED: This real-time window measures retry attempts, not UI readiness. + // eslint-disable-next-line playwright/no-wait-for-timeout -- A bounded observation interval must elapse to detect retry storms. + await this.page.waitForTimeout(remaining); + return this.snapshot(); + } + + async replaceText(text: string): Promise { + await this.paragraph.editorViewLines.click(); + const isMacintosh = await this.page.evaluate(() => navigator.userAgent.includes('Macintosh')); + await this.page.keyboard.press(isMacintosh ? 'Meta+A' : 'Control+A'); + await this.page.keyboard.insertText(text); + } +} diff --git a/zeppelin-web-angular/e2e/tests/notebook/collaboration/offline-recovery.spec.ts b/zeppelin-web-angular/e2e/tests/notebook/collaboration/offline-recovery.spec.ts new file mode 100644 index 00000000000..3278691ef28 --- /dev/null +++ b/zeppelin-web-angular/e2e/tests/notebook/collaboration/offline-recovery.spec.ts @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { randomUUID } from 'node:crypto'; +import { expect, test, type BrowserContext, type Page } from '@playwright/test'; +import { E2E_TEST_FOLDER } from '../../../models/base-page'; +import { NotebookOfflineRecoveryPage, type ConnectivitySnapshot } from '../../../models/notebook-offline-recovery-page'; +import { addPageAnnotationBeforeEach, PAGES, waitForZeppelinReady } from '../../../utils'; + +const paragraphText = async (page: Page, noteId: string): Promise => { + const response = await page.request.get(`/api/notebook/${noteId}`); + if (!response.ok()) throw new Error(`Canonical notebook fetch failed: ${response.status()}`); + return (await response.json()).body.paragraphs[0].text as string; +}; + +const attemptsInFirstSecond = (snapshot: ConnectivitySnapshot, transition: 'offlineAt' | 'onlineAt'): number => { + const started = snapshot[transition]; + if (started === null) throw new Error(`No ${transition} event was observed`); + return snapshot.attempts.filter(attempt => attempt >= started && attempt < started + 1000).length; +}; + +test.describe('Notebook offline recovery', () => { + addPageAnnotationBeforeEach(PAGES.WORKSPACE.NOTEBOOK); + let view: NotebookOfflineRecoveryPage; + + test.beforeEach(async ({ page }) => { + view = new NotebookOfflineRecoveryPage(page); + await view.install(); + }); + + test('pauses offline retries and recovers missed and live peer edits without reloading', async ({ + page, + context, + browser, + baseURL + }, testInfo) => { + const marker = randomUUID(); + const initialText = `%md initial_${marker}`; + const missedText = `%md missed_${marker}`; + const liveText = `%md live_${marker}`; + const created = await page.request.post('/api/notebook', { + data: { + notePath: `${E2E_TEST_FOLDER}/OfflineRecovery_${marker}`, + paragraphs: [{ text: initialText }] + } + }); + expect(created.ok()).toBe(true); + const noteId = (await created.json()).body as string; + let peerContext: BrowserContext | undefined; + const observations: Record = {}; + + try { + // Both viewers inherit the suite's principal; this also works with anonymous storageState. + peerContext = await browser.newContext({ baseURL, storageState: await context.storageState() }); + const peerPage = await peerContext.newPage(); + const peer = new NotebookOfflineRecoveryPage(peerPage); + await peer.install(); + + await test.step('Given two real notebook viewers share an isolated note', async () => { + await Promise.all([page.goto(`/#/notebook/${noteId}`), peerPage.goto(`/#/notebook/${noteId}`)]); + await Promise.all([waitForZeppelinReady(page), waitForZeppelinReady(peerPage)]); + await expect(view.editorText).toContainText(initialText); + await expect(peer.editorText).toContainText(initialText); + observations.initial = await view.snapshot(); + expect(observations.initial).toMatchObject({ + online: true, + open: 1, + active: 1, + maxOpen: 1, + maxActive: 1, + socketStates: [1] + }); + expect(observations.initial.attempts).toHaveLength(1); + await expect.poll(async () => (await peer.snapshot()).open).toBe(1); + expect((await peer.snapshot()).documentId).not.toBe(observations.initial.documentId); + }); + + await test.step('When the browser context goes offline, the socket closes without a retry storm', async () => { + await view.setOffline(true); + await expect.poll(async () => (await view.snapshot()).online).toBe(false); + await expect.poll(async () => (await view.snapshot()).offlineAt).not.toBeNull(); + await expect(view.reconnecting).toBeVisible(); + observations.firstSecond = await view.observe('offlineAt', 1000); + expect(attemptsInFirstSecond(observations.firstSecond, 'offlineAt')).toBeLessThanOrEqual(1); + expect(observations.firstSecond.documentId).toBe(observations.initial.documentId); + await expect.poll(async () => (await view.snapshot()).active, { timeout: 15000 }).toBe(0); + }); + + await test.step('And the peer persists a change while the first viewer remains paused offline', async () => { + await peer.replaceText(missedText); + await expect(peer.editorText).toContainText(missedText); + await expect.poll(() => paragraphText(peerPage, noteId), { timeout: 15000 }).toBe(missedText); + observations.paused = await view.observe('offlineAt', 3500); + expect(observations.paused).toMatchObject({ online: false, open: 0, active: 0, maxOpen: 1, maxActive: 1 }); + expect(observations.paused.attempts).toHaveLength(observations.firstSecond.attempts.length); + expect(observations.paused.documentId).toBe(observations.initial.documentId); + expect(observations.paused.socketStates.every(state => state === 3)).toBe(true); + await expect(view.editorText).toContainText(initialText); + await expect(view.editorText).not.toContainText(missedText); + }); + + await test.step('Then going online rehydrates the missed change in the same document with one socket', async () => { + await view.setOffline(false); + await expect.poll(async () => (await view.snapshot()).onlineAt).not.toBeNull(); + await expect(view.editorText).toContainText(missedText, { timeout: 30000 }); + await expect(view.reconnecting).toBeHidden(); + observations.recovered = await view.observe('onlineAt', 1000); + expect(attemptsInFirstSecond(observations.recovered, 'onlineAt')).toBeLessThanOrEqual(1); + expect(observations.recovered.attempts).toHaveLength(observations.paused.attempts.length + 1); + expect(observations.recovered).toMatchObject({ online: true, open: 1, active: 1, maxOpen: 1, maxActive: 1 }); + expect(observations.recovered.documentId).toBe(observations.initial.documentId); + expect(observations.recovered.socketStates).toEqual([...observations.paused.socketStates, 1]); + await expect(page).toHaveURL(new RegExp(`/#/notebook/${noteId}$`)); + }); + + await test.step('And a subsequent peer edit arrives live without another reconnect or reload', async () => { + await peer.replaceText(liveText); + await expect(peer.editorText).toContainText(liveText); + await expect(view.editorText).toContainText(liveText, { timeout: 30000 }); + observations.live = await view.snapshot(); + expect(observations.live.documentId).toBe(observations.initial.documentId); + expect(observations.live.attempts).toHaveLength(observations.recovered.attempts.length); + expect(observations.live).toMatchObject({ online: true, open: 1, active: 1, maxOpen: 1, maxActive: 1 }); + expect(await peer.snapshot()).toMatchObject({ online: true, open: 1, active: 1, maxOpen: 1, maxActive: 1 }); + }); + } finally { + try { + await testInfo.attach('native-websocket-lifecycle', { + body: JSON.stringify(observations, null, 2), + contentType: 'application/json' + }); + } finally { + try { + const removed = await page.request.delete(`/api/notebook/${noteId}`); + expect(removed.ok()).toBe(true); + } finally { + await Promise.all([context.setOffline(false), peerContext?.close()]); + } + } + } + }); +}); diff --git a/zeppelin-web-angular/projects/zeppelin-sdk/src/message-connection.ts b/zeppelin-web-angular/projects/zeppelin-sdk/src/message-connection.ts new file mode 100644 index 00000000000..719cfd0debd --- /dev/null +++ b/zeppelin-web-angular/projects/zeppelin-sdk/src/message-connection.ts @@ -0,0 +1,384 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { interval, Observable, Observer, Subject, Subscription } from 'rxjs'; +import { webSocket, WebSocketSubject } from 'rxjs/webSocket'; + +import { Ticket } from './interfaces/message-common.interface'; +import { + MessageDataTypeMap, + MessageReceiveDataTypeMap, + MessageSendDataTypeMap +} from './interfaces/message-data-type-map.interface'; +import { WebSocketMessage } from './interfaces/websocket-message.interface'; + +interface ConnectionIdentity { + readonly url: string; + readonly principal: string | undefined; +} + +interface ConnectionAttempt { + identity: ConnectionIdentity; + socket: WebSocket | null; + subject: WebSocketSubject> | null; + subscription: Subscription; + retired: boolean; + opened: boolean; +} + +interface MessageConnectionHandlers { + received: Observer>; + nextMessageId(): string; + interceptReceived(message: WebSocketMessage): WebSocketMessage; + ping(): void; +} + +/** + * SDK-internal owner of the physical connection and its pending outbound commands. + * Requested user/endpoint identity is snapshotted separately from mutable configuration. + * Changing that identity cancels old unsent work and waits for physical close before reconnecting. + */ +export class MessageConnection { + public connectedStatus = false; + public connectedStatus$ = new Subject(); + private ws: WebSocketSubject> | null = null; + private attempt: ConnectionAttempt | null = null; + private retryTimer: ReturnType | null = null; + // Retry after 1, 2, 4, 8, 16, then at most 30 seconds; a valid open resets to 1 second. + private retryDelay = 1000; + private online = true; + private connectionRequested = false; + private identity: ConnectionIdentity | null = null; + private destroyed = false; + private open$ = new Subject(); + private close$ = new Subject(); + private sent$ = new Subject>(); + private pendingMessages: Array> = []; + private pingIntervalSubscription = new Subscription(); + private wsUrl?: string; + private ticket?: Ticket; + private readonly normalCloseCode = 1000; + + constructor(private readonly handlers: MessageConnectionHandlers) {} + + setOnline(online: boolean): void { + if (this.destroyed || this.online === online) { + return; + } + this.online = online; + if (online) { + this.startConnection(); + } else { + this.cancelRetry(); + if (this.attempt) { + this.retireAttempt(this.attempt); + } + this.disconnect(); + } + } + + bootstrap(ticket: Ticket, wsUrl: string): void { + if (this.destroyed) { + return; + } + this.setTicket(ticket); + this.setWsUrl(wsUrl); + this.connect(); + } + + getWsInstance(): WebSocketSubject> | null { + return this.ws; + } + + setWsUrl(wsUrl: string): void { + this.wsUrl = wsUrl; + } + + setTicket(ticket: Ticket): void { + this.ticket = ticket; + } + + connect(): void { + if (this.destroyed) { + return; + } + if (!this.wsUrl) { + throw new Error('WebSocket URL is not set. Please call setWsUrl() before connect()'); + } + this.connectionRequested = true; + this.startConnection(); + } + + close(): void { + if (this.destroyed) { + return; + } + this.connectionRequested = false; + this.identity = null; + this.cancelRetry(); + this.pendingMessages.length = 0; + if (this.attempt) { + this.retireAttempt(this.attempt); + } + this.disconnect(); + } + + opened(): Observable { + return this.open$.asObservable(); + } + + closed(): Observable { + return this.close$.asObservable(); + } + + sent(): Observable> { + return this.sent$.asObservable(); + } + + send(op: K, data: MessageSendDataTypeMap[K] | undefined): string { + if (this.destroyed || (!this.ws && !this.connectionRequested)) { + throw new Error('WebSocket is not connected. Bootstrap first.'); + } + this.startConnection(); + if (this.destroyed || !this.connectionRequested) { + throw new Error('WebSocket is not connected. Bootstrap first.'); + } + const message = { + op, + msgId: this.handlers.nextMessageId(), + data, + ...this.ticket + }; + console.log('Send:', message.op, message.principal); + + const attempt = this.attempt; + // Reporting is writable; only the current physical attempt determines whether a frame can be sent. + if (!attempt || !this.isCurrent(attempt) || !attempt.opened || attempt.socket?.readyState !== 1) { + this.pendingMessages.push(message); + } else { + this.flushPendingMessages(attempt); + attempt.subject!.next(message); + } + this.sent$.next(message); + return message.msgId; + } + + destroy(): void { + if (this.destroyed) { + return; + } + this.destroyed = true; + this.connectionRequested = false; + this.identity = null; + this.cancelRetry(); + this.pendingMessages.length = 0; + if (this.attempt) { + this.retireAttempt(this.attempt); + } + this.disconnect(); + this.connectedStatus$.complete(); + this.open$.complete(); + this.close$.complete(); + this.sent$.complete(); + this.handlers.received.complete(); + } + + private startConnection(): void { + if (this.destroyed || !this.connectionRequested) { + return; + } + const principal = this.ticket?.principal; + if (!this.identity) { + this.identity = { url: this.wsUrl!, principal }; + } else if (this.identity.url !== this.wsUrl || this.identity.principal !== principal) { + // Bind the new request before notifications can reenter connect(), close(), or destroy(). + this.identity = { url: this.wsUrl!, principal }; + this.cancelRetry(); + this.pendingMessages.length = 0; + if (this.attempt) { + this.retireAttempt(this.attempt); + } + this.disconnect(); + } + if (this.destroyed || !this.connectionRequested || !this.online || this.attempt || this.retryTimer !== null) { + return; + } + + this.startAttempt(this.identity!); + } + + private startAttempt(identity: ConnectionIdentity): void { + const attempt: ConnectionAttempt = { + identity, + socket: null, + subject: null, + subscription: new Subscription(), + retired: false, + opened: false + }; + this.attempt = attempt; + const onClose = (event: CloseEvent) => this.onSocketClose(attempt, event); + try { + const subject = webSocket>({ + url: attempt.identity.url, + // RxJS resets its socket reference on error, before the native close event. + // Keep the physical socket until close, even after retiring its callbacks. + WebSocketCtor: class extends WebSocket { + constructor(url: string) { + super(url); + attempt.socket = this; + this.addEventListener('close', onClose); + } + }, + openObserver: { next: event => this.onSocketOpen(attempt, event) } + }); + attempt.subject = subject; + this.ws = subject; + // A constructor failure can notify error before subscribe() returns. + // The pre-existing parent subscription also tears down that synchronous case. + attempt.subscription.add( + subject.subscribe({ + next: message => { + if (!this.isCurrent(attempt)) { + return; + } + const received = this.handlers.interceptReceived(message as WebSocketMessage); + if (this.isCurrent(attempt)) { + console.log('Receive:', received.op); + this.handlers.received.next(received); + } + }, + error: () => this.onSocketError(attempt) + }) + ); + } catch { + this.onSocketError(attempt); + } + } + + private isCurrent(attempt: ConnectionAttempt): boolean { + return ( + this.attempt === attempt && + attempt.identity === this.identity && + !attempt.retired && + !this.destroyed && + this.online && + this.connectionRequested + ); + } + + private onSocketOpen(attempt: ConnectionAttempt, event: Event): void { + if (!this.isCurrent(attempt) || attempt.opened) { + return; + } + this.startConnection(); + if (!this.isCurrent(attempt)) { + return; + } + attempt.opened = true; + this.retryDelay = 1000; + this.flushPendingMessages(attempt); + this.pingIntervalSubscription.unsubscribe(); + this.pingIntervalSubscription = interval(1000 * 10).subscribe(this.handlers.ping); + this.connectedStatus = true; + this.connectedStatus$.next(true); + if (this.isCurrent(attempt)) { + this.open$.next(event); + } + } + + private flushPendingMessages(attempt: ConnectionAttempt): void { + if (this.pendingMessages.length === 0) { + return; + } + const pending = this.pendingMessages; + this.pendingMessages = []; + for (const message of pending) { + attempt.subject!.next({ ...message, ...this.ticket }); + } + } + + private onSocketError(attempt: ConnectionAttempt): void { + if (!this.isCurrent(attempt)) { + return; + } + this.scheduleRetry(); + this.retireAttempt(attempt); + this.disconnect(); + } + + private onSocketClose(attempt: ConnectionAttempt, event: CloseEvent): void { + if (this.attempt !== attempt) { + return; + } + const current = this.isCurrent(attempt); + this.retireAttempt(attempt); + if (!current) { + // Offline/close/destroy retired application callbacks, not this close barrier. + // An online transition or explicit connect may now resume the saved intent. + this.startConnection(); + return; + } + if (event.code === this.normalCloseCode) { + this.connectionRequested = false; + this.identity = null; + this.pendingMessages.length = 0; + } else { + this.scheduleRetry(); + } + this.disconnect(); + if (!this.destroyed) { + this.close$.next(event); + } + } + + private retireAttempt(attempt: ConnectionAttempt): void { + attempt.retired = true; + if (this.ws === attempt.subject) { + this.ws = null; + } + attempt.subscription.unsubscribe(); + attempt.subject?.unsubscribe(); + const socket = attempt.socket; + if (!socket || socket.readyState === 3) { + if (this.attempt === attempt) { + this.attempt = null; + } + } else if (socket.readyState === 0 || socket.readyState === 1) { + socket.close(); + } + } + + private disconnect(): void { + this.pingIntervalSubscription.unsubscribe(); + this.connectedStatus = false; + this.connectedStatus$.next(false); + } + + private scheduleRetry(): void { + if (this.destroyed || !this.connectionRequested || !this.online || this.retryTimer !== null) { + return; + } + this.retryTimer = setTimeout(() => { + this.retryTimer = null; + this.startConnection(); + }, this.retryDelay); + this.retryDelay = Math.min(this.retryDelay * 2, 30000); + } + + private cancelRetry(): void { + if (this.retryTimer !== null) { + clearTimeout(this.retryTimer); + this.retryTimer = null; + } + } +} diff --git a/zeppelin-web-angular/projects/zeppelin-sdk/src/message-integration.spec.ts b/zeppelin-web-angular/projects/zeppelin-sdk/src/message-integration.spec.ts new file mode 100644 index 00000000000..18df19e8fe7 --- /dev/null +++ b/zeppelin-web-angular/projects/zeppelin-sdk/src/message-integration.spec.ts @@ -0,0 +1,408 @@ +// @vitest-environment node + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { once } from 'node:events'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { config, firstValueFrom, fromEvent } from 'rxjs'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { WebSocket, WebSocketServer, type RawData } from 'ws'; + +import { OP } from './interfaces/message-operator.interface'; +import { Message } from './message'; + +interface ClientConnection { + socket: WebSocket; + startedAt: number; + closed: Promise; +} + +const clients: ClientConnection[] = []; +const messages: Message[] = []; +const servers = new Set(); +const peers: WebSocket[] = []; +const serverErrors: unknown[] = []; +const unhandledErrors: unknown[] = []; +let maxConcurrent = 0; +let previousUnhandledError: typeof config.onUnhandledError; + +// Only observe construction and physical closure; all handshakes, frames, errors, +// and close events go through the real ws implementation and a TCP listener. +class ObservedWebSocket extends WebSocket { + constructor(url: string) { + super(url); + clients.push({ + socket: this, + startedAt: performance.now(), + closed: firstValueFrom(fromEvent(this, 'close')) + }); + maxConcurrent = Math.max( + maxConcurrent, + clients.filter(client => client.socket.readyState !== WebSocket.CLOSED).length + ); + } +} + +const startServer = async (port = 0) => { + const server = new WebSocketServer({ host: '127.0.0.1', port, path: '/ws' }); + servers.add(server); + server.on('error', error => serverErrors.push(error)); + server.on('connection', peer => { + peers.push(peer); + peer.on('error', error => serverErrors.push(error)); + }); + await once(server, 'listening'); + const address = server.address(); + if (typeof address === 'string' || address === null) { + throw new Error('Expected a TCP WebSocket listener'); + } + return { server, port: address.port, url: `ws://127.0.0.1:${address.port}/ws` }; +}; + +const stopServer = async (server: WebSocketServer) => { + const closed = once(server, 'close'); + server.close(); + await closed; + servers.delete(server); +}; + +const createMessage = (url: string) => { + const message = new Message(); + messages.push(message); + message.bootstrap({ principal: 'anonymous', ticket: 'integration-ticket', roles: '[]' }, url); + return message; +}; + +const expectRoundTrip = async (message: Message, peer: WebSocket, noteId: string) => { + const requests: string[] = []; + const received: unknown[] = []; + const onRequest = (data: RawData) => requests.push(data.toString()); + const subscription = message.receive(OP.NOTE).subscribe(data => received.push(data)); + peer.on('message', onRequest); + try { + message.ping(); + await expect.poll(() => requests.length, { timeout: 1000 }).toBe(1); + expect(JSON.parse(requests[0])).toMatchObject({ + op: OP.PING, + principal: 'anonymous', + ticket: 'integration-ticket' + }); + const data = { note: { id: noteId } }; + peer.send(JSON.stringify({ op: OP.NOTE, data })); + await expect.poll(() => received, { timeout: 1000 }).toEqual([data]); + } finally { + subscription.unsubscribe(); + peer.off('message', onRequest); + } +}; + +const destroyAndWait = async (message: Message) => { + message.destroy(); + await expect + .poll(() => clients.every(client => client.socket.readyState === WebSocket.CLOSED), { timeout: 1000 }) + .toBe(true); + await Promise.all(clients.map(client => client.closed)); + expect(message.connectedStatus).toBe(false); + expect(message.getWsInstance()).toBeNull(); +}; + +beforeEach(() => { + clients.length = 0; + messages.length = 0; + peers.length = 0; + serverErrors.length = 0; + unhandledErrors.length = 0; + maxConcurrent = 0; + previousUnhandledError = config.onUnhandledError; + config.onUnhandledError = error => unhandledErrors.push(error); + vi.stubGlobal('WebSocket', ObservedWebSocket); + vi.spyOn(console, 'log').mockImplementation(() => {}); +}); + +afterEach(async () => { + try { + for (const message of messages) { + message.destroy(); + } + // Failed assertions must not leave handles behind. Successful cases already + // await the Message-owned shutdown in destroyAndWait, without termination. + for (const { socket } of clients) { + if (socket.readyState !== WebSocket.CLOSED) { + socket.terminate(); + } + } + for (const peer of peers) { + if (peer.readyState !== WebSocket.CLOSED) { + peer.terminate(); + } + } + await Promise.all(clients.map(client => client.closed)); + await Promise.all([...servers].map(stopServer)); + // RxJS reports unhandled observer errors on a later timer turn. + await delay(25); + // eslint-disable-next-line vitest/no-standalone-expect -- Teardown must catch asynchronous RxJS errors in every scenario. + expect(unhandledErrors).toEqual([]); + // eslint-disable-next-line vitest/no-standalone-expect -- Server-side teardown errors must fail the scenario too. + expect(serverErrors).toEqual([]); + } finally { + config.onUnhandledError = previousUnhandledError; + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + } +}); + +describe('Message with real WebSockets', () => { + it.each([ + { + name: 'an explicit abnormal close frame', + code: 1011, + disconnect: (peer: WebSocket) => peer.close(1011, 'retry') + }, + { name: 'an abruptly terminated connection', code: 1006, disconnect: (peer: WebSocket) => peer.terminate() } + ])( + 'recovers from $name with one delayed replacement', + async ({ code, disconnect }) => { + const { url } = await startServer(); + const message = createMessage(url); + const closeCodes: number[] = []; + message.closed().subscribe(event => closeCodes.push(event.code)); + await expect.poll(() => message.connectedStatus, { timeout: 1000 }).toBe(true); + expect(peers).toHaveLength(1); + + disconnect(peers[0]); + await clients[0].closed; + expect(message.connectedStatus).toBe(false); + expect(closeCodes).toEqual([code]); + await delay(250); + expect(clients).toHaveLength(1); + + await expect.poll(() => message.connectedStatus, { timeout: 2000 }).toBe(true); + expect(clients).toHaveLength(2); + expect(peers).toHaveLength(2); + expect(clients[1].startedAt - clients[0].startedAt).toBeGreaterThanOrEqual(900); + expect(clients.filter(client => client.socket.readyState === WebSocket.OPEN)).toHaveLength(1); + expect(maxConcurrent).toBe(1); + await expectRoundTrip(message, peers[1], 'recovered-after-server-close'); + + await destroyAndWait(message); + await delay(1100); + expect(clients).toHaveLength(2); + }, + 10000 + ); + + it('backs off a refused endpoint, recovers when it returns, and shuts down without unhandled errors', async () => { + // Allocate an ephemeral local endpoint, then really stop listening. Both + // failed attempts must therefore perform an actual refused TCP connection. + const endpoint = await startServer(); + await stopServer(endpoint.server); + const message = createMessage(endpoint.url); + await clients[0].closed; + expect(message.connectedStatus).toBe(false); + await delay(250); + expect(clients).toHaveLength(1); + + await expect.poll(() => clients.length, { timeout: 1500 }).toBe(2); + await clients[1].closed; + expect(message.connectedStatus).toBe(false); + const returned = await startServer(endpoint.port); + expect(returned.url).toBe(endpoint.url); + await delay(250); + expect(clients).toHaveLength(2); + + await expect.poll(() => message.connectedStatus, { timeout: 2500 }).toBe(true); + expect(clients).toHaveLength(3); + expect(peers).toHaveLength(1); + expect(clients[1].startedAt - clients[0].startedAt).toBeGreaterThanOrEqual(900); + expect(clients[2].startedAt - clients[1].startedAt).toBeGreaterThanOrEqual(1900); + expect(clients.filter(client => client.socket.readyState === WebSocket.OPEN)).toHaveLength(1); + expect(maxConcurrent).toBe(1); + await expectRoundTrip(message, peers[0], 'recovered-after-refusal'); + + await destroyAndWait(message); + message.setOnline(false); + message.setOnline(true); + message.connect(); + await delay(1100); + expect(clients).toHaveLength(3); + }, 10000); + + it.each(['bootstrap', 'setters and connect', 'setters and bootstrap'] as const)( + 'registers a fresh principal-bound socket with %s so user-targeted replies reach it', + async configuration => { + const { server, url } = await startServer(); + const byUser = new Map(); + const frames: Array<{ principal: string; op: OP; ticket: string }> = []; + server.on('connection', peer => { + let principal: string | undefined; + peer.on('message', data => { + const envelope: { principal: string; op: OP; ticket: string } = JSON.parse(data.toString()); + frames.push(envelope); + if (principal === undefined) { + principal = envelope.principal; + byUser.set(principal, peer); + } + }); + peer.on('close', () => { + if (principal !== undefined && byUser.get(principal) === peer) { + byUser.delete(principal); + } + }); + }); + const message = new Message(); + messages.push(message); + message.bootstrap({ principal: 'A', ticket: 'A-ticket', roles: '[]' }, url); + await expect.poll(() => message.connectedStatus, { timeout: 1000 }).toBe(true); + message.ping(); + await expect.poll(() => byUser.has('A'), { timeout: 1000 }).toBe(true); + + const ticket = { principal: 'B', ticket: 'B-ticket', roles: '[]' }; + if (configuration !== 'bootstrap') { + message.setTicket(ticket); + message.setWsUrl(url); + } + if (configuration === 'setters and connect') { + message.connect(); + } else { + message.bootstrap(ticket, url); + } + await clients[0].closed; + await expect.poll(() => message.connectedStatus, { timeout: 1000 }).toBe(true); + message.ping(); + await expect.poll(() => byUser.has('B'), { timeout: 1000 }).toBe(true); + expect(frames).toEqual([ + expect.objectContaining({ op: OP.PING, principal: 'A', ticket: 'A-ticket' }), + expect.objectContaining({ op: OP.PING, principal: 'B', ticket: 'B-ticket' }) + ]); + const received: unknown[] = []; + const subscription = message.receive(OP.NOTE).subscribe(data => received.push(data)); + try { + byUser.get('B')!.send(JSON.stringify({ op: OP.NOTE, data: { note: { id: 'B-only' } } })); + await expect.poll(() => received, { timeout: 1000 }).toEqual([{ note: { id: 'B-only' } }]); + expect(byUser.has('A')).toBe(false); + expect(clients).toHaveLength(2); + expect(maxConcurrent).toBe(1); + await destroyAndWait(message); + } finally { + subscription.unsubscribe(); + } + } + ); + + it('moves requests to a newly configured endpoint only after the old physical socket closes', async () => { + const first = await startServer(); + const second = await startServer(); + const firstFrames: Array<{ op: OP }> = []; + const secondFrames: Array<{ op: OP }> = []; + first.server.on('connection', peer => { + peer.on('message', data => firstFrames.push(JSON.parse(data.toString()))); + }); + second.server.on('connection', peer => { + peer.on('message', data => secondFrames.push(JSON.parse(data.toString()))); + }); + const message = createMessage(first.url); + await expect.poll(() => message.connectedStatus, { timeout: 1000 }).toBe(true); + message.ping(); + await expect.poll(() => firstFrames.length, { timeout: 1000 }).toBe(1); + expect(firstFrames.map(frame => frame.op)).toEqual([OP.PING]); + + message.setWsUrl(second.url); + message.connect(); + await clients[0].closed; + await expect.poll(() => message.connectedStatus, { timeout: 1000 }).toBe(true); + message.listNodes(); + await expect.poll(() => secondFrames.length, { timeout: 1000 }).toBe(1); + expect(secondFrames.map(frame => frame.op)).toEqual([OP.LIST_NOTES]); + expect(firstFrames.map(frame => frame.op)).toEqual([OP.PING]); + expect(clients).toHaveLength(2); + expect(clients[0].socket.readyState).toBe(WebSocket.CLOSED); + expect(clients[1].socket.url).toBe(second.url); + expect(clients.filter(client => client.socket.readyState === WebSocket.OPEN)).toHaveLength(1); + expect(maxConcurrent).toBe(1); + await expectRoundTrip(message, peers[1], 'new-endpoint'); + expect(firstFrames.map(frame => frame.op)).toEqual([OP.PING]); + expect(secondFrames.map(frame => frame.op)).toEqual([OP.LIST_NOTES, OP.PING]); + await destroyAndWait(message); + }); + + it('delivers commands in order on the open socket despite a writable disconnected status', async () => { + const { server, url } = await startServer(); + const frames: Array<{ op: OP }> = []; + server.on('connection', peer => { + peer.on('message', data => frames.push(JSON.parse(data.toString()))); + }); + const message = createMessage(url); + await expect.poll(() => message.connectedStatus, { timeout: 1000 }).toBe(true); + expect(clients[0].socket.readyState).toBe(WebSocket.OPEN); + + message.connectedStatus = false; + message.listNodes(); + message.connectedStatus = true; + message.ping(); + + await expect.poll(() => frames.length, { timeout: 1000 }).toBe(2); + expect(frames.map(frame => frame.op)).toEqual([OP.LIST_NOTES, OP.PING]); + expect(clients).toHaveLength(1); + expect(clients[0].socket.readyState).toBe(WebSocket.OPEN); + expect(maxConcurrent).toBe(1); + await destroyAndWait(message); + }); + + it('delivers queued offline and connecting commands once in their original order', async () => { + const { server, url } = await startServer(); + const received: Array<{ op: OP; msgId: string; data?: { paragraph?: string } }> = []; + server.on('connection', peer => { + peer.on('message', data => received.push(JSON.parse(data.toString()))); + }); + const message = createMessage(url); + await expect.poll(() => message.connectedStatus, { timeout: 1000 }).toBe(true); + message.setOnline(false); + await clients[0].closed; + message.listNodes(); + const first = message.commitParagraph('p1', undefined, 'offline edit', {}, {}, 'n1'); + message.setOnline(true); + expect(message.connectedStatus).toBe(false); + const second = message.commitParagraph('p1', undefined, 'newer edit', {}, {}, 'n1'); + await expect.poll(() => received.length, { timeout: 1000 }).toBe(3); + expect(received.map(frame => frame.op)).toEqual([OP.LIST_NOTES, OP.COMMIT_PARAGRAPH, OP.COMMIT_PARAGRAPH]); + expect(received.slice(1).map(frame => [frame.msgId, frame.data?.paragraph])).toEqual([ + [first, 'offline edit'], + [second, 'newer edit'] + ]); + expect(clients).toHaveLength(2); + expect(maxConcurrent).toBe(1); + await destroyAndWait(message); + }); + + it.each(['close', 'destroy'] as const)( + 'handles %s during a real pending connection without resurrection', + async action => { + const endpoint = await startServer(); + await stopServer(endpoint.server); + const message = createMessage(endpoint.url); + expect(clients[0].socket.readyState).toBe(WebSocket.CONNECTING); + + message[action](); + await clients[0].closed; + expect(message.connectedStatus).toBe(false); + expect(message.getWsInstance()).toBeNull(); + message.setOnline(false); + message.setOnline(true); + await delay(1100); + expect(clients).toHaveLength(1); + await destroyAndWait(message); + }, + 5000 + ); +}); diff --git a/zeppelin-web-angular/projects/zeppelin-sdk/src/message-lifecycle.spec.ts b/zeppelin-web-angular/projects/zeppelin-sdk/src/message-lifecycle.spec.ts new file mode 100644 index 00000000000..4bf00df303e --- /dev/null +++ b/zeppelin-web-angular/projects/zeppelin-sdk/src/message-lifecycle.spec.ts @@ -0,0 +1,830 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { OP } from './interfaces/message-operator.interface'; +import { Message } from './message'; + +class ControlledWebSocket extends EventTarget { + static instances: ControlledWebSocket[] = []; + static attempts = 0; + static failuresRemaining = 0; + static maxConcurrent = 0; + + readyState = 0; + onopen: ((event: Event) => void) | null = null; + onclose: ((event: CloseEvent) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + readonly sent: string[] = []; + readonly close = vi.fn(() => { + if (this.readyState < 2) { + this.readyState = 2; + } + }); + + constructor(readonly url: string) { + super(); + ControlledWebSocket.attempts++; + if (ControlledWebSocket.failuresRemaining > 0) { + ControlledWebSocket.failuresRemaining--; + throw new Error('WebSocket constructor failed'); + } + ControlledWebSocket.instances.push(this); + ControlledWebSocket.maxConcurrent = Math.max( + ControlledWebSocket.maxConcurrent, + ControlledWebSocket.instances.filter(socket => socket.readyState !== 3).length + ); + } + + send(data: string): void { + if (this.readyState !== 1) { + throw new Error('Socket is not open'); + } + this.sent.push(data); + } + + open(): void { + this.readyState = 1; + this.onopen?.(new Event('open')); + } + + fail(): void { + this.onerror?.(new Event('error')); + } + + finishClose(code = 1006): void { + this.readyState = 3; + const event = new CloseEvent('close', { code, wasClean: code === 1000 }); + this.dispatchEvent(event); + this.onclose?.(event); + } + + receive(data: unknown): void { + this.onmessage?.(new MessageEvent('message', { data: JSON.stringify(data) })); + } +} + +const ticket = { principal: 'anonymous', ticket: 'ticket', roles: '[]' }; +const url = 'ws://zeppelin.test/ws'; +const messages: Message[] = []; +const createMessage = () => { + const message = new Message(); + messages.push(message); + return message; +}; +const latestSocket = () => ControlledWebSocket.instances[ControlledWebSocket.instances.length - 1]; + +beforeEach(() => { + vi.useFakeTimers(); + ControlledWebSocket.instances = []; + ControlledWebSocket.attempts = 0; + ControlledWebSocket.failuresRemaining = 0; + ControlledWebSocket.maxConcurrent = 0; + vi.stubGlobal('WebSocket', ControlledWebSocket); + vi.spyOn(console, 'log').mockImplementation(() => {}); +}); + +afterEach(() => { + for (const message of messages.splice(0)) { + message.destroy(); + } + for (const socket of ControlledWebSocket.instances) { + if (socket.readyState !== 3) { + socket.finishClose(1000); + } + } + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe('Message connection lifecycle', () => { + it('requires connection intent, stores offline bootstrap, and makes repeated bootstrap idempotent', () => { + const message = createMessage(); + message.setOnline(false); + message.setOnline(true); + vi.advanceTimersByTime(60000); + expect(ControlledWebSocket.attempts).toBe(0); + + message.setOnline(false); + message.bootstrap(ticket, url); + message.bootstrap({ ...ticket, ticket: 'new-ticket' }, url); + vi.advanceTimersByTime(60000); + expect(ControlledWebSocket.attempts).toBe(0); + + message.setOnline(true); + const subject = message.getWsInstance(); + message.setOnline(true); + message.connect(); + message.bootstrap({ ...ticket, ticket: 'new-ticket' }, url); + expect(ControlledWebSocket.attempts).toBe(1); + expect(message.getWsInstance()).toBe(subject); + expect(latestSocket().url).toBe(url); + + latestSocket().open(); + message.connect(); + message.bootstrap({ ...ticket, ticket: 'new-ticket' }, url); + message.ping(); + expect(ControlledWebSocket.attempts).toBe(1); + expect(JSON.parse(latestSocket().sent[0])).toMatchObject({ + op: OP.PING, + principal: ticket.principal, + ticket: 'new-ticket' + }); + }); + + it('uses refreshed credentials for a command queued while initially connecting', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + message.listNodes(); + message.bootstrap({ ...ticket, ticket: 'refreshed-ticket' }, url); + latestSocket().open(); + expect(latestSocket().sent.map(data => JSON.parse(data))).toEqual([ + expect.objectContaining({ op: OP.LIST_NOTES, principal: ticket.principal, ticket: 'refreshed-ticket' }) + ]); + expect(ControlledWebSocket.attempts).toBe(1); + }); + + it.each(['connect', 'bootstrap'] as const)( + 'uses the original connection identity after setters run before %s', + configuration => { + const message = createMessage(); + message.bootstrap(ticket, url); + const previous = latestSocket(); + previous.open(); + const nextTicket = { ...ticket, principal: 'next-user', ticket: 'next-ticket' }; + const nextUrl = 'ws://next-zeppelin.test/ws'; + message.setTicket(nextTicket); + message.setWsUrl(nextUrl); + if (configuration === 'connect') { + message.connect(); + } else { + message.bootstrap(nextTicket, nextUrl); + } + expect(ControlledWebSocket.attempts).toBe(1); + previous.finishClose(1000); + const replacement = latestSocket(); + expect(replacement.url).toBe(nextUrl); + replacement.open(); + message.ping(); + expect(replacement.sent.map(data => JSON.parse(data))).toEqual([ + expect.objectContaining({ op: OP.PING, principal: 'next-user', ticket: 'next-ticket' }) + ]); + expect(ControlledWebSocket.attempts).toBe(2); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + } + ); + + it.each(['offline', 'retry'] as const)('cancels the old identity backlog and retry while %s', state => { + const message = createMessage(); + message.setOnline(false); + message.bootstrap(ticket, url); + if (state === 'retry') { + message.setOnline(true); + latestSocket().finishClose(); + } + message.commitParagraph('p1', undefined, 'old-user edit', {}, {}, 'n1'); + message.setTicket({ ...ticket, principal: 'next-user', ticket: 'next-ticket' }); + message.connect(); + message.listNodes(); + message.setOnline(true); + latestSocket().open(); + vi.advanceTimersByTime(2000); + expect(latestSocket().sent.map(data => JSON.parse(data))).toEqual([ + expect.objectContaining({ op: OP.LIST_NOTES, principal: 'next-user', ticket: 'next-ticket' }) + ]); + expect(ControlledWebSocket.attempts).toBe(state === 'offline' ? 1 : 2); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('does not flush a previous principal backlog when its connecting socket opens after a setter change', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + const previous = latestSocket(); + message.commitParagraph('p1', undefined, 'old-user edit', {}, {}, 'n1'); + message.setTicket({ ...ticket, principal: 'next-user', ticket: 'next-ticket' }); + previous.open(); + expect(previous.sent).toEqual([]); + expect(message.connectedStatus).toBe(false); + previous.finishClose(1000); + latestSocket().open(); + message.ping(); + expect(latestSocket().sent.map(data => JSON.parse(data))).toEqual([ + expect.objectContaining({ op: OP.PING, principal: 'next-user', ticket: 'next-ticket' }) + ]); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('moves a command to the new principal instead of sending it on the old open socket', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + const previous = latestSocket(); + previous.open(); + message.setTicket({ ...ticket, principal: 'next-user', ticket: 'next-ticket' }); + message.listNodes(); + expect(previous.sent).toEqual([]); + previous.finishClose(1000); + latestSocket().open(); + expect(latestSocket().sent.map(data => JSON.parse(data))).toEqual([ + expect.objectContaining({ op: OP.LIST_NOTES, principal: 'next-user', ticket: 'next-ticket' }) + ]); + expect(ControlledWebSocket.attempts).toBe(2); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('keeps sending in order on an open socket while its public status is changed', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + latestSocket().open(); + message.connectedStatus = false; + message.listNodes(); + message.connectedStatus = true; + message.ping(); + expect(latestSocket().sent.map(data => JSON.parse(data).op)).toEqual([OP.LIST_NOTES, OP.PING]); + expect(ControlledWebSocket.attempts).toBe(1); + }); + + it('keeps connecting commands queued with current credentials even if public status is set true', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + message.connectedStatus = true; + message.listNodes(); + message.setTicket({ ...ticket, ticket: 'refreshed-ticket' }); + message.connect(); + expect(latestSocket().sent).toEqual([]); + latestSocket().open(); + message.ping(); + expect(latestSocket().sent.map(data => JSON.parse(data))).toEqual([ + expect.objectContaining({ op: OP.LIST_NOTES, ticket: 'refreshed-ticket' }), + expect.objectContaining({ op: OP.PING, ticket: 'refreshed-ticket' }) + ]); + expect(ControlledWebSocket.attempts).toBe(1); + }); + + it.each(['connecting', 'open', 'retry', 'offline', 'closing'] as const)( + 'replaces the server-bound principal while %s without overlapping sockets', + state => { + const message = createMessage(); + message.bootstrap(ticket, url); + const previous = latestSocket(); + if (state === 'open' || state === 'offline' || state === 'closing') { + previous.open(); + } + if (state === 'retry') { + previous.finishClose(); + } else if (state === 'offline') { + message.setOnline(false); + message.listNodes(); + } else if (state === 'closing') { + message.close(); + } + + message.bootstrap({ ...ticket, principal: 'next-user', ticket: 'next-ticket' }, url); + expect(ControlledWebSocket.attempts).toBe(state === 'retry' ? 2 : 1); + if (previous.readyState !== 3) { + previous.finishClose(1000); + } + expect(ControlledWebSocket.attempts).toBe(state === 'offline' ? 1 : 2); + if (state === 'offline') { + message.setOnline(true); + } + const replacement = latestSocket(); + expect(replacement).not.toBe(previous); + replacement.open(); + message.ping(); + expect(replacement.sent.map(data => JSON.parse(data))).toEqual([ + expect.objectContaining({ op: OP.PING, principal: 'next-user', ticket: 'next-ticket' }) + ]); + expect(ControlledWebSocket.attempts).toBe(2); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + } + ); + + it('uses a changed endpoint even when a disconnected-status subscriber requests connection', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + const previous = latestSocket(); + previous.open(); + message.connectedStatus$.subscribe(connected => { + if (!connected) { + message.connect(); + } + }); + + message.bootstrap(ticket, 'ws://other-zeppelin.test/ws'); + previous.finishClose(1000); + expect(latestSocket().url).toBe('ws://other-zeppelin.test/ws'); + expect(ControlledWebSocket.attempts).toBe(2); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it.each(['close', 'destroy'] as const)('honors reentrant %s while reconciling a changed connection', action => { + const message = createMessage(); + message.bootstrap(ticket, url); + const previous = latestSocket(); + previous.open(); + const subscription = message.connectedStatus$.subscribe(connected => { + if (!connected) { + subscription.unsubscribe(); + message[action](); + } + }); + message.setWsUrl('ws://next-zeppelin.test/ws'); + message.connect(); + previous.finishClose(1000); + vi.advanceTimersByTime(60000); + expect(ControlledWebSocket.attempts).toBe(1); + expect(message.connectedStatus).toBe(false); + expect(() => message.listNodes()).toThrow('Bootstrap first'); + }); + + it('uses the latest reentrant identity rather than an earlier replacement snapshot', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + const previous = latestSocket(); + previous.open(); + const subscription = message.connectedStatus$.subscribe(connected => { + if (!connected) { + subscription.unsubscribe(); + message.setTicket({ ...ticket, principal: 'third-user', ticket: 'third-ticket' }); + message.setWsUrl('ws://third-zeppelin.test/ws'); + message.connect(); + } + }); + message.setTicket({ ...ticket, principal: 'next-user', ticket: 'next-ticket' }); + message.setWsUrl('ws://next-zeppelin.test/ws'); + message.connect(); + previous.finishClose(1000); + latestSocket().open(); + message.ping(); + expect(ControlledWebSocket.instances.map(socket => socket.url)).toEqual([url, 'ws://third-zeppelin.test/ws']); + expect(latestSocket().sent.map(data => JSON.parse(data))).toEqual([ + expect.objectContaining({ op: OP.PING, principal: 'third-user', ticket: 'third-ticket' }) + ]); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('preserves offline and connecting send order before the host rehydrates the note', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + const previous = latestSocket(); + previous.open(); + message.setOnline(false); + message.listNodes(); + const first = message.commitParagraph('p1', undefined, 'offline edit', {}, {}, 'n1'); + message.setOnline(true); + previous.finishClose(1000); + const second = message.commitParagraph('p1', undefined, 'newer edit', {}, {}, 'n1'); + message.bootstrap({ ...ticket, ticket: 'refreshed-ticket' }, url); + message.connectedStatus$.subscribe(connected => { + if (connected) { + message.getNote('n1'); + } + }); + latestSocket().open(); + + const frames = latestSocket().sent.map(data => JSON.parse(data)); + expect(frames.map(frame => frame.op)).toEqual([ + OP.LIST_NOTES, + OP.COMMIT_PARAGRAPH, + OP.COMMIT_PARAGRAPH, + OP.GET_NOTE + ]); + expect(frames.slice(1, 3).map(frame => [frame.msgId, frame.data.paragraph])).toEqual([ + [first, 'offline edit'], + [second, 'newer edit'] + ]); + expect(frames.map(frame => frame.ticket)).toEqual(Array(4).fill('refreshed-ticket')); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('discards unsent work when an explicit close cancels the connection request', () => { + const message = createMessage(); + message.setOnline(false); + message.bootstrap(ticket, url); + message.commitParagraph('p1', undefined, 'cancelled edit', {}, {}, 'n1'); + message.close(); + expect(() => message.listNodes()).toThrow('Bootstrap first'); + message.connect(); + message.setOnline(true); + latestSocket().open(); + message.ping(); + expect(latestSocket().sent.map(data => JSON.parse(data).op)).toEqual([OP.PING]); + }); + + it('uses one exponential retry from one second to the thirty-second cap and resets after open', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + + for (const delay of [1000, 2000, 4000, 8000, 16000, 30000, 30000]) { + const attempts = ControlledWebSocket.attempts; + latestSocket().fail(); + latestSocket().fail(); + latestSocket().finishClose(); + message.connect(); + message.bootstrap(ticket, url); + vi.advanceTimersByTime(delay - 1); + expect(ControlledWebSocket.attempts).toBe(attempts); + vi.advanceTimersByTime(1); + expect(ControlledWebSocket.attempts).toBe(attempts + 1); + } + + latestSocket().open(); + const attempts = ControlledWebSocket.attempts; + latestSocket().finishClose(); + vi.advanceTimersByTime(999); + expect(ControlledWebSocket.attempts).toBe(attempts); + vi.advanceTimersByTime(1); + expect(ControlledWebSocket.attempts).toBe(attempts + 1); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('reports valid open, receives messages, and only pings an open connection', () => { + const message = createMessage(); + const status = vi.fn(); + const opened = vi.fn(); + const closed = vi.fn(); + const received = vi.fn(); + message.connectedStatus$.subscribe(status); + message.opened().subscribe(opened); + message.closed().subscribe(closed); + message.received().subscribe(received); + message.bootstrap(ticket, url); + const socket = latestSocket(); + socket.open(); + socket.open(); + socket.receive({ op: OP.NOTE, data: { note: { id: 'note-1' } } }); + vi.advanceTimersByTime(10000); + + expect(message.connectedStatus).toBe(true); + expect(status).toHaveBeenLastCalledWith(true); + expect(opened).toHaveBeenCalledTimes(1); + expect(received).toHaveBeenCalledWith({ op: OP.NOTE, data: { note: { id: 'note-1' } } }); + expect(socket.sent.map(data => JSON.parse(data).op)).toEqual([OP.PING]); + + socket.finishClose(); + expect(message.connectedStatus).toBe(false); + expect(status).toHaveBeenLastCalledWith(false); + expect(closed).toHaveBeenCalledOnce(); + expect(closed.mock.calls[0][0].code).toBe(1006); + vi.advanceTimersByTime(20000); + expect(socket.sent).toHaveLength(1); + expect(latestSocket().sent).toEqual([]); + }); + + it('waits for physical close even after retry is due and ignores callbacks from the old socket', () => { + const message = createMessage(); + const opened = vi.fn(); + const closed = vi.fn(); + const received = vi.fn(); + message.opened().subscribe(opened); + message.closed().subscribe(closed); + message.received().subscribe(received); + message.bootstrap(ticket, url); + const old = latestSocket(); + old.open(); + const staleOpen = old.onopen; + const staleClose = old.onclose; + const staleError = old.onerror; + const staleReceive = old.onmessage; + old.fail(); + expect(old.readyState).toBe(2); + expect(message.connectedStatus).toBe(false); + expect(message.getWsInstance()).toBeNull(); + + vi.advanceTimersByTime(60000); + message.connect(); + message.bootstrap(ticket, url); + expect(ControlledWebSocket.attempts).toBe(1); + old.finishClose(); + expect(ControlledWebSocket.attempts).toBe(2); + const replacement = latestSocket(); + replacement.open(); + + staleOpen?.(new Event('open')); + staleError?.(new Event('error')); + staleClose?.(new CloseEvent('close', { code: 1006 })); + old.finishClose(); + staleReceive?.(new MessageEvent('message', { data: JSON.stringify({ op: OP.NOTE, data: {} }) })); + vi.advanceTimersByTime(5000); + expect(message.connectedStatus).toBe(true); + expect(opened).toHaveBeenCalledTimes(2); + expect(closed).not.toHaveBeenCalled(); + expect(received).not.toHaveBeenCalled(); + expect(ControlledWebSocket.attempts).toBe(2); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('does not reset retry backoff or status for a retired open callback', () => { + const message = createMessage(); + const opened = vi.fn(); + message.opened().subscribe(opened); + message.bootstrap(ticket, url); + const first = latestSocket(); + const staleOpen = first.onopen; + first.fail(); + first.finishClose(); + vi.advanceTimersByTime(1000); + const second = latestSocket(); + const retiredOpen = second.onopen; + second.fail(); + second.finishClose(); + + staleOpen?.(new Event('open')); + retiredOpen?.(new Event('open')); + expect(message.connectedStatus).toBe(false); + expect(opened).not.toHaveBeenCalled(); + vi.advanceTimersByTime(1999); + expect(ControlledWebSocket.attempts).toBe(2); + vi.advanceTimersByTime(1); + expect(ControlledWebSocket.attempts).toBe(3); + latestSocket().open(); + expect(message.connectedStatus).toBe(true); + expect(opened).toHaveBeenCalledOnce(); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('pauses connecting and open sockets offline, then resumes each intent only once', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + const connecting = latestSocket(); + message.setOnline(false); + expect(connecting.close).toHaveBeenCalledOnce(); + expect(connecting.readyState).toBe(2); + expect(message.getWsInstance()).toBeNull(); + message.setOnline(true); + message.setOnline(true); + vi.advanceTimersByTime(60000); + expect(ControlledWebSocket.attempts).toBe(1); + + connecting.finishClose(); + expect(ControlledWebSocket.attempts).toBe(2); + const open = latestSocket(); + open.open(); + message.setOnline(false); + expect(message.connectedStatus).toBe(false); + open.finishClose(); + vi.advanceTimersByTime(60000); + expect(ControlledWebSocket.attempts).toBe(2); + expect(open.sent).toEqual([]); + + message.setOnline(true); + message.setOnline(true); + expect(ControlledWebSocket.attempts).toBe(3); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('cancels a pending retry offline and immediately resumes once when online', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + latestSocket().finishClose(); + vi.advanceTimersByTime(500); + message.setOnline(false); + vi.advanceTimersByTime(60000); + expect(ControlledWebSocket.attempts).toBe(1); + message.setOnline(true); + message.setOnline(true); + expect(ControlledWebSocket.attempts).toBe(2); + vi.advanceTimersByTime(5000); + expect(ControlledWebSocket.attempts).toBe(2); + }); + + it('physically closes, cancels intent, and permits only a new explicit connect to reopen', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + const socket = latestSocket(); + socket.open(); + message.close(); + message.close(); + expect(socket.close).toHaveBeenCalledOnce(); + expect(message.connectedStatus).toBe(false); + expect(message.getWsInstance()).toBeNull(); + message.setOnline(false); + message.setOnline(true); + socket.finishClose(); + vi.advanceTimersByTime(60000); + expect(ControlledWebSocket.attempts).toBe(1); + + message.connect(); + expect(ControlledWebSocket.attempts).toBe(2); + latestSocket().finishClose(); + message.close(); + vi.advanceTimersByTime(60000); + expect(ControlledWebSocket.attempts).toBe(2); + }); + + it('waits for an explicitly closed socket before honoring a new bootstrap', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + const socket = latestSocket(); + socket.open(); + message.close(); + message.bootstrap(ticket, url); + message.connect(); + expect(ControlledWebSocket.attempts).toBe(1); + socket.finishClose(1000); + expect(ControlledWebSocket.attempts).toBe(2); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('cancels a connecting socket despite late open and error callbacks', () => { + const message = createMessage(); + const opened = vi.fn(); + const received = vi.fn(); + message.opened().subscribe(opened); + message.received().subscribe(received); + message.bootstrap(ticket, url); + const socket = latestSocket(); + const staleOpen = socket.onopen; + const staleError = socket.onerror; + const staleReceive = socket.onmessage; + + message.close(); + expect(socket.close).toHaveBeenCalledOnce(); + staleOpen?.(new Event('open')); + staleError?.(new Event('error')); + staleReceive?.(new MessageEvent('message', { data: JSON.stringify({ op: OP.NOTE, data: {} }) })); + message.setOnline(false); + message.setOnline(true); + socket.finishClose(); + vi.advanceTimersByTime(120000); + + expect(message.connectedStatus).toBe(false); + expect(message.getWsInstance()).toBeNull(); + expect(opened).not.toHaveBeenCalled(); + expect(received).not.toHaveBeenCalled(); + expect(socket.sent).toEqual([]); + expect(ControlledWebSocket.attempts).toBe(1); + expect(vi.getTimerCount()).toBe(0); + }); + + it('does not deliver a message when its interceptor closes the connection', () => { + class ClosingMessage extends Message { + interceptReceived(data: Parameters[0]) { + this.close(); + return data; + } + } + const message = new ClosingMessage(); + messages.push(message); + const received = vi.fn(); + message.received().subscribe(received); + message.bootstrap(ticket, url); + const socket = latestSocket(); + socket.open(); + socket.receive({ op: OP.NOTE, data: {} }); + socket.finishClose(); + vi.advanceTimersByTime(120000); + + expect(received).not.toHaveBeenCalled(); + expect(message.connectedStatus).toBe(false); + expect(ControlledWebSocket.attempts).toBe(1); + expect(vi.getTimerCount()).toBe(0); + }); + + it('does not automatically retry a normal server close', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + latestSocket().open(); + latestSocket().finishClose(1000); + vi.advanceTimersByTime(120000); + expect(message.connectedStatus).toBe(false); + expect(ControlledWebSocket.attempts).toBe(1); + message.setOnline(false); + message.setOnline(true); + expect(ControlledWebSocket.attempts).toBe(1); + }); + + it('handles synchronous constructor failure without an unhandled error or duplicate retry', () => { + ControlledWebSocket.failuresRemaining = 2; + const message = createMessage(); + expect(() => message.bootstrap(ticket, url)).not.toThrow(); + expect(ControlledWebSocket.attempts).toBe(1); + message.connect(); + vi.advanceTimersByTime(999); + expect(ControlledWebSocket.attempts).toBe(1); + vi.advanceTimersByTime(1); + expect(ControlledWebSocket.attempts).toBe(2); + vi.advanceTimersByTime(1999); + expect(ControlledWebSocket.attempts).toBe(2); + vi.advanceTimersByTime(1); + expect(ControlledWebSocket.attempts).toBe(3); + latestSocket().open(); + expect(message.connectedStatus).toBe(true); + expect(ControlledWebSocket.maxConcurrent).toBe(1); + }); + + it('allows destruction by an open-status subscriber without resurrecting ping or open events', () => { + const message = createMessage(); + const opened = vi.fn(); + message.opened().subscribe(opened); + message.connectedStatus$.subscribe(connected => { + if (connected) { + message.destroy(); + } + }); + message.bootstrap(ticket, url); + const socket = latestSocket(); + socket.open(); + socket.finishClose(); + vi.advanceTimersByTime(120000); + expect(message.connectedStatus).toBe(false); + expect(opened).not.toHaveBeenCalled(); + expect(socket.sent).toEqual([]); + expect(ControlledWebSocket.attempts).toBe(1); + expect(vi.getTimerCount()).toBe(0); + }); + + it('dispatches heartbeat through the current Message override', () => { + const message = createMessage(); + message.bootstrap(ticket, url); + latestSocket().open(); + message.ping = () => message.listNodes(); + vi.advanceTimersByTime(10000); + expect(latestSocket().sent.map(data => JSON.parse(data).op)).toEqual([OP.LIST_NOTES]); + message.close(); + vi.advanceTimersByTime(10000); + expect(latestSocket().sent.map(data => JSON.parse(data).op)).toEqual([OP.LIST_NOTES]); + }); + + it('completes facade streams once in order even when disconnect triggers another destroy', () => { + const message = createMessage(); + const events: string[] = []; + message.connectedStatus$.subscribe({ + next: connected => { + if (!connected) { + events.push('disconnected'); + message.destroy(); + } + }, + complete: () => events.push('status complete') + }); + message.opened().subscribe({ complete: () => events.push('open complete') }); + message.closed().subscribe({ complete: () => events.push('close complete') }); + message.sent().subscribe({ complete: () => events.push('sent complete') }); + message.received().subscribe({ complete: () => events.push('received complete') }); + message.bootstrap(ticket, url); + latestSocket().open(); + message.destroy(); + latestSocket().finishClose(1000); + message.destroy(); + expect(events).toEqual([ + 'disconnected', + 'status complete', + 'open complete', + 'close complete', + 'sent complete', + 'received complete' + ]); + }); + + it.each(['connecting', 'open', 'retry', 'offline', 'closing'] as const)('makes destroy terminal while %s', state => { + const message = createMessage(); + const completed = vi.fn(); + const received = vi.fn(); + message.connectedStatus$.subscribe({ complete: completed }); + message.received().subscribe(received); + message.bootstrap(ticket, url); + const socket = latestSocket(); + if (state === 'open' || state === 'offline' || state === 'closing') { + socket.open(); + } else if (state === 'retry') { + socket.finishClose(); + } + if (state === 'offline') { + message.setOnline(false); + } else if (state === 'closing') { + message.close(); + } + + message.destroy(); + message.destroy(); + expect(message.connectedStatus).toBe(false); + expect(message.getWsInstance()).toBeNull(); + expect(completed).toHaveBeenCalledOnce(); + expect(() => message.connect()).not.toThrow(); + message.bootstrap(ticket, url); + message.setOnline(false); + message.setOnline(true); + socket.finishClose(); + socket.onopen?.(new Event('open')); + socket.fail(); + socket.receive({ op: OP.NOTE, data: {} }); + vi.advanceTimersByTime(120000); + expect(received).not.toHaveBeenCalled(); + expect(socket.sent).toEqual([]); + expect(ControlledWebSocket.attempts).toBe(1); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/zeppelin-web-angular/projects/zeppelin-sdk/src/message.spec.ts b/zeppelin-web-angular/projects/zeppelin-sdk/src/message.spec.ts index 4fc8cbbc2ae..a564dc08a55 100644 --- a/zeppelin-web-angular/projects/zeppelin-sdk/src/message.spec.ts +++ b/zeppelin-web-angular/projects/zeppelin-sdk/src/message.spec.ts @@ -164,15 +164,25 @@ describe('Message.receive', () => { }); describe('Message.send', () => { - const connectedMessage = () => { + const messages: Message[] = []; + afterEach(() => { + for (const message of messages.splice(0)) { + message.destroy(); + } + }); + + const offlineMessage = () => { const message = new Message(); + messages.push(message); + message.setOnline(false); + message.bootstrap({ principal: 'anonymous', ticket: 'ticket', roles: '[]' }, 'ws://zeppelin.test/ws'); const next = vi.fn(); - (message as unknown as { ws: { next: typeof next } }).ws = { next }; + message.sent().subscribe(next); return { message, next }; }; it('returns the msgId of each sent message', () => { - const { message, next } = connectedMessage(); + const { message, next } = offlineMessage(); const commitMsgId = message.commitParagraph('p1', undefined, 'text', {}, {}, 'n1'); const insertMsgId = message.insertParagraph(0); @@ -183,7 +193,7 @@ describe('Message.send', () => { }); it('returns the msgId of a note read and a revision read', () => { - const { message, next } = connectedMessage(); + const { message, next } = offlineMessage(); const noteMsgId = message.getNote('n1'); const revisionMsgId = message.noteRevision('n1', 'r1'); diff --git a/zeppelin-web-angular/projects/zeppelin-sdk/src/message.ts b/zeppelin-web-angular/projects/zeppelin-sdk/src/message.ts index 21fb29bb94f..1deb7f68d49 100644 --- a/zeppelin-web-angular/projects/zeppelin-sdk/src/message.ts +++ b/zeppelin-web-angular/projects/zeppelin-sdk/src/message.ts @@ -10,9 +10,9 @@ * limitations under the License. */ -import { interval, Observable, Subject, Subscription } from 'rxjs'; -import { delay, filter, map, mergeMap, retryWhen, take } from 'rxjs/operators'; -import { webSocket, WebSocketSubject } from 'rxjs/webSocket'; +import { Observable, Subject } from 'rxjs'; +import { filter, map } from 'rxjs/operators'; +import { WebSocketSubject } from 'rxjs/webSocket'; import { Ticket } from './interfaces/message-common.interface'; import { @@ -25,6 +25,7 @@ import { OP } from './interfaces/message-operator.interface'; import { ParagraphConfig, ParagraphParams, SendParagraph } from './interfaces/message-paragraph.interface'; import { WebSocketMessage } from './interfaces/websocket-message.interface'; +import { MessageConnection } from './message-connection'; import { getMessagePayloadGuard } from './message-payload-guards'; export type ArgumentsType = T extends (...args: infer U) => void ? U : never; @@ -37,57 +38,51 @@ export type ReceiveArgumentsType = MessageReceiveDataTypeMap[K] extends undefined ? () => void : (data: MessageReceiveDataTypeMap[K]) => void; export class Message { - public connectedStatus = false; - public connectedStatus$ = new Subject(); - private ws: WebSocketSubject> | null = null; - private wsSubscription: Subscription | null = null; - private open$ = new Subject(); - private close$ = new Subject(); - private sent$ = new Subject>(); private received$ = new Subject>(); - private pingIntervalSubscription = new Subscription(); - private wsUrl?: string; - private ticket?: Ticket; private uniqueClientId = Math.random().toString(36).substring(2, 7); // TODO: Clean up this variable with `msgId` in server-side. See ZEPPELIN-6419, ZEPPELIN-4985 private lastMsgIdSeqSent = 0; - private readonly normalCloseCode = 1000; - - constructor() { - this.open$.subscribe(() => { - this.connectedStatus = true; - this.connectedStatus$.next(this.connectedStatus); - this.pingIntervalSubscription.unsubscribe(); - this.pingIntervalSubscription = interval(1000 * 10).subscribe(() => this.ping()); - }); - this.close$.subscribe(event => { - this.connectedStatus = false; - this.connectedStatus$.next(this.connectedStatus); - this.pingIntervalSubscription.unsubscribe(); - - if (event.code !== this.normalCloseCode) { - console.log('WebSocket closed unexpectedly. Reconnecting...'); - this.connect(); - } - }); + private readonly connection = new MessageConnection({ + nextMessageId: () => `${this.uniqueClientId}-${++this.lastMsgIdSeqSent}`, + interceptReceived: message => this.interceptReceived(message), + received: this.received$, + ping: () => this.ping() + }); + + get connectedStatus(): boolean { + return this.connection.connectedStatus; + } + + get connectedStatus$(): Subject { + return this.connection.connectedStatus$; + } + + set connectedStatus(connected: boolean) { + this.connection.connectedStatus = connected; + } + + set connectedStatus$(status: Subject) { + this.connection.connectedStatus$ = status; + } + + setOnline(online: boolean): void { + this.connection.setOnline(online); } bootstrap(ticket: Ticket, wsUrl: string) { - this.setTicket(ticket); - this.setWsUrl(wsUrl); - this.connect(); + this.connection.bootstrap(ticket, wsUrl); } - getWsInstance(): Message['ws'] { - return this.ws; + getWsInstance(): WebSocketSubject> | null { + return this.connection.getWsInstance(); } setWsUrl(wsUrl: string): void { - this.wsUrl = wsUrl; + this.connection.setWsUrl(wsUrl); } setTicket(ticket: Ticket): void { - this.ticket = ticket; + this.connection.setTicket(ticket); } interceptReceived(data: WebSocketMessage): WebSocketMessage { @@ -95,37 +90,7 @@ export class Message { } connect() { - if (!this.wsUrl) { - throw new Error('WebSocket URL is not set. Please call setWsUrl() before connect()'); - } - - // Unsubscribe from existing subscription first - if (this.wsSubscription) { - this.wsSubscription.unsubscribe(); - this.wsSubscription = null; - } - - // Then close existing WebSocket - if (this.ws) { - this.ws.complete(); - this.ws = null; - } - - this.ws = webSocket>({ - url: this.wsUrl, - openObserver: this.open$, - closeObserver: this.close$ - }); - - this.wsSubscription = this.ws - .pipe( - // reconnect - retryWhen(errors => errors.pipe(mergeMap(() => this.close$.pipe(take(1), delay(4000))))) - ) - .subscribe(e => { - console.log('Receive:', e.op); - this.received$.next(this.interceptReceived(e as WebSocketMessage)); - }); + this.connection.connect(); } ping() { @@ -133,19 +98,19 @@ export class Message { } close() { - this.close$.next(new CloseEvent('close', { code: this.normalCloseCode })); + this.connection.close(); } opened(): Observable { - return this.open$.asObservable(); + return this.connection.opened(); } closed(): Observable { - return this.close$.asObservable(); + return this.connection.closed(); } sent(): Observable> { - return this.sent$.asObservable(); + return this.connection.sent(); } received(): Observable> { @@ -153,21 +118,8 @@ export class Message { } send(...args: SendArgumentsType): string { - if (!this.ws) { - throw new Error('WebSocket is not connected. Bootstrap first.'); - } const [op, data] = args; - const message = { - op, - msgId: `${this.uniqueClientId}-${++this.lastMsgIdSeqSent}`, - data, - ...this.ticket - }; - console.log('Send:', message.op, message.principal); - - this.ws.next(message); - this.sent$.next(message); - return message.msgId; + return this.connection.send(op, data); } receive(op: K): Observable[K]> { @@ -187,14 +139,7 @@ export class Message { } destroy(): void { - if (this.wsSubscription) { - this.wsSubscription.unsubscribe(); - this.wsSubscription = null; - } - if (this.ws) { - this.ws.complete(); - this.ws = null; - } + this.connection.destroy(); } getHomeNote(): void { diff --git a/zeppelin-web-angular/src/app/pages/workspace/workspace.component.ts b/zeppelin-web-angular/src/app/pages/workspace/workspace.component.ts index ca2b8b9a3f3..83469a9586a 100644 --- a/zeppelin-web-angular/src/app/pages/workspace/workspace.component.ts +++ b/zeppelin-web-angular/src/app/pages/workspace/workspace.component.ts @@ -45,13 +45,17 @@ export class WorkspaceComponent implements OnInit, OnDestroy { } /** - * Close the old connection manually when the network is offline - * and connect a new, the {@link MessageService} will auto-retry + * Close the old connection when the network is offline. + * {@link MessageService} pauses retries until the browser comes online again. */ @HostListener('window:offline') onOffline() { - this.messageService.close(); - this.messageService.connect(); + this.messageService.setOnline(false); + } + + @HostListener('window:online') + onOnline() { + this.messageService.setOnline(true); } setUpWebsocketReconnectMessage() { diff --git a/zeppelin-web-angular/src/app/services/message.service.spec.ts b/zeppelin-web-angular/src/app/services/message.service.spec.ts index e1269c32e5f..be6cb0a6d32 100644 --- a/zeppelin-web-angular/src/app/services/message.service.spec.ts +++ b/zeppelin-web-angular/src/app/services/message.service.spec.ts @@ -10,25 +10,41 @@ * limitations under the License. */ -import { describe, expect, it, vi } from 'vitest'; +import { TestBed } from '@angular/core/testing'; +import { NzMessageService } from 'ng-zorro-antd/message'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { MessageInterceptor } from '@zeppelin/interfaces'; +import { WorkspaceComponent } from '../pages/workspace/workspace.component'; import { BaseUrlService } from './base-url.service'; import { MessageService } from './message.service'; import { TicketService } from './ticket.service'; -const connectedMessageService = (): MessageService => { +vi.mock('@zeppelin/visualizations', () => ({ setTheme: vi.fn() })); + +const offlineServices: MessageService[] = []; +const offlineMessageService = (): MessageService => { const service = new MessageService({} as BaseUrlService, {} as TicketService, null as unknown as MessageInterceptor); - (service as unknown as { ws: { next: () => void } }).ws = { next: vi.fn() }; + offlineServices.push(service); + service.setOnline(false); + service.setTicket({ principal: 'viewer', ticket: 'ticket', roles: '[]' }); + service.setWsUrl('ws://localhost/ws'); + service.connect(); return service; }; describe('MessageService local add focus', () => { + afterEach(() => { + for (const service of offlineServices.splice(0)) { + service.ngOnDestroy(); + } + }); + it.each([ ['insertParagraph', (service: MessageService) => service.insertParagraph(0)], ['copyParagraph', (service: MessageService) => service.copyParagraph(1, undefined, 'text', {}, {})] ])('records the msgId returned by %s once', (_method, add) => { - const service = connectedMessageService(); + const service = offlineMessageService(); const msgId = add(service); @@ -37,9 +53,163 @@ describe('MessageService local add focus', () => { }); it('does not record a msgId for a paragraph added by another client', () => { - const service = connectedMessageService(); + const service = offlineMessageService(); service.insertParagraph(0); expect(service.consumeLocalAddFocusMsgId('other-client-1')).toBe(false); }); }); + +class BootstrapWebSocket extends EventTarget { + static instances: BootstrapWebSocket[] = []; + readyState = 0; + onopen: ((event: Event) => void) | null = null; + onclose: ((event: CloseEvent) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + send = vi.fn(); + + constructor(readonly url: string) { + super(); + BootstrapWebSocket.instances.push(this); + } + + open(): void { + this.readyState = 1; + const event = new Event('open'); + this.onopen?.(event); + this.dispatchEvent(event); + } + + close(): void { + this.readyState = 3; + const event = new CloseEvent('close', { code: 1000, wasClean: true }); + this.onclose?.(event); + this.dispatchEvent(event); + } +} + +describe('MessageService browser connectivity at bootstrap', () => { + let service: MessageService; + const ticket = { principal: 'viewer', ticket: 'viewer-ticket', roles: '[]' }; + const url = 'ws://localhost/ws'; + + beforeEach(() => { + vi.useFakeTimers(); + BootstrapWebSocket.instances = []; + vi.stubGlobal('WebSocket', BootstrapWebSocket); + service = new MessageService( + { getWebsocketUrl: () => url } as BaseUrlService, + { originTicket: ticket } as TicketService, + null as unknown as MessageInterceptor + ); + }); + + afterEach(() => { + service.ngOnDestroy(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it('does not attempt an offline bootstrap and resumes once with the original URL and ticket', () => { + vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); + const connected = vi.fn(); + service.connectedStatus$.subscribe(connected); + + service.bootstrap(); + vi.advanceTimersByTime(60000); + + expect(BootstrapWebSocket.instances).toHaveLength(0); + expect(service.connectedStatus).toBe(false); + + service.setOnline(true); + service.setOnline(true); + + expect(BootstrapWebSocket.instances).toHaveLength(1); + const socket = BootstrapWebSocket.instances[0]; + expect(socket.url).toBe(url); + socket.open(); + expect(service.connectedStatus).toBe(true); + expect(connected).toHaveBeenLastCalledWith(true); + + service.ping(); + expect(JSON.parse(socket.send.mock.calls[0][0] as string)).toMatchObject({ op: 'PING', ...ticket }); + }); + + it('opens an online bootstrap immediately without duplicating an in-flight connection', () => { + vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(true); + + service.bootstrap(); + service.bootstrap(); + + expect(BootstrapWebSocket.instances).toHaveLength(1); + BootstrapWebSocket.instances[0].open(); + expect(service.connectedStatus).toBe(true); + }); + + it('applies a new principal before resuming an offline connection at bootstrap', () => { + const online = vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); + const auth = { originTicket: ticket }; + service = new MessageService( + { getWebsocketUrl: () => url } as BaseUrlService, + auth as TicketService, + null as unknown as MessageInterceptor + ); + service.bootstrap(); + service.listNodes(); + auth.originTicket = { ...ticket, principal: 'next-user', ticket: 'next-ticket' }; + online.mockReturnValue(true); + service.bootstrap(); + + expect(BootstrapWebSocket.instances).toHaveLength(1); + const socket = BootstrapWebSocket.instances[0]; + socket.open(); + service.ping(); + expect(socket.send.mock.calls.map(([frame]) => JSON.parse(frame as string))).toEqual([ + expect.objectContaining({ op: 'PING', principal: 'next-user', ticket: 'next-ticket' }) + ]); + }); + + it('pauses and resumes physical connections through workspace browser events, including an initially offline page', () => { + const online = vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); + TestBed.configureTestingModule({ + declarations: [WorkspaceComponent], + providers: [ + { provide: MessageService, useValue: service }, + { provide: NzMessageService, useValue: { loading: () => ({ messageId: 'connecting' }), remove: vi.fn() } } + ] + }).overrideComponent(WorkspaceComponent, { + set: { template: '', templateUrl: undefined, styles: [], styleUrls: [] } + }); + const workspace = TestBed.createComponent(WorkspaceComponent); + workspace.detectChanges(); + service.bootstrap(); + vi.advanceTimersByTime(60000); + expect(BootstrapWebSocket.instances).toHaveLength(0); + + online.mockReturnValue(true); + window.dispatchEvent(new Event('online')); + window.dispatchEvent(new Event('online')); + expect(BootstrapWebSocket.instances).toHaveLength(1); + BootstrapWebSocket.instances[0].open(); + expect(service.connectedStatus).toBe(true); + + online.mockReturnValue(false); + window.dispatchEvent(new Event('offline')); + window.dispatchEvent(new Event('offline')); + expect(BootstrapWebSocket.instances[0].readyState).toBe(3); + expect(service.connectedStatus).toBe(false); + vi.advanceTimersByTime(60000); + expect(BootstrapWebSocket.instances).toHaveLength(1); + + online.mockReturnValue(true); + window.dispatchEvent(new Event('online')); + window.dispatchEvent(new Event('online')); + expect(BootstrapWebSocket.instances).toHaveLength(2); + BootstrapWebSocket.instances[1].open(); + expect(service.connectedStatus).toBe(true); + expect(BootstrapWebSocket.instances.filter(socket => socket.readyState === 1)).toHaveLength(1); + workspace.destroy(); + }); +}); diff --git a/zeppelin-web-angular/src/app/services/message.service.ts b/zeppelin-web-angular/src/app/services/message.service.ts index 7f714585105..02f226f2678 100644 --- a/zeppelin-web-angular/src/app/services/message.service.ts +++ b/zeppelin-web-angular/src/app/services/message.service.ts @@ -53,7 +53,12 @@ export class MessageService extends Message implements OnDestroy { } bootstrap(): void { + const online = navigator.onLine; + if (!online) { + this.setOnline(false); + } super.bootstrap(this.ticketService.originTicket, this.baseUrlService.getWebsocketUrl()); + this.setOnline(online); } ping() {