Prototype improved a11y buffer view (#4340)

Author: Tyriar

css/xterm.css

@@ -150,6 +150,25 @@
     color: transparent;
 }
 
+.xterm .xterm-accessibility-buffer {
+    position: absolute;
+    left: 0;
+    top: 0;
+    bottom: 0;
+    right: 0;
+    padding: .5em;
+    background: #000;
+    color: #fff;
+    opacity: 0;
+    overflow: scroll;
+    overflow-x: hidden;
+}
+
+.xterm .xterm-accessibility-buffer:focus {
+    opacity: 1;
+    z-index: 20;
+}
+
 .xterm .live-region {
     position: absolute;
     left: -9999px;

src/browser/AccessibilityManager.ts

@@ -4,14 +4,16 @@
  */
 
 import * as Strings from 'browser/LocalizableStrings';
-import { ITerminal, IRenderDebouncer } from 'browser/Types';
+import { ITerminal, IRenderDebouncer, ReadonlyColorSet } from 'browser/Types';
 import { IBuffer } from 'common/buffer/Types';
 import { isMac } from 'common/Platform';
 import { TimeBasedDebouncer } from 'browser/TimeBasedDebouncer';
 import { addDisposableDomListener } from 'browser/Lifecycle';
 import { Disposable, toDisposable } from 'common/Lifecycle';
 import { ScreenDprMonitor } from 'browser/ScreenDprMonitor';
-import { IRenderService } from 'browser/services/Services';
+import { IRenderService, IThemeService } from 'browser/services/Services';
+import { IOptionsService } from 'common/services/Services';
+import { ITerminalOptions } from 'xterm';
 
 const MAX_ROWS_TO_READ = 20;
 
@@ -26,6 +28,7 @@ export class AccessibilityManager extends Disposable {
   private _rowElements: HTMLElement[];
   private _liveRegion: HTMLElement;
   private _liveRegionLineCount: number = 0;
+  private _accessiblityBuffer: HTMLElement;
 
   private _renderRowsDebouncer: IRenderDebouncer;
   private _screenDprMonitor: ScreenDprMonitor;
@@ -48,7 +51,9 @@ export class AccessibilityManager extends Disposable {
 
   constructor(
     private readonly _terminal: ITerminal,
-    private readonly _renderService: IRenderService
+    @IOptionsService optionsService: IOptionsService,
+    @IRenderService private readonly _renderService: IRenderService,
+    @IThemeService themeService: IThemeService
   ) {
     super();
     this._accessibilityTreeRoot = document.createElement('div');
@@ -83,7 +88,16 @@ export class AccessibilityManager extends Disposable {
     if (!this._terminal.element) {
       throw new Error('Cannot enable accessibility before Terminal.open');
     }
-    this._terminal.element.insertAdjacentElement('afterbegin', this._accessibilityTreeRoot);
+
+    this._accessiblityBuffer = document.createElement('div');
+    this._accessiblityBuffer.ariaLabel = Strings.accessibilityBuffer;
+    this._accessiblityBuffer.classList.add('xterm-accessibility-buffer');
+
+    // TODO: this is needed when content editable is false
+    this._refreshAccessibilityBuffer();
+    this._accessiblityBuffer.addEventListener('focus', () => this._refreshAccessibilityBuffer());
+    this._terminal.element.insertAdjacentElement('afterbegin', this._accessiblityBuffer);
+
 
     this.register(this._renderRowsDebouncer);
     this.register(this._terminal.onResize(e => this._handleResize(e.rows)));
@@ -97,6 +111,11 @@ export class AccessibilityManager extends Disposable {
     this.register(this._terminal.onBlur(() => this._clearLiveRegion()));
     this.register(this._renderService.onDimensionsChange(() => this._refreshRowsDimensions()));
 
+    this._handleColorChange(themeService.colors);
+    this.register(themeService.onChangeColors(e => this._handleColorChange(e)));
+    this._handleFontOptionChange(optionsService.options);
+    this.register(optionsService.onMultipleOptionChange(['fontSize', 'fontFamily'], () => this._handleFontOptionChange(optionsService.options)));
+
     this._screenDprMonitor = new ScreenDprMonitor(window);
     this.register(this._screenDprMonitor);
     this._screenDprMonitor.setListener(() => this._refreshRowsDimensions());
@@ -299,4 +318,33 @@ export class AccessibilityManager extends Disposable {
     this._liveRegion.textContent += this._charsToAnnounce;
     this._charsToAnnounce = '';
   }
+
+
+  private _refreshAccessibilityBuffer(): void {
+    if (!this._terminal.viewport) {
+      return;
+    }
+
+    const { bufferElements, cursorElement } = this._terminal.viewport.getBufferElements(0);
+    for (const element of bufferElements) {
+      if (element.textContent) {
+        element.textContent = element.textContent.replace(new RegExp(' ', 'g'), '\xA0');
+      }
+    }
+    this._accessiblityBuffer.tabIndex = 0;
+    this._accessiblityBuffer.ariaRoleDescription = 'document';
+    this._accessiblityBuffer.replaceChildren(...bufferElements);
+    this._accessiblityBuffer.scrollTop = this._accessiblityBuffer.scrollHeight;
+    this._accessiblityBuffer.focus();
+  }
+
+  private _handleColorChange(colorSet: ReadonlyColorSet): void {
+    this._accessiblityBuffer.style.backgroundColor = colorSet.background.css;
+    this._accessiblityBuffer.style.color = colorSet.foreground.css;
+  }
+
+  private _handleFontOptionChange(options: Required<ITerminalOptions>): void {
+    this._accessiblityBuffer.style.fontFamily = options.fontFamily;
+    this._accessiblityBuffer.style.fontSize = `${options.fontSize}px`;
+  }
 }

src/browser/LocalizableStrings.ts

@@ -10,3 +10,5 @@ export let promptLabel = 'Terminal input';
 
 // eslint-disable-next-line prefer-const
 export let tooMuchOutput = 'Too much output to announce, navigate to rows manually to read';
+
+export const accessibilityBuffer = 'Accessibility buffer';

src/browser/Terminal.ts

@@ -267,7 +267,7 @@ export class Terminal extends CoreTerminal implements ITerminal {
   private _handleScreenReaderModeOptionChange(value: boolean): void {
     if (value) {
       if (!this._accessibilityManager && this._renderService) {
-        this._accessibilityManager = new AccessibilityManager(this, this._renderService);
+        this._accessibilityManager = this._instantiationService.createInstance(AccessibilityManager, this);
       }
     } else {
       this._accessibilityManager?.dispose();
@@ -419,7 +419,6 @@ export class Terminal extends CoreTerminal implements ITerminal {
     this.element.dir = 'ltr';   // xterm.css assumes LTR
     this.element.classList.add('terminal');
     this.element.classList.add('xterm');
-    this.element.setAttribute('tabindex', '0');
     parent.appendChild(this.element);
 
     // Performance: Use a document fragment to build the terminal
@@ -553,7 +552,7 @@ export class Terminal extends CoreTerminal implements ITerminal {
     if (this.options.screenReaderMode) {
       // Note that this must be done *after* the renderer is created in order to
       // ensure the correct order of the dprchange event
-      this._accessibilityManager = new AccessibilityManager(this, this._renderService);
+      this._accessibilityManager = this._instantiationService.createInstance(AccessibilityManager, this);
     }
     this.register(this.optionsService.onSpecificOptionChange('screenReaderMode', e => this._handleScreenReaderModeOptionChange(e)));
 

src/browser/TestUtils.test.ts

@@ -140,6 +140,9 @@ export class MockTerminal implements ITerminal {
   public write(data: string): void {
     throw new Error('Method not implemented.');
   }
+  public getBufferElements(startLine: number, endLine?: number | undefined): { bufferElements: HTMLElement[], cursorElement?: HTMLElement | undefined } {
+    throw new Error('Method not implemented.');
+  }
   public bracketedPasteMode!: boolean;
   public renderer!: IRenderer;
   public linkifier2!: ILinkifier2;
@@ -310,6 +313,9 @@ export class MockViewport implements IViewport {
   public getLinesScrolled(ev: WheelEvent): number {
     throw new Error('Method not implemented.');
   }
+  public getBufferElements(startLine: number, endLine?: number | undefined): { bufferElements: HTMLElement[], cursorElement?: HTMLElement | undefined } {
+    throw new Error('Method not implemented.');
+  }
 }
 
 export class MockCompositionHelper implements ICompositionHelper {

src/browser/Types.d.ts

@@ -144,6 +144,7 @@ export interface IViewport extends IDisposable {
   scrollBarWidth: number;
   syncScrollArea(immediate?: boolean): void;
   getLinesScrolled(ev: WheelEvent): number;
+  getBufferElements(startLine: number, endLine?: number): { bufferElements: HTMLElement[], cursorElement?: HTMLElement };
   handleWheel(ev: WheelEvent): boolean;
   handleTouchStart(ev: TouchEvent): void;
   handleTouchMove(ev: TouchEvent): boolean;

src/browser/Viewport.ts

@@ -279,6 +279,33 @@ export class Viewport extends Disposable implements IViewport {
     return amount;
   }
 
+
+  public getBufferElements(startLine: number, endLine?: number): { bufferElements: HTMLElement[], cursorElement?: HTMLElement } {
+    let currentLine: string = '';
+    let cursorElement: HTMLElement | undefined;
+    const bufferElements: HTMLElement[] = [];
+    const end = endLine ?? this._bufferService.buffer.lines.length;
+    const lines = this._bufferService.buffer.lines;
+    for (let i = startLine; i < end; i++) {
+      const line = lines.get(i);
+      if (!line) {
+        continue;
+      }
+      const isWrapped = lines.get(i + 1)?.isWrapped;
+      currentLine += line.translateToString(!isWrapped);
+      if (!isWrapped || i === lines.length - 1) {
+        const div = document.createElement('div');
+        div.textContent = currentLine;
+        bufferElements.push(div);
+        if (currentLine.length > 0) {
+          cursorElement = div;
+        }
+        currentLine = '';
+      }
+    }
+    return { bufferElements, cursorElement };
+  }
+
   /**
    * Gets the number of pixels scrolled by the mouse event taking into account what type of delta
    * is being used.

typings/xterm.d.ts

@@ -498,6 +498,11 @@ declare module 'xterm' {
      * being printed to the terminal when `screenReaderMode` is enabled.
      */
     tooMuchOutput: string;
+
+    /**
+     * The aria label for the accessibility buffer
+     */
+    accessibilityBuffer: string;
   }
 
   /**