diff options
Diffstat (limited to 'src/vs/editor/common/model/textModel.ts')
| -rw-r--r-- | src/vs/editor/common/model/textModel.ts | 406 |
1 files changed, 56 insertions, 350 deletions
diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index 78f17dd2669..8aa71a280d5 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -5,7 +5,6 @@ import { ArrayQueue, pushMany } from 'vs/base/common/arrays'; import { VSBuffer, VSBufferReadableStream } from 'vs/base/common/buffer'; -import { CharCode } from 'vs/base/common/charCode'; import { Color } from 'vs/base/common/color'; import { onUnexpectedError } from 'vs/base/common/errors'; import { Emitter, Event } from 'vs/base/common/event'; @@ -15,39 +14,35 @@ import { listenStream } from 'vs/base/common/stream'; import * as strings from 'vs/base/common/strings'; import { Constants } from 'vs/base/common/uint'; import { URI } from 'vs/base/common/uri'; -import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { ISingleEditOperation } from 'vs/editor/common/core/editOperation'; +import { countEOL } from 'vs/editor/common/core/eolCounter'; +import { normalizeIndentation } from 'vs/editor/common/core/indentation'; import { IPosition, Position } from 'vs/editor/common/core/position'; import { IRange, Range } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; +import { TextChange } from 'vs/editor/common/core/textChange'; +import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/core/textModelDefaults'; +import { IWordAtPosition } from 'vs/editor/common/core/wordHelper'; +import { FormattingOptions } from 'vs/editor/common/languages'; +import { ILanguageService } from 'vs/editor/common/languages/language'; +import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import * as model from 'vs/editor/common/model'; -import { IBracketPairsTextModelPart } from 'vs/editor/common/textModelBracketPairs'; import { BracketPairsTextModelPart } from 'vs/editor/common/model/bracketPairsTextModelPart/bracketPairsImpl'; import { ColorizedBracketPairsDecorationProvider } from 'vs/editor/common/model/bracketPairsTextModelPart/colorizedBracketPairsDecorationProvider'; import { EditStack } from 'vs/editor/common/model/editStack'; import { GuidesTextModelPart } from 'vs/editor/common/model/guidesTextModelPart'; -import { IGuidesTextModelPart } from 'vs/editor/common/textModelGuides'; import { guessIndentation } from 'vs/editor/common/model/indentationGuesser'; import { IntervalNode, IntervalTree, recomputeMaxEnd } from 'vs/editor/common/model/intervalTree'; import { PieceTreeTextBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer'; import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder'; -import { TextChange } from 'vs/editor/common/core/textChange'; -import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, LineInjectedText, ModelInjectedTextChangedEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/textModelEvents'; import { SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch'; -import { TextModelTokenization } from 'vs/editor/common/model/textModelTokens'; -import { countEOL } from 'vs/editor/common/core/eolCounter'; -import { ContiguousMultilineTokens } from 'vs/editor/common/tokens/contiguousMultilineTokens'; -import { SparseMultilineTokens } from 'vs/editor/common/tokens/sparseMultilineTokens'; -import { ContiguousTokensStore } from 'vs/editor/common/tokens/contiguousTokensStore'; -import { SparseTokensStore } from 'vs/editor/common/tokens/sparseTokensStore'; -import { getWordAtText, IWordAtPosition } from 'vs/editor/common/core/wordHelper'; -import { FormattingOptions, StandardTokenType } from 'vs/editor/common/languages'; -import { ILanguageConfigurationService, ResolvedLanguageConfiguration } from 'vs/editor/common/languages/languageConfigurationRegistry'; -import { ILanguageService } from 'vs/editor/common/languages/language'; +import { TokenizationTextModelPart } from 'vs/editor/common/model/tokenizationTextModelPart'; +import { IBracketPairsTextModelPart } from 'vs/editor/common/textModelBracketPairs'; +import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelOptionsChangedEvent, InternalModelContentChangeEvent, LineInjectedText, ModelInjectedTextChangedEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/textModelEvents'; +import { IGuidesTextModelPart } from 'vs/editor/common/textModelGuides'; +import { ITokenizationTextModelPart } from 'vs/editor/common/tokenizationTextModelPart'; import { IColorTheme, ThemeColor } from 'vs/platform/theme/common/themeService'; import { IUndoRedoService, ResourceEditStackSnapshot } from 'vs/platform/undoRedo/common/undoRedo'; -import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/core/textModelDefaults'; -import { normalizeIndentation } from 'vs/editor/common/core/indentation'; -import { ISingleEditOperation } from 'vs/editor/common/core/editOperation'; function createTextBufferBuilder() { return new PieceTreeTextBufferBuilder(); @@ -172,12 +167,6 @@ const enum StringOffsetValidationType { SurrogatePairs = 1, } -export const enum BackgroundTokenizationState { - Uninitialized = 0, - InProgress = 1, - Completed = 2, -} - export class TextModel extends Disposable implements model.ITextModel, IDecorationsTreesHost { private static readonly MODEL_SYNC_LIMIT = 50 * 1024 * 1024; // 50 MB @@ -227,14 +216,9 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati private readonly _onDidChangeDecorations: DidChangeDecorationsEmitter = this._register(new DidChangeDecorationsEmitter(affectedInjectedTextLines => this.handleBeforeFireDecorationsChangedEvent(affectedInjectedTextLines))); public readonly onDidChangeDecorations: Event<IModelDecorationsChangedEvent> = this._onDidChangeDecorations.event; - private readonly _onDidChangeLanguage: Emitter<IModelLanguageChangedEvent> = this._register(new Emitter<IModelLanguageChangedEvent>()); - public readonly onDidChangeLanguage: Event<IModelLanguageChangedEvent> = this._onDidChangeLanguage.event; - - private readonly _onDidChangeLanguageConfiguration: Emitter<IModelLanguageConfigurationChangedEvent> = this._register(new Emitter<IModelLanguageConfigurationChangedEvent>()); - public readonly onDidChangeLanguageConfiguration: Event<IModelLanguageConfigurationChangedEvent> = this._onDidChangeLanguageConfiguration.event; - - private readonly _onDidChangeTokens: Emitter<IModelTokensChangedEvent> = this._register(new Emitter<IModelTokensChangedEvent>()); - public readonly onDidChangeTokens: Event<IModelTokensChangedEvent> = this._onDidChangeTokens.event; + public get onDidChangeLanguage() { return this._tokenizationTextModelPart.onDidChangeLanguage; } + public get onDidChangeLanguageConfiguration() { return this._tokenizationTextModelPart.onDidChangeLanguageConfiguration; } + public get onDidChangeTokens() { return this._tokenizationTextModelPart.onDidChangeTokens; } private readonly _onDidChangeOptions: Emitter<IModelOptionsChangedEvent> = this._register(new Emitter<IModelOptionsChangedEvent>()); public readonly onDidChangeOptions: Event<IModelOptionsChangedEvent> = this._onDidChangeOptions.event; @@ -265,7 +249,8 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati private _options: model.TextModelResolvedOptions; private _isDisposed: boolean; - private _isDisposing: boolean; + private __isDisposing: boolean; + public _isDisposing(): boolean { return this.__isDisposing; } private _versionId: number; /** * Unlike, versionId, this can go down (via undo) or go to previous values (via redo) @@ -294,40 +279,15 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati private readonly _decorationProvider: ColorizedBracketPairsDecorationProvider; //#endregion - //#region Tokenization - private _languageId: string; - private readonly _languageRegistryListener: IDisposable; - private readonly _tokens: ContiguousTokensStore; - private readonly _semanticTokens: SparseTokensStore; - private readonly _tokenization: TextModelTokenization; - //#endregion + private readonly _tokenizationTextModelPart: TokenizationTextModelPart; + public get tokenization(): ITokenizationTextModelPart { return this._tokenizationTextModelPart; } - private readonly _bracketPairColorizer: BracketPairsTextModelPart; - public get bracketPairs(): IBracketPairsTextModelPart { return this._bracketPairColorizer; } + private readonly _bracketPairs: BracketPairsTextModelPart; + public get bracketPairs(): IBracketPairsTextModelPart { return this._bracketPairs; } private readonly _guidesTextModelPart: GuidesTextModelPart; public get guides(): IGuidesTextModelPart { return this._guidesTextModelPart; } - private _backgroundTokenizationState = BackgroundTokenizationState.Uninitialized; - public get backgroundTokenizationState(): BackgroundTokenizationState { - return this._backgroundTokenizationState; - } - private handleTokenizationProgress(completed: boolean) { - if (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) { - // We already did a full tokenization and don't go back to progressing. - return; - } - const newState = completed ? BackgroundTokenizationState.Completed : BackgroundTokenizationState.InProgress; - if (this._backgroundTokenizationState !== newState) { - this._backgroundTokenizationState = newState; - this._bracketPairColorizer.handleDidChangeBackgroundTokenizationState(); - this._onBackgroundTokenizationStateChanged.fire(); - } - } - - private readonly _onBackgroundTokenizationStateChanged = this._register(new Emitter<void>()); - public readonly onBackgroundTokenizationStateChanged: Event<void> = this._onBackgroundTokenizationStateChanged.event; - constructor( source: string | model.ITextBufferFactory, languageId: string, @@ -356,6 +316,17 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati this._options = TextModel.resolveOptions(this._buffer, creationOptions); + this._bracketPairs = this._register(new BracketPairsTextModelPart(this, this._languageConfigurationService)); + this._guidesTextModelPart = this._register(new GuidesTextModelPart(this, this._languageConfigurationService)); + this._decorationProvider = this._register(new ColorizedBracketPairsDecorationProvider(this)); + this._tokenizationTextModelPart = new TokenizationTextModelPart( + this._languageService, + this._languageConfigurationService, + this, + this._bracketPairs, + languageId + ); + const bufferLineCount = this._buffer.getLineCount(); const bufferTextLength = this._buffer.getValueLengthInRange(new Range(1, 1, bufferLineCount, this._buffer.getLineLength(bufferLineCount) + 1), model.EndOfLinePreference.TextDefined); @@ -378,17 +349,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati this._initialUndoRedoSnapshot = null; this._isDisposed = false; - this._isDisposing = false; - - this._languageId = languageId; - - this._languageRegistryListener = this._languageConfigurationService.onDidChange( - e => { - if (e.affects(this._languageId)) { - this._onDidChangeLanguageConfiguration.fire({}); - } - } - ); + this.__isDisposing = false; this._instanceId = strings.singleLetterHash(MODEL_ID); this._lastDecorationId = 0; @@ -400,13 +361,6 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati this._isRedoing = false; this._trimAutoWhitespaceLines = null; - this._tokens = new ContiguousTokensStore(this._languageService.languageIdCodec); - this._semanticTokens = new SparseTokensStore(this._languageService.languageIdCodec); - this._tokenization = new TextModelTokenization(this, this._languageService.languageIdCodec); - - this._bracketPairColorizer = this._register(new BracketPairsTextModelPart(this, this._languageConfigurationService)); - this._guidesTextModelPart = this._register(new GuidesTextModelPart(this, this._languageConfigurationService)); - this._decorationProvider = this._register(new ColorizedBracketPairsDecorationProvider(this)); this._register(this._decorationProvider.onDidChange(() => { this._onDidChangeDecorations.beginDeferredEmit(); @@ -416,14 +370,13 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati } public override dispose(): void { - this._isDisposing = true; + this.__isDisposing = true; this._onWillDispose.fire(); - this._languageRegistryListener.dispose(); - this._tokenization.dispose(); + this._tokenizationTextModelPart.dispose(); this._isDisposed = true; super.dispose(); this._bufferDisposable.dispose(); - this._isDisposing = false; + this.__isDisposing = false; // Manually release reference to previous text buffer to avoid large leaks // in case someone leaks a TextModel reference const emptyDisposedTextBuffer = new PieceTreeTextBuffer([], '', '\n', false, false, true, true); @@ -436,14 +389,11 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati return ( this._onWillDispose.hasListeners() || this._onDidChangeDecorations.hasListeners() - || this._onDidChangeLanguage.hasListeners() - || this._onDidChangeLanguageConfiguration.hasListeners() - || this._onDidChangeTokens.hasListeners() + || this._tokenizationTextModelPart._hasListeners() || this._onDidChangeOptions.hasListeners() || this._onDidChangeAttached.hasListeners() || this._onDidChangeInjectedText.hasListeners() || this._eventEmitter.hasListeners() - || this._onBackgroundTokenizationStateChanged.hasListeners() ); } @@ -464,12 +414,12 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati } private _emitContentChangedEvent(rawChange: ModelRawContentChangedEvent, change: IModelContentChangedEvent): void { - if (this._isDisposing) { + if (this.__isDisposing) { // Do not confuse listeners by emitting any event after disposing return; } - this._bracketPairColorizer.handleDidChangeContent(change); - this._tokenization.handleDidChangeContent(change); + this._tokenizationTextModelPart.handleDidChangeContent(change); + this._bracketPairs.handleDidChangeContent(change); this._eventEmitter.fire(new InternalModelContentChangeEvent(rawChange, change)); } @@ -513,8 +463,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati this._increaseVersionId(); // Flush all tokens - this._tokens.flush(); - this._semanticTokens.flush(); + this._tokenizationTextModelPart.flush(); // Destroy all my decorations this._decorations = Object.create(null); @@ -600,7 +549,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati public onBeforeAttached(): void { this._attachedEditorCount++; if (this._attachedEditorCount === 1) { - this._tokenization.handleDidChangeAttached(); + this._tokenizationTextModelPart.handleDidChangeAttached(); this._onDidChangeAttached.fire(undefined); } } @@ -608,7 +557,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati public onBeforeDetached(): void { this._attachedEditorCount--; if (this._attachedEditorCount === 0) { - this._tokenization.handleDidChangeAttached(); + this._tokenizationTextModelPart.handleDidChangeAttached(); this._onDidChangeAttached.fire(undefined); } } @@ -697,7 +646,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati const e = this._options.createChangeEvent(newOpts); this._options = newOpts; - this._bracketPairColorizer.handleDidChangeOptions(e); + this._bracketPairs.handleDidChangeOptions(e); this._decorationProvider.handleDidChangeOptions(e); this._onDidChangeOptions.fire(e); } @@ -1463,8 +1412,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati for (let i = 0, len = contentChanges.length; i < len; i++) { const change = contentChanges[i]; const [eolCount, firstLineLength, lastLineLength] = countEOL(change.text); - this._tokens.acceptEdit(change.range, eolCount, firstLineLength); - this._semanticTokens.acceptEdit(change.range, eolCount, firstLineLength, lastLineLength, change.text.length > 0 ? change.text.charCodeAt(0) : CharCode.Null); + this._tokenizationTextModelPart.acceptEdit(change.range, change.text, eolCount, firstLineLength, lastLineLength); this._decorationsTree.acceptReplace(change.rangeOffset, change.rangeLength, change.text.length, change.forceMoveMarkers); } @@ -1944,270 +1892,28 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati //#region Tokenization - public setLineTokens(lineNumber: number, tokens: Uint32Array | ArrayBuffer | null): void { - if (lineNumber < 1 || lineNumber > this.getLineCount()) { - throw new Error('Illegal value for lineNumber'); - } - - this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), tokens, false); - } - - public setTokens(tokens: ContiguousMultilineTokens[], backgroundTokenizationCompleted: boolean = false): void { - if (tokens.length !== 0) { - const ranges: { fromLineNumber: number; toLineNumber: number }[] = []; - - for (let i = 0, len = tokens.length; i < len; i++) { - const element = tokens[i]; - let minChangedLineNumber = 0; - let maxChangedLineNumber = 0; - let hasChange = false; - for (let lineNumber = element.startLineNumber; lineNumber <= element.endLineNumber; lineNumber++) { - if (hasChange) { - this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.getLineTokens(lineNumber), false); - maxChangedLineNumber = lineNumber; - } else { - const lineHasChange = this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.getLineTokens(lineNumber), true); - if (lineHasChange) { - hasChange = true; - minChangedLineNumber = lineNumber; - maxChangedLineNumber = lineNumber; - } - } - } - if (hasChange) { - ranges.push({ fromLineNumber: minChangedLineNumber, toLineNumber: maxChangedLineNumber }); - } - } - - if (ranges.length > 0) { - this._emitModelTokensChangedEvent({ - tokenizationSupportChanged: false, - semanticTokensApplied: false, - ranges: ranges - }); - } - } - this.handleTokenizationProgress(backgroundTokenizationCompleted); - } - - public setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void { - this._semanticTokens.set(tokens, isComplete); - - this._emitModelTokensChangedEvent({ - tokenizationSupportChanged: false, - semanticTokensApplied: tokens !== null, - ranges: [{ fromLineNumber: 1, toLineNumber: this.getLineCount() }] - }); - } - - public hasCompleteSemanticTokens(): boolean { - return this._semanticTokens.isComplete(); - } - - public hasSomeSemanticTokens(): boolean { - return !this._semanticTokens.isEmpty(); - } - - public setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[]): void { - if (this.hasCompleteSemanticTokens()) { - return; - } - const changedRange = this.validateRange(this._semanticTokens.setPartial(range, tokens)); - - this._emitModelTokensChangedEvent({ - tokenizationSupportChanged: false, - semanticTokensApplied: true, - ranges: [{ fromLineNumber: changedRange.startLineNumber, toLineNumber: changedRange.endLineNumber }] - }); - } - - public tokenizeViewport(startLineNumber: number, endLineNumber: number): void { - startLineNumber = Math.max(1, startLineNumber); - endLineNumber = Math.min(this._buffer.getLineCount(), endLineNumber); - this._tokenization.tokenizeViewport(startLineNumber, endLineNumber); - } - - public clearTokens(): void { - this._tokens.flush(); - this._emitModelTokensChangedEvent({ - tokenizationSupportChanged: true, - semanticTokensApplied: false, - ranges: [{ - fromLineNumber: 1, - toLineNumber: this._buffer.getLineCount() - }] - }); - } - - public clearSemanticTokens(): void { - this._semanticTokens.flush(); - - this._emitModelTokensChangedEvent({ - tokenizationSupportChanged: false, - semanticTokensApplied: false, - ranges: [{ fromLineNumber: 1, toLineNumber: this.getLineCount() }] - }); - } - - private _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void { - if (!this._isDisposing) { - this._bracketPairColorizer.handleDidChangeTokens(e); - this._onDidChangeTokens.fire(e); - } - } - - public resetTokenization(): void { - this._tokenization.reset(); - } - - public forceTokenization(lineNumber: number): void { - if (lineNumber < 1 || lineNumber > this.getLineCount()) { - throw new Error('Illegal value for lineNumber'); - } - - this._tokenization.forceTokenization(lineNumber); - } - - public isCheapToTokenize(lineNumber: number): boolean { - return this._tokenization.isCheapToTokenize(lineNumber); - } - - public tokenizeIfCheap(lineNumber: number): void { - if (this.isCheapToTokenize(lineNumber)) { - this.forceTokenization(lineNumber); - } - } - - public getLineTokens(lineNumber: number): LineTokens { - if (lineNumber < 1 || lineNumber > this.getLineCount()) { - throw new Error('Illegal value for lineNumber'); - } - - return this._getLineTokens(lineNumber); - } - - private _getLineTokens(lineNumber: number): LineTokens { - const lineText = this.getLineContent(lineNumber); - const syntacticTokens = this._tokens.getTokens(this._languageId, lineNumber - 1, lineText); - return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens); - } - + // TODO move them to the tokenization part. public getLanguageId(): string { - return this._languageId; + return this.tokenization.getLanguageId(); } public setMode(languageId: string): void { - if (this._languageId === languageId) { - // There's nothing to do - return; - } - - const e: IModelLanguageChangedEvent = { - oldLanguage: this._languageId, - newLanguage: languageId - }; - - this._languageId = languageId; - - this._bracketPairColorizer.handleDidChangeLanguage(e); - this._tokenization.handleDidChangeLanguage(e); - this._onDidChangeLanguage.fire(e); - this._onDidChangeLanguageConfiguration.fire({}); + this.tokenization.setLanguageId(languageId); } public getLanguageIdAtPosition(lineNumber: number, column: number): string { - const position = this.validatePosition(new Position(lineNumber, column)); - const lineTokens = this.getLineTokens(position.lineNumber); - return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1)); + return this.tokenization.getLanguageIdAtPosition(lineNumber, column); } - - public getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType { - const position = this.validatePosition(new Position(lineNumber, column)); - return this._tokenization.getTokenTypeIfInsertingCharacter(position, character); - } - - tokenizeLineWithEdit(position: IPosition, length: number, newText: string): LineTokens | null { - const validatedPosition = this.validatePosition(position); - return this._tokenization.tokenizeLineWithEdit(validatedPosition, length, newText); - } - - private getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration { - return this._languageConfigurationService.getLanguageConfiguration(languageId); - } - - // Having tokens allows implementing additional helper methods - - public getWordAtPosition(_position: IPosition): IWordAtPosition | null { - this._assertNotDisposed(); - const position = this.validatePosition(_position); - const lineContent = this.getLineContent(position.lineNumber); - const lineTokens = this._getLineTokens(position.lineNumber); - const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1); - - // (1). First try checking right biased word - const [rbStartOffset, rbEndOffset] = TextModel._findLanguageBoundaries(lineTokens, tokenIndex); - const rightBiasedWord = getWordAtText( - position.column, - this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(), - lineContent.substring(rbStartOffset, rbEndOffset), - rbStartOffset - ); - // Make sure the result touches the original passed in position - if (rightBiasedWord && rightBiasedWord.startColumn <= _position.column && _position.column <= rightBiasedWord.endColumn) { - return rightBiasedWord; - } - - // (2). Else, if we were at a language boundary, check the left biased word - if (tokenIndex > 0 && rbStartOffset === position.column - 1) { - // edge case, where `position` sits between two tokens belonging to two different languages - const [lbStartOffset, lbEndOffset] = TextModel._findLanguageBoundaries(lineTokens, tokenIndex - 1); - const leftBiasedWord = getWordAtText( - position.column, - this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(), - lineContent.substring(lbStartOffset, lbEndOffset), - lbStartOffset - ); - // Make sure the result touches the original passed in position - if (leftBiasedWord && leftBiasedWord.startColumn <= _position.column && _position.column <= leftBiasedWord.endColumn) { - return leftBiasedWord; - } - } - - return null; + public setLineTokens(lineNumber: number, tokens: Uint32Array | ArrayBuffer | null): void { + this._tokenizationTextModelPart.setLineTokens(lineNumber, tokens); } - private static _findLanguageBoundaries(lineTokens: LineTokens, tokenIndex: number): [number, number] { - const languageId = lineTokens.getLanguageId(tokenIndex); - - // go left until a different language is hit - let startOffset = 0; - for (let i = tokenIndex; i >= 0 && lineTokens.getLanguageId(i) === languageId; i--) { - startOffset = lineTokens.getStartOffset(i); - } - - // go right until a different language is hit - let endOffset = lineTokens.getLineContent().length; - for (let i = tokenIndex, tokenCount = lineTokens.getCount(); i < tokenCount && lineTokens.getLanguageId(i) === languageId; i++) { - endOffset = lineTokens.getEndOffset(i); - } - - return [startOffset, endOffset]; + public getWordAtPosition(position: IPosition): IWordAtPosition | null { + return this._tokenizationTextModelPart.getWordAtPosition(position); } public getWordUntilPosition(position: IPosition): IWordAtPosition { - const wordAtPosition = this.getWordAtPosition(position); - if (!wordAtPosition) { - return { - word: '', - startColumn: position.column, - endColumn: position.column - }; - } - return { - word: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn), - startColumn: wordAtPosition.startColumn, - endColumn: position.column - }; + return this._tokenizationTextModelPart.getWordUntilPosition(position); } //#endregion |