From 5854ef865c12cd0da295485dc676416779da106e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Moreno?= Date: Thu, 12 May 2022 09:16:03 +0200 Subject: Generate policies again (#149322) * Generate ADMX/ADML policy declaration files from static analysis Fixes: #148941 * fix build * move to optional dependencies --- build/lib/policies.ts | 674 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 674 insertions(+) create mode 100644 build/lib/policies.ts (limited to 'build/lib/policies.ts') diff --git a/build/lib/policies.ts b/build/lib/policies.ts new file mode 100644 index 00000000000..62ea4d561e5 --- /dev/null +++ b/build/lib/policies.ts @@ -0,0 +1,674 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { spawn } from 'child_process'; +import { promises as fs } from 'fs'; +import * as path from 'path'; +import * as byline from 'byline'; +import { rgPath } from '@vscode/ripgrep'; +import * as Parser from 'tree-sitter'; +import fetch from 'node-fetch'; +const { typescript } = require('tree-sitter-typescript'); +const product = require('../../product.json'); + +type NlsString = { value: string; nlsKey: string }; + +function isNlsString(value: string | NlsString | undefined): value is NlsString { + return value ? typeof value !== 'string' : false; +} + +function isStringArray(value: (string | NlsString)[]): value is string[] { + return !value.some(s => isNlsString(s)); +} + +function isNlsStringArray(value: (string | NlsString)[]): value is NlsString[] { + return value.every(s => isNlsString(s)); +} + +interface Category { + readonly moduleName: string; + readonly name: NlsString; +} + +enum PolicyType { + StringEnum +} + +interface Policy { + readonly category: Category; + readonly minimumVersion: string; + renderADMX(regKey: string): string[]; + renderADMLStrings(translations?: LanguageTranslations): string[]; + renderADMLPresentation(): string; +} + +function renderADMLString(prefix: string, moduleName: string, nlsString: NlsString, translations?: LanguageTranslations): string { + let value: string | undefined; + + if (translations) { + const moduleTranslations = translations[moduleName]; + + if (moduleTranslations) { + value = moduleTranslations[nlsString.nlsKey]; + } + } + + if (!value) { + value = nlsString.value; + } + + return `${value}`; +} + +abstract class BasePolicy implements Policy { + constructor( + protected policyType: PolicyType, + protected name: string, + readonly category: Category, + readonly minimumVersion: string, + protected description: NlsString, + protected moduleName: string, + ) { } + + protected renderADMLString(nlsString: NlsString, translations?: LanguageTranslations): string { + return renderADMLString(this.name, this.moduleName, nlsString, translations); + } + + renderADMX(regKey: string) { + return [ + ``, + ` `, + ` `, + ` `, + ...this.renderADMXElements(), + ` `, + `` + ]; + } + + protected abstract renderADMXElements(): string[]; + + renderADMLStrings(translations?: LanguageTranslations) { + return [ + `${this.name}`, + this.renderADMLString(this.description, translations) + ]; + } + + renderADMLPresentation(): string { + return `${this.renderADMLPresentationContents()}`; + } + + protected abstract renderADMLPresentationContents(): string; +} + +class BooleanPolicy extends BasePolicy { + + static from( + name: string, + category: Category, + minimumVersion: string, + description: NlsString, + moduleName: string, + settingNode: Parser.SyntaxNode + ): BooleanPolicy | undefined { + const type = getStringProperty(settingNode, 'type'); + + if (type !== 'boolean') { + return undefined; + } + + return new BooleanPolicy(name, category, minimumVersion, description, moduleName); + } + + private constructor( + name: string, + category: Category, + minimumVersion: string, + description: NlsString, + moduleName: string, + ) { + super(PolicyType.StringEnum, name, category, minimumVersion, description, moduleName); + } + + protected renderADMXElements(): string[] { + return [ + ``, + ` `, + `` + ]; + } + + renderADMLPresentationContents() { + return `${this.name}`; + } +} + +class IntPolicy extends BasePolicy { + + static from( + name: string, + category: Category, + minimumVersion: string, + description: NlsString, + moduleName: string, + settingNode: Parser.SyntaxNode + ): IntPolicy | undefined { + const type = getStringProperty(settingNode, 'type'); + + if (type !== 'number') { + return undefined; + } + + const defaultValue = getIntProperty(settingNode, 'default'); + + if (typeof defaultValue === 'undefined') { + throw new Error(`Missing required 'default' property.`); + } + + return new IntPolicy(name, category, minimumVersion, description, moduleName, defaultValue); + } + + private constructor( + name: string, + category: Category, + minimumVersion: string, + description: NlsString, + moduleName: string, + protected readonly defaultValue: number, + ) { + super(PolicyType.StringEnum, name, category, minimumVersion, description, moduleName); + } + + protected renderADMXElements(): string[] { + return [ + `` + // `` + ]; + } + + renderADMLPresentationContents() { + return `${this.name}`; + } +} + +class StringPolicy extends BasePolicy { + + static from( + name: string, + category: Category, + minimumVersion: string, + description: NlsString, + moduleName: string, + settingNode: Parser.SyntaxNode + ): StringPolicy | undefined { + const type = getStringProperty(settingNode, 'type'); + + if (type !== 'string') { + return undefined; + } + + return new StringPolicy(name, category, minimumVersion, description, moduleName); + } + + private constructor( + name: string, + category: Category, + minimumVersion: string, + description: NlsString, + moduleName: string, + ) { + super(PolicyType.StringEnum, name, category, minimumVersion, description, moduleName); + } + + protected renderADMXElements(): string[] { + return [``]; + } + + renderADMLPresentationContents() { + return ``; + } +} + +class StringEnumPolicy extends BasePolicy { + + static from( + name: string, + category: Category, + minimumVersion: string, + description: NlsString, + moduleName: string, + settingNode: Parser.SyntaxNode + ): StringEnumPolicy | undefined { + const type = getStringProperty(settingNode, 'type'); + + if (type !== 'string') { + return undefined; + } + + const enum_ = getStringArrayProperty(settingNode, 'enum'); + + if (!enum_) { + return undefined; + } + + if (!isStringArray(enum_)) { + throw new Error(`Property 'enum' should not be localized.`); + } + + const enumDescriptions = getStringArrayProperty(settingNode, 'enumDescriptions'); + + if (!enumDescriptions) { + throw new Error(`Missing required 'enumDescriptions' property.`); + } else if (!isNlsStringArray(enumDescriptions)) { + throw new Error(`Property 'enumDescriptions' should be localized.`); + } + + return new StringEnumPolicy(name, category, minimumVersion, description, moduleName, enum_, enumDescriptions); + } + + private constructor( + name: string, + category: Category, + minimumVersion: string, + description: NlsString, + moduleName: string, + protected enum_: string[], + protected enumDescriptions: NlsString[], + ) { + super(PolicyType.StringEnum, name, category, minimumVersion, description, moduleName); + } + + protected renderADMXElements(): string[] { + return [ + ``, + ...this.enum_.map((value, index) => ` ${value}`), + `` + ]; + } + + renderADMLStrings(translations?: LanguageTranslations) { + return [ + ...super.renderADMLStrings(translations), + ...this.enumDescriptions.map(e => this.renderADMLString(e, translations)) + ]; + } + + renderADMLPresentationContents() { + return ``; + } +} + +interface QType { + Q: string; + value(matches: Parser.QueryMatch[]): T | undefined; +} + +const IntQ: QType = { + Q: `(number) @value`, + + value(matches: Parser.QueryMatch[]): number | undefined { + const match = matches[0]; + + if (!match) { + return undefined; + } + + const value = match.captures.filter(c => c.name === 'value')[0]?.node.text; + + if (!value) { + throw new Error(`Missing required 'value' property.`); + } + + return parseInt(value); + } +}; + +const StringQ: QType = { + Q: `[ + (string (string_fragment) @value) + (call_expression function: (identifier) @localizeFn arguments: (arguments (string (string_fragment) @nlsKey) (string (string_fragment) @value)) (#eq? @localizeFn localize)) + ]`, + + value(matches: Parser.QueryMatch[]): string | NlsString | undefined { + const match = matches[0]; + + if (!match) { + return undefined; + } + + const value = match.captures.filter(c => c.name === 'value')[0]?.node.text; + + if (!value) { + throw new Error(`Missing required 'value' property.`); + } + + const nlsKey = match.captures.filter(c => c.name === 'nlsKey')[0]?.node.text; + + if (nlsKey) { + return { value, nlsKey }; + } else { + return value; + } + } +}; + +const StringArrayQ: QType<(string | NlsString)[]> = { + Q: `(array ${StringQ.Q})`, + + value(matches: Parser.QueryMatch[]): (string | NlsString)[] | undefined { + if (matches.length === 0) { + return undefined; + } + + return matches.map(match => { + return StringQ.value([match]) as string | NlsString; + }); + } +}; + +function getProperty(qtype: QType, node: Parser.SyntaxNode, key: string): T | undefined { + const query = new Parser.Query( + typescript, + `( + (pair + key: [(property_identifier)(string)] @key + value: ${qtype.Q} + ) + (#eq? @key ${key}) + )` + ); + + return qtype.value(query.matches(node)); +} + +function getIntProperty(node: Parser.SyntaxNode, key: string): number | undefined { + return getProperty(IntQ, node, key); +} + +function getStringProperty(node: Parser.SyntaxNode, key: string): string | NlsString | undefined { + return getProperty(StringQ, node, key); +} + +function getStringArrayProperty(node: Parser.SyntaxNode, key: string): (string | NlsString)[] | undefined { + return getProperty(StringArrayQ, node, key); +} + +// TODO: add more policy types +const PolicyTypes = [ + BooleanPolicy, + IntPolicy, + StringEnumPolicy, + StringPolicy, +]; + +function getPolicy( + moduleName: string, + configurationNode: Parser.SyntaxNode, + settingNode: Parser.SyntaxNode, + policyNode: Parser.SyntaxNode, + categories: Map +): Policy { + const name = getStringProperty(policyNode, 'name'); + + if (!name) { + throw new Error(`Missing required 'name' property.`); + } else if (isNlsString(name)) { + throw new Error(`Property 'name' should be a literal string.`); + } + + const categoryName = getStringProperty(configurationNode, 'title'); + + if (!categoryName) { + throw new Error(`Missing required 'title' property.`); + } else if (!isNlsString(categoryName)) { + throw new Error(`Property 'title' should be localized.`); + } + + const categoryKey = `${categoryName.nlsKey}:${categoryName.value}`; + let category = categories.get(categoryKey); + + if (!category) { + category = { moduleName, name: categoryName }; + categories.set(categoryKey, category); + } + + const minimumVersion = getStringProperty(policyNode, 'minimumVersion'); + + if (!minimumVersion) { + throw new Error(`Missing required 'minimumVersion' property.`); + } else if (isNlsString(minimumVersion)) { + throw new Error(`Property 'minimumVersion' should be a literal string.`); + } + + const description = getStringProperty(settingNode, 'description'); + + if (!description) { + throw new Error(`Missing required 'description' property.`); + } if (!isNlsString(description)) { + throw new Error(`Property 'description' should be localized.`); + } + + let result: Policy | undefined; + + for (const policyType of PolicyTypes) { + if (result = policyType.from(name, category, minimumVersion, description, moduleName, settingNode)) { + break; + } + } + + if (!result) { + throw new Error(`Failed to parse policy '${name}'.`); + } + + return result; +} + +function getPolicies(moduleName: string, node: Parser.SyntaxNode): Policy[] { + const query = new Parser.Query(typescript, ` + ( + (call_expression + function: (member_expression property: (property_identifier) @registerConfigurationFn) (#eq? @registerConfigurationFn registerConfiguration) + arguments: (arguments (object (pair + key: [(property_identifier)(string)] @propertiesKey (#eq? @propertiesKey properties) + value: (object (pair + key: [(property_identifier)(string)] + value: (object (pair + key: [(property_identifier)(string)] @policyKey (#eq? @policyKey policy) + value: (object) @policy + )) @setting + )) + )) @configuration) + ) + ) + `); + + const categories = new Map(); + + return query.matches(node).map(m => { + const configurationNode = m.captures.filter(c => c.name === 'configuration')[0].node; + const settingNode = m.captures.filter(c => c.name === 'setting')[0].node; + const policyNode = m.captures.filter(c => c.name === 'policy')[0].node; + return getPolicy(moduleName, configurationNode, settingNode, policyNode, categories); + }); +} + +async function getFiles(root: string): Promise { + return new Promise((c, e) => { + const result: string[] = []; + const rg = spawn(rgPath, ['-l', 'registerConfiguration\\(', '-g', 'src/**/*.ts', '-g', '!src/**/test/**', root]); + const stream = byline(rg.stdout.setEncoding('utf8')); + stream.on('data', path => result.push(path)); + stream.on('error', err => e(err)); + stream.on('end', () => c(result)); + }); +} + +function renderADMX(regKey: string, versions: string[], categories: Category[], policies: Policy[]) { + versions = versions.map(v => v.replace(/\./g, '_')); + + return ` + + + + + + + + ${versions.map(v => ``).join(`\n `)} + + + + + ${categories.map(c => ``).join(`\n `)} + + + ${policies.map(p => p.renderADMX(regKey)).flat().join(`\n `)} + + +`; +} + +function renderADML(appName: string, versions: string[], categories: Category[], policies: Policy[], translations?: LanguageTranslations) { + return ` + + + + + + ${appName} + ${versions.map(v => `${appName} >= ${v}`)} + ${categories.map(c => renderADMLString('Category', c.moduleName, c.name, translations))} + ${policies.map(p => p.renderADMLStrings(translations)).flat().join(`\n `)} + + + ${policies.map(p => p.renderADMLPresentation()).join(`\n `)} + + + +`; +} + +function renderGP(policies: Policy[], translations: Translations) { + const appName = product.nameLong; + const regKey = product.win32RegValueName; + + const versions = [...new Set(policies.map(p => p.minimumVersion)).values()].sort(); + const categories = [...new Set(policies.map(p => p.category))]; + + return { + admx: renderADMX(regKey, versions, categories, policies), + adml: [ + { languageId: 'en-us', contents: renderADML(appName, versions, categories, policies) }, + ...translations.map(({ languageId, languageTranslations }) => + ({ languageId, contents: renderADML(appName, versions, categories, policies, languageTranslations) })) + ] + }; +} + +const Languages = { + 'fr': 'fr-fr', + 'it': 'it-it', + 'de': 'de-de', + 'es': 'es-es', + 'ru': 'ru-ru', + 'zh-hans': 'zh-cn', + 'zh-hant': 'zh-tw', + 'ja': 'ja-jp', + 'ko': 'ko-kr', + 'cs': 'cs-cz', + 'pt-br': 'pt-br', + 'tr': 'tr-tr', + 'pl': 'pl-pl', +}; + +type LanguageTranslations = { [moduleName: string]: { [nlsKey: string]: string } }; +type Translations = { languageId: string; languageTranslations: LanguageTranslations }[]; + +async function getLatestStableVersion(updateUrl: string) { + const res = await fetch(`${updateUrl}/api/update/darwin/stable/latest`); + const { name: version } = await res.json() as { name: string }; + return version; +} + +async function getNLS(resourceUrlTemplate: string, languageId: string, version: string) { + const resource = { + publisher: 'ms-ceintl', + name: `vscode-language-pack-${languageId}`, + version, + path: 'extension/translations/main.i18n.json' + }; + + const url = resourceUrlTemplate.replace(/\{([^}]+)\}/g, (_, key) => resource[key as keyof typeof resource]); + const res = await fetch(url); + const { contents: result } = await res.json() as { contents: LanguageTranslations }; + return result; +} + +async function parsePolicies(): Promise { + const parser = new Parser(); + parser.setLanguage(typescript); + + const files = await getFiles(process.cwd()); + const base = path.join(process.cwd(), 'src'); + const policies = []; + + for (const file of files) { + const moduleName = path.relative(base, file).replace(/\.ts$/i, '').replace(/\\/g, '/'); + const contents = await fs.readFile(file, { encoding: 'utf8' }); + const tree = parser.parse(contents); + policies.push(...getPolicies(moduleName, tree.rootNode)); + } + + return policies; +} + +async function getTranslations(): Promise { + const updateUrl = product.updateUrl; + + if (!updateUrl) { + console.warn(`Skipping policy localization: No 'updateUrl' found in 'product.json'.`); + return []; + } + + const resourceUrlTemplate = product.extensionsGallery?.resourceUrlTemplate; + + if (!resourceUrlTemplate) { + console.warn(`Skipping policy localization: No 'resourceUrlTemplate' found in 'product.json'.`); + return []; + } + + const version = await getLatestStableVersion(updateUrl); + const languageIds = Object.keys(Languages); + + return await Promise.all(languageIds.map( + languageId => getNLS(resourceUrlTemplate, languageId, version) + .then(languageTranslations => ({ languageId, languageTranslations })) + )); +} + +async function main() { + const [policies, translations] = await Promise.all([parsePolicies(), getTranslations()]); + const { admx, adml } = await renderGP(policies, translations); + + const root = '.build/policies/win32'; + await fs.rm(root, { recursive: true, force: true }); + await fs.mkdir(root, { recursive: true }); + + await fs.writeFile(path.join(root, `${product.win32RegValueName}.admx`), admx.replace(/\r?\n/g, '\n')); + + for (const { languageId, contents } of adml) { + const languagePath = path.join(root, languageId === 'en-us' ? 'en-us' : Languages[languageId as keyof typeof Languages]); + await fs.mkdir(languagePath, { recursive: true }); + await fs.writeFile(path.join(languagePath, `${product.win32RegValueName}.adml`), contents.replace(/\r?\n/g, '\n')); + } +} + +if (require.main === module) { + main().catch(err => { + console.error(err); + process.exit(1); + }); +} -- cgit v1.2.3