From 2bb4f344163c98f5bc2383c89812e11ab7f19f63 Mon Sep 17 00:00:00 2001 From: lovetox Date: Sun, 27 Mar 2022 13:49:56 +0200 Subject: chore: Move all ci scripts into .ci folder --- .ci/appveyor.yml | 57 +++++++++++++++++++++++++ .ci/appveyor_build.py | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++ .ci/deploy.py | 97 ++++++++++++++++++++++++++++++++++++++++++ .ci/link-gtk.py | 48 +++++++++++++++++++++ .ci/pylint.sh | 88 ++++++++++++++++++++++++++++++++++++++ .ci/sdist.py | 50 ++++++++++++++++++++++ 6 files changed, 455 insertions(+) create mode 100644 .ci/appveyor.yml create mode 100644 .ci/appveyor_build.py create mode 100644 .ci/deploy.py create mode 100755 .ci/link-gtk.py create mode 100755 .ci/pylint.sh create mode 100755 .ci/sdist.py (limited to '.ci') diff --git a/.ci/appveyor.yml b/.ci/appveyor.yml new file mode 100644 index 000000000..163cb9e4e --- /dev/null +++ b/.ci/appveyor.yml @@ -0,0 +1,57 @@ +version: 1.1.{build} + +image: Visual Studio 2019 + +environment: + matrix: + - MSYSTEM: MINGW64 + MSYS_ARCH: "x86_64" + ARCH: "64bit" + + - MSYSTEM: MINGW32 + MSYS_ARCH: "i686" + ARCH: "32bit" + +branches: + only: + - master + +clone_depth: 1 + +# init: +# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + +install: + - set PATH=C:\msys64\usr\bin;%PATH% + - bash -lc "pacman --needed --noconfirm -Syu" + # This is needed because without killing all processes -Su will fail + - ps: Get-Process | Where-Object {$_.path -like 'C:\msys64*'} | Stop-Process + - bash -lc "pacman -Sydd --noconfirm filesystem" + - bash -lc "pacman --needed --noconfirm -Su" + +build_script: + + - ps: | + $filename = "Gajim-$($env:GAJIM_VERSION)-$($env:ARCH)" + $filename_portable = "Gajim-$($env:GAJIM_VERSION)-Portable-$($env:ARCH)" + + if ($env:GAJIM_VERSION -eq "Nightly") { + $time_string=(get-date -UFormat "%Y-%m-%d").ToString() + $filename = $filename + "-" + $time_string + $filename_portable = $filename_portable + "-" + $time_string + } + + $buildroot="C:\msys64\home\appveyor\gajim\win\_build_root" + + function bash($command) { + Write-Host $command -NoNewline + C:\msys64\usr\bin\sh.exe --login -c $command + } + + bash "git clone C:/projects/gajim C:/msys64/home/appveyor/gajim" + bash "C:/msys64/home/appveyor/gajim/win/build.sh $($env:MSYS_ARCH)" + Push-AppveyorArtifact "$($buildroot)/Gajim.exe" -FileName "$($filename).exe" + Push-AppveyorArtifact "$($buildroot)/Gajim-Portable.exe" -FileName "$($filename_portable).exe" + +# on_finish: +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) diff --git a/.ci/appveyor_build.py b/.ci/appveyor_build.py new file mode 100644 index 000000000..907f41d7f --- /dev/null +++ b/.ci/appveyor_build.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +import os +import requests +import time +from pathlib import Path + +from rich.console import Console + +ACCOUNT = 'lovetox' +PROJECT_SLUG = 'gajim' +BRANCH = 'master' +BASE_URL = 'https://ci.appveyor.com/api' +API_KEY = os.environ['APPVEYOR_API_KEY'] +HEADERS = {'Authorization': f'Bearer {API_KEY}'} +RETRY_TIMEOUT = 2 * 60 +INITIAL_START_DELAY = 20 * 60 + +SETTINGS_API_URL = f'{BASE_URL}/projects/{ACCOUNT}/{PROJECT_SLUG}/settings/yaml' +BUILDS_API_URL = f'{BASE_URL}/builds' +PROJECT_API_URL = f'{BASE_URL}/projects/{ACCOUNT}/{PROJECT_SLUG}' + + +console = Console() + + +def get_gajim_version() -> str: + if os.environ.get('GAJIM_NIGHTLY_BUILD') is not None: + return 'Nightly' + + tag = os.environ.get('CI_COMMIT_TAG') + if tag is None: + exit('No tag found') + return tag.removesuffix('gajim-') + + +def push_yaml_to_project() -> None: + console.print('Push settings ...') + with open('.ci/appveyor.yml', 'r') as file: + yaml = file.read() + + req = requests.put(SETTINGS_API_URL, data=yaml, headers=HEADERS) + req.raise_for_status() + + +def start_build() -> str: + console.print('Start build ...') + payload = { + 'accountName': ACCOUNT, + 'projectSlug': PROJECT_SLUG, + 'branch': BRANCH, + 'commitId': os.environ['CI_COMMIT_SHA'], + 'environmentVariables': { + 'GAJIM_VERSION': get_gajim_version(), + } + } + req = requests.post(BUILDS_API_URL, headers=HEADERS, json=payload) + req.raise_for_status() + response = req.json() + return response['buildId'] + + +def is_build_finished(build: dict[str, str]) -> bool: + if build['status'] in ('failed', 'cancelled'): + exit('Found failed job') + + return build['status'] == 'success' + + +def get_artifacts(build_id: str) -> None: + time.sleep(INITIAL_START_DELAY) + while True: + time.sleep(RETRY_TIMEOUT) + + console.print('Check build status ...') + req = requests.get(PROJECT_API_URL, headers=HEADERS) + req.raise_for_status() + response = req.json() + build = response['build'] + if build_id != build['buildId']: + exit('Unable to find buildid: %s' % build_id) + + if is_build_finished(build): + break + + console.print('Build status:', build['status']) + + for job in build['jobs']: + download_job_artifacts(job['jobId']) + + console.print('All artifacts downloaded!') + + +def download_job_artifacts(job_id: str) -> None: + artifacts_api_url = f'{BASE_URL}/buildjobs/{job_id}/artifacts' + req = requests.get(artifacts_api_url, headers=HEADERS) + req.raise_for_status() + response = req.json() + + build_folder = Path.cwd() / 'build' + + for artifact in response: + filename = artifact['fileName'] + console.print('Download', filename, '...') + file_url = f'{artifacts_api_url}/{filename}' + req = requests.get(file_url, headers=HEADERS) + req.raise_for_status() + with open(build_folder / filename, 'wb') as file: + file.write(req.content) + + +if __name__ == '__main__': + push_yaml_to_project() + build_id = start_build() + get_artifacts(build_id) diff --git a/.ci/deploy.py b/.ci/deploy.py new file mode 100644 index 000000000..e8099d22d --- /dev/null +++ b/.ci/deploy.py @@ -0,0 +1,97 @@ +import os +import sys +from ftplib import FTP_TLS +from pathlib import Path +import functools +from typing import Any + +from rich.console import Console + +FTP_URL = 'panoramix.gajim.org' +FTP_USER = os.environ['FTP_USER'] +FTP_PASS = os.environ['FTP_PASS'] + +WINDOWS_NIGHTLY_FOLDER = 'win_snap' + +console = Console() + + +def ftp_connection(func: Any) -> Any: + @functools.wraps(func) + def func_wrapper(filedir: Path) -> None: + ftp = FTP_TLS(FTP_URL, FTP_USER, FTP_PASS) + console.print('Successfully connected to', FTP_URL) + func(ftp, filedir) + ftp.quit() + console.print('Quit') + return + return func_wrapper + + +def get_release_folder_from_tag(tag: str) -> str: + numbers = tag.split('.') + return '.'.join(numbers[:2]) + + +def get_gajim_tag() -> str: + tag = os.environ.get('CI_COMMIT_TAG') + if tag is None: + exit('No tag found') + return tag.removesuffix('gajim-') + + +def get_dir_list(ftp: FTP_TLS) -> list[str]: + return [x[0] for x in ftp.mlsd()] + + +def ensure_folder_exists(ftp: FTP_TLS, dirname: str) -> None: + dir_list = get_dir_list(ftp) + if dirname not in dir_list: + ftp.mkd(dirname) + + +def upload_all_from_dir(ftp: FTP_TLS, dir: Path) -> None: + for file_path in dir.iterdir(): + console.print('Upload file', file_path.name) + with open(file_path, 'rb') as f: + ftp.storbinary('STOR ' + file_path.name, f) + + +def get_deploy_method() -> str: + deploy_type = os.environ['DEPLOY_TYPE'] + is_nightly = bool(os.environ.get('GAJIM_NIGHTLY_BUILD')) + if is_nightly: + return f'deploy_{deploy_type}_nightly' + return f'deploy_{deploy_type}_release' + + +@ftp_connection +def deploy_windows_nightly(ftp: FTP_TLS, filedir: Path) -> None: + ftp.cwd(WINDOWS_NIGHTLY_FOLDER) + upload_all_from_dir(ftp, filedir) + + +@ftp_connection +def deploy_windows_release(ftp: FTP_TLS, filedir: Path) -> None: + tag = get_gajim_tag() + folder = get_release_folder_from_tag(tag) + ensure_folder_exists(ftp, folder) + ftp.cwd(folder) + upload_all_from_dir(ftp, filedir) + + +@ftp_connection +def deploy_linux_nightly(): + raise NotImplementedError + + +@ftp_connection +def deploy_linux_release(): + raise NotImplementedError + + +if __name__ == '__main__': + filedir = Path(sys.argv[1]) + current_module = sys.modules[__name__] + method = getattr(current_module, get_deploy_method()) + method(filedir) diff --git a/.ci/link-gtk.py b/.ci/link-gtk.py new file mode 100755 index 000000000..0690563f7 --- /dev/null +++ b/.ci/link-gtk.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Creates links from gui folder to all files in the gtk folder +# This is needed for pyright to work correctly with the dynamic gui imports + +from pathlib import Path + +IGNORED_FILES = ['__init__.py'] +IGNORED_DIRS = ['__pycache__'] + +cwd = Path.cwd() + +if cwd.name != 'gajim': + exit('Script needs to be excecuted from gajim repository root directory') + +gui_path = cwd / 'gajim' / 'gui' +gtk_path = cwd / 'gajim' / 'gtk' + + +def cleanup_dir(target_dir: Path) -> None: + for path in target_dir.iterdir(): + if path.name in IGNORED_FILES: + continue + if path.name in IGNORED_DIRS: + continue + path.unlink() + + +def link(target: Path) -> None: + source = str(target) + source = source.replace('gajim/gtk', 'gajim/gui') + source = Path(source) + source.symlink_to(target) + print('create symlink from', source, 'to', target) + + +def link_files(source_dir: Path) -> None: + for path in source_dir.iterdir(): + if path.is_dir(): + if path.name not in IGNORED_DIRS: + link(path) + + elif path.name not in IGNORED_FILES: + link(path) + + +cleanup_dir(gui_path) +link_files(gtk_path) diff --git a/.ci/pylint.sh b/.ci/pylint.sh new file mode 100755 index 000000000..300a6d10f --- /dev/null +++ b/.ci/pylint.sh @@ -0,0 +1,88 @@ +#!/bin/sh + +# exit when any command fails +set -e + +PYLINT=${PYLINT:-pylint} + +# Errors which are covered by pyright +# +# E1101 no-member +# E1102 not-callable +# E1111 assignment-from-no-return +# E1120 no-value-for-parameter +# E1121 too-many-function-args +# E1123 unexpected-keyword-arg +# E1124 redundant-keyword-arg +# E1125 missing-kwoa +# E1126 invalid-sequence-index +# E1127 invalid-slice-index +# E1128 assignment-from-none +# E1129 not-context-manager +# E1130 invalid-unary-operand-type +# E1131 unsupported-binary-operation +# E1132 repeated-keyword +# E1133 not-an-iterable +# E1134 not-a-mapping +# E1135 unsupported-membership-test +# E1136 unsubscriptable-object +# E1137 unsupported-assignment-operation +# E1138 unsupported-delete-operation +# E1139 invalid-metaclass +# E1140 unhashable-dict-key +# E1141 dict-iter-missing-items +# E1142 await-outside-async +# I1101 c-extension-no-member +# W0212 protected-access +# W1113 keyword-arg-before-vararg +# W1114 arguments-out-of-order +# W1115 non-str-assignment-to-dunder-name +# W1116 isinstance-second-argument-not-valid-type + +IGNORE_PYRIGHT='E1101,E1102,E1111,E1120,E1121,E1123,E1124,E1125,E1126,E1127,E1128,E1129,E1130,E1131,E1132,E1133,E1134,E1135,E1136,E1137,E1138,E1139,E1140,E1141,E1142,I1101,W0212,W1113,W1114,W1115,W1116' + +IGNORE_ALWAYS='R0801,C0209,W0237,W0707,R1732,W1404,W1406,R1725' + +IGNORE_ERRORS=C0103,C0302,C0301,C0330,C0411,C0415,E0401,E0611,R0201,R0901,R0904,R0912,R0913,R0914,R0915,R0916,R1702,R1710,W0201,W0221,W0223,W0231,W0233,W0603,W0613 + +IGNORE_GTK_MODULE_ERRORS=C0103,C0301,C0330,C0415,E0401,E0611,R0201,R0904,R0915,R1710,W0201,W0233,W0221,W0613 + +"$PYLINT" --version + +"$PYLINT" --disable="$IGNORE_PYRIGHT,$IGNORE_ALWAYS,$IGNORE_ERRORS" --ignore=modules,dbus,gtk "$@" +"$PYLINT" --disable="$IGNORE_PYRIGHT,$IGNORE_ALWAYS,$IGNORE_GTK_MODULE_ERRORS" "$@/gtk" +"$PYLINT" --disable="$IGNORE_PYRIGHT,$IGNORE_ALWAYS,E0401,C0415" "$@/common/modules" +"$PYLINT" --disable="$IGNORE_PYRIGHT,$IGNORE_ALWAYS" "$@/common/dbus" + +# C0103 invalid-name +# C0209 use-f-string +# C0301 line-too-long +# C0302 too-many-lines +# C0330 bad-continuation +# C0415 import-outside-toplevel +# E0401 import-error +# E0611 no-name-in-module +# R0201 no-self-use +# R0801 duplicat-code +# R0901 too-many-ancestors +# R0904 too-many-public-methods +# R0913 too-many-arguments +# R0912 too-many-branches +# R0914 too-many-locals +# R0915 too-many-statements +# R0916 too-many-boolean-expressions +# R1702 too-many-nested-blocks +# R1710 inconsistent-return-statements +# R1732 consider-using-with +# R1725 super-with-arguments +# W0201 attribute-defined-outside-init +# W0221 arguments-differ +# W0223 abstract-method +# W0231 super-init-not-called +# W0233 non-parent-init-called +# W0237 arguments-renamed +# W0603 global-statement +# W0613 unused-argument +# W0707 raise-missing-from +# W1404 implicit-str-concat +# W1406 redundant-u-string-prefix diff --git a/.ci/sdist.py b/.ci/sdist.py new file mode 100755 index 000000000..8aeda48b8 --- /dev/null +++ b/.ci/sdist.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import io +import zipfile +import subprocess +import shutil +from pathlib import Path +import requests + +PLUGINS = [ + 'plugin_installer', +] + +PLUGINS_BASE_URL = 'https://ftp.gajim.org' +PLUGINS_FOLDER = Path('./gajim/data/plugins') + + +def get_plugins_url(plugin): + return f'{PLUGINS_BASE_URL}/plugins_master_zip/{plugin}.zip' + + +def extraxt_zip(zip_bytes, path): + print('Extract to', path) + with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zip_file: + zip_file.extractall(path) + + +def download_plugins(): + PLUGINS_FOLDER.mkdir(parents=True) + for plugin in PLUGINS: + url = get_plugins_url(plugin) + print('Download', url) + req = requests.get(url) + req.raise_for_status() + extraxt_zip(req.content, PLUGINS_FOLDER) + + +def setup(): + print('Setup') + subprocess.call(['python3', 'setup.py', 'sdist']) + + +def cleanup(): + print('Cleanup') + shutil.rmtree(PLUGINS_FOLDER) + + +download_plugins() +setup() +cleanup() -- cgit v1.2.3