Refactor CI Pipelines

- Move tasks to python scripts
- Rework appveyor build
- Add deploy jobs

5 files changed, 403 insertions, 68 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 57ead0c60..44e98e98c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,14 +1,19 @@
 image: ci-gajim:master
 
-include:
-- project: 'gajim/ci'
-  ref: master
-  file: '.gajim-ci.yml'
+workflow:
+  rules:
+    - if: $GAJIM_NIGHTLY_BUILD
+    - if: $CI_PIPELINE_SOURCE == "push"
+
+stages:
+  - test
+  - build
+  - deploy
 
 pyright:
   stage: test
   script:
-    - pip3 install git+https://dev.gajim.org/gajim/python-nbxmpp.git@$NBXMPP_BRANCH
+    - pip3 install git+https://dev.gajim.org/gajim/python-nbxmpp.git
     - pip3 install git+https://github.com/pygobject/pygobject-stubs.git
     - npm install pyright
     - python3 -V
@@ -16,8 +21,77 @@ pyright:
     - $(npm bin)/pyright --version
     - $(npm bin)/pyright
   interruptible: true
-  allow_failure: false
 
-variables:
-  NBXMPP_BRANCH: master
-  PLUGINS_BRANCH: master
+code-quality:
+  stage: test
+  script:
+    - python3 -V
+    - pip3 install -I git+https://dev.gajim.org/gajim/python-nbxmpp.git
+    - scripts/ci/pylint.sh --jobs=2 gajim
+    - coverage run --source=gajim -m unittest discover -s test -v
+    - coverage report -mi
+    - coverage xml -i
+    - codespell -I codespell.conf --skip="*__pycache__*,gajim/data/icons,gajim/data/sounds,gajim/data/emoticons" gajim
+    - python3 setup.py build
+    - appstream-util validate build/data/org.gajim.Gajim.appdata.xml
+  coverage: "/TOTAL.+ ([0-9]{1,3}%)/"
+  artifacts:
+    reports:
+      cobertura: coverage.xml
+  interruptible: true
+
+build-linux:
+  stage: build
+  rules:
+    - if: '$GAJIM_NIGHTLY_BUILD'
+    - if: '$CI_COMMIT_TAG'
+      when: manual
+  script:
+    - python3 ./scripts/ci/sdist.py
+
+  artifacts:
+    name: "gajim-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA"
+    expire_in: 1 week
+    paths:
+      - dist/gajim-*.tar.gz
+
+build-windows:
+  stage: build
+  rules:
+    - if: '$GAJIM_NIGHTLY_BUILD'
+    - if: '$CI_COMMIT_TAG'
+  script:
+    - python3 ./win/ci/appveyor_build.py
+
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - build/Gajim-*.exe
+
+deploy-windows:
+  stage: deploy
+  needs: ['build-windows']
+  rules:
+    - if: '$GAJIM_NIGHTLY_BUILD'
+    - if: '$CI_COMMIT_TAG'
+      when: manual
+  variables:
+    DEPLOY_TYPE: "windows"
+  script:
+    - python3 ./scripts/ci/deploy.py build
+
+deploy-flatpak:
+  stage: deploy
+  needs: []
+  rules:
+    - if: '$CI_COMMIT_TAG'
+  when: manual
+  script:
+    - git clone https://github.com/flathub/org.gajim.Gajim.git
+    - cd org.gajim.Gajim
+    - git config user.email "ci@gajim.org"
+    - git config user.name "CI Deploy"
+    - mv ../flatpak/org.gajim.Gajim.yaml org.gajim.Gajim.yaml
+    - git diff
+    - git add org.gajim.Gajim.yaml
+    - git commit -m "$CI_COMMIT_TAG"
diff --git a/scripts/ci/deploy.py b/scripts/ci/deploy.py
new file mode 100644
index 000000000..8e77c6644
--- /dev/null
+++ b/scripts/ci/deploy.py
@@ -0,0 +1,97 @@
+import os
+import sys
+from ftplib import FTP_TLS, ftpcp
+from pathlib import Path
+import functools
+from typing import Any
+
+from rich.console import Console
+
+FTP_URL = 'panoramix.gajim.org'
+FTP_USER = os.environ['FTP_USER']
+FTP_PASS = os.environ['FTP_PASS']
+
+WINDOWS_NIGHTLY_FOLDER = 'win_snap'
+
+console = Console()
+
+
+def ftp_connection(func: Any) -> Any:
+    @functools.wraps(func)
+    def func_wrapper(filedir: Path) -> None:
+        ftp = FTP_TLS(FTP_URL, FTP_USER, FTP_PASS)
+        console.print('Successfully connected to', FTP_URL)
+        func(ftp, filedir)
+        ftp.quit()
+        console.print('Quit')
+        return
+    return func_wrapper
+
+
+def get_release_folder_from_tag(tag: str) -> str:
+    numbers = tag.split('.')
+    return '.'.join(numbers[:2])
+
+
+def get_gajim_tag() -> str:
+    tag = os.environ.get('CI_COMMIT_TAG')
+    if tag is None:
+        exit('No tag found')
+    return tag.removesuffix('gajim-')
+
+
+def get_dir_list(ftp: FTP_TLS) -> list[str]:
+    return [x[0] for x in ftp.mlsd()]
+
+
+def ensure_folder_exists(ftp: FTP_TLS, dirname: str) -> None:
+    dir_list = get_dir_list(ftp)
+    if dirname not in dir_list:
+        ftp.mkd(dirname)
+
+
+def upload_all_from_dir(ftp: FTP_TLS, dir: Path) -> None:
+    for file_path in dir.iterdir():
+        console.print('Upload file', file_path.name)
+        with open(file_path, 'rb') as f:
+            ftp.storbinary('STOR ' + file_path.name, f)
+
+
+def get_deploy_method() -> str:
+    deploy_type = os.environ['DEPLOY_TYPE']
+    is_nightly = bool(os.environ.get('GAJIM_NIGHTLY_BUILD'))
+    if is_nightly:
+        return f'deploy_{deploy_type}_nightly'
+    return f'deploy_{deploy_type}_release'
+
+
+@ftp_connection
+def deploy_windows_nightly(ftp: FTP_TLS, filedir: Path) -> None:
+    ftp.cwd(WINDOWS_NIGHTLY_FOLDER)
+    upload_all_from_dir(ftp, filedir)
+
+
+@ftp_connection
+def deploy_windows_release(ftp: FTP_TLS, filedir: Path) -> None:
+    tag = get_gajim_tag()
+    folder = get_release_folder_from_tag(tag)
+    ensure_folder_exists(ftp, folder)
+    ftp.cwd(folder)
+    upload_all_from_dir(ftp, filedir)
+
+
+@ftp_connection
+def deploy_linux_nightly():
+    raise NotImplementedError
+
+
+@ftp_connection
+def deploy_linux_release():
+    raise NotImplementedError
+
+
+if __name__ == '__main__':
+    filedir = Path(sys.argv[1])
+    current_module = sys.modules[__name__]
+    method = getattr(current_module, get_deploy_method())
+    method(filedir)
diff --git a/scripts/ci/sdist.py b/scripts/ci/sdist.py
new file mode 100755
index 000000000..8aeda48b8
--- /dev/null
+++ b/scripts/ci/sdist.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import io
+import zipfile
+import subprocess
+import shutil
+from pathlib import Path
+import requests
+
+PLUGINS = [
+    'plugin_installer',
+]
+
+PLUGINS_BASE_URL = 'https://ftp.gajim.org'
+PLUGINS_FOLDER = Path('./gajim/data/plugins')
+
+
+def get_plugins_url(plugin):
+    return f'{PLUGINS_BASE_URL}/plugins_master_zip/{plugin}.zip'
+
+
+def extraxt_zip(zip_bytes, path):
+    print('Extract to', path)
+    with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zip_file:
+        zip_file.extractall(path)
+
+
+def download_plugins():
+    PLUGINS_FOLDER.mkdir(parents=True)
+    for plugin in PLUGINS:
+        url = get_plugins_url(plugin)
+        print('Download', url)
+        req = requests.get(url)
+        req.raise_for_status()
+        extraxt_zip(req.content, PLUGINS_FOLDER)
+
+
+def setup():
+    print('Setup')
+    subprocess.call(['python3', 'setup.py', 'sdist'])
+
+
+def cleanup():
+    print('Cleanup')
+    shutil.rmtree(PLUGINS_FOLDER)
+
+
+download_plugins()
+setup()
+cleanup()
diff --git a/appveyor.yml b/win/ci/appveyor.yml
index d51340cdd..163cb9e4e 100644
--- a/appveyor.yml
+++ b/win/ci/appveyor.yml
@@ -1,59 +1,57 @@
-image: Visual Studio 2019

-

-environment:

-  matrix:

-    - MSYSTEM: MINGW64

-      MSYS_ARCH: "x86_64"

-      ARCH: "64bit"

-

-    - MSYSTEM: MINGW32

-      MSYS_ARCH: "i686"

-      ARCH: "32bit"

-

-branches:

-  only:

-    - master

-

-clone_depth: 1

-

-# init:

-#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

-

-install:

-  - set PATH=C:\msys64\usr\bin;%PATH%

-  - bash -lc "pacman --needed --noconfirm -Syu"

-  # This is needed because without killing all processes -Su will fail

-  - ps: Get-Process | Where-Object {$_.path -like 'C:\msys64*'} | Stop-Process

-  - bash -lc "pacman -Sydd --noconfirm filesystem"

-  - bash -lc "pacman --needed --noconfirm -Su"

-

-build_script:

-

-  - ps: |

-        $env:TIME_STRING=(get-date -UFormat "%Y-%m-%d").ToString()

-        $env:BUILDROOT="C:\msys64\home\appveyor\gajim\win\_build_root"

-

-        function bash($command) {

-          Write-Host $command -NoNewline

-          C:\msys64\usr\bin\sh.exe --login -c $command

-        }

-

-        bash "git clone C:/projects/gajim C:/msys64/home/appveyor/gajim"

-        bash "C:/msys64/home/appveyor/gajim/win/build.sh $($env:MSYS_ARCH)"

-        Push-AppveyorArtifact "$($env:BUILDROOT)/Gajim.exe" -FileName "Gajim-Master-$($env:ARCH)-$($env:TIME_STRING).exe"

-        Push-AppveyorArtifact "$($env:BUILDROOT)/Gajim-Portable.exe" -FileName "Gajim-Portable-Master-$($env:ARCH)-$($env:TIME_STRING).exe"

-

-# on_finish:

-#   - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

- 

-deploy:

-    # FTP deployment provider settings

-  - provider: FTP

-    protocol: ftp

-    host: panoramix.gajim.org

-    username:

-      secure: SNcOJDhUtBjfddbKXudE2w==

-    password:

-      secure: tQkYbcUb6nChCp0cdqo4CA==

-    folder: win_snap

-    debug: true

+version: 1.1.{build}
+
+image: Visual Studio 2019
+
+environment:
+  matrix:
+    - MSYSTEM: MINGW64
+      MSYS_ARCH: "x86_64"
+      ARCH: "64bit"
+
+    - MSYSTEM: MINGW32
+      MSYS_ARCH: "i686"
+      ARCH: "32bit"
+
+branches:
+  only:
+    - master
+
+clone_depth: 1
+
+# init:
+#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+
+install:
+  - set PATH=C:\msys64\usr\bin;%PATH%
+  - bash -lc "pacman --needed --noconfirm -Syu"
+  # This is needed because without killing all processes -Su will fail
+  - ps: Get-Process | Where-Object {$_.path -like 'C:\msys64*'} | Stop-Process
+  - bash -lc "pacman -Sydd --noconfirm filesystem"
+  - bash -lc "pacman --needed --noconfirm -Su"
+
+build_script:
+
+  - ps: |
+        $filename = "Gajim-$($env:GAJIM_VERSION)-$($env:ARCH)"
+        $filename_portable = "Gajim-$($env:GAJIM_VERSION)-Portable-$($env:ARCH)"
+
+        if ($env:GAJIM_VERSION -eq "Nightly") {
+            $time_string=(get-date -UFormat "%Y-%m-%d").ToString()
+            $filename = $filename + "-" + $time_string
+            $filename_portable = $filename_portable + "-" + $time_string
+        }
+
+        $buildroot="C:\msys64\home\appveyor\gajim\win\_build_root"
+
+        function bash($command) {
+          Write-Host $command -NoNewline
+          C:\msys64\usr\bin\sh.exe --login -c $command
+        }
+
+        bash "git clone C:/projects/gajim C:/msys64/home/appveyor/gajim"
+        bash "C:/msys64/home/appveyor/gajim/win/build.sh $($env:MSYS_ARCH)"
+        Push-AppveyorArtifact "$($buildroot)/Gajim.exe" -FileName "$($filename).exe"
+        Push-AppveyorArtifact "$($buildroot)/Gajim-Portable.exe" -FileName "$($filename_portable).exe"
+
+# on_finish:
+#   - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
diff --git a/win/ci/appveyor_build.py b/win/ci/appveyor_build.py
new file mode 100644
index 000000000..4a4d9bbf1
--- /dev/null
+++ b/win/ci/appveyor_build.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+from typing import Iterator
+
+import os
+import requests
+import time
+from pathlib import Path
+
+from rich.console import Console
+
+ACCOUNT = 'lovetox'
+PROJECT_SLUG = 'gajim'
+BRANCH = 'master'
+BASE_URL = 'https://ci.appveyor.com/api'
+API_KEY = os.environ['APPVEYOR_API_KEY']
+HEADERS = {'Authorization': f'Bearer {API_KEY}'}
+RETRY_TIMEOUT = 2 * 60
+INITIAL_START_DELAY = 20 * 60
+
+SETTINGS_API_URL = f'{BASE_URL}/projects/{ACCOUNT}/{PROJECT_SLUG}/settings/yaml'
+BUILDS_API_URL = f'{BASE_URL}/builds'
+PROJECT_API_URL = f'{BASE_URL}/projects/{ACCOUNT}/{PROJECT_SLUG}'
+
+
+console = Console()
+
+
+def get_gajim_version() -> str:
+    if os.environ.get('GAJIM_NIGHTLY_BUILD') is not None:
+        return 'Nightly'
+
+    tag = os.environ.get('CI_COMMIT_TAG')
+    if tag is None:
+        exit('No tag found')
+    return tag.removesuffix('gajim-')
+
+
+def push_yaml_to_project() -> None:
+    console.print('Push settings ...')
+    with open('./win/ci/appveyor.yml', 'r') as file:
+        yaml = file.read()
+
+    req = requests.put(SETTINGS_API_URL, data=yaml, headers=HEADERS)
+    req.raise_for_status()
+
+
+def start_build() -> str:
+    console.print('Start build ...')
+    payload = {
+        'accountName': ACCOUNT,
+        'projectSlug': PROJECT_SLUG,
+        'branch': BRANCH,
+        'environmentVariables': {
+           'GAJIM_VERSION': get_gajim_version(),
+        }
+    }
+    req = requests.post(BUILDS_API_URL, headers=HEADERS, json=payload)
+    req.raise_for_status()
+    response = req.json()
+    return response['buildId']
+
+
+def is_build_finished(build: dict[str, str]) -> bool:
+    if build['status'] in ('failed', 'cancelled'):
+        exit('Found failed job')
+
+    return build['status'] == 'success'
+
+
+def get_artifacts(build_id: str) -> None:
+    time.sleep(INITIAL_START_DELAY)
+    while True:
+        time.sleep(RETRY_TIMEOUT)
+
+        console.print('Check build status ...')
+        req = requests.get(PROJECT_API_URL, headers=HEADERS)
+        req.raise_for_status()
+        response = req.json()
+        build = response['build']
+        if build_id != build['buildId']:
+            exit('Unable to find buildid: %s' % build_id)
+
+        if is_build_finished(build):
+            break
+
+        console.print('Build status:', build['status'])
+
+    for job in build['jobs']:
+        download_job_artifacts(job['jobId'])
+
+    console.print('All artifacts downloaded!')
+
+
+def download_job_artifacts(job_id: str) -> None:
+    artifacts_api_url = f'{BASE_URL}/buildjobs/{job_id}/artifacts'
+    req = requests.get(artifacts_api_url, headers=HEADERS)
+    req.raise_for_status()
+    response = req.json()
+
+    build_folder = Path.cwd() / 'build'
+
+    for artifact in response:
+        filename = artifact['fileName']
+        console.print('Download', filename, '...')
+        file_url = f'{artifacts_api_url}/{filename}'
+        req = requests.get(file_url, headers=HEADERS)
+        req.raise_for_status()
+        with open(build_folder / filename, 'wb') as file:
+            file.write(req.content)
+
+
+if __name__ == '__main__':
+    push_yaml_to_project()
+    build_id = start_build()
+    get_artifacts(build_id)