diff options
author | Adrien Ferrand <ferrand.ad@gmail.com> | 2020-02-18 15:58:18 +0300 |
---|---|---|
committer | Brad Warren <bmw@eff.org> | 2020-04-22 02:36:49 +0300 |
commit | 2bcabe6626ad4aa0073269bcee9719c24da469cb (patch) | |
tree | 83f60cee0d512be0cb26748e7e8a35950554731b /snap | |
parent | c12baf7d8c873a1492088f844fe885a16bf19cd0 (diff) |
Fix certbot part build in snap
* Declare properly source-subdir to build the certbot part
* Use snapcraft 3.10+, remove the custom python plugin
Diffstat (limited to 'snap')
-rw-r--r-- | snap/.travis.yml | 3 | ||||
-rw-r--r-- | snap/snap/plugins/x_python.py | 499 | ||||
-rw-r--r-- | snap/snap/snapcraft.yaml | 11 |
3 files changed, 8 insertions, 505 deletions
diff --git a/snap/.travis.yml b/snap/.travis.yml index 418407558..125893985 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -27,7 +27,8 @@ notifications: addons: snaps: - name: snapcraft - channel: stable + # TODO: move back to stable channel once snapcraft 3.10 has been pushed to it. + channel: candidate classic: true - name: lxd channel: stable diff --git a/snap/snap/plugins/x_python.py b/snap/snap/plugins/x_python.py deleted file mode 100644 index 5c8c3b86a..000000000 --- a/snap/snap/plugins/x_python.py +++ /dev/null @@ -1,499 +0,0 @@ -# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- -# -# This is an almost verbatim copy of what is in the snapcraft -# tree, can be completely removed once the functionality in -# snapcraft is in place. (LP: #1841861) -# -# Copyright (C) 2016-2020 Canonical Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -"""The python plugin can be used for python 2 or 3 based parts. - -It can be used for python projects where you would want to do: - - - import python modules with a requirements.txt - - build a python project that has a setup.py - - install packages straight from pip - -This plugin uses the common plugin keywords as well as those for "sources". -For more information check the 'plugins' topic for the former and the -'sources' topic for the latter. - -Additionally, this plugin uses the following plugin-specific keywords: - - - requirements: - (list of strings) - List of paths to requirements files. - - constraints: - (list of strings) - List of paths to constraint files. - - process-dependency-links: - (bool; default: false) - Enable the processing of dependency links in pip, which allow one - project to provide places to look for another project - - python-packages: - (list) - A list of dependencies to get from PyPI - - python-version: - (string; default: python3) - The python version to use. Valid options are: python2 and python3 - -If the plugin finds a python interpreter with a basename that matches -`python-version` in the <stage> directory on the following fixed path: -`<stage-dir>/usr/bin/<python-interpreter>` then this interpreter would -be preferred instead and no interpreter would be brought in through -`stage-packages` mechanisms. -""" - -import collections -import contextlib -import os -import re -from shutil import which -from textwrap import dedent -from typing import List, Optional, Set - -import requests - -import snapcraft -from snapcraft.common import isurl -from snapcraft.internal import errors, mangling -from snapcraft.internal.errors import SnapcraftPluginCommandError -from snapcraft.plugins import _python - - -class UnsupportedPythonVersionError(snapcraft.internal.errors.SnapcraftError): - - fmt = "Unsupported python version: {python_version!r}" - - -class SnapcraftPluginPythonFileMissing(snapcraft.internal.errors.SnapcraftError): - - fmt = ( - "Failed to find the referred {plugin_property} file at the given " - "path: {plugin_property_value!r}.\n" - "Check the property and ensure the file exists." - ) - - def __init__(self, *, plugin_property, plugin_property_value): - super().__init__( - plugin_property=plugin_property, plugin_property_value=plugin_property_value - ) - - -class PythonPlugin(snapcraft.BasePlugin): - @classmethod - def schema(cls): - schema = super().schema() - schema["properties"]["requirements"] = { - "type": "array", - "minitems": 1, - "uniqueItems": True, - "items": {"type": "string"}, - "default": [], - } - schema["properties"]["constraints"] = { - "type": "array", - "minitems": 1, - "uniqueItems": True, - "items": {"type": "string"}, - "default": [], - } - schema["properties"]["python-packages"] = { - "type": "array", - "minitems": 1, - "uniqueItems": True, - "items": {"type": "string"}, - "default": [], - } - schema["properties"]["process-dependency-links"] = { - "type": "boolean", - "default": False, - } - schema["properties"]["python-version"] = { - "type": "string", - "default": "python3", - "enum": ["python2", "python3"], - } - schema["anyOf"] = [{"required": ["source"]}, {"required": ["python-packages"]}] - - return schema - - @classmethod - def get_pull_properties(cls): - # Inform Snapcraft of the properties associated with pulling. If these - # change in the YAML Snapcraft will consider the pull step dirty. - return [ - "requirements", - "constraints", - "python-packages", - "process-dependency-links", - "python-version", - ] - - @property - def plugin_stage_packages(self): - if self.options.python_version == "python2": - python_base = "python" - elif self.options.python_version == "python3": - python_base = "python3" - - if self.project.info.get_build_base() in ("core", "core16", "core18"): - stage_packages = [python_base] - else: - stage_packages = [] - - if self.project.info.get_build_base() == "core18" and python_base == "python3": - stage_packages.append("{}-distutils".format(python_base)) - - return stage_packages - - # ignore mypy error: Read-only property cannot override read-write property - @property # type: ignore - def stage_packages(self): - try: - _python.get_python_command( - self._python_major_version, - stage_dir=self.project.stage_dir, - install_dir=self.installdir, - ) - except _python.errors.MissingPythonCommandError: - return super().stage_packages + self.plugin_stage_packages - else: - return super().stage_packages - - @property - def _pip(self): - if not self.__pip: - self.__pip = _python.Pip( - python_major_version=self._python_major_version, - part_dir=self.partdir, - install_dir=self.installdir, - stage_dir=self.project.stage_dir, - ) - return self.__pip - - def __init__(self, name, options, project): - super().__init__(name, options, project) - - self._setup_base_tools(project.info.get_build_base()) - - self._manifest = collections.OrderedDict() - - # Pip requires only the major version of python rather than the command - # name like our option requires. - match = re.match(r"python(?P<major_version>\d).*", self.options.python_version) - if not match: - raise UnsupportedPythonVersionError( - python_version=self.options.python_version - ) - - self._python_major_version = match.group("major_version") - self.__pip = None - - def _setup_base_tools(self, base): - # NOTE: stage-packages are lazily loaded. - if base in ("core", "core16", "core18"): - if self.options.python_version == "python3": - self.build_packages.extend( - [ - "python3-dev", - "python3-pip", - "python3-pkg-resources", - "python3-setuptools", - ] - ) - elif self.options.python_version == "python2": - self.build_packages.extend( - [ - "python-dev", - "python-pip", - "python-pkg-resources", - "python-setuptools", - ] - ) - else: - raise errors.PluginBaseError(part_name=self.name, base=base) - - def pull(self): - super().pull() - - self._pip.setup() - - with simple_env_bzr(os.path.join(self.installdir, "bin")): - # Download this project, using its setup.py if present. This will - # also download any python-packages requested. - self._download_project() - - def clean_pull(self): - super().clean_pull() - self._pip.clean_packages() - - def build(self): - super().build() - - with simple_env_bzr(os.path.join(self.installdir, "bin")): - # Install the packages that have already been downloaded - installed_pipy_packages = self._install_project() - - requirements = self._get_list_of_packages_from_property( - self.options.requirements - ) - if requirements: - self._manifest["requirements-contents"] = requirements - - constraints = self._get_list_of_packages_from_property(self.options.constraints) - if constraints: - self._manifest["constraints-contents"] = constraints - - self._manifest["python-packages"] = [ - "{}={}".format(name, installed_pipy_packages[name]) - for name in installed_pipy_packages - ] - - try: - _python.generate_sitecustomize( - self._python_major_version, - stage_dir=self.project.stage_dir, - install_dir=self.installdir, - ) - except _python.errors.MissingUserSitePackagesError as site_error: - print("Part {!r} generated to site-packages: {!s}.".format( - self.name, site_error)) - - def _find_file(self, *, filename: str) -> Optional[str]: - # source-subdir defaults to '' - for basepath in [self.builddir, self.sourcedir]: - if basepath == self.sourcedir: - # This is overwritten in the base plugin - # TODO add consistency - source_subdir = self.options.source_subdir - else: - source_subdir = "" - filepath = os.path.join(basepath, source_subdir, filename) - if os.path.exists(filepath): - return filepath - - return None - - def _get_setup_py_dir(self): - setup_py_dir = None - setup_py_path = self._find_file(filename="setup.py") - if setup_py_path: - setup_py_dir = os.path.dirname(setup_py_path) - - return setup_py_dir - - def _get_list_of_packages_from_property(self, property_list: Set[str]) -> List[str]: - """Return a sorted list of all packages found in property.""" - package_list = list() # type: List[str] - for entry in property_list: - contents = self._get_file_contents(entry) - package_list.extend(contents.splitlines()) - return package_list - - def _get_normalized_property_set( - self, property_name, property_list: List[str] - ) -> Set[str]: - """Return a normalized set from a requirements or constraints list.""" - normalized = set() # type: Set[str] - for entry in property_list: - if isurl(entry): - normalized.add(entry) - else: - entry_file = self._find_file(filename=entry) - if not entry_file: - raise SnapcraftPluginPythonFileMissing( - plugin_property=property_name, plugin_property_value=entry - ) - normalized.add(entry_file) - - return normalized - - def _install_wheels(self, wheels): - installed = self._pip.list() - wheel_names = [os.path.basename(w).split("-")[0] for w in wheels] - - # we want to avoid installing what is already provided in - # stage-packages - need_install = [k for k in wheel_names if k not in installed] - self._pip.install( - need_install, - upgrade=True, - install_deps=False, - process_dependency_links=self.options.process_dependency_links, - ) - - def _download_project(self): - constraints = self._get_normalized_property_set( - "constraints", self.options.constraints - ) - requirements = self._get_normalized_property_set( - "requirements", self.options.requirements - ) - - self._pip.download( - self.options.python_packages, - setup_py_dir=None, - constraints=constraints, - requirements=requirements, - process_dependency_links=self.options.process_dependency_links, - ) - - def _install_project(self): - setup_py_dir = self._get_setup_py_dir() - constraints = self._get_normalized_property_set( - "constraints", self.options.constraints - ) - requirements = self._get_normalized_property_set( - "requirements", self.options.requirements - ) - - # setup.py is handled in a different step as some projects may - # need to satisfy dependencies for setup.py to be parsed. - wheels = self._pip.wheel( - self.options.python_packages, - setup_py_dir=None, - constraints=constraints, - requirements=requirements, - process_dependency_links=self.options.process_dependency_links, - ) - - if wheels: - self._install_wheels(wheels) - - if setup_py_dir is not None: - self._pip.download( - [], - setup_py_dir=setup_py_dir, - constraints=constraints, - requirements=set(), - process_dependency_links=self.options.process_dependency_links, - ) - wheels = self._pip.wheel( - [], - setup_py_dir=setup_py_dir, - constraints=constraints, - requirements=set(), - process_dependency_links=self.options.process_dependency_links, - ) - - if wheels: - self._install_wheels(wheels) - - setup_py_path = os.path.join(setup_py_dir, "setup.py") - if os.path.exists(setup_py_path): - self._pip.install( - [], - setup_py_dir=setup_py_dir, - constraints=constraints, - process_dependency_links=self.options.process_dependency_links, - upgrade=True, - ) - # pbr and others don't work using `pip install .` - # LP: #1670852 - # There is also a chance that this setup.py is distutils based - # in which case we will rely on the `pip install .` ran before - # this. - with contextlib.suppress(SnapcraftPluginCommandError): - self._setup_tools_install(setup_py_path) - - return self._pip.list() - - def _setup_tools_install(self, setup_file): - command = [ - _python.get_python_command( - self._python_major_version, - stage_dir=self.project.stage_dir, - install_dir=self.installdir, - ), - os.path.basename(setup_file), - "--no-user-cfg", - "install", - "--single-version-externally-managed", - "--user", - "--record", - "install.txt", - ] - self.run(command, env=self._pip.env(), cwd=os.path.dirname(setup_file)) - - # Fix all shebangs to use the in-snap python. The stuff installed from - # pip has already been fixed, but anything done in this step has not. - mangling.rewrite_python_shebangs(self.installdir) - - def _get_file_contents(self, path): - if isurl(path): - return requests.get(path).text - else: - file_path = os.path.join(self.sourcedir, path) - with open(file_path) as _file: - return _file.read() - - def get_manifest(self): - return self._manifest - - def snap_fileset(self): - fileset = super().snap_fileset() - fileset.append("-bin/pip") - fileset.append("-bin/pip2") - fileset.append("-bin/pip3") - fileset.append("-bin/pip2.7") - fileset.append("-bin/pip3.*") - fileset.append("-bin/easy_install*") - fileset.append("-bin/wheel") - # Holds all the .pyc files. It is a major cause of inter part - # conflict. - fileset.append("-**/__pycache__") - fileset.append("-**/*.pyc") - # The RECORD files include hashes useful when uninstalling packages. - # In the snap they will cause conflicts when more than one part uses - # the python plugin. - fileset.append("-lib/python*/site-packages/*/RECORD") - return fileset - - -@contextlib.contextmanager -def simple_env_bzr(bin_dir): - """Create an appropriate environment to run bzr. - - The python plugin sets up PYTHONUSERBASE and PYTHONHOME which - conflicts with bzr when using python3 as those two environment - variables will make bzr look for modules in the wrong location. - """ - os.makedirs(bin_dir, exist_ok=True) - bzr_bin = os.path.join(bin_dir, "bzr") - real_bzr_bin = which("bzr") - if real_bzr_bin: - exec_line = 'exec {} "$@"'.format(real_bzr_bin) - else: - exec_line = "echo bzr needs to be in PATH; exit 1" - with open(bzr_bin, "w") as f: - f.write( - dedent( - """#!/bin/sh - unset PYTHONUSERBASE - unset PYTHONHOME - {} - """.format( - exec_line - ) - ) - ) - os.chmod(bzr_bin, 0o777) - try: - yield - finally: - os.remove(bzr_bin) - if not os.listdir(bin_dir): - os.rmdir(bin_dir) diff --git a/snap/snap/snapcraft.yaml b/snap/snap/snapcraft.yaml index 0e4a12204..53643865a 100644 --- a/snap/snap/snapcraft.yaml +++ b/snap/snap/snapcraft.yaml @@ -38,20 +38,21 @@ apps: parts: python-augeas: - plugin: x-python + plugin: python source: git://github.com/basak/python-augeas source-branch: snap python-version: python3 build-packages: [libaugeas-dev] acme: - plugin: x-python + plugin: python source: certbot source-subdir: acme constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 certbot: - plugin: x-python + plugin: python source: certbot + source-subdir: certbot constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [acme] @@ -62,7 +63,7 @@ parts: stage: - -usr/lib/python3.6/sitecustomize.py certbot-apache: - plugin: x-python + plugin: python source: certbot source-subdir: certbot-apache constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] @@ -73,7 +74,7 @@ parts: # Prefer cffi - -lib/python3.6/site-packages/augeas.py certbot-nginx: - plugin: x-python + plugin: python source: certbot source-subdir: certbot-nginx constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] |