1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
#!/usr/bin/env python
# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
# to 1, a combination of tools/oldest_constraints.txt,
# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
# top level of the package's directory is used, otherwise, a combination of
# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
# other file always takes precedence over tools/dev_constraints.txt. If
# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
# no other arguments.
from __future__ import absolute_import
from __future__ import print_function
import contextlib
import os
import re
import shutil
import subprocess
import sys
import tempfile
import merge_requirements as merge_module
import readlink
import strip_hashes
# Once this code doesn't need to support Python 2, we can simply use
# tempfile.TemporaryDirectory.
@contextlib.contextmanager
def temporary_directory():
dirpath = tempfile.mkdtemp()
try:
yield dirpath
finally:
shutil.rmtree(dirpath)
def find_tools_path():
return os.path.dirname(readlink.main(__file__))
def certbot_oldest_processing(tools_path, args, test_constraints):
if args[0] != '-e' or len(args) != 2:
raise ValueError('When CERTBOT_OLDEST is set, this script must be run '
'with a single -e <path> argument.')
# remove any extras such as [dev]
pkg_dir = re.sub(r'\[\w+\]', '', args[1])
requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt')
shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints)
# packages like acme don't have any local oldest requirements
if not os.path.isfile(requirements):
return None
return requirements
def certbot_normal_processing(tools_path, test_constraints):
repo_path = os.path.dirname(tools_path)
certbot_requirements = os.path.normpath(os.path.join(
repo_path, 'tools/certbot_constraints.txt'))
with open(certbot_requirements, 'r') as fd:
certbot_reqs = fd.readlines()
with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd:
pipstrap_reqs = fd.readlines()
with open(test_constraints, 'w') as fd:
data_certbot = "\n".join(strip_hashes.process_entries(certbot_reqs))
data_pipstrap = "\n".join(strip_hashes.process_entries(pipstrap_reqs))
data = "\n".join([data_certbot, data_pipstrap])
fd.write(data)
def merge_requirements(tools_path, requirements, test_constraints, all_constraints):
# Order of the files in the merge function matters.
# Indeed version retained for a given package will be the last version
# found when following all requirements in the given order.
# Here is the order by increasing priority:
# 1) The general development constraints (tools/dev_constraints.txt)
# 2) The general tests constraints (oldest_requirements.txt or
# certbot_constraints.txt + pipstrap's constraints for the normal processing)
# 3) The local requirement file, typically local-oldest-requirement in oldest tests
files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints]
if requirements:
files.append(requirements)
merged_requirements = merge_module.main(*files)
with open(all_constraints, 'w') as fd:
fd.write(merged_requirements)
def call_with_print(command, env=None):
if not env:
env = os.environ
print(command)
subprocess.check_call(command, shell=True, env=env)
def pip_install_with_print(args_str, env=None):
if not env:
env = os.environ
command = ['"', sys.executable, '" -m pip install --disable-pip-version-check ', args_str]
call_with_print(''.join(command), env=env)
def main(args):
tools_path = find_tools_path()
with temporary_directory() as working_dir:
test_constraints = os.path.join(working_dir, 'test_constraints.txt')
all_constraints = os.path.join(working_dir, 'all_constraints.txt')
if os.environ.get('CERTBOT_NO_PIN') == '1':
# With unpinned dependencies, there is no constraint
pip_install_with_print(' '.join(args))
else:
# Otherwise, we merge requirements to build the constraints and pin dependencies
requirements = None
if os.environ.get('CERTBOT_OLDEST') == '1':
requirements = certbot_oldest_processing(tools_path, args, test_constraints)
else:
certbot_normal_processing(tools_path, test_constraints)
env = os.environ.copy()
env["PIP_CONSTRAINT"] = all_constraints
merge_requirements(tools_path, requirements, test_constraints, all_constraints)
if requirements: # This branch is executed during the oldest tests
# First step, install the transitive dependencies of oldest requirements
# in respect with oldest constraints.
pip_install_with_print('--requirement "{0}"'.format(requirements),
env=env)
# Second step, ensure that oldest requirements themselves are effectively
# installed using --force-reinstall, and avoid corner cases like the one described
# in https://github.com/certbot/certbot/issues/7014.
pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"'
.format(requirements))
print(' '.join(args))
pip_install_with_print(' '.join(args), env=env)
if __name__ == '__main__':
main(sys.argv[1:])
|
