hit database less to improve api performance (#515)

4 files changed, 72 insertions, 26 deletions

diff --git a/scripts/development/__init__.py b/scripts/development/__init__.py
new file mode 100644
index 0000000000..16439a892f
--- /dev/null
+++ b/scripts/development/__init__.py
@@ -0,0 +1,34 @@
+import sys
+from typing import Tuple
+
+from requests import post
+
+REGISTER_URL = 'http://127.0.0.1:8000/api/v1/apps'
+RELEASE_URL = 'http://127.0.0.1:8000/api/v1/apps/releases'
+ADMIN = ('admin', 'admin')
+
+
+def handle_response(response) -> None:
+    if response.status_code > 299:
+        msg = 'Request to url %s failed with status code %d'
+        print(msg % (response.url, response.status_code), file=sys.stderr)
+        print(response.text, file=sys.stderr)
+
+
+def import_app(certificate: str, signature: str, auth=Tuple[str, str]) -> None:
+    response = post(REGISTER_URL, auth=auth, json={
+        'signature': signature,
+        'certificate': certificate
+    })
+    handle_response(response)
+
+
+def import_release(url: str, signature: str, nightly: bool,
+                   auth=Tuple[str, str]) -> None:
+    print('Downloading app from %s' % url)
+    response = post(RELEASE_URL, auth=auth, json={
+        'download': url,
+        'signature': signature,
+        'nightly': nightly
+    })
+    handle_response(response)
diff --git a/scripts/development/proddata.py b/scripts/development/proddata.py
new file mode 100644
index 0000000000..90459ad515
--- /dev/null
+++ b/scripts/development/proddata.py
@@ -0,0 +1,24 @@
+"""
+Queries the live API and import all releases locally for a Nextcloud version
+"""
+import sys
+
+from requests import get
+
+from . import import_app, ADMIN, import_release
+
+APPS_URL = 'https://apps.nextcloud.com/api/v1/platform/%s/apps.json'
+
+
+def main():
+    version = sys.argv[1]
+    apps = get(APPS_URL % version).json()
+    for app in apps:
+        import_app(app['certificate'], 'signature', ADMIN)
+        for release in app['releases']:
+            import_release(release['download'], release['signature'],
+                           release['isNightly'], ADMIN)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/development/settings/base.py b/scripts/development/settings/base.py
index 38fe7961a1..cfd5401e52 100644
--- a/scripts/development/settings/base.py
+++ b/scripts/development/settings/base.py
@@ -21,3 +21,9 @@ LOGGING['handlers']['console'] = {
     'class': 'logging.StreamHandler',
 }
 LOGGING['loggers']['django']['handlers'] += ['console']
+
+# make it possible to run debug toolbar for api
+CSP_EXCLUDE_URL_PREFIXES = ('/api/v1',)
+
+REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['app_upload'] = '10000/day'
+REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['app_register'] = '10000/day'
diff --git a/scripts/development/testdata.py b/scripts/development/testdata.py
index 89b8fbdfaa..f8493c97c4 100644
--- a/scripts/development/testdata.py
+++ b/scripts/development/testdata.py
@@ -1,4 +1,4 @@
-import requests
+from . import import_app, import_release, ADMIN
 
 twofactor_cert = """
 -----BEGIN CERTIFICATE-----
@@ -101,31 +101,13 @@ apps = [{
     }]
 }]
 
-admin = ('admin', 'admin')
 
+def main():
+    for app in apps:
+        import_app(app['certificate'], 'signature', ADMIN)
+        for release in app['releases']:
+            import_release(release['url'], 'signature', False, ADMIN)
 
-def handle_response(response):
-    if response.status_code > 299:
-        msg = 'Request to url %s failed with status code %d'
-        print(msg % (response.url, response.status_code))
-        print(response.text)
 
-
-for app in apps:
-    response = requests.post('http://127.0.0.1:8000/api/v1/apps',
-                             auth=admin,
-                             json={
-                                 'signature': 'signature',
-                                 'certificate': app['certificate']
-                             })
-    handle_response(response)
-    for release in app['releases']:
-        print('Downloading app from %s' % release['url'])
-        response = requests.post('http://127.0.0.1:8000/api/v1/apps/releases',
-                                 auth=admin,
-                                 json={
-                                     'download': release['url'],
-                                     'signature': 'signature',
-                                     'nightly': False
-                                 })
-        handle_response(response)
+if __name__ == '__main__':
+    main()