run pipeline steps in parallel with dependent steps

refactor pipelines

1 files changed, 98 insertions, 62 deletions

diff --git a/.drone.star b/.drone.star
index 28c4e45d9..8c00b95ba 100644
--- a/.drone.star
+++ b/.drone.star
@@ -65,7 +65,7 @@ def main(ctx):
             trigger = build_trigger,
             depends_on = [],
         ),
-        build_and_test_client(
+        ctest_pipeline(
             ctx,
             "clang",
             "clang++",
@@ -73,11 +73,11 @@ def main(ctx):
             "Ninja",
             trigger = build_trigger,
         ),
-        gui_tests(ctx, trigger = build_trigger, version = "latest"),
+        gui_test_pipeline(ctx, trigger = build_trigger, version = "latest"),
     ]
     cron_pipelines = [
         # Build client
-        build_and_test_client(
+        ctest_pipeline(
             ctx,
             "gcc",
             "g++",
@@ -85,7 +85,7 @@ def main(ctx):
             "Unix Makefiles",
             trigger = cron_trigger,
         ),
-        build_and_test_client(
+        ctest_pipeline(
             ctx,
             "clang",
             "clang++",
@@ -93,7 +93,7 @@ def main(ctx):
             "Ninja",
             trigger = cron_trigger,
         ),
-        gui_tests(ctx, trigger = cron_trigger),
+        gui_test_pipeline(ctx, trigger = cron_trigger),
         notification(
             name = "build",
             trigger = cron_trigger,
@@ -167,7 +167,7 @@ def check_starlark(ctx, trigger = {}, depends_on = []):
         "trigger": trigger,
     }
 
-def build_and_test_client(ctx, c_compiler, cxx_compiler, build_type, generator, trigger = {}, depends_on = []):
+def ctest_pipeline(ctx, c_compiler, cxx_compiler, build_type, generator, trigger = {}, depends_on = []):
     build_command = "ninja" if generator == "Ninja" else "make"
     pipeline_name = c_compiler + "-" + build_type.lower() + "-" + build_command
     build_dir = "build-" + pipeline_name
@@ -179,45 +179,32 @@ def build_and_test_client(ctx, c_compiler, cxx_compiler, build_type, generator,
             "os": "linux",
             "arch": "amd64",
         },
-        "steps": skipIfUnchanged(ctx, "unit-tests") + [
-                     {
-                         "name": "submodules",
-                         "image": DOCKER_GIT,
-                         "commands": [
-                             "git submodule update --init --recursive",
-                         ],
-                     },
-                 ] +
-                 build_client(ctx, c_compiler, cxx_compiler, build_type, generator, build_command, build_dir) +
-                 [
-                     {
-                         "name": "ctest",
-                         "image": OC_CI_CLIENT,
-                         "environment": {
-                             "LC_ALL": "C.UTF-8",
-                         },
-                         "commands": [
-                             'cd "' + build_dir + '"',
-                             "useradd -m -s /bin/bash tester",
-                             "chown -R tester:tester .",
-                             "su-exec tester ctest --output-on-failure -LE nodrone",
-                         ],
-                     },
-                 ],
+        "steps": skipIfUnchanged(ctx, "unit-tests") +
+                 gitSubModules() +
+                 build_client(c_compiler, cxx_compiler, build_type, generator, build_command, build_dir) +
+                 c_tests(build_dir, [build_command]),
         "trigger": trigger,
         "depends_on": depends_on,
     }
 
-def gui_tests(ctx, trigger = {}, depends_on = [], filterTags = [], version = "daily-master-qa"):
+def gui_test_pipeline(ctx, trigger = {}, depends_on = [], filterTags = [], version = "daily-master-qa"):
     pipeline_name = "GUI-tests"
     build_dir = "build-" + pipeline_name
-    squish_parameters = "--reportgen html,%s --envvar QT_LOGGING_RULES=sync.httplogger=true;gui.socketapi=false --tags ~@skip" % GUI_TEST_REPORT_DIR
+    squish_parameters = "--reportgen html,%s --envvar QT_LOGGING_RULES=sync.httplogger=true;gui.socketapi=false --tags @test" % GUI_TEST_REPORT_DIR
 
     if (len(filterTags) > 0):
         for tags in filterTags:
             squish_parameters += " --tags " + tags
             pipeline_name += "-" + tags
 
+    build_config = {
+        "c_compiler": "gcc",
+        "cxx_compiler": "g++",
+        "build_type": "Debug",
+        "generator": "Ninja",
+        "build_command": "ninja",
+    }
+
     return {
         "kind": "pipeline",
         "name": pipeline_name,
@@ -225,38 +212,22 @@ def gui_tests(ctx, trigger = {}, depends_on = [], filterTags = [], version = "da
             "os": "linux",
             "arch": "amd64",
         },
-        "steps": skipIfUnchanged(ctx, "gui-tests") + [
-                     {
-                         "name": "submodules",
-                         "image": DOCKER_GIT,
-                         "commands": [
-                             "git submodule update --init --recursive",
-                         ],
-                     },
-                 ] +
+        "steps": skipIfUnchanged(ctx, "gui-tests") +
+                 gitSubModules() +
                  installCore(version) +
                  setupServerAndApp(2) +
                  fixPermissions() +
                  owncloudLog() +
                  setGuiTestReportDir() +
-                 build_client(ctx, "gcc", "g++", "Debug", "Ninja", "ninja", build_dir) +
-                 [
-                     {
-                         "name": "GUItests",
-                         "image": OC_CI_SQUISH,
-                         "environment": {
-                             "LICENSEKEY": from_secret("squish_license_server"),
-                             "GUI_TEST_REPORT_DIR": GUI_TEST_REPORT_DIR,
-                             "CLIENT_REPO": "/drone/src/",
-                             "MIDDLEWARE_URL": "http://testmiddleware:3000/",
-                             "BACKEND_HOST": "http://owncloud/",
-                             "SECURE_BACKEND_HOST": "https://owncloud/",
-                             "SERVER_INI": "/drone/src/test/gui/drone/server.ini",
-                             "SQUISH_PARAMETERS": squish_parameters,
-                             "STACKTRACE_FILE": STACKTRACE_FILE,
-                         },
-                     },
-                 ] +
+                 build_client(
+                     build_config["c_compiler"],
+                     build_config["cxx_compiler"],
+                     build_config["build_type"],
+                     build_config["generator"],
+                     build_config["build_command"],
+                     build_dir,
+                 ) +
+                 gui_tests(squish_parameters, [build_config["build_command"]]) +
                  # GUI test result has been disabled for now, as we squish can not produce the result in both html and json format.
                  # Disabled untill the feature to generate json result is implemented in squish, or some other method to reuse the log parser is implemented.
                  #  showGuiTestResult() +
@@ -276,10 +247,10 @@ def gui_tests(ctx, trigger = {}, depends_on = [], filterTags = [], version = "da
         ],
     }
 
-def build_client(ctx, c_compiler, cxx_compiler, build_type, generator, build_command, build_dir):
+def build_client(c_compiler, cxx_compiler, build_type, generator, build_command, build_dir):
     return [
         {
-            "name": "cmake",
+            "name": "generate",
             "image": OC_CI_CLIENT,
             "environment": {
                 "LC_ALL": "C.UTF-8",
@@ -300,6 +271,45 @@ def build_client(ctx, c_compiler, cxx_compiler, build_type, generator, build_com
                 'cd "' + build_dir + '"',
                 build_command + " -j4",
             ],
+            "depends_on": ["generate"],
+        },
+    ]
+
+def c_tests(build_dir, depends_on = []):
+    return [
+        {
+            "name": "ctest",
+            "image": OC_CI_CLIENT,
+            "environment": {
+                "LC_ALL": "C.UTF-8",
+            },
+            "commands": [
+                'cd "' + build_dir + '"',
+                "useradd -m -s /bin/bash tester",
+                "chown -R tester:tester .",
+                "su-exec tester ctest --output-on-failure -LE nodrone",
+            ],
+            "depends_on": depends_on,
+        },
+    ]
+
+def gui_tests(squish_parameters, depends_on = []):
+    return [
+        {
+            "name": "GUItests",
+            "image": OC_CI_SQUISH,
+            "environment": {
+                "LICENSEKEY": from_secret("squish_license_server"),
+                "GUI_TEST_REPORT_DIR": GUI_TEST_REPORT_DIR,
+                "CLIENT_REPO": "/drone/src/",
+                "MIDDLEWARE_URL": "http://testmiddleware:3000/",
+                "BACKEND_HOST": "http://owncloud/",
+                "SECURE_BACKEND_HOST": "https://owncloud/",
+                "SERVER_INI": "/drone/src/test/gui/drone/server.ini",
+                "SQUISH_PARAMETERS": squish_parameters,
+                "STACKTRACE_FILE": STACKTRACE_FILE,
+            },
+            "depends_on": depends_on,
         },
     ]
 
@@ -501,6 +511,7 @@ def setupServerAndApp(logLevel):
             "php occ config:system:set skeletondirectory --value=/var/www/owncloud/server/apps/testing/data/tinySkeleton",
             "php occ config:system:set sharing.federation.allowHttpFallback --value=true --type=bool",
         ],
+        "depends_on": stepDependsOn(installCore("")),
     }]
 
 def owncloudService():
@@ -547,6 +558,7 @@ def owncloudLog():
         "commands": [
             "tail -f /drone/src/server/data/owncloud.log",
         ],
+        "depends_on": stepDependsOn(installCore("")),
     }]
 
 def fixPermissions():
@@ -557,6 +569,16 @@ def fixPermissions():
             "cd /drone/src/server",
             "chown www-data * -R",
         ],
+        "depends_on": stepDependsOn(setupServerAndApp("")),
+    }]
+
+def gitSubModules():
+    return [{
+        "name": "submodules",
+        "image": DOCKER_GIT,
+        "commands": [
+            "git submodule update --init --recursive",
+        ],
     }]
 
 def setGuiTestReportDir():
@@ -607,6 +629,7 @@ def uploadGuiTestLogs():
                 "from_secret": "cache_public_s3_secret_key",
             },
         },
+        "depends_on": stepDependsOn(gui_tests("")),
         "when": {
             "status": [
                 "failure",
@@ -630,6 +653,7 @@ def buildGithubComment(suite):
                 "from_secret": "cache_public_s3_bucket",
             },
         },
+        "depends_on": stepDependsOn(uploadGuiTestLogs()),
         "when": {
             "status": [
                 "failure",
@@ -656,6 +680,7 @@ def githubComment(alternateSuiteName):
         "commands": [
             "if [ -s %s/comments.file ]; then echo '%s' | cat - %s/comments.file > temp && mv temp %s/comments.file && /bin/drone-github-comment; fi" % (GUI_TEST_REPORT_DIR, prefix, GUI_TEST_REPORT_DIR, GUI_TEST_REPORT_DIR),
         ],
+        "depends_on": stepDependsOn(buildGithubComment("")),
         "when": {
             "status": [
                 "failure",
@@ -729,3 +754,14 @@ def skipIfUnchanged(ctx, type):
             ],
         },
     }]
+
+def stepDependsOn(steps):
+    depends_on_steps = []
+
+    if type(steps) == dict:
+        steps = [steps]
+
+    for step in steps:
+        depends_on_steps.append(step["name"])
+
+    return depends_on_steps