#!/usr/bin/python3
import os
import sys
import yaml
import tarfile
import tempfile
import argparse
import subprocess
import multiprocessing
from components import CommonUtils, Package, EnvironmentHandler, TestHandler, PlatformFlavor, EnvFileUtils, MergeFolders
from components.CiConfigurationUtils import *
import shutil
import copy
import time
import datetime
import json

environmentManual='''
Environment variables:

Required variables:

KDECI_PACKAGE_PROJECT=<string>: gitlab project containing registry, where packages are downloaded
    from and uploaded to; required, inless --skip-dependencies-fetch is passed
KDECI_GITLAB_SERVER=<url>: gitlab server where KDECI_PACKAGE_PROJECT registry is 
    located; required, inless --skip-dependencies-fetch is passed
KDECI_CACHE_PATH=<path>: path to the cache folder for package downloads (required always)
KDECI_CC_CACHE=<path>: path to ccache directory; if not present, ccache is disabled
KDECI_GITLAB_TOKEN=<string>: gitlab token for uploading the package; if not present, 
    package uploading step is skipped

Optional variables:

KDECI_ONLY_BUILD=<bool>: forcefully enable --only-build option
KDECI_PUBLISH_TO_CACHE=<bool>: forcefully enable --publish-to-cache option
KDECI_SKIP_ECM_ANDROID_TOOLCHAIN=<bool>: don't activate ECM's android toolchain when
    ANDROID_HOME is found, the project is expected to provide its own toolchain via
    KDECI_EXTRA_CMAKE_ARGS environment variable
KDECI_REMOVE_INSTALL_FOLDERS_AFTER_BUILD=<bool> remove _install and _build folders when the
    build is finished. This option is useful when using seed-package-registry.py for
    batch-building the packages. Removing the build and install folders prevents cross-links
    between the libraries via absolute rpaths.
KDECI_COMPRESS_PACKAGES_ON_DOWNLOAD=<bool>: use compression when downloading dependency packages;
    the packages are stored as raw .tar files, so using gz compression for their downloads may
    improve performance when running this script locally
KDECI_WORKDIR_PATH=<path>: use external folder for _build, _staging and _install folders;
    might be necessary for builds on Windows to overcome path limitation (just set it to
    `C:\\_`)
KDECI_SHARED_INSTALL_PATH=<path>: shared install folder for the packages **and** the project
    itself. May be used for setting up build environment on the local developer's system.
KDECI_BUILD_TYPE=<string>: overrides build type used for the project; when present, the option
    from config.yml is ignored
KDECI_BUILD_TARGET=<string>: overrides build target for the project
    (default: 'force-build-target' config option or 'all' if option is empty)
KDECI_INSTALL_TARGET=<string>: overrides build target for the project
    (default: 'force-install-target' config option or 'install' is option is empty)
KDECI_EXTRA_CMAKE_ARGS=<string>: appends arguments to the CMake configuration call for the project
KDECI_POST_INSTALL_SCRIPTS_FILTER=<semicolon-separated-list-of-paths>: the list of scripts from
    .kde-ci.yml that should be run for this project. The post-install scripts not present in this
    list will not be executed
KDECI_PACKAGE_ALIASES_YAML=<dictionary in yaml format>: a dictionary of aliases used for fetching
    packages; may be used for switching a library to a debug or asan version. The valies from the
    environment variable will override the values from PackageAliases section of .kde-ci.yml.
KRITACI_DRY_RUN=<bool>: enables dry run, no actual build actions will be performed. Also enables --skip-dependencies-fetch.
KRITACI_PARALLEL_JOBS=<integer>: the number of parallel jobs to use during the build

where <bool> is either 'True' or 'False'
'''


# Capture our command line parameters
parser = argparse.ArgumentParser(description='Utility to perform a CI run for a KDE project.',
                                 formatter_class=argparse.RawDescriptionHelpFormatter,
                                 epilog=environmentManual)
parser.add_argument('--project', type=str, required=True)
parser.add_argument('--branch', type=str, required=True)
parser.add_argument('--platform', type=str, required=True)
parser.add_argument('--only-build', default=False, action='store_true')
parser.add_argument('--only-deps', default=False, action='store_true')
parser.add_argument('-e', '--env', type=str, help='The name of the environment initialization file; relative paths are resolved relative to the workdir')
parser.add_argument('--only-env', action='store_true', help='Fetch deps, generate environment file and exit')
parser.add_argument('--extra-cmake-args', type=str, nargs='+', action='append', required=False)
parser.add_argument('--skip-publishing', default=False, action='store_true')
parser.add_argument('--publish-to-cache', default=False, action='store_true')
parser.add_argument('--skip-dependencies-fetch', default=False, action='store_true')
parser.add_argument('--fail-on-leaked-stage-files', default=False, action='store_true')
parser.add_argument('-s','--skip-deps', nargs='+', help='A space-separated list of dependencies to skip fetching', required=False)
parser.add_argument('--dry-run', default=False, action='store_true')
arguments = parser.parse_args()
platform = PlatformFlavor.PlatformFlavor(arguments.platform)

if 'KRITACI_DRY_RUN' in os.environ:
    arguments.dry_run = (os.environ['KRITACI_DRY_RUN'].lower() in ['true', '1', 't', 'y', 'yes'])

if arguments.dry_run:
    arguments.skip_dependencies_fetch = True

if arguments.only_deps:
    if arguments.only_build:
        print ("WARNING: argument --only-build is ignored, since --only-deps is preset")
    if arguments.extra_cmake_args:
        print ("WARNING: argument --extra-cmake-args is ignored, since --only-deps is preset")
    if arguments.skip_publishing:
        print ("WARNING: argument --skip-publishing is ignored, since --only-deps is preset")
    if arguments.fail_on_leaked_stage_files:
        print ("WARNING: argument --fail-on-leaked-stage-files is ignored, since --only-deps is preset")
    if arguments.skip_dependencies_fetch:
        print ("ERROR: argument --skip-dependencies-fetch conflicts with --only-deps")

if arguments.only_env:
    if arguments.only_build:
        print ("WARNING: argument --only-build is ignored, since --only-env is preset")
    if arguments.extra_cmake_args:
        print ("WARNING: argument --extra-cmake-args is ignored, since --only-env is preset")
    if arguments.skip_publishing:
        print ("WARNING: argument --skip-publishing is ignored, since --only-env is preset")
    if arguments.fail_on_leaked_stage_files:
        print ("WARNING: argument --fail-on-leaked-stage-files is ignored, since --only-env is preset")

if 'KDECI_ONLY_BUILD' in os.environ:
    arguments.only_build = CommonUtils.boolFromEnv(os.environ['KDECI_ONLY_BUILD'])
    print ('## Overriding --only-build from environment: {}'.format(arguments.only_build))

if 'KDECI_PUBLISH_TO_CACHE' in os.environ:
    arguments.publish_to_cache = CommonUtils.boolFromEnv(os.environ['KDECI_PUBLISH_TO_CACHE'])
    print ('## Overriding --publish-to-cache from environment: {}'.format(arguments.publish_to_cache))

skipECMAndroidToolchain = False

if 'KDECI_SKIP_ECM_ANDROID_TOOLCHAIN' in os.environ:
    skipECMAndroidToolchain = CommonUtils.boolFromEnv(os.environ['KDECI_SKIP_ECM_ANDROID_TOOLCHAIN'])
    print ('## Disable ECM\'s Android toolchain file: {}'.format(skipECMAndroidToolchain))

removeInstallFoldersAfterBuild = False

if 'KDECI_REMOVE_INSTALL_FOLDERS_AFTER_BUILD' in os.environ:
    removeInstallFoldersAfterBuild = CommonUtils.boolFromEnv(os.environ['KDECI_REMOVE_INSTALL_FOLDERS_AFTER_BUILD'])
    print ('## Enable "remove install folder after build": {}'.format(removeInstallFoldersAfterBuild))

    if removeInstallFoldersAfterBuild and 'KDECI_SHARED_INSTALL_PATH' in os.environ:
        print ('## ERROR: conflicting environment variables are set: KDECI_SHARED_INSTALL_PATH and KDECI_REMOVE_INSTALL_FOLDERS_AFTER_BUILD')
        sys.exit(1)


# Determine the appropriate number of CPU cores we should use when running builds
cpuCount = int(multiprocessing.cpu_count())

if 'KRITACI_PARALLEL_JOBS' in os.environ:
    cpuCount = max(1, int(os.environ['KRITACI_PARALLEL_JOBS']))

####
# Load the project configuration
####

configuration = loadProjectConfiguration('', arguments.project)

# Determine whether we will be running tests
# This is only done if they're enabled for this project and we haven't been asked to just build the project
run_tests = configuration['Options']['run-tests'] and configuration['Options']['build-tests'] and not arguments.only_build
# We also Can't run tests in cross compilation environments, so don't run tests there either
if platform.os in ['Android']:
    run_tests = False

####
# Determine a number of paths we will need later on
####

# Our sources are assumed to be in the current working directory
sourcesPath = os.getcwd()

baseWorkDirectoryPath = os.getcwd()

if 'KDECI_WORKDIR_PATH' in os.environ:
    baseWorkDirectoryPath = os.path.join(os.environ['KDECI_WORKDIR_PATH'], arguments.project)
    potentialBuildFolder = os.path.join(baseWorkDirectoryPath, '_build')
    if os.path.isdir(potentialBuildFolder):
        print('## WARNING: workdir already contains _build folder: {}'.format(potentialBuildFolder))

# Determine where to locate the project build
buildPath = os.path.join( baseWorkDirectoryPath, '_build' )
if configuration['Options']['in-source-build']:
    buildPath = os.getcwd()

# Determine where to unpack the dependencies to
installPath = os.path.join( baseWorkDirectoryPath, '_install' )

if 'KDECI_SHARED_INSTALL_PATH' in os.environ:
    installPath = os.environ['KDECI_SHARED_INSTALL_PATH']

# Determine where we will stage the installation
installStagingPath = os.path.join( baseWorkDirectoryPath, '_staging' )

####
# Fetch project sources
####

# In order to resolve project dependencies later, we need to ensure all remote refs have been fetched
if 'CI_REPOSITORY_URL' in os.environ and os.path.exists('.git/shallow'):
    subprocess.check_call("git fetch --quiet --unshallow --tags {0} +refs/heads/*:refs/heads/*".format(os.environ['CI_REPOSITORY_URL']), shell=True)
elif 'CI_REPOSITORY_URL' in os.environ:
    subprocess.check_call("git fetch --quiet --tags {0} +refs/heads/*:refs/heads/*".format(os.environ['CI_REPOSITORY_URL']), shell=True)

####
# Prepare to resolve and fetch our project dependencies
####

# Bring our dependency resolver online...
dependencyResolver = prepareDependenciesResolver(platform)

defaultBuildType = 'Debug'

# Have we explicitly requested a release build?
# This should only ever be used by applications, and never libraries
# On Windows we do this because building our dependencies in Debug mode is just too hard and MSVC requires everything to be in either Debug or Release (you can't mix/match)
if configuration['Options']['release-build'] or sys.platform == 'win32':
    # Switch the Debug build for a Release one then!
    defaultBuildType = 'Release'


# Retrieve some key bits of information from our environment
# All of these come from environment variables due to needing to be set on either the CI Agent level or the group project level
# We remove them from the environment as they are sensitive (in the case of KDECI_GITLAB_TOKEN especially) and aren't needed by anything else
# Of these values, KDECI_GITLAB_TOKEN is optional as it is only used for writing changes back to the package archive
# The remainder are required for storing packages locally and fetching them
localCachePath = os.environ.pop('KDECI_CACHE_PATH')
gitlabToken    = os.environ.pop('KDECI_GITLAB_TOKEN', None)
buildType = os.environ.get('KDECI_BUILD_TYPE', defaultBuildType)

defaultBuildTarget = configuration['Options']['force-build-target']
if not defaultBuildTarget:
    defaultBuildTarget = 'all'
buildTarget = os.environ.get('KDECI_BUILD_TARGET', defaultBuildTarget)

defaultInstallTarget = configuration['Options']['force-install-target']
if not defaultInstallTarget:
    defaultInstallTarget = 'install'
installTarget = os.environ.get('KDECI_INSTALL_TARGET', defaultInstallTarget)

packageAliases = {}
if 'KDECI_PACKAGE_ALIASES_YAML' in os.environ:
    aliases = str(os.environ['KDECI_PACKAGE_ALIASES_YAML'])
    packageAliases = yaml.safe_load(aliases)
    print(f'type: {type(packageAliases)}')
    print(f'aliases: {packageAliases}')
    if not isinstance(packageAliases, dict):
        print("## Failed to parse KDECI_PACKAGE_ALIASES_YAML environment variable: {}".format(aliases))
        sys.exit(1)

if 'PackageAliases' in configuration:
    CommonUtils.recursiveUpdate(configuration['PackageAliases'], packageAliases)

def lazyAliasPackage(package):
    return package if not package in packageAliases else packageAliases[package]

# We can skip all communication with invent if we're not fetching dependencies, testing or publishing
if not (arguments.skip_dependencies_fetch and arguments.only_build):
    packageProject = os.environ.pop('KDECI_PACKAGE_PROJECT')
    gitlabInstance = os.environ.pop('KDECI_GITLAB_SERVER')

    # Bring the package archive up
    packageRegistry = Package.Registry( localCachePath, gitlabInstance, gitlabToken,
                                        packageProject, packageAliases=packageAliases)

    ####
    # Now resolve both build and runtime dependencies, then fetch the build dependencies!
    ####

    # Resolve the dependencies of this project
    projectPublicDependencies = dependencyResolver.resolve( configuration['Dependencies'], arguments.branch )
    # As well as the runtime dependencies
    projectRuntimeDependencies = dependencyResolver.resolve( configuration['RuntimeDependencies'], arguments.branch )
    # As well as the build-only dependencies
    projectPrivateDependencies = dependencyResolver.resolve( configuration['BuildDependencies'], arguments.branch )

    # Calculate the actual set of dependencies required for the build stage
    projectBuildDependencies = projectPublicDependencies | projectPrivateDependencies

dependenciesToUnpack = []

if not arguments.skip_dependencies_fetch:
    # skip retrieving dependencies which are already prepared
    dependenciesToRetrieve = \
        projectBuildDependencies \
        if arguments.skip_deps is None \
        else dict(item for item in projectBuildDependencies.items() if item[0] not in arguments.skip_deps)

    # Now we can retrieve the build time dependencies
    allDependencies = packageRegistry.retrieveDependencies( dependenciesToRetrieve, userFeedback=True )

    dependenciesToUnpack = \
        allDependencies \
        if arguments.skip_deps is None \
        else [item for item in allDependencies if item[1]['identifier'] not in arguments.skip_deps]

    # sort the dependencies in the correct order

    projectToDepMap = []

    for packageContents, packageMetadata, cacheStatus in dependenciesToUnpack:
        projectId = packageMetadata['identifier']
        deps = packageMetadata['dependencies']
        projectToDepMap.append((projectId, list(map(lazyAliasPackage, deps.keys()))))

    # for project, deps in projectToDepMap:
    #     print (f'{project}: {deps}')

    providedDeps = []
    installDepsOrder = []

    if not arguments.skip_deps is None:
        providedDeps.extend(arguments.skip_deps)

    while projectToDepMap:
        batchToInstall = []
        for project, deps in projectToDepMap:
            if all(dep in providedDeps for dep in deps):
                batchToInstall.append(project)

        if not batchToInstall:
            print ("## ERROR: infinite loop while sorting the deps")
            sys.exit(1)

        projectToDepMap = [(project, dep)
                        for project,dep in projectToDepMap
                        if not project in batchToInstall]

        installDepsOrder.extend(batchToInstall)
        providedDeps.extend(batchToInstall)

    # And then unpack them
    for project in installDepsOrder:
        packageContents, packageMetadata, cacheStatus = \
            next((c,m,s) for c,m,s in dependenciesToUnpack if m['identifier'] == project)

        print('## Unpacking dependency: {} ({})'.format(packageMetadata['identifier'], cacheStatus.name))

        # Open the archive file
        archive = tarfile.open( name=packageContents, mode='r' )

        with tempfile.TemporaryDirectory() as tmpDir:
            # Extract it's contents into a temporary directory
            archive.extractall( path=tmpDir )
            # Merge it into the install directory
            MergeFolders.merge_folders(tmpDir, installPath, move_files=True)

if arguments.only_deps:
    sys.exit(0)

####
# Perform final steps needed to get ready to start the build process
####

# Determine what our build environment should be comprised of....
buildEnvironment = EnvironmentHandler.generateFor( installPrefix=installPath )

# Apply any environment changes from our configuration
for key, value in configuration['Environment'].items():
    # Apply each key in turn
    buildEnvironment[ key ] = value

def ccacheSupportsVerbose():
    result = True

    print( "## Test if ccache supports verbose (\'-vvv\') option")
    commandToRun = 'ccache -vvvs'

    # Run the command
    try:
        print( "## RUNNING: " + commandToRun )
        subprocess.check_call( commandToRun, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True )
    except Exception:
        result = False

    print ('## ccache supports verbose output: {}'.format(result))
    return result

useCcacheForBuilds = configuration['Options']['use-ccache'] and 'KDECI_CC_CACHE' in buildEnvironment
ccacheVerboseArg = '-vvv' if useCcacheForBuilds and ccacheSupportsVerbose() else ''

if arguments.only_env or arguments.only_deps:
    try:
        subprocess.check_call( 'ccache --version', stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=sourcesPath, env=buildEnvironment )
    except:
        useCcacheForBuilds = False

# Do we need to get ccache ready to use?
if useCcacheForBuilds:
    # Setup the path used for the cache....
    buildEnvironment['CCACHE_DIR'] = os.path.join( buildEnvironment['KDECI_CC_CACHE'], arguments.project )
    buildEnvironment['CMAKE_C_COMPILER_LAUNCHER'] = 'ccache'
    buildEnvironment['CMAKE_CXX_COMPILER_LAUNCHER'] = 'ccache'
    # Ensure ccache is setup for use
    if configuration['Options']['ccache-large-cache']:
        subprocess.check_call( 'ccache -M 20G', stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=sourcesPath, env=buildEnvironment )
    else:
        subprocess.check_call( 'ccache -M 2G', stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=sourcesPath, env=buildEnvironment )
    # Reset cache-hit stats
    subprocess.check_call( 'ccache -z {}'.format(ccacheVerboseArg), stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=sourcesPath, env=buildEnvironment )
    # Dump intial stats for ccache (to estimate the size)
    subprocess.check_call( 'ccache -s {}'.format(ccacheVerboseArg), stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=sourcesPath, env=buildEnvironment )

# Python can create __pycache__ directories that will trigger DESTDIR leak detection so we disable this
buildEnvironment['PYTHONDONTWRITEBYTECODE'] = '1'

# Make sure the build directory exists
if not os.path.exists( buildPath ):
    os.makedirs( buildPath )

# Let the user know what we are about to do
print("## KDE Continuous Integration System")
print("##")
print("## Starting build for {0} on {1}".format(arguments.project, arguments.branch))
print("##")
print("## Installing project to {0}".format(installPath))
print("## Building project in {0}".format(buildPath))
print("## Staging project directory {0}".format(installStagingPath))
print("##")
print("## Project build target: {0}".format(buildTarget))
print("## Project install target: {0}".format(installTarget))
print("##")
print("## Project CI configuration as follows:")
for key, value in configuration['Options'].items():
    print("##    {0}: {1}".format(key, value))
print("##")
print("## Providing the following dependencies:")
if packageAliases:
    print("##    Using package aliases:")
    for package, alias in packageAliases.items():
        print("##    {0} -> {1}".format(package, alias))
    print("##")
for packageContents, packageMetadata, cacheStatus in dependenciesToUnpack:
    print("##    {0} - {1} ({2})".format(packageMetadata['identifier'], packageMetadata['branch'], packageMetadata['gitRevision']))
print("##")
if arguments.skip_deps:
    print("## Skipped the following deps (provided with --skip-deps option):")
    for depId in arguments.skip_deps:
        print("##     {}".format(depId))
print("##")
print("## Building with the following environment variables:")
for variable, contents in sorted(buildEnvironment.items()):
    print("##    {0}={1}".format(variable, contents))
print("##")

if not arguments.env is None or arguments.only_env:
    envFile = arguments.env

    if envFile is None:
        envFile = 'env'

    if not os.path.isabs(envFile):
        envFile = os.path.abspath(os.path.join(baseWorkDirectoryPath, envFile))

    print("## Generating env file: {}".format(envFile))

    EnvFileUtils.writeEnvFile(os.path.dirname(envFile), os.path.basename(envFile),
                              buildEnvironment)
    if arguments.only_env:
        print("## env file generated, exiting...")
        sys.exit(0)

if arguments.dry_run:
    print("WARNING: dry run action {0} considered built successfully".format(arguments.project))
    sys.exit(0)

print("## Starting build process...")

####
# Configure the project!
####

# Begin building up our configure command
# There are some parameters which are universal to all platforms..
cmakeCommand = [
    # Run CMake itself
    'cmake',
    # We want a Debug build to allow for good backtraces
    '-DCMAKE_BUILD_TYPE={}'.format(buildType),
    # We want tests to be built!
    '-DBUILD_TESTING={}'.format('ON' if configuration['Options']['build-tests'] else 'OFF'),
    # And we want to be installed in a given directory
    '-DCMAKE_INSTALL_PREFIX="' + installPath + '"',
    # Generate compile_commands.json for tooling
    '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
    # Plus any project specific parameters
    configuration['Options']['cmake-options']
]

useCoverageBuild = False

# Are we on Linux (but not Android)?
# If we are on Linux then we also need to check to see whether we are on a MUSL based system - as ASAN does not work there
if platform.os == 'Linux' and not os.path.exists('/lib/libc.musl-x86_64.so.1'):
    if configuration['Options']['run-gcovr']:
        # Then we also want Coverage by default
        cmakeCommand.append('-DBUILD_COVERAGE=ON')
        useCoverageBuild = True
    if configuration['Options']['use-asan']:
        # We also want to enable ASAN for our builds
        cmakeCommand.append("-DECM_ENABLE_SANITIZERS='address'")

# Are we on Windows?
if sys.platform == 'win32':
    if configuration['Options']['force-ninja-on-windows']:
        # We want a Ninja based build, rather than the default MSBuild
        cmakeCommand.append('-G "Ninja"')
    else:
        cmakeCommand.append('-G "MinGW Makefiles"')

if sys.platform == 'darwin':
    # use ninja if available
    try:
        subprocess.check_call('which ninja', stdout=sys.stdout, stderr=sys.stderr, shell=True, close_fds=True, cwd=buildPath, env=buildEnvironment)
        cmakeCommand.append('-G "Ninja"')
    except Exception:
        print("## Using Makefile Generator")

# Are we building for Android?
if platform.os == 'Android' and not skipECMAndroidToolchain:
    # We want CMake to cross compile appropriately
    cmakeCommand.append('-DKF5_HOST_TOOLING=/opt/nativetooling/lib/x86_64-linux-gnu/cmake/')
    cmakeCommand.append('-DKF6_HOST_TOOLING=/opt/nativetooling6/lib/x86_64-linux-gnu/cmake/')
    # CMake also needs additional guidance to find things
    # First though, apply a necessary transformation to allow CMake to parse the list we pass it
    ecmAdditionalRoots = buildEnvironment['CMAKE_PREFIX_PATH'].replace(':', ';')
    # Then give that list to CMake
    cmakeCommand.append('-DECM_ADDITIONAL_FIND_ROOT_PATH="' + ecmAdditionalRoots + '"')

    # Next we need to set the appropriate toolchain
    ecmToolchainLocations = [
        # First we should prefer toolchains shipped with this project
        os.path.join( sourcesPath, "toolchain/Android.cmake" ),
        # Next we  prefer a toolchain provided by the project dependencies
        os.path.join( installPath, "share/ECM/toolchain/Android.cmake" ),
        # As a final fallback use a toolchain baked into our SDK/Image
        "/opt/nativetooling/share/ECM/toolchain/Android.cmake",
    ]
    ecmToolchain = CommonUtils.firstPresentFile("/", ecmToolchainLocations)
    qt6Toolchain = "/home/user/android-arm-clang/lib/cmake/Qt6/qt.toolchain.cmake"

    # Now we make sure we found something (this should never happen)
    if ecmToolchain == "":
        print("## Unable to locate a suitable toolchain for this build!")
        print("## Aborting build - Android compilation requires use of an appropriate CMake toolchain to enable cross-compilation (please add a dependency on extra-cmake-modules)")
        sys.exit(1)

    # Determine whether we are in a Qt 6 or Qt 5 environment
    if os.path.exists( qt6Toolchain ):
        cmakeCommand.append("-DCMAKE_TOOLCHAIN_FILE=" + qt6Toolchain)
        cmakeCommand.append("-DQT_CHAINLOAD_TOOLCHAIN_FILE=" + ecmToolchain)
    else:
        cmakeCommand.append("-DCMAKE_TOOLCHAIN_FILE=" + ecmToolchain)

# Extra CMake arguments provided by the Gitlab template
if arguments.extra_cmake_args:
    # necessary since we cannot use the 'extend' action for the arguments due to requiring Python < 3.8
    flat_args = [item for sublist in arguments.extra_cmake_args for item in sublist]
    cmakeCommand.extend(flat_args)

if 'KDECI_EXTRA_CMAKE_ARGS' in os.environ:
    cmakeCommand.extend(os.environ['KDECI_EXTRA_CMAKE_ARGS'].split())

# Lucky last, we add the path to our sources
cmakeCommand.append( '"' + sourcesPath + '"' )

# Now glue it all together!
commandToRun = ' '.join( cmakeCommand )

# Run the command
try:
    print( "## RUNNING: " + commandToRun )
    subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=buildPath, env=buildEnvironment )
except Exception:
    print("## Failed to configure the project")
    sys.exit(1)

####
# Compile the project!!
####

beforeInstallTimestamp = datetime.datetime.now().timestamp()

makeCommand = "cmake --build . --parallel {cpuCount} --target {customTarget}"

# Finalise the command we will be running
commandToRun = makeCommand.format( cpuCount=cpuCount, customTarget = buildTarget )

# Compile the project
try:
    print( "## RUNNING: " + commandToRun )
    subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=buildPath, env=buildEnvironment )
except Exception:
    print("## Failed to build the project")
    sys.exit(1)

####
# Run tests before installation if needed
####

if run_tests and configuration['Options']['test-before-installing']:
    # Run the tests!
    print("## RUNNING PROJECT TESTS")
    testResult = TestHandler.run( configuration, sourcesPath, buildPath, installPath, buildEnvironment )
    if not testResult and platform.matches(configuration['Options']['require-passing-tests-on']):
        print("## Tests failed")
        sys.exit(1)

####
# Install the project...
####

# Set the appropriate environment variables to ensure we can capture make install's output later on
buildEnvironment['DESTDIR'] = installStagingPath
buildEnvironment['INSTALL_ROOT'] = installStagingPath
buildEnvironment['KDECI_REAL_PREFIX_PATH'] = installPath

# Now determine the path we should be archiving
# Because we could potentially be running on Windows we have to ensure our second path has been converted to a suitable form
# This conversion is necessary as os.path.join can't handle the presence of drive letters in paths other than the first argument
pathToArchive = os.path.join( installStagingPath, CommonUtils.makePathRelative(installPath) )

makeCommand = "cmake --build . --target {customTarget}"
commandToRun = makeCommand.format(customTarget = installTarget)

# Install the project
try:
    print("## RUNNING: " + commandToRun)
    subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=buildPath, env=buildEnvironment )
except Exception:
    print("## Failed to install the project")
    sys.exit(1)

####
# Run post-install scripts
####

scriptsAllowed = None

if 'KDECI_POST_INSTALL_SCRIPTS_FILTER' in buildEnvironment:
    scriptsAllowed = buildEnvironment['KDECI_POST_INSTALL_SCRIPTS_FILTER'].split(';')

def iterateThroughPostInstallScripts():
    if isinstance(configuration['PostInstallScripts'], dict):
        for name, script in configuration['PostInstallScripts'].items():
            yield (name, script)
    elif isinstance(configuration['PostInstallScripts'], list):
        for item in configuration['PostInstallScripts']:
            if platform.matches(item['on']):
                for name, script in item['require'].items():
                    yield (name, script)

for name, script in iterateThroughPostInstallScripts():

    if not scriptsAllowed is None and not name in scriptsAllowed:
        print('## Skipping script \"{}\" due to the filter active'.format(name))
        continue

    scriptEnvironment = copy.deepcopy(buildEnvironment)
    scriptEnvironment['KDECI_CACHE_PATH'] = localCachePath
    scriptEnvironment['KDECI_BUILD_TYPE'] = buildType
    scriptEnvironment['KDECI_INTERNAL_USE_CCACHE'] = str(useCcacheForBuilds)
    scriptEnvironment['KDECI_SOURCES_DIR'] = sourcesPath
    scriptEnvironment['KDECI_TARGET_PLATFORM'] = str(platform)

    scriptEnvironment = EnvironmentHandler.addEnvironmentPrefix(pathToArchive, scriptEnvironment)

    if script.endswith('.py') and not (script.startswith('python ') or script.startswith('python3')):
        commandToRun = sys.executable + " " + os.path.join(sourcesPath, script)
    else:
        commandToRun = script if os.path.isabs(script) else os.path.join(sourcesPath, script)

    # Run post-install scripts
    try:
        print('## RUNNING POST-INSTALL SCRIPT \"{}\": {}'.format(name, commandToRun))
        subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=baseWorkDirectoryPath, env=scriptEnvironment )
    except Exception:
        print("## Failed to run post-install script the project")
        sys.exit(1)

leakedFiles = []

for root, dirs, files in os.walk(installPath):
    for name in files:
        filePath = os.path.join(root, name)
        fileTimeStamp = os.path.getctime(filePath)
        #print("file: {} timestamp: {}".format(filePath, fileTimeStamp))
        if fileTimeStamp > beforeInstallTimestamp:
            leakedFiles.append(filePath)

if leakedFiles:
    print("## ERROR: some files seem to have been installed bypassing the _staging directory ($DESTDIR environment variable)!")
    print("##  timestamp: {}".format(time.asctime(time.localtime(beforeInstallTimestamp))))
    for filePath in leakedFiles:
        print("##  leaked file: {}, {}".format(filePath, time.asctime(time.localtime(os.path.getctime(filePath)))))
    if arguments.fail_on_leaked_stage_files:
        print('## Exiting... (\'--fail-on-leaked-stage-files\' is set)')
        sys.exit(1)
    else:
        print('## Ignoring... (set \'--fail-on-leaked-stage-files\' to fail on leaked files)')

if configuration['Options']['pkg-config-sanity-check'] != 'none':
    commandToRun = '{} -u {} --prefix {} --destdir {} {}'.format(
        sys.executable,
        os.path.join(CommonUtils.scriptsBaseDirectory(), 'sanity-check-pkg-config-files.py'),
        installPath,
        installStagingPath,
        '-f' if configuration['Options']['pkg-config-sanity-check'] == 'error' else '')

    # Run pkg-config sanity checks
    try:
        print('## RUNNING PKG-CONFIG SANITY-CHECKS: {}'.format(commandToRun))
        subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=os.getcwd(), env=os.environ )
    except Exception:
        print("## Failed to run pkg-config sanity-check script")
        sys.exit(1)

if sys.platform == 'darwin':
    try:
        if configuration['Options']['macos-rpath-fix']:
            commandToRun = '{} {} --debug --prefix {} --destdir {}'.format(
                "zsh",
                os.path.join(CommonUtils.scriptsBaseDirectory(), 'macos-fix-rpath.sh'),
                installPath,
                installStagingPath)
            
            # Run macos-fix-rpath 
            try:
                print('## RUNNING MACOS-RPATH-FIX: {}'.format(commandToRun))
                subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=os.getcwd(), env=os.environ )
            except Exception:
                print("## Failed to run macOS rpath-fix script")
                sys.exit(1)
    except Exception:
        pass


# Cleanup the capture variables to ensure they don't interfere with later runs
del buildEnvironment['DESTDIR']
del buildEnvironment['INSTALL_ROOT']
del buildEnvironment['KDECI_REAL_PREFIX_PATH']

# Dump ccache stats if applicable
if useCcacheForBuilds:
    # Dump cache-hit stats
    subprocess.check_call( 'ccache -s {}'.format(ccacheVerboseArg), stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=sourcesPath, env=buildEnvironment )

####
# Capture the installation if needed and deploy the staged install to the final install directory
####

# We want to capture the tree as it is inside the install directory and don't want any trailing slashes in the archive as this isn't standards compliant
# Therefore we list everything in the install directory and add each of those to the archive, rather than adding the whole install directory
filesToInclude = os.listdir( pathToArchive )

# Copy the files into the installation directory
# This is so later tests can rely on the project having been installed
# While we ran 'make install' just before this didn't install it as we diverted the installation to allow us to cleanly capture it
for filename in filesToInclude:
    fullPath = os.path.join(pathToArchive, filename)
    print("Copying {} -> {}".format(fullPath, os.path.join(installPath, filename)))
    if os.path.isdir(fullPath):
        dstFullPath = os.path.join(installPath, filename)
        MergeFolders.merge_folders(fullPath, dstFullPath)
    else:
        shutil.copy2(fullPath, os.path.join(installPath, filename))

# Are we supposed to be publishing this particular package to the archive?
if (gitlabToken is not None or arguments.publish_to_cache) and not arguments.skip_publishing:
    # Create a temporary file, then open the file as a tar archive for writing
    # We don't want it to be deleted as storePackage will move the archive into it's cache
    archiveFile = tempfile.NamedTemporaryFile(delete=False)
    archive = tarfile.open( fileobj=archiveFile, mode='w' )

    # Add all the files which need to be in the archive into the archive
    for filename in filesToInclude:
        fullPath = os.path.join(pathToArchive, filename)
        archive.add( fullPath, arcname=filename, recursive=True )

    # Close the archive, which will write it out to disk, finishing what we need to do here
    archive.close()
    archiveFile.close()

    # With the archive being generated, we can now prepare some metadata...
    packageMetadata = {
        'dependencies': projectPublicDependencies,
        'runtime-dependencies': projectRuntimeDependencies
    }

    if gitlabToken is not None:
        # Grab the Git revision (SHA-1 hash) we are building
        # This is always present in Gitlab CI builds
        gitRevision = os.environ['CI_COMMIT_SHA']

        print('## Publishing package: {} branch: {} sha1: {}'.format(arguments.project, arguments.branch, gitRevision))
        print('##    metadata: {}'.format(packageMetadata))

        # Publish our package to the registry
        packageRegistry.upload(archiveFile.name, arguments.project, arguments.branch, gitRevision, packageMetadata)

    if arguments.publish_to_cache:
        normalisedBranch = packageRegistry._normaliseBranchName(arguments.branch)

        packageNameFile = '{}-{}.tar'.format(arguments.project, normalisedBranch)
        packageMetadataFile = '{}-{}.json'.format(arguments.project, normalisedBranch)

        packageTimestamp = int( os.path.getmtime(archiveFile.name) )
        versionForGitlab = "{0}-{1}".format(normalisedBranch, packageTimestamp)

        gitRevision = os.environ.get('CI_COMMIT_SHA', 'unknown')
        fullPackageMetadata = packageRegistry.generateMetadata(archiveFile.name, arguments.project, arguments.branch, gitRevision, packageMetadata)

        print('## Copying package to chache: {} branch: {}'.format(arguments.project, arguments.branch))
        print('##    metadata: {}'.format(fullPackageMetadata))
        print('##    package file: {}'.format(packageNameFile))
        print('##    location: {}'.format(localCachePath))

        shutil.copy2(archiveFile.name, os.path.join(localCachePath, packageNameFile))
        with open(os.path.join(localCachePath, packageMetadataFile), 'w') as f:
            f.write(json.dumps(fullPackageMetadata, indent = 4))

    # Cleanup the temporary archive file as it is no longer needed
    os.remove( archiveFile.name )

if removeInstallFoldersAfterBuild:
    print('## Removing install folder: {}'.format(installPath))
    shutil.rmtree(installPath)
    print('## Removing staging folder: {}'.format(installStagingPath))
    shutil.rmtree(installStagingPath)

# If this is a build only run then bail here
if arguments.only_build:
    sys.exit(0)

####
# Retrieve runtime dependencies if they are needed, and rebuild our environment
####

# Now we can retrieve the build time dependencies
dependenciesToUnpack = packageRegistry.retrieveDependencies( projectRuntimeDependencies, runtime=True, userFeedback=True )
# And then unpack them
for packageContents, packageMetadata, cacheStatus in dependenciesToUnpack:
    # Open the archive file
    archive = tarfile.open( name=packageContents, mode='r' )
    # Extract it's contents into the install directory
    archive.extractall( path=installPath )

# Regenerate our environment in case the newly installed software uses directories previously not used
buildEnvironment = EnvironmentHandler.generateFor( installPrefix=installPath )

# Apply any environment changes from our configuration as well
for key, value in configuration['Environment'].items():
    # Apply each key in turn
    buildEnvironment[ key ] = value

####
# Run tests if we didn't do that already
####

if run_tests and not configuration['Options']['test-before-installing']:
    # Run the tests!
    print("## RUNNING PROJECT TESTS")
    testResult = TestHandler.run( configuration, sourcesPath, buildPath, installPath, buildEnvironment )
    requirePassingTestsOn = configuration['Options']['require-passing-tests-on']
    if not testResult and platform.matches(requirePassingTestsOn):
        print("## Tests failed")
        sys.exit(1)

####
# Extract test coverage results for processing by Gitlab 
####

# If we aren't running on Linux then we skip this, as we consider that to be the canonical platform for code coverage...
# Additionally, as coverage information requires tests to have been run, skip extracting coverage information if tests have been disabled
if run_tests and useCoverageBuild:
    # Determine the command we need to run
    # We ask GCovr to exclude the build directory by default as we don't want generated artifacts (like moc files) getting included as well
    # Sometimes projects will want to customise things slightly so we provide for that as well
    commandToRun = 'gcovr --txt --xml "CoberturaLcovResults.xml" --exclude "_build/.*" --root "{sources}" {otherArguments}'
    commandToRun = commandToRun.format( sources=sourcesPath, otherArguments=configuration['Options']['gcovr-arguments'] )

    # Now run it!
    # If gcovr bails we ignore it, as failures to extract lcov results shouldn't cause builds to fail.
    try:
        print("## RUNNING: " + commandToRun)
        subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment )
    except Exception:
        pass

####
# Run complete!
####

print("## CI Run Completed Successfully!")
sys.exit(0)