571 lines
25 KiB
Python
Executable file
571 lines
25 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
import os
|
|
import re
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import argparse
|
|
import subprocess
|
|
import multiprocessing
|
|
import glob
|
|
import lzma
|
|
import shutil
|
|
import yaml
|
|
from packaging.version import Version as PackagingVersion
|
|
from components import CommonUtils, Dependencies, Package, EnvironmentHandler, TestHandler, PlatformFlavor
|
|
|
|
# Capture our command line parameters
|
|
parser = argparse.ArgumentParser(description='Utility to perform a CI run for a KDE project.')
|
|
parser.add_argument('--project', type=str, required=True)
|
|
parser.add_argument('--branch', type=str, required=True)
|
|
parser.add_argument('--platform', type=str, required=True)
|
|
parser.add_argument('--only-build', default=False, action='store_true')
|
|
parser.add_argument('--only-setup-environment', default=False, action='store_true')
|
|
parser.add_argument('--extra-cmake-args', type=str, nargs='+', action='append', required=False)
|
|
parser.add_argument('--skip-publishing', default=False, action='store_true')
|
|
arguments = parser.parse_args()
|
|
platform = PlatformFlavor.PlatformFlavor(arguments.platform)
|
|
|
|
####
|
|
# Load the project configuration
|
|
####
|
|
|
|
# This consists of:
|
|
# 0) Global configuration
|
|
configuration = yaml.safe_load( open(os.path.join(CommonUtils.scriptsBaseDirectory(), 'config', 'global.yml')) )
|
|
|
|
# 1) Project/branch specific configuration contained within the repository
|
|
if os.path.exists('.kde-ci.yml'):
|
|
localConfig = yaml.safe_load( open('.kde-ci.yml') )
|
|
CommonUtils.recursiveUpdate( configuration, localConfig )
|
|
|
|
# 2) Global overrides applied to the project configuration
|
|
projectConfigFile = os.path.join(CommonUtils.scriptsBaseDirectory(), 'config', arguments.project + '.yml')
|
|
if os.path.exists( projectConfigFile ):
|
|
projectConfig = yaml.safe_load( open(projectConfigFile) )
|
|
CommonUtils.recursiveUpdate( configuration, projectConfig )
|
|
|
|
# If the global flag to disable running tests is set, ensure that the per platform setting matches that request
|
|
if not configuration['Options']['run-tests']:
|
|
configuration['Options']['run-tests-on'] = []
|
|
|
|
# Determine whether we will be running tests
|
|
# This is only done if they're enabled for this project and we haven't been asked to just build the project
|
|
shouldRunTests = platform.matches(configuration['Options']['run-tests-on']) and not arguments.only_build
|
|
|
|
# Determine whether test failures should be considered fatal
|
|
requirePassingTests = platform.matches(configuration['Options']['require-passing-tests-on'])
|
|
allowFailingTests = platform.matches(configuration['Options']['allow-failing-tests-on'])
|
|
if requirePassingTests and allowFailingTests:
|
|
print("## ERROR: Both require-passing-tests-on and allow-failing-tests-on are set. This is not a valid configuration.")
|
|
print("## Set only allow-failing-tests-on in order to migrate the configuration.")
|
|
sys.exit(1)
|
|
|
|
if configuration['Options']['per-test-timeout'] > 1000:
|
|
print("## ERROR: per test timeout is too large")
|
|
print("## The per test time out is meant to prevent accidental")
|
|
print("## infinite loops or other that occupies a builder wrongly")
|
|
print("## for a long time, so individial tests should not be ")
|
|
print("## too long running ")
|
|
print("## This can be worked around by splitting long running ")
|
|
print("## tests into multiple individual ones ")
|
|
sys.exit(1)
|
|
|
|
####
|
|
# Determine a number of paths we will need later on
|
|
####
|
|
|
|
# Our sources are assumed to be in the current working directory
|
|
sourcesPath = os.getcwd()
|
|
|
|
# Determine where to locate the project build
|
|
buildPath = os.path.join( os.getcwd(), '_build' )
|
|
if configuration['Options']['in-source-build']:
|
|
buildPath = os.getcwd()
|
|
|
|
# Determine where to unpack the dependencies to
|
|
installPath = os.path.join( os.getcwd(), '_install' )
|
|
|
|
# Determine where we will stage the installation
|
|
installStagingPath = os.path.join( os.getcwd(), '_staging' )
|
|
|
|
# Public artifact path to publish gcov data
|
|
publicArtifactPath = os.path.join( os.getcwd(), 'public' )
|
|
|
|
####
|
|
# Fetch project sources
|
|
####
|
|
|
|
# In order to resolve project dependencies later, we need to ensure all remote refs have been fetched
|
|
if 'CI_REPOSITORY_URL' in os.environ and os.path.exists('.git/shallow'):
|
|
subprocess.check_call("git fetch --quiet --unshallow --tags {0} +refs/heads/*:refs/heads/*".format(os.environ['CI_REPOSITORY_URL']), shell=True)
|
|
elif 'CI_REPOSITORY_URL' in os.environ:
|
|
subprocess.check_call("git fetch --quiet --tags {0} +refs/heads/*:refs/heads/*".format(os.environ['CI_REPOSITORY_URL']), shell=True)
|
|
|
|
####
|
|
# Prepare to resolve and fetch our project dependencies
|
|
####
|
|
|
|
# Determine where some key resources we need for resolving dependencies will be found...
|
|
projectsMetadataPath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'repo-metadata', 'projects-invent' )
|
|
branchRulesPath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'repo-metadata', 'branch-rules.yml' )
|
|
|
|
# Bring our dependency resolver online...
|
|
dependencyResolver = Dependencies.Resolver( projectsMetadataPath, branchRulesPath, platform )
|
|
|
|
# Retrieve some key bits of information from our environment
|
|
# All of these come from environment variables due to needing to be set on either the CI Agent level or the group project level
|
|
# We remove them from the environment as they are sensitive (in the case of KDECI_GITLAB_TOKEN especially) and aren't needed by anything else
|
|
# Of these values, KDECI_GITLAB_TOKEN is optional as it is only used for writing changes back to the package archive
|
|
# The remainder are required for storing packages locally and fetching them
|
|
localCachePath = os.environ.pop('KDECI_CACHE_PATH')
|
|
gitlabInstance = os.environ.pop('KDECI_GITLAB_SERVER')
|
|
gitlabToken = os.environ.pop('KDECI_GITLAB_TOKEN', None)
|
|
packageProject = os.environ.pop('KDECI_PACKAGE_PROJECT')
|
|
servicesKey = os.environ.pop('KDECI_SECURE_SERVICES_KEY', None)
|
|
|
|
# Bring the package archive up
|
|
packageRegistry = Package.Registry( localCachePath, gitlabInstance, gitlabToken, packageProject )
|
|
|
|
####
|
|
# Now resolve both build and runtime dependencies, then fetch the build dependencies!
|
|
####
|
|
|
|
# Resolve the dependencies of this project
|
|
projectBuildDependencies = dependencyResolver.resolve( configuration['Dependencies'], arguments.branch )
|
|
# As well as the runtime dependencies
|
|
projectRuntimeDependencies = dependencyResolver.resolve( configuration['RuntimeDependencies'], arguments.branch )
|
|
|
|
# Now we can retrieve the build time dependencies
|
|
dependenciesToUnpack = packageRegistry.retrieveDependencies( projectBuildDependencies )
|
|
# And then unpack them
|
|
for packageContents, packageMetadata in dependenciesToUnpack:
|
|
# Open the archive file
|
|
archive = tarfile.open( name=packageContents, mode='r' )
|
|
# Extract it's contents into the install directory
|
|
archive.extractall( path=installPath, filter='fully_trusted' )
|
|
|
|
####
|
|
# Perform final steps needed to get ready to start the build process
|
|
####
|
|
|
|
# Determine what our build environment should be comprised of....
|
|
buildEnvironment = EnvironmentHandler.generateFor( installPrefix=installPath, configuration=configuration )
|
|
|
|
# Apply any environment changes from our configuration
|
|
for key, value in configuration['Environment'].items():
|
|
# Apply each key in turn
|
|
buildEnvironment[ key ] = value
|
|
|
|
# Do we need to get ccache ready to use?
|
|
if configuration['Options']['use-ccache'] and 'KDECI_CC_CACHE' in buildEnvironment:
|
|
# Setup the path used for the cache....
|
|
buildEnvironment['CCACHE_DIR'] = os.path.join( buildEnvironment['KDECI_CC_CACHE'], arguments.project )
|
|
# Ensure ccache is setup for use
|
|
if configuration['Options']['ccache-large-cache']:
|
|
subprocess.check_call( 'ccache -M 10G', stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=sourcesPath, env=buildEnvironment )
|
|
else:
|
|
subprocess.check_call( 'ccache -M 2G', stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=sourcesPath, env=buildEnvironment )
|
|
|
|
# Make sure the build directory exists
|
|
if not os.path.exists( buildPath ):
|
|
os.makedirs( buildPath )
|
|
|
|
# Figure out what CMake version we are running
|
|
# The only way to do this unfortunately is by parsing the output of cmake --version so run that
|
|
commandResult = subprocess.run("cmake --version", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=buildEnvironment)
|
|
cmakeOutput = commandResult.stdout.decode('utf-8')
|
|
|
|
# Pull the version out
|
|
versionMatch = re.search("version ([0-9].[0-9]+.[0-9]+)", cmakeOutput)
|
|
if versionMatch:
|
|
cmakeVersion = PackagingVersion( versionMatch.group(1) )
|
|
else:
|
|
print("## Failed to determine the CMake version in use")
|
|
sys.exit(1)
|
|
|
|
# Let the user know what we are about to do
|
|
print("## KDE Continuous Integration System")
|
|
print("##")
|
|
print("## Starting build for {0} on {1}".format(arguments.project, arguments.branch))
|
|
print("##")
|
|
print("## Installing project to {0}".format(installPath))
|
|
print("## Building project in {0}".format(buildPath))
|
|
print("##")
|
|
print("## Project CI configuration as follows:")
|
|
for key, value in configuration['Options'].items():
|
|
print("## {0}: {1}".format(key, value))
|
|
print("##")
|
|
print("## Providing the following dependencies:")
|
|
for packageContents, packageMetadata in dependenciesToUnpack:
|
|
print("## {0} - {1} ({2})".format(packageMetadata['identifier'], packageMetadata['branch'], packageMetadata['gitRevision']))
|
|
print("##")
|
|
print("## Building with the following environment variables:")
|
|
for variable, contents in buildEnvironment.items():
|
|
print("## {0}={1}".format(variable, contents))
|
|
print("##")
|
|
# If this is a setup only job, stop here
|
|
if arguments.only_setup_environment:
|
|
sys.exit(0)
|
|
|
|
print("## Starting build process...")
|
|
|
|
####
|
|
# Configure the project!
|
|
####
|
|
|
|
# Begin building up our configure command
|
|
# There are some parameters which are universal to all platforms..
|
|
cmakeTracePath = os.path.join( buildPath, 'cmake-trace.log' )
|
|
cmakeCommand = [
|
|
# Run CMake itself
|
|
'cmake',
|
|
# Capture the first configure, before find_package results are cached
|
|
'--trace-expand',
|
|
'--trace-redirect="' + cmakeTracePath + '"',
|
|
# We want a Debug build to allow for good backtraces
|
|
'-DCMAKE_BUILD_TYPE=Debug',
|
|
# And we want to be installed in a given directory
|
|
'-DCMAKE_INSTALL_PREFIX="' + installPath + '"',
|
|
# Generate compile_commands.json for tooling
|
|
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
|
|
# Our downstreams don't have internet, so we also shouldn't rely on it at cmake time
|
|
'-DFETCHCONTENT_FULLY_DISCONNECTED=ON',
|
|
# Plus any project specific parameters
|
|
configuration['Options']['cmake-options']
|
|
]
|
|
|
|
if shouldRunTests:
|
|
cmakeCommand.append('-DBUILD_TESTING=ON')
|
|
else:
|
|
cmakeCommand.append('-DBUILD_TESTING=OFF')
|
|
|
|
# Are we able to suppress CMake policy warnings?
|
|
# This flag was added in CMake 4.4.0 and later so make sure that is the CMake version we have
|
|
if cmakeVersion >= PackagingVersion("4.4.0"):
|
|
cmakeCommand.append('-Wno-policy')
|
|
|
|
# Do we need to make use of ccache?
|
|
if configuration['Options']['use-ccache'] and 'KDECI_CC_CACHE' in buildEnvironment:
|
|
# Then instruct CMake accordingly...
|
|
cmakeCommand.append('-DCMAKE_C_COMPILER_LAUNCHER=ccache')
|
|
cmakeCommand.append('-DCMAKE_CXX_COMPILER_LAUNCHER=ccache')
|
|
|
|
# Have we explicitly requested a release build?
|
|
# This should only ever be used by applications, and never libraries
|
|
# Windows is always a release build so there is no need to do anything here
|
|
if configuration['Options']['release-build'] and sys.platform != 'win32':
|
|
# Switch the Debug build for a Release one then!
|
|
cmakeCommand.remove('-DCMAKE_BUILD_TYPE=Debug')
|
|
cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release')
|
|
|
|
# Are we on Linux (but not Android)?
|
|
# If we are on Linux then we also need to check to see whether we are on a MUSL based system - as ASAN does not work there
|
|
if platform.os == 'Linux' and CommonUtils.ismanylinux():
|
|
# Then we also want Coverage by default
|
|
cmakeCommand.append('-DBUILD_COVERAGE=ON')
|
|
|
|
# We also want to enable ASAN for our builds
|
|
# Except for Android (where we don't run binaries anyway) and MUSL, where it's not supported
|
|
if not platform.os == 'Android' and not os.path.exists('/lib/libc.musl-aarch64.so.1'):
|
|
cmakeCommand.append("-DECM_ENABLE_SANITIZERS='address'")
|
|
|
|
# Are we on Windows?
|
|
if sys.platform == 'win32':
|
|
# We want a Ninja based build, rather than the default MSBuild
|
|
cmakeCommand.append('-G "Ninja"')
|
|
# Switch the Debug build for a Release one
|
|
# We do this because building our dependencies in Debug mode is just too hard and MSVC requires everything to be in either Debug or Release (you can't mix/match)
|
|
cmakeCommand.remove('-DCMAKE_BUILD_TYPE=Debug')
|
|
cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release')
|
|
# we are building with ASAN, where Windows really wants us to have debug info (see https://learn.microsoft.com/en-gb/cpp/error-messages/compiler-warnings/compiler-warning-c5072)
|
|
# force debug info without going full debug build
|
|
cmakeCommand.append('-DCMAKE_CXX_FLAGS=/Z7')
|
|
|
|
# Are we building for Android?
|
|
if platform.os == 'Android':
|
|
# We want CMake to cross compile appropriately
|
|
cmakeCommand.append('-DKF5_HOST_TOOLING=/opt/nativetooling/lib/x86_64-linux-gnu/cmake/')
|
|
cmakeCommand.append('-DKF6_HOST_TOOLING=/opt/nativetooling6/lib/x86_64-linux-gnu/cmake/')
|
|
# CMake also needs additional guidance to find things
|
|
# First though, apply a necessary transformation to allow CMake to parse the list we pass it
|
|
ecmAdditionalRoots = buildEnvironment['CMAKE_PREFIX_PATH'].replace(':', ';')
|
|
# Then give that list to CMake
|
|
cmakeCommand.append('-DECM_ADDITIONAL_FIND_ROOT_PATH="' + ecmAdditionalRoots + '"')
|
|
|
|
# Next we need to set the appropriate ECM toolchain
|
|
ecmToolchainLocations = [
|
|
# First we should prefer toolchains shipped with this project
|
|
os.path.join( sourcesPath, "toolchain/Android.cmake" ),
|
|
# Next we prefer a toolchain provided by the project dependencies
|
|
os.path.join( installPath, "share/ECM/toolchain/Android.cmake" ),
|
|
# As a final fallback use a toolchain baked into our SDK/Image
|
|
"/opt/nativetooling/share/ECM/toolchain/Android.cmake",
|
|
]
|
|
ecmToolchain = CommonUtils.firstPresentFile("/", ecmToolchainLocations)
|
|
|
|
# As well as the Qt 6 toolchain
|
|
qt6ToolchainLocations = [
|
|
"/home/user/android-arm-clang/lib/cmake/Qt6/qt.toolchain.cmake",
|
|
"/home/user/android-arm64-clang/lib/cmake/Qt6/qt.toolchain.cmake"
|
|
]
|
|
qt6Toolchain = CommonUtils.firstPresentFile("/", qt6ToolchainLocations)
|
|
|
|
# Now we make sure we found something (this should never happen)
|
|
if ecmToolchain == "":
|
|
print("## Unable to locate a suitable toolchain for this build!")
|
|
print("## Aborting build - Android compilation requires use of an appropriate CMake toolchain to enable cross-compilation (please add a dependency on extra-cmake-modules)")
|
|
sys.exit(1)
|
|
|
|
# Determine whether we are in a Qt 6 or Qt 5 environment
|
|
if os.path.exists( qt6Toolchain ):
|
|
cmakeCommand.append("-DCMAKE_TOOLCHAIN_FILE=" + qt6Toolchain)
|
|
cmakeCommand.append("-DQT_CHAINLOAD_TOOLCHAIN_FILE=" + ecmToolchain)
|
|
else:
|
|
cmakeCommand.append("-DCMAKE_TOOLCHAIN_FILE=" + ecmToolchain)
|
|
|
|
# Qt 6.11+ needs a newer Android API level specified, but ECM does not default that
|
|
# We can assume we are on a newer version if we are 64-bit based
|
|
# This is a hack/workaround and should be fixed in ECM's toolchain to do the right thing
|
|
if os.path.exists("/home/user/android-arm64-clang"):
|
|
cmakeCommand.append("-DCMAKE_ANDROID_API=28")
|
|
|
|
# Extra CMake arguments provided by the Gitlab template
|
|
if arguments.extra_cmake_args:
|
|
# necessary since we cannot use the 'extend' action for the arguments due to requiring Python < 3.8
|
|
flat_args = [item for sublist in arguments.extra_cmake_args for item in sublist]
|
|
cmakeCommand.extend(flat_args)
|
|
|
|
# Lucky last, we add the path to our sources
|
|
cmakeCommand.append( '"' + sourcesPath + '"' )
|
|
|
|
# Now glue it all together!
|
|
commandToRun = ' '.join( cmakeCommand )
|
|
|
|
# Run the command
|
|
try:
|
|
print( "## RUNNING: " + commandToRun )
|
|
subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=buildPath, env=buildEnvironment )
|
|
except Exception:
|
|
print("## Failed to configure the project")
|
|
sys.exit(1)
|
|
|
|
# Parse the trace captured while the project was configured
|
|
cmakeTracePathXz = None
|
|
try:
|
|
compressedTracePath = cmakeTracePath + '.xz'
|
|
with open(cmakeTracePath, 'rb') as cmakeTrace, lzma.open(compressedTracePath, 'wb') as cmakeTraceXz:
|
|
shutil.copyfileobj(cmakeTrace, cmakeTraceXz)
|
|
cmakeTracePathXz = compressedTracePath
|
|
except Exception:
|
|
print("## Failed to collect CMake traces")
|
|
|
|
####
|
|
# Compile the project!!
|
|
####
|
|
|
|
# Determine the appropriate number of CPU cores we should use when running builds
|
|
cpuCount = int(multiprocessing.cpu_count())
|
|
if configuration['Options']['core-usage-limit'] > 0:
|
|
cpuCount = min( cpuCount, configuration['Options']['core-usage-limit'] )
|
|
|
|
# Let CMake invoke the build tool
|
|
commandToRun = f'cmake --build "{buildPath}" -j {cpuCount}'
|
|
|
|
# Compile the project
|
|
try:
|
|
print( "## RUNNING: " + commandToRun )
|
|
subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment )
|
|
except Exception:
|
|
print("## Failed to build the project")
|
|
sys.exit(1)
|
|
|
|
####
|
|
# Run qmllint if desired
|
|
####
|
|
|
|
if configuration['Options']['run-qmllint'] and arguments.platform == "Linux/Qt6Next/Shared" and not arguments.only_build:
|
|
print("## RUNNING QMLLINT")
|
|
|
|
commandToRun = f'cmake --build "{buildPath}" -t all_qmllint_json'
|
|
subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment )
|
|
|
|
# all_qmllint_json produces a _qmllint.json for each QML module
|
|
files = [buildPath + "/" + file for file in glob.glob("*_qmllint.json", root_dir=buildPath)]
|
|
TestHandler.qmllintToCodeClimate(files, sourcesPath)
|
|
|
|
####
|
|
# Run tests before installation if needed
|
|
####
|
|
|
|
def runTests() -> None:
|
|
# Run the tests!
|
|
print("## RUNNING PROJECT TESTS")
|
|
testResult = TestHandler.run( configuration, sourcesPath, buildPath, installPath, buildEnvironment )
|
|
if not testResult:
|
|
if requirePassingTests:
|
|
print("## Tests failed, and are required to pass on this platform due to configuration")
|
|
sys.exit(1)
|
|
elif allowFailingTests:
|
|
print("## Tests failed, but are allowed to fail on this platform due to configuration")
|
|
else:
|
|
print("## Tests failed, but are not considered fatal by default")
|
|
|
|
if shouldRunTests and configuration['Options']['test-before-installing']:
|
|
runTests()
|
|
|
|
####
|
|
# Install the project...
|
|
####
|
|
|
|
# Set the appropriate environment variables to ensure we can capture make install's output later on
|
|
buildEnvironment['DESTDIR'] = installStagingPath
|
|
buildEnvironment['INSTALL_ROOT'] = installStagingPath
|
|
|
|
# Let CMake install the project
|
|
commandToRun = f'cmake --install "{buildPath}"'
|
|
|
|
# Install the project
|
|
try:
|
|
print("## RUNNING: " + commandToRun)
|
|
subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment )
|
|
except Exception:
|
|
print("## Failed to install the project")
|
|
sys.exit(1)
|
|
|
|
# Cleanup the capture variables to ensure they don't interfere with later runs
|
|
del buildEnvironment['DESTDIR']
|
|
del buildEnvironment['INSTALL_ROOT']
|
|
|
|
####
|
|
# Capture the installation if needed and deploy the staged install to the final install directory
|
|
####
|
|
|
|
# Create a temporary file, then open the file as a tar archive for writing
|
|
# We don't want it to be deleted as storePackage will move the archive into it's cache
|
|
archiveFile = tempfile.NamedTemporaryFile(delete=False)
|
|
archive = tarfile.open( fileobj=archiveFile, mode='w' )
|
|
|
|
# Now determine the path we should be archiving
|
|
# Because we could potentially be running on Windows we have to ensure our second path has been converted to a suitable form
|
|
# This conversion is necessary as os.path.join can't handle the presence of drive letters in paths other than the first argument
|
|
pathToArchive = os.path.join( installStagingPath, CommonUtils.makePathRelative(installPath) )
|
|
|
|
# Add all the files which need to be in the archive into the archive
|
|
# We want to capture the tree as it is inside the install directory and don't want any trailing slashes in the archive as this isn't standards compliant
|
|
# Therefore we list everything in the install directory and add each of those to the archive, rather than adding the whole install directory
|
|
filesToInclude = os.listdir( pathToArchive )
|
|
for filename in filesToInclude:
|
|
fullPath = os.path.join(pathToArchive, filename)
|
|
archive.add( fullPath, arcname=filename, recursive=True )
|
|
|
|
# Close the archive, which will write it out to disk, finishing what we need to do here
|
|
archive.close()
|
|
archiveFile.close()
|
|
|
|
# Are we supposed to be publishing this particular package to the archive?
|
|
if gitlabToken is not None and not arguments.skip_publishing:
|
|
# With the archive being generated, we can now prepare some metadata...
|
|
packageMetadata = {
|
|
'dependencies': projectBuildDependencies,
|
|
'runtime-dependencies': projectRuntimeDependencies
|
|
}
|
|
|
|
# Grab the Git revision (SHA-1 hash) we are building
|
|
# This is always present in Gitlab CI builds
|
|
gitRevision = os.environ['CI_COMMIT_SHA']
|
|
|
|
# Publish our package to the
|
|
packageRegistry.upload(archiveFile.name, arguments.project, arguments.branch, gitRevision, packageMetadata, cmakeTracePathXz)
|
|
|
|
# Now open the archive - so we can extract it's contents over the install prefix
|
|
# This is so later tests can rely on the project having been installed
|
|
# While we ran 'make install' just before this didn't install it as we diverted the installation to allow us to cleanly capture it
|
|
archive = tarfile.open( name=archiveFile.name, mode='r' )
|
|
archive.extractall( path=installPath, filter='fully_trusted' )
|
|
archive.close()
|
|
|
|
# Cleanup the temporary archive file as it is no longer needed
|
|
os.remove( archiveFile.name )
|
|
|
|
# If this is a build only run then bail here
|
|
if arguments.only_build:
|
|
sys.exit(0)
|
|
|
|
####
|
|
# Run installed library checks if requested
|
|
####
|
|
|
|
if platform.matches(configuration['Options']['run-installed-library-check-on']) and platform.os not in ['Android']:
|
|
print("## RUNNING INSTALLED LIBRARY CHECKS")
|
|
|
|
commandToRun = f'cmake --build "{buildPath}" -j {cpuCount} -t all_installed_library_check'
|
|
subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment )
|
|
|
|
####
|
|
# Retrieve runtime dependencies if they are needed, and rebuild our environment
|
|
####
|
|
|
|
# Ensure that we don't end up unpacking a packaged copy of ourselves
|
|
# This can potentially occur when something the current project has a runtime dependency on has a build dependency on the current project being built
|
|
# This is a bit of compatibility nightmare but not much else we can do here - if something breaks then the developer gets to hold the pieces
|
|
dependenciesToIgnore = [
|
|
arguments.project
|
|
]
|
|
|
|
# Now we can retrieve the build time dependencies
|
|
dependenciesToUnpack = packageRegistry.retrieveDependencies( projectRuntimeDependencies, dependenciesToIgnore=dependenciesToIgnore, runtime=True )
|
|
# And then unpack them
|
|
for packageContents, packageMetadata in dependenciesToUnpack:
|
|
# Open the archive file
|
|
archive = tarfile.open( name=packageContents, mode='r' )
|
|
# Extract it's contents into the install directory
|
|
archive.extractall( path=installPath, filter='fully_trusted' )
|
|
|
|
# Regenerate our environment in case the newly installed software uses directories previously not used
|
|
buildEnvironment = EnvironmentHandler.generateFor( installPrefix=installPath, configuration=configuration )
|
|
|
|
# Apply any environment changes from our configuration as well
|
|
for key, value in configuration['Environment'].items():
|
|
# Apply each key in turn
|
|
buildEnvironment[ key ] = value
|
|
|
|
####
|
|
# Run tests if we didn't do that already
|
|
####
|
|
|
|
if shouldRunTests and not configuration['Options']['test-before-installing']:
|
|
runTests()
|
|
|
|
####
|
|
# Extract test coverage results for processing by Gitlab
|
|
####
|
|
|
|
# If we aren't running on Linux then we skip this, as we consider that to be the canonical platform for code coverage...
|
|
# Additionally, as coverage information requires tests to have been run, skip extracting coverage information if tests have been disabled
|
|
if shouldRunTests and platform.os == 'Linux' and configuration['Options']['run-gcovr']:
|
|
# Determine the command we need to run
|
|
# We ask GCovr to exclude the build directory by default as we don't want generated artifacts (like moc files) getting included as well
|
|
# Sometimes projects will want to customise things slightly so we provide for that as well
|
|
commandToRun = 'gcovr --txt --xml "CoberturaLcovResults.xml" --html-details {publicArtifacts}/coverage/index.html --exclude "_build/.*" --exclude "_install/.*" --root "{sources}" {otherArguments}'
|
|
commandToRun = commandToRun.format( publicArtifacts=publicArtifactPath, sources=sourcesPath, otherArguments=configuration['Options']['gcovr-arguments'] )
|
|
|
|
# Now run it!
|
|
# If gcovr bails we ignore it, as failures to extract lcov results shouldn't cause builds to fail.
|
|
try:
|
|
print("## RUNNING: " + commandToRun)
|
|
coveragePath = os.path.join( publicArtifactPath, 'coverage' )
|
|
os.makedirs ( coveragePath )
|
|
subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment )
|
|
except Exception:
|
|
pass
|
|
|
|
coverageUrl = "https://{0}.local-kde.org/-/{1}/-/jobs/{2}/artifacts/public/coverage/index.html"
|
|
print("Coverage data can be viewed at " + coverageUrl.format( os.environ['CI_PROJECT_NAMESPACE'], os.environ['CI_PROJECT_NAME'], os.environ['CI_JOB_ID'] ))
|
|
|
|
####
|
|
# Run complete!
|
|
####
|
|
|
|
print("## CI Run Completed Successfully!")
|
|
sys.exit(0)
|