#!/usr/bin/env python3 import os import re import sys import tarfile import tempfile import argparse import subprocess import multiprocessing import glob import lzma import shutil import yaml from packaging.version import Version as PackagingVersion from components import CommonUtils, Dependencies, Package, EnvironmentHandler, TestHandler, PlatformFlavor # Capture our command line parameters parser = argparse.ArgumentParser(description='Utility to perform a CI run for a KDE project.') parser.add_argument('--project', type=str, required=True) parser.add_argument('--branch', type=str, required=True) parser.add_argument('--platform', type=str, required=True) parser.add_argument('--only-build', default=False, action='store_true') parser.add_argument('--only-setup-environment', default=False, action='store_true') parser.add_argument('--extra-cmake-args', type=str, nargs='+', action='append', required=False) parser.add_argument('--skip-publishing', default=False, action='store_true') arguments = parser.parse_args() platform = PlatformFlavor.PlatformFlavor(arguments.platform) #### # Load the project configuration #### # This consists of: # 0) Global configuration configuration = yaml.safe_load( open(os.path.join(CommonUtils.scriptsBaseDirectory(), 'config', 'global.yml')) ) # 1) Project/branch specific configuration contained within the repository if os.path.exists('.kde-ci.yml'): localConfig = yaml.safe_load( open('.kde-ci.yml') ) CommonUtils.recursiveUpdate( configuration, localConfig ) # 2) Global overrides applied to the project configuration projectConfigFile = os.path.join(CommonUtils.scriptsBaseDirectory(), 'config', arguments.project + '.yml') if os.path.exists( projectConfigFile ): projectConfig = yaml.safe_load( open(projectConfigFile) ) CommonUtils.recursiveUpdate( configuration, projectConfig ) # If the global flag to disable running tests is set, ensure that the per platform setting matches that request if not configuration['Options']['run-tests']: configuration['Options']['run-tests-on'] = [] # Determine whether we will be running tests # This is only done if they're enabled for this project and we haven't been asked to just build the project shouldRunTests = platform.matches(configuration['Options']['run-tests-on']) and not arguments.only_build # Determine whether test failures should be considered fatal requirePassingTests = platform.matches(configuration['Options']['require-passing-tests-on']) allowFailingTests = platform.matches(configuration['Options']['allow-failing-tests-on']) if requirePassingTests and allowFailingTests: print("## ERROR: Both require-passing-tests-on and allow-failing-tests-on are set. This is not a valid configuration.") print("## Set only allow-failing-tests-on in order to migrate the configuration.") sys.exit(1) if configuration['Options']['per-test-timeout'] > 1000: print("## ERROR: per test timeout is too large") print("## The per test time out is meant to prevent accidental") print("## infinite loops or other that occupies a builder wrongly") print("## for a long time, so individial tests should not be ") print("## too long running ") print("## This can be worked around by splitting long running ") print("## tests into multiple individual ones ") sys.exit(1) #### # Determine a number of paths we will need later on #### # Our sources are assumed to be in the current working directory sourcesPath = os.getcwd() # Determine where to locate the project build buildPath = os.path.join( os.getcwd(), '_build' ) if configuration['Options']['in-source-build']: buildPath = os.getcwd() # Determine where to unpack the dependencies to installPath = os.path.join( os.getcwd(), '_install' ) # Determine where we will stage the installation installStagingPath = os.path.join( os.getcwd(), '_staging' ) # Public artifact path to publish gcov data publicArtifactPath = os.path.join( os.getcwd(), 'public' ) #### # Fetch project sources #### # In order to resolve project dependencies later, we need to ensure all remote refs have been fetched if 'CI_REPOSITORY_URL' in os.environ and os.path.exists('.git/shallow'): subprocess.check_call("git fetch --quiet --unshallow --tags {0} +refs/heads/*:refs/heads/*".format(os.environ['CI_REPOSITORY_URL']), shell=True) elif 'CI_REPOSITORY_URL' in os.environ: subprocess.check_call("git fetch --quiet --tags {0} +refs/heads/*:refs/heads/*".format(os.environ['CI_REPOSITORY_URL']), shell=True) #### # Prepare to resolve and fetch our project dependencies #### # Determine where some key resources we need for resolving dependencies will be found... projectsMetadataPath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'repo-metadata', 'projects-invent' ) branchRulesPath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'repo-metadata', 'branch-rules.yml' ) # Bring our dependency resolver online... dependencyResolver = Dependencies.Resolver( projectsMetadataPath, branchRulesPath, platform ) # Retrieve some key bits of information from our environment # All of these come from environment variables due to needing to be set on either the CI Agent level or the group project level # We remove them from the environment as they are sensitive (in the case of KDECI_GITLAB_TOKEN especially) and aren't needed by anything else # Of these values, KDECI_GITLAB_TOKEN is optional as it is only used for writing changes back to the package archive # The remainder are required for storing packages locally and fetching them localCachePath = os.environ.pop('KDECI_CACHE_PATH') gitlabInstance = os.environ.pop('KDECI_GITLAB_SERVER') gitlabToken = os.environ.pop('KDECI_GITLAB_TOKEN', None) packageProject = os.environ.pop('KDECI_PACKAGE_PROJECT') servicesKey = os.environ.pop('KDECI_SECURE_SERVICES_KEY', None) # Bring the package archive up packageRegistry = Package.Registry( localCachePath, gitlabInstance, gitlabToken, packageProject ) #### # Now resolve both build and runtime dependencies, then fetch the build dependencies! #### # Resolve the dependencies of this project projectBuildDependencies = dependencyResolver.resolve( configuration['Dependencies'], arguments.branch ) # As well as the runtime dependencies projectRuntimeDependencies = dependencyResolver.resolve( configuration['RuntimeDependencies'], arguments.branch ) # Now we can retrieve the build time dependencies dependenciesToUnpack = packageRegistry.retrieveDependencies( projectBuildDependencies ) # And then unpack them for packageContents, packageMetadata in dependenciesToUnpack: # Open the archive file archive = tarfile.open( name=packageContents, mode='r' ) # Extract it's contents into the install directory archive.extractall( path=installPath, filter='fully_trusted' ) #### # Perform final steps needed to get ready to start the build process #### # Determine what our build environment should be comprised of.... buildEnvironment = EnvironmentHandler.generateFor( installPrefix=installPath, configuration=configuration ) # Apply any environment changes from our configuration for key, value in configuration['Environment'].items(): # Apply each key in turn buildEnvironment[ key ] = value # Do we need to get ccache ready to use? if configuration['Options']['use-ccache'] and 'KDECI_CC_CACHE' in buildEnvironment: # Setup the path used for the cache.... buildEnvironment['CCACHE_DIR'] = os.path.join( buildEnvironment['KDECI_CC_CACHE'], arguments.project ) # Ensure ccache is setup for use if configuration['Options']['ccache-large-cache']: subprocess.check_call( 'ccache -M 10G', stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=sourcesPath, env=buildEnvironment ) else: subprocess.check_call( 'ccache -M 2G', stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=sourcesPath, env=buildEnvironment ) # Make sure the build directory exists if not os.path.exists( buildPath ): os.makedirs( buildPath ) # Figure out what CMake version we are running # The only way to do this unfortunately is by parsing the output of cmake --version so run that commandResult = subprocess.run("cmake --version", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=buildEnvironment) cmakeOutput = commandResult.stdout.decode('utf-8') # Pull the version out versionMatch = re.search("version ([0-9].[0-9]+.[0-9]+)", cmakeOutput) if versionMatch: cmakeVersion = PackagingVersion( versionMatch.group(1) ) else: print("## Failed to determine the CMake version in use") sys.exit(1) # Let the user know what we are about to do print("## KDE Continuous Integration System") print("##") print("## Starting build for {0} on {1}".format(arguments.project, arguments.branch)) print("##") print("## Installing project to {0}".format(installPath)) print("## Building project in {0}".format(buildPath)) print("##") print("## Project CI configuration as follows:") for key, value in configuration['Options'].items(): print("## {0}: {1}".format(key, value)) print("##") print("## Providing the following dependencies:") for packageContents, packageMetadata in dependenciesToUnpack: print("## {0} - {1} ({2})".format(packageMetadata['identifier'], packageMetadata['branch'], packageMetadata['gitRevision'])) print("##") print("## Building with the following environment variables:") for variable, contents in buildEnvironment.items(): print("## {0}={1}".format(variable, contents)) print("##") # If this is a setup only job, stop here if arguments.only_setup_environment: sys.exit(0) print("## Starting build process...") #### # Configure the project! #### # Begin building up our configure command # There are some parameters which are universal to all platforms.. cmakeTracePath = os.path.join( buildPath, 'cmake-trace.log' ) cmakeCommand = [ # Run CMake itself 'cmake', # Capture the first configure, before find_package results are cached '--trace-expand', '--trace-redirect="' + cmakeTracePath + '"', # We want a Debug build to allow for good backtraces '-DCMAKE_BUILD_TYPE=Debug', # And we want to be installed in a given directory '-DCMAKE_INSTALL_PREFIX="' + installPath + '"', # Generate compile_commands.json for tooling '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', # Our downstreams don't have internet, so we also shouldn't rely on it at cmake time '-DFETCHCONTENT_FULLY_DISCONNECTED=ON', # Plus any project specific parameters configuration['Options']['cmake-options'] ] if shouldRunTests: cmakeCommand.append('-DBUILD_TESTING=ON') else: cmakeCommand.append('-DBUILD_TESTING=OFF') # Are we able to suppress CMake policy warnings? # This flag was added in CMake 4.4.0 and later so make sure that is the CMake version we have if cmakeVersion >= PackagingVersion("4.4.0"): cmakeCommand.append('-Wno-policy') # Do we need to make use of ccache? if configuration['Options']['use-ccache'] and 'KDECI_CC_CACHE' in buildEnvironment: # Then instruct CMake accordingly... cmakeCommand.append('-DCMAKE_C_COMPILER_LAUNCHER=ccache') cmakeCommand.append('-DCMAKE_CXX_COMPILER_LAUNCHER=ccache') # Have we explicitly requested a release build? # This should only ever be used by applications, and never libraries # Windows is always a release build so there is no need to do anything here if configuration['Options']['release-build'] and sys.platform != 'win32': # Switch the Debug build for a Release one then! cmakeCommand.remove('-DCMAKE_BUILD_TYPE=Debug') cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release') # Are we on Linux (but not Android)? # If we are on Linux then we also need to check to see whether we are on a MUSL based system - as ASAN does not work there if platform.os == 'Linux' and CommonUtils.ismanylinux(): # Then we also want Coverage by default cmakeCommand.append('-DBUILD_COVERAGE=ON') # We also want to enable ASAN for our builds # Except for Android (where we don't run binaries anyway) and MUSL, where it's not supported if not platform.os == 'Android' and not os.path.exists('/lib/libc.musl-aarch64.so.1'): cmakeCommand.append("-DECM_ENABLE_SANITIZERS='address'") # Are we on Windows? if sys.platform == 'win32': # We want a Ninja based build, rather than the default MSBuild cmakeCommand.append('-G "Ninja"') # Switch the Debug build for a Release one # We do this because building our dependencies in Debug mode is just too hard and MSVC requires everything to be in either Debug or Release (you can't mix/match) cmakeCommand.remove('-DCMAKE_BUILD_TYPE=Debug') cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release') # we are building with ASAN, where Windows really wants us to have debug info (see https://learn.microsoft.com/en-gb/cpp/error-messages/compiler-warnings/compiler-warning-c5072) # force debug info without going full debug build cmakeCommand.append('-DCMAKE_CXX_FLAGS=/Z7') # Are we building for Android? if platform.os == 'Android': # We want CMake to cross compile appropriately cmakeCommand.append('-DKF5_HOST_TOOLING=/opt/nativetooling/lib/x86_64-linux-gnu/cmake/') cmakeCommand.append('-DKF6_HOST_TOOLING=/opt/nativetooling6/lib/x86_64-linux-gnu/cmake/') # CMake also needs additional guidance to find things # First though, apply a necessary transformation to allow CMake to parse the list we pass it ecmAdditionalRoots = buildEnvironment['CMAKE_PREFIX_PATH'].replace(':', ';') # Then give that list to CMake cmakeCommand.append('-DECM_ADDITIONAL_FIND_ROOT_PATH="' + ecmAdditionalRoots + '"') # Next we need to set the appropriate ECM toolchain ecmToolchainLocations = [ # First we should prefer toolchains shipped with this project os.path.join( sourcesPath, "toolchain/Android.cmake" ), # Next we prefer a toolchain provided by the project dependencies os.path.join( installPath, "share/ECM/toolchain/Android.cmake" ), # As a final fallback use a toolchain baked into our SDK/Image "/opt/nativetooling/share/ECM/toolchain/Android.cmake", ] ecmToolchain = CommonUtils.firstPresentFile("/", ecmToolchainLocations) # As well as the Qt 6 toolchain qt6ToolchainLocations = [ "/home/user/android-arm-clang/lib/cmake/Qt6/qt.toolchain.cmake", "/home/user/android-arm64-clang/lib/cmake/Qt6/qt.toolchain.cmake" ] qt6Toolchain = CommonUtils.firstPresentFile("/", qt6ToolchainLocations) # Now we make sure we found something (this should never happen) if ecmToolchain == "": print("## Unable to locate a suitable toolchain for this build!") print("## Aborting build - Android compilation requires use of an appropriate CMake toolchain to enable cross-compilation (please add a dependency on extra-cmake-modules)") sys.exit(1) # Determine whether we are in a Qt 6 or Qt 5 environment if os.path.exists( qt6Toolchain ): cmakeCommand.append("-DCMAKE_TOOLCHAIN_FILE=" + qt6Toolchain) cmakeCommand.append("-DQT_CHAINLOAD_TOOLCHAIN_FILE=" + ecmToolchain) else: cmakeCommand.append("-DCMAKE_TOOLCHAIN_FILE=" + ecmToolchain) # Qt 6.11+ needs a newer Android API level specified, but ECM does not default that # We can assume we are on a newer version if we are 64-bit based # This is a hack/workaround and should be fixed in ECM's toolchain to do the right thing if os.path.exists("/home/user/android-arm64-clang"): cmakeCommand.append("-DCMAKE_ANDROID_API=28") # Extra CMake arguments provided by the Gitlab template if arguments.extra_cmake_args: # necessary since we cannot use the 'extend' action for the arguments due to requiring Python < 3.8 flat_args = [item for sublist in arguments.extra_cmake_args for item in sublist] cmakeCommand.extend(flat_args) # Lucky last, we add the path to our sources cmakeCommand.append( '"' + sourcesPath + '"' ) # Now glue it all together! commandToRun = ' '.join( cmakeCommand ) # Run the command try: print( "## RUNNING: " + commandToRun ) subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=buildPath, env=buildEnvironment ) except Exception: print("## Failed to configure the project") sys.exit(1) # Parse the trace captured while the project was configured cmakeTracePathXz = None try: compressedTracePath = cmakeTracePath + '.xz' with open(cmakeTracePath, 'rb') as cmakeTrace, lzma.open(compressedTracePath, 'wb') as cmakeTraceXz: shutil.copyfileobj(cmakeTrace, cmakeTraceXz) cmakeTracePathXz = compressedTracePath except Exception: print("## Failed to collect CMake traces") #### # Compile the project!! #### # Determine the appropriate number of CPU cores we should use when running builds cpuCount = int(multiprocessing.cpu_count()) if configuration['Options']['core-usage-limit'] > 0: cpuCount = min( cpuCount, configuration['Options']['core-usage-limit'] ) # Let CMake invoke the build tool commandToRun = f'cmake --build "{buildPath}" -j {cpuCount}' # Compile the project try: print( "## RUNNING: " + commandToRun ) subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment ) except Exception: print("## Failed to build the project") sys.exit(1) #### # Run qmllint if desired #### if configuration['Options']['run-qmllint'] and arguments.platform == "Linux/Qt6Next/Shared" and not arguments.only_build: print("## RUNNING QMLLINT") commandToRun = f'cmake --build "{buildPath}" -t all_qmllint_json' subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment ) # all_qmllint_json produces a _qmllint.json for each QML module files = [buildPath + "/" + file for file in glob.glob("*_qmllint.json", root_dir=buildPath)] TestHandler.qmllintToCodeClimate(files, sourcesPath) #### # Run tests before installation if needed #### def runTests() -> None: # Run the tests! print("## RUNNING PROJECT TESTS") testResult = TestHandler.run( configuration, sourcesPath, buildPath, installPath, buildEnvironment ) if not testResult: if requirePassingTests: print("## Tests failed, and are required to pass on this platform due to configuration") sys.exit(1) elif allowFailingTests: print("## Tests failed, but are allowed to fail on this platform due to configuration") else: print("## Tests failed, but are not considered fatal by default") if shouldRunTests and configuration['Options']['test-before-installing']: runTests() #### # Install the project... #### # Set the appropriate environment variables to ensure we can capture make install's output later on buildEnvironment['DESTDIR'] = installStagingPath buildEnvironment['INSTALL_ROOT'] = installStagingPath # Let CMake install the project commandToRun = f'cmake --install "{buildPath}"' # Install the project try: print("## RUNNING: " + commandToRun) subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment ) except Exception: print("## Failed to install the project") sys.exit(1) # Cleanup the capture variables to ensure they don't interfere with later runs del buildEnvironment['DESTDIR'] del buildEnvironment['INSTALL_ROOT'] #### # Capture the installation if needed and deploy the staged install to the final install directory #### # Create a temporary file, then open the file as a tar archive for writing # We don't want it to be deleted as storePackage will move the archive into it's cache archiveFile = tempfile.NamedTemporaryFile(delete=False) archive = tarfile.open( fileobj=archiveFile, mode='w' ) # Now determine the path we should be archiving # Because we could potentially be running on Windows we have to ensure our second path has been converted to a suitable form # This conversion is necessary as os.path.join can't handle the presence of drive letters in paths other than the first argument pathToArchive = os.path.join( installStagingPath, CommonUtils.makePathRelative(installPath) ) # Add all the files which need to be in the archive into the archive # We want to capture the tree as it is inside the install directory and don't want any trailing slashes in the archive as this isn't standards compliant # Therefore we list everything in the install directory and add each of those to the archive, rather than adding the whole install directory filesToInclude = os.listdir( pathToArchive ) for filename in filesToInclude: fullPath = os.path.join(pathToArchive, filename) archive.add( fullPath, arcname=filename, recursive=True ) # Close the archive, which will write it out to disk, finishing what we need to do here archive.close() archiveFile.close() # Are we supposed to be publishing this particular package to the archive? if gitlabToken is not None and not arguments.skip_publishing: # With the archive being generated, we can now prepare some metadata... packageMetadata = { 'dependencies': projectBuildDependencies, 'runtime-dependencies': projectRuntimeDependencies } # Grab the Git revision (SHA-1 hash) we are building # This is always present in Gitlab CI builds gitRevision = os.environ['CI_COMMIT_SHA'] # Publish our package to the packageRegistry.upload(archiveFile.name, arguments.project, arguments.branch, gitRevision, packageMetadata, cmakeTracePathXz) # Now open the archive - so we can extract it's contents over the install prefix # This is so later tests can rely on the project having been installed # While we ran 'make install' just before this didn't install it as we diverted the installation to allow us to cleanly capture it archive = tarfile.open( name=archiveFile.name, mode='r' ) archive.extractall( path=installPath, filter='fully_trusted' ) archive.close() # Cleanup the temporary archive file as it is no longer needed os.remove( archiveFile.name ) # If this is a build only run then bail here if arguments.only_build: sys.exit(0) #### # Run installed library checks if requested #### if platform.matches(configuration['Options']['run-installed-library-check-on']) and platform.os not in ['Android']: print("## RUNNING INSTALLED LIBRARY CHECKS") commandToRun = f'cmake --build "{buildPath}" -j {cpuCount} -t all_installed_library_check' subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment ) #### # Retrieve runtime dependencies if they are needed, and rebuild our environment #### # Ensure that we don't end up unpacking a packaged copy of ourselves # This can potentially occur when something the current project has a runtime dependency on has a build dependency on the current project being built # This is a bit of compatibility nightmare but not much else we can do here - if something breaks then the developer gets to hold the pieces dependenciesToIgnore = [ arguments.project ] # Now we can retrieve the build time dependencies dependenciesToUnpack = packageRegistry.retrieveDependencies( projectRuntimeDependencies, dependenciesToIgnore=dependenciesToIgnore, runtime=True ) # And then unpack them for packageContents, packageMetadata in dependenciesToUnpack: # Open the archive file archive = tarfile.open( name=packageContents, mode='r' ) # Extract it's contents into the install directory archive.extractall( path=installPath, filter='fully_trusted' ) # Regenerate our environment in case the newly installed software uses directories previously not used buildEnvironment = EnvironmentHandler.generateFor( installPrefix=installPath, configuration=configuration ) # Apply any environment changes from our configuration as well for key, value in configuration['Environment'].items(): # Apply each key in turn buildEnvironment[ key ] = value #### # Run tests if we didn't do that already #### if shouldRunTests and not configuration['Options']['test-before-installing']: runTests() #### # Extract test coverage results for processing by Gitlab #### # If we aren't running on Linux then we skip this, as we consider that to be the canonical platform for code coverage... # Additionally, as coverage information requires tests to have been run, skip extracting coverage information if tests have been disabled if shouldRunTests and platform.os == 'Linux' and configuration['Options']['run-gcovr']: # Determine the command we need to run # We ask GCovr to exclude the build directory by default as we don't want generated artifacts (like moc files) getting included as well # Sometimes projects will want to customise things slightly so we provide for that as well commandToRun = 'gcovr --txt --xml "CoberturaLcovResults.xml" --html-details {publicArtifacts}/coverage/index.html --exclude "_build/.*" --exclude "_install/.*" --root "{sources}" {otherArguments}' commandToRun = commandToRun.format( publicArtifacts=publicArtifactPath, sources=sourcesPath, otherArguments=configuration['Options']['gcovr-arguments'] ) # Now run it! # If gcovr bails we ignore it, as failures to extract lcov results shouldn't cause builds to fail. try: print("## RUNNING: " + commandToRun) coveragePath = os.path.join( publicArtifactPath, 'coverage' ) os.makedirs ( coveragePath ) subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, env=buildEnvironment ) except Exception: pass coverageUrl = "https://{0}.local-kde.org/-/{1}/-/jobs/{2}/artifacts/public/coverage/index.html" print("Coverage data can be viewed at " + coverageUrl.format( os.environ['CI_PROJECT_NAMESPACE'], os.environ['CI_PROJECT_NAME'], os.environ['CI_JOB_ID'] )) #### # Run complete! #### print("## CI Run Completed Successfully!") sys.exit(0)