222 lines
9.7 KiB
Python
222 lines
9.7 KiB
Python
#!/usr/bin/python3
|
|
import os
|
|
import sys
|
|
import copy
|
|
import yaml
|
|
import argparse
|
|
import shutil
|
|
import subprocess
|
|
from components import CommonUtils, Dependencies, PlatformFlavor, Package, MergeFolders
|
|
from components.CiConfigurationUtils import *
|
|
|
|
|
|
# Capture our command line parameters
|
|
parser = argparse.ArgumentParser(description='Utility to seed a Package Registry for use with run-ci-build.py')
|
|
parser.add_argument('--seed-file', type=str, required=True)
|
|
parser.add_argument('--platform', type=str, required=True)
|
|
parser.add_argument('--extra-cmake-args', type=str, nargs='+', action='append', required=False)
|
|
parser.add_argument('--skip-dependencies-fetch', default=False, action='store_true')
|
|
parser.add_argument('--publish-to-cache', default=False, action='store_true')
|
|
parser.add_argument('--missing-only', default=False, action='store_true')
|
|
arguments = parser.parse_args()
|
|
platform = PlatformFlavor.PlatformFlavor(arguments.platform)
|
|
|
|
if arguments.missing_only and not arguments.publish_to_cache:
|
|
print ("WARNING: argument --missing-only has no effect without --publish-to-cache")
|
|
|
|
####
|
|
# Prepare to work
|
|
####
|
|
|
|
# Make sure if we are running under Gitlab CI that there are no variables around that will interfere in things
|
|
if 'CI_REPOSITORY_URL' in os.environ:
|
|
del os.environ['CI_REPOSITORY_URL']
|
|
|
|
# Because we are a seed job it is assumed we are building a release branch
|
|
os.environ['CI_COMMIT_REF_PROTECTED'] = "true"
|
|
|
|
# Load the seed file
|
|
# This file will contain a list of definitions of projects we should be building
|
|
seedConfiguration = yaml.safe_load( open( arguments.seed_file ) )
|
|
|
|
# Determine where we will be working
|
|
workingDirectory = os.getcwd()
|
|
|
|
# Bring our dependency resolver online...
|
|
dependencyResolver = prepareDependenciesResolver(platform)
|
|
|
|
# And use it to determine the projects we will be building
|
|
# The seed file uses the same definition format as the project dependencies, so we can reuse that logic
|
|
# In this case the branch name is only used to resolve @same and that is unsupported in a seed file, so a dummy value of None is sufficient here
|
|
projectsToBuild = dependencyResolver.resolve( seedConfiguration, None )
|
|
|
|
####
|
|
# Fetch the projects we will be building and resolve their dependencies
|
|
####
|
|
|
|
# Setup a place to store the information
|
|
projectBuildDependencies = {}
|
|
|
|
# Go over all the projects we will be building
|
|
for identifier, branch in projectsToBuild.items():
|
|
# Retrieve the full details from the Dependencies project database
|
|
project = dependencyResolver.projectsByIdentifier[ identifier ]
|
|
projectFolder = os.path.join(workingDirectory, identifier)
|
|
|
|
if project['hasrepo']:
|
|
# Construct the URL to clone
|
|
gitUrl = "https://invent.kde.org/{0}.git".format( project['repopath'] )
|
|
|
|
# Clone it!
|
|
commandToRun = "git clone {0} --branch={1} {2}/".format( gitUrl, branch, identifier )
|
|
subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=workingDirectory )
|
|
elif 'reuse-directory' in project:
|
|
if os.path.isdir(projectFolder):
|
|
for item in os.listdir(projectFolder):
|
|
print (f'Trying to remove {item}')
|
|
if not item in ['.kde-ci-override.yml', '.gitignore']:
|
|
print (f' removing... {item}')
|
|
itemPath = os.path.join(projectFolder, item)
|
|
if os.path.isdir(itemPath):
|
|
shutil.rmtree()
|
|
else:
|
|
os.remove(itemPath)
|
|
pass
|
|
|
|
MergeFolders.merge_folders(os.path.join(workingDirectory, project['reuse-directory']),
|
|
projectFolder,
|
|
skip_paths=['_build', '_install', '_staging'])
|
|
|
|
projectConfigFile = os.path.join(projectFolder, '.kde-ci.yml')
|
|
overrideFile = os.path.join(projectFolder, '.kde-ci-override.yml')
|
|
if os.path.isfile(overrideFile):
|
|
if not os.path.isfile(projectConfigFile):
|
|
shutil.copy2(overrideFile, projectConfigFile)
|
|
else:
|
|
with open(projectConfigFile, 'r') as f:
|
|
localConfig = yaml.safe_load(f)
|
|
with open(overrideFile, 'r') as f:
|
|
overrideConfig = yaml.safe_load(f)
|
|
|
|
CommonUtils.recursiveUpdate( localConfig, overrideConfig )
|
|
|
|
with open(projectConfigFile, 'w') as f:
|
|
yaml.dump(localConfig, f, indent = 2)
|
|
|
|
configuration = loadProjectConfiguration(projectFolder, identifier)
|
|
|
|
# The Dependency Resolver requires the current working directory to be in the project it is resolving (due to @same)
|
|
# Therefore we need to switch there first
|
|
os.chdir( projectFolder )
|
|
# Resolve the dependencies for this project now
|
|
dependencies = dependencyResolver.resolve( allBuildDependencies(configuration), branch )
|
|
# And save them to our list...
|
|
projectBuildDependencies[ identifier ] = dependencies
|
|
# Now that we are done we can change back
|
|
os.chdir( workingDirectory )
|
|
|
|
####
|
|
# Now we can start to build these projects
|
|
####
|
|
|
|
builtProjects = {}
|
|
allPresentProjectsInSharedInstall = set()
|
|
|
|
while len(projectsToBuild) != 0:
|
|
# Make a copy to work on...
|
|
workingProjectsList = copy.deepcopy(projectsToBuild)
|
|
# Go over all the products in our list
|
|
for identifier, branch in workingProjectsList.items():
|
|
# Grab the products dependencies
|
|
projectDeps = projectBuildDependencies[ identifier ]
|
|
# Eliminate from this any dependencies this seed file is not building
|
|
# For those dependencies we simply assume another seed job has built them
|
|
# This also has the nice side effect of telling us if there is anything left we are still waiting to build
|
|
remainingDependencies = list( set(projectDeps).intersection(projectsToBuild) )
|
|
|
|
# Do we have anything left?
|
|
if len(remainingDependencies) > 0:
|
|
# Not it's turn unfortunately
|
|
continue
|
|
|
|
# We have a winner!
|
|
# Remove it from the list of projects to build....
|
|
del projectsToBuild[ identifier ]
|
|
|
|
localCachePath = os.environ.get('KDECI_CACHE_PATH', None)
|
|
if not localCachePath is None and arguments.publish_to_cache and arguments.missing_only:
|
|
normalisedBranch = Package.Registry._normaliseBranchName( branch )
|
|
if os.path.exists(os.path.join(localCachePath, '{}-{}.json'.format(identifier, normalisedBranch))):
|
|
print('## Skipping build of {} ({}) since a package exists in the cache'.format(identifier, normalisedBranch))
|
|
continue
|
|
|
|
# Then start the build process - find where the sources are...
|
|
projectSources = os.path.join( workingDirectory, identifier )
|
|
|
|
# We need to set CI_COMMIT_SHA in the environment to match the hash of the project we are building
|
|
process = subprocess.Popen("git log --format=%H -1", stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, cwd=projectSources)
|
|
os.environ['CI_COMMIT_SHA'] = process.stdout.readline().strip().decode('utf-8')
|
|
|
|
# Prepare the command needed to build the project...
|
|
commandToRun = "{0} -u {1}/run-ci-build.py --project {2} --branch {3} --platform {4} --only-build --fail-on-leaked-stage-files".format(
|
|
sys.executable,
|
|
CommonUtils.scriptsBaseDirectory(),
|
|
identifier,
|
|
branch,
|
|
platform
|
|
)
|
|
|
|
if arguments.skip_dependencies_fetch:
|
|
# just forward skip-dependencies-fetch argument to the lower-level tool
|
|
commandToRun += ' --skip-dependencies-fetch'
|
|
|
|
if arguments.publish_to_cache:
|
|
# just forward publish-to-cache argument to the lower-level tool
|
|
commandToRun += ' --publish-to-cache'
|
|
|
|
if arguments.extra_cmake_args:
|
|
# necessary since we cannot use the 'extend' action for the arguments due to requiring Python < 3.8
|
|
flat_args = [item for sublist in arguments.extra_cmake_args for item in sublist]
|
|
commandToRun += ' ' + ' '.join(['--extra-cmake-args=' + arg for arg in flat_args])
|
|
|
|
if 'KDECI_SHARED_INSTALL_PATH' in os.environ:
|
|
requiredBuildDepsSet = set()
|
|
for dep, depBranch in projectBuildDependencies[identifier].items():
|
|
requiredBuildDepsSet.update(
|
|
lazyResolveProjectDeps(workingDirectory,
|
|
dep, depBranch,
|
|
builtProjects.keys(),
|
|
dependencyResolver))
|
|
|
|
depsToExclude = requiredBuildDepsSet.intersection(allPresentProjectsInSharedInstall)
|
|
|
|
if depsToExclude:
|
|
commandToRun += ' --skip-deps ' + ' '.join(depsToExclude)
|
|
|
|
print('## Run project build: {}'.format(commandToRun))
|
|
|
|
# Then run it!
|
|
try:
|
|
subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=projectSources )
|
|
pass
|
|
except:
|
|
print('## Failed building a project: {}'.format(identifier))
|
|
print('## Projects built: \"{}\"'.format((' '.join(builtProjects.keys()))))
|
|
print('## Projects **not** built: \"{}\"'.format(' '.join(projectsToBuild.keys())))
|
|
raise
|
|
|
|
# Add it to the list of projects we've built
|
|
builtProjects[ identifier ] = branch
|
|
if 'KDECI_SHARED_INSTALL_PATH' in os.environ:
|
|
allPresentProjectsInSharedInstall.update(
|
|
lazyResolveProjectDeps(workingDirectory,
|
|
identifier, branch,
|
|
builtProjects.keys(),
|
|
dependencyResolver))
|
|
|
|
|
|
####
|
|
# We're done!
|
|
####
|
|
|
|
sys.exit(0)
|