133 lines
5.4 KiB
Python
133 lines
5.4 KiB
Python
import os
|
|
import sys
|
|
import yaml
|
|
from components import CommonUtils, Dependencies, Package
|
|
|
|
|
|
####
|
|
# Load the project configuration
|
|
####
|
|
def loadProjectConfiguration(projectRoot, projectName):
|
|
# This consists of:
|
|
# 0) Global configuration
|
|
configuration = yaml.safe_load( open(os.path.join(CommonUtils.scriptsBaseDirectory(), 'config', 'global.yml')) )
|
|
|
|
# 1) Global configuration from environment
|
|
if 'KDECI_GLOBAL_CONFIG_OVERRIDE_PATH' in os.environ:
|
|
overridePath = os.environ['KDECI_GLOBAL_CONFIG_OVERRIDE_PATH']
|
|
if os.path.exists( overridePath ):
|
|
overrideConfig = yaml.safe_load( open(overridePath) )
|
|
CommonUtils.recursiveUpdate( configuration, overrideConfig )
|
|
else:
|
|
print('## Error: $KDECI_GLOBAL_CONFIG_OVERRIDE_PATH({}) is present, but the file doesn\'t exist'.format(overridePath))
|
|
sys.exit(-1)
|
|
|
|
# 2) Project/branch specific configuration contained within the repository
|
|
localConfigFile = os.path.join( projectRoot, '.kde-ci.yml' )
|
|
if os.path.exists( localConfigFile ):
|
|
localConfig = yaml.safe_load( open(localConfigFile) )
|
|
CommonUtils.recursiveUpdate( configuration, localConfig )
|
|
|
|
# 3) Global overrides applied to the project configuration
|
|
projectConfigFile = os.path.join(CommonUtils.scriptsBaseDirectory(), 'config', projectName + '.yml')
|
|
if os.path.exists( projectConfigFile ):
|
|
projectConfig = yaml.safe_load( open(projectConfigFile) )
|
|
CommonUtils.recursiveUpdate( configuration, projectConfig )
|
|
|
|
return configuration
|
|
|
|
def allBuildDependencies(configuration):
|
|
result = []
|
|
if configuration['Dependencies']:
|
|
result.extend(configuration['Dependencies'])
|
|
if configuration['BuildDependencies']:
|
|
result.extend(configuration['BuildDependencies'])
|
|
return result
|
|
####
|
|
# Prepare to resolve and fetch our project dependencies
|
|
####
|
|
def prepareDependenciesResolver(platform):
|
|
metadataFolderPath = os.environ.get('KDECI_REPO_METADATA_PATH', os.path.join(CommonUtils.scriptsBaseDirectory(), 'repo-metadata'))
|
|
|
|
# Determine where some key resources we need for resolving dependencies will be found...
|
|
projectsMetadataPath = os.path.join( metadataFolderPath, 'projects-invent' )
|
|
branchRulesPath = os.path.join( metadataFolderPath, 'branch-rules.yml' )
|
|
|
|
# Bring our dependency resolver online...
|
|
return Dependencies.Resolver( projectsMetadataPath, branchRulesPath, platform )
|
|
|
|
####
|
|
# Lazily retrieve project dependency information either from a local build or
|
|
# from the package registry
|
|
####
|
|
packageRegistry = None
|
|
def lazyResolveProjectDeps(workingDirectory, projectId, projectBranch, locallyBuiltProjectIds, dependencyResolver):
|
|
exisitingDeps = set()
|
|
projectDirectory = os.path.join(workingDirectory, projectId)
|
|
|
|
if not projectId in locallyBuiltProjectIds:
|
|
localCachePath = os.environ['KDECI_CACHE_PATH']
|
|
gitlabInstance = os.environ['KDECI_GITLAB_SERVER']
|
|
packageProject = os.environ['KDECI_PACKAGE_PROJECT']
|
|
|
|
packageRegistry = Package.Registry( localCachePath, gitlabInstance, None, packageProject )
|
|
allDependencies = packageRegistry.retrieveDependencies( {projectId: projectBranch}, onlyMetadata=True )
|
|
|
|
exisitingDeps.update([item[1]['identifier'] for item in allDependencies])
|
|
else:
|
|
configuration = loadProjectConfiguration(projectDirectory, projectId)
|
|
projectBuildDependencies = dependencyResolver.resolve( allBuildDependencies(configuration), projectBranch )
|
|
|
|
exisitingDeps.add(projectId)
|
|
for childDep, childBranch in projectBuildDependencies.items():
|
|
exisitingDeps.add(childDep)
|
|
exisitingDeps.update(lazyResolveProjectDeps(workingDirectory, childDep, childBranch, locallyBuiltProjectIds, dependencyResolver))
|
|
|
|
return exisitingDeps
|
|
|
|
####
|
|
# Generate reverse dependencies mapping, e.g.
|
|
# ext_zlib -> ext_qt
|
|
# ext_qt -> ext_kimageformats
|
|
####
|
|
def genReverseDeps(workingDirectory, dependencyResolver, branch, debug = False, onlyPlatformDeps = None):
|
|
reverseDeps = {}
|
|
for subdir, dirs, files in os.walk(workingDirectory):
|
|
relative = os.path.relpath(subdir, workingDirectory)
|
|
depth = os.path.normpath(relative).count(os.sep) + 1
|
|
|
|
if depth >= 2:
|
|
dirs.clear()
|
|
|
|
subDirName = os.path.basename(subdir)
|
|
|
|
if subDirName == os.path.basename(workingDirectory):
|
|
continue
|
|
|
|
if not subDirName.startswith('ext_'):
|
|
dirs.clear()
|
|
continue
|
|
|
|
checkAllowed = lambda name: (name in onlyPlatformDeps) if not onlyPlatformDeps is None else True
|
|
|
|
projectName = subDirName
|
|
|
|
if not checkAllowed(projectName): continue
|
|
|
|
if os.path.exists(os.path.join(subdir, 'CMakeLists.txt')):
|
|
configuration = loadProjectConfiguration(subdir, projectName)
|
|
projectBuildDependencies = list(dependencyResolver.resolve( allBuildDependencies(configuration), branch ).keys()) # all build
|
|
projectBuildDependencies = list(filter(checkAllowed, projectBuildDependencies))
|
|
|
|
if debug:
|
|
print ("## project: {} depends: {}".format(projectName, projectBuildDependencies))
|
|
|
|
# found the project, don't check subdirs anymore
|
|
dirs.clear()
|
|
|
|
for dep in projectBuildDependencies:
|
|
if dep in reverseDeps:
|
|
reverseDeps[dep].add(projectName)
|
|
else:
|
|
reverseDeps[dep] = set([projectName])
|
|
return reverseDeps
|