400 lines
19 KiB
Python
400 lines
19 KiB
Python
import os
|
|
import sys
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import gitlab
|
|
import shutil
|
|
import tempfile
|
|
import packaging.version
|
|
from enum import Enum
|
|
|
|
class CacheStatus(Enum):
|
|
FromRemote = 0,
|
|
FromCache = 1
|
|
|
|
class Registry(object):
|
|
|
|
# Record all the details we need for later use
|
|
def __init__(self, localCachePath, gitlabInstance, gitlabToken, gitlabPackageProject, packageAliases = {}):
|
|
# Store all the details we have been given for later use
|
|
self.localCachePath = localCachePath
|
|
|
|
# First prepare to gather details from both the cache and the remote registry
|
|
self.cachedPackages = []
|
|
self.remotePackages = []
|
|
self.packageAliases = packageAliases
|
|
|
|
# Make sure the local cache path exists
|
|
if not os.path.exists( self.localCachePath ):
|
|
os.makedirs( self.localCachePath )
|
|
|
|
# Determine what we have locally first
|
|
cacheContents = os.listdir( self.localCachePath )
|
|
for filename in cacheContents:
|
|
# Make sure we are dealing with a metadata *.json file here
|
|
if not filename.endswith('.json'):
|
|
continue
|
|
|
|
# Construct the full path to the file, then load it's contents
|
|
fullPath = os.path.join( self.localCachePath, filename )
|
|
packageMetadata = json.load( open(fullPath) )
|
|
|
|
# Determine which package we have here and add it to our list of known packages
|
|
self.cachedPackages.append( packageMetadata )
|
|
|
|
# Now we reach out to the remote registry...
|
|
# First establish a connection to Gitlab
|
|
# For reasons unknown using the native OAuth token support in python-gitlab doesn't work here, but oauth:token as a HTTP password does
|
|
if gitlabToken is not None:
|
|
gitlabServer = gitlab.Gitlab( gitlabInstance, private_token=gitlabToken )
|
|
else:
|
|
gitlabServer = gitlab.Gitlab( gitlabInstance )
|
|
|
|
# Then retrieve our registry project
|
|
self.remoteRegistry = gitlabServer.projects.get( gitlabPackageProject )
|
|
|
|
# Now that we have that setup, let's find out what packages our Gitlab package project knows about
|
|
for package in self.remoteRegistry.packages.list( iterator=True ):
|
|
# Grab the version (branch+timestamp) and break it out into the corresponding components
|
|
# We use the version snapshotted at the time the package was created to ensure that we agree with the metadata file
|
|
branch, timestamp = package.version.rsplit('-', 1)
|
|
|
|
# Create the details we will be saving
|
|
packageMetadata = {
|
|
'id': package.id,
|
|
'identifier': package.name,
|
|
'version': package.version,
|
|
'branch': branch,
|
|
'timestamp': int(timestamp),
|
|
}
|
|
|
|
# Save it to the list and move on to the next one
|
|
self.remotePackages.append( packageMetadata )
|
|
|
|
# Convert a branch name into a standardised form
|
|
@staticmethod
|
|
def _normaliseBranchName( branch ):
|
|
# Cleanup a Git branch name for use in our Package Registry
|
|
return branch.replace('/', '-')
|
|
|
|
# Calculate sha256sum of a file on the filesystem
|
|
@staticmethod
|
|
def _calcPackageSha256Sum( archivePath ):
|
|
if hasattr(hashlib, "file_digest"):
|
|
# Python 3.11+ version
|
|
with open(archivePath, "rb") as f:
|
|
packageArchiveDigest = hashlib.file_digest(f, "sha256")
|
|
return packageArchiveDigest.hexdigest()
|
|
|
|
else:
|
|
# Python 3.10 version
|
|
hashObj = hashlib.new("sha256")
|
|
# Read in 64KB chunks and hash them
|
|
with open(archivePath, "rb") as f:
|
|
for chunk in iter(lambda: f.read(65536), b""):
|
|
hashObj.update(chunk)
|
|
return hashObj.hexdigest()
|
|
|
|
# Choose between two branches to determine which one is "newer"
|
|
def _selectNewerBranch( self, firstBranch, secondBranch ):
|
|
# First we need to check whether either of these are 'master'
|
|
# If they are, then we don't need to do anything further as master always wins
|
|
if firstBranch == 'master' or secondBranch == 'master':
|
|
return 'master'
|
|
|
|
# Otherwise we do a version comparison...
|
|
if packaging.version.parse( firstBranch ) > packaging.version.parse( secondBranch ):
|
|
# Then the first branch is newer
|
|
return firstBranch
|
|
|
|
# Otherwise we can assume the second branch wins
|
|
return secondBranch
|
|
|
|
# Retrieve a package matching the supplied parameters
|
|
# Returns a tuple containing a handle to the package archive and a dictionary of metadata surrounding the package
|
|
def retrieve(self, identifier, branch, onlyMetadata = False, userFeedback = False):
|
|
# Get ready to search
|
|
remotePackage = None
|
|
cachedPackage = None
|
|
|
|
if identifier in self.packageAliases:
|
|
identifier = self.packageAliases[identifier]
|
|
|
|
# Prepare a normalised branch name
|
|
normalisedBranch = self._normaliseBranchName( branch )
|
|
|
|
def searchNewestPackage(packagesList, newestPackage):
|
|
resultPackage = newestPackage
|
|
|
|
for entry in packagesList:
|
|
# If the identifier and branch don't match then skip over to the next one
|
|
# We have to use the normalised branch name when doing the comparison, as the entries returned from Gitlab's API will be normalised
|
|
# But the entries in self.cachedPackages use non-normalized branch name
|
|
if entry['identifier'] != identifier or \
|
|
(entry['branch'] != normalisedBranch and entry['branch'] != branch):
|
|
continue
|
|
|
|
# Do we have an existing match?
|
|
if resultPackage is None:
|
|
resultPackage = entry
|
|
|
|
# Is this match newer than our previous match?
|
|
if entry['timestamp'] > resultPackage['timestamp']:
|
|
resultPackage = entry
|
|
|
|
return resultPackage
|
|
|
|
# We start this process by searching through our remote details
|
|
remotePackage = searchNewestPackage(self.remotePackages, None)
|
|
|
|
# if requested explicitly, allow local cache to override the
|
|
# packages in the registry
|
|
allowPackageOverrideByCache = os.environ.get('KDECI_ALLOW_PACKAGE_OVERRIDE_BY_CACHE', '0') in ['1', 'True', 'true']
|
|
if allowPackageOverrideByCache:
|
|
remotePackage = searchNewestPackage(self.cachedPackages, remotePackage)
|
|
|
|
# Before we continue, did we find something?
|
|
# If we found nothing, bow out gracefully here...
|
|
if remotePackage is None:
|
|
return ( None, None, None )
|
|
|
|
# Determine the name we use for the files in the cache as well as the path to the files
|
|
packageName = "{0}-{1}".format( remotePackage['identifier'], normalisedBranch )
|
|
localContentsPath = os.path.join( self.localCachePath, packageName + ".tar" )
|
|
localMetadataPath = os.path.join( self.localCachePath, packageName + ".json" )
|
|
|
|
# Next we check to see if we have a local cache entry
|
|
# If we do, then we can rely on that rather than retrieving a fresh copy from the remote archive
|
|
for entry in self.cachedPackages:
|
|
# Make sure the identifier, branch and timestamp all agree
|
|
# If they do, then we have found the package in the cache
|
|
# (By definition the package cannot be newer as we have the latest remote version - if it is then something is seriously wrong)
|
|
if entry['identifier'] == identifier and entry['branch'] == branch and entry['timestamp'] == remotePackage['timestamp']:
|
|
cachedPackage = entry
|
|
break
|
|
|
|
shouldCheckSha256Sum = (
|
|
# if package override is disallowed, we should compare against remote
|
|
# package unconditionally, not looking at the presence of remote id;
|
|
# if id is missing, it is a bug in this script
|
|
not allowPackageOverrideByCache or
|
|
# if package override is allowed, we should compare only in case the
|
|
# package is really a remote package, i.e. it has an id
|
|
'id' in remotePackage
|
|
)
|
|
|
|
if cachedPackage and shouldCheckSha256Sum:
|
|
remotePkgObject = self.remoteRegistry.packages.get(remotePackage['id'])
|
|
packageSha256Sum = ""
|
|
for file in remotePkgObject.package_files.list(iterator=True):
|
|
if file.file_name == 'archive.tar':
|
|
packageSha256Sum = file.file_sha256
|
|
|
|
localCachePackageSha256Sum = self._calcPackageSha256Sum(localContentsPath)
|
|
|
|
if packageSha256Sum != localCachePackageSha256Sum:
|
|
print ( 'WARNING: the cached package has the same timestamp, but a inconsistent sha256sum,')
|
|
print ( ' skipping the cached package...')
|
|
print (f' package name: {packageName}')
|
|
print (f' remote sha256sum metadata: {packageSha256Sum}')
|
|
print (f' actual package sha256sum: {localCachePackageSha256Sum}')
|
|
cachedPackage = None
|
|
|
|
# If we have a cachedPackage entry then we can assume we have a cache hit and we should use that
|
|
if cachedPackage:
|
|
# Return the contents file and the corresponding metadata
|
|
return ( localContentsPath, cachedPackage, CacheStatus.FromCache )
|
|
|
|
if userFeedback:
|
|
print (f'Downloading package: {packageName}')
|
|
|
|
# Let's retrieve the file if we need to now...
|
|
# First we need to formulate the original version string
|
|
# Download the metadata first...
|
|
response = self.remoteRegistry.generic_packages.download(
|
|
package_name=remotePackage['identifier'],
|
|
package_version=remotePackage['version'],
|
|
file_name="metadata.json"
|
|
)
|
|
|
|
if onlyMetadata:
|
|
return ( None, json.loads(response), CacheStatus.FromRemote )
|
|
|
|
latestMetadata = tempfile.NamedTemporaryFile(delete=False, mode='wb', dir=self.localCachePath)
|
|
latestMetadata.write( response )
|
|
latestMetadata.close()
|
|
|
|
extraDownloadArgs = {}
|
|
|
|
if os.environ.get('KDECI_COMPRESS_PACKAGES_ON_DOWNLOAD', '0') in ['1', 'True', 'true']:
|
|
extraDownloadArgs = {
|
|
'headers' : {'Accept-Encoding': 'gzip, deflate'}
|
|
}
|
|
|
|
# Now the metadata...
|
|
response = self.remoteRegistry.generic_packages.download(
|
|
package_name=remotePackage['identifier'],
|
|
package_version=remotePackage['version'],
|
|
file_name="archive.tar",
|
|
**extraDownloadArgs
|
|
)
|
|
latestContent = tempfile.NamedTemporaryFile(delete=False, mode='wb', dir=self.localCachePath)
|
|
latestContent.write( response )
|
|
latestContent.close()
|
|
|
|
# Remove the old versions first (Windows is odd sometimes)
|
|
# On Windows virtiofs behaves weirdly and doesn't support
|
|
# overwriting existing files
|
|
if os.path.exists( localMetadataPath ):
|
|
os.remove( localMetadataPath )
|
|
if os.path.exists( localContentsPath ):
|
|
os.remove( localContentsPath )
|
|
|
|
# Move both to the cache for future use
|
|
shutil.move( latestContent.name, localContentsPath )
|
|
shutil.move( latestMetadata.name, localMetadataPath )
|
|
|
|
# All done, we can return a tuple of the archive and metadata now
|
|
localContentsPath = localContentsPath
|
|
localMetadataFile = json.load( open(localMetadataPath) )
|
|
return ( localContentsPath, localMetadataFile, CacheStatus.FromRemote )
|
|
|
|
# Takes a dict of projects (with values being the branches), and fetches them and any dependencies they have
|
|
# Returns the complete list for further processing
|
|
def retrieveDependencies(self, dependenciesToFetch, runtime=False, onlyMetadata = False, userFeedback = False):
|
|
# Prepare a list of the details we need to keep
|
|
fetchedPackages = {}
|
|
packageBranches = {}
|
|
|
|
# Create the list of dependencies we will need to resolve
|
|
dependenciesToResolve = copy.deepcopy( dependenciesToFetch )
|
|
|
|
# Go over the list of dependencies we need to process
|
|
# As we go we will add the dependencies of that package to the list, so we just check to see if it is empty
|
|
while len(dependenciesToResolve) > 0:
|
|
# Grab the first one from the list...
|
|
identifier, branch = dependenciesToResolve.popitem()
|
|
|
|
# First, have we already encountered this package?
|
|
# If so there is nothing to do
|
|
if identifier in fetchedPackages:
|
|
continue
|
|
|
|
# Given we have not previously fetched this dependency, we should do so...
|
|
try:
|
|
fetchedPackages[ identifier ] = self.retrieve( identifier, branch, onlyMetadata=onlyMetadata, userFeedback = userFeedback)
|
|
except Exception:
|
|
aliasString = ', alias: {}'.format(self.packageAliases[identifier]) if identifier in self.packageAliases else ''
|
|
raise Exception("Unable to locate requested dependency in the registry: {} (branch: {}{})".format( identifier, branch, aliasString ))
|
|
|
|
# We should also register the branch we have fetched
|
|
packageBranches[ identifier ] = branch
|
|
|
|
# Pull the metadata out...
|
|
packageContents, packageMetadata, cacheStatus = fetchedPackages[ identifier ]
|
|
|
|
# Make sure we have received a usable package
|
|
# Otherwise throw an exception and bail
|
|
if packageMetadata is None:
|
|
aliasString = ', alias: {}'.format(self.packageAliases[identifier]) if identifier in self.packageAliases else ''
|
|
raise Exception("Unable to locate requested dependency in the registry: {} (branch: {}{})".format( identifier, branch, aliasString ))
|
|
|
|
# Go over all the dependencies this package has and build a list to examine
|
|
# If we have been asked to include runtime dependencies, we need to capture them as well
|
|
packageDependencies = {}
|
|
packageDependencies.update( packageMetadata['dependencies'] )
|
|
if runtime and 'runtime-dependencies' in packageMetadata:
|
|
packageDependencies.update( packageMetadata['runtime-dependencies'] )
|
|
|
|
# Go over all the dependencies this package has
|
|
# If we haven't fetched it already, then we should add it to the list to process
|
|
for dependency, dependencyBranch in packageDependencies.items():
|
|
# Is this package one we have seen before?
|
|
# and if it is, then is the branch the same?
|
|
if dependency in packageBranches and packageBranches[ dependency ] == dependencyBranch:
|
|
# Then there is nothing for us to do here - this package has been seen already
|
|
# We don't need to worry about it any further, as it's dependencies will have been resolved already :)
|
|
continue
|
|
|
|
# However, if we have seen it before and the branch isn't the same then we have a problem!
|
|
# This means a project wants two different versions of the same bit of software, which is not going to work
|
|
if dependency in packageBranches:
|
|
# To work around this, we simply assume the newer version is what we should provide
|
|
# This is not ideal, but the Developer can hold the broken pieces if this does not work out
|
|
dependenciesToResolve[ dependency ] = self._selectNewerBranch( packageBranches[ dependency ], dependencyBranch )
|
|
continue
|
|
|
|
# Then we know we are safe to add it to the list
|
|
dependenciesToResolve[ dependency ] = dependencyBranch
|
|
|
|
# Processing complete!
|
|
return list( fetchedPackages.values() )
|
|
|
|
def generateMetadata(self, archivePath, identifier, branch, gitRevision, additionalMetadata = {}):
|
|
# Formulate the remote version number
|
|
# While Git branches may contain slashes, the Gitlab generic package registry does not allow this so we need to normalise it first
|
|
normalisedBranch = self._normaliseBranchName( branch )
|
|
|
|
# Make sure that the archive path we have been given exists
|
|
if not os.path.exists( archivePath ):
|
|
return None
|
|
|
|
# With the branch name normalised, we can now generate the version string to provide to Gitlab's package registry
|
|
packageTimestamp = int( os.path.getmtime( archivePath ) )
|
|
versionForGitlab = "{0}-{1}".format( normalisedBranch, packageTimestamp )
|
|
|
|
# Prepare the metadata, ensuring that the minimum bits of information are being included
|
|
packageMetadata = {
|
|
'identifier': identifier,
|
|
'branch': branch,
|
|
'version': versionForGitlab,
|
|
'timestamp': packageTimestamp,
|
|
'gitRevision': gitRevision,
|
|
'dependencies': {},
|
|
'runtime-dependencies': {},
|
|
}
|
|
# Include the additional information we have been provided
|
|
packageMetadata.update( additionalMetadata )
|
|
|
|
return packageMetadata
|
|
|
|
def upload(self, archivePath, identifier, branch, gitRevision, additionalMetadata = {}):
|
|
# Make sure that the archive path we have been given exists
|
|
if not os.path.exists( archivePath ):
|
|
return False
|
|
|
|
# Prepare the metadata, ensuring that the minimum bits of information are being included
|
|
packageMetadata = self.generateMetadata(archivePath, identifier, branch, gitRevision, additionalMetadata)
|
|
|
|
if packageMetadata is None:
|
|
return False
|
|
|
|
# With the branch name normalised, we can now generate the version string to provide to Gitlab's package registry
|
|
versionForGitlab = packageMetadata['version']
|
|
|
|
# Turn this into a file for upload
|
|
latestMetadata = tempfile.NamedTemporaryFile(delete=False, mode='w')
|
|
json.dump( packageMetadata, latestMetadata, indent = 4 )
|
|
latestMetadata.close()
|
|
|
|
# Start by uploading the archive to Gitlab
|
|
# For the Tarball we cannot use the python-gitlab method as it reads the whole thing into memory
|
|
# We therefore reach into the innards of python-gitlab and do it ourselves directly - bit of a pity that it tries to read it into memory as in theory it should work fine if it did not
|
|
tarballFile = open( archivePath, 'rb' )
|
|
tarballUploadUrl = f"{self.remoteRegistry.generic_packages._computed_path}/{identifier}/{versionForGitlab}/archive.tar"
|
|
package = self.remoteRegistry.manager.gitlab.http_put(tarballUploadUrl, post_data=tarballFile, raw=True)
|
|
|
|
# Then upload the metadata
|
|
package = self.remoteRegistry.generic_packages.upload(
|
|
package_name=identifier,
|
|
package_version=versionForGitlab,
|
|
file_name="metadata.json",
|
|
path=latestMetadata.name
|
|
)
|
|
|
|
# Cleanup!
|
|
os.remove( latestMetadata.name )
|
|
|
|
# All done now!
|
|
return True
|