324 lines
16 KiB
Python
324 lines
16 KiB
Python
import os
|
|
import copy
|
|
import json
|
|
import shutil
|
|
import tarfile
|
|
import tempfile
|
|
from typing import cast, Dict, List, Optional, Tuple
|
|
import gitlab
|
|
import packaging.version
|
|
|
|
class Registry(object):
|
|
|
|
# Record all the details we need for later use
|
|
def __init__(self, localCachePath: str, gitlabInstance: str, gitlabToken: Optional[str], gitlabPackageProject: str) -> None:
|
|
# Store all the details we have been given for later use
|
|
self.localCachePath = localCachePath
|
|
|
|
# First prepare to gather details from both the cache and the remote registry
|
|
self.cachedPackages = []
|
|
self.remotePackages = []
|
|
|
|
# Make sure the local cache path exists
|
|
if not os.path.exists( self.localCachePath ):
|
|
os.makedirs( self.localCachePath )
|
|
|
|
# Determine what we have locally first
|
|
cacheContents = os.listdir( self.localCachePath )
|
|
for filename in cacheContents:
|
|
# Make sure we are dealing with a metadata *.json file here
|
|
if not filename.endswith('.json'):
|
|
continue
|
|
|
|
# Construct the full path to the file, then load it's contents
|
|
fullPath = os.path.join( self.localCachePath, filename )
|
|
packageMetadata = json.load( open(fullPath) )
|
|
|
|
# Determine which package we have here and add it to our list of known packages
|
|
self.cachedPackages.append( packageMetadata )
|
|
|
|
# Now we reach out to the remote registry...
|
|
# First establish a connection to Gitlab
|
|
# For reasons unknown using the native OAuth token support in python-gitlab doesn't work here, but oauth:token as a HTTP password does
|
|
if gitlabToken is not None:
|
|
gitlabServer = gitlab.Gitlab( gitlabInstance, private_token=gitlabToken )
|
|
else:
|
|
gitlabServer = gitlab.Gitlab( gitlabInstance )
|
|
|
|
# Then retrieve our registry project
|
|
self.remoteRegistry = gitlabServer.projects.get( gitlabPackageProject )
|
|
|
|
# Now that we have that setup, let's find out what packages our Gitlab package project knows about
|
|
for package in self.remoteRegistry.packages.list( iterator=True ):
|
|
# Grab the version (branch+timestamp) and break it out into the corresponding components
|
|
# We use the version snapshotted at the time the package was created to ensure that we agree with the metadata file
|
|
branch, timestamp = package.version.rsplit('-', 1)
|
|
|
|
# Create the details we will be saving
|
|
packageMetadata = {
|
|
'identifier': package.name,
|
|
'version': package.version,
|
|
'branch': branch,
|
|
'timestamp': int(timestamp),
|
|
}
|
|
|
|
# Save it to the list and move on to the next one
|
|
self.remotePackages.append( packageMetadata )
|
|
|
|
# Convert a branch name into a standardised form
|
|
def _normaliseBranchName( self, branch: str ) -> str:
|
|
# Cleanup a Git branch name for use in our Package Registry
|
|
return branch.replace('/', '-')
|
|
|
|
# Choose between two branches to determine which one is "newer"
|
|
def _selectNewerBranch( self, firstBranch: str, secondBranch: str ) -> str:
|
|
# First we need to check whether either of these are 'master'
|
|
# If they are, then we don't need to do anything further as master always wins
|
|
if firstBranch == 'master' or secondBranch == 'master':
|
|
return 'master'
|
|
|
|
# Otherwise we do a version comparison...
|
|
if packaging.version.parse( firstBranch ) > packaging.version.parse( secondBranch ):
|
|
# Then the first branch is newer
|
|
return firstBranch
|
|
|
|
# Otherwise we can assume the second branch wins
|
|
return secondBranch
|
|
|
|
# Retrieve a package matching the supplied parameters
|
|
# Returns a tuple containing a handle to the package archive and a dictionary of metadata surrounding the package
|
|
def retrieve(self, identifier: str, branch: str) -> Tuple[str, Dict[str, Dict[str, str]]]:
|
|
# Get ready to search
|
|
remotePackage = None
|
|
cachedPackage = None
|
|
|
|
# Prepare a normalised branch name
|
|
normalisedBranch = self._normaliseBranchName( branch )
|
|
|
|
# We start this process by searching through our remote details
|
|
for entry in self.remotePackages:
|
|
# If the identifier and branch don't match then skip over to the next one
|
|
# We have to use the normalised branch name when doing the comparison, as the entries returned from Gitlab's API will be normalised
|
|
if entry['identifier'] != identifier or entry['branch'] != normalisedBranch:
|
|
continue
|
|
|
|
# Do we have an existing match?
|
|
if remotePackage is None:
|
|
remotePackage = entry
|
|
|
|
# Is this match newer than our previous match?
|
|
if entry['timestamp'] > remotePackage['timestamp']:
|
|
remotePackage = entry
|
|
|
|
# Before we continue, did we find something?
|
|
if remotePackage is None:
|
|
raise Exception(f"Package {identifier} was not found in branch {branch}")
|
|
|
|
# Determine the name we use for the files in the cache as well as the path to the files
|
|
packageName = "{0}-{1}".format( remotePackage['identifier'], remotePackage['branch'] )
|
|
localContentsPath = os.path.join( self.localCachePath, packageName + ".tar" )
|
|
localMetadataPath = os.path.join( self.localCachePath, packageName + ".json" )
|
|
|
|
# Next we check to see if we have a local cache entry
|
|
# If we do, then we can rely on that rather than retrieving a fresh copy from the remote archive
|
|
for entry in self.cachedPackages:
|
|
# Make sure the identifier, branch and timestamp all agree
|
|
# If they do, then we have found the package in the cache
|
|
# (By definition the package cannot be newer as we have the latest remote version - if it is then something is seriously wrong)
|
|
if entry['identifier'] == identifier and entry['branch'] == branch and entry['timestamp'] == remotePackage['timestamp']:
|
|
cachedPackage = entry
|
|
|
|
# If we have a cachedPackage entry then we can assume we have a cache hit and we should use that
|
|
if cachedPackage:
|
|
# Return the contents file and the corresponding metadata
|
|
return ( localContentsPath, cachedPackage )
|
|
|
|
# Let's retrieve the file if we need to now...
|
|
# First we need to formulate the original version string
|
|
# Download the archive first...
|
|
response = self.remoteRegistry.generic_packages.download(
|
|
package_name=remotePackage['identifier'],
|
|
package_version=remotePackage['version'],
|
|
file_name="archive.tar"
|
|
)
|
|
if not isinstance(response, bytes):
|
|
raise ValueError("'response' was expected to be of type 'bytes'.")
|
|
latestContent = tempfile.NamedTemporaryFile(delete=False, mode='wb', dir=self.localCachePath)
|
|
latestContent.write( response )
|
|
latestContent.close()
|
|
|
|
# Now the metadata file...
|
|
response = self.remoteRegistry.generic_packages.download(
|
|
package_name=remotePackage['identifier'],
|
|
package_version=remotePackage['version'],
|
|
file_name="metadata.json"
|
|
)
|
|
if not isinstance(response, bytes):
|
|
raise ValueError("'response' was expected to be of type 'bytes'.")
|
|
latestMetadata = tempfile.NamedTemporaryFile(delete=False, mode='wb', dir=self.localCachePath)
|
|
latestMetadata.write( response )
|
|
latestMetadata.close()
|
|
|
|
# Remove the old versions first (Windows is odd sometimes)
|
|
if os.path.exists( localMetadataPath ):
|
|
os.remove( localMetadataPath )
|
|
if os.path.exists( localContentsPath ):
|
|
os.remove( localContentsPath )
|
|
|
|
# Move both to the cache for future use
|
|
shutil.move( latestContent.name, localContentsPath )
|
|
shutil.move( latestMetadata.name, localMetadataPath )
|
|
|
|
# All done, we can return a tuple of the archive and metadata now
|
|
localMetadataFile = json.load( open(localMetadataPath) )
|
|
return ( localContentsPath, localMetadataFile )
|
|
|
|
# Takes a dict of projects (with values being the branches), and fetches them and any dependencies they have
|
|
# Returns the complete list for further processing
|
|
def retrieveDependencies(self, dependenciesToFetch: Dict[str, str], dependenciesToIgnore: Optional[List[str]]=None, runtime: bool=False) -> List[Tuple[str, Dict[str, Dict[str,str]]]]:
|
|
# Prepare a list of the details we need to keep
|
|
fetchedPackages: Dict[str, Tuple[str, Dict[str, Dict[str,str]]]] = {}
|
|
packageBranches: Dict[str, str] = {}
|
|
|
|
# Make sure the dependencies to be ignored list is initialised
|
|
if dependenciesToIgnore is None:
|
|
dependenciesToIgnore = []
|
|
|
|
# Create the list of dependencies we will need to resolve
|
|
dependenciesToResolve = copy.deepcopy( dependenciesToFetch )
|
|
|
|
# Go over the list of dependencies we need to process
|
|
# As we go we will add the dependencies of that package to the list, so we just check to see if it is empty
|
|
while len(dependenciesToResolve) > 0:
|
|
# Grab the first one from the list...
|
|
identifier, branch = dependenciesToResolve.popitem()
|
|
|
|
# First, have we already encountered this package?
|
|
# Otherwise, is it on the list of packages we should be ignoring?
|
|
# If so there is nothing to do
|
|
if identifier in fetchedPackages or identifier in dependenciesToIgnore:
|
|
continue
|
|
|
|
# Given we have not previously fetched this dependency, we should do so...
|
|
try:
|
|
fetchedPackages[ identifier ] = self.retrieve( identifier, branch )
|
|
except Exception:
|
|
raise Exception("Unable to locate requested dependency in the registry: {} (branch: {})".format( identifier, branch ))
|
|
|
|
# We should also register the branch we have fetched
|
|
packageBranches[ identifier ] = branch
|
|
|
|
# Pull the metadata out...
|
|
_packageContents, packageMetadata = fetchedPackages[ identifier ]
|
|
|
|
# Make sure we have received a usable package
|
|
# Otherwise throw an exception and bail
|
|
if packageMetadata is None:
|
|
raise Exception("Unable to locate requested dependency in the registry: {} (branch: {})".format( identifier, branch ))
|
|
|
|
# Go over all the dependencies this package has and build a list to examine
|
|
# If we have been asked to include runtime dependencies, we need to capture them as well
|
|
packageDependencies = {}
|
|
packageDependencies.update( packageMetadata['dependencies'] )
|
|
if runtime and 'runtime-dependencies' in packageMetadata:
|
|
packageDependencies.update( packageMetadata['runtime-dependencies'] )
|
|
|
|
# Go over all the dependencies this package has
|
|
# If we haven't fetched it already, then we should add it to the list to process
|
|
for dependency, dependencyBranch in packageDependencies.items():
|
|
# Is this package one we have seen before?
|
|
# and if it is, then is the branch the same?
|
|
if dependency in packageBranches and packageBranches[ dependency ] == dependencyBranch:
|
|
# Then there is nothing for us to do here - this package has been seen already
|
|
# We don't need to worry about it any further, as it's dependencies will have been resolved already :)
|
|
continue
|
|
|
|
# However, if we have seen it before and the branch isn't the same then we have a problem!
|
|
# This means a project wants two different versions of the same bit of software, which is not going to work
|
|
if dependency in packageBranches:
|
|
# To work around this, we simply assume the newer version is what we should provide
|
|
# This is not ideal, but the Developer can hold the broken pieces if this does not work out
|
|
print("WARNING: Different versions of {} have been requested: {} vs. {} (requested by: {})".format( dependency, packageBranches[ dependency ], dependencyBranch, identifier ))
|
|
dependenciesToResolve[ dependency ] = self._selectNewerBranch( packageBranches[ dependency ], dependencyBranch )
|
|
continue
|
|
|
|
# Then we know we are safe to add it to the list
|
|
dependenciesToResolve[ dependency ] = dependencyBranch
|
|
|
|
# Processing complete!
|
|
return list( fetchedPackages.values() )
|
|
|
|
def upload(self, archivePath: str, identifier: str, branch: str, gitRevision: str, additionalMetadata: Optional[Dict[str, Dict[str,str]]] = None, additionalFileName: Optional[str] = None) -> bool:
|
|
# Formulate the remote version number
|
|
# While Git branches may contain slashes, the Gitlab generic package registry does not allow this so we need to normalise it first
|
|
normalisedBranch = self._normaliseBranchName( branch )
|
|
|
|
# Make sure that the archive path we have been given exists
|
|
if not os.path.exists( archivePath ):
|
|
return False
|
|
|
|
# only pass an additionalFileName if it exists
|
|
assert not additionalFileName or os.path.exists(additionalFileName)
|
|
|
|
# With the branch name normalised, we can now generate the version string to provide to Gitlab's package registry
|
|
packageTimestamp = int( os.path.getmtime( archivePath ) )
|
|
versionForGitlab = "{0}-{1}".format( normalisedBranch, packageTimestamp )
|
|
|
|
# Prepare the metadata, ensuring that the minimum bits of information are being included
|
|
packageMetadata = {
|
|
'identifier': identifier,
|
|
'branch': branch,
|
|
'version': versionForGitlab,
|
|
'timestamp': packageTimestamp,
|
|
'gitRevision': gitRevision,
|
|
'dependencies': {},
|
|
'runtime-dependencies': {},
|
|
}
|
|
# Include the additional information we have been provided
|
|
if additionalMetadata is None:
|
|
additionalMetadata = {}
|
|
packageMetadata.update( additionalMetadata )
|
|
# Turn this into a file for upload
|
|
latestMetadata = tempfile.NamedTemporaryFile(delete=False, mode='w')
|
|
json.dump( packageMetadata, latestMetadata, indent = 4 )
|
|
latestMetadata.close()
|
|
|
|
# Start by uploading the archive to Gitlab
|
|
# For the Tarball we cannot use the python-gitlab method as it reads the whole thing into memory
|
|
# We therefore reach into the innards of python-gitlab and do it ourselves directly - bit of a pity that it tries to read it into memory as in theory it should work fine if it did not
|
|
with open( archivePath, 'rb' ) as file:
|
|
# The type annotation for post_data in gitlab.http_put omits
|
|
# BufferedReader, so we tell it that the BufferedReader has type
|
|
# `bytes`
|
|
tarballFile = cast(bytes, file)
|
|
tarballUploadUrl = f"{self.remoteRegistry.generic_packages._computed_path}/{identifier}/{versionForGitlab}/archive.tar"
|
|
self.remoteRegistry.manager.gitlab.http_put(tarballUploadUrl, post_data=tarballFile, raw=True)
|
|
|
|
# Then upload the metadata
|
|
self.remoteRegistry.generic_packages.upload(
|
|
package_name=identifier,
|
|
package_version=versionForGitlab,
|
|
file_name="metadata.json",
|
|
path=latestMetadata.name
|
|
)
|
|
|
|
if additionalFileName is not None:
|
|
self.remoteRegistry.generic_packages.upload(
|
|
package_name=identifier,
|
|
package_version=versionForGitlab,
|
|
file_name=os.path.basename(additionalFileName),
|
|
path=additionalFileName
|
|
)
|
|
|
|
# Cleanup!
|
|
os.remove( latestMetadata.name )
|
|
|
|
# All done now!
|
|
return True
|
|
|
|
|
|
# Retrieve a package matching the supplied parameters and install it in destination folder
|
|
def retrieveInstall(self, identifier: str, branch: str, destination: str) -> None:
|
|
local_contents_path, _ = self.retrieve(identifier, branch)
|
|
archive = tarfile.open(local_contents_path, mode="r")
|
|
archive.extractall(destination, filter='fully_trusted')
|