419 lines
16 KiB
Python
419 lines
16 KiB
Python
# SPDX-License-Identifier: BSD-2-Clause
|
|
# SPDX-FileCopyrightText: 2023 KDE e.V.
|
|
# SPDX-FileContributor: Ingo Klöcker <dev@ingo-kloecker.de>
|
|
|
|
import base64
|
|
import configparser
|
|
import dataclasses
|
|
import logging
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from typing import Optional
|
|
|
|
from sftpnotary import config, projects, util
|
|
from sftpnotary.core import Job, Request, Response, TaskProcessor, Worker
|
|
from sftpnotary.exceptions import Error
|
|
from sftpnotary.sftp import SFTPClient, SFTPError
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class SignFlatpakRequest(Request):
|
|
fileName: Optional[str] = None
|
|
localeBundleFileName: Optional[str] = None
|
|
debugBundleFileName: Optional[str] = None
|
|
|
|
def validate(self):
|
|
util.validateFileName(self.fileName, strict=True)
|
|
if self.localeBundleFileName:
|
|
util.validateFileName(self.localeBundleFileName, strict=True)
|
|
if self.debugBundleFileName:
|
|
util.validateFileName(self.debugBundleFileName, strict=True)
|
|
|
|
|
|
SignFlatpakResponse = Response
|
|
|
|
|
|
def fileNameOrNone(path: Optional[Path]):
|
|
if path is not None:
|
|
return path.name
|
|
return None
|
|
|
|
|
|
class SignFlatpakJob(Job):
|
|
def __init__(
|
|
self,
|
|
sftp: SFTPClient,
|
|
flatpakFilePath: Path,
|
|
localeBundlePath: Optional[Path] = None,
|
|
debugBundlePath: Optional[Path] = None,
|
|
token: Optional[str] = None,
|
|
projectPath: Optional[str] = None,
|
|
branch: Optional[str] = None,
|
|
):
|
|
super().__init__(sftp)
|
|
|
|
self.flatpakBundlePath = flatpakFilePath
|
|
self.localeBundlePath = localeBundlePath
|
|
self.debugBundlePath = debugBundlePath
|
|
|
|
self.request = SignFlatpakRequest(
|
|
fileName=flatpakFilePath.name,
|
|
localeBundleFileName=fileNameOrNone(localeBundlePath),
|
|
debugBundleFileName=fileNameOrNone(debugBundlePath),
|
|
token=token,
|
|
projectPath=projectPath,
|
|
branch=branch,
|
|
)
|
|
|
|
def start(self):
|
|
self.sftp.upload(self.flatpakBundlePath, self.task.path() / self.request.fileName)
|
|
if self.localeBundlePath:
|
|
self.sftp.upload(self.localeBundlePath, self.task.path() / self.request.localeBundleFileName)
|
|
if self.debugBundlePath:
|
|
self.sftp.upload(self.debugBundlePath, self.task.path() / self.request.debugBundleFileName)
|
|
self.task.putRequest(self.request)
|
|
self.taskQueue.commitTask(self.task)
|
|
|
|
def waitForCompletion(self):
|
|
self.taskQueue.waitForCompletion(self.task)
|
|
log.debug(f"Fetching response of task '{self.task.id}' ...")
|
|
response = None
|
|
try:
|
|
response = self.task.fetchResponse(SignFlatpakResponse)
|
|
except SFTPError as e:
|
|
log.error("Error: %s", e)
|
|
if response is None:
|
|
log.error("Error: Got no response. Task failed.")
|
|
else:
|
|
self.success = True
|
|
self.downloadLogFiles(self.flatpakBundlePath.parent)
|
|
self.taskQueue.removeTask(self.task)
|
|
|
|
|
|
def getRefFromFlatpakBundleHeader(header):
|
|
# This code is based on the code of the function `build_bundle()` in
|
|
# `app/flatpak-builtins-build-bundle.c` of flatpak. It reads the "ref" of
|
|
# the flatpak bundle from a serialized GVariant array.
|
|
if len(header) < 32:
|
|
raise Exception("Truncated or invalid Flatpak bundle (smaller than 32 bytes)")
|
|
# Flatpak bundles start with "flatpak\0" 01 00 89 e5 (or e5 89 00 01 in big endian)
|
|
# where the 01 can be used as a version, so that we ignore it
|
|
if not header.startswith(b"flatpak\0"):
|
|
raise Exception("Invalid Flatpak bundle header (doesn't start with 'flatpak')")
|
|
if header[9:12] != b"\x00\x89\xe5" and header[8:11] != b"\xe5\x89\x00":
|
|
raise Exception("Invalid Flatpak bundle header (wrong magic number)")
|
|
refKey = header.find(b"ref\0", 16)
|
|
if refKey == -1:
|
|
raise Exception("Start of 'ref' entry not found")
|
|
refValueStart = refKey + 8 # 0-terminated "ref" padded to 8 bytes
|
|
refValueEnd = header.find(b"\0", refValueStart)
|
|
if refValueEnd == -1:
|
|
raise Exception("End of 'ref' value not found")
|
|
ref = header[refValueStart:refValueEnd].decode(errors="replace")
|
|
return ref
|
|
|
|
|
|
def getRefFromFlatpakBundle(flatpakFilePath):
|
|
with open(flatpakFilePath, "rb") as f:
|
|
b = f.read(256)
|
|
try:
|
|
return getRefFromFlatpakBundleHeader(b)
|
|
except Exception as e:
|
|
log.debug("Error:", exc_info=e)
|
|
return None
|
|
|
|
|
|
def getAppIdFromRef(ref):
|
|
if ref is None:
|
|
raise Error("Failed to get application ID from flatpak bundle")
|
|
refParts = ref.split("/")
|
|
return refParts[1] if len(refParts) > 1 else ""
|
|
|
|
|
|
def getBranchFromRef(ref):
|
|
if ref is None:
|
|
raise Error("Failed to get branch from flatpak bundle")
|
|
refParts = ref.split("/")
|
|
return refParts[3] if len(refParts) > 3 else ""
|
|
|
|
|
|
def generateFlatpakref(cwd, appId, branch, repoUrlPrefix, repoName, runtimeRepoUrl, publicKeyPath, isRuntime):
|
|
# See https://docs.flatpak.org/en/latest/repositories.html#flatpakref-files
|
|
flatpakrefConfig = configparser.ConfigParser()
|
|
# overwrite optionxform to avoid key conversion to lowercase
|
|
flatpakrefConfig.optionxform = lambda option: option
|
|
ref = {}
|
|
ref["Name"] = appId
|
|
ref["Branch"] = branch
|
|
ref["Title"] = f"{appId} from {repoName}"
|
|
ref["SuggestRemoteName"] = repoName
|
|
ref["RuntimeRepo"] = runtimeRepoUrl
|
|
ref["IsRuntime"] = "true" if isRuntime else "false"
|
|
|
|
if not repoUrlPrefix.endswith("/"):
|
|
repoUrlPrefix += "/"
|
|
|
|
ref["Url"] = f"{repoUrlPrefix}{repoName}/"
|
|
|
|
with open(publicKeyPath, "rb") as f:
|
|
encoded = base64.b64encode(f.read())
|
|
ref["GPGKey"] = encoded.decode("utf-8")
|
|
|
|
flatpakrefConfig["Flatpak Ref"] = ref
|
|
|
|
filename = f"{appId}.flatpakref"
|
|
with open(cwd / filename, "w") as outputfile:
|
|
flatpakrefConfig.write(outputfile)
|
|
|
|
return filename
|
|
|
|
|
|
def generateFlatpakrepo(cwd, repoUrlPrefix, repoName, iconFileName, publicKeyPath):
|
|
# See https://docs.flatpak.org/en/latest/hosting-a-repository.html#flatpakrepo-files
|
|
flatpakrepoConfig = configparser.ConfigParser()
|
|
# overwrite optionxform to avoid key conversion to lowercase
|
|
flatpakrepoConfig.optionxform = lambda option: option
|
|
ref = {}
|
|
ref["Title"] = repoName
|
|
ref["Homepage"] = "https://kde.org/"
|
|
ref["Comment"] = f"KDE {repoName} Flatpak repository"
|
|
ref["Description"] = f"KDE {repoName} Flatpak repository"
|
|
|
|
if not repoUrlPrefix.endswith("/"):
|
|
repoUrlPrefix += "/"
|
|
|
|
ref["Url"] = f"{repoUrlPrefix}{repoName}/"
|
|
if iconFileName:
|
|
ref["Icon"] = f"{repoUrlPrefix}{repoName}/{iconFileName}"
|
|
|
|
with open(publicKeyPath, "rb") as f:
|
|
encoded = base64.b64encode(f.read())
|
|
ref["GPGKey"] = encoded.decode("utf-8")
|
|
|
|
flatpakrepoConfig["Flatpak Repo"] = ref
|
|
|
|
filename = f"{repoName}.flatpakrepo"
|
|
with open(cwd / filename, "w") as outputfile:
|
|
flatpakrepoConfig.write(outputfile)
|
|
|
|
return filename
|
|
|
|
|
|
class SignFlatpakProcessor(TaskProcessor):
|
|
requestClass = SignFlatpakRequest
|
|
|
|
def getProjectSetting(self, name, required=True):
|
|
value = projects.settings.get(self.request.projectPath, self.request.branch, name)
|
|
if required and not value:
|
|
projectSettingsLocation = config.settings.get(
|
|
"General",
|
|
"ProjectSettings",
|
|
"flatpaksigner-projects.yaml next to the service's config file on the server",
|
|
)
|
|
raise Error(
|
|
f"Project setting '{name}' not set for branch {self.request.branch} "
|
|
f"of project {self.request.projectPath}. Check configuration in {projectSettingsLocation}."
|
|
)
|
|
return value
|
|
|
|
def doProcess(self):
|
|
repoName = self.getProjectSetting("repository")
|
|
signingKey = self.getProjectSetting("signingkey")
|
|
|
|
remoteFilePath = self.task.path() / self.request.fileName
|
|
with TemporaryDirectory() as workPathName:
|
|
workPath = Path(workPathName)
|
|
# use simple file names locally to avoid issues with user provided file names
|
|
localFilePath = workPath / "unsigned.flatpak"
|
|
self.sftp.download(remoteFilePath, localFilePath)
|
|
self.verifyApplicationId(localFilePath)
|
|
localLocaleBundlePath = None
|
|
if self.request.localeBundleFileName:
|
|
remoteLocaleBundlePath = self.task.path() / self.request.localeBundleFileName
|
|
localLocaleBundlePath = workPath / "locale.flatpak"
|
|
self.sftp.download(remoteLocaleBundlePath, localLocaleBundlePath)
|
|
self.verifyApplicationId(localLocaleBundlePath, expectedSuffix=".Locale")
|
|
localDebugBundlePath = None
|
|
if self.request.debugBundleFileName:
|
|
remoteDebugBundlePath = self.task.path() / self.request.debugBundleFileName
|
|
localDebugBundlePath = workPath / "debug.flatpak"
|
|
self.sftp.download(remoteDebugBundlePath, localDebugBundlePath)
|
|
self.verifyApplicationId(localDebugBundlePath, expectedSuffix=".Debug")
|
|
self.signFlatpak(localFilePath, localLocaleBundlePath, localDebugBundlePath, repoName, signingKey)
|
|
|
|
def verifyApplicationId(self, flatpakFilePath, expectedSuffix=None):
|
|
ref = getRefFromFlatpakBundle(flatpakFilePath)
|
|
applicationId = getAppIdFromRef(ref)
|
|
allowedApplicationIds = self.getProjectSetting("applicationid")
|
|
if isinstance(allowedApplicationIds, str):
|
|
allowedApplicationIds = [allowedApplicationIds]
|
|
if expectedSuffix:
|
|
# append expected suffix to allowed application IDs and make them AppStream compatible
|
|
allowedApplicationIds = [appId.replace("-", "_") + expectedSuffix for appId in allowedApplicationIds]
|
|
if applicationId not in allowedApplicationIds:
|
|
projectSettingsLocation = config.settings.get(
|
|
"General",
|
|
"ProjectSettings",
|
|
"flatpaksigner-projects.yaml next to the service's config file on the server",
|
|
)
|
|
raise Error(
|
|
f"The application ID of the Flatpak ({applicationId}) did not match any of the allowed application IDs "
|
|
f"({', '.join(allowedApplicationIds)}). Check configuration in {projectSettingsLocation}."
|
|
)
|
|
|
|
def signFlatpak(self, flatpakFilePath, localeBundlePath, debugBundlePath, repoName, signingKey):
|
|
workingDirectory = flatpakFilePath.parent
|
|
flatpakFileName = flatpakFilePath.name
|
|
remoteRepoPath = Path(config.settings.get("FlatpakSigning", "RemoteRepoPrefix")) / repoName
|
|
localRepoPath = workingDirectory / repoName
|
|
|
|
# 1. Fetch the remote repository
|
|
# rsync the remote base path including only the wanted project repo;
|
|
# this way rsync doesn't fail if there isn't a repo for the project yet
|
|
command = [
|
|
"rsync",
|
|
"-Ha",
|
|
"--delete",
|
|
f"--include={remoteRepoPath.name}",
|
|
f"--include={remoteRepoPath.name}/**",
|
|
"--exclude=*",
|
|
f"{remoteRepoPath.parent}/",
|
|
f"{localRepoPath.parent}/",
|
|
]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# 1.5 Create a new local repo if there wasn't a remote repo
|
|
if not localRepoPath.exists():
|
|
command = [
|
|
"ostree",
|
|
"init",
|
|
f"--repo={localRepoPath}",
|
|
"--mode",
|
|
"archive-z2",
|
|
]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# 2. Import the file bundle into the local repository signing the commit
|
|
command = [
|
|
"flatpak",
|
|
"build-import-bundle",
|
|
"--update-appstream",
|
|
f"--gpg-sign={signingKey}",
|
|
f"{localRepoPath}",
|
|
f"{flatpakFileName}",
|
|
]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# 2.1 Import the locale bundle into the local repository signing the commit
|
|
if localeBundlePath:
|
|
command = [
|
|
"flatpak",
|
|
"build-import-bundle",
|
|
"--update-appstream",
|
|
f"--gpg-sign={signingKey}",
|
|
f"{localRepoPath}",
|
|
f"{localeBundlePath.name}",
|
|
]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# 2.2 Import the debug bundle into the local repository signing the commit
|
|
if debugBundlePath:
|
|
command = [
|
|
"flatpak",
|
|
"build-import-bundle",
|
|
"--update-appstream",
|
|
f"--gpg-sign={signingKey}",
|
|
f"{localRepoPath}",
|
|
f"{debugBundlePath.name}",
|
|
]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# 3. Generate static deltas for all references
|
|
command = [
|
|
"flatpak",
|
|
"build-update-repo",
|
|
"--prune",
|
|
"--prune-depth=10",
|
|
"--generate-static-deltas",
|
|
f"--gpg-sign={signingKey}",
|
|
f"{localRepoPath}",
|
|
]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# 4. Push changes to the remote repository
|
|
command = [
|
|
"rsync",
|
|
"-Ha",
|
|
"--delete",
|
|
f"{localRepoPath}/",
|
|
f"{remoteRepoPath}/",
|
|
]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# 5. Create a .flatpakref file
|
|
ref = getRefFromFlatpakBundle(flatpakFilePath)
|
|
applicationId = getAppIdFromRef(ref)
|
|
branch = getBranchFromRef(ref)
|
|
repoUrlPrefix = config.settings.get("FlatpakSigning", "RepoURLPrefix")
|
|
publicKey = config.settings.getPath("FlatpakSigning", "RepoKeysLocation") / f"{signingKey}.gpg"
|
|
runtimeRepoUrl = projects.settings.get(self.request.projectPath, self.request.branch, "runtimerepourl")
|
|
isRuntime = projects.settings.getBool(self.request.projectPath, self.request.branch, "isruntime", False)
|
|
|
|
refFileName = generateFlatpakref(
|
|
cwd=workingDirectory,
|
|
appId=applicationId,
|
|
branch=branch,
|
|
repoUrlPrefix=repoUrlPrefix,
|
|
repoName=repoName,
|
|
runtimeRepoUrl=runtimeRepoUrl,
|
|
publicKeyPath=publicKey,
|
|
isRuntime=isRuntime,
|
|
)
|
|
|
|
# 6. Push .flatpakref to the server
|
|
command = [
|
|
"rsync",
|
|
"-Ha",
|
|
f"{workingDirectory}/{refFileName}",
|
|
f"{remoteRepoPath}/{refFileName}",
|
|
]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# 7. Upload icon
|
|
iconPath = config.settings.getPath("FlatpakSigning", "RepoIconFile")
|
|
if iconPath is None:
|
|
iconFileName = ""
|
|
else:
|
|
iconFileName = "icon.svg"
|
|
|
|
command = [
|
|
"rsync",
|
|
"-Ha",
|
|
iconPath,
|
|
f"{remoteRepoPath}/{iconFileName}",
|
|
]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# 8. Create a .flatpakrepo file
|
|
repoFileName = generateFlatpakrepo(
|
|
cwd=workingDirectory,
|
|
repoUrlPrefix=repoUrlPrefix,
|
|
repoName=repoName,
|
|
iconFileName=iconFileName,
|
|
publicKeyPath=publicKey,
|
|
)
|
|
|
|
# 9. Push .flatpakrepo to the server
|
|
command = [
|
|
"rsync",
|
|
"-Ha",
|
|
f"{workingDirectory}/{repoFileName}",
|
|
f"{remoteRepoPath}/{repoFileName}",
|
|
]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
|
|
class SignFlatpakWorker(Worker):
|
|
processorClass = SignFlatpakProcessor
|