# SPDX-License-Identifier: BSD-2-Clause # SPDX-FileCopyrightText: 2023 KDE e.V. # SPDX-FileContributor: Ingo Klöcker import dataclasses import logging import os import time import zipfile from pathlib import Path from tempfile import TemporaryDirectory from typing import List, Optional from sftpnotary import apkutils, config, projects, util from sftpnotary.core import Job, Request, Response, TaskProcessor, Worker from sftpnotary.exceptions import Error, InvalidFileName from sftpnotary.sftp import SFTPClient, SFTPError log = logging.getLogger(__name__) @dataclasses.dataclass class PublishOnFDroidRequest(Request): apkFileNames: Optional[List[str]] = None fastlaneFileName: Optional[str] = None def validate(self): if self.apkFileNames is None: raise InvalidFileName() for apkFileName in self.apkFileNames: util.validateFileName(apkFileName, strict=True) util.validateFileName(self.fastlaneFileName, strict=True) PublishOnFDroidResponse = Response class PublishOnFDroidJob(Job): def __init__( self, sftp: SFTPClient, apkFilePaths: List[Path], fastlaneFilePath: Path, token: Optional[str] = None, projectPath: Optional[str] = None, branch: Optional[str] = None, ): super().__init__(sftp) self.apkFilePaths = apkFilePaths self.fastlaneFilePath = fastlaneFilePath self.request = PublishOnFDroidRequest( apkFileNames=[], fastlaneFileName=None, token=token, projectPath=projectPath, branch=branch, ) self.resultPath = self.apkFilePaths[0].parent def start(self): for apkFilePath in self.apkFilePaths: apkFileName = apkFilePath.name self.sftp.upload(apkFilePath, self.task.path() / apkFileName) self.request.apkFileNames.append(apkFileName) fastlaneFileName = self.fastlaneFilePath.name self.sftp.upload(self.fastlaneFilePath, self.task.path() / fastlaneFileName) self.request.fastlaneFileName = fastlaneFileName self.task.putRequest(self.request) self.taskQueue.commitTask(self.task) def waitForCompletion(self): self.taskQueue.waitForCompletion(self.task) log.debug(f"Fetching response of task '{self.task.id}' ...") response = None try: response = self.task.fetchResponse(PublishOnFDroidResponse) except SFTPError as e: log.error("Error: %s", e) if response is None: log.error("Error: Got no response. Task failed.") else: self.success = True self.downloadLogFiles(self.resultPath) self.taskQueue.removeTask(self.task) def removeOldFiles(path, maxAgeInDays): oldFilesPurgeThreshold = time.time() - (3600 * 24 * maxAgeInDays) log.debug(f"Removing files that are older than {maxAgeInDays} days.") for root, dirs, files in os.walk(path, topdown=False, onerror=None, followlinks=False): # first iterate over all files for name in files: filePath = Path(root) / name # ignore anything that's not a regular file if filePath.is_symlink() or not filePath.is_file(): log.debug(f"Ignoring {filePath} which is not a regular file.") continue # purge ancient files if os.path.getmtime(filePath) < oldFilesPurgeThreshold: log.info(f"Removing outdated file: {filePath}") filePath.unlink() # then delete empty subfolders for name in dirs: filePath = Path(root) / name try: # try to remove the folder filePath.rmdir() log.info(f"Removed empty folder: {filePath}") except OSError: # ignore failed removal; maybe the folder wasn't empty pass class PublishOnFDroidProcessor(TaskProcessor): requestClass = PublishOnFDroidRequest def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.sdkToolsPath = config.settings.getPath("ApkSigning", "SDKToolsPath") if self.sdkToolsPath is None: raise Error("SDKToolsPath not set in config.") self.sdkPath = self.sdkToolsPath.parent.parent def getProjectSetting(self, name, required=True): value = projects.settings.get(self.request.projectPath, self.request.branch, name) if required and not value: raise Error( f"Project setting '{name}' not set for branch {self.request.branch} " f"of project {self.request.projectPath}" ) return value def doProcess(self): repoName = self.getProjectSetting("repository") with TemporaryDirectory() as workPathName: workPath = Path(workPathName) localApkFilePaths = [] for apkFileName in self.request.apkFileNames: localApkFilePath = workPath / apkFileName self.sftp.download(self.task.path() / apkFileName, localApkFilePath) localApkFilePaths.append(localApkFilePath) localFastlaneFilePath = workPath / self.request.fastlaneFileName self.sftp.download(self.task.path() / self.request.fastlaneFileName, localFastlaneFilePath) self.verifyApplicationId(localApkFilePaths) self.publishApks(localApkFilePaths, localFastlaneFilePath, repoName) def verifyApplicationId(self, apkFilePaths): allowedApplicationId = projects.settings.get(self.request.projectPath, self.request.branch, "applicationid") for apkFilePath in apkFilePaths: applicationId = apkutils.getApplicationId(apkFilePath) if applicationId != allowedApplicationId: raise Error( f"The application ID of the APK ({applicationId}) did not match the allowed application ID " f"({allowedApplicationId})." ) def publishApks(self, apkFilePaths, fastlaneFilePath, repoName): workingDirectory = fastlaneFilePath.parent localRepoPath = config.settings.getPath("FDroidPublishing", "LocalRepoPrefix") / repoName / "fdroid" metadataPath = localRepoPath / "metadata" fastlaneDirPath = localRepoPath.parent / "fastlane" # Make sure the repository has been set up. The local repository should be the folder which contains the repo/ # folder. if not (localRepoPath / "repo").is_dir(): raise Error( "The specified F-Droid repository does not exist. Please run 'fdroid init' to setup the repository." ) # Move the APKs into the local repository # Run "mv" instead of a using a Python function because the limitations of os.rename, os.replace, shutil.move, # etc., gives me headaches command = ["mv", "-f"] + apkFilePaths + [localRepoPath / "repo"] util.runCommand(command, cwd=workingDirectory) # Create fastlane directory (if it doesn't exist yet) and move fastlane zip files into it os.makedirs(fastlaneDirPath, exist_ok=True) command = ["mv", "-f", fastlaneFilePath, fastlaneDirPath / fastlaneFilePath.name] util.runCommand(command, cwd=workingDirectory) # Remove files that haven't been updated in a long time, e.g. because of # renamed or removed builds maxAgeInDays = int(config.settings.get(f"FDroidRepository {repoName}", "Expiry", "365")) removeOldFiles(localRepoPath / "repo", maxAgeInDays=maxAgeInDays) # Let F-Droid prepare the metadata skeletons for us # We'll then fill in these templates with the information from the APK files self.runFdroidCommand(["update", "-c"], repoPath=localRepoPath) # Remove old APKs that have been moved to the archive repo localArchivePath = localRepoPath / "archive" oldApkPaths = localArchivePath.glob("*.apk") for oldApkPath in oldApkPaths: log.info(f"Removing old APK: {oldApkPath}") oldApkPath.unlink() # Extract the meta data of all apps from the fastlane files self.prepareMetaDataForAllApks(localRepoPath / "repo", fastlaneDirPath, metadataPath) # Ask F-Droid to do a full update pass self.runFdroidCommand(["update"], repoPath=localRepoPath) # Publish the repository on a webserver for clients to use self.runFdroidCommand(["deploy", "-v"], repoPath=localRepoPath) def runFdroidCommand(self, command, *, repoPath): command = ["fdroid"] + command # prepend the Android SDK Tools path to PATH to ensure that `fdroid` finds the correct `apksigner` environment = { **os.environ, "ANDROID_HOME": str(self.sdkPath), "PATH": str(self.sdkToolsPath) + os.pathsep + os.environ["PATH"], } util.runCommand(command, cwd=repoPath, env=environment) # def prepareMetaDataForAllApks(self, repoFolder: Path, fastlaneDirPath: Path, metadataPath: Path): def prepareMetaDataForAllApks(self, repoFolder, fastlaneDirPath, metadataPath): processedApplications = set() # Extract the metadata for all APKs in the repository. with os.scandir(repoFolder) as dirEntryIterator: for entry in dirEntryIterator: # ignore anything that's not a regular file named *.apk if not entry.is_file(follow_symlinks=False) or not entry.name.endswith(".apk"): continue # First, determine the ID of the application we have here # This is needed in order to locate the metadata files within the APK that have the information we need applicationId = apkutils.getApplicationId(Path(entry.path)) # Skip APKs of already processed applications if applicationId in processedApplications: log.debug(f"Meta data for {applicationId} already extracted") continue # Now we know we have a file that we are interested in; process it self.processApkFile(applicationId, fastlaneDirPath, metadataPath) processedApplications.add(applicationId) def processApkFile(self, applicationId: str, fastlaneDirPath: Path, metadataPath: Path): log.info(f"Preparing meta data for {applicationId}") # Check if we have metadata provided by the build process, and if so apply that zipName = fastlaneDirPath / f"fastlane-{applicationId}.zip" if os.path.exists(zipName): log.info(f"Using meta data from build process: {zipName.name}") with zipfile.ZipFile(zipName, "r") as zipFile: filePaths = apkutils.filterFastlaneFileNames(zipFile.namelist(), applicationId) zipFile.extractall(path=metadataPath, members=filePaths) else: log.error(f"Missing meta data package {zipName}") class PublishOnFDroidWorker(Worker): processorClass = PublishOnFDroidProcessor