262 lines
11 KiB
Python
262 lines
11 KiB
Python
# SPDX-License-Identifier: BSD-2-Clause
|
|
# SPDX-FileCopyrightText: 2023 KDE e.V.
|
|
# SPDX-FileContributor: Ingo Klöcker <dev@ingo-kloecker.de>
|
|
|
|
import dataclasses
|
|
import logging
|
|
import os
|
|
import time
|
|
import zipfile
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from typing import List, Optional
|
|
|
|
from sftpnotary import apkutils, config, projects, util
|
|
from sftpnotary.core import Job, Request, Response, TaskProcessor, Worker
|
|
from sftpnotary.exceptions import Error, InvalidFileName
|
|
from sftpnotary.sftp import SFTPClient, SFTPError
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class PublishOnFDroidRequest(Request):
|
|
apkFileNames: Optional[List[str]] = None
|
|
fastlaneFileName: Optional[str] = None
|
|
|
|
def validate(self):
|
|
if self.apkFileNames is None:
|
|
raise InvalidFileName()
|
|
for apkFileName in self.apkFileNames:
|
|
util.validateFileName(apkFileName, strict=True)
|
|
util.validateFileName(self.fastlaneFileName, strict=True)
|
|
|
|
|
|
PublishOnFDroidResponse = Response
|
|
|
|
|
|
class PublishOnFDroidJob(Job):
|
|
def __init__(
|
|
self,
|
|
sftp: SFTPClient,
|
|
apkFilePaths: List[Path],
|
|
fastlaneFilePath: Path,
|
|
token: Optional[str] = None,
|
|
projectPath: Optional[str] = None,
|
|
branch: Optional[str] = None,
|
|
):
|
|
super().__init__(sftp)
|
|
|
|
self.apkFilePaths = apkFilePaths
|
|
self.fastlaneFilePath = fastlaneFilePath
|
|
self.request = PublishOnFDroidRequest(
|
|
apkFileNames=[],
|
|
fastlaneFileName=None,
|
|
token=token,
|
|
projectPath=projectPath,
|
|
branch=branch,
|
|
)
|
|
self.resultPath = self.apkFilePaths[0].parent
|
|
|
|
def start(self):
|
|
for apkFilePath in self.apkFilePaths:
|
|
apkFileName = apkFilePath.name
|
|
self.sftp.upload(apkFilePath, self.task.path() / apkFileName)
|
|
self.request.apkFileNames.append(apkFileName)
|
|
fastlaneFileName = self.fastlaneFilePath.name
|
|
self.sftp.upload(self.fastlaneFilePath, self.task.path() / fastlaneFileName)
|
|
self.request.fastlaneFileName = fastlaneFileName
|
|
self.task.putRequest(self.request)
|
|
self.taskQueue.commitTask(self.task)
|
|
|
|
def waitForCompletion(self):
|
|
self.taskQueue.waitForCompletion(self.task)
|
|
log.debug(f"Fetching response of task '{self.task.id}' ...")
|
|
response = None
|
|
try:
|
|
response = self.task.fetchResponse(PublishOnFDroidResponse)
|
|
except SFTPError as e:
|
|
log.error("Error: %s", e)
|
|
if response is None:
|
|
log.error("Error: Got no response. Task failed.")
|
|
else:
|
|
self.success = True
|
|
self.downloadLogFiles(self.resultPath)
|
|
self.taskQueue.removeTask(self.task)
|
|
|
|
|
|
def removeOldFiles(path, maxAgeInDays):
|
|
oldFilesPurgeThreshold = time.time() - (3600 * 24 * maxAgeInDays)
|
|
log.debug(f"Removing files that are older than {maxAgeInDays} days.")
|
|
for root, dirs, files in os.walk(path, topdown=False, onerror=None, followlinks=False):
|
|
# first iterate over all files
|
|
for name in files:
|
|
filePath = Path(root) / name
|
|
# ignore anything that's not a regular file
|
|
if filePath.is_symlink() or not filePath.is_file():
|
|
log.debug(f"Ignoring {filePath} which is not a regular file.")
|
|
continue
|
|
|
|
# purge ancient files
|
|
if os.path.getmtime(filePath) < oldFilesPurgeThreshold:
|
|
log.info(f"Removing outdated file: {filePath}")
|
|
filePath.unlink()
|
|
|
|
# then delete empty subfolders
|
|
for name in dirs:
|
|
filePath = Path(root) / name
|
|
try:
|
|
# try to remove the folder
|
|
filePath.rmdir()
|
|
log.info(f"Removed empty folder: {filePath}")
|
|
except OSError:
|
|
# ignore failed removal; maybe the folder wasn't empty
|
|
pass
|
|
|
|
|
|
class PublishOnFDroidProcessor(TaskProcessor):
|
|
requestClass = PublishOnFDroidRequest
|
|
|
|
def __init__(self, *args, **kwargs) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
|
|
self.sdkToolsPath = config.settings.getPath("ApkSigning", "SDKToolsPath")
|
|
if self.sdkToolsPath is None:
|
|
raise Error("SDKToolsPath not set in config.")
|
|
self.sdkPath = self.sdkToolsPath.parent.parent
|
|
|
|
def getProjectSetting(self, name, required=True):
|
|
value = projects.settings.get(self.request.projectPath, self.request.branch, name)
|
|
if required and not value:
|
|
raise Error(
|
|
f"Project setting '{name}' not set for branch {self.request.branch} "
|
|
f"of project {self.request.projectPath}"
|
|
)
|
|
return value
|
|
|
|
def doProcess(self):
|
|
repoName = self.getProjectSetting("repository")
|
|
|
|
with TemporaryDirectory() as workPathName:
|
|
workPath = Path(workPathName)
|
|
|
|
localApkFilePaths = []
|
|
for apkFileName in self.request.apkFileNames:
|
|
localApkFilePath = workPath / apkFileName
|
|
self.sftp.download(self.task.path() / apkFileName, localApkFilePath)
|
|
localApkFilePaths.append(localApkFilePath)
|
|
localFastlaneFilePath = workPath / self.request.fastlaneFileName
|
|
self.sftp.download(self.task.path() / self.request.fastlaneFileName, localFastlaneFilePath)
|
|
|
|
self.verifyApplicationId(localApkFilePaths)
|
|
self.publishApks(localApkFilePaths, localFastlaneFilePath, repoName)
|
|
|
|
def verifyApplicationId(self, apkFilePaths):
|
|
allowedApplicationId = projects.settings.get(self.request.projectPath, self.request.branch, "applicationid")
|
|
for apkFilePath in apkFilePaths:
|
|
applicationId = apkutils.getApplicationId(apkFilePath)
|
|
if applicationId != allowedApplicationId:
|
|
raise Error(
|
|
f"The application ID of the APK ({applicationId}) did not match the allowed application ID "
|
|
f"({allowedApplicationId})."
|
|
)
|
|
|
|
def publishApks(self, apkFilePaths, fastlaneFilePath, repoName):
|
|
workingDirectory = fastlaneFilePath.parent
|
|
localRepoPath = config.settings.getPath("FDroidPublishing", "LocalRepoPrefix") / repoName / "fdroid"
|
|
metadataPath = localRepoPath / "metadata"
|
|
fastlaneDirPath = localRepoPath.parent / "fastlane"
|
|
|
|
# Make sure the repository has been set up. The local repository should be the folder which contains the repo/
|
|
# folder.
|
|
if not (localRepoPath / "repo").is_dir():
|
|
raise Error(
|
|
"The specified F-Droid repository does not exist. Please run 'fdroid init' to setup the repository."
|
|
)
|
|
|
|
# Move the APKs into the local repository
|
|
# Run "mv" instead of a using a Python function because the limitations of os.rename, os.replace, shutil.move,
|
|
# etc., gives me headaches
|
|
command = ["mv", "-f"] + apkFilePaths + [localRepoPath / "repo"]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# Create fastlane directory (if it doesn't exist yet) and move fastlane zip files into it
|
|
os.makedirs(fastlaneDirPath, exist_ok=True)
|
|
command = ["mv", "-f", fastlaneFilePath, fastlaneDirPath / fastlaneFilePath.name]
|
|
util.runCommand(command, cwd=workingDirectory)
|
|
|
|
# Remove files that haven't been updated in a long time, e.g. because of
|
|
# renamed or removed builds
|
|
maxAgeInDays = int(config.settings.get(f"FDroidRepository {repoName}", "Expiry", "365"))
|
|
removeOldFiles(localRepoPath / "repo", maxAgeInDays=maxAgeInDays)
|
|
|
|
# Let F-Droid prepare the metadata skeletons for us
|
|
# We'll then fill in these templates with the information from the APK files
|
|
self.runFdroidCommand(["update", "-c"], repoPath=localRepoPath)
|
|
|
|
# Remove old APKs that have been moved to the archive repo
|
|
localArchivePath = localRepoPath / "archive"
|
|
oldApkPaths = localArchivePath.glob("*.apk")
|
|
for oldApkPath in oldApkPaths:
|
|
log.info(f"Removing old APK: {oldApkPath}")
|
|
oldApkPath.unlink()
|
|
|
|
# Extract the meta data of all apps from the fastlane files
|
|
self.prepareMetaDataForAllApks(localRepoPath / "repo", fastlaneDirPath, metadataPath)
|
|
|
|
# Ask F-Droid to do a full update pass
|
|
self.runFdroidCommand(["update"], repoPath=localRepoPath)
|
|
|
|
# Publish the repository on a webserver for clients to use
|
|
self.runFdroidCommand(["deploy", "-v"], repoPath=localRepoPath)
|
|
|
|
def runFdroidCommand(self, command, *, repoPath):
|
|
command = ["fdroid"] + command
|
|
# prepend the Android SDK Tools path to PATH to ensure that `fdroid` finds the correct `apksigner`
|
|
environment = {
|
|
**os.environ,
|
|
"ANDROID_HOME": str(self.sdkPath),
|
|
"PATH": str(self.sdkToolsPath) + os.pathsep + os.environ["PATH"],
|
|
}
|
|
util.runCommand(command, cwd=repoPath, env=environment)
|
|
|
|
# def prepareMetaDataForAllApks(self, repoFolder: Path, fastlaneDirPath: Path, metadataPath: Path):
|
|
def prepareMetaDataForAllApks(self, repoFolder, fastlaneDirPath, metadataPath):
|
|
processedApplications = set()
|
|
|
|
# Extract the metadata for all APKs in the repository.
|
|
with os.scandir(repoFolder) as dirEntryIterator:
|
|
for entry in dirEntryIterator:
|
|
# ignore anything that's not a regular file named *.apk
|
|
if not entry.is_file(follow_symlinks=False) or not entry.name.endswith(".apk"):
|
|
continue
|
|
|
|
# First, determine the ID of the application we have here
|
|
# This is needed in order to locate the metadata files within the APK that have the information we need
|
|
applicationId = apkutils.getApplicationId(Path(entry.path))
|
|
|
|
# Skip APKs of already processed applications
|
|
if applicationId in processedApplications:
|
|
log.debug(f"Meta data for {applicationId} already extracted")
|
|
continue
|
|
|
|
# Now we know we have a file that we are interested in; process it
|
|
self.processApkFile(applicationId, fastlaneDirPath, metadataPath)
|
|
processedApplications.add(applicationId)
|
|
|
|
def processApkFile(self, applicationId: str, fastlaneDirPath: Path, metadataPath: Path):
|
|
log.info(f"Preparing meta data for {applicationId}")
|
|
|
|
# Check if we have metadata provided by the build process, and if so apply that
|
|
zipName = fastlaneDirPath / f"fastlane-{applicationId}.zip"
|
|
if os.path.exists(zipName):
|
|
log.info(f"Using meta data from build process: {zipName.name}")
|
|
with zipfile.ZipFile(zipName, "r") as zipFile:
|
|
filePaths = apkutils.filterFastlaneFileNames(zipFile.namelist(), applicationId)
|
|
zipFile.extractall(path=metadataPath, members=filePaths)
|
|
else:
|
|
log.error(f"Missing meta data package {zipName}")
|
|
|
|
|
|
class PublishOnFDroidWorker(Worker):
|
|
processorClass = PublishOnFDroidProcessor
|