147 lines
5.5 KiB
Python
147 lines
5.5 KiB
Python
# SPDX-License-Identifier: BSD-2-Clause
|
|
# SPDX-FileCopyrightText: 2023 KDE e.V.
|
|
# SPDX-FileContributor: Ingo Klöcker <dev@ingo-kloecker.de>
|
|
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import tarfile
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Iterator, List, Optional, Union
|
|
|
|
import requests
|
|
|
|
import sftpnotary.log
|
|
from sftpnotary import config, projects
|
|
from sftpnotary.exceptions import Error, InvalidFileName
|
|
from sftpnotary.sftp import SFTPClient
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def setUpServiceLogging(serviceName: str, verbosity: int) -> None:
|
|
# if verbosity level is greater than 2 then decrease the root logger level
|
|
rootLoggerLevel = max(logging.DEBUG, logging.WARNING - 10 * max(0, verbosity - 2))
|
|
logFilePath = config.settings.getPath("Logging", "LogFile")
|
|
logging.basicConfig(
|
|
format=sftpnotary.log.standardFormat,
|
|
level=rootLoggerLevel,
|
|
filename=str(logFilePath) if logFilePath is not None else None,
|
|
)
|
|
|
|
# default to WARNING level for the sftpnotary logger
|
|
logLevel = max(logging.DEBUG, logging.WARNING - 10 * verbosity)
|
|
logging.getLogger("sftpnotary").setLevel(logLevel)
|
|
sftpnotary.log.setNameForTaskLogs(serviceName)
|
|
|
|
|
|
def setUpClientLogging(clientLogger: logging.Logger, verbosity: int) -> None:
|
|
# if verbosity level is greater than 1 then decrease the root logger level
|
|
rootLoggerLevel = max(logging.DEBUG, logging.WARNING - 10 * max(0, verbosity - 1))
|
|
logging.basicConfig(format="%(asctime)s %(levelname)s %(name)s %(message)s", level=rootLoggerLevel)
|
|
|
|
# default to INFO level for the client logger and the sftpnotary logger
|
|
logLevel = max(logging.DEBUG, logging.INFO - 10 * verbosity)
|
|
clientLogger.setLevel(logLevel)
|
|
logging.getLogger("sftpnotary").setLevel(logLevel)
|
|
|
|
|
|
@contextmanager
|
|
def connectToSftpServer():
|
|
with SFTPClient() as sftp:
|
|
sftp.connect(
|
|
config.settings.get("SFTP", "Server"),
|
|
config.settings.get("SFTP", "BasePath"),
|
|
port=config.settings.get("SFTP", "Port", 22),
|
|
username=config.settings.get("SFTP", "Username"),
|
|
knownHostsFile=config.settings.getPath("SFTP", "KnownHostsFile"),
|
|
rsaKeyFile=config.settings.getPath("SFTP", "RsaKeyFile"),
|
|
)
|
|
yield sftp
|
|
|
|
|
|
def loadProjectSettings(path: Optional[Union[str, Path]], relativeTo: Optional[Path] = None):
|
|
if path is None:
|
|
return
|
|
if isinstance(path, str) and path.startswith(("https://", "http://")):
|
|
log.debug(f"Fetching project settings: {path}")
|
|
r = requests.get(path, timeout=5)
|
|
r.raise_for_status()
|
|
projects.readProjectSettings(r.text)
|
|
else:
|
|
projectSettingsPath = Path(path).expanduser()
|
|
if not projectSettingsPath.is_absolute() and relativeTo is not None:
|
|
projectSettingsPath = Path(relativeTo) / projectSettingsPath
|
|
with open(projectSettingsPath, "r") as f:
|
|
projects.readProjectSettings(f)
|
|
|
|
|
|
def runCommand(command, commandToLog=None, **kwargs) -> subprocess.CompletedProcess:
|
|
try:
|
|
log.info(f"Running {command if commandToLog is None else commandToLog} ...")
|
|
kwargs.update({"capture_output": True, "check": True, "text": True})
|
|
result = subprocess.run(command, **kwargs) # nosec subprocess_without_shell_equals_true
|
|
log.info("The process exited successfully")
|
|
log.debug(f"stdout:\n{result.stdout}")
|
|
log.debug(f"stderr:\n{result.stderr}")
|
|
return result
|
|
except subprocess.CalledProcessError as e:
|
|
log.error(f"The process exited with {e.returncode}")
|
|
log.debug(f"stdout:\n{e.stdout}")
|
|
log.debug(f"stderr:\n{e.stderr}")
|
|
raise Error(f"Running {command[0]} failed")
|
|
|
|
|
|
def collectArgumentsUpToSize(arguments: List[str], maxSize: int) -> Iterator[List[str]]:
|
|
currentSize = 0
|
|
firstFile = 0
|
|
for i, argument in enumerate(arguments):
|
|
argumentSize = len(argument)
|
|
if argumentSize > maxSize:
|
|
raise Error(f"Size of argument {argument!r} exceeds available size {maxSize}.")
|
|
if currentSize + argumentSize <= maxSize:
|
|
currentSize += argumentSize + 1
|
|
continue
|
|
yield arguments[firstFile:i]
|
|
firstFile = i
|
|
currentSize = argumentSize + 1
|
|
|
|
yield arguments[firstFile:]
|
|
|
|
|
|
def validateFileName(fileName: str, strict: bool = False) -> None:
|
|
"""Verify that the file name does not contain Unix path separators to
|
|
prevent directory traversal problems on the SFTP server.
|
|
|
|
If strict is True, then only printable ASCII characters are allowed and
|
|
the file name must not contain spaces."""
|
|
if fileName is None:
|
|
raise InvalidFileName()
|
|
if not fileName or "/" in fileName:
|
|
raise InvalidFileName(fileName)
|
|
if strict and (" " in fileName or not fileName.isascii() or not fileName.isprintable()):
|
|
raise InvalidFileName(fileName)
|
|
|
|
|
|
def getFileSuffix(filePath: Path):
|
|
for ext in [".tar.gz"]:
|
|
if filePath.name.endswith(ext):
|
|
return ext
|
|
|
|
return filePath.suffix
|
|
|
|
|
|
def removeFileSuffix(fileName: str):
|
|
suffixLen = len(getFileSuffix(Path(fileName)))
|
|
if suffixLen:
|
|
return fileName[:-suffixLen]
|
|
else:
|
|
return fileName
|
|
|
|
|
|
def extractTarArchive(archivePath: Path, targetPath: Path):
|
|
os.makedirs(targetPath, exist_ok=True)
|
|
with tarfile.open(name=archivePath, mode="r") as archive:
|
|
# Set "nosec B202" because of https://github.com/PyCQA/bandit/issues/1038#issuecomment-1635863856
|
|
archive.extractall(filter="data", path=targetPath) # nosec B202
|