186 lines
7.8 KiB
Python
186 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
|
import os
|
|
import tempfile
|
|
import argparse
|
|
import glob
|
|
import subprocess
|
|
import shutil
|
|
import tarfile
|
|
from components import Package
|
|
|
|
def prepare(locale: str, base_dir: str, doc_dir: str, subdir: str):
|
|
tmp_dir = tempfile.TemporaryDirectory(delete=False)
|
|
gen_folder = os.path.join(tmp_dir.name, subdir)
|
|
os.makedirs(gen_folder)
|
|
|
|
potential_customization_folders = ["_install/share/kf5/kdoctools/customization/", "_install/share/kf6/kdoctools/customization/"]
|
|
customization_folder = ""
|
|
for folder in potential_customization_folders:
|
|
if os.path.isdir(os.path.join(base_dir, folder)):
|
|
customization_folder = os.path.join(base_dir, folder)
|
|
break
|
|
shutil.copytree(customization_folder, gen_folder, dirs_exist_ok=True)
|
|
|
|
for file in glob.glob(os.path.join(base_dir, "ci-utilities/resources/documentation/*.xsl")):
|
|
shutil.copy(file, gen_folder)
|
|
|
|
# Copy original files
|
|
shutil.copytree(os.path.join(doc_dir, "en", subdir), gen_folder, dirs_exist_ok=True)
|
|
# Copy overriden files per language (docbook and image files)
|
|
if locale != "en":
|
|
shutil.copytree(os.path.join(doc_dir, locale, subdir), gen_folder, dirs_exist_ok=True)
|
|
|
|
return gen_folder, customization_folder
|
|
|
|
|
|
def generate_html(locale: str, base_dir: str, doc_dir: str, subdir: str, out_dir: str):
|
|
print(f"Generating html for locale {locale}")
|
|
gen_folder, _ = prepare(locale, base_dir, doc_dir, subdir)
|
|
|
|
# Mapping between kdoctools version and where to find the correspoding
|
|
# customization folder on the docs.kde.org website
|
|
customization_folders_mapping = {
|
|
"_install/share/kf5/kdoctools/customization/": "/trunk_kf5/en/kdoctools5-common/",
|
|
"_install/share/kf6/kdoctools/customization/": "/trunk_kf6/en/kdoctools6-common/"
|
|
}
|
|
docs_kde_org_customization_folder = ""
|
|
for kdoctools_folder, customization_folder in customization_folders_mapping.items():
|
|
if os.path.isdir(os.path.join(base_dir, kdoctools_folder)):
|
|
docs_kde_org_customization_folder = customization_folder
|
|
break
|
|
|
|
cmd_html = ["xsltproc", "--stringparam",
|
|
"kde.common", docs_kde_org_customization_folder,
|
|
os.path.join(gen_folder, "kde-chunk-online.xsl"),
|
|
"index.docbook"]
|
|
proc = subprocess.Popen(cmd_html, cwd=gen_folder,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
proc_out, proc_err = proc.communicate()
|
|
proc_rc = proc.returncode
|
|
if proc_rc != 0:
|
|
proc_data = [("stdout", proc_out), ("stderr", proc_err)]
|
|
for p_msg in proc_data:
|
|
print(f"{p_msg[0]}:\n{p_msg[1]}")
|
|
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
for file in glob.glob(os.path.join(gen_folder, "*.html")):
|
|
shutil.copy(file, out_dir)
|
|
for file in glob.glob(os.path.join(gen_folder, "*.png")):
|
|
shutil.copy(file, out_dir)
|
|
shutil.rmtree(gen_folder)
|
|
|
|
|
|
def generate_pdf(locale: str, base_dir: str, doc_dir: str, subdir: str, out_dir: str):
|
|
print(f"Generating pdf for locale {locale}")
|
|
gen_folder, customization_folder = prepare(locale, base_dir, doc_dir, subdir)
|
|
env_pdf = os.environ.copy()
|
|
env_pdf["SGML_CATALOG_FILES"] = f"{customization_folder}/catalog.xml"
|
|
env_pdf["DBLATEX_BASE_DIR"] = f"{base_dir}/docs-kde-org"
|
|
cmd_pdf = [os.path.join(base_dir, "docs-kde-org/buildpdf.sh"),
|
|
os.path.join(gen_folder, "index.docbook")]
|
|
proc = subprocess.Popen(cmd_pdf, cwd=gen_folder,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
env=env_pdf)
|
|
proc_out, proc_err = proc.communicate()
|
|
proc_rc = proc.returncode
|
|
if proc_rc != 0:
|
|
proc_data = [("stdout", proc_out), ("stderr", proc_err)]
|
|
for p_msg in proc_data:
|
|
print(f"{p_msg[0]}:\n{p_msg[1]}")
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
for file in glob.glob(os.path.join(gen_folder, "*.pdf")):
|
|
shutil.copy(file, out_dir)
|
|
shutil.rmtree(gen_folder)
|
|
|
|
|
|
def publish_documentation(project: str, branch: str, doc_folder: str, gitlab_instance: str, gitlab_token: str):
|
|
print("Publish documentation")
|
|
registry = "documentation"
|
|
localCachePath = f"/mnt/{registry}/artifacts/"
|
|
packageProject = f"teams/ci-artifacts/{registry}"
|
|
|
|
# Are we supposed to be publishing this particular package to the archive?
|
|
if gitlab_token is None:
|
|
return
|
|
|
|
# Bring the package archive up
|
|
packageRegistry = Package.Registry( localCachePath, gitlab_instance, gitlab_token, packageProject )
|
|
|
|
####
|
|
# Capture the installation if needed and deploy the staged install to the final install directory
|
|
####
|
|
|
|
# Create a temporary file, then open the file as a tar archive for writing
|
|
# We don't want it to be deleted as storePackage will move the archive into it's cache
|
|
archiveFile = tempfile.NamedTemporaryFile(delete=False)
|
|
archive = tarfile.open( fileobj=archiveFile, mode='w' )
|
|
|
|
# Now determine the path we should be archiving
|
|
pathToArchive = doc_folder
|
|
|
|
# Add all the files which need to be in the archive into the archive
|
|
# We want to capture the tree as it is inside the install directory and don't want any trailing slashes in the archive as this isn't standards compliant
|
|
# Therefore we list everything in the install directory and add each of those to the archive, rather than adding the whole install directory
|
|
filesToInclude = os.listdir( pathToArchive )
|
|
for filename in filesToInclude:
|
|
fullPath = os.path.join(pathToArchive, filename)
|
|
archive.add( fullPath, arcname=filename, recursive=True )
|
|
|
|
# Close the archive, which will write it out to disk, finishing what we need to do here
|
|
archive.close()
|
|
archiveFile.close()
|
|
|
|
# With the archive being generated, we can now prepare some metadata...
|
|
packageMetadata = {
|
|
}
|
|
|
|
# Grab the Git revision (SHA-1 hash) we are building
|
|
# This is always present in Gitlab CI builds
|
|
gitRevision = os.environ['CI_COMMIT_SHA']
|
|
|
|
# Publish our package to the
|
|
packageRegistry.upload(archiveFile.name, project, branch, gitRevision, packageMetadata)
|
|
|
|
####
|
|
# Run complete!
|
|
####
|
|
print("## Archived Successfully!")
|
|
|
|
|
|
# Capture our command line parameters
|
|
parser = argparse.ArgumentParser(description="Utility to generate the documentation.")
|
|
parser.add_argument("--project", type=str, required=True)
|
|
parser.add_argument("--branch", type=str, required=True)
|
|
parser.add_argument("--doc-dir", type=str, required=True)
|
|
parser.add_argument("--output-folder", type=str, required=True)
|
|
parser.add_argument("--format", type=str, required=True, nargs="+", choices=["html", "pdf"])
|
|
parser.add_argument("--publish", action='store_true', default=False)
|
|
arguments = parser.parse_args()
|
|
|
|
gitlab_instance = os.environ.pop('KDECI_GITLAB_SERVER')
|
|
gitlab_token = os.environ.pop('KDECI_GITLAB_TOKEN', None)
|
|
|
|
base_dir = os.getcwd()
|
|
|
|
# Find all docbook in the install folder
|
|
doc_dir = arguments.doc_dir
|
|
docbook_files = glob.glob(os.path.join(doc_dir, "**/index.docbook"), recursive=True)
|
|
docbook_files = [item.removeprefix(doc_dir) for item in docbook_files]
|
|
|
|
# make sure the folder is absolute
|
|
output_folder = os.path.realpath(arguments.output_folder)
|
|
shutil.rmtree(output_folder, ignore_errors=True)
|
|
os.makedirs(output_folder)
|
|
|
|
EXCLUDED_PDF_LOCALES = ["ar", "ja", "ko", "zh_CN"]
|
|
|
|
for docbook in docbook_files:
|
|
folder = os.path.dirname(docbook)
|
|
[locale, subdir] = folder.split("/", 1)
|
|
if "html" in arguments.format:
|
|
generate_html(locale, base_dir, doc_dir, subdir, os.path.join(output_folder, locale, arguments.project, subdir))
|
|
if "pdf" in arguments.format and not locale in EXCLUDED_PDF_LOCALES:
|
|
generate_pdf(locale, base_dir, doc_dir, subdir, os.path.join(output_folder, locale, arguments.project, subdir))
|
|
|
|
if arguments.publish:
|
|
publish_documentation(arguments.project, arguments.branch, output_folder, gitlab_instance, gitlab_token)
|