diff --git a/INSTALL.md b/INSTALL.md index 74029fb1a..aa5a6de88 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -68,7 +68,7 @@ For more details, see the documentation at [fonts.mss](style/fonts.mss). To download the fonts, run the following script ``` -scripts/get-fonts.sh +scripts/get-fonts.py ``` ## Dependencies diff --git a/scripts/change-fonts-cjk.sh b/scripts/change-fonts-cjk.sh index 1ef492942..b951c8dc1 100755 --- a/scripts/change-fonts-cjk.sh +++ b/scripts/change-fonts-cjk.sh @@ -8,7 +8,7 @@ mkdir -p "${FONTDIR}" # download filename url download() { ## Download if newer, and if curl fails, clean up and exit - curl --fail --compressed -A "get-fonts.sh/osm-carto" -o "$1" -z "$1" -L "$2" || { echo "Failed to download $1 $2"; rm -f "$1"; exit 1; } + curl --fail --compressed -A "change-fonts-cjk.sh/osm-carto" -o "$1" -z "$1" -L "$2" || { echo "Failed to download $1 $2"; rm -f "$1"; exit 1; } } case "$1" in diff --git a/scripts/docker-startup.sh b/scripts/docker-startup.sh index c3177ee9d..45abc15c6 100644 --- a/scripts/docker-startup.sh +++ b/scripts/docker-startup.sh @@ -56,7 +56,7 @@ EOF scripts/get-external-data.py $EXTERNAL_DATA_SCRIPT_FLAGS # Download fonts - scripts/get-fonts.sh + scripts/get-fonts.py ;; kosmtik) diff --git a/scripts/get-fonts.py b/scripts/get-fonts.py new file mode 100755 index 000000000..2eb6df5d4 --- /dev/null +++ b/scripts/get-fonts.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# This script downloads several built fonts from repos of https://github.com/notofonts +# Additional fonts can be found on https://notofonts.github.io + +import os +import requests +import tempfile +import shutil +import warnings +import zipfile + +FONTDIR = os.environ.get("FONTDIR", "./fonts") + +try: + os.mkdir(FONTDIR) +except FileExistsError: + warnings.warn("Font directory already exists") + +# Fonts available in regular, bold, and italic +REGULAR_BOLD_ITALIC = ["NotoSans"] + +# Fonts available in regular and bold +REGULAR_BOLD = [ + "NotoSansAdlamUnjoined", + "NotoSansArabic", + "NotoSansArmenian", + "NotoSansBalinese", + "NotoSansBamum", + "NotoSansBengaliUI", + "NotoSansCanadianAboriginal", + "NotoSansCham", + "NotoSansCherokee", + "NotoSansDevanagariUI", + "NotoSansEthiopic", + "NotoSansGeorgian", + "NotoSansGujaratiUI", + "NotoSansGurmukhiUI", + "NotoSansHebrew", + "NotoSansJavanese", + "NotoSansKannadaUI", + "NotoSansKayahLi", + "NotoSansKhmerUI", + "NotoSansLaoUI", + "NotoSansLisu", + "NotoSansMalayalamUI", + "NotoSansMyanmarUI", + "NotoSansOlChiki", + "NotoSansOriya", + "NotoSansSinhalaUI", + "NotoSansSundanese", + "NotoSansSymbols", + "NotoSansTaiTham", + "NotoSansTamilUI", + "NotoSansTeluguUI", + "NotoSansThaana", + "NotoSansThaiUI", + "NotoSerifTibetan", +] + +# Fonts with regular and black, but no bold +REGULAR_BLACK = ["NotoSansSyriac"] + +# Fonts only available in regular +REGULAR = [ + "NotoSansBatak", + "NotoSansBuginese", + "NotoSansBuhid", + "NotoSansChakma", + "NotoSansCoptic", + "NotoSansHanunoo", + "NotoSansLepcha", + "NotoSansLimbu", + "NotoSansMandaic", + "NotoSansMongolian", + "NotoSansNewTaiLue", + "NotoSansNKo", + "NotoSansOsage", + "NotoSansOsmanya", + "NotoSansSamaritan", + "NotoSansSaurashtra", + "NotoSansShavian", + "NotoSansSymbols2", + "NotoSansTagalog", + "NotoSansTagbanwa", + "NotoSansTaiLe", + "NotoSansTaiViet", + "NotoSansTifinagh", + "NotoSansVai", + "NotoSansYi", +] + +SOUTH_ASIAN_UI_FONTS = [ + "BengaliUI", + "DevanagariUI", + "GujaratiUI", + "GurmukhiUI", + "KannadaUI", + "MalayalamUI", + "SinhalaUI", + "TamilUI", + "TeluguUI", +] + +# hyphenated or irregular Noto repo names +NOTO_REPO_FOR_FONT = { + "NotoSans": "latin-greek-cyrillic", + "NotoSansAdlamUnjoined": "adlam", + "NotoSansCanadianAboriginal": "canadian-aboriginal", + "NotoSansKayahLi": "kayah-li", + "NotoSansNewTaiLue": "new-tai-lue", + "NotoSansOlChiki": "ol-chiki", + "NotoSansSymbols2": "symbols", + "NotoSansTaiLe": "tai-le", + "NotoSansTaiTham": "tai-tham", + "NotoSansTaiViet": "tai-viet", + "NotoSerifTibetan": "tibetan", +} + + +# Download the fonts in the lists above +def findFontUrls(fontName, modifier, useRepo=False): + # remove 'UI' from South Asian font names for subfolder name + if fontName.replace("NotoSans", "") in SOUTH_ASIAN_UI_FONTS: + subfolder = fontName.replace("UI", "") + else: + subfolder = fontName + + # pick up regular and irregular repo names for path + repo = NOTO_REPO_FOR_FONT.get( + fontName, fontName.replace("NotoSans", "").replace("UI", "").lower() + ) + + return [ + f"https://cdn.jsdelivr.net/gh/notofonts/notofonts.github.io/fonts/{subfolder}/hinted/ttf/{fontName}-{modifier}.ttf", + f"https://notofonts.github.io/{repo}/fonts/{subfolder}/hinted/ttf/{fontName}-{modifier}.ttf", + ] + + +def downloadToFile(urls, destination, dir=FONTDIR): + headers = {"User-Agent": "get-fonts.py/osm-carto"} + + try: + r = requests.get(urls[0], headers=headers) + if r.status_code != 200: + if len(urls) > 1: + warnings.warn(f"Failed to download {urls[0]}, retrying with repo HEAD") + downloadToFile(urls[1:], destination, dir=dir) + else: + raise Exception + with open(os.path.join(dir, destination), "wb") as f: + f.write(r.content) + except: + raise Exception(f"Failed to download {urls}") + + +for font in REGULAR_BOLD + REGULAR_BOLD_ITALIC + REGULAR_BLACK + REGULAR: + regularFontUrls = findFontUrls(font, "Regular") + downloadToFile(regularFontUrls, f"{font}-Regular.ttf") + + if (font in REGULAR_BOLD) or (font in REGULAR_BOLD_ITALIC): + boldFontUrls = findFontUrls(font, "Bold") + downloadToFile(boldFontUrls, f"{font}-Bold.ttf") + + if font in REGULAR_BOLD_ITALIC: + italicFontUrls = findFontUrls(font, "Italic") + downloadToFile(italicFontUrls, f"{font}-Italic.ttf") + + if font in REGULAR_BLACK: + blackFontUrls = findFontUrls(font, "Black") + downloadToFile(blackFontUrls, f"{font}-Black.ttf") + +# Other noto fonts which don't follow the URL pattern above + +# CJK fonts +downloadToFile( + [ + "https://github.com/notofonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Regular.otf" + ], + "NotoSansCJKjp-Regular.otf", +) +downloadToFile( + [ + "https://github.com/notofonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Bold.otf" + ], + "NotoSansCJKjp-Bold.otf", +) + +# Fonts in zipfiles need a temporary directory +TMPDIR = tempfile.mkdtemp(prefix="get-fonts.") + +# Noto Emoji B&W isn't available as a separate download, so we need to download the package and unzip it +downloadToFile( + ["https://archive.org/download/noto-emoji/Noto_Emoji.zip"], + "Noto_Emoji.zip", + dir=TMPDIR, +) +emojiPath = os.path.join(TMPDIR, "Noto_Emoji.zip") +emojiExtract = ["NotoEmoji-Regular.ttf", "NotoEmoji-Bold.ttf"] +with zipfile.ZipFile(emojiPath, "r") as zip_ref: + for file in emojiExtract: + source = zip_ref.getinfo(f"static/{file}") + zip_ref.extract(source, FONTDIR) + # move from FONTDIR/static/x to overwrite FONTDIR/x + unzipSrc = os.path.join(FONTDIR, file) + if os.path.exists(unzipSrc): + os.remove(unzipSrc) + shutil.move(os.path.join(FONTDIR, "static", file), FONTDIR) + +downloadToFile( + ["https://mirrors.dotsrc.org/osdn/hanazono-font/68253/hanazono-20170904.zip"], + "hanazono.zip", + dir=TMPDIR, +) +hanazonoPath = os.path.join(TMPDIR, "hanazono.zip") +with zipfile.ZipFile(hanazonoPath, "r") as zip_ref: + for file in ["HanaMinA.ttf", "HanaMinB.ttf"]: + source = zip_ref.getinfo(file) + zip_ref.extract(source, FONTDIR) + +# clean up tmp directories +shutil.rmtree(TMPDIR) +fontdir_static = os.path.join(FONTDIR, "static") +if os.path.exists(fontdir_static): + shutil.rmtree(fontdir_static) diff --git a/scripts/get-fonts.sh b/scripts/get-fonts.sh deleted file mode 100755 index b25d3ad2f..000000000 --- a/scripts/get-fonts.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/sh -set -e - -FONTDIR="./fonts" - -mkdir -p "${FONTDIR}" - -# download filename url -download() { - ## Download if newer, and if curl fails, clean up and exit - curl --fail --compressed -A "get-fonts.sh/osm-carto" -o "$1" -z "$1" -L "$2" || { echo "Failed to download $1 $2"; rm -f "$1"; exit 1; } -} - -# TTF Hinted Noto Fonts - -# Fonts available in regular, bold, and italic -REGULAR_BOLD_ITALIC="NotoSans" - -# Fonts available in regular and bold -REGULAR_BOLD="NotoSansAdlamUnjoined -NotoSansArabicUI -NotoSansArmenian -NotoSansBalinese -NotoSansBamum -NotoSansBengaliUI -NotoSansCanadianAboriginal -NotoSansCham -NotoSansCherokee -NotoSansDevanagariUI -NotoSansEthiopic -NotoSansGeorgian -NotoSansGujaratiUI -NotoSansGurmukhiUI -NotoSansHebrew -NotoSansJavanese -NotoSansKannadaUI -NotoSansKayahLi -NotoSansKhmerUI -NotoSansLaoUI -NotoSansLisu -NotoSansMalayalamUI -NotoSansMyanmarUI -NotoSansOlChiki -NotoSansOriyaUI -NotoSansSinhalaUI -NotoSansSundanese -NotoSansSymbols -NotoSansTaiTham -NotoSansTamilUI -NotoSansTeluguUI -NotoSansThaana -NotoSansThaiUI -NotoSerifTibetan" - -# Fonts with regular and black, but no bold -REGULAR_BLACK="NotoSansSyriac" - -# Fonts only available in regular -REGULAR="NotoSansBatak -NotoSansBuginese -NotoSansBuhid -NotoSansChakma -NotoSansCoptic -NotoSansHanunoo -NotoSansLepcha -NotoSansLimbu -NotoSansMandaic -NotoSansMongolian -NotoSansNewTaiLue -NotoSansNKo -NotoSansOsage -NotoSansOsmanya -NotoSansSamaritan -NotoSansSaurashtra -NotoSansShavian -NotoSansSymbols2 -NotoSansTagalog -NotoSansTagbanwa -NotoSansTaiLe -NotoSansTaiViet -NotoSansTifinagh -NotoSansVai -NotoSansYi" - -# Download the fonts in the lists above - -for font in $REGULAR_BOLD_ITALIC; do - regular="$font-Regular.ttf" - bold="$font-Bold.ttf" - italic="$font-Italic.ttf" - download "${FONTDIR}/${regular}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${regular}" - download "${FONTDIR}/${bold}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${bold}" - download "${FONTDIR}/${italic}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${italic}" -done - -for font in $REGULAR_BOLD; do - regular="$font-Regular.ttf" - bold="$font-Bold.ttf" - download "${FONTDIR}/${regular}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${regular}" - download "${FONTDIR}/${bold}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${bold}" -done - -for font in $REGULAR_BLACK; do - regular="$font-Regular.ttf" - black="$font-Black.ttf" - download "${FONTDIR}/${regular}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${regular}" - download "${FONTDIR}/${black}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${black}" -done - -for font in $REGULAR; do - regular="$font-Regular.ttf" - download "${FONTDIR}/${regular}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${regular}" -done - -# Other noto fonts which don't follow the URL pattern above -download "${FONTDIR}/NotoSansCJKjp-Regular.otf" "https://github.com/googlefonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Regular.otf" -download "${FONTDIR}/NotoSansCJKjp-Bold.otf" "https://github.com/googlefonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Bold.otf" - -# Fonts in zipfiles need a temporary directory -TMPDIR=$(mktemp -d -t get-fonts.XXXXXXXXX) -trap "rm -rf ${TMPDIR} ${FONTDIR}/static" EXIT - -# Noto Emoji B&W isn't available as a separate download, so we need to download the package and unzip it -curl --fail -A "get-fonts.sh/osm-carto" -o "${TMPDIR}/Noto_Emoji.zip" -L 'https://fonts.google.com/download?family=Noto%20Emoji' - -unzip -oqq "${TMPDIR}/Noto_Emoji.zip" static/NotoEmoji-Regular.ttf static/NotoEmoji-Bold.ttf -d "${FONTDIR}" -mv "${FONTDIR}/static/NotoEmoji-Regular.ttf" "${FONTDIR}" -mv "${FONTDIR}/static/NotoEmoji-Bold.ttf" "${FONTDIR}" - -curl --fail -A "get-fonts.sh/osm-carto" -o "${TMPDIR}/hanazono.zip" -L 'https://mirrors.dotsrc.org/osdn/hanazono-font/68253/hanazono-20170904.zip' - -unzip -oqq "${TMPDIR}/hanazono.zip" HanaMinA.ttf HanaMinB.ttf -d "${FONTDIR}" diff --git a/style/fonts.mss b/style/fonts.mss index e4ca039cd..ba57bd1fe 100644 --- a/style/fonts.mss +++ b/style/fonts.mss @@ -58,7 +58,7 @@ A regular style. */ @book-fonts: "Noto Sans Regular", "Noto Sans Adlam Unjoined Regular", - "Noto Sans Arabic UI Regular", + "Noto Sans Arabic Regular", "Noto Sans Armenian Regular", "Noto Sans Balinese Regular", "Noto Sans Bamum Regular", @@ -93,7 +93,7 @@ A regular style. "Noto Sans New Tai Lue Regular", "Noto Sans NKo Regular", "Noto Sans Ol Chiki Regular", - "Noto Sans Oriya UI Regular", + "Noto Sans Oriya Regular", "Noto Sans Osage Regular", "Noto Sans Osmanya Regular", "Noto Sans Samaritan Regular", @@ -129,7 +129,7 @@ regular text and can be used for emphasis. Fallback is a regular style. */ @bold-fonts: "Noto Sans Bold", "Noto Sans Adlam Unjoined Bold", - "Noto Sans Arabic UI Bold", + "Noto Sans Arabic Bold", "Noto Sans Armenian Bold", "Noto Sans Balinese Bold", "Noto Sans Bamum Bold", @@ -152,7 +152,7 @@ regular text and can be used for emphasis. Fallback is a regular style. "Noto Sans Malayalam UI Bold", "Noto Sans Myanmar UI Bold", "Noto Sans Ol Chiki Bold", - "Noto Sans Oriya UI Bold", + "Noto Sans Oriya Bold", "Noto Sans Sinhala UI Bold", "Noto Sans Sundanese Bold", "Noto Sans Symbols Bold",