From b9ad82a489d2fd3097115fa8bc4f0e247a5b02e9 Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Sat, 28 Oct 2023 12:52:36 -0500 Subject: [PATCH 01/12] Use notofonts.github.io for newer font releases --- scripts/get-fonts.sh | 160 ++++++++++++++++++++++++------------------- 1 file changed, 89 insertions(+), 71 deletions(-) diff --git a/scripts/get-fonts.sh b/scripts/get-fonts.sh index b25d3ad2f..6abf219b5 100755 --- a/scripts/get-fonts.sh +++ b/scripts/get-fonts.sh @@ -17,70 +17,70 @@ download() { REGULAR_BOLD_ITALIC="NotoSans" # Fonts available in regular and bold -REGULAR_BOLD="NotoSansAdlamUnjoined -NotoSansArabicUI -NotoSansArmenian -NotoSansBalinese -NotoSansBamum -NotoSansBengaliUI -NotoSansCanadianAboriginal -NotoSansCham -NotoSansCherokee -NotoSansDevanagariUI -NotoSansEthiopic -NotoSansGeorgian -NotoSansGujaratiUI -NotoSansGurmukhiUI -NotoSansHebrew -NotoSansJavanese -NotoSansKannadaUI -NotoSansKayahLi -NotoSansKhmerUI -NotoSansLaoUI -NotoSansLisu -NotoSansMalayalamUI -NotoSansMyanmarUI -NotoSansOlChiki -NotoSansOriyaUI -NotoSansSinhalaUI -NotoSansSundanese -NotoSansSymbols -NotoSansTaiTham -NotoSansTamilUI -NotoSansTeluguUI -NotoSansThaana -NotoSansThaiUI -NotoSerifTibetan" +REGULAR_BOLD="NotoSansAdlamUnjoined;adlam +NotoSansArabicUI;arabic +NotoSansArmenian;armenian +NotoSansBalinese;balinese +NotoSansBamum;bamum +NotoSansBengaliUI;bengali +NotoSansCanadianAboriginal;canadian-aboriginal +NotoSansCham;cham +NotoSansCherokee;cherokee +NotoSansDevanagariUI;devanagari +NotoSansEthiopic;ethiopic +NotoSansGeorgian;georgian +NotoSansGujaratiUI;gujarati +NotoSansGurmukhiUI;gurmukhi +NotoSansHebrew;hebrew +NotoSansJavanese;javanese +NotoSansKannadaUI;kannada +NotoSansKayahLi;kayah-li +NotoSansKhmerUI;khmer +NotoSansLaoUI;lao +NotoSansLisu;lisu +NotoSansMalayalamUI;malayalam +NotoSansMyanmarUI;myanmar +NotoSansOlChiki;ol-chiki +NotoSansOriya;oriya +NotoSansSinhalaUI;sinhala +NotoSansSundanese;sundanese +NotoSansSymbols;symbols +NotoSansTaiTham;tai-tham +NotoSansTamilUI;tamil +NotoSansTeluguUI;telugu +NotoSansThaana;thaana +NotoSansThaiUI;thai +NotoSerifTibetan;tibetan" # Fonts with regular and black, but no bold -REGULAR_BLACK="NotoSansSyriac" +REGULAR_BLACK="NotoSansSyriac;syriac" # Fonts only available in regular -REGULAR="NotoSansBatak -NotoSansBuginese -NotoSansBuhid -NotoSansChakma -NotoSansCoptic -NotoSansHanunoo -NotoSansLepcha -NotoSansLimbu -NotoSansMandaic -NotoSansMongolian -NotoSansNewTaiLue -NotoSansNKo -NotoSansOsage -NotoSansOsmanya -NotoSansSamaritan -NotoSansSaurashtra -NotoSansShavian -NotoSansSymbols2 -NotoSansTagalog -NotoSansTagbanwa -NotoSansTaiLe -NotoSansTaiViet -NotoSansTifinagh -NotoSansVai -NotoSansYi" +REGULAR="NotoSansBatak;batak +NotoSansBuginese;buginese +NotoSansBuhid;buhid +NotoSansChakma;chakma +NotoSansCoptic;coptic +NotoSansHanunoo;hanunoo +NotoSansLepcha;lepcha +NotoSansLimbu;limbu +NotoSansMandaic;mandaic +NotoSansMongolian;mongolian +NotoSansNewTaiLue;new-tai-lue +NotoSansNKo;nko +NotoSansOsage;osage +NotoSansOsmanya;osmanya +NotoSansSamaritan;samaritan +NotoSansSaurashtra;saurashtra +NotoSansShavian;shavian +NotoSansSymbols2;symbols +NotoSansTagalog;tagalog +NotoSansTagbanwa;tagbanwa +NotoSansTaiLe;tai-le +NotoSansTaiViet;tai-viet +NotoSansTifinagh;tifinagh +NotoSansVai;vai +NotoSansYi;yi" # Download the fonts in the lists above @@ -88,28 +88,46 @@ for font in $REGULAR_BOLD_ITALIC; do regular="$font-Regular.ttf" bold="$font-Bold.ttf" italic="$font-Italic.ttf" - download "${FONTDIR}/${regular}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${regular}" - download "${FONTDIR}/${bold}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${bold}" - download "${FONTDIR}/${italic}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${italic}" + download "${FONTDIR}/${regular}" "https://notofonts.github.io/latin-greek-cyrillic/fonts/${font}/hinted/ttf/${regular}" + download "${FONTDIR}/${bold}" "https://notofonts.github.io/latin-greek-cyrillic/fonts/${font}/hinted/ttf/${bold}" + download "${FONTDIR}/${italic}" "https://notofonts.github.io/latin-greek-cyrillic/fonts/${font}/hinted/ttf/${italic}" done -for font in $REGULAR_BOLD; do +for fontDir in $REGULAR_BOLD; do + IFS=";" read -r -a arr <<< "${fontDir}" + font="${arr[0]}" + subfolder="${arr[1]}" regular="$font-Regular.ttf" bold="$font-Bold.ttf" - download "${FONTDIR}/${regular}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${regular}" - download "${FONTDIR}/${bold}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${bold}" + # remove UI from South Asian scripts path (not consistent with Arabic, Khmer, Thai) + font="${font/BengaliUI/Bengali}" + font="${font/GujaratiUI/Gujarati}" + font="${font/GurmukhiUI/Gurmukhi}" + font="${font/KannadaUI/Kannada}" + font="${font/MalayalamUI/Malayalam}" + font="${font/SinhalaUI/Sinhala}" + font="${font/TamilUI/Tamil}" + font="${font/TeluguUI/Telugu}" + download "${FONTDIR}/${regular}" "https://notofonts.github.io/${subfolder}/fonts/${font}/hinted/ttf/${regular}" + download "${FONTDIR}/${bold}" "https://notofonts.github.io/${subfolder}/fonts/${font}/hinted/ttf/${bold}" done -for font in $REGULAR_BLACK; do +for fontDir in $REGULAR_BLACK; do + IFS=";" read -r -a arr <<< "${fontDir}" + font="${arr[0]}" + subfolder="${arr[1]}" regular="$font-Regular.ttf" black="$font-Black.ttf" - download "${FONTDIR}/${regular}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${regular}" - download "${FONTDIR}/${black}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${black}" + download "${FONTDIR}/${regular}" "https://notofonts.github.io/${subfolder}/fonts/${font}/hinted/ttf/${regular}" + download "${FONTDIR}/${black}" "https://notofonts.github.io/${subfolder}/fonts/${font}/hinted/ttf/${black}" done -for font in $REGULAR; do +for fontDir in $REGULAR; do + IFS=";" read -r -a arr <<< "${fontDir}" + font="${arr[0]}" + subfolder="${arr[1]}" regular="$font-Regular.ttf" - download "${FONTDIR}/${regular}" "https://github.com/notofonts/noto-fonts/raw/main/hinted/ttf/${font}/${regular}" + download "${FONTDIR}/${regular}" "https://notofonts.github.io/${subfolder}/fonts/${font}/hinted/ttf/${regular}" done # Other noto fonts which don't follow the URL pattern above From e68aeebff96fbdff0b98c654b961bd6d596601e2 Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Sat, 28 Oct 2023 13:05:59 -0500 Subject: [PATCH 02/12] oriya --- style/fonts.mss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/style/fonts.mss b/style/fonts.mss index e4ca039cd..353cfbb65 100644 --- a/style/fonts.mss +++ b/style/fonts.mss @@ -93,7 +93,7 @@ A regular style. "Noto Sans New Tai Lue Regular", "Noto Sans NKo Regular", "Noto Sans Ol Chiki Regular", - "Noto Sans Oriya UI Regular", + "Noto Sans Oriya Regular", "Noto Sans Osage Regular", "Noto Sans Osmanya Regular", "Noto Sans Samaritan Regular", @@ -152,7 +152,7 @@ regular text and can be used for emphasis. Fallback is a regular style. "Noto Sans Malayalam UI Bold", "Noto Sans Myanmar UI Bold", "Noto Sans Ol Chiki Bold", - "Noto Sans Oriya UI Bold", + "Noto Sans Oriya Bold", "Noto Sans Sinhala UI Bold", "Noto Sans Sundanese Bold", "Noto Sans Symbols Bold", From c4dcb53598bfe5473f3f798ce757659e6b8a94a8 Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Mon, 13 Nov 2023 17:42:29 -0600 Subject: [PATCH 03/12] change bash script header --- scripts/get-fonts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-fonts.sh b/scripts/get-fonts.sh index 6abf219b5..1a6686683 100755 --- a/scripts/get-fonts.sh +++ b/scripts/get-fonts.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash set -e FONTDIR="./fonts" From 648ccbdaed8c058494bd6169c083379df8401c6b Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Thu, 4 Jan 2024 18:06:03 -0600 Subject: [PATCH 04/12] customizable font directory Co-authored-by: Hummeltech <6109326+hummeltech@users.noreply.github.com> --- scripts/get-fonts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-fonts.sh b/scripts/get-fonts.sh index 1a6686683..7281752cb 100755 --- a/scripts/get-fonts.sh +++ b/scripts/get-fonts.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -FONTDIR="./fonts" +FONTDIR="${FONTDIR:-./fonts}" mkdir -p "${FONTDIR}" From 431a50d446c7c4dc875ecb50261e4ec3cf05877a Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Sun, 28 Jan 2024 21:21:46 -0600 Subject: [PATCH 05/12] Python script takes over functions of get-fonts.sh --- scripts/get-fonts.py | 210 +++++++++++++++++++++++++++++++++++++++++++ scripts/get-fonts.sh | 150 ------------------------------- 2 files changed, 210 insertions(+), 150 deletions(-) create mode 100644 scripts/get-fonts.py delete mode 100755 scripts/get-fonts.sh diff --git a/scripts/get-fonts.py b/scripts/get-fonts.py new file mode 100644 index 000000000..b0f55cc01 --- /dev/null +++ b/scripts/get-fonts.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +# This script downloads several built fonts from repos of https://github.com/notofonts +# Additional fonts can be found on https://notofonts.github.io + +import os +import requests +import tempfile +import shutil +import warnings +import zipfile + +FONTDIR = os.environ.get("FONTDIR", "./fonts") + +try: + os.mkdir(FONTDIR) +except FileExistsError: + warnings.warn("Font directory already exists") + +# Fonts available in regular, bold, and italic +REGULAR_BOLD_ITALIC = ["NotoSans"] + +# Fonts available in regular and bold +REGULAR_BOLD = [ + "NotoSansAdlamUnjoined", + "NotoSansArabicUI", + "NotoSerifArmenian", + "NotoSansBalinese", + "NotoSansBamum", + "NotoSansBengaliUI", + "NotoSansCanadianAboriginal", + "NotoSansCham", + "NotoSansCherokee", + "NotoSansDevanagariUI", + "NotoSansEthiopic", + "NotoSansGeorgian", + "NotoSansGujaratiUI", + "NotoSansGurmukhiUI", + "NotoSansHebrew", + "NotoSansJavanese", + "NotoSansKannadaUI", + "NotoSansKayahLi", + "NotoSansKhmerUI", + "NotoSansLaoUI", + "NotoSansLisu", + "NotoSansMalayalamUI", + "NotoSansMyanmarUI", + "NotoSansOlChiki", + "NotoSansOriya", + "NotoSansSinhalaUI", + "NotoSansSundanese", + "NotoSansSymbols", + "NotoSansTaiTham", + "NotoSansTamilUI", + "NotoSansTeluguUI", + "NotoSansThaana", + "NotoSansThaiUI", + "NotoSerifTibetan", +] + +# Fonts with regular and black, but no bold +REGULAR_BLACK = ["NotoSansSyriac"] + +# Fonts only available in regular +REGULAR = [ + "NotoSansBatak", + "NotoSansBuginese", + "NotoSansBuhid", + "NotoSansChakma", + "NotoSansCoptic", + "NotoSansHanunoo", + "NotoSansLepcha", + "NotoSansLimbu", + "NotoSansMandaic", + "NotoSansMongolian", + "NotoSansNewTaiLue", + "NotoSansNKo", + "NotoSansOsage", + "NotoSansOsmanya", + "NotoSansSamaritan", + "NotoSansSaurashtra", + "NotoSansShavian", + "NotoSansSymbols2", + "NotoSansTagalog", + "NotoSansTagbanwa", + "NotoSansTaiLe", + "NotoSansTaiViet", + "NotoSansTifinagh", + "NotoSansVai", + "NotoSansYi", +] + +SOUTH_ASIAN_UI_FONTS = [ + "BengaliUI", + "GujaratiUI", + "GurmukhiUI", + "KannadaUI", + "MalayalamUI", + "SinhalaUI", + "TamilUI", + "TeluguUI", +] + +# hyphenated or irregular Noto repo names +NOTO_REPO_FOR_FONT = { + "NotoSans": "latin-greek-cyrillic", + "NotoSansAdlamUnjoined": "adlam", + "NotoSansCanadianAboriginal": "canadian-aboriginal", + "NotoSansKayahLi": "kayah-li", + "NotoSansNewTaiLue": "new-tai-lue", + "NotoSansOlChiki": "ol-chiki", + "NotoSansSymbols2": "symbols", + "NotoSansTaiLe": "tai-le", + "NotoSansTaiTham": "tai-tham", + "NotoSansTaiViet": "tai-viet", + "NotoSerifArmenian": "armenian", + "NotoSerifTibetan": "tibetan", +} + +# Download the fonts in the lists above +def findFontUrl(fontName, modifier): + # remove 'UI' from South Asian font names (not consistent with Arabic, Khmer, Thai) + if fontName.replace("NotoSans", "") in SOUTH_ASIAN_UI_FONTS: + fontName = fontName.replace("UI", "") + + # pick up regular and irregular repo names for path + repo = NOTO_REPO_FOR_FONT.get( + fontName, fontName.replace("NotoSans", "").replace("UI", "").lower() + ) + + return f"{repo}/fonts/{fontName}/hinted/ttf/{fontName}-{modifier}.ttf" + + +def downloadToFile(url, destination, dir=FONTDIR): + headers = {"User-Agent": "get-fonts.py/osm-carto"} + try: + r = requests.get(url, headers=headers) + if r.status_code != 200: + raise Exception + with open(os.path.join(dir, destination), "wb") as f: + f.write(r.content) + except: + raise Exception(f"Failed to download {url}") + + +for font in REGULAR_BOLD + REGULAR_BOLD_ITALIC + REGULAR_BLACK + REGULAR: + regular = f"{font}-Regular.ttf" + regularFontUrl = findFontUrl(font, "Regular") + downloadToFile(f"https://notofonts.github.io/{regularFontUrl}", regular) + + if (font in REGULAR_BOLD) or (font in REGULAR_BOLD_ITALIC): + bold = f"{font}-Bold.ttf" + boldFontUrl = findFontUrl(font, "Bold") + downloadToFile(f"https://notofonts.github.io/{boldFontUrl}", bold) + + if font in REGULAR_BOLD_ITALIC: + italic = f"{font}-Italic.ttf" + italicFontUrl = findFontUrl(font, "Italic") + downloadToFile(f"https://notofonts.github.io/{italicFontUrl}", italic) + + if font in REGULAR_BLACK: + black = f"{font}-Black.ttf" + blackFontUrl = findFontUrl(font, "Black") + downloadToFile(f"https://notofonts.github.io/{blackFontUrl}", black) + +# Other noto fonts which don't follow the URL pattern above + +# CJK fonts +downloadToFile( + "https://github.com/notofonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Regular.otf", + "NotoSansCJKjp-Regular.otf", +) +downloadToFile( + "https://github.com/notofonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Bold.otf", + "NotoSansCJKjp-Bold.otf", +) + +# Fonts in zipfiles need a temporary directory +TMPDIR = tempfile.mkdtemp(prefix="get-fonts.") + +# Noto Emoji B&W isn't available as a separate download, so we need to download the package and unzip it +downloadToFile( + "https://fonts.google.com/download?family=Noto%20Emoji", + "Noto_Emoji.zip", + dir=TMPDIR, +) +emojiPath = os.path.join(TMPDIR, "Noto_Emoji.zip") +emojiExtract = ["static/NotoEmoji-Regular.ttf", "static/NotoEmoji-Bold.ttf"] +with zipfile.ZipFile(emojiPath, "r") as zip_ref: + for file in emojiExtract: + source = zip_ref.getinfo(file) + zip_ref.extract(source, FONTDIR) + # move from FONTDIR/static/x to FONTDIR + shutil.move(os.path.join(FONTDIR, file), FONTDIR) + +downloadToFile( + "https://mirrors.dotsrc.org/osdn/hanazono-font/68253/hanazono-20170904.zip", + "hanazono.zip", + dir=TMPDIR, +) +hanazonoPath = os.path.join(TMPDIR, "hanazono.zip") +with zipfile.ZipFile(hanazonoPath, "r") as zip_ref: + for file in ["HanaMinA.ttf", "HanaMinB.ttf"]: + source = zip_ref.getinfo(file) + zip_ref.extract(source, FONTDIR) + +# clean up tmp directories +shutil.rmtree(TMPDIR) +fontdir_static = os.path.join(FONTDIR, "static") +if os.path.exists(fontdir_static): + shutil.rmtree(fontdir_static) diff --git a/scripts/get-fonts.sh b/scripts/get-fonts.sh deleted file mode 100755 index 7281752cb..000000000 --- a/scripts/get-fonts.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash -set -e - -FONTDIR="${FONTDIR:-./fonts}" - -mkdir -p "${FONTDIR}" - -# download filename url -download() { - ## Download if newer, and if curl fails, clean up and exit - curl --fail --compressed -A "get-fonts.sh/osm-carto" -o "$1" -z "$1" -L "$2" || { echo "Failed to download $1 $2"; rm -f "$1"; exit 1; } -} - -# TTF Hinted Noto Fonts - -# Fonts available in regular, bold, and italic -REGULAR_BOLD_ITALIC="NotoSans" - -# Fonts available in regular and bold -REGULAR_BOLD="NotoSansAdlamUnjoined;adlam -NotoSansArabicUI;arabic -NotoSansArmenian;armenian -NotoSansBalinese;balinese -NotoSansBamum;bamum -NotoSansBengaliUI;bengali -NotoSansCanadianAboriginal;canadian-aboriginal -NotoSansCham;cham -NotoSansCherokee;cherokee -NotoSansDevanagariUI;devanagari -NotoSansEthiopic;ethiopic -NotoSansGeorgian;georgian -NotoSansGujaratiUI;gujarati -NotoSansGurmukhiUI;gurmukhi -NotoSansHebrew;hebrew -NotoSansJavanese;javanese -NotoSansKannadaUI;kannada -NotoSansKayahLi;kayah-li -NotoSansKhmerUI;khmer -NotoSansLaoUI;lao -NotoSansLisu;lisu -NotoSansMalayalamUI;malayalam -NotoSansMyanmarUI;myanmar -NotoSansOlChiki;ol-chiki -NotoSansOriya;oriya -NotoSansSinhalaUI;sinhala -NotoSansSundanese;sundanese -NotoSansSymbols;symbols -NotoSansTaiTham;tai-tham -NotoSansTamilUI;tamil -NotoSansTeluguUI;telugu -NotoSansThaana;thaana -NotoSansThaiUI;thai -NotoSerifTibetan;tibetan" - -# Fonts with regular and black, but no bold -REGULAR_BLACK="NotoSansSyriac;syriac" - -# Fonts only available in regular -REGULAR="NotoSansBatak;batak -NotoSansBuginese;buginese -NotoSansBuhid;buhid -NotoSansChakma;chakma -NotoSansCoptic;coptic -NotoSansHanunoo;hanunoo -NotoSansLepcha;lepcha -NotoSansLimbu;limbu -NotoSansMandaic;mandaic -NotoSansMongolian;mongolian -NotoSansNewTaiLue;new-tai-lue -NotoSansNKo;nko -NotoSansOsage;osage -NotoSansOsmanya;osmanya -NotoSansSamaritan;samaritan -NotoSansSaurashtra;saurashtra -NotoSansShavian;shavian -NotoSansSymbols2;symbols -NotoSansTagalog;tagalog -NotoSansTagbanwa;tagbanwa -NotoSansTaiLe;tai-le -NotoSansTaiViet;tai-viet -NotoSansTifinagh;tifinagh -NotoSansVai;vai -NotoSansYi;yi" - -# Download the fonts in the lists above - -for font in $REGULAR_BOLD_ITALIC; do - regular="$font-Regular.ttf" - bold="$font-Bold.ttf" - italic="$font-Italic.ttf" - download "${FONTDIR}/${regular}" "https://notofonts.github.io/latin-greek-cyrillic/fonts/${font}/hinted/ttf/${regular}" - download "${FONTDIR}/${bold}" "https://notofonts.github.io/latin-greek-cyrillic/fonts/${font}/hinted/ttf/${bold}" - download "${FONTDIR}/${italic}" "https://notofonts.github.io/latin-greek-cyrillic/fonts/${font}/hinted/ttf/${italic}" -done - -for fontDir in $REGULAR_BOLD; do - IFS=";" read -r -a arr <<< "${fontDir}" - font="${arr[0]}" - subfolder="${arr[1]}" - regular="$font-Regular.ttf" - bold="$font-Bold.ttf" - # remove UI from South Asian scripts path (not consistent with Arabic, Khmer, Thai) - font="${font/BengaliUI/Bengali}" - font="${font/GujaratiUI/Gujarati}" - font="${font/GurmukhiUI/Gurmukhi}" - font="${font/KannadaUI/Kannada}" - font="${font/MalayalamUI/Malayalam}" - font="${font/SinhalaUI/Sinhala}" - font="${font/TamilUI/Tamil}" - font="${font/TeluguUI/Telugu}" - download "${FONTDIR}/${regular}" "https://notofonts.github.io/${subfolder}/fonts/${font}/hinted/ttf/${regular}" - download "${FONTDIR}/${bold}" "https://notofonts.github.io/${subfolder}/fonts/${font}/hinted/ttf/${bold}" -done - -for fontDir in $REGULAR_BLACK; do - IFS=";" read -r -a arr <<< "${fontDir}" - font="${arr[0]}" - subfolder="${arr[1]}" - regular="$font-Regular.ttf" - black="$font-Black.ttf" - download "${FONTDIR}/${regular}" "https://notofonts.github.io/${subfolder}/fonts/${font}/hinted/ttf/${regular}" - download "${FONTDIR}/${black}" "https://notofonts.github.io/${subfolder}/fonts/${font}/hinted/ttf/${black}" -done - -for fontDir in $REGULAR; do - IFS=";" read -r -a arr <<< "${fontDir}" - font="${arr[0]}" - subfolder="${arr[1]}" - regular="$font-Regular.ttf" - download "${FONTDIR}/${regular}" "https://notofonts.github.io/${subfolder}/fonts/${font}/hinted/ttf/${regular}" -done - -# Other noto fonts which don't follow the URL pattern above -download "${FONTDIR}/NotoSansCJKjp-Regular.otf" "https://github.com/googlefonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Regular.otf" -download "${FONTDIR}/NotoSansCJKjp-Bold.otf" "https://github.com/googlefonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Bold.otf" - -# Fonts in zipfiles need a temporary directory -TMPDIR=$(mktemp -d -t get-fonts.XXXXXXXXX) -trap "rm -rf ${TMPDIR} ${FONTDIR}/static" EXIT - -# Noto Emoji B&W isn't available as a separate download, so we need to download the package and unzip it -curl --fail -A "get-fonts.sh/osm-carto" -o "${TMPDIR}/Noto_Emoji.zip" -L 'https://fonts.google.com/download?family=Noto%20Emoji' - -unzip -oqq "${TMPDIR}/Noto_Emoji.zip" static/NotoEmoji-Regular.ttf static/NotoEmoji-Bold.ttf -d "${FONTDIR}" -mv "${FONTDIR}/static/NotoEmoji-Regular.ttf" "${FONTDIR}" -mv "${FONTDIR}/static/NotoEmoji-Bold.ttf" "${FONTDIR}" - -curl --fail -A "get-fonts.sh/osm-carto" -o "${TMPDIR}/hanazono.zip" -L 'https://mirrors.dotsrc.org/osdn/hanazono-font/68253/hanazono-20170904.zip' - -unzip -oqq "${TMPDIR}/hanazono.zip" HanaMinA.ttf HanaMinB.ttf -d "${FONTDIR}" From 524c5e46e03d11713b831696774c0517776f00cb Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Sun, 28 Jan 2024 21:26:35 -0600 Subject: [PATCH 06/12] install / docker script changes --- INSTALL.md | 2 +- scripts/change-fonts-cjk.sh | 2 +- scripts/docker-startup.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 74029fb1a..aa5a6de88 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -68,7 +68,7 @@ For more details, see the documentation at [fonts.mss](style/fonts.mss). To download the fonts, run the following script ``` -scripts/get-fonts.sh +scripts/get-fonts.py ``` ## Dependencies diff --git a/scripts/change-fonts-cjk.sh b/scripts/change-fonts-cjk.sh index 1ef492942..b951c8dc1 100755 --- a/scripts/change-fonts-cjk.sh +++ b/scripts/change-fonts-cjk.sh @@ -8,7 +8,7 @@ mkdir -p "${FONTDIR}" # download filename url download() { ## Download if newer, and if curl fails, clean up and exit - curl --fail --compressed -A "get-fonts.sh/osm-carto" -o "$1" -z "$1" -L "$2" || { echo "Failed to download $1 $2"; rm -f "$1"; exit 1; } + curl --fail --compressed -A "change-fonts-cjk.sh/osm-carto" -o "$1" -z "$1" -L "$2" || { echo "Failed to download $1 $2"; rm -f "$1"; exit 1; } } case "$1" in diff --git a/scripts/docker-startup.sh b/scripts/docker-startup.sh index c3177ee9d..45abc15c6 100644 --- a/scripts/docker-startup.sh +++ b/scripts/docker-startup.sh @@ -56,7 +56,7 @@ EOF scripts/get-external-data.py $EXTERNAL_DATA_SCRIPT_FLAGS # Download fonts - scripts/get-fonts.sh + scripts/get-fonts.py ;; kosmtik) From bb848f8452f56e21dc41bbfde76af63e49eb6d5a Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Sun, 28 Jan 2024 21:28:37 -0600 Subject: [PATCH 07/12] armenian --- scripts/get-fonts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-fonts.py b/scripts/get-fonts.py index b0f55cc01..cab069137 100644 --- a/scripts/get-fonts.py +++ b/scripts/get-fonts.py @@ -23,7 +23,7 @@ REGULAR_BOLD = [ "NotoSansAdlamUnjoined", "NotoSansArabicUI", - "NotoSerifArmenian", + "NotoSansArmenian", "NotoSansBalinese", "NotoSansBamum", "NotoSansBengaliUI", From ea888403533d715df3245c3f6b7b6e732568520f Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Sun, 28 Jan 2024 21:39:10 -0600 Subject: [PATCH 08/12] better handling UI fonts and copying files --- scripts/get-fonts.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/get-fonts.py b/scripts/get-fonts.py index cab069137..e8dfd5526 100644 --- a/scripts/get-fonts.py +++ b/scripts/get-fonts.py @@ -120,14 +120,16 @@ def findFontUrl(fontName, modifier): # remove 'UI' from South Asian font names (not consistent with Arabic, Khmer, Thai) if fontName.replace("NotoSans", "") in SOUTH_ASIAN_UI_FONTS: - fontName = fontName.replace("UI", "") + subfolder = fontName.replace("UI", "") + else: + subfolder = fontName # pick up regular and irregular repo names for path repo = NOTO_REPO_FOR_FONT.get( fontName, fontName.replace("NotoSans", "").replace("UI", "").lower() ) - return f"{repo}/fonts/{fontName}/hinted/ttf/{fontName}-{modifier}.ttf" + return f"{repo}/fonts/{subfolder}/hinted/ttf/{fontName}-{modifier}.ttf" def downloadToFile(url, destination, dir=FONTDIR): @@ -189,7 +191,10 @@ def downloadToFile(url, destination, dir=FONTDIR): for file in emojiExtract: source = zip_ref.getinfo(file) zip_ref.extract(source, FONTDIR) - # move from FONTDIR/static/x to FONTDIR + # move from FONTDIR/static/x to overwrite FONTDIR/x + unzipSrc = os.path.join(FONTDIR, file) + if os.path.exists(unzipSrc): + os.remove(unzipSrc) shutil.move(os.path.join(FONTDIR, file), FONTDIR) downloadToFile( From 6ab9467081f433eccc3431af2d7311d4c83aa747 Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Mon, 29 Jan 2024 00:29:52 -0600 Subject: [PATCH 09/12] fix static/Noto and fallback URL for notofonts --- scripts/get-fonts.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/get-fonts.py b/scripts/get-fonts.py index e8dfd5526..5ed22b3e2 100644 --- a/scripts/get-fonts.py +++ b/scripts/get-fonts.py @@ -137,7 +137,17 @@ def downloadToFile(url, destination, dir=FONTDIR): try: r = requests.get(url, headers=headers) if r.status_code != 200: - raise Exception + if "notofonts.github.io" in url: + warnings.warn( + f"Failed to download {url}, retrying with raw.githubusercontent.com" + ) + modurl = ( + "https://raw.githubusercontent.com/notofonts/notofonts.github.io/main" + + url[url.index("/fonts") :] + ) + downloadToFile(modurl, destination, dir=dir) + else: + raise Exception with open(os.path.join(dir, destination), "wb") as f: f.write(r.content) except: @@ -186,16 +196,16 @@ def downloadToFile(url, destination, dir=FONTDIR): dir=TMPDIR, ) emojiPath = os.path.join(TMPDIR, "Noto_Emoji.zip") -emojiExtract = ["static/NotoEmoji-Regular.ttf", "static/NotoEmoji-Bold.ttf"] +emojiExtract = ["NotoEmoji-Regular.ttf", "NotoEmoji-Bold.ttf"] with zipfile.ZipFile(emojiPath, "r") as zip_ref: for file in emojiExtract: - source = zip_ref.getinfo(file) + source = zip_ref.getinfo(f"static/{file}") zip_ref.extract(source, FONTDIR) # move from FONTDIR/static/x to overwrite FONTDIR/x unzipSrc = os.path.join(FONTDIR, file) if os.path.exists(unzipSrc): os.remove(unzipSrc) - shutil.move(os.path.join(FONTDIR, file), FONTDIR) + shutil.move(os.path.join(FONTDIR, "static", file), FONTDIR) downloadToFile( "https://mirrors.dotsrc.org/osdn/hanazono-font/68253/hanazono-20170904.zip", From ea80304bdb9816ff1da6a1b0abdd0962d01a27b8 Mon Sep 17 00:00:00 2001 From: ndoiron Date: Mon, 5 Feb 2024 12:47:10 +0700 Subject: [PATCH 10/12] prefer jsdelivr then language-specific repo; Arabic UI deprecated --- scripts/get-fonts.py | 63 ++++++++++++++++++++++---------------------- style/fonts.mss | 4 +-- 2 files changed, 33 insertions(+), 34 deletions(-) mode change 100644 => 100755 scripts/get-fonts.py diff --git a/scripts/get-fonts.py b/scripts/get-fonts.py old mode 100644 new mode 100755 index 5ed22b3e2..ee8f4c9d2 --- a/scripts/get-fonts.py +++ b/scripts/get-fonts.py @@ -22,7 +22,7 @@ # Fonts available in regular and bold REGULAR_BOLD = [ "NotoSansAdlamUnjoined", - "NotoSansArabicUI", + "NotoSansArabic", "NotoSansArmenian", "NotoSansBalinese", "NotoSansBamum", @@ -91,6 +91,7 @@ SOUTH_ASIAN_UI_FONTS = [ "BengaliUI", + "DevanagariUI", "GujaratiUI", "GurmukhiUI", "KannadaUI", @@ -112,13 +113,13 @@ "NotoSansTaiLe": "tai-le", "NotoSansTaiTham": "tai-tham", "NotoSansTaiViet": "tai-viet", - "NotoSerifArmenian": "armenian", "NotoSerifTibetan": "tibetan", } + # Download the fonts in the lists above -def findFontUrl(fontName, modifier): - # remove 'UI' from South Asian font names (not consistent with Arabic, Khmer, Thai) +def findFontUrls(fontName, modifier, useRepo=False): + # remove 'UI' from South Asian font names for subfolder name if fontName.replace("NotoSans", "") in SOUTH_ASIAN_UI_FONTS: subfolder = fontName.replace("UI", "") else: @@ -129,23 +130,21 @@ def findFontUrl(fontName, modifier): fontName, fontName.replace("NotoSans", "").replace("UI", "").lower() ) - return f"{repo}/fonts/{subfolder}/hinted/ttf/{fontName}-{modifier}.ttf" + return [ + f"https://cdn.jsdelivr.net/gh/notofonts/notofonts.github.io/fonts/{subfolder}/hinted/ttf/{fontName}-{modifier}.ttf", + f"https://notofonts.github.io/{repo}/fonts/{subfolder}/hinted/ttf/{fontName}-{modifier}.ttf", + ] -def downloadToFile(url, destination, dir=FONTDIR): +def downloadToFile(urls, destination, dir=FONTDIR): headers = {"User-Agent": "get-fonts.py/osm-carto"} + try: - r = requests.get(url, headers=headers) + r = requests.get(urls[0], headers=headers) if r.status_code != 200: - if "notofonts.github.io" in url: - warnings.warn( - f"Failed to download {url}, retrying with raw.githubusercontent.com" - ) - modurl = ( - "https://raw.githubusercontent.com/notofonts/notofonts.github.io/main" - + url[url.index("/fonts") :] - ) - downloadToFile(modurl, destination, dir=dir) + if len(urls) > 1: + warnings.warn(f"Failed to download {urls[0]}, retrying with repo HEAD") + downloadToFile(urls[1:], destination, dir=dir) else: raise Exception with open(os.path.join(dir, destination), "wb") as f: @@ -155,34 +154,34 @@ def downloadToFile(url, destination, dir=FONTDIR): for font in REGULAR_BOLD + REGULAR_BOLD_ITALIC + REGULAR_BLACK + REGULAR: - regular = f"{font}-Regular.ttf" - regularFontUrl = findFontUrl(font, "Regular") - downloadToFile(f"https://notofonts.github.io/{regularFontUrl}", regular) + regularFontUrls = findFontUrls(font, "Regular") + downloadToFile(regularFontUrls, f"{font}-Regular.ttf") if (font in REGULAR_BOLD) or (font in REGULAR_BOLD_ITALIC): - bold = f"{font}-Bold.ttf" - boldFontUrl = findFontUrl(font, "Bold") - downloadToFile(f"https://notofonts.github.io/{boldFontUrl}", bold) + boldFontUrls = findFontUrls(font, "Bold") + downloadToFile(boldFontUrls, f"{font}-Bold.ttf") if font in REGULAR_BOLD_ITALIC: - italic = f"{font}-Italic.ttf" - italicFontUrl = findFontUrl(font, "Italic") - downloadToFile(f"https://notofonts.github.io/{italicFontUrl}", italic) + italicFontUrls = findFontUrls(font, "Italic") + downloadToFile(italicFontUrls, f"{font}-Italic.ttf") if font in REGULAR_BLACK: - black = f"{font}-Black.ttf" - blackFontUrl = findFontUrl(font, "Black") - downloadToFile(f"https://notofonts.github.io/{blackFontUrl}", black) + blackFontUrls = findFontUrls(font, "Black") + downloadToFile(blackFontUrls, f"{font}-Black.ttf") # Other noto fonts which don't follow the URL pattern above # CJK fonts downloadToFile( - "https://github.com/notofonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Regular.otf", + [ + "https://github.com/notofonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Regular.otf" + ], "NotoSansCJKjp-Regular.otf", ) downloadToFile( - "https://github.com/notofonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Bold.otf", + [ + "https://github.com/notofonts/noto-cjk/raw/main/Sans/OTF/Japanese/NotoSansCJKjp-Bold.otf" + ], "NotoSansCJKjp-Bold.otf", ) @@ -191,7 +190,7 @@ def downloadToFile(url, destination, dir=FONTDIR): # Noto Emoji B&W isn't available as a separate download, so we need to download the package and unzip it downloadToFile( - "https://fonts.google.com/download?family=Noto%20Emoji", + ["https://fonts.google.com/download?family=Noto%20Emoji"], "Noto_Emoji.zip", dir=TMPDIR, ) @@ -208,7 +207,7 @@ def downloadToFile(url, destination, dir=FONTDIR): shutil.move(os.path.join(FONTDIR, "static", file), FONTDIR) downloadToFile( - "https://mirrors.dotsrc.org/osdn/hanazono-font/68253/hanazono-20170904.zip", + ["https://mirrors.dotsrc.org/osdn/hanazono-font/68253/hanazono-20170904.zip"], "hanazono.zip", dir=TMPDIR, ) diff --git a/style/fonts.mss b/style/fonts.mss index 353cfbb65..ba57bd1fe 100644 --- a/style/fonts.mss +++ b/style/fonts.mss @@ -58,7 +58,7 @@ A regular style. */ @book-fonts: "Noto Sans Regular", "Noto Sans Adlam Unjoined Regular", - "Noto Sans Arabic UI Regular", + "Noto Sans Arabic Regular", "Noto Sans Armenian Regular", "Noto Sans Balinese Regular", "Noto Sans Bamum Regular", @@ -129,7 +129,7 @@ regular text and can be used for emphasis. Fallback is a regular style. */ @bold-fonts: "Noto Sans Bold", "Noto Sans Adlam Unjoined Bold", - "Noto Sans Arabic UI Bold", + "Noto Sans Arabic Bold", "Noto Sans Armenian Bold", "Noto Sans Balinese Bold", "Noto Sans Bamum Bold", From 40d6b76aaeaefd034b22cfe7285f5fde61eb839e Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Sat, 1 Jun 2024 13:50:26 -0500 Subject: [PATCH 11/12] use archive.org link from #4956 --- scripts/get-fonts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-fonts.py b/scripts/get-fonts.py index ee8f4c9d2..14425232e 100755 --- a/scripts/get-fonts.py +++ b/scripts/get-fonts.py @@ -190,7 +190,7 @@ def downloadToFile(urls, destination, dir=FONTDIR): # Noto Emoji B&W isn't available as a separate download, so we need to download the package and unzip it downloadToFile( - ["https://fonts.google.com/download?family=Noto%20Emoji"], + ["https://archive.org/download/noto-emoji/Noto_Emoji.zip"], "Noto_Emoji.zip", dir=TMPDIR, ) From ee511e46c0d2cea9d36428dd44e9da4154188e0a Mon Sep 17 00:00:00 2001 From: Nick Doiron Date: Sun, 17 Nov 2024 07:42:16 -0600 Subject: [PATCH 12/12] Update scripts/get-fonts.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Pétillon --- scripts/get-fonts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-fonts.py b/scripts/get-fonts.py index 14425232e..2eb6df5d4 100755 --- a/scripts/get-fonts.py +++ b/scripts/get-fonts.py @@ -150,7 +150,7 @@ def downloadToFile(urls, destination, dir=FONTDIR): with open(os.path.join(dir, destination), "wb") as f: f.write(r.content) except: - raise Exception(f"Failed to download {url}") + raise Exception(f"Failed to download {urls}") for font in REGULAR_BOLD + REGULAR_BOLD_ITALIC + REGULAR_BLACK + REGULAR: