From 792193d230e6e68ccf79af0bfb7acf6f07d8e730 Mon Sep 17 00:00:00 2001 From: me <> Date: Tue, 9 Jan 2024 20:05:01 +0100 Subject: [PATCH] init --- .gitignore | 1 + README.md | 61 ++++++ hgh.json | 27 +++ hgh.py | 353 ++++++++++++++++++++++++++++++ templates/default.jinja | 16 ++ templates/fotos_letzte_tage.jinja | 16 ++ 6 files changed, 474 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 hgh.json create mode 100755 hgh.py create mode 100644 templates/default.jinja create mode 100644 templates/fotos_letzte_tage.jinja diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..722d5e7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vscode diff --git a/README.md b/README.md new file mode 100644 index 0000000..e27f13d --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +# hgh – Hugo Helper + +A Python script to do magic when working with Hugo websites. + +It was created for my websites. So this is not for general purpose. If it works for you, great. If not, fine. + +Most things should work for others without change, but commands like `--create-pagebundle` probably need to be adapted for everyone else. + +Ich nix English. Habs versucht. :P + +## Dependencies + +### Python + +argparse, os, time, re, subprocess, json, sys, datetime, pytz, jinja2, tempfile, glob, pathlib + +### Tools + +- find (Should be already installed on most Linux systems) +- ripgrep (binary is rg) +- fzf (Fuzzy Find for some commands) +- hugo +- yq (only for --create-tags-categories-caches) +- VSCode (optional and only used to open a new created page bundle) + +## Installation + +- Install dependencies (Python modules and tools). +- Create `~/.config/hgh/` and copy `hgh.json` into this directory. +- Create `~/.cache/hgh/` directory. +- Move `hgh.py` file and `templates` directory to anywhere on your system. +- In `hgh.json` config file adapt at least: (Do not use `~` for paths within the config file.) + - "settings" -> "template_path" + - "settings" -> "shell" (default: /bin/bash) + - "settings" -> "bin_editor" (default: micro) + - "sites" -> everything… +- run `hgh.py --printconfig` or `hgh.py --site xyz -cdw` for testing.. + +## Find tags and/or categories + +To get a list of tags and categories used within a website, run from time to time: + +``` +hgh.py --site xyz --create-tags-categories-caches +``` + +This creates two cache files with counts of tags and categories. See `sites -> "list_tag"` and `sites -> "list_categories"` in `hgh.json` for the file paths. + +## Create Page Bundles + +If you want to use creation of page bundles with `--create-pagebundle`, please adapt paths and logic in `if args.create_pagebundle:` of `hgh.py`. Without changes it works only for Natenoms blog. + +## Misc + +### Tipps + +Um nicht dauernd `hgh.py --site blog` eingeben zu müssen, kann man einen Alias in der Shell einrichten und in die `.bashrc` eintragen: + +``` bash +alias hgb='hgh.py --site blog' +``` diff --git a/hgh.json b/hgh.json new file mode 100644 index 0000000..770eb29 --- /dev/null +++ b/hgh.json @@ -0,0 +1,27 @@ +{"settings": + { + "default_template": "default", + "template_path": "/home/user/.config/hgh/templates", + "shell": "/bin/bash", + "open_new_pagebundle_with_vscode": true, + "slug_max_length": 160, + "bin_vscode": "flatpak run com.vscodium.codium", + "bin_editor": "nano", + "comments_filename": "comments.json" + }, + "sites": + { + "blog": + { "basedir": "/home/user/websites/domain.de/web_blog", + "contentdir": "/home/user/websites/domain.de/web_blog/content", + "contentdir-light": "/home/user/somewhere_else/domain.de/light_blog_for_testing/content", + "build_dir": "public/", + "remote_user": "user", + "remote_host": "host", + "remote_path": "/var/www/domain.de/htdocs", + "base_url": "https://domain.de/", + "list_tags": "/home/user/.cache/hgh/domain.de_tags.list", + "list_categories": "/home/user/.cache/hgh/domain.de_categories.list", + "serve_port": 1313 } + } +} diff --git a/hgh.py b/hgh.py new file mode 100755 index 0000000..b8219f8 --- /dev/null +++ b/hgh.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 + +import argparse, os, time, re, subprocess, json, sys + +config_json = os.path.expanduser("~/.config/hgh/hgh.json") + +parser = argparse.ArgumentParser(description='Handles hugo stuff..') + +group0 = parser.add_mutually_exclusive_group() +#group1 = parser.add_mutually_exclusive_group() +group2 = parser.add_mutually_exclusive_group() +group3 = parser.add_mutually_exclusive_group() + +group0.add_argument("--site", help="Name of site to work with. Use --sites for a list of all available sites.", dest='site_name', action="store", type=str, default = None) +group0.add_argument("-p", "--printconfig", help="Print complete json config.", dest='printconfig', action="store_true") +group0.add_argument("--sites", help="Print all sites from from configuration", dest='sites', action="store_true") + +group2.add_argument("--create-comments-file", help="Create a comments.json in current working directory with given amount of comments.", dest='create_comments_file', action="store_true") +group2.add_argument("-cpb", "--create-pagebundle", help="Create new page bundle. Add --template for specific template to use.", dest='create_pagebundle', action="store", type=str, default = None) +group2.add_argument("--time", help="Copy current date and time to clipboard.", dest='time', action="store_true") +group2.add_argument("-cdb", help="Change directory to base directory of site.", dest='cdb', action="store_true") +group2.add_argument("-cdc", help="Change directory to contentdir of site.", dest='cdc', action="store_true") +group2.add_argument("-cdl", help="Change directory to alternative contentdir of site.", dest='cdl', action="store_true") +group2.add_argument("--build", help="Build a site.", dest='site_build', action="store_true") +group2.add_argument("--upload", help="Build AND upload site to remote.", dest='site_upload', action="store_true") +group2.add_argument("--deploy", help="Deploy (build AND upload) a site.", dest='deploy', action="store_true") +group2.add_argument("--tmux", help="Start the tmux session for --site.", dest='tmux', action="store_true") +group2.add_argument("-hs", "--hugo-server", help="Serves site (full). Use --hugo-server-environment for different environments.", dest='hugo_server', action="store_true") +group2.add_argument("-hsl", "--hugo-server-light", help="Serves site (light).", dest='hugo_server_light', action="store_true") +group2.add_argument("-ctcc", "--create-tags-categories-caches", help="Serves site (light).", dest='create_tags_categories_caches', action="store_true") + +#group3.add_argument("--deprecated_searchpost", help="Find pages/posts containing search term in 'path, slug, title, date or permalink'. Strings and Regular Expressions (regex) are allowed.", dest='searchpost', action="store", type=str, default = None) +group3.add_argument("-hse", "--hugo-server-env", help="Use this environment for hugo server. Default on hugo and here is 'development'. Full render is normally 'production'.", dest='hugo_server_environment', action="store", type=str, default = None) +group3.add_argument("-f", "--find", help="Find pages/posts containing search term in file content. Uses 'grep -r'. Strings and regex allowed.", dest='find_anything', action="store", type=str, default = None) +group3.add_argument("-ff", "--find-fuzzy", help="Fuzzy find pages/posts containing search term. Press F4 to edit a selected file with \"bin_editor\". Strings and regex allowed.", dest='find_anything_fuzzy', action="store", type=str, default = None) +group3.add_argument("-ft", "--find-tags", help="Find tags containing search term. Strings and regex are allowed.", dest='find_tags', action="store", type=str, default = None) +group3.add_argument("-fc", "--find-categories", help="Find categories containing search term. Strings and regex are allowed.", dest='find_categories', action="store", type=str, default = None) +group3.add_argument("--template", help="Use template for --create-pagebundle command.", dest='template', action="store", type=str, default = None) +group3.add_argument("--list-templates", help="List available templates.", dest='list_templates', action="store_true") + +def get_config(filename): + with open(filename, "r") as config_file: + config = json.load(config_file) + + return(config) + +def title_to_slug(title): + slug_max_length=config["settings"]["slug_max_length"] + + slug=title.replace(" ", "-") + slug=slug.lower() + if len(slug) > slug_max_length: + slug=slug[:slug_max_length].rstrip() + + special_char_map = {ord('ä'):'ae', ord('ü'):'ue', ord('ö'):'oe', ord('ß'):'ss'} + slug = slug.translate(special_char_map) + + slug = re.sub('[^0-9a-zA-Z-]+', '', slug) + + return(slug) + +def get_datetime(): + import datetime, pytz + + datetime = pytz.timezone('Europe/Berlin').localize(datetime.datetime.now()).strftime("%Y-%m-%dT%H:%M:%S%z") + datetime = "{0}:{1}".format( + datetime[:-2], + datetime[-2:] + ) # https://gist.github.com/mattstibbs/a283f55a124d2de1b1d732daaac1f7f8 + + return(datetime) + +def hugo_server(environment="development"): + if args.hugo_server_environment: + environment = args.hugo_server_environment + + print("Starting hugo-full for {site} (port:{port})\n".format(site=args.site_name, port=config["sites"][args.site_name]["serve_port"])) + subprocess.run(["hugo", "server", "-e", environment, "--logLevel", "info", "--cleanDestinationDir", "--renderToDisk", "--disableFastRender", "-E", "-D", "-F", "-p", str(config["sites"][args.site_name]["serve_port"]) ], cwd=config["sites"][args.site_name]["basedir"]) + + # ohne --disableFastRender gibt es bei meinem Theme im Blog das Problem, dass CSS komplett fehlt. Wer das nicht hat, kann das entfernen. + +def hugo_server_light(): + print("Starting hugo-light for {site} (port:{port})\n".format(site=args.site_name, port=config["sites"][args.site_name]["serve_port"])) + subprocess.run(["hugo", "server", "--logLevel", "info", "--cleanDestinationDir", "--renderToDisk", "-E", "-D", "-F", "-c", config["sites"][args.site_name]["contentdir-light"], "-p", str(config["sites"][args.site_name]["serve_port"]) ], cwd=config["sites"][args.site_name]["basedir"]) + +def start_tmux(): + # Attach to a already running session named --site or create a new one. + process = subprocess.run(["/usr/bin/tmux", "has-session", "-t", args.site_name], cwd=config["sites"][args.site_name]["basedir"]) + if process.returncode == 0: + # tmux already exists, attach + print("Attaching to tmux for {site}".format(site=args.site_name)) + subprocess.run(["/usr/bin/tmux", "attach", "-t", args.site_name], cwd=config["sites"][args.site_name]["basedir"]) + else: + # Create a new session + print("Starting tmux for {site}".format(site=args.site_name)) + os.chdir(config["sites"][args.site_name]["basedir"]) + os.system("/usr/bin/tmux new-session -s '{site_name}' -n 'Full' -c '{cwd}' \; send-keys '{editor} $(fzf)' \; split-window -v -p 5 \; new-window -n 'Light' -c '{cwd_light}' \; send-keys '{editor} $(fzf)' \; split-window -v -p 5 \; new-window -n 'Serve' \; send-keys 'hg.py --site {site_name} -sl' \; select-window -t 1 \; select-pane -t 0 \;".format(editor=config["settings"]["bin_editor"], site_name=args.site_name, cwd=config["sites"][args.site_name]["basedir"], cwd_light=config["sites"][args.site_name]["contentdir-light"])) + # first window: basedir + # second window: contentdir-light + # third window: serve with prefilled command prompt + +def site_build(): + print("Building {site}".format(site=args.site_name)) + subprocess.run(["hugo", "--minify", "--cleanDestinationDir" ], cwd=config["sites"][args.site_name]["basedir"]) + +# def searchpost(search): # hugo list all zeigt nur beiträge an, die auch im aktuellen zustand gebaut würden. Bei development bei mir also nur die Beiträge von 2023, findet also nicht alles. +# process = subprocess.run(["hugo", "list", "all" ], cwd=config["sites"][args.site_name]["basedir"], capture_output=True, universal_newlines=True) +# +# _counter = 0 +# for line in iter(process.stdout.split('\n')): +# if re.search(search, line, re.IGNORECASE): +# _counter = _counter + 1 +# print('\n' + line) +# +# +# if _counter > 0: +# print("\nFormat: path,slug,title,date,expiryDate,publishDate,draft,permalink") +# +# print("Results for '{}': {}".format(search, _counter)) + +def interactive_tags(search): + os.system("/usr/bin/cat {filename} | fzf --delimiter=':' -n 2 -q '{search}'".format(filename = config["sites"][args.site_name]["list_tags"], search=search)) + +def interactive_categories(search): + os.system("/usr/bin/cat {filename} | fzf --delimiter=':' -n 2 -q '{search}'".format(filename = config["sites"][args.site_name]["list_categories"], search=search)) + +def find_anything(search): + os.chdir(config["sites"][args.site_name]["basedir"]) + os.system("/usr/bin/rg -i --column --line-number --no-heading \'{search}\'".format(search=search)) + +def find_anything_fuzzy(search): + os.chdir(config["sites"][args.site_name]["basedir"]) + os.system("/usr/bin/rg -i --column --line-number --no-heading \'{search}\' content | fzf --bind 'f4:execute({editor} {{1}})' {bind_explorer} {preview} {preview_window} --with-nth=1.. -e --delimiter=':'".format(search=search, editor=config["settings"]["bin_editor"], preview="--preview 'batcat --style=full --color=always --line-range :500 {1}'", preview_window="--preview-window=top:50%:wrap", bind_explorer="--bind \'f3:execute(xdg-open \"$(dirname {})\")\'")) + +def site_upload(): + remote_user = config["sites"][args.site_name]["remote_user"] + remote_host = config["sites"][args.site_name]["remote_host"] + remote_path = config["sites"][args.site_name]["remote_path"] + build_dir = config["sites"][args.site_name]["build_dir"] + + subprocess.run(["/usr/bin/rsync", "-avz", "--delete-delay", build_dir, "{}@{}:{}".format(remote_user, remote_host, remote_path) ], cwd=config[args.site_name]["basedir"]) + +def get_year_month_date(): + from datetime import date + todays_date = date.today() + year = todays_date.strftime('%Y') + month = todays_date.strftime('%m') + day = todays_date.strftime('%d') + + return(year, month, day) + +def process_template(template, title, slug, datetime): + from jinja2 import Template + + filename = config["settings"]["template_path"] + template + ".jinja" + + with open(filename) as f: + tmpl = Template(f.read()) + + return(tmpl.render(title = title, slug = slug, datetime = datetime)) + +def count_and_unify_occurences_from_file(filename): + """ + Liest eine Datei ein, die tausende gleiche Zeilen mit Tags enthält, die aus vielen Ausgaben von yq kamen und erstellt daraus ein dict, in dem jede Zeile nur einmal vorkommt und zusätzlich die Anzahl der Vorkommen enthält. + Ausgabe. + Code von https://www.geeksforgeeks.org/python-count-occurrences-of-each-word-in-given-text-file/ + """ + f = open(filename, "r") + d = dict() + + for line in f: + tag = line.strip("- ") + tag = tag.strip() + + if tag in d: + d[tag] = d[tag] + 1 + else: + d[tag] = 1 + + return(d) + +def write_unified_occurences_to_cache_file(filename, occurences): + f = open(filename, "w") + + for key in list(sorted(occurences.keys())): + f.write("{count}:{tag}\n".format(count=occurences[key], tag=key)) + + f.close() + +def create_tags_categories_caches(): + """ Nutzt yq, um alle Blogbeiträge und Seiten nach genutzen Tags und Kategorien zu durchsuchen und eine Liste dieser und deren Häufigkeit zu erstellen, die dann mit "hgh.py -ft" genutzt werden kann. """ + from tempfile import mkstemp + + fd_tags, filename_tags = mkstemp() + fd_categories, filename_categories = mkstemp() + + os.chdir(config["sites"][args.site_name]["contentdir"]) + + command_tags = "find -iname \"*.md\" -exec yq --front-matter=extract \"{typeof}\" '{{}}' >> \"{tmpfile}\" \;".format(typeof = ".tags", tmpfile = filename_tags) + command_categories = "find -iname \"*.md\" -exec yq --front-matter=extract \"{typeof}\" '{{}}' >> \"{tmpfile}\" \;".format(typeof = ".categories", tmpfile = filename_categories) + + os.system(command_tags) + os.system(command_categories) + + # tags + write_unified_occurences_to_cache_file(config["sites"][args.site_name]["list_tags"], count_and_unify_occurences_from_file(filename_tags)) + + # categories + write_unified_occurences_to_cache_file(config["sites"][args.site_name]["list_categories"], count_and_unify_occurences_from_file(filename_categories)) + +""" Start des Tools ab hier """ + +config = get_config(config_json) +args = parser.parse_args() + +if not args.site_name and not (args.sites or args.time or args.printconfig or args.list_templates or args.create_comments_file): + parser.error("--site is required for all arguments except (--time, --printconfig, --templates, --create-comments-file).") + +if args.create_comments_file: + # Creates a comment.json from template + filename = config["settings"]["comments_filename"] + + comment = """[{ +"id": 0, +"author": "", +"text": "", +"website": "", +"date": "", +"time": "", +"reply_to": 0 +}]""" + + if not os.path.isfile(filename): + with open(filename, "w") as w: + w.write(comment) + + print("{filename} created.\n\nDon't forget to add \"jsoncomments: true\" to frontmatter.".format(filename=filename)) + else: + print("Aborted. There is already a file named \"{}\"".format(filename)) + +if args.sites: + for key in config["sites"].keys(): + print("{}: {}".format(key, config["sites"][key]["basedir"])) + +if args.create_tags_categories_caches: + create_tags_categories_caches() + +if args.list_templates: + import glob + print("Templates available in {}".format(config["settings"]["template_path"])) + for file in glob.glob(config["settings"]["template_path"] + "*.jinja"): + print("- \"{}\"".format(file.lstrip(config["settings"]["template_path"]).rstrip(".jinja"))) + +if args.time: + #get_datetime() + os.system("date +'%Y-%m-%dT%H:%M:%S%:z' | xsel -i -b") # geht bei mir leider nicht mit pyperclip3 + print("Check your clipboard :)") + +if args.printconfig: + print("Config file: {path}\n{dump}".format(path = config_json, dump = json.dumps(config, indent=4))) + +if args.hugo_server: + hugo_server() + +if args.hugo_server_light: + hugo_server_light() + +if args.tmux: + start_tmux() + +# if args.searchpost: +# print("Deprecated") +# searchpost(args.searchpost) + +if args.find_tags: + interactive_tags(args.find_tags) + +if args.find_categories: + interactive_categories(args.find_categories) + +if args.find_anything: + find_anything(args.find_anything) + +if args.find_anything_fuzzy: + find_anything_fuzzy(args.find_anything_fuzzy) + +if args.site_build: + site_build() + +if args.site_upload: + site_upload() + +if args.deploy: + site_build() + site_upload() + + +# Wie gehtn das, dass man dort dann auch landet und das Python sich beendet? Gar nicht, da man auf die Shell, in der Python aufgerufen wurde, keinerlei Zugriff hat. +# Man kann nur mit chdir das cwd wechseln und dort eine Shell aufrufen. + +if args.cdb: + subprocess.run([config["settings"]["shell"]], cwd=config["sites"][args.site_name]["basedir"]) + +if args.cdc: + subprocess.run([config["settings"]["shell"]], cwd=config["sites"][args.site_name]["contentdir"]) + +if args.cdl: + subprocess.run([config["settings"]["shell"]], cwd=config["sites"][args.site_name]["contentdir-light"]) + +if args.create_pagebundle: # Erstellt ein Page Bundle mit Dateinamen aus slug, der automatisch aus dem angegebenen Titel generiert wird. --create-pagebundle "Dies ist ein neuer Blogbeitrag" + # Das funktioniert nur angepasst für Natenoms Blog. Bitte im Code anpassen. Todo... + title = args.create_pagebundle + slug = title_to_slug(title) + if args.template == None: + template = config["settings"]["default_template"] + else: + template = args.template + + blog_directory_posts = config["sites"][args.site_name]["contentdir"] + '/posts/' + + # Page Bundle (nur Verzeichnis) verarbeiten + year, month, day = get_year_month_date() + + directory_of_pagebundle = blog_directory_posts + year + "/" + month + "/" + year + "-" + month + "-" + day + "-" + slug + + if os.path.isdir(directory_of_pagebundle): # Page Bundle Verzeichnis existiert schon. + directory_of_pagebundle_exists = True + # Checking if the list is empty or not + if len(os.listdir(directory_of_pagebundle)) == 0: + print("Info: Directory exists, but is empty, weiter gehts: {}".format(directory_of_pagebundle)) + else: # Page Bundle Verzeichnis existiert noch nicht. + from pathlib import Path + #directory_of_pagebundle_exists = False + Path(directory_of_pagebundle).mkdir(parents=True, exist_ok=True) + + # index.md verarbeiten + if os.path.isfile(directory_of_pagebundle + "/index.md"): + print("Es gibt schon eine index.md im Page Bundle") + sys.exit(1) + else: + filename = directory_of_pagebundle + "/index.md" + filecontent = process_template(template, title, slug, get_datetime()) # erweitern, dass eine Liste mit allen möglichen Dingen übergeben wird, die man in einem Template nutzen könnte. + + with open(filename, "w") as w: + w.write(filecontent) + + if config["settings"]["open_new_pagebundle_with_vscode"]: + # tut nicht bei flatpak und normal gleichzeichtig: subprocess.run([config["settings"]["bin_vscode"].split(" "), config["sites"][args.site_name]["basedir"], directory_of_pagebundle + "/index.md"]) + os.system("{bin_vscode} {basedir} {filename} &".format(bin_vscode = config["settings"]["bin_vscode"], basedir = config["sites"][args.site_name]["basedir"], filename = directory_of_pagebundle + "/index.md")) + else: + print("Page Bundle erfolgreich erstellt:\n\tTemplate: {}\n\tSlug: {}\n\tVerzeichnis: {}".format(template, slug, directory_of_pagebundle)) + os.system("echo '{path}' | xsel -i -b".format(path=directory_of_pagebundle + "/index.md")) + print("Hinweise:\n\tPfad zur Datei in Zwischenablage eingefügt. In VSCodium öffnen mit 'Strg + o' und einfügen, Leerzeichen am Ende entfernen. (Das Leerzeichen am Ende kommt von VSCode.)") diff --git a/templates/default.jinja b/templates/default.jinja new file mode 100644 index 0000000..43c619d --- /dev/null +++ b/templates/default.jinja @@ -0,0 +1,16 @@ +--- +title: "{{ title }}" +slug: {{ slug }} +author: Natenom +draft: true +# comments: true +# jsoncomments: true +date: {{ datetime }} +categories: +- +tags: +- + +--- + + diff --git a/templates/fotos_letzte_tage.jinja b/templates/fotos_letzte_tage.jinja new file mode 100644 index 0000000..43c619d --- /dev/null +++ b/templates/fotos_letzte_tage.jinja @@ -0,0 +1,16 @@ +--- +title: "{{ title }}" +slug: {{ slug }} +author: Natenom +draft: true +# comments: true +# jsoncomments: true +date: {{ datetime }} +categories: +- +tags: +- + +--- + +