diff options
| author | Lukasz Kasprzak <lukasz.kasprzak@pm.me> | 2026-03-19 15:05:32 +0100 |
|---|---|---|
| committer | Lukasz Kasprzak <lukasz.kasprzak@pm.me> | 2026-03-19 15:05:32 +0100 |
| commit | b1844d805a8f190af00faf3f6e5bed7d997ecaae (patch) | |
| tree | d6296ba5e85034836e203a542d2ad34a9e5c5b32 | |
| parent | c74163f1ecbd809c83bb465ee89fa04369a89972 (diff) | |
| download | bin-b1844d805a8f190af00faf3f6e5bed7d997ecaae.tar.gz bin-b1844d805a8f190af00faf3f6e5bed7d997ecaae.zip | |
created sorter for sorting downloads with custom rules; created straper for recreation of server
36 files changed, 5150 insertions, 0 deletions
diff --git a/__pycache__/dumppdf.cpython-313.pyc b/__pycache__/dumppdf.cpython-313.pyc Binary files differnew file mode 100644 index 0000000..20a2388 --- /dev/null +++ b/__pycache__/dumppdf.cpython-313.pyc diff --git a/__pycache__/pdf2txt.cpython-313.pyc b/__pycache__/pdf2txt.cpython-313.pyc Binary files differnew file mode 100644 index 0000000..84e0609 --- /dev/null +++ b/__pycache__/pdf2txt.cpython-313.pyc diff --git a/docs/kvm-lab-readme.md b/docs/kvm-lab-readme.md new file mode 100644 index 0000000..89da6c1 --- /dev/null +++ b/docs/kvm-lab-readme.md @@ -0,0 +1,162 @@ +# `KVM Lab Workflow` + +**Purpose:** +Fast creation of disposable or persistent KVM labs with strong isolation, +no host contamination, and reversible state management. + +--- + +## Directory Layout + +``` +/var/lib/libvirt/images/ +├── base/ +│ ├── debian13-template.qcow2 +│ └── work01-base-YYYY-MM-DD_HHMMSS.qcow2 +├── overlays/ +│ └── work01.qcow2 +├── lab/ +│ └── <lab-name>.qcow2 +├── whonix/ +│ ├── Whonix-Gateway-*.qcow2 +│ └── Whonix-Workstation-*.qcow2 +└── archive/ +``` + +--- + +## Installed Tools + +| Script | Location | Purpose | +| --------------------- | ---------------------------------- | -------------------------------- | +| `kvm-lab-create` | `~/.local/bin/kvm-lab-create` | Create lab VMs from chosen base | +| `kvm-lab-destroy` | `~/.local/bin/kvm-lab-destroy` | Destroy lab VM + disk safely | +| `kvm-promote-to-base` | `~/.local/bin/kvm-promote-to-base` | Freeze any VM into reusable base | +| `kvm--lab-status` | `~/.local/bin/kvm-lab-status` | Prints the current status | +Ensure: + +``` +echo $PATH | grep "$HOME/.local/bin" +``` + +--- + +## Core Concepts + +| Term | Meaning | +| ---------- | ------------------------------------------------ | +| Base image | Immutable qcow2 used as parent for new labs | +| Overlay | Writable qcow2 layer for a specific VM | +| Promote | Freeze a VM’s current disk state into a new base | + +--- + +## Normal Workflow + +### 1. Promote a VM into a base + +Freeze current VM state for reuse. + +``` +virsh -c qemu:///system shutdown work01 +kvm-promote-to-base work01 +``` + +Creates: + +``` +/var/lib/libvirt/images/base/work01-base-YYYY-MM-DD_HHMMSS.qcow2 +``` + +--- + +### 2. Create a new lab + +From template: + +``` +kvm-lab-create lab10 --base template --start +``` + +From newest work01 base: + +``` +kvm-lab-create lab11 --base work01 --start +``` + +From specific base: + +``` +kvm-lab-create lab12 --base /var/lib/libvirt/images/base/work01-base-2026-01-09_150905.qcow2 --start +``` + +--- + +### 3. Destroy a lab safely + +``` +kvm-lab-destroy lab11 +``` + +**Guards enforced:** + +* Only deletes VMs whose disks are under `/var/lib/libvirt/images/lab/` +* Refuses to touch `work01`, template, Whonix, or base images + +--- + +## Debugging & Verification + +List all VMs: + +``` +virsh -c qemu:///system list --all +``` + +Check disk chain: + +``` +sudo qemu-img info --backing-chain /var/lib/libvirt/images/lab/lab10.qcow2 +``` + +Confirm no base corruption: + +``` +sudo qemu-img info /var/lib/libvirt/images/base/work01-base-*.qcow2 +``` + +--- + +## Invariants (Never Broken) + +* `work01` remains intact +* Base images are never modified +* Lab VMs cannot access host or other VMs +* No filesystem passthrough +* All labs use NAT-only `default` network + +--- + +## Recovery + +Delete a bad base: + +``` +sudo rm /var/lib/libvirt/images/base/<base-file>.qcow2 +``` + +Delete a stuck lab: + +``` +kvm-lab-destroy <lab-name> +``` + +--- + +## Philosophy + +> **One-way operations** +> Promote → Create → Destroy +> Host always stays clean. + + diff --git a/lib/bashsimplecurses b/lib/bashsimplecurses deleted file mode 160000 -Subproject 006b9cafd4d381b693c5e4f55140f2834cf3087 diff --git a/mail-i2p-fetch b/mail-i2p-fetch new file mode 100755 index 0000000..7153cfb --- /dev/null +++ b/mail-i2p-fetch @@ -0,0 +1,2 @@ +#!/bin/sh +getmail --rcfile "$HOME/.config/getmail/i2p/getmailrc" @@ -0,0 +1,95 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2006 Søren Roug, European Environment Agency +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): +# +from odf.odf2xhtml import ODF2XHTML +import zipfile +import sys, os, smtplib, getopt + +from email.mime.multipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.text import MIMEText +from email.encoders import encode_base64 + +if sys.version_info[0]==3: unicode=str + +def usage(): + sys.stderr.write("Usage: %s [-f from] [-s subject] inputfile recipients...\n" % sys.argv[0]) + +try: + opts, args = getopt.getopt(sys.argv[1:], "f:s:", ["from=", "subject="]) +except getopt.GetoptError: + usage() + sys.exit(2) + +fromaddr = os.getlogin() + "@" + os.getenv('HOSTNAME','localhost') +subject = None +for o, a in opts: + if o in ("-f", "--from"): + fromaddr = a + if o in ("-s", "--subject"): + subject = a + +if len(args) < 2: + usage() + sys.exit(2) + +suffices = { + 'wmf':('image','x-wmf'), + 'png':('image','png'), + 'gif':('image','gif'), + 'jpg':('image','jpeg'), + 'jpeg':('image','jpeg') + } + +msg = MIMEMultipart('related',type="text/html") +msg['From'] = fromaddr +# msg['Date'] = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) +msg['To'] = ','.join(args[1:]) +msg.preamble = 'This is a multi-part message in MIME format.' +msg.epilogue = '' +odhandler = ODF2XHTML() +result = odhandler.odf2xhtml(unicode(args[0])) +if subject: + msg['Subject'] = subject +else: + msg['Subject'] = odhandler.title +htmlpart = MIMEText(result,'html','us-ascii') +htmlpart['Content-Location'] = 'index.html' +msg.attach(htmlpart) +z = zipfile.ZipFile(unicode(args[0])) +for file in z.namelist(): + if file[0:9] == 'Pictures/': + suffix = file[file.rfind(".")+1:] + main,sub = suffices.get(suffix,('application','octet-stream')) + img = MIMENonMultipart(main,sub) + img.set_payload(z.read(file)) + img['Content-Location'] = "" + file + encode_base64(img) + msg.attach(img) +z.close() + +server = smtplib.SMTP('localhost') +#server.set_debuglevel(1) +server.sendmail(fromaddr, args[1:], msg.as_string()) +server.quit() + + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/media-toggle b/media-toggle new file mode 100755 index 0000000..6aecc69 --- /dev/null +++ b/media-toggle @@ -0,0 +1,10 @@ +#!/bin/sh + +if mpc status | grep -q '^\[playing\]'; then + mpc pause + playerctl --all-players pause 2>/dev/null +else + mpc play + playerctl --all-players play 2>/dev/null +fi + diff --git a/mutt-pdf b/mutt-pdf new file mode 100755 index 0000000..be0ab56 --- /dev/null +++ b/mutt-pdf @@ -0,0 +1,5 @@ +#!/bin/sh +mkdir -p ~/Downloads/mail +dest=~/Downloads/mail/mutt_$$.pdf +cp "$1" "$dest" +zathura "$dest" diff --git a/officium_curses.sh b/officium_curses.sh new file mode 100755 index 0000000..7afdd32 --- /dev/null +++ b/officium_curses.sh @@ -0,0 +1,212 @@ +#!/bin/bash +# ============================================================================= +# officium_curses.sh — Liturgy of the Hours viewer using bashsimplecurses +# +# Full-screen TUI inspired by bashmount: +# +# ┌─ <liturgical day title> ──────────────────────────────────────────────┐ +# │ 1) Laudes 2) Prima 3) Tertia 4) Sexta 5) Nona 6) Vesperae │ +# └───────────────────────────────────────────────────────────────────────┘ +# ┌─ <Hora> ───────────────────────────────────────────────────────────────┐ +# │ Prayer text ... │ +# └───────────────────────────────────────────────────────────────────────┘ +# ┌─ Commands ─────────────────────────────────────────────────────────────┐ +# │ [1-6]: select hora [Enter]: refresh [q]: quit │ +# └───────────────────────────────────────────────────────────────────────┘ +# Command: _ +# +# Authors : Adam Gomułka, Łukasz Kasprzak +# Created : 2026-03-05 +# Version : 0.2 +# ============================================================================= + +set -eo pipefail + +# ── Library ─────────────────────────────────────────────────────────────────── + +BSC_LIB="${HOME}/.local/lib/bashsimplecurses/simple_curses.sh" + +if [ ! -f "$BSC_LIB" ]; then + echo "Error: bashsimplecurses not found at $BSC_LIB" + echo "Expected path: $BSC_LIB" + exit 1 +fi + +source "$BSC_LIB" + +# ── Configuration ───────────────────────────────────────────────────────────── + +BASE_URL="https://www.divinumofficium.com/cgi-bin/horas/officium.pl" +HORAS=(Laudes Prima Tertia Sexta Nona Vesperae) + +# Word-wrap width: full terminal width minus window border/padding +TERM_WIDTH=$(tput cols 2>/dev/null || echo 80) +WRAP="${WRAP:-$((TERM_WIDTH - 6))}" + +# ── State ───────────────────────────────────────────────────────────────────── + +SELECTED=1 # currently selected hora (0-based index into HORAS) +DIES="" # liturgical day title (from purple font tag) +STATUS="" # shown in the commands bar +declare -a PRAYER # prayer text, pre-split into lines for BSC + +HTML_TMP="$(mktemp)" +trap 'rm -f "$HTML_TMP"' EXIT + +# ── Fetch and extract ───────────────────────────────────────────────────────── + +# Fetch the HTML for a given hora, extract the day title and prayer text, +# and populate the DIES and PRAYER globals. +fetch_hora() { + local hora="$1" + local url="${BASE_URL}?command=pray${hora}" + + STATUS="Fetching ${hora}..." + + if ! curl -s --max-time 30 "$url" -o "$HTML_TMP" 2>/dev/null; then + STATUS="Error: curl failed" + PRAYER=("Could not reach divinumofficium.com." "Check your network connection.") + DIES="" + return 1 + fi + if [ ! -s "$HTML_TMP" ]; then + STATUS="Error: empty response" + PRAYER=("Server returned an empty response.") + DIES="" + return 1 + fi + + # Extract the liturgical day title from the purple font tag near the top + DIES=$(grep -o '<FONT COLOR="purple">[^<]*</FONT>' "$HTML_TMP" \ + | sed 's/<[^>]*>//g; s/\r//') + + # Extract and clean the Latin prayer text using the same pipeline as officium.sh + local raw_text + raw_text=$(awk ' + /COLOR="red"><B><I>Incipit<\/I><\/B><\/FONT>/ { printing=1 } + printing { + if (/COLOR=.?green.?>[0-9]/) td_col=2 + if (/<\/TD>/ && td_col==2) td_col=0 + if (/<\/TR>/) { td_col=0; print "" } + if (td_col != 2) print + } + /<\/TABLE>/ && printing { exit } + ' "$HTML_TMP" \ + | sed 's/<[^>]*>//g' \ + | sed 's/\r$//' \ + | sed 's/[[:space:]]*$//' \ + | sed -E 's/^[0-9]+:[0-9]+[[:space:]]*//' \ + | sed 's/ / /g' \ + | sed 's/ / /g' \ + | sed 's/</</g' \ + | sed 's/>/>/g' \ + | sed 's/&/\&/g' \ + | sed 's/&#[0-9]*;//g' \ + | sed 's/&#x[0-9a-fA-F]*;//g' \ + | sed -E 's/^(Psalmus[[:space:]]+[0-9]+)\([^)]*\)/\1/' \ + | cat -s \ + | grep -vE "^(Top[[:space:]]+Next|Top|Next)$" \ + | grep -vE "^[A-Z][a-z]+ [A-Z][a-z]+\{" \ + | grep -v "^[0-9]$") + + # Word-wrap long lines and load into the PRAYER array. + # BSC's append takes one line at a time and does not wrap, so we must + # pre-wrap here before rendering. + PRAYER=() + while IFS= read -r line; do + if [ -z "$line" ]; then + PRAYER+=("") + elif [ "${#line}" -gt "$WRAP" ]; then + while IFS= read -r wrapped; do + PRAYER+=("$wrapped") + done < <(echo "$line" | fold -s -w "$WRAP") + else + PRAYER+=("$line") + fi + done <<< "$raw_text" + + STATUS="$(date '+%A, %d %B %Y') — ${#PRAYER[@]} lines" +} + +# ── BSC layout ──────────────────────────────────────────────────────────────── + +main() { + local hora="${HORAS[$SELECTED]}" + + # ── Window 1: Hora selector ────────────────────────────────────────────── + # Spans the full width; shows the liturgical day title and numbered hora list. + + local win_title="Divinum Officium" + [ -n "$DIES" ] && win_title="$DIES" + + window "$win_title" "red" "100%" + # Build one line of numbered hora buttons; selected hora is wrapped in [] + local hora_line=" " + for i in "${!HORAS[@]}"; do + local n=$((i + 1)) + if [ "$i" -eq "$SELECTED" ]; then + hora_line+="${n}) [${HORAS[$i]}] " + else + hora_line+="${n}) ${HORAS[$i]} " + fi + done + append "$hora_line" + endwin + + # ── Window 2: Prayer text ──────────────────────────────────────────────── + + window "$hora" "blue" "100%" + for line in "${PRAYER[@]}"; do + # BSC collapses truly empty appends in some versions, so use a single + # space for blank lines to preserve paragraph spacing + if [ -z "$line" ]; then + append " " + else + append "$line" + fi + done + endwin + + # ── Window 3: Commands ─────────────────────────────────────────────────── + + window "Commands" "green" "100%" + append " [1-6]: select hora [Enter]: refresh [q]: quit" + addsep + append " $STATUS" + endwin +} + +# ── Interactive loop ────────────────────────────────────────────────────────── + +# Fetch the default hora (Prima) before showing the UI for the first time +fetch_hora "${HORAS[$SELECTED]}" + +while true; do + # Render the current state using BSC + main_loop + + # Print the command prompt below the BSC windows (outside any window frame) + printf "\nCommand: " + + # Read one character without requiring Enter; -s suppresses echo. + # Timeout of 0.5s so the loop stays responsive. + IFS= read -r -s -n1 -t 0.5 key 2>/dev/null || true + + case "$key" in + 1|2|3|4|5|6) + # Select hora by number (1-6) and fetch it + SELECTED=$(( key - 1 )) + fetch_hora "${HORAS[$SELECTED]}" + ;; + "") + # Enter (empty string from -n1) — refresh current hora + fetch_hora "${HORAS[$SELECTED]}" + ;; + q|Q) + clear + echo "Finis." + exit 0 + ;; + # Any other key: just redraw without re-fetching + esac +done diff --git a/pdf-open b/pdf-open new file mode 100755 index 0000000..ca5c427 --- /dev/null +++ b/pdf-open @@ -0,0 +1,6 @@ +#!/bin/bash +echo "Got file: $1" >> /tmp/pdf-debug.log +echo "File exists: $(test -f "$1" && echo yes || echo no)" >> /tmp/pdf-debug.log +echo "File size: $(wc -c < "$1" 2>/dev/null)" >> /tmp/pdf-debug.log +cp "$1" /tmp/mutt_pdf_$$.pdf +zathura /tmp/mutt_pdf_$$.pdf diff --git a/pip-bins/csv2ods b/pip-bins/csv2ods new file mode 100755 index 0000000..c9ece9f --- /dev/null +++ b/pip-bins/csv2ods @@ -0,0 +1,229 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2008 Agustin Henze -> agustinhenze at gmail.com +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): +# +# Søren Roug +# +# Oct 2014: Georges Khaznadar <georgesk@debian.org> +# - ported to Python3 +# - imlemented the missing switch -c / --encoding, with an extra +# feature for POSIX platforms which can guess encoding. + +from odf.opendocument import OpenDocumentSpreadsheet +from odf.style import Style, TextProperties, ParagraphProperties, TableColumnProperties +from odf.text import P +from odf.table import Table, TableColumn, TableRow, TableCell +from optparse import OptionParser +import sys,csv,re, os, codecs + +if sys.version_info[0]==3: unicode=str + +if sys.version_info[0]==2: + class UTF8Recoder: + """ + Iterator that reads an encoded stream and reencodes the input to UTF-8 + """ + def __init__(self, f, encoding): + self.reader = codecs.getreader(encoding)(f) + + def __iter__(self): + return self + + def next(self): + return self.reader.next().encode("utf-8") + + class UnicodeReader: + """ + A CSV reader which will iterate over lines in the CSV file "f", + which is encoded in the given encoding. + """ + + def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): + f = UTF8Recoder(f, encoding) + self.reader = csv.reader(f, dialect=dialect, **kwds) + + def next(self): + row = self.reader.next() + return [unicode(s, "utf-8") for s in row] + + def __iter__(self): + return self + + +def csvToOds( pathFileCSV, pathFileODS, tableName='table', + delimiter=',', quoting=csv.QUOTE_MINIMAL, + quotechar = '"', escapechar = None, + skipinitialspace = False, lineterminator = '\r\n', + encoding="utf-8"): + textdoc = OpenDocumentSpreadsheet() + # Create a style for the table content. One we can modify + # later in the word processor. + tablecontents = Style(name="Table Contents", family="paragraph") + tablecontents.addElement(ParagraphProperties(numberlines="false", linenumber="0")) + tablecontents.addElement(TextProperties(fontweight="bold")) + textdoc.styles.addElement(tablecontents) + + # Start the table + table = Table( name=tableName ) + + if sys.version_info[0]==3: + reader = csv.reader(open(pathFileCSV, encoding=encoding), + delimiter=delimiter, + quoting=quoting, + quotechar=quotechar, + escapechar=escapechar, + skipinitialspace=skipinitialspace, + lineterminator=lineterminator) + else: + reader = UnicodeReader(open(pathFileCSV), + encoding=encoding, + delimiter=delimiter, + quoting=quoting, + quotechar=quotechar, + escapechar=escapechar, + skipinitialspace=skipinitialspace, + lineterminator=lineterminator) + fltExp = re.compile('^\s*[-+]?\d+(\.\d+)?\s*$') + + for row in reader: + tr = TableRow() + table.addElement(tr) + for val in row: + if fltExp.match(val): + tc = TableCell(valuetype="float", value=val.strip()) + else: + tc = TableCell(valuetype="string") + tr.addElement(tc) + p = P(stylename=tablecontents,text=val) + tc.addElement(p) + + textdoc.spreadsheet.addElement(table) + textdoc.save( pathFileODS ) + +if __name__ == "__main__": + usage = "%prog -i file.csv -o file.ods -d" + parser = OptionParser(usage=usage, version="%prog 0.1") + parser.add_option('-i','--input', action='store', + dest='input', help='File input in csv') + parser.add_option('-o','--output', action='store', + dest='output', help='File output in ods') + parser.add_option('-d','--delimiter', action='store', + dest='delimiter', help='specifies a one-character string to use as the field separator. It defaults to ",".') + + parser.add_option('-c','--encoding', action='store', + dest='encoding', help='specifies the encoding the file csv. It defaults to utf-8') + + parser.add_option('-t','--table', action='store', + dest='tableName', help='The table name in the output file') + + parser.add_option('-s','--skipinitialspace', + dest='skipinitialspace', help='''specifies how to interpret whitespace which + immediately follows a delimiter. It defaults to False, which + means that whitespace immediately following a delimiter is part + of the following field.''') + + parser.add_option('-l','--lineterminator', action='store', + dest='lineterminator', help='''specifies the character sequence which should + terminate rows.''') + + parser.add_option('-q','--quoting', action='store', + dest='quoting', help='''It can take on any of the following module constants: + 0 = QUOTE_MINIMAL means only when required, for example, when a field contains either the quotechar or the delimiter + 1 = QUOTE_ALL means that quotes are always placed around fields. + 2 = QUOTE_NONNUMERIC means that quotes are always placed around fields which do not parse as integers or floating point numbers. + 3 = QUOTE_NONE means that quotes are never placed around fields. + It defaults is QUOTE_MINIMAL''') + + parser.add_option('-e','--escapechar', action='store', + dest='escapechar', help='''specifies a one-character string used to escape the delimiter when quoting is set to QUOTE_NONE.''') + + parser.add_option('-r','--quotechar', action='store', + dest='quotechar', help='''specifies a one-character string to use as the quoting character. It defaults to ".''') + + (options, args) = parser.parse_args() + + if options.input: + pathFileCSV = options.input + else: + parser.print_help() + exit( 0 ) + + if options.output: + pathFileODS = options.output + else: + parser.print_help() + exit( 0 ) + + if options.delimiter: + delimiter = options.delimiter + else: + delimiter = "," + + if options.skipinitialspace: + skipinitialspace = True + else: + skipinitialspace=False + + if options.lineterminator: + lineterminator = options.lineterminator + else: + lineterminator ="\r\n" + + if options.escapechar: + escapechar = options.escapechar + else: + escapechar=None + + if options.tableName: + tableName = options.tableName + else: + tableName = "table" + + if options.quotechar: + quotechar = options.quotechar + else: + quotechar = "\"" + + encoding = "utf-8" # default setting + ########################################################### + ## try to guess the encoding; this is implemented only with + ## POSIX platforms. Can it be improved? + output = os.popen('/usr/bin/file ' + pathFileCSV).read() + m=re.match(r'^.*: ([-a-zA-Z0-9]+) text$', output) + if m: + encoding=m.group(1) + if 'ISO-8859' in encoding: + encoding="latin-1" + else: + encoding="utf-8" + ############################################################ + # when the -c or --coding switch is used, it takes precedence + if options.encoding: + encoding = options.encoding + + csvToOds( pathFileCSV=unicode(pathFileCSV), + pathFileODS=unicode(pathFileODS), + delimiter=delimiter, skipinitialspace=skipinitialspace, + escapechar=escapechar, + lineterminator=unicode(lineterminator), + tableName=tableName, quotechar=quotechar, + encoding=encoding) + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/pip-bins/dumppdf.py b/pip-bins/dumppdf.py new file mode 100755 index 0000000..c2066aa --- /dev/null +++ b/pip-bins/dumppdf.py @@ -0,0 +1,468 @@ +#!/usr/bin/python3 +"""Extract pdf structure in XML format""" + +import logging +import os.path +import re +import sys +from argparse import ArgumentParser +from collections.abc import Container, Iterable +from typing import Any, TextIO, cast + +import pdfminer +from pdfminer.pdfdocument import PDFDocument, PDFNoOutlines, PDFXRefFallback +from pdfminer.pdfexceptions import ( + PDFIOError, + PDFObjectNotFound, + PDFTypeError, + PDFValueError, +) +from pdfminer.pdfpage import PDFPage +from pdfminer.pdfparser import PDFParser +from pdfminer.pdftypes import PDFObjRef, PDFStream, resolve1, stream_value +from pdfminer.psparser import LIT, PSKeyword, PSLiteral +from pdfminer.utils import isnumber + +logging.basicConfig() +logger = logging.getLogger(__name__) + +ESC_PAT = re.compile(r'[\000-\037&<>()"\042\047\134\177-\377]') + + +def escape(s: str | bytes) -> str: + us = str(s, "latin-1") if isinstance(s, bytes) else s + return ESC_PAT.sub(lambda m: f"&#{ord(m.group(0))};", us) + + +def dumpxml(out: TextIO, obj: object, codec: str | None = None) -> None: + if obj is None: + out.write("<null />") + return + + if isinstance(obj, dict): + out.write(f'<dict size="{len(obj)}">\n') + for k, v in obj.items(): + out.write(f"<key>{k}</key>\n") + out.write("<value>") + dumpxml(out, v) + out.write("</value>\n") + out.write("</dict>") + return + + if isinstance(obj, list): + out.write(f'<list size="{len(obj)}">\n') + for v in obj: + dumpxml(out, v) + out.write("\n") + out.write("</list>") + return + + if isinstance(obj, (str, bytes)): + out.write(f'<string size="{len(obj)}">{escape(obj)}</string>') + return + + if isinstance(obj, PDFStream): + if codec == "raw": + # Bug: writing bytes to text I/O. This will raise TypeError. + out.write(obj.get_rawdata()) # type: ignore [arg-type] + elif codec == "binary": + # Bug: writing bytes to text I/O. This will raise TypeError. + out.write(obj.get_data()) # type: ignore [arg-type] + else: + out.write("<stream>\n<props>\n") + dumpxml(out, obj.attrs) + out.write("\n</props>\n") + if codec == "text": + data = obj.get_data() + out.write(f'<data size="{len(data)}">{escape(data)}</data>\n') + out.write("</stream>") + return + + if isinstance(obj, PDFObjRef): + out.write(f'<ref id="{obj.objid}" />') + return + + if isinstance(obj, PSKeyword): + # Likely bug: obj.name is bytes, not str + out.write(f"<keyword>{obj.name}</keyword>") # type: ignore [str-bytes-safe] + return + + if isinstance(obj, PSLiteral): + # Likely bug: obj.name may be bytes, not str + out.write(f"<literal>{obj.name}</literal>") # type: ignore [str-bytes-safe] + return + + if isnumber(obj): + out.write(f"<number>{obj}</number>") + return + + raise PDFTypeError(obj) + + +def dumptrailers( + out: TextIO, + doc: PDFDocument, + show_fallback_xref: bool = False, +) -> None: + for xref in doc.xrefs: + if not isinstance(xref, PDFXRefFallback) or show_fallback_xref: + out.write("<trailer>\n") + dumpxml(out, xref.get_trailer()) + out.write("\n</trailer>\n\n") + no_xrefs = all(isinstance(xref, PDFXRefFallback) for xref in doc.xrefs) + if no_xrefs and not show_fallback_xref: + msg = ( + "This PDF does not have an xref. Use --show-fallback-xref if " + "you want to display the content of a fallback xref that " + "contains all objects." + ) + logger.warning(msg) + + +def dumpallobjs( + out: TextIO, + doc: PDFDocument, + codec: str | None = None, + show_fallback_xref: bool = False, +) -> None: + visited = set() + out.write("<pdf>") + for xref in doc.xrefs: + for objid in xref.get_objids(): + if objid in visited: + continue + visited.add(objid) + try: + obj = doc.getobj(objid) + if obj is None: + continue + out.write(f'<object id="{objid}">\n') + dumpxml(out, obj, codec=codec) + out.write("\n</object>\n\n") + except PDFObjectNotFound as e: + print(f"not found: {e!r}") + dumptrailers(out, doc, show_fallback_xref) + out.write("</pdf>") + + +def dumpoutline( + outfp: TextIO, + fname: str, + objids: Any, + pagenos: Container[int], + password: str = "", + dumpall: bool = False, + codec: str | None = None, + extractdir: str | None = None, +) -> None: + with open(fname, "rb") as fp: + parser = PDFParser(fp) + doc = PDFDocument(parser, password) + pages = { + page.pageid: pageno + for (pageno, page) in enumerate(PDFPage.create_pages(doc), 1) + } + + def resolve_dest(dest: object) -> Any: + if isinstance(dest, (str, bytes)): + dest = resolve1(doc.get_dest(dest)) + elif isinstance(dest, PSLiteral): + dest = resolve1(doc.get_dest(dest.name)) + if isinstance(dest, dict): + dest = dest["D"] + if isinstance(dest, PDFObjRef): + dest = dest.resolve() + return dest + + try: + outlines = doc.get_outlines() + outfp.write("<outlines>\n") + for level, title, dest, a, _se in outlines: + pageno = None + if dest: + dest = resolve_dest(dest) + pageno = pages[dest[0].objid] + elif a: + action = a + if isinstance(action, dict): + subtype = action.get("S") + if subtype and repr(subtype) == "/'GoTo'" and action.get("D"): + dest = resolve_dest(action["D"]) + pageno = pages[dest[0].objid] + s = escape(title) + outfp.write(f'<outline level="{level!r}" title="{s}">\n') + if dest is not None: + outfp.write("<dest>") + dumpxml(outfp, dest) + outfp.write("</dest>\n") + if pageno is not None: + outfp.write(f"<pageno>{pageno!r}</pageno>\n") + outfp.write("</outline>\n") + outfp.write("</outlines>\n") + except PDFNoOutlines: + pass + parser.close() + + +LITERAL_FILESPEC = LIT("Filespec") +LITERAL_EMBEDDEDFILE = LIT("EmbeddedFile") + + +def extractembedded(fname: str, password: str, extractdir: str) -> None: + def extract1(objid: int, obj: dict[str, Any]) -> None: + filename = os.path.basename(obj.get("UF") or cast(bytes, obj.get("F")).decode()) + fileref = obj["EF"].get("UF") or obj["EF"].get("F") + fileobj = doc.getobj(fileref.objid) + if not isinstance(fileobj, PDFStream): + error_msg = ( + f"unable to process PDF: reference for {filename!r} is not a PDFStream" + ) + raise PDFValueError(error_msg) + if fileobj.get("Type") is not LITERAL_EMBEDDEDFILE: + raise PDFValueError( + f"unable to process PDF: reference for {filename!r} " + "is not an EmbeddedFile", + ) + path = os.path.join(extractdir, f"{objid:06d}-{filename}") + if os.path.exists(path): + raise PDFIOError(f"file exists: {path!r}") + print(f"extracting: {path!r}") + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as out: + out.write(fileobj.get_data()) + + with open(fname, "rb") as fp: + parser = PDFParser(fp) + doc = PDFDocument(parser, password) + extracted_objids = set() + for xref in doc.xrefs: + for objid in xref.get_objids(): + obj = doc.getobj(objid) + if ( + objid not in extracted_objids + and isinstance(obj, dict) + and obj.get("Type") is LITERAL_FILESPEC + ): + extracted_objids.add(objid) + extract1(objid, obj) + + +def dumppdf( + outfp: TextIO, + fname: str, + objids: Iterable[int], + pagenos: Container[int], + password: str = "", + dumpall: bool = False, + codec: str | None = None, + extractdir: str | None = None, + show_fallback_xref: bool = False, +) -> None: + with open(fname, "rb") as fp: + parser = PDFParser(fp) + doc = PDFDocument(parser, password) + if objids: + for objid in objids: + obj = doc.getobj(objid) + dumpxml(outfp, obj, codec=codec) + if pagenos: + for pageno, page in enumerate(PDFPage.create_pages(doc)): + if pageno in pagenos: + if codec: + for obj in page.contents: + obj = stream_value(obj) + dumpxml(outfp, obj, codec=codec) + else: + dumpxml(outfp, page.attrs) + if dumpall: + dumpallobjs(outfp, doc, codec, show_fallback_xref) + if (not objids) and (not pagenos) and (not dumpall): + dumptrailers(outfp, doc, show_fallback_xref) + if codec not in ("raw", "binary"): + outfp.write("\n") + + +def create_parser() -> ArgumentParser: + parser = ArgumentParser(description=__doc__, add_help=True) + parser.add_argument( + "files", + type=str, + default=None, + nargs="+", + help="One or more paths to PDF files.", + ) + + parser.add_argument( + "--version", + "-v", + action="version", + version=f"pdfminer.six v{pdfminer.__version__}", + ) + parser.add_argument( + "--debug", + "-d", + default=False, + action="store_true", + help="Use debug logging level.", + ) + procedure_parser = parser.add_mutually_exclusive_group() + procedure_parser.add_argument( + "--extract-toc", + "-T", + default=False, + action="store_true", + help="Extract structure of outline", + ) + procedure_parser.add_argument( + "--extract-embedded", + "-E", + type=str, + help="Extract embedded files", + ) + + parse_params = parser.add_argument_group( + "Parser", + description="Used during PDF parsing", + ) + parse_params.add_argument( + "--page-numbers", + type=int, + default=None, + nargs="+", + help="A space-seperated list of page numbers to parse.", + ) + parse_params.add_argument( + "--pagenos", + "-p", + type=str, + help="A comma-separated list of page numbers to parse. Included for " + "legacy applications, use --page-numbers for more idiomatic " + "argument entry.", + ) + parse_params.add_argument( + "--objects", + "-i", + type=str, + help="Comma separated list of object numbers to extract", + ) + parse_params.add_argument( + "--all", + "-a", + default=False, + action="store_true", + help="If the structure of all objects should be extracted", + ) + parse_params.add_argument( + "--show-fallback-xref", + action="store_true", + help="Additionally show the fallback xref. Use this if the PDF " + "has zero or only invalid xref's. This setting is ignored if " + "--extract-toc or --extract-embedded is used.", + ) + parse_params.add_argument( + "--password", + "-P", + type=str, + default="", + help="The password to use for decrypting PDF file.", + ) + + output_params = parser.add_argument_group( + "Output", + description="Used during output generation.", + ) + output_params.add_argument( + "--outfile", + "-o", + type=str, + default="-", + help='Path to file where output is written. Or "-" (default) to ' + "write to stdout.", + ) + codec_parser = output_params.add_mutually_exclusive_group() + codec_parser.add_argument( + "--raw-stream", + "-r", + default=False, + action="store_true", + help="Write stream objects without encoding", + ) + codec_parser.add_argument( + "--binary-stream", + "-b", + default=False, + action="store_true", + help="Write stream objects with binary encoding", + ) + codec_parser.add_argument( + "--text-stream", + "-t", + default=False, + action="store_true", + help="Write stream objects as plain text", + ) + + return parser + + +def main(argv: list[str] | None = None) -> None: + parser = create_parser() + args = parser.parse_args(args=argv) + + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + + objids = [int(x) for x in args.objects.split(",")] if args.objects else [] + + if args.page_numbers: + pagenos = {x - 1 for x in args.page_numbers} + elif args.pagenos: + pagenos = {int(x) - 1 for x in args.pagenos.split(",")} + else: + pagenos = set() + + password = args.password + + if args.raw_stream: + codec: str | None = "raw" + elif args.binary_stream: + codec = "binary" + elif args.text_stream: + codec = "text" + else: + codec = None + + # Use context manager for file output, ensuring proper cleanup + with sys.stdout if args.outfile == "-" else open(args.outfile, "w") as outfp: + for fname in args.files: + if args.extract_toc: + dumpoutline( + outfp, + fname, + objids, + pagenos, + password=password, + dumpall=args.all, + codec=codec, + extractdir=None, + ) + elif args.extract_embedded: + extractembedded( + fname, password=password, extractdir=args.extract_embedded + ) + else: + dumppdf( + outfp, + fname, + objids, + pagenos, + password=password, + dumpall=args.all, + codec=codec, + extractdir=None, + show_fallback_xref=args.show_fallback_xref, + ) + + +if __name__ == "__main__": + main() diff --git a/pip-bins/odf2mht b/pip-bins/odf2mht new file mode 100755 index 0000000..e10e63b --- /dev/null +++ b/pip-bins/odf2mht @@ -0,0 +1,72 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2006 Søren Roug, European Environment Agency +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): +# +from __future__ import print_function +from odf.odf2xhtml import ODF2XHTML +import zipfile +import sys +#from time import gmtime, strftime + +from email.mime.multipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.text import MIMEText +from email import encoders + +if sys.version_info[0]==3: unicode=str + +if len(sys.argv) != 2: + sys.stderr.write("Usage: %s inputfile\n" % sys.argv[0]) + sys.exit(1) + +suffices = { + 'wmf':('image','x-wmf'), + 'png':('image','png'), + 'gif':('image','gif'), + 'jpg':('image','jpeg'), + 'jpeg':('image','jpeg') + } + +msg = MIMEMultipart('related',type="text/html") +# msg['Subject'] = 'Subject here' +# msg['From'] = '<Saved by ODT2MHT>' +# msg['Date'] = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) +msg.preamble = 'This is a multi-part message in MIME format.' +msg.epilogue = '' +odhandler = ODF2XHTML() +result = odhandler.odf2xhtml(unicode(sys.argv[1])) +htmlpart = MIMEText(result,'html','us-ascii') +htmlpart['Content-Location'] = 'index.html' +msg.attach(htmlpart) +z = zipfile.ZipFile(sys.argv[1]) +for file in z.namelist(): + if file[0:9] == 'Pictures/': + suffix = file[file.rfind(".")+1:] + main,sub = suffices.get(suffix,('application','octet-stream')) + img = MIMENonMultipart(main,sub) + img.set_payload(z.read(file)) + img['Content-Location'] = "" + file + encoders.encode_base64(img) + msg.attach(img) +z.close() +print (msg.as_string()) + + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/pip-bins/odf2xhtml b/pip-bins/odf2xhtml new file mode 100755 index 0000000..d70dfb0 --- /dev/null +++ b/pip-bins/odf2xhtml @@ -0,0 +1,59 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2007 Søren Roug, European Environment Agency +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): +# +from odf.odf2xhtml import ODF2XHTML +import sys, getopt + +if sys.version_info[0]==3: unicode=str + +from io import StringIO + +def usage(): + sys.stderr.write("Usage: %s [-p] inputfile\n" % sys.argv[0]) + +try: + opts, args = getopt.getopt(sys.argv[1:], "ep", ["plain","embedable"]) +except getopt.GetoptError: + usage() + sys.exit(2) + +generatecss = True +embedable = False +for o, a in opts: + if o in ("-p", "--plain"): + generatecss = False + if o in ("-e", "--embedable"): + embedable = True + +if len(args) != 1: + usage() + sys.exit(2) + +odhandler = ODF2XHTML(generatecss, embedable) +try: + result = odhandler.odf2xhtml(unicode(args[0])) +except: + sys.stderr.write("Unable to open file %s or file is not OpenDocument\n" % args[0]) + sys.exit(1) +sys.stdout.write(result) + + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/pip-bins/odf2xml b/pip-bins/odf2xml new file mode 100755 index 0000000..f04d9f1 --- /dev/null +++ b/pip-bins/odf2xml @@ -0,0 +1,81 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2008 Søren Roug, European Environment Agency +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): +# +# + +# OpenDocument can be a complete office document in a single +# XML document. This script will create such a document. +import sys, getopt, base64 +from odf.opendocument import load +from odf.draw import Image, ObjectOle +from odf.style import BackgroundImage +from odf.text import ListLevelStyleImage +from odf.office import BinaryData + +if sys.version_info[0]==3: unicode=str + +def usage(): + sys.stderr.write("Usage: %s [-e] [-o outputfile] [inputfile]\n" % sys.argv[0]) + + +if __name__ == "__main__": + embedimage = False + try: + opts, args = getopt.getopt(sys.argv[1:], "o:e", ["output="]) + except getopt.GetoptError: + usage() + sys.exit(2) + + outputfile = '-' + + for o, a in opts: + if o in ("-o", "--output"): + outputfile = a + if o == '-e': + embedimage = True + + if len(args) > 1: + usage() + sys.exit(2) + if len(args) == 0: + d = load(sys.stdin) + else: + d = load(unicode(args[0])) + if embedimage: + images = d.getElementsByType(Image) + \ + d.getElementsByType(BackgroundImage) + \ + d.getElementsByType(ObjectOle) + \ + d.getElementsByType(ListLevelStyleImage) + for image in images: + href = image.getAttribute('href') + if href and href[:9] == "Pictures/": + p = d.Pictures[href] + bp = base64.encodestring(p[1]) + image.addElement(BinaryData(text=bp)) + image.removeAttribute('href') + xml = d.xml() + if outputfile == '-': + print (xml) + else: + open(outputfile,"wb").write(xml) + + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/pip-bins/odfimgimport b/pip-bins/odfimgimport new file mode 100755 index 0000000..bd95fed --- /dev/null +++ b/pip-bins/odfimgimport @@ -0,0 +1,190 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2007-2009 Søren Roug, European Environment Agency +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): +# +from __future__ import print_function + +import zipfile, sys, getopt, mimetypes +try: + from urllib2 import urlopen, quote, unquote +except ImportError: + from urllib.request import urlopen, quote, unquote +try: + from urlparse import urlunsplit, urlsplit +except ImportError: + from urllib.parse import urlunsplit, urlsplit +from odf.opendocument import load +from odf.draw import Image + +if sys.version_info[0]==3: unicode=str + +#sys.tracebacklimit = 0 + +# Variable to count the number of retrieval failures +failures = 0 + +# Set to one if quiet behaviour is wanted +quiet = 0 + +# If set will write every url to import +verbose = 0 + +# Dictionary with new pictures. Key is original file path +# Item is newfilename +newpictures = {} +doc = None + +def importpicture(href): + """ Add the picture to the ZIP file + Returns the new path name to the file in the zip archive + If it is unable to import, then it returns the original href + Sideeffect: add line to manifest + """ + global doc, newpictures, failures, verbose + + # Check that it is not already in the manifest + if href in doc.Pictures: return href + + image = None + if verbose: print ("Importing", href, file=sys.stderr) + if href[:7] == "http://" or href[:8] == "https://" or href[:6] == "ftp://": + # There is a bug in urlopen: It can't open urls with non-ascii unicode + # characters. Convert to UTF-8 and then use percent encoding + try: + goodhref = href.encode('ascii') + except: + o = list(urlsplit(href)) + o[2] = quote(o[2].encode('utf-8')) + goodhref = urlunsplit(o) + if goodhref in newpictures: + if verbose: print ("already imported", file=sys.stderr) + return newpictures[goodhref] # Already imported + try: + f = urlopen(goodhref.decode("utf-8")) + image = f.read() + headers = f.info() + f.close() + # Get the mimetype from the headerlines + c_t = headers['Content-Type'].split(';')[0].strip() + if c_t: mediatype = c_t.split(';')[0].strip() + if verbose: print ("OK", file=sys.stderr) + except: + failures += 1 + if verbose: print ("failed", file=sys.stderr) + return href + # Remove query string + try: href= href[:href.rindex('?')] + except: pass + try: + lastslash = href[href.rindex('/'):] + ext = lastslash[lastslash.rindex('.'):] + except: ext = mimetypes.guess_extension(mediatype) + # Everything is a simple path. + else: + goodhref = href + if href[:3] == '../': + if directory is None: + goodhref = unquote(href[3:]) + else: + goodhref = unquote(directory + href[2:]) + if goodhref in newpictures: + if verbose: print ("already imported", file=sys.stderr) + return newpictures[goodhref] # Already imported + mediatype, encoding = mimetypes.guess_type(goodhref) + if mediatype is None: + mediatype = '' + try: ext = goodhref[goodhref.rindex('.'):] + except: ext='' + else: + ext = mimetypes.guess_extension(mediatype) + try: + image = file(goodhref).read() + if verbose: print ("OK", file=sys.stderr) + except: + failures += 1 + if verbose: print ("failed", file=sys.stderr) + return href + # If we have a picture to import, the image variable contains it + # and manifestfn, ext and mediatype has a value + if image: + manifestfn = doc.addPictureFromString(image, unicode(mediatype)) + newpictures[goodhref] = manifestfn + return manifestfn + + if verbose: print ("not imported", file=sys.stderr) + return href + +def exitwithusage(exitcode=2): + """ Print out usage information and exit """ + print ("Usage: %s [-q] [-v] [-o output] [inputfile]" % sys.argv[0], file=sys.stderr) + print ("\tInputfile must be OpenDocument format", file=sys.stderr) + sys.exit(exitcode) + +outputfile = None +writefile = True + +try: + opts, args = getopt.getopt(sys.argv[1:], "qvo:") +except getopt.GetoptError: + exitwithusage() + +for o, a in opts: + if o == "-o": + outputfile = a + writefile = True + if o == "-q": + quiet = 1 + if o == "-v": + verbose = 1 + +if len(args) == 0: + try: + doc = load(sys.stdin) + directory = None + except: + print ("Couldn't open OpenDocument file", file=sys.stderr) + exitwithusage() +else: + fn = unicode(args[0]) + if not zipfile.is_zipfile(fn): + exitwithusage() + dirinx = max(fn.rfind('\\'), fn.rfind('/')) + if dirinx >= 0: directory = fn[:dirinx] + else: directory = "." + doc = load(fn) + +for image in doc.getElementsByType(Image): + href = image.getAttribute('href') + newhref = importpicture(href) + image.setAttribute('href',newhref) + +if writefile: + if outputfile is None: + doc.save(fn) + else: + doc.save(unicode(outputfile)) + + +if quiet == 0 and failures > 0: + print ("Couldn't import %d image(s)" % failures, file=sys.stderr) +sys.exit( int(failures > 0) ) + + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/pip-bins/odflint b/pip-bins/odflint new file mode 100755 index 0000000..e5da6bb --- /dev/null +++ b/pip-bins/odflint @@ -0,0 +1,216 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2009 Søren Roug, European Environment Agency +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): +# +import zipfile +from xml.sax import make_parser,handler +from xml.sax.xmlreader import InputSource +import xml.sax.saxutils +import sys +from odf.opendocument import OpenDocument +from odf import element, grammar +from odf.namespaces import * +from odf.attrconverters import attrconverters, cnv_string + +from io import BytesIO + +if sys.version_info[0]==3: unicode=str + +extension_attributes = { + "OpenOffice.org" : { + (METANS,u'template'): ( + (XLINKNS,u'role'), + ), + (STYLENS,u'graphic-properties'): ( + (STYLENS,u'background-transparency'), + ), + (STYLENS,u'paragraph-properties'): ( + (TEXTNS,u'enable-numbering'), + (STYLENS,u'join-border'), + ), + (STYLENS,u'table-cell-properties'): ( + (STYLENS,u'writing-mode'), + ), + (STYLENS,u'table-row-properties'): ( + (STYLENS,u'keep-together'), + ), + }, + "KOffice" : { + (STYLENS,u'graphic-properties'): ( + (KOFFICENS,u'frame-behavior-on-new-page'), + ), + (DRAWNS,u'page'): ( + (KOFFICENS,u'name'), + ), + (PRESENTATIONNS,u'show-shape'): ( + (KOFFICENS,u'order-id'), + ), + (PRESENTATIONNS,u'hide-shape'): ( + (KOFFICENS,u'order-id'), + ), + (CHARTNS,u'legend'): ( + (KOFFICENS,u'title'), + ), + } +} + +printed_errors = [] + +def print_error(str): + if str not in printed_errors: + printed_errors.append(str) + print (str) + +def chop_arg(arg): + if len(arg) > 20: + return "%s..." % arg[0:20] + return arg + +def make_qname(tag): + return "%s:%s" % (nsdict.get(tag[0],tag[0]), tag[1]) + +def allowed_attributes(tag): + return grammar.allowed_attributes.get(tag) + + +class ODFElementHandler(handler.ContentHandler): + """ Extract headings from content.xml of an ODT file """ + def __init__(self, document): + self.doc = document + self.tagstack = [] + self.data = [] + self.currtag = None + + def characters(self, data): + self.data.append(data) + + def startElementNS(self, tag, qname, attrs): + """ Pseudo-create an element + """ + allowed_attrs = grammar.allowed_attributes.get(tag) + attrdict = {} + for (att,value) in attrs.items(): + prefix = nsdict.get(att[0],att[0]) + # Check if it is a known extension + notan_extension = True + for product, ext_attrs in extension_attributes.items(): + allowed_ext_attrs = ext_attrs.get(tag) + if allowed_ext_attrs and att in allowed_ext_attrs: + print_error("Warning: Attribute %s in element <%s> is illegal - %s extension" % ( make_qname(att), make_qname(tag), product)) + notan_extension = False + # Check if it is an allowed attribute + if notan_extension and allowed_attrs and att not in allowed_attrs: + print_error("Error: Attribute %s:%s is not allowed in element <%s>" % ( prefix, att[1], make_qname(tag))) + # Check the value + try: + convert = attrconverters.get(att, cnv_string) + convert(att, value, tag) + except ValueError as res: + print_error("Error: Bad value '%s' for attribute %s:%s in tag: <%s> - %s" % + (chop_arg(value), prefix, att[1], make_qname(tag), res)) + + self.tagstack.append(tag) + self.data = [] + # Check that the parent allows this child element + if tag not in ( (OFFICENS, 'document'), (OFFICENS, 'document-content'), (OFFICENS, 'document-styles'), + (OFFICENS, 'document-meta'), (OFFICENS, 'document-settings'), + (MANIFESTNS,'manifest')): + try: + parent = self.tagstack[-2] + allowed_children = grammar.allowed_children.get(parent) + except: + print_error("Error: This document starts with the wrong tag: <%s>" % make_qname(tag)) + allowed_children = None + if allowed_children and tag not in allowed_children: + print_error("Error: Element %s is not allowed in element %s" % ( make_qname(tag), make_qname(parent))) + # Test that all mandatory attributes have been added. + required = grammar.required_attributes.get(tag) + if required: + for r in required: + if attrs.get(r) is None: + print_error("Error: Required attribute missing: %s in <%s>" % (make_qname(r), make_qname(tag))) + + + def endElementNS(self, tag, qname): + self.currtag = self.tagstack.pop() + str = ''.join(self.data).strip() + # Check that only elements that can take text have text + # But only elements we know exist in grammar + if tag in grammar.allowed_children: + if str != '' and tag not in grammar.allows_text: + print_error("Error: %s does not allow text data" % make_qname(tag)) + self.data = [] + +class ODFDTDHandler(handler.DTDHandler): + def notationDecl(self, name, public_id, system_id): + """ Ignore DTDs """ + print_error("Warning: ODF doesn't use DOCTYPEs") + +def exitwithusage(exitcode=2): + """ print out usage information """ + sys.stderr.write("Usage: %s inputfile\n" % sys.argv[0]) + sys.stderr.write("\tInputfile must be OpenDocument format\n") + sys.exit(exitcode) + +def lint(odffile): + if not zipfile.is_zipfile(odffile): + print_error("Error: This is not a zipped file") + return + zfd = zipfile.ZipFile(odffile) + try: + mimetype = zfd.read('mimetype') + except: + mimetype='' + d = OpenDocument(unicode(mimetype)) + first = True + for zi in zfd.infolist(): + if first: + if zi.filename == 'mimetype': + if zi.compress_type != zipfile.ZIP_STORED: + print_error("Error: The 'mimetype' member must be stored - not deflated") + if zi.comment != "": + print_error("Error: The 'mimetype' member must not have extra header info") + else: + print_error("Warning: The first member in the archive should be the mimetype") + first = False + if zi.filename in ('META-INF/manifest.xml', 'content.xml', 'meta.xml', 'styles.xml', 'settings.xml'): + content = zfd.read(zi.filename) + parser = make_parser() + parser.setFeature(handler.feature_namespaces, True) + parser.setFeature(handler.feature_external_ges, False) + parser.setContentHandler(ODFElementHandler(d)) + dtdh = ODFDTDHandler() + parser.setDTDHandler(dtdh) + parser.setErrorHandler(handler.ErrorHandler()) + + inpsrc = InputSource() + if not isinstance(content, str): + content=content + inpsrc.setByteStream(BytesIO(content)) + parser.parse(inpsrc) + + +if len(sys.argv) != 2: + exitwithusage() +lint(unicode(sys.argv[1])) + + + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/pip-bins/odfmeta b/pip-bins/odfmeta new file mode 100755 index 0000000..b99be94 --- /dev/null +++ b/pip-bins/odfmeta @@ -0,0 +1,266 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2006-2009 Søren Roug, European Environment Agency +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): +# +import zipfile, time, sys, getopt, re +import xml.sax, xml.sax.saxutils +from odf.namespaces import TOOLSVERSION, OFFICENS, XLINKNS, DCNS, METANS +from io import BytesIO + +OUTENCODING="utf-8" + +whitespace = re.compile(r'\s+') + +fields = { +'title': (DCNS,u'title'), +'description': (DCNS,u'description'), +'subject': (DCNS,u'subject'), +'creator': (DCNS,u'creator'), +'date': (DCNS,u'date'), +'language': (DCNS,u'language'), +'generator': (METANS,u'generator'), +'initial-creator': (METANS,u'initial-creator'), +'keyword': (METANS,u'keyword'), +'editing-duration': (METANS,u'editing-duration'), +'editing-cycles': (METANS,u'editing-cycles'), +'printed-by': (METANS,u'printed-by'), +'print-date': (METANS,u'print-date'), +'creation-date': (METANS,u'creation-date'), +'user-defined': (METANS,u'user-defined'), +#'template': (METANS,u'template'), +} + +xfields = [] +Xfields = [] +addfields = {} +deletefields = {} +yieldfields = {} +showversion = None + +def exitwithusage(exitcode=2): + """ print out usage information """ + sys.stderr.write("Usage: %s [-cdlvV] [-xXaAI metafield]... [-o output] [inputfile]\n" % sys.argv[0]) + sys.stderr.write("\tInputfile must be OpenDocument format\n") + sys.exit(exitcode) + +def normalize(str): + """ + The normalize-space function returns the argument string with whitespace + normalized by stripping leading and trailing whitespace and replacing + sequences of whitespace characters by a single space. + """ + return whitespace.sub(' ', str).strip() + +class MetaCollector: + """ + The MetaCollector is a pseudo file object, that can temporarily ignore write-calls + It could probably be replaced with a StringIO object. + """ + def __init__(self): + self._content = [] + self.dowrite = True + + def write(self, str): + if self.dowrite: + self._content.append(str) + + def content(self): + return ''.join(self._content) + + +base = xml.sax.saxutils.XMLGenerator + +class odfmetaparser(base): + """ Parse a meta.xml file with an event-driven parser and replace elements. + It would probably be a cleaner approach to use a DOM based parser and + then manipulate in memory. + Small issue: Reorders elements + """ + version = 'Unknown' + + def __init__(self): + self._mimetype = '' + self.output = MetaCollector() + self._data = [] + self.seenfields = {} + base.__init__(self, self.output, OUTENCODING) + + def startElementNS(self, name, qname, attrs): + self._data = [] + field = name +# I can't modify the template until the tool replaces elements at the same +# location and not at the end +# if name == (METANS,u'template'): +# self._data = [attrs.get((XLINKNS,u'title'),'')] + if showversion and name == (OFFICENS,u'document-meta'): + if showversion == '-V': + print ("version:%s" % attrs.get((OFFICENS,u'version'),'Unknown').decode('utf-8')) + else: + print ("%s" % attrs.get((OFFICENS,u'version'),'Unknown').decode('utf-8')) + if name == (METANS,u'user-defined'): + field = attrs.get((METANS,u'name')) + if field in deletefields: + self.output.dowrite = False + elif field in yieldfields: + del addfields[field] + base.startElementNS(self, name, qname, attrs) + else: + base.startElementNS(self, name, qname, attrs) + self._tag = field + + def endElementNS(self, name, qname): + field = name + if name == (METANS,u'user-defined'): + field = self._tag + if name == (OFFICENS,u'meta'): + for k,v in addfields.items(): + if len(v) > 0: + if type(k) == type(''): + base.startElementNS(self,(METANS,u'user-defined'),None,{(METANS,u'name'):k}) + base.characters(self, v) + base.endElementNS(self, (METANS,u'user-defined'),None) + else: + base.startElementNS(self, k, None, {}) + base.characters(self, v) + base.endElementNS(self, k, None) + if name in xfields: + print ("%s" % self.data()) + if name in Xfields: + if isinstance(self._tag, tuple): + texttag = self._tag[1] + else: + texttag = self._tag + print ("%s:%s" % (texttag, self.data())) + if field in deletefields: + self.output.dowrite = True + else: + base.endElementNS(self, name, qname) + + def characters(self, content): + base.characters(self, content) + self._data.append(content) + + def meta(self): + return self.output.content() + + def data(self): + if usenormalize: + return normalize(''.join(self._data)) + else: + return ''.join(self._data) + +now = time.localtime()[:6] +outputfile = "-" +writemeta = False # Do we change any meta data? +usenormalize = False + +try: + opts, args = getopt.getopt(sys.argv[1:], "cdlvVI:A:a:o:x:X:") +except getopt.GetoptError: + exitwithusage() + +if len(opts) == 0: + opts = [ ('-l','') ] + +for o, a in opts: + if o in ('-a','-A','-I'): + writemeta = True + if a.find(":") >= 0: + k,v = a.split(":",1) + else: + k,v = (a, "") + if len(k) == 0: + exitwithusage() + k = fields.get(k,k) + addfields[k] = unicode(v,'utf-8') + if o == '-a': + yieldfields[k] = True + if o == '-I': + deletefields[k] = True + if o == '-d': + writemeta = True + addfields[(DCNS,u'date')] = "%04d-%02d-%02dT%02d:%02d:%02d" % now + deletefields[(DCNS,u'date')] = True + if o == '-c': + usenormalize = True + if o in ('-v', '-V'): + showversion = o + if o == '-l': + Xfields = fields.values() + if o == "-x": + xfields.append(fields.get(a,a)) + if o == "-X": + Xfields.append(fields.get(a,a)) + if o == "-o": + outputfile = a + +# The specification says we should change the element to our own, +# and must not export the original identifier. +if writemeta: + addfields[(METANS,u'generator')] = TOOLSVERSION + deletefields[(METANS,u'generator')] = True + +odfs = odfmetaparser() +parser = xml.sax.make_parser() +parser.setFeature(xml.sax.handler.feature_namespaces, 1) +parser.setContentHandler(odfs) + +if len(args) == 0: + zin = zipfile.ZipFile(sys.stdin,'r') +else: + if not zipfile.is_zipfile(args[0]): + exitwithusage() + zin = zipfile.ZipFile(args[0], 'r') + +try: + content = zin.read('meta.xml').decode('utf-8') +except: + sys.stderr.write("File has no meta data\n") + sys.exit(1) +parser.parse(BytesIO(content.encode('utf-8'))) + +if writemeta: + if outputfile == '-': + if sys.stdout.isatty(): + sys.stderr.write("Won't write ODF file to terminal\n") + sys.exit(1) + zout = zipfile.ZipFile(sys.stdout,"w") + else: + zout = zipfile.ZipFile(outputfile,"w") + + + + # Loop through the input zipfile and copy the content to the output until we + # get to the meta.xml. Then substitute. + for zinfo in zin.infolist(): + if zinfo.filename == "meta.xml": + # Write meta + zi = zipfile.ZipInfo("meta.xml", now) + zi.compress_type = zipfile.ZIP_DEFLATED + zout.writestr(zi,odfs.meta() ) + else: + payload = zin.read(zinfo.filename) + zout.writestr(zinfo, payload) + + zout.close() +zin.close() + + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/pip-bins/odfoutline b/pip-bins/odfoutline new file mode 100755 index 0000000..0d26071 --- /dev/null +++ b/pip-bins/odfoutline @@ -0,0 +1,144 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2006 Søren Roug, European Environment Agency +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): +# +from __future__ import print_function +import zipfile +from xml.sax import make_parser,handler +from xml.sax.xmlreader import InputSource +import xml.sax.saxutils +import sys +from odf.namespaces import TEXTNS, TABLENS, DRAWNS + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + + +def getxmlpart(odffile, xmlfile): + """ Get the content out of the ODT file""" + z = zipfile.ZipFile(odffile) + content = z.read(xmlfile) + z.close() + return content + + + +# +# Extract headings from content.xml +# +class ODTHeadingHandler(handler.ContentHandler): + """ Extract headings from content.xml of an ODT file """ + def __init__(self, eater): + self.r = eater + self.data = [] + self.level = 0 + + def characters(self, data): + self.data.append(data) + + def startElementNS(self, tag, qname, attrs): + if tag == (TEXTNS, 'h'): + self.level = 0 + for (att,value) in attrs.items(): + if att == (TEXTNS, 'outline-level'): + self.level = int(value) + self.data = [] + + def endElementNS(self, tag, qname): + if tag == (TEXTNS, 'h'): + str = ''.join(self.data) + self.data = [] + self.r.append("%d%*s%s" % (self.level, self.level, '', str)) + +class ODTSheetHandler(handler.ContentHandler): + """ Extract sheet names from content.xml of an ODS file """ + def __init__(self, eater): + self.r = eater + + def startElementNS(self, tag, qname, attrs): + if tag == (TABLENS, 'table'): + sheetname = attrs.get((TABLENS, 'name')) + if sheetname: + self.r.append(sheetname) + +class ODTSlideHandler(handler.ContentHandler): + """ Extract headings from content.xml of an ODT file """ + def __init__(self, eater): + self.r = eater + self.data = [] + self.pagenum = 0 + + def characters(self, data): + self.data.append(data) + + def startElementNS(self, tag, qname, attrs): + if tag == (DRAWNS, 'page'): + self.pagenum = self.pagenum + 1 + self.r.append("SLIDE %d: %s" % ( self.pagenum, attrs.get((DRAWNS, 'name'),''))) + if tag == (TEXTNS, 'p'): + self.data = [] + + def endElementNS(self, tag, qname): + if tag == (TEXTNS, 'p'): + str = ''.join(self.data) + self.data = [] + if len(str) > 0: + self.r.append(" " + str) + +def odtheadings(odtfile): + mimetype = getxmlpart(odtfile,'mimetype') + content = getxmlpart(odtfile,'content.xml') + lines = [] + parser = make_parser() + parser.setFeature(handler.feature_namespaces, 1) + if not isinstance(mimetype, str): + mimetype=mimetype.decode("utf-8") + if mimetype in ('application/vnd.oasis.opendocument.text', + 'application/vnd.oasis.opendocument.text-template'): + parser.setContentHandler(ODTHeadingHandler(lines)) + elif mimetype in ('application/vnd.oasis.opendocument.spreadsheet', + 'application/vnd.oasis.opendocument.spreadsheet-template'): + parser.setContentHandler(ODTSheetHandler(lines)) + elif mimetype in ('application/vnd.oasis.opendocument.presentation' + 'application/vnd.oasis.opendocument.presentation-template'): + parser.setContentHandler(ODTSlideHandler(lines)) + else: + print ("Unsupported fileformat") + sys.exit(2) + parser.setErrorHandler(handler.ErrorHandler()) + + inpsrc = InputSource() + if not isinstance(content, str): + content=content.decode("utf-8") + inpsrc.setByteStream(StringIO(content)) + parser.parse(inpsrc) + return lines + + +if __name__ == "__main__": + filler = " " + for heading in odtheadings(sys.argv[1]): + print (heading) + + + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/pip-bins/odfuserfield b/pip-bins/odfuserfield new file mode 100755 index 0000000..cae1574 --- /dev/null +++ b/pip-bins/odfuserfield @@ -0,0 +1,101 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2006-2007 Søren Roug, European Environment Agency +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): Michael Howitz, gocept gmbh & co. kg + +import sys +import getopt + +import odf.userfield + +if sys.version_info[0]==3: unicode=str + +listfields = False +Listfields = False +xfields = [] +Xfields = [] +setfields = {} +outputfile = None +inputfile = None + + +def exitwithusage(exitcode=2): + """ print out usage information """ + sys.stderr.write("Usage: %s [-lL] [-xX metafield] [-s metafield:value]... " + "[-o output] [inputfile]\n" % sys.argv[0]) + sys.stderr.write("\tInputfile must be OpenDocument format\n") + sys.exit(exitcode) + + +try: + opts, args = getopt.getopt(sys.argv[1:], "lLs:o:x:X:") +except getopt.GetoptError: + exitwithusage() + +if len(opts) == 0: + exitwithusage() + +for o, a in opts: + if o == '-s': + if a.find(":") >= 0: + k,v = a.split(":",1) + else: + k,v = (a, "") + if len(k) == 0: + exitwithusage() + setfields[unicode(k)] = unicode(v) + if o == '-l': + listfields = True + Listfields = False + if o == '-L': + Listfields = True + listfields = False + if o == "-x": + xfields.append(unicode(a)) + if o == "-X": + Xfields.append(unicode(a)) + if o == "-o": + outputfile = unicode(a) + +if len(args) != 0: + inputfile = unicode(args[0]) + +user_fields = odf.userfield.UserFields(inputfile, outputfile) + +if xfields: + for value in user_fields.list_values(xfields): + print (value) + +if Listfields or Xfields: + if Listfields: + Xfields = None + for field_name, value_type, value in user_fields.list_fields_and_values( + Xfields): + print ("%s#%s:%s" % (field_name, value_type, value)) + +if listfields: + for value in user_fields.list_fields(): + print (value) + +if setfields: + user_fields.update(setfields) + + + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/pip-bins/pdf2txt.py b/pip-bins/pdf2txt.py new file mode 100755 index 0000000..cda5b88 --- /dev/null +++ b/pip-bins/pdf2txt.py @@ -0,0 +1,324 @@ +#!/usr/bin/python3 +"""A command line tool for extracting text and images from PDF and +output it to plain text, html, xml or tags. +""" + +import argparse +import logging +import sys +from collections.abc import Container, Iterable +from typing import Any + +import pdfminer.high_level +from pdfminer.layout import LAParams +from pdfminer.pdfexceptions import PDFValueError +from pdfminer.utils import AnyIO + +logging.basicConfig() + +OUTPUT_TYPES = ((".htm", "html"), (".html", "html"), (".xml", "xml"), (".tag", "tag")) + + +def float_or_disabled(x: str) -> float | None: + if x.lower().strip() == "disabled": + return None + try: + return float(x) + except ValueError as err: + raise argparse.ArgumentTypeError(f"invalid float value: {x}") from err + + +def extract_text( + files: Iterable[str] = [], + outfile: str = "-", + laparams: LAParams | None = None, + output_type: str = "text", + codec: str = "utf-8", + strip_control: bool = False, + maxpages: int = 0, + page_numbers: Container[int] | None = None, + password: str = "", + scale: float = 1.0, + rotation: int = 0, + layoutmode: str = "normal", + output_dir: str | None = None, + debug: bool = False, + disable_caching: bool = False, + **kwargs: Any, +) -> None: + if not files: + raise PDFValueError("Must provide files to work upon!") + + if output_type == "text" and outfile != "-": + for override, alttype in OUTPUT_TYPES: + if outfile.endswith(override): + output_type = alttype + + if outfile == "-": + outfp: AnyIO = sys.stdout + if sys.stdout.encoding is not None: + codec = "utf-8" + for fname in files: + with open(fname, "rb") as fp: + pdfminer.high_level.extract_text_to_fp(fp, **locals()) + else: + # Use context manager for file output, ensuring proper cleanup + with open(outfile, "wb") as outfp: + for fname in files: + with open(fname, "rb") as fp: + pdfminer.high_level.extract_text_to_fp(fp, **locals()) + + +def create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, add_help=True) + parser.add_argument( + "files", + type=str, + default=None, + nargs="+", + help="One or more paths to PDF files.", + ) + + parser.add_argument( + "--version", + "-v", + action="version", + version=f"pdfminer.six v{pdfminer.__version__}", + ) + parser.add_argument( + "--debug", + "-d", + default=False, + action="store_true", + help="Use debug logging level.", + ) + parser.add_argument( + "--disable-caching", + "-C", + default=False, + action="store_true", + help="If caching or resources, such as fonts, should be disabled.", + ) + + parse_params = parser.add_argument_group( + "Parser", + description="Used during PDF parsing", + ) + parse_params.add_argument( + "--page-numbers", + type=int, + default=None, + nargs="+", + help="A space-seperated list of page numbers to parse.", + ) + parse_params.add_argument( + "--pagenos", + "-p", + type=str, + help="A comma-separated list of page numbers to parse. " + "Included for legacy applications, use --page-numbers " + "for more idiomatic argument entry.", + ) + parse_params.add_argument( + "--maxpages", + "-m", + type=int, + default=0, + help="The maximum number of pages to parse.", + ) + parse_params.add_argument( + "--password", + "-P", + type=str, + default="", + help="The password to use for decrypting PDF file.", + ) + parse_params.add_argument( + "--rotation", + "-R", + default=0, + type=int, + help="The number of degrees to rotate the PDF " + "before other types of processing.", + ) + + la_params = LAParams() # will be used for defaults + la_param_group = parser.add_argument_group( + "Layout analysis", + description="Used during layout analysis.", + ) + la_param_group.add_argument( + "--no-laparams", + "-n", + default=False, + action="store_true", + help="If layout analysis parameters should be ignored.", + ) + la_param_group.add_argument( + "--detect-vertical", + "-V", + default=la_params.detect_vertical, + action="store_true", + help="If vertical text should be considered during layout analysis", + ) + la_param_group.add_argument( + "--line-overlap", + type=float, + default=la_params.line_overlap, + help="If two characters have more overlap than this they " + "are considered to be on the same line. The overlap is specified " + "relative to the minimum height of both characters.", + ) + la_param_group.add_argument( + "--char-margin", + "-M", + type=float, + default=la_params.char_margin, + help="If two characters are closer together than this margin they " + "are considered to be part of the same line. The margin is " + "specified relative to the width of the character.", + ) + la_param_group.add_argument( + "--word-margin", + "-W", + type=float, + default=la_params.word_margin, + help="If two characters on the same line are further apart than this " + "margin then they are considered to be two separate words, and " + "an intermediate space will be added for readability. The margin " + "is specified relative to the width of the character.", + ) + la_param_group.add_argument( + "--line-margin", + "-L", + type=float, + default=la_params.line_margin, + help="If two lines are close together they are considered to " + "be part of the same paragraph. The margin is specified " + "relative to the height of a line.", + ) + la_param_group.add_argument( + "--boxes-flow", + "-F", + type=float_or_disabled, + default=la_params.boxes_flow, + help="Specifies how much a horizontal and vertical position of a " + "text matters when determining the order of lines. The value " + "should be within the range of -1.0 (only horizontal position " + "matters) to +1.0 (only vertical position matters). You can also " + "pass `disabled` to disable advanced layout analysis, and " + "instead return text based on the position of the bottom left " + "corner of the text box.", + ) + la_param_group.add_argument( + "--all-texts", + "-A", + default=la_params.all_texts, + action="store_true", + help="If layout analysis should be performed on text in figures.", + ) + + output_params = parser.add_argument_group( + "Output", + description="Used during output generation.", + ) + output_params.add_argument( + "--outfile", + "-o", + type=str, + default="-", + help="Path to file where output is written. " + 'Or "-" (default) to write to stdout.', + ) + output_params.add_argument( + "--output_type", + "-t", + type=str, + default="text", + help="Type of output to generate {text,html,xml,tag}.", + ) + output_params.add_argument( + "--codec", + "-c", + type=str, + default="utf-8", + help="Text encoding to use in output file.", + ) + output_params.add_argument( + "--output-dir", + "-O", + default=None, + help="The output directory to put extracted images in. If not given, " + "images are not extracted.", + ) + output_params.add_argument( + "--layoutmode", + "-Y", + default="normal", + type=str, + help="Type of layout to use when generating html " + "{normal,exact,loose}. If normal,each line is" + " positioned separately in the html. If exact" + ", each character is positioned separately in" + " the html. If loose, same result as normal " + "but with an additional newline after each " + "text line. Only used when output_type is html.", + ) + output_params.add_argument( + "--scale", + "-s", + type=float, + default=1.0, + help="The amount of zoom to use when generating html file. " + "Only used when output_type is html.", + ) + output_params.add_argument( + "--strip-control", + "-S", + default=False, + action="store_true", + help="Remove control statement from text. Only used when output_type is xml.", + ) + + return parser + + +def parse_args(args: list[str] | None) -> argparse.Namespace: + parsed_args = create_parser().parse_args(args=args) + + # Propagate parsed layout parameters to LAParams object + if parsed_args.no_laparams: + parsed_args.laparams = None + else: + parsed_args.laparams = LAParams( + line_overlap=parsed_args.line_overlap, + char_margin=parsed_args.char_margin, + line_margin=parsed_args.line_margin, + word_margin=parsed_args.word_margin, + boxes_flow=parsed_args.boxes_flow, + detect_vertical=parsed_args.detect_vertical, + all_texts=parsed_args.all_texts, + ) + + if parsed_args.page_numbers: + parsed_args.page_numbers = {x - 1 for x in parsed_args.page_numbers} + + if parsed_args.pagenos: + parsed_args.page_numbers = {int(x) - 1 for x in parsed_args.pagenos.split(",")} + + if parsed_args.output_type == "text" and parsed_args.outfile != "-": + for override, alttype in OUTPUT_TYPES: + if parsed_args.outfile.endswith(override): + parsed_args.output_type = alttype + + return parsed_args + + +def main(args: list[str] | None = None) -> int: + parsed_args = parse_args(args) + extract_text(**vars(parsed_args)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pip-bins/pdfplumber b/pip-bins/pdfplumber new file mode 100755 index 0000000..cfb94ab --- /dev/null +++ b/pip-bins/pdfplumber @@ -0,0 +1,8 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from pdfplumber.cli import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/pip-bins/pypdfium2 b/pip-bins/pypdfium2 new file mode 100755 index 0000000..ec81a5f --- /dev/null +++ b/pip-bins/pypdfium2 @@ -0,0 +1,8 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +import re +import sys +from pypdfium2.__main__ import cli_main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(cli_main()) diff --git a/pip-bins/xml2odf b/pip-bins/xml2odf new file mode 100755 index 0000000..e7114a8 --- /dev/null +++ b/pip-bins/xml2odf @@ -0,0 +1,241 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (C) 2006 Søren Roug, European Environment Agency +# +# This is free software. You may redistribute it under the terms +# of the Apache license and the GNU General Public License Version +# 2 or at your option any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Contributor(s): +# +# + +# OpenDocument can be a complete office document in a single +# XML document. This script will take such a document and create +# a package +import io +import zipfile,time, sys, getopt +import xml.sax, xml.sax.saxutils +from odf import manifest + +class SplitWriter: + def __init__(self): + self.activefiles = [] + self._content = [] + self._meta = [] + self._styles = [] + self._settings = [] + + self.files = {'content': self._content, 'meta': self._meta, + 'styles':self._styles, 'settings': self._settings } + + def write(self, str): + for f in self.activefiles: + f.append(str) + + def activate(self, filename): + file = self.files[filename] + if file not in self.activefiles: + self.activefiles.append(file) + + def deactivate(self, filename): + file = self.files[filename] + if file in self.activefiles: + self.activefiles.remove(file) + +odmimetypes = { + 'application/vnd.oasis.opendocument.text': '.odt', + 'application/vnd.oasis.opendocument.text-template': '.ott', + 'application/vnd.oasis.opendocument.graphics': '.odg', + 'application/vnd.oasis.opendocument.graphics-template': '.otg', + 'application/vnd.oasis.opendocument.presentation': '.odp', + 'application/vnd.oasis.opendocument.presentation-template': '.otp', + 'application/vnd.oasis.opendocument.spreadsheet': '.ods', + 'application/vnd.oasis.opendocument.spreadsheet-template': '.ots', + 'application/vnd.oasis.opendocument.chart': '.odc', + 'application/vnd.oasis.opendocument.chart-template': '.otc', + 'application/vnd.oasis.opendocument.image': '.odi', + 'application/vnd.oasis.opendocument.image-template': '.oti', + 'application/vnd.oasis.opendocument.formula': '.odf', + 'application/vnd.oasis.opendocument.formula-template': '.otf', + 'application/vnd.oasis.opendocument.text-master': '.odm', + 'application/vnd.oasis.opendocument.text-web': '.oth', +} + +OFFICENS = u"urn:oasis:names:tc:opendocument:xmlns:office:1.0" +base = xml.sax.saxutils.XMLGenerator + +class odfsplitter(base): + + def __init__(self): + self._mimetype = '' + self.output = SplitWriter() + self._prefixes = [] + base.__init__(self, self.output, 'utf-8') + + def startPrefixMapping(self, prefix, uri): + base.startPrefixMapping(self, prefix, uri) + self._prefixes.append('xmlns:%s="%s"' % (prefix, uri)) + + def startElementNS(self, name, qname, attrs): + if name == (OFFICENS, u"document"): + self._mimetype = attrs.get((OFFICENS, "mimetype")) + elif name == (OFFICENS, u"meta"): + self.output.activate('meta') + + elif name == (OFFICENS, u"settings"): + self.output.activate('settings') + elif name == (OFFICENS, u"scripts"): + self.output.activate('content') + elif name == (OFFICENS, u"font-face-decls"): + self.output.activate('content') + self.output.activate('styles') + elif name == (OFFICENS, u"styles"): + self.output.activate('styles') + elif name == (OFFICENS, u"automatic-styles"): + self.output.activate('content') + self.output.activate('styles') + elif name == (OFFICENS, u"master-styles"): + self.output.activate('styles') + elif name == (OFFICENS, u"body"): + self.output.activate('content') + base.startElementNS(self, name, qname, attrs) + + def endElementNS(self, name, qname): + base.endElementNS(self, name, qname) + if name == (OFFICENS, u"meta"): + self.output.deactivate('meta') + elif name == (OFFICENS, u"settings"): + self.output.deactivate('settings') + elif name == (OFFICENS, u"scripts"): + self.output.deactivate('content') + elif name == (OFFICENS, u"font-face-decls"): + self.output.deactivate('content') + self.output.deactivate('styles') + elif name == (OFFICENS, u"styles"): + self.output.deactivate('styles') + elif name == (OFFICENS, u"automatic-styles"): + self.output.deactivate('content') + self.output.deactivate('styles') + elif name == (OFFICENS, u"master-styles"): + self.output.deactivate('styles') + elif name == (OFFICENS, u"body"): + self.output.deactivate('content') + + + def content(self): + """ Return the content inside a wrapper called <office:document-content> + """ + prefixes = ' '.join(self._prefixes) + return ''.join(['<?xml version="1.0" encoding="UTF-8"?>\n<office:document-content %s office:version="1.0">' % prefixes] + list(map(lambda x: x.decode("utf-8"), self.output._content)) + ['</office:document-content>']) + + def settings(self): + prefixes = ' '.join(self._prefixes).encode('utf-8') + return ''.join( ['<?xml version="1.0" encoding="UTF-8"?>\n<office:document-settings %s office:version="1.0">' % prefixes] + self.output._settings + ['''</office:document-settings>''']) + + def styles(self): + prefixes = ' '.join(self._prefixes) + return ''.join( ['<?xml version="1.0" encoding="UTF-8"?>\n<office:document-styles %s office:version="1.0">' % prefixes] + list(map(lambda x: x.decode("utf-8"), self.output._styles)) + ['''</office:document-styles>''']) + + def meta(self): + prefixes = ' '.join(self._prefixes) + return ''.join( ['<?xml version="1.0" encoding="UTF-8"?>\n<office:document-meta %s office:version="1.0">' % prefixes] + list(map(lambda x: x.decode("utf-8"), self.output._meta)) + ['''</office:document-meta>''']) + +def usage(): + sys.stderr.write("Usage: %s [-o outputfile] [-s] inputfile\n" % sys.argv[0]) + +def manifestxml(m): + """ Generates the content of the manifest.xml file """ + xml=io.StringIO() + xml.write(u"<?xml version='1.0' encoding='UTF-8'?>\n") + m.toXml(0,xml) + return xml.getvalue() + +try: + opts, args = getopt.getopt(sys.argv[1:], "o:s", ["output=","suffix"]) +except getopt.GetoptError: + usage() + sys.exit(2) + +outputfile = '-' +addsuffix = False + +for o, a in opts: + if o in ("-o", "--output"): + outputfile = a + if o in ("-s", "--suffix"): + addsuffix = True + +if len(args) > 1: + usage() + sys.exit(2) + +odfs = odfsplitter() +parser = xml.sax.make_parser() +parser.setFeature(xml.sax.handler.feature_namespaces, 1) +parser.setContentHandler(odfs) +if len(args) == 0: + parser.parse(sys.stdin) +else: + parser.parse(open(args[0],"r")) + +mimetype = odfs._mimetype +suffix = odmimetypes.get(mimetype,'.xxx') + +if outputfile == '-': + if sys.stdout.isatty(): + sys.stderr.write("Won't write ODF file to terminal\n") + sys.exit(1) + z = zipfile.ZipFile(sys.stdout,"w") +else: + if addsuffix: + outputfile = outputfile + suffix + z = zipfile.ZipFile(outputfile,"w") + +now = time.localtime()[:6] + +# Write mimetype +zi = zipfile.ZipInfo('mimetype', now) +zi.compress_type = zipfile.ZIP_STORED +z.writestr(zi,mimetype) + +# Write content +zi = zipfile.ZipInfo("content.xml", now) +zi.compress_type = zipfile.ZIP_DEFLATED +z.writestr(zi,odfs.content() ) +# Write styles +zi = zipfile.ZipInfo("styles.xml", now) +zi.compress_type = zipfile.ZIP_DEFLATED +z.writestr(zi,odfs.styles() ) + +# Write meta +zi = zipfile.ZipInfo("meta.xml", now) +zi.compress_type = zipfile.ZIP_DEFLATED +z.writestr(zi,odfs.meta() ) + +m = manifest.Manifest() +m.addElement(manifest.FileEntry(fullpath="/", mediatype=mimetype)) +m.addElement(manifest.FileEntry(fullpath="content.xml",mediatype="text/xml")) +m.addElement(manifest.FileEntry(fullpath="styles.xml", mediatype="text/xml")) +m.addElement(manifest.FileEntry(fullpath="meta.xml", mediatype="text/xml")) + +# Write manifest +zi = zipfile.ZipInfo("META-INF/manifest.xml", now) +zi.compress_type = zipfile.ZIP_DEFLATED +z.writestr(zi, manifestxml(m).encode("utf-8") ) +z.close() + + + +# Local Variables: *** +# mode: python *** +# End: *** diff --git a/simplex-chat b/simplex-chat Binary files differnew file mode 100755 index 0000000..c697643 --- /dev/null +++ b/simplex-chat diff --git a/sorter/readme.md b/sorter/readme.md new file mode 100644 index 0000000..e81c411 --- /dev/null +++ b/sorter/readme.md @@ -0,0 +1,210 @@ +# Downloads Sorter + +A Python script that automatically sorts files from your `~/Downloads` folder into organised subdirectories based on filename keywords and — more importantly — the **actual text content** of each file. + +Built for Debian Linux, Polish and English documents, Python 3.11+. + +--- + +## How it works + +When you run the script it does the following: + +1. Scans every supported file in `~/Downloads` (not subdirectories) +2. Extracts the full text content from each file +3. Checks that text against the keyword rules in `rules.toml` +4. Falls back to checking the filename if no content match is found +5. Shows you a **dry-run preview** of where each file would go +6. Asks for confirmation before moving anything + +Content always beats filename — so a file called `scan001.pdf` that contains an Arc of Asia invoice will correctly go to `Work/ARC`, even though the filename gives no hint. + +### Supported file types + +`.pdf` `.docx` `.doc` `.txt` `.xlsx` `.xls` `.md` `.odt` + +You can add more in `rules.toml` (see below). + +--- + +## Files + +``` +sorter/ +├── sort_downloads.sh # Run this — installs deps, then calls the Python script +├── sort_downloads.py # The actual logic +└── rules.toml # Your rules — edit this to add keywords and categories +``` + +Keep all three files in the same directory. + +--- + +## Daily use + +```bash +cd ~/.local/bin/sorter +./sort_downloads.sh +``` + +That's it. The script will show a preview, then ask: + +``` +Proceed with moving files? [y/N] +``` + +Type `y` to move, or just press Enter to cancel without touching anything. + +### Useful flags + +```bash +# Skip the confirmation prompt and move immediately +./sort_downloads.sh --yes + +# Preview rules without scanning any files +./sort_downloads.sh --list-rules + +# Sort a different folder instead of ~/Downloads +./sort_downloads.sh --dir /path/to/folder + +# Use a different rules file +./sort_downloads.sh --config /path/to/other-rules.toml +``` + +### Output folders + +Subfolders are created automatically inside `~/Downloads` the first time a file routes there. Based on the default rules you will get: + +``` +~/Downloads/ +├── Work/ +│ ├── ARC/ +│ └── LKIT/ +├── AKW/ +└── Other/ +``` + +If two files would land on the same destination path, the script appends `_1`, `_2` etc. rather than overwriting. + +--- + +## How to extend it + +Everything is controlled by `rules.toml`. You never need to touch the Python script to add new keywords or categories. + +### Adding a keyword to an existing category + +Open `rules.toml` and add a line to the relevant list: + +```toml +[[categories]] +name = "Work/ARC" +content_keywords = [ + "arc of asia", + "9571181577", + "your new keyword here", # ← add here +] +filename_keywords = [ + "arc", + "new_filename_hint", # ← or here +] +``` + +`content_keywords` are matched against the full extracted text of the file. +`filename_keywords` are matched against the filename only, and only used if no content match was found first. + +Both are case-insensitive and partial — `"kasprzak"` will match `"Łukasz Kasprzak International Trade"`. + +### Adding a new category + +Append a new `[[categories]]` block anywhere in the list: + +```toml +[[categories]] +name = "Finance/Banking" +description = "Bank statements and account exports" +content_keywords = [ + "revolut", + "account statement", + "wyciąg bankowy", + "mbank", +] +filename_keywords = [ + "revolut", + "statement", + "wyciag", +] +``` + +The folder path (`Finance/Banking`, `Personal/Tax`, or any depth you like) will be created automatically inside `~/Downloads`. + +**Order matters** — categories are checked top to bottom and the first match wins. Put more specific categories above broader ones. + +### Adding a new file extension + +Add it to `supported_extensions` in `rules.toml`: + +```toml +supported_extensions = [ + ".pdf", + ".docx", + ".txt", + # ... existing entries ... + ".log", # plain text — works out of the box + ".csv", # plain text — works out of the box +] +``` + +Plain text formats (`.log`, `.csv`, `.json`, `.xml`, `.ini`, `.conf`) work immediately with no code changes. Binary formats that need a dedicated parser (e.g. `.pptx`, `.ods`) would require a small addition to `sort_downloads.py`. + +### Changing the catch-all folder + +Files that match no category go here: + +```toml +fallback_folder = "Other" +``` + +Change it to anything you like, e.g. `"Unsorted"` or `"Inbox"`. + +--- + +## Keyword tips + +- **NIP / REGON / KRS numbers** are the most reliable content keywords — they are unique per company and appear on every invoice and document +- Put **broad keywords** (like `"invoice"`) lower in the list so they don't accidentally catch documents that should match a more specific category above +- If a bank statement matches the wrong category because a supplier's address appears as a payee, move that supplier's address out of `content_keywords` and rely on the NIP/REGON instead +- Polish characters work fine in both content and filename keywords (`ł`, `ó`, `ą`, `ś`, `ź`, etc.) +- Run `./sort_downloads.sh --list-rules` after editing to confirm your changes loaded correctly + +--- + +## Dependencies + +| Library | Purpose | Installed by | +|---|---|---| +| `pdfplumber` | Read text from PDF files | `sort_downloads.sh` automatically | +| `python-docx` | Read text from .docx/.doc files | `sort_downloads.sh` automatically | +| `openpyxl` | Read text from .xlsx/.xls files | `sort_downloads.sh` automatically | +| `odfpy` | Read text from .odt files | `sort_downloads.sh` automatically | +| `tomllib` | Parse rules.toml | Built into Python 3.11+ — nothing to install | + +The bash wrapper (`sort_downloads.sh`) checks for and installs any missing libraries automatically on each run using `pip install --break-system-packages`. + +**Python 3.11 or newer is required.** On Debian 12+ this is the default. + +--- + +## Troubleshooting + +**A file landed in the wrong folder** +Run `--list-rules` to check what keywords are loaded. The preview also shows the match reason in brackets, e.g. `[content: 'wiercany 60a']` — use this to identify which keyword caused the misroute and either remove it or move a more specific category above it in `rules.toml`. + +**A file ended up in Other** +The script found no matching keyword in the file's content or filename. Open the file, find a unique phrase or number, and add it as a `content_keyword` to the appropriate category. + +**PDF content is not being read** +Check that `pdfplumber` is installed (`pip show pdfplumber`). Some PDFs are image-only scans with no embedded text — these cannot be read without OCR, which is not currently supported. + +**TOML syntax error on startup** +TOML is strict about quoting — all strings must be in double quotes. Numbers like NIP/REGON must also be quoted (`"9571181577"`, not `9571181577`) to be treated as text for matching. diff --git a/sorter/rules.toml b/sorter/rules.toml new file mode 100644 index 0000000..0c0a7ac --- /dev/null +++ b/sorter/rules.toml @@ -0,0 +1,124 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Downloads Sorter — Rules Configuration (TOML) +# +# PRIORITY: content_keywords > filename_keywords +# File contents are scanned first; filename is used as fallback. +# +# FOLDER PATHS: relative to your Downloads directory. +# Use / for subfolders, e.g. "Work/ARC" +# +# KEYWORD TIPS: +# - Case-insensitive ("arc" matches "ARC", "Arc", "arc") +# - Partial match ("kasprzak" matches "Łukasz Kasprzak International Trade") +# - Always quote strings — avoids TOML type surprises with numbers +# - NIP/REGON/KRS numbers are the most reliable content keywords +# - Polish characters work fine (ł, ó, ą, ś, ź, etc.) +# - Categories are checked in order — first match wins +# ───────────────────────────────────────────────────────────────────────────── + +# Catch-all folder for files that match no category +fallback_folder = "Other" + +# File types to scan (add or remove extensions as needed) +supported_extensions = [ + ".pdf", + ".docx", + ".doc", + ".txt", + ".xlsx", + ".xls", + ".md", + ".odt", +] + +# ── Categories ──────────────────────────────────────────────────────────────── +# Each [[categories]] block defines one destination folder. +# Add as many blocks as you need. + +[[categories]] +name = "Work/ARC" +description = "Arc of Asia Sp. z o.o. — documents, invoices, correspondence" +content_keywords = [ + "arc of asia", + "arc of asia spółka z ograniczoną odpowiedzialnością", + "arc of asia spolka z ograniczona odpowiedzialnoscia", + "aleja grunwaldzka 56", + "9571181577", # NIP + "540356138", # REGON + "0001140839", # KRS + "pl957118157700000", # EORI +] +filename_keywords = [ + "arc", + "aoa", + "arc_of_asia", +] + +[[categories]] +name = "Work/LKIT" +description = "Łukasz Kasprzak International Trade — documents, invoices, correspondence" +content_keywords = [ + "łukasz kasprzak international trade", + "lukasz kasprzak international trade", + "8181739189", # NIP + "540804571", # REGON +] +filename_keywords = [ + "lkit", + "kasprzak", +] + +[[categories]] +name = "AKW" +description = "Akademia Katolicka w Warszawie" +content_keywords = [ + "akademia katolicka w warszawie", + "akademia katolicka", +] +filename_keywords = [ + "akw", +] + +# ── Add more categories below ───────────────────────────────────────────────── + +# [[categories]] +# name = "Finance/Banking" +# description = "Bank statements, Revolut, account history" +# content_keywords = [ +# "revolut", +# "account statement", +# "wyciąg bankowy", +# ] +# filename_keywords = [ +# "revolut", +# "statement", +# "wyciag", +# ] + +# [[categories]] +# name = "Finance/Invoices" +# description = "VAT invoices and receipts" +# content_keywords = [ +# "faktura vat", +# "invoice", +# "23% vat", +# ] +# filename_keywords = [ +# "faktura", +# "invoice", +# ] + +# [[categories]] +# name = "Personal/Tax" +# description = "Tax documents and PIT forms" +# content_keywords = [ +# "pit-37", +# "pit-36", +# "urząd skarbowy", +# "zeznanie podatkowe", +# ] +# filename_keywords = [ +# "pit", +# "podatek", +# "tax", +# ] diff --git a/sorter/sort_downloads.py b/sorter/sort_downloads.py new file mode 100644 index 0000000..10cc696 --- /dev/null +++ b/sorter/sort_downloads.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +""" +Downloads Folder Sorter +Rules are loaded from rules.toml — edit that file to add keywords/categories. +Content match takes priority over filename match. +Requires Python 3.11+ (uses built-in tomllib). +""" + +import sys +import shutil +import tomllib +import argparse +from pathlib import Path + +# ── Optional deps (graceful fallback) ──────────────────────────────────────── +try: + import pdfplumber + HAS_PDF = True +except ImportError: + HAS_PDF = False + +try: + from docx import Document + HAS_DOCX = True +except ImportError: + HAS_DOCX = False + +try: + import openpyxl + HAS_XLSX = True +except ImportError: + HAS_XLSX = False + +try: + from odf import teletype + from odf.opendocument import load as odf_load + HAS_ODT = True +except ImportError: + HAS_ODT = False + +# ── Config loading ──────────────────────────────────────────────────────────── + +def load_config(config_path: Path) -> dict: + if not config_path.exists(): + print(f"Error: Config file not found: {config_path}") + print("Make sure rules.toml is in the same folder as this script.") + sys.exit(1) + with open(config_path, "rb") as f: + config = tomllib.load(f) + return config + +def validate_config(config: dict): + if "categories" not in config or not config["categories"]: + print("Error: 'categories' is missing or empty in rules.toml") + sys.exit(1) + for cat in config["categories"]: + if "name" not in cat: + print("Error: A category in rules.toml is missing a 'name' field.") + sys.exit(1) + +# ── Text extraction ─────────────────────────────────────────────────────────── + +def extract_text_pdf(path: Path) -> str: + if not HAS_PDF: + return "" + try: + with pdfplumber.open(path) as pdf: + return "\n".join(page.extract_text() or "" for page in pdf.pages) + except Exception: + return "" + +def extract_text_docx(path: Path) -> str: + if not HAS_DOCX: + return "" + try: + doc = Document(path) + return "\n".join(p.text for p in doc.paragraphs) + except Exception: + return "" + +def extract_text_xlsx(path: Path) -> str: + if not HAS_XLSX: + return "" + try: + wb = openpyxl.load_workbook(path, read_only=True, data_only=True) + parts = [] + for ws in wb.worksheets: + for row in ws.iter_rows(values_only=True): + parts.append(" ".join(str(c) for c in row if c is not None)) + return "\n".join(parts) + except Exception: + return "" + +def extract_text_odt(path: Path) -> str: + if not HAS_ODT: + return "" + try: + doc = odf_load(path) + return teletype.extractText(doc.text) + except Exception: + return "" + +def extract_text_plain(path: Path) -> str: + for enc in ("utf-8", "utf-16", "latin-1"): + try: + return path.read_text(encoding=enc) + except Exception: + continue + return "" + +def extract_text(path: Path) -> str: + ext = path.suffix.lower() + if ext == ".pdf": + return extract_text_pdf(path) + elif ext in (".docx", ".doc"): + return extract_text_docx(path) + elif ext in (".xlsx", ".xls"): + return extract_text_xlsx(path) + elif ext == ".odt": + return extract_text_odt(path) + elif ext in (".txt", ".md"): + return extract_text_plain(path) + return "" + +# ── Classification ──────────────────────────────────────────────────────────── + +def classify(text: str, filename: str, categories: list, fallback: str) -> tuple: + """ + Returns (destination_folder, match_reason). + Content keywords checked first, then filename keywords. + Categories are evaluated in order — first match wins. + """ + text_low = text.lower() + name_low = filename.lower() + + for cat in categories: + for kw in cat.get("content_keywords", []): + if str(kw).lower() in text_low: + return cat["name"], f"content: '{kw}'" + + for cat in categories: + for kw in cat.get("filename_keywords", []): + if str(kw).lower() in name_low: + return cat["name"], f"filename: '{kw}'" + + return fallback, "no match" + +# ── Core ────────────────────────────────────────────────────────────────────── + +def collect_files(downloads: Path, extensions: set) -> list: + files = [] + for entry in downloads.iterdir(): + if entry.is_file() and entry.suffix.lower() in extensions: + files.append(entry) + return sorted(files) + +def plan_moves(files: list, downloads: Path, config: dict) -> list: + categories = config["categories"] + fallback = config.get("fallback_folder", "Other") + moves = [] + for f in files: + print(f" Scanning: {f.name} ...", end=" ", flush=True) + text = extract_text(f) + dest_folder, reason = classify(text, f.name, categories, fallback) + dest_path = downloads / dest_folder / f.name + moves.append((f, dest_path, reason)) + print(f"-> {dest_folder}/ ({reason})") + return moves + +def print_plan(moves: list, downloads: Path): + print("\n" + "=" * 70) + print(" DRY-RUN PREVIEW") + print("=" * 70) + col_w = max(len(src.name) for src, _, _ in moves) + 2 + for src, dst, reason in moves: + rel = dst.relative_to(downloads) + print(f" {src.name:<{col_w}} -> {rel} [{reason}]") + print("=" * 70) + +def execute_moves(moves: list, downloads: Path) -> list: + errors = [] + for src, dst, _ in moves: + try: + dst.parent.mkdir(parents=True, exist_ok=True) + final_dst = dst + counter = 1 + while final_dst.exists(): + final_dst = dst.with_stem(f"{dst.stem}_{counter}") + counter += 1 + shutil.move(str(src), str(final_dst)) + print(f" OK {src.name} -> {final_dst.relative_to(downloads)}") + except Exception as e: + errors.append((src, e)) + print(f" ERR {src.name} -> {e}") + return errors + +def check_deps() -> list: + missing = [] + if not HAS_PDF: + missing.append("pdfplumber") + if not HAS_DOCX: + missing.append("python-docx") + if not HAS_XLSX: + missing.append("openpyxl") + if not HAS_ODT: + missing.append("odfpy") + return missing + +# ── Entry point ─────────────────────────────────────────────────────────────── + +def main(): + if sys.version_info < (3, 11): + print("Error: Python 3.11 or newer is required (for built-in tomllib).") + print(f" You have Python {sys.version}") + sys.exit(1) + + parser = argparse.ArgumentParser( + description="Sort ~/Downloads using rules defined in rules.toml" + ) + parser.add_argument( + "--dir", + default=str(Path.home() / "Downloads"), + help="Path to Downloads folder (default: ~/Downloads)", + ) + parser.add_argument( + "--config", + default=None, + help="Path to rules.toml (default: same folder as this script)", + ) + parser.add_argument( + "--yes", "-y", + action="store_true", + help="Skip confirmation prompt and move immediately", + ) + parser.add_argument( + "--list-rules", + action="store_true", + help="Print loaded categories and keywords, then exit", + ) + args = parser.parse_args() + + script_dir = Path(__file__).parent + config_path = Path(args.config) if args.config else script_dir / "rules.toml" + config = load_config(config_path) + validate_config(config) + + extensions = set(config.get("supported_extensions", [ + ".pdf", ".docx", ".doc", ".txt", ".xlsx", ".xls", ".md", ".odt" + ])) + + if args.list_rules: + print(f"\nConfig: {config_path}\n") + for cat in config["categories"]: + print(f" [folder] {cat['name']}") + if cat.get("description"): + print(f" {cat['description']}") + if cat.get("content_keywords"): + kws = ", ".join(str(k) for k in cat["content_keywords"]) + print(f" content : {kws}") + if cat.get("filename_keywords"): + kws = ", ".join(str(k) for k in cat["filename_keywords"]) + print(f" filename: {kws}") + print() + print(f" [folder] {config.get('fallback_folder', 'Other')} (catch-all)\n") + return + + downloads = Path(args.dir).expanduser().resolve() + if not downloads.is_dir(): + print(f"Error: '{downloads}' is not a directory.") + sys.exit(1) + + missing = check_deps() + if missing: + print("Warning: some content-extraction libraries are missing.") + print(" Install them for full content scanning:") + for m in missing: + print(f" pip install {m}") + print() + + print(f"Scanning: {downloads}") + print(f"Rules: {config_path}\n") + + files = collect_files(downloads, extensions) + if not files: + print("No supported files found.") + sys.exit(0) + + print(f"Found {len(files)} file(s). Classifying...\n") + moves = plan_moves(files, downloads, config) + + print_plan(moves, downloads) + + if not args.yes: + answer = input("\nProceed with moving files? [y/N] ").strip().lower() + if answer != "y": + print("Aborted. No files were moved.") + sys.exit(0) + + print("\nMoving files...\n") + errors = execute_moves(moves, downloads) + + print() + if errors: + print(f"Done with {len(errors)} error(s).") + else: + print("All files moved successfully.") + +if __name__ == "__main__": + main() diff --git a/sorter/sort_downloads.sh b/sorter/sort_downloads.sh new file mode 100755 index 0000000..d2023a8 --- /dev/null +++ b/sorter/sort_downloads.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# sort_downloads.sh — wrapper for sort_downloads.py +# Installs missing Python deps, then runs the sorter. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PYTHON_SCRIPT="$SCRIPT_DIR/sort_downloads.py" + +# ── Colour helpers ──────────────────────────────────────────────────────────── +GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; NC='\033[0m' +info() { echo -e "${GREEN}[sort]${NC} $*"; } +warn() { echo -e "${YELLOW}[warn]${NC} $*"; } +error() { echo -e "${RED}[error]${NC} $*" >&2; } + +# ── Check Python 3.11+ ──────────────────────────────────────────────────────── +if ! command -v python3 &>/dev/null; then + error "python3 not found. Install it with: sudo apt install python3" + exit 1 +fi + +PYVER=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") +PYMAJ=$(python3 -c "import sys; print(sys.version_info.major)") +PYMIN=$(python3 -c "import sys; print(sys.version_info.minor)") + +if [[ "$PYMAJ" -lt 3 ]] || [[ "$PYMAJ" -eq 3 && "$PYMIN" -lt 11 ]]; then + error "Python 3.11+ required for built-in tomllib. You have Python $PYVER." + exit 1 +fi + +info "Using Python $(python3 --version) — tomllib built-in, no extra config deps needed." + +# ── Install pip if missing ──────────────────────────────────────────────────── +if ! python3 -m pip --version &>/dev/null; then + warn "pip not found — installing..." + sudo apt-get install -y python3-pip +fi + +# ── Install content-extraction libraries ───────────────────────────────────── +# Note: pyyaml is no longer needed — TOML is built into Python 3.11+ +declare -A DEPS=( + [pdfplumber]="pdfplumber" + [docx]="python-docx" + [openpyxl]="openpyxl" + [odf]="odfpy" +) + +for import_name in "${!DEPS[@]}"; do + pip_name="${DEPS[$import_name]}" + if ! python3 -c "import $import_name" &>/dev/null 2>&1; then + info "Installing $pip_name..." + python3 -m pip install --quiet "$pip_name" --break-system-packages + fi +done + +info "All dependencies ready." +echo "" + +# ── Run the sorter ──────────────────────────────────────────────────────────── +exec python3 "$PYTHON_SCRIPT" "$@" diff --git a/straper/README.md b/straper/README.md new file mode 100644 index 0000000..a076f8f --- /dev/null +++ b/straper/README.md @@ -0,0 +1,123 @@ +# sanctum / labunix rebuild toolkit + +A modular, idempotent, Unix-style rebuild toolkit designed to reduce recovery time from weeks to hours. + +## Design goals + +- Separate **install**, **restore**, and **verify**. +- Keep base connectivity stable before touching overlay networking. +- Make secrets and identities opt-in. +- Never abort the whole run because one restore item failed. +- Back up before overwrite and write a machine-readable report. +- Work on both **VM** and **real hardware**. + +## Files + +- `lib/common.sh` — shared helpers +- `install-base.sh` — installs packages and base structure only +- `restore-configs.sh` — restores configs from `capture-full.sh` database +- `doctor.sh` — read-only health and readiness checks + +## Defaults + +- Safe by default +- No automatic identity transplant +- No blind `/etc/network` overwrite unless explicitly requested +- No immutable `/etc/resolv.conf` +- No automatic WireGuard activation unless explicitly requested + +## Roles + +- `lab` — VM / test clone, least invasive +- `hardware` — real machine, still conservative +- `replacement` — real disaster-recovery target, identities allowed when requested + +## Recommended recovery order + +### 1. Base install + +```bash +sudo ./install-base.sh --role replacement --profile core --start-safe-services +``` + +### 2. Restore essentials first + +```bash +sudo ./restore-configs.sh \ + --role replacement \ + --category system-basics \ + --category users \ + --category ssh \ + --category apt-sources \ + --start-services +``` + +### 3. Restore network/DNS/firewall carefully + +```bash +sudo ./restore-configs.sh \ + --role replacement \ + --category network-base \ + --category dns \ + --category firewall \ + --network-mode source \ + --dns-mode chain \ + --start-services +``` + +### 4. Restore services + +```bash +sudo ./restore-configs.sh \ + --role replacement \ + --category nginx \ + --category mariadb \ + --category postfix \ + --category prosody \ + --category docker \ + --restore-secrets \ + --start-services +``` + +### 5. Restore identities last + +```bash +sudo ./restore-configs.sh \ + --role replacement \ + --category privacy \ + --category tls \ + --category identities \ + --restore-secrets \ + --restore-identities \ + --start-services +``` + +### 6. Verify + +```bash +sudo ./doctor.sh --role replacement --strict +``` + +## Reports and state + +Each run writes: + +- `/var/log/labunix-rebuild/report-<RUN_ID>.tsv` +- `/var/log/labunix-rebuild/doctor-<RUN_ID>.txt` +- `/var/lib/labunix-rebuild/state-<RUN_ID>.env` +- `/var/lib/labunix-rebuild/backups/<RUN_ID>/...` + +## Notes + +- `install-base.sh` does **not** restore your server identity. +- `restore-configs.sh` is intentionally interactive unless `--yes` is used. +- `doctor.sh` is read-only. +- For network restore, use local console access when possible. + + +## Latest patch notes + +- v0.1.1 fixes early exit in `load_optional_config()` under `set -e` +- `install-base.sh` now has a bootstrap DNS fallback for broken `systemd-resolved` stub setups +- initial `apt-get update` is now validated and fails loudly instead of being reported as success +- locale handling no longer requires locale tools before base packages install them diff --git a/straper/common.sh b/straper/common.sh new file mode 100644 index 0000000..32f3581 --- /dev/null +++ b/straper/common.sh @@ -0,0 +1,330 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Shared helpers for the labunix/sanctum rebuild toolkit. + +COMMON_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +TOOLKIT_DIR="$(cd -- "${COMMON_DIR}/.." && pwd)" + +TOOLKIT_NAME="labunix-rebuild" +TOOLKIT_VERSION="0.1.1" + +: "${DB_DIR:=/srv/sanctum-rebuild}" +: "${PUBLIC_DIR:=${DB_DIR}/db/public}" +: "${SECRET_DIR:=${DB_DIR}/db/secret}" +: "${ROLE:=lab}" # lab | hardware | replacement +: "${PROFILE:=core}" # minimal | core | full +: "${RUN_ID:=$(date +%Y%m%dT%H%M%S)}" +: "${STATE_DIR:=/var/lib/${TOOLKIT_NAME}}" +: "${LOG_DIR:=/var/log/${TOOLKIT_NAME}}" +: "${BACKUP_DIR:=${STATE_DIR}/backups/${RUN_ID}}" +: "${REPORT_FILE:=${LOG_DIR}/report-${RUN_ID}.tsv}" +: "${STATE_FILE:=${STATE_DIR}/state-${RUN_ID}.env}" +: "${DRY_RUN:=false}" +: "${ASSUME_YES:=false}" +: "${VERBOSE:=false}" +: "${START_SERVICES:=false}" +: "${RESTORE_IDENTITIES:=false}" +: "${RESTORE_SECRETS:=false}" +: "${NETWORK_MODE:=safe}" # safe | source +: "${DNS_MODE:=auto}" # auto | resolved | dnsmasq | unbound | chain +: "${RESTORE_PURGE:=false}" + +umask 022 + +now_iso() { date +"%Y-%m-%dT%H:%M:%S%z"; } +log() { printf '[%s] %s\n' "$(now_iso)" "$*"; } +warn() { printf '[%s] WARN: %s\n' "$(now_iso)" "$*" >&2; } +die() { printf '[%s] ERROR: %s\n' "$(now_iso)" "$*" >&2; exit 1; } + +ensure_root() { + [[ ${EUID} -eq 0 ]] || die "must be run as root" +} + +ensure_runtime_dirs() { + mkdir -p -- "${STATE_DIR}" "${LOG_DIR}" "${BACKUP_DIR}" + touch -- "${REPORT_FILE}" "${STATE_FILE}" + chmod 700 -- "${STATE_DIR}" "${BACKUP_DIR}" + chmod 755 -- "${LOG_DIR}" + if [[ ! -s ${REPORT_FILE} ]]; then + printf 'timestamp\tscript\tcategory\titem\taction\tstatus\tnote\tbackup\n' > "${REPORT_FILE}" + fi +} + +load_optional_config() { + local conf + for conf in /etc/labunix/rebuild.conf /root/.config/labunix/rebuild.conf; do + if [[ -f ${conf} ]]; then + # shellcheck disable=SC1090 + source "${conf}" + fi + done + return 0 +} + +require_cmd() { + local c + for c in "$@"; do + command -v -- "${c}" >/dev/null 2>&1 || die "required command not found: ${c}" + done +} + +have_cmd() { + command -v -- "$1" >/dev/null 2>&1 +} + +set_state() { + local key="$1" value="$2" tmp + tmp="$(mktemp)" + if [[ -f ${STATE_FILE} ]]; then + grep -v -E "^${key}=" "${STATE_FILE}" > "${tmp}" || true + fi + printf '%s=%q\n' "${key}" "${value}" >> "${tmp}" + mv -- "${tmp}" "${STATE_FILE}" + chmod 600 -- "${STATE_FILE}" +} + +get_state() { + local key="$1" + [[ -f ${STATE_FILE} ]] || return 1 + awk -F= -v k="${key}" '$1==k {sub(/^[^=]*=/,""); print; found=1} END {exit(found?0:1)}' "${STATE_FILE}" +} + +report() { + local script="$1" category="$2" item="$3" action="$4" status="$5" note="$6" backup="${7:-}" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$(now_iso)" "${script}" "${category}" "${item}" "${action}" "${status}" "${note}" "${backup}" \ + >> "${REPORT_FILE}" +} + +run() { + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] %s\n' "$*" + return 0 + fi + "$@" +} + +run_capture() { + local outfile="$1"; shift + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] capture %s <= %s\n' "${outfile}" "$*" + return 0 + fi + "$@" > "${outfile}" 2>&1 +} + +backup_target() { + local target="$1" rel backup + [[ -e ${target} || -L ${target} ]] || return 1 + rel="${target#/}" + backup="${BACKUP_DIR}/${rel}" + mkdir -p -- "$(dirname -- "${backup}")" + cp -a -- "${target}" "${backup}" + printf '%s\n' "${backup}" +} + +files_equal() { + local src="$1" dst="$2" + [[ -f ${src} && -f ${dst} ]] || return 1 + cmp -s -- "${src}" "${dst}" +} + +dirs_equal() { + local src="$1" dst="$2" + [[ -d ${src} && -d ${dst} ]] || return 1 + diff -qr -- "${src}" "${dst}" >/dev/null 2>&1 +} + +copy_path() { + local src="$1" dst="$2" + if [[ -d ${src} ]]; then + mkdir -p -- "${dst}" + if [[ ${RESTORE_PURGE} == true ]]; then + rsync -a --delete -- "${src}/" "${dst}/" + else + rsync -a -- "${src}/" "${dst}/" + fi + else + mkdir -p -- "$(dirname -- "${dst}")" + cp -a -- "${src}" "${dst}" + fi +} + +restore_path() { + local script="$1" category="$2" item="$3" src="$4" dst="$5" mode="${6:-}" owner="${7:-}" group="${8:-}" + local backup="" changed_note="restored" + + if [[ ! -e ${src} && ! -L ${src} ]]; then + report "${script}" "${category}" "${item}" "restore" "skipped" "source missing: ${src}" "" + return 0 + fi + + if [[ -f ${src} && -f ${dst} ]] && files_equal "${src}" "${dst}"; then + if [[ -z ${mode} && -z ${owner} && -z ${group} ]]; then + report "${script}" "${category}" "${item}" "restore" "ok" "already up to date" "" + return 0 + fi + changed_note="metadata normalized" + fi + + if [[ -d ${src} && -d ${dst} ]] && dirs_equal "${src}" "${dst}"; then + if [[ -z ${mode} && -z ${owner} && -z ${group} ]]; then + report "${script}" "${category}" "${item}" "restore" "ok" "already up to date" "" + return 0 + fi + changed_note="metadata normalized" + fi + + if [[ -e ${dst} || -L ${dst} ]]; then + if [[ ${DRY_RUN} == true ]]; then + backup="${BACKUP_DIR}/${dst#/}" + printf '[dry] backup %s -> %s\n' "${dst}" "${backup}" + else + backup="$(backup_target "${dst}")" + fi + fi + + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] restore %s -> %s\n' "${src}" "${dst}" + [[ -n ${mode} ]] && printf '[dry] chmod %s %s\n' "${mode}" "${dst}" + [[ -n ${owner} ]] && printf '[dry] chown -R %s%s %s\n' "${owner}" "${group:+:${group}}" "${dst}" + else + if [[ "${changed_note}" == "metadata normalized" ]]; then + : + else + copy_path "${src}" "${dst}" + fi + [[ -n ${mode} ]] && chmod "${mode}" -- "${dst}" 2>/dev/null || true + [[ -n ${owner} ]] && chown -R "${owner}${group:+:${group}}" -- "${dst}" 2>/dev/null || true + fi + + report "${script}" "${category}" "${item}" "restore" "changed" "${changed_note}" "${backup}" + return 0 +} + +prompt_yes_no() { + local prompt="$1" default="${2:-no}" answer="" + if [[ ${ASSUME_YES} == true ]]; then + return 0 + fi + case "${default}" in + yes) read -r -p "${prompt} [Y/n]: " answer ;; + no) read -r -p "${prompt} [y/N]: " answer ;; + *) read -r -p "${prompt} [y/n]: " answer ;; + esac + answer="${answer:-${default}}" + [[ ${answer} =~ ^([Yy]|[Yy][Ee][Ss])$ ]] +} + +list_from_file() { + local f="$1" + [[ -f ${f} ]] || return 0 + grep -Ev '^(#|$)' "${f}" +} + +apt_install_if_missing() { + local pkgs=() pkg + for pkg in "$@"; do + dpkg -s -- "${pkg}" >/dev/null 2>&1 || pkgs+=("${pkg}") + done + [[ ${#pkgs[@]} -eq 0 ]] && return 0 + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] apt-get install -y --no-install-recommends %s\n' "${pkgs[*]}" + else + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${pkgs[@]}" + fi +} + +systemctl_safe_enable() { + local unit="$1" + if ! systemctl list-unit-files --type=service --type=target --type=timer --type=socket | awk '{print $1}' | grep -qx -- "${unit}"; then + return 1 + fi + run systemctl enable "${unit}" +} + +systemctl_safe_start() { + local unit="$1" + if ! systemctl list-unit-files --type=service --type=target --type=timer --type=socket | awk '{print $1}' | grep -qx -- "${unit}"; then + return 1 + fi + run systemctl restart "${unit}" +} + +validate_sshd() { have_cmd sshd && sshd -t; } +validate_nginx() { have_cmd nginx && nginx -t; } +validate_nft() { [[ -f /etc/nftables.conf ]] && have_cmd nft && nft -c -f /etc/nftables.conf; } +validate_unbound() { have_cmd unbound-checkconf && unbound-checkconf; } +validate_dnsmasq() { have_cmd dnsmasq && dnsmasq --test; } +validate_postfix() { have_cmd postfix && postfix check; } +validate_mariadb() { have_cmd mariadbd && mariadbd --verbose --help >/dev/null 2>&1; } + +service_is_active() { + systemctl is-active --quiet "$1" +} + +service_is_enabled() { + systemctl is-enabled --quiet "$1" +} + +detect_virtualization() { + if have_cmd systemd-detect-virt; then + systemd-detect-virt || true + else + true + fi +} + +primary_iface() { + ip route get 1.1.1.1 2>/dev/null | awk '/dev/ {for(i=1;i<=NF;i++) if ($i=="dev") {print $(i+1); exit}}' +} + +primary_src_ip() { + ip route get 1.1.1.1 2>/dev/null | awk '/src/ {for(i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}' +} + +has_default_route() { + ip route show default | grep -q . +} + +can_reach_ip() { + local ip="$1" + ping -c1 -W3 "${ip}" >/dev/null 2>&1 +} + +can_resolve_name() { + local name="$1" + getent ahostsv4 "${name}" >/dev/null 2>&1 +} + +guess_dns_owner() { + if service_is_active unbound 2>/dev/null && service_is_active dnsmasq 2>/dev/null; then + printf 'chain\n' + elif service_is_active unbound 2>/dev/null; then + printf 'unbound\n' + elif service_is_active dnsmasq 2>/dev/null; then + printf 'dnsmasq\n' + elif service_is_active systemd-resolved 2>/dev/null; then + printf 'resolved\n' + else + printf 'unknown\n' + fi +} + +role_allows_identities() { + [[ ${ROLE} == replacement ]] +} + +role_allows_overlay() { + [[ ${ROLE} == replacement || ${ROLE} == hardware ]] +} + +mark_manual() { + local script="$1" category="$2" item="$3" note="$4" + report "${script}" "${category}" "${item}" "manual" "manual" "${note}" "" +} + +note_failure() { + local script="$1" category="$2" item="$3" action="$4" note="$5" backup="${6:-}" + report "${script}" "${category}" "${item}" "${action}" "failed" "${note}" "${backup}" +} diff --git a/straper/doctor.sh b/straper/doctor.sh new file mode 100644 index 0000000..6aed53e --- /dev/null +++ b/straper/doctor.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/common.sh" + +SCRIPT_NAME="doctor.sh" +STRICT=false +OUTFILE="" + +usage() { + cat <<USAGE +Usage: sudo ./doctor.sh [options] + +Purpose: + Read-only health and readiness checks for a rebuilt sanctum/labunix host. + +Options: + --db-dir PATH + --role lab|hardware|replacement + --strict Exit non-zero if any FAIL is found + --output PATH Write human report to this path + --dry-run Still read-only; only affects report/state writes + --help +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --db-dir) DB_DIR="$2"; PUBLIC_DIR="${DB_DIR}/db/public"; SECRET_DIR="${DB_DIR}/db/secret"; shift ;; + --role) ROLE="$2"; shift ;; + --strict) STRICT=true ;; + --output) OUTFILE="$2"; shift ;; + --dry-run) DRY_RUN=true ;; + --help) usage; exit 0 ;; + *) die "unknown option: $1" ;; + esac + shift +done + +ensure_root +load_optional_config +ensure_runtime_dirs +require_cmd ip systemctl ss awk sed grep + +OUTFILE="${OUTFILE:-${LOG_DIR}/doctor-${RUN_ID}.txt}" +: > "${OUTFILE}" + +FAILS=0 +WARNS=0 +OKS=0 +MANUALS=0 + +emit() { + printf '%s\n' "$*" | tee -a "${OUTFILE}" +} + +ok_line() { + ((OKS+=1)) + emit "OK | $1" + report "${SCRIPT_NAME}" doctor "$1" check ok "$1" "" +} +warn_line() { + ((WARNS+=1)) + emit "WARN | $1" + report "${SCRIPT_NAME}" doctor "$1" check warn "$1" "" +} +fail_line() { + ((FAILS+=1)) + emit "FAIL | $1" + report "${SCRIPT_NAME}" doctor "$1" check failed "$1" "" +} +manual_line() { + ((MANUALS+=1)) + emit "MANUAL| $1" + report "${SCRIPT_NAME}" doctor "$1" manual manual "$1" "" +} + +service_report() { + local svc="$1" + local unit="$svc" + local load active enabled + + [[ "${unit}" == *.* ]] || unit="${unit}.service" + + load="$(systemctl show -P LoadState "${unit}" 2>/dev/null || true)" + if [[ -z "${load}" || "${load}" == "not-found" ]]; then + warn_line "service ${unit} not installed" + return 0 + fi + + active="$(systemctl show -P ActiveState "${unit}" 2>/dev/null || true)" + enabled="$(systemctl show -P UnitFileState "${unit}" 2>/dev/null || true)" + + if [[ "${active}" == "active" ]]; then + ok_line "service ${unit} active (${enabled})" + else + warn_line "service ${unit} ${active:-unknown} (${enabled:-unknown})" + fi +} + +emit "labunix/sanctum rebuild doctor" +emit "timestamp: $(now_iso)" +emit "role: ${ROLE}" +emit "db: ${DB_DIR}" +emit "" + +virt="$(detect_virtualization)" +if [[ -n ${virt} ]]; then + warn_line "virtualization detected: ${virt}" +else + ok_line "bare metal or virtualization not detected" +fi + +current_host="$(hostname -s 2>/dev/null || true)" +captured_host="$(head -n1 "${PUBLIC_DIR}/system/etc-hostname" 2>/dev/null || true)" +if [[ -n ${captured_host} && ${current_host} == ${captured_host} ]]; then + ok_line "hostname matches DB (${current_host})" +elif [[ -n ${captured_host} ]]; then + warn_line "hostname differs: current=${current_host} captured=${captured_host}" +else + warn_line "captured hostname not found in DB" +fi + +if has_default_route; then + ok_line "default route present" +else + fail_line "no default route" +fi + +iface="$(primary_iface)" +src_ip="$(primary_src_ip)" +[[ -n ${iface} ]] && ok_line "primary route uses iface=${iface} src=${src_ip}" || fail_line "could not determine primary interface via ip route get 1.1.1.1" + +if can_reach_ip 1.1.1.1; then + ok_line "outbound IP reachability to 1.1.1.1" +else + fail_line "cannot reach 1.1.1.1" +fi + +if can_resolve_name deb.debian.org; then + ok_line "DNS resolves deb.debian.org" +else + fail_line "DNS resolution failed for deb.debian.org" +fi + +owner="$(guess_dns_owner)" +case "${owner}" in + resolved) ok_line "DNS owner appears to be systemd-resolved" ;; + dnsmasq) ok_line "DNS owner appears to be dnsmasq" ;; + unbound) ok_line "DNS owner appears to be unbound" ;; + chain) warn_line "DNS owner appears chained (dnsmasq + unbound); verify port ownership intentionally" ;; + *) warn_line "DNS owner unclear" ;; +esac + +failed_units="$(systemctl list-units --state=failed --no-legend --plain 2>/dev/null | awk '{print $1}')" +if [[ -z ${failed_units} ]]; then + ok_line "no failed systemd units" +else + fail_line "failed units present: $(echo "${failed_units}" | paste -sd, -)" +fi + +# Config validators +have_cmd sshd && validate_sshd && ok_line "sshd config validates" || warn_line "sshd config validation failed or sshd missing" +have_cmd nginx && validate_nginx && ok_line "nginx config validates" || warn_line "nginx config validation failed or nginx missing" +have_cmd nft && [[ -f /etc/nftables.conf ]] && validate_nft && ok_line "nftables config validates" || warn_line "nftables config validation failed or /etc/nftables.conf missing" +have_cmd unbound-checkconf && validate_unbound && ok_line "unbound config validates" || warn_line "unbound config validation failed or unbound missing" +have_cmd dnsmasq && validate_dnsmasq && ok_line "dnsmasq config validates" || warn_line "dnsmasq config validation failed or dnsmasq missing" +have_cmd postfix && validate_postfix && ok_line "postfix config validates" || warn_line "postfix config validation failed or postfix missing" + +emit "" +emit "Service snapshot" +service_report ssh +service_report systemd-resolved +service_report dnsmasq +service_report unbound +service_report nftables +service_report nginx +service_report mariadb +service_report postfix +service_report prosody +service_report docker +service_report prometheus +service_report grafana-server +service_report tor +service_report i2pd + +emit "" +emit "Listening ports snapshot" +ss -tlnp 2>/dev/null | tee -a "${OUTFILE}" >/dev/null || true + +if have_cmd docker; then + emit "" + emit "Docker snapshot" + docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' 2>/dev/null | tee -a "${OUTFILE}" >/dev/null || warn_line "docker present but docker ps failed" +fi + +if [[ -d ${PUBLIC_DIR}/docker/compose-redacted || -d ${SECRET_DIR}/docker/compose-full ]]; then + local_count="$(find /srv /opt /home /root -maxdepth 4 \( -name 'docker-compose.yml' -o -name 'docker-compose.yaml' -o -name 'compose.yml' -o -name 'compose.yaml' \) 2>/dev/null | wc -l | tr -d ' ')" + ok_line "compose files found locally: ${local_count}" +fi + +if [[ -f /etc/resolv.conf ]]; then + emit "" + emit "/etc/resolv.conf" + sed -n '1,20p' /etc/resolv.conf | tee -a "${OUTFILE}" >/dev/null || true +fi + +if [[ -f ${STATE_DIR}/state-${RUN_ID}.env ]]; then + manual_line "current run state file: ${STATE_DIR}/state-${RUN_ID}.env" +fi + +emit "" +emit "Summary: ok=${OKS} warn=${WARNS} fail=${FAILS} manual=${MANUALS}" + +set_state doctor_done true +report "${SCRIPT_NAME}" run finish exit ok "ok=${OKS}; warn=${WARNS}; fail=${FAILS}; manual=${MANUALS}" "" + +if [[ ${STRICT} == true && ${FAILS} -gt 0 ]]; then + exit 1 +fi diff --git a/straper/install-base.sh b/straper/install-base.sh new file mode 100644 index 0000000..e2a2d42 --- /dev/null +++ b/straper/install-base.sh @@ -0,0 +1,342 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/common.sh" + +SCRIPT_NAME="install-base.sh" +USE_DB_APT_SOURCES=false +FROM_DB_MANUAL=false +HOSTNAME_WANTED="" +TIMEZONE_WANTED="Europe/Warsaw" +LOCALE_WANTED="en_US.UTF-8" + +usage() { + cat <<USAGE +Usage: sudo ./install-base.sh [options] + +Purpose: + Install a reproducible Debian base for sanctum/labunix recovery without + importing host-specific configs or identities. + +Options: + --db-dir PATH Capture DB root (default: ${DB_DIR}) + --role lab|hardware|replacement + --profile minimal|core|full + --hostname NAME Set hostname + --timezone TZ Set timezone (default: ${TIMEZONE_WANTED}) + --locale LOCALE Set locale (default: ${LOCALE_WANTED}) + --use-db-apt-sources Restore apt sources from db/public/packages/apt + --from-db-manual Install packages listed in apt-mark-manual.txt + --start-safe-services Enable/start low-risk services after install + --yes Non-interactive yes to prompts + --dry-run Print actions only + --verbose More logging + --help +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --db-dir) DB_DIR="$2"; PUBLIC_DIR="${DB_DIR}/db/public"; SECRET_DIR="${DB_DIR}/db/secret"; shift ;; + --role) ROLE="$2"; shift ;; + --profile) PROFILE="$2"; shift ;; + --hostname) HOSTNAME_WANTED="$2"; shift ;; + --timezone) TIMEZONE_WANTED="$2"; shift ;; + --locale) LOCALE_WANTED="$2"; shift ;; + --use-db-apt-sources) USE_DB_APT_SOURCES=true ;; + --from-db-manual) FROM_DB_MANUAL=true ;; + --start-safe-services) START_SERVICES=true ;; + --yes) ASSUME_YES=true ;; + --dry-run) DRY_RUN=true ;; + --verbose) VERBOSE=true ;; + --help) usage; exit 0 ;; + *) die "unknown option: $1" ;; + esac + shift +done + +ensure_root +load_optional_config +ensure_runtime_dirs +require_cmd apt-get apt-cache dpkg hostnamectl rsync + +mkdir -p -- /etc/labunix + +log "${SCRIPT_NAME}: role=${ROLE} profile=${PROFILE} db=${DB_DIR}" +report "${SCRIPT_NAME}" "run" "start" "init" "ok" "role=${ROLE} profile=${PROFILE}" "" + +case "${ROLE}" in + lab|hardware|replacement) ;; + *) die "invalid role: ${ROLE}" ;; +esac +case "${PROFILE}" in + minimal|core|full) ;; + *) die "invalid profile: ${PROFILE}" ;; +esac + +BASE_PACKAGES=( + ca-certificates curl wget gnupg2 jq rsync vim zsh sudo + locales tzdata lsb-release apt-transport-https + net-tools iproute2 iputils-ping dnsutils ethtool + openssh-server systemd-resolved nftables fail2ban +) + +CORE_PACKAGES=( + wireguard wireguard-tools dnsmasq unbound nginx mariadb-server postfix prosody + tor i2pd pygopherd mumble-server prometheus prometheus-node-exporter loki + apparmor apparmor-utils +) + +FULL_EXTRA_PACKAGES=( + git tmux htop build-essential pkg-config + docker.io +) + +bootstrap_dns_prepare() { + local test_host="${BOOTSTRAP_TEST_HOST:-deb.debian.org}" + local backup="/etc/resolv.conf.labunix-preinstall.bak" + + if getent ahostsv4 "${test_host}" >/dev/null 2>&1; then + report "${SCRIPT_NAME}" "dns" "bootstrap" "check" "ok" "resolver already working" "" + return 0 + fi + + if ! ping -c 1 -W 3 1.1.1.1 >/dev/null 2>&1; then + note_failure "${SCRIPT_NAME}" "dns" "bootstrap" "check" "no outbound IP connectivity" + die "bootstrap DNS check failed: no outbound IP connectivity" + fi + + if command -v dig >/dev/null 2>&1 && dig +time=2 +tries=1 +short @1.1.1.1 "${test_host}" >/dev/null 2>&1; then + [[ -e /etc/resolv.conf && ! -e "${backup}" ]] && cp -a /etc/resolv.conf "${backup}" || true + printf 'nameserver 1.1.1.1 +nameserver 9.9.9.9 +' > /etc/resolv.conf + chmod 644 /etc/resolv.conf || true + + if getent ahostsv4 "${test_host}" >/dev/null 2>&1; then + report "${SCRIPT_NAME}" "dns" "bootstrap" "fallback" "changed" "static resolv.conf applied" "${backup}" + return 0 + fi + fi + + note_failure "${SCRIPT_NAME}" "dns" "bootstrap" "check" "DNS still broken after fallback attempt" + die "bootstrap DNS check failed" +} + +install_pkg_group() { + local group_name="$1"; shift + local pkgs=("$@") + log "installing package group: ${group_name}" + if apt_install_if_missing "${pkgs[@]}"; then + report "${SCRIPT_NAME}" "packages" "${group_name}" "install" "ok" "group processed" "" + else + note_failure "${SCRIPT_NAME}" "packages" "${group_name}" "install" "apt install failed" + die "package group failed: ${group_name}" + fi +} + +apply_hostname_locale_timezone() { + if [[ -n ${HOSTNAME_WANTED} ]]; then + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] hostnamectl set-hostname %s\n' "${HOSTNAME_WANTED}" + else + hostnamectl set-hostname "${HOSTNAME_WANTED}" + printf '%s\n' "${HOSTNAME_WANTED}" > /etc/hostname + fi + report "${SCRIPT_NAME}" "system" "hostname" "set" "changed" "${HOSTNAME_WANTED}" "" + fi + + if [[ -f /etc/locale.gen ]] && grep -Eq "^#?\s*${LOCALE_WANTED}\b" /etc/locale.gen 2>/dev/null; then + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] enable locale %s in /etc/locale.gen\n' "${LOCALE_WANTED}" + else + if have_cmd locale-gen && have_cmd update-locale; then + sed -i "s/^# *\(${LOCALE_WANTED}\b.*\)/\1/" /etc/locale.gen + locale-gen "${LOCALE_WANTED}" >/dev/null + update-locale LANG="${LOCALE_WANTED}" >/dev/null + else + report "${SCRIPT_NAME}" "system" "locale" "manual" "manual" "locale tools missing; install locales first or rerun after base packages" "" + return 0 + fi + fi + report "${SCRIPT_NAME}" "system" "locale" "set" "changed" "${LOCALE_WANTED}" "" + else + report "${SCRIPT_NAME}" "system" "locale" "manual" "manual" "locale missing in /etc/locale.gen: ${LOCALE_WANTED}" "" + fi + + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] set timezone %s\n' "${TIMEZONE_WANTED}" + else + ln -sf "/usr/share/zoneinfo/${TIMEZONE_WANTED}" /etc/localtime + dpkg-reconfigure -f noninteractive tzdata >/dev/null + fi + report "${SCRIPT_NAME}" "system" "timezone" "set" "changed" "${TIMEZONE_WANTED}" "" +} + +restore_apt_sources_from_db() { + local src="${PUBLIC_DIR}/packages/apt" + [[ -d ${src} ]] || { mark_manual "${SCRIPT_NAME}" "apt" "sources" "db/public/packages/apt missing"; return 0; } + + if ! prompt_yes_no "Restore APT sources from DB now?" no; then + report "${SCRIPT_NAME}" "apt" "sources" "restore" "skipped" "operator skipped" "" + return 0 + fi + + restore_path "${SCRIPT_NAME}" "apt" "sources.list" "${src}/sources.list" "/etc/apt/sources.list" + restore_path "${SCRIPT_NAME}" "apt" "sources.list.d" "${src}/sources.list.d" "/etc/apt/sources.list.d" + restore_path "${SCRIPT_NAME}" "apt" "preferences" "${src}/preferences" "/etc/apt/preferences" + restore_path "${SCRIPT_NAME}" "apt" "preferences.d" "${src}/preferences.d" "/etc/apt/preferences.d" + restore_path "${SCRIPT_NAME}" "apt" "apt.conf.d" "${src}/apt.conf.d" "/etc/apt/apt.conf.d" + + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] apt-get update\n' + else + if apt-get update; then + report "${SCRIPT_NAME}" "apt" "update-after-restore" "update" "ok" "apt metadata refreshed" "" + else + note_failure "${SCRIPT_NAME}" "apt" "update-after-restore" "update" "apt-get update failed after restoring sources" + fi + fi +} + +package_available() { + local pkg="$1" + apt-cache show -- "${pkg}" >/dev/null 2>&1 +} + +install_from_db_manual() { + local f="${PUBLIC_DIR}/packages/apt-mark-manual.txt" pkg installed=0 skipped=0 failed=0 + [[ -f ${f} ]] || { mark_manual "${SCRIPT_NAME}" "packages" "db-manual" "apt-mark-manual.txt missing"; return 0; } + + while IFS= read -r pkg; do + [[ -n ${pkg} ]] || continue + [[ ${pkg} =~ ^# ]] && continue + if dpkg -s -- "${pkg}" >/dev/null 2>&1; then + ((installed+=1)) + continue + fi + if ! package_available "${pkg}"; then + ((skipped+=1)) + report "${SCRIPT_NAME}" "packages" "${pkg}" "install" "skipped" "package not available in current apt sources" "" + continue + fi + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] apt-get install -y --no-install-recommends %s\n' "${pkg}" + ((installed+=1)) + continue + fi + if DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${pkg}"; then + ((installed+=1)) + report "${SCRIPT_NAME}" "packages" "${pkg}" "install" "changed" "installed from apt-mark-manual" "" + else + ((failed+=1)) + note_failure "${SCRIPT_NAME}" "packages" "${pkg}" "install" "apt install failed" + fi + done < <(list_from_file "${f}") + + report "${SCRIPT_NAME}" "packages" "db-manual-summary" "install" "ok" "installed=${installed} skipped=${skipped} failed=${failed}" "" +} +normalize_service_baseline() { + local keep_active=( + ssh.service + systemd-resolved.service + fail2ban.service + ) + + local stop_disable=( + dnsmasq.service + unbound.service + unbound-resolvconf.service + nginx.service + mariadb.service + postfix.service + prosody.service + prometheus.service + tor.service + i2pd.service + mumble-server.service + pygopherd.service + ) + + local unit + + for unit in "${stop_disable[@]}"; do + if systemctl list-unit-files --type=service --no-legend --no-pager 2>/dev/null | awk '{print $1}' | grep -qx -- "${unit}"; then + systemctl disable --now "${unit}" >/dev/null 2>&1 || true + fi + done + + systemctl reset-failed "${stop_disable[@]}" >/dev/null 2>&1 || true + + for unit in "${keep_active[@]}"; do + if systemctl list-unit-files --type=service --no-legend --no-pager 2>/dev/null | awk '{print $1}' | grep -qx -- "${unit}"; then + systemctl enable "${unit}" >/dev/null 2>&1 || true + systemctl restart "${unit}" >/dev/null 2>&1 || true + fi + done + + report "${SCRIPT_NAME}" "services" "baseline" "normalize" "changed" "safe services kept; app/conflicting services stopped" "" +} +start_safe_services() { + local svc + for svc in ssh systemd-resolved fail2ban; do + systemctl_safe_enable "${svc}" || true + systemctl_safe_start "${svc}" || true + done + + case "${DNS_MODE}" in + resolved) + systemctl_safe_enable systemd-resolved || true + systemctl_safe_start systemd-resolved || true + ;; + esac + report "${SCRIPT_NAME}" "services" "safe-start" "start" "ok" "safe services processed" "" +} + +if [[ ${DRY_RUN} == false ]]; then + bootstrap_dns_prepare + if apt-get update; then + report "${SCRIPT_NAME}" "apt" "update" "update" "ok" "initial apt update" "" + else + note_failure "${SCRIPT_NAME}" "apt" "update" "update" "initial apt-get update failed" + die "initial apt-get update failed" + fi +else + report "${SCRIPT_NAME}" "apt" "update" "update" "ok" "initial apt update (dry-run)" "" +fi + +install_pkg_group base "${BASE_PACKAGES[@]}" +case "${PROFILE}" in + minimal) ;; + core) install_pkg_group core "${CORE_PACKAGES[@]}" ;; + full) + install_pkg_group core "${CORE_PACKAGES[@]}" + install_pkg_group full-extra "${FULL_EXTRA_PACKAGES[@]}" + ;; +esac + +normalize_service_baseline +apply_hostname_locale_timezone + +if [[ ${USE_DB_APT_SOURCES} == true ]]; then + restore_apt_sources_from_db +fi + +if [[ ${FROM_DB_MANUAL} == true ]]; then + install_from_db_manual +fi + +install -d -m 0755 /srv /srv/www /srv/docs /etc/wireguard /etc/labunix /var/lib/labunix +report "${SCRIPT_NAME}" "filesystem" "base-dirs" "create" "ok" "/srv /etc/wireguard /etc/labunix prepared" "" + +if [[ ${START_SERVICES} == true ]]; then + start_safe_services +else + report "${SCRIPT_NAME}" "services" "safe-start" "start" "skipped" "--start-safe-services not requested" "" +fi + +set_state install_base done +report "${SCRIPT_NAME}" "run" "finish" "exit" "ok" "completed" "" +log "${SCRIPT_NAME}: done; report=${REPORT_FILE}" diff --git a/straper/rebuild.conf.example b/straper/rebuild.conf.example new file mode 100644 index 0000000..ba1e9ab --- /dev/null +++ b/straper/rebuild.conf.example @@ -0,0 +1,10 @@ +# Example: /etc/labunix/rebuild.conf +DB_DIR=/srv/sanctum-rebuild +ROLE=lab +PROFILE=core +DNS_MODE=auto +NETWORK_MODE=safe +START_SERVICES=false +RESTORE_SECRETS=false +RESTORE_IDENTITIES=false +ASSUME_YES=false diff --git a/straper/restore-configs.sh b/straper/restore-configs.sh new file mode 100644 index 0000000..db81487 --- /dev/null +++ b/straper/restore-configs.sh @@ -0,0 +1,521 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/common.sh" + +SCRIPT_NAME="restore-configs.sh" +CATEGORIES=() +LIST_CATEGORIES=false + +usage() { + cat <<USAGE +Usage: sudo ./restore-configs.sh [options] + +Purpose: + Restore selected configs from a capture DB in a controlled, category-based way. + Intended to be non-fatal per item whenever possible. Sensitive/identity-heavy + categories should be restored deliberately. + +Options: + --db-dir PATH + --role lab|hardware|replacement + --category NAME May be used multiple times + --list-categories + --yes Non-interactive yes to prompts + --dry-run + --verbose + --help + +Categories: + system-basics + users + ssh + network + dns + firewall + nginx + mariadb + postfix + prosody + tor + i2pd + docker + monitoring +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --db-dir) DB_DIR="$2"; PUBLIC_DIR="${DB_DIR}/db/public"; SECRET_DIR="${DB_DIR}/db/secret"; shift ;; + --role) ROLE="$2"; shift ;; + --category) CATEGORIES+=("$2"); shift ;; + --list-categories) LIST_CATEGORIES=true ;; + --yes) ASSUME_YES=true ;; + --dry-run) DRY_RUN=true ;; + --verbose) VERBOSE=true ;; + --help) usage; exit 0 ;; + *) die "unknown option: $1" ;; + esac + shift +done + +if [[ "${LIST_CATEGORIES}" == true ]]; then + printf '%s\n' \ + system-basics \ + users \ + ssh \ + network \ + dns \ + firewall \ + nginx \ + mariadb \ + postfix \ + prosody \ + tor \ + i2pd \ + docker \ + monitoring + exit 0 +fi + +ensure_root +load_optional_config +ensure_runtime_dirs +require_cmd cp chmod chown find grep sed awk systemctl + +[[ -d "${PUBLIC_DIR}" ]] || die "public DB not found: ${PUBLIC_DIR}" +[[ ${#CATEGORIES[@]} -gt 0 ]] || die "no categories specified; use --category or --list-categories" + +log "${SCRIPT_NAME}: role=${ROLE} categories=${CATEGORIES[*]} db=${DB_DIR}" +report "${SCRIPT_NAME}" "run" "start" "init" "ok" "role=${ROLE} categories=${CATEGORIES[*]}" "" + +case "${ROLE}" in + lab|hardware|replacement) ;; + *) die "invalid role: ${ROLE}" ;; +esac + +fix_sudoers_perms() { + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] normalize sudoers ownership and permissions\n' + return 0 + fi + + if [[ -f /etc/sudoers ]]; then + chown root:root /etc/sudoers || true + chmod 0440 /etc/sudoers || true + fi + + if [[ -d /etc/sudoers.d ]]; then + chown root:root /etc/sudoers.d || true + chmod 0755 /etc/sudoers.d || true + find /etc/sudoers.d -maxdepth 1 -type f -exec chown root:root {} \; || true + find /etc/sudoers.d -maxdepth 1 -type f -exec chmod 0440 {} \; || true + fi + + if visudo -c >/dev/null 2>&1; then + report "${SCRIPT_NAME}" "users" "sudoers-perms" "fix" "changed" "sudoers ownership and permissions normalized" "" + return 0 + fi + + note_failure "${SCRIPT_NAME}" "users" "sudoers-perms" "fix" "visudo validation failed after permission normalization" + return 1 +} + +maybe_restore() { + local category="$1" item="$2" src="$3" dst="$4" + local prompt_text="Restore ${category}/${item} -> ${dst}?" + + if ! prompt_yes_no "${prompt_text}" yes; then + report "${SCRIPT_NAME}" "${category}" "${item}" "restore" "skipped" "operator skipped" "" + return 0 + fi + + restore_path "${SCRIPT_NAME}" "${category}" "${item}" "${src}" "${dst}" || true +} + +category_enabled() { + local want="$1" x + for x in "${CATEGORIES[@]}"; do + [[ "${x}" == "${want}" ]] && return 0 + done + return 1 +} + +restore_system_basics() { + local base="${PUBLIC_DIR}/system" + [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "system-basics" "db" "missing ${base}"; return 0; } + + if ! prompt_yes_no "Restore category 'system-basics'?" yes; then + report "${SCRIPT_NAME}" "system-basics" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + maybe_restore "system-basics" "etc-hostname" "${base}/etc-hostname" "/etc/hostname" + maybe_restore "system-basics" "etc-hosts" "${base}/etc-hosts" "/etc/hosts" + maybe_restore "system-basics" "etc-environment" "${base}/etc-environment" "/etc/environment" + maybe_restore "system-basics" "locale.gen" "${base}/locale.gen" "/etc/locale.gen" + maybe_restore "system-basics" "timezone" "${base}/timezone" "/etc/timezone" + + if [[ -f "${base}/etc-hostname" ]]; then + local captured_hostname + captured_hostname="$(head -n1 "${base}/etc-hostname" 2>/dev/null || true)" + if [[ -n "${captured_hostname}" ]]; then + if prompt_yes_no "Apply captured hostname now?" yes; then + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] hostnamectl set-hostname %s\n' "${captured_hostname}" + else + hostnamectl set-hostname "${captured_hostname}" || note_failure "${SCRIPT_NAME}" "system-basics" "hostnamectl" "set" "hostnamectl failed" + fi + report "${SCRIPT_NAME}" "system-basics" "hostnamectl" "set" "changed" "${captured_hostname}" "" + else + report "${SCRIPT_NAME}" "system-basics" "hostnamectl" "set" "skipped" "operator skipped hostnamectl" "" + fi + fi + fi +} + +restore_users() { + local base="${PUBLIC_DIR}/users" + [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "users" "db" "missing ${base}"; return 0; } + + if ! prompt_yes_no "Restore category 'users'?" yes; then + report "${SCRIPT_NAME}" "users" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + maybe_restore "users" "sudoers" "${base}/sudoers" "/etc/sudoers" + maybe_restore "users" "sudoers.d" "${base}/sudoers.d" "/etc/sudoers.d" + fix_sudoers_perms || true + maybe_restore "users" "shells" "${base}/shells" "/etc/shells" + maybe_restore "users" "login.defs" "${base}/login.defs" "/etc/login.defs" +} + +restore_ssh() { + local pub_users="${PUBLIC_DIR}/users" + local sec_ssh="${SECRET_DIR}/ssh" + + if ! prompt_yes_no "Restore category 'ssh'?" yes; then + report "${SCRIPT_NAME}" "ssh" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + maybe_restore "ssh" "sshd_config" "${pub_users}/sshd_config" "/etc/ssh/sshd_config" + maybe_restore "ssh" "sshd_config.d" "${pub_users}/sshd_config.d" "/etc/ssh/sshd_config.d" + + if [[ "${ROLE}" == "replacement" || "${ROLE}" == "hardware" ]]; then + if [[ -d "${sec_ssh}/etc-ssh" ]]; then + if prompt_yes_no "Restore SSH host keys from secret DB?" no; then + restore_path "${SCRIPT_NAME}" "ssh" "host-keys" "${sec_ssh}/etc-ssh" "/etc/ssh" || true + else + report "${SCRIPT_NAME}" "ssh" "host-keys" "restore" "skipped" "operator skipped host keys" "" + fi + fi + else + report "${SCRIPT_NAME}" "ssh" "host-keys" "restore" "skipped" "lab role: host keys not restored" "" + fi + + if [[ -d "${sec_ssh}/user-lukasz" ]]; then + if prompt_yes_no "Restore lukasz user SSH material from secret DB?" no; then + restore_path "${SCRIPT_NAME}" "ssh" "user-lukasz" "${sec_ssh}/user-lukasz" "/home/lukasz/.ssh" || true + if [[ ${DRY_RUN} == false ]]; then + chown -R lukasz:lukasz /home/lukasz/.ssh 2>/dev/null || true + chmod 700 /home/lukasz/.ssh 2>/dev/null || true + find /home/lukasz/.ssh -maxdepth 1 -type f -exec chmod 600 {} \; 2>/dev/null || true + fi + else + report "${SCRIPT_NAME}" "ssh" "user-lukasz" "restore" "skipped" "operator skipped user ssh material" "" + fi + fi + + if have_cmd sshd; then + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] sshd -t\n' + report "${SCRIPT_NAME}" "ssh" "validate" "check" "ok" "dry-run only" "" + else + if sshd -t; then + report "${SCRIPT_NAME}" "ssh" "validate" "check" "ok" "sshd config validates" "" + systemctl restart ssh >/dev/null 2>&1 || true + else + note_failure "${SCRIPT_NAME}" "ssh" "validate" "check" "sshd validation failed" + fi + fi + fi +} + +restore_network() { + local base="${PUBLIC_DIR}/network" + [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "network" "db" "missing ${base}"; return 0; } + + if ! prompt_yes_no "Restore category 'network'? This can disrupt connectivity." no; then + report "${SCRIPT_NAME}" "network" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + if [[ "${ROLE}" == "lab" ]]; then + manual_line="lab role: full network restore intentionally skipped" + report "${SCRIPT_NAME}" "network" "category" "manual" "manual" "${manual_line}" "" + return 0 + fi + + maybe_restore "network" "etc-network" "${base}/etc-network" "/etc/network" + maybe_restore "network" "etc-netplan" "${base}/etc-netplan" "/etc/netplan" + maybe_restore "network" "systemd-network" "${base}/systemd-network" "/etc/systemd/network" + maybe_restore "network" "nsswitch.conf" "${base}/nsswitch.conf" "/etc/nsswitch.conf" + maybe_restore "network" "hosts.allow" "${base}/hosts.allow" "/etc/hosts.allow" + maybe_restore "network" "hosts.deny" "${base}/hosts.deny" "/etc/hosts.deny" +} + +restore_dns() { + local base="${PUBLIC_DIR}/dns" + [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "dns" "db" "missing ${base}"; return 0; } + + if ! prompt_yes_no "Restore category 'dns'? This can disrupt resolver state." no; then + report "${SCRIPT_NAME}" "dns" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + if [[ "${ROLE}" == "lab" ]]; then + report "${SCRIPT_NAME}" "dns" "category" "manual" "manual" "lab role: DNS restore intentionally skipped" "" + return 0 + fi + + maybe_restore "dns" "dnsmasq.conf" "${base}/dnsmasq.conf" "/etc/dnsmasq.conf" + maybe_restore "dns" "dnsmasq.d" "${base}/dnsmasq.d" "/etc/dnsmasq.d" + maybe_restore "dns" "etc-unbound" "${base}/etc-unbound" "/etc/unbound" + maybe_restore "dns" "resolv.conf" "${base}/resolv.conf" "/etc/resolv.conf" +} + +restore_firewall() { + local base="${PUBLIC_DIR}/firewall" + [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "firewall" "db" "missing ${base}"; return 0; } + + if ! prompt_yes_no "Restore category 'firewall'? This can disrupt connectivity." no; then + report "${SCRIPT_NAME}" "firewall" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + if [[ "${ROLE}" == "lab" ]]; then + report "${SCRIPT_NAME}" "firewall" "category" "manual" "manual" "lab role: firewall restore intentionally skipped" "" + return 0 + fi + + maybe_restore "firewall" "nftables.conf" "${base}/nftables.conf" "/etc/nftables.conf" + maybe_restore "firewall" "nftables.d" "${base}/nftables.d" "/etc/nftables.d" + + if [[ ${DRY_RUN} == false && -f /etc/nftables.conf ]] && have_cmd nft; then + nft -c -f /etc/nftables.conf >/dev/null 2>&1 || note_failure "${SCRIPT_NAME}" "firewall" "validate" "check" "nftables config validation failed" + fi +} + +restore_nginx() { + local base="${PUBLIC_DIR}/nginx/etc-nginx" + [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "nginx" "db" "missing ${base}"; return 0; } + + if ! prompt_yes_no "Restore category 'nginx'?" no; then + report "${SCRIPT_NAME}" "nginx" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + maybe_restore "nginx" "etc-nginx" "${base}" "/etc/nginx" + + if [[ "${ROLE}" == "lab" ]]; then + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] sanitize /etc/nginx for lab role\n' + printf '[dry] rm -rf /etc/nginx/nginx\n' + printf '[dry] find /etc/nginx -maxdepth 1 -type d -name '\''sites-available.bak.*'\'' -exec rm -rf {} +\n' + printf '[dry] rm -f /etc/nginx/sites-enabled/*\n' + printf '[dry] ln -sf ../sites-available/default /etc/nginx/sites-enabled/default\n' + else + rm -rf /etc/nginx/nginx 2>/dev/null || true + find /etc/nginx -maxdepth 1 -type d -name 'sites-available.bak.*' -exec rm -rf {} + 2>/dev/null || true + + mkdir -p /etc/nginx/sites-enabled + find /etc/nginx/sites-enabled -mindepth 1 -maxdepth 1 -exec rm -f {} + 2>/dev/null || true + + if [[ -e /etc/nginx/sites-available/default ]]; then + ln -sf ../sites-available/default /etc/nginx/sites-enabled/default + fi + fi + + report "${SCRIPT_NAME}" "nginx" "lab-sanitize" "restore" "changed" \ + "lab role: dropped captured production sites-enabled and restored only default site" "" + fi +} + +restore_mariadb() { + local base="${PUBLIC_DIR}/mariadb/etc-mysql" + [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "mariadb" "db" "missing ${base}"; return 0; } + + if ! prompt_yes_no "Restore category 'mariadb'?" no; then + report "${SCRIPT_NAME}" "mariadb" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + maybe_restore "mariadb" "etc-mysql" "${base}" "/etc/mysql" +} + +restore_postfix() { + local base="${PUBLIC_DIR}/postfix" + [[ -d "${base}" ]] || { + mark_manual "${SCRIPT_NAME}" "postfix" "db" "missing ${base}" + return 0 + } + + if ! prompt_yes_no "Restore category 'postfix'?" no; then + report "${SCRIPT_NAME}" "postfix" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + if prompt_yes_no "Restore postfix/main.cf -> /etc/postfix/main.cf?" yes; then + restore_path "${SCRIPT_NAME}" "postfix" "main.cf" \ + "${base}/main.cf" "/etc/postfix/main.cf" \ + 0644 root root || true + else + report "${SCRIPT_NAME}" "postfix" "main.cf" "restore" "skipped" "operator skipped" "" + fi + + if prompt_yes_no "Restore postfix/master.cf -> /etc/postfix/master.cf?" yes; then + restore_path "${SCRIPT_NAME}" "postfix" "master.cf" \ + "${base}/master.cf" "/etc/postfix/master.cf" \ + 0644 root root || true + else + report "${SCRIPT_NAME}" "postfix" "master.cf" "restore" "skipped" "operator skipped" "" + fi +} + +restore_prosody() { + local pub_base="${PUBLIC_DIR}/prosody/etc-prosody" + local sec_base="${SECRET_DIR}/prosody/etc-prosody" + local base="" + local label="" + + if ! prompt_yes_no "Restore category 'prosody'?" no; then + report "${SCRIPT_NAME}" "prosody" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + if [[ -d "${sec_base}" ]]; then + base="${sec_base}" + label="etc-prosody(secret)" + elif [[ -d "${pub_base}" ]]; then + base="${pub_base}" + label="etc-prosody(public)" + else + mark_manual "${SCRIPT_NAME}" "prosody" "db" "missing public/secret prosody config" + return 0 + fi + + if prompt_yes_no "Restore prosody/${label} -> /etc/prosody?" yes; then + restore_path "${SCRIPT_NAME}" "prosody" "${label}" "${base}" "/etc/prosody" || true + + if [[ ${DRY_RUN} == true ]]; then + printf '[dry] chown -R root:root /etc/prosody\n' + printf '[dry] find /etc/prosody -type d -exec chmod 0755 {} +\n' + printf '[dry] find /etc/prosody -type f -exec chmod 0644 {} +\n' + printf '[dry] find /etc/prosody/certs -type f -exec chmod 0640 {} + 2>/dev/null || true\n' + else + chown -R root:root /etc/prosody 2>/dev/null || true + find /etc/prosody -type d -exec chmod 0755 {} + 2>/dev/null || true + find /etc/prosody -type f -exec chmod 0644 {} + 2>/dev/null || true + [[ -d /etc/prosody/certs ]] && find /etc/prosody/certs -type f -exec chmod 0640 {} + 2>/dev/null || true + fi + + report "${SCRIPT_NAME}" "prosody" "metadata" "restore" "changed" \ + "normalized /etc/prosody ownership=root:root dirs=0755 files=0644 certs=0640" "" + else + report "${SCRIPT_NAME}" "prosody" "${label}" "restore" "skipped" "operator skipped" "" + fi +} + +restore_tor() { + local pub_base="${PUBLIC_DIR}/tor" + local sec_base="${SECRET_DIR}/tor" + + if ! prompt_yes_no "Restore category 'tor'?" no; then + report "${SCRIPT_NAME}" "tor" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + maybe_restore "tor" "torrc" "${pub_base}/torrc" "/etc/tor/torrc" + maybe_restore "tor" "torrc.d" "${pub_base}/torrc.d" "/etc/tor/torrc.d" + + if [[ "${ROLE}" == "replacement" && -d "${sec_base}/var-lib-tor" ]]; then + maybe_restore "tor" "var-lib-tor" "${sec_base}/var-lib-tor" "/var/lib/tor" + else + report "${SCRIPT_NAME}" "tor" "identity" "restore" "skipped" "tor private data not restored in this role" "" + fi +} + +restore_i2pd() { + local pub_base="${PUBLIC_DIR}/i2pd" + local sec_base="${SECRET_DIR}/i2pd" + + if ! prompt_yes_no "Restore category 'i2pd'?" no; then + report "${SCRIPT_NAME}" "i2pd" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + maybe_restore "i2pd" "etc-i2pd" "${pub_base}/etc-i2pd" "/etc/i2pd" + + if [[ "${ROLE}" == "replacement" && -d "${sec_base}/var-lib-i2pd" ]]; then + maybe_restore "i2pd" "var-lib-i2pd" "${sec_base}/var-lib-i2pd" "/var/lib/i2pd" + else + report "${SCRIPT_NAME}" "i2pd" "identity" "restore" "skipped" "i2pd private data not restored in this role" "" + fi +} + +restore_docker() { + local pub_base="${PUBLIC_DIR}/docker" + local sec_base="${SECRET_DIR}/docker" + + if ! prompt_yes_no "Restore category 'docker'?" no; then + report "${SCRIPT_NAME}" "docker" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + maybe_restore "docker" "daemon.json" "${pub_base}/daemon.json" "/etc/docker/daemon.json" + + if [[ -d "${sec_base}/compose-full" ]]; then + report "${SCRIPT_NAME}" "docker" "compose-full" "manual" "manual" "compose files available in secret DB; restore manually per stack" "" + fi +} + +restore_monitoring() { + if ! prompt_yes_no "Restore category 'monitoring'?" no; then + report "${SCRIPT_NAME}" "monitoring" "category" "restore" "skipped" "operator skipped category" "" + return 0 + fi + + [[ -d "${PUBLIC_DIR}/prometheus/etc-prometheus" ]] && maybe_restore "monitoring" "prometheus" "${PUBLIC_DIR}/prometheus/etc-prometheus" "/etc/prometheus" + [[ -f "${PUBLIC_DIR}/prometheus/node-exporter-defaults" ]] && maybe_restore "monitoring" "prometheus-node-exporter" "${PUBLIC_DIR}/prometheus/node-exporter-defaults" "/etc/default/prometheus-node-exporter" + [[ -d "${PUBLIC_DIR}/loki/etc-loki" ]] && maybe_restore "monitoring" "loki" "${PUBLIC_DIR}/loki/etc-loki" "/etc/loki" + [[ -d "${PUBLIC_DIR}/grafana/etc-grafana" ]] && maybe_restore "monitoring" "grafana" "${PUBLIC_DIR}/grafana/etc-grafana" "/etc/grafana" + [[ -d "${SECRET_DIR}/alloy/etc-alloy" ]] && maybe_restore "monitoring" "alloy" "${SECRET_DIR}/alloy/etc-alloy" "/etc/alloy" +} + +for category in "${CATEGORIES[@]}"; do + log "processing category: ${category}" + case "${category}" in + system-basics) restore_system_basics ;; + users) restore_users ;; + ssh) restore_ssh ;; + network) restore_network ;; + dns) restore_dns ;; + firewall) restore_firewall ;; + nginx) restore_nginx ;; + mariadb) restore_mariadb ;; + postfix) restore_postfix ;; + prosody) restore_prosody ;; + tor) restore_tor ;; + i2pd) restore_i2pd ;; + docker) restore_docker ;; + monitoring) restore_monitoring ;; + *) note_failure "${SCRIPT_NAME}" "category" "${category}" "restore" "unknown category" ;; + esac +done + +set_state restore_configs done +report "${SCRIPT_NAME}" "run" "finish" "exit" "ok" "completed" "" +log "${SCRIPT_NAME}: done; report=${REPORT_FILE}" diff --git a/straper/sanctum-rebuild-toolkit.tar.gz b/straper/sanctum-rebuild-toolkit.tar.gz Binary files differnew file mode 100644 index 0000000..5a49146 --- /dev/null +++ b/straper/sanctum-rebuild-toolkit.tar.gz |
